diff options
| -rw-r--r-- | .auto/log.jsonl | 2 | ||||
| -rw-r--r-- | .auto/wip-decompmap.patch | 49 |
2 files changed, 51 insertions, 0 deletions
diff --git a/.auto/log.jsonl b/.auto/log.jsonl index 4ecfac1..4a618f5 100644 --- a/.auto/log.jsonl +++ b/.auto/log.jsonl @@ -30,3 +30,5 @@ {"run":25,"commit":"8218b91","metric":18746.2,"metrics":{"lg_boot_ms":713.2,"lg_decomp_ms":2575.4,"lg_graph_ms":1010.9,"lg_hex_ms":457.3,"lg_index_ms":97.3,"lg_listing_cold_ms":561.7,"lg_listing_warm_ms":411.8,"lg_nav_ms":6936.8,"lg_palette_ms":4.7,"lg_render_ms":225.9,"lg_search_ms":1442.5,"pure_graph_ms":212.9,"sm_boot_ms":433.3,"sm_decomp_ms":1304.2,"sm_graph_ms":710.8,"sm_hex_ms":456.7,"sm_index_ms":2.3,"sm_listing_cold_ms":263.9,"sm_listing_warm_ms":285.7,"sm_nav_ms":334.7,"sm_palette_ms":0.3,"sm_render_ms":257.2,"sm_search_ms":46.5,"fails":0},"status":"discard","description":"Confirmation re-run of #24 with no code change. 18608 -> 18746, i.e. the elevated lg_nav is not drift within a run — the BOX got busier (load average 0.62 at session start, 1.55 now). The direct paging microbenchmark is unchanged at 24.4 us/row, so the code is the same speed; the wall clock is not.","timestamp":1786068203628,"segment":4,"confidence":7.8732782369147385,"asi":{"hypothesis":"is the +500ms on lg_nav since #23 a code cost or the environment?","answer":"environment. /tmp/domain_break.py (a direct cold walk of bash's .text through the worker, no UI) reads 20.31us/row in the heads call + 4.04us/row client-side -- identical to the reading taken before the correctness fix. Meanwhile the box's load average went 0.62 -> 1.55 during the session.","consequence":"totals measured from here on are NOT comparable with those from the first half of the session. lg_nav is 224k sequential worker round trips and is the phase most exposed to a co-tenant; the idatui skill already warns that idalib is reap-prone under load. Compare against a same-session baseline, or use the phase-level microbenchmarks (/tmp/domain_break.py, /tmp/spanbench2.py, .auto/diff_spans.py) which are far less exposed.","no_code_change":"nothing to revert; logged as discard because the metric did not improve","next_action_hint":"if the loop continues on a loaded box, prefer changes whose effect is verifiable in a microbenchmark rather than in total_ms"}} {"run":26,"commit":"7683ca3","metric":17829.9,"metrics":{"lg_boot_ms":724.4,"lg_decomp_ms":2353.5,"lg_graph_ms":1041,"lg_hex_ms":434.9,"lg_index_ms":95.5,"lg_listing_cold_ms":545.6,"lg_listing_warm_ms":407.9,"lg_nav_ms":6922,"lg_palette_ms":4.8,"lg_render_ms":218.1,"lg_search_ms":884.3,"pure_graph_ms":213.6,"sm_boot_ms":432,"sm_decomp_ms":1250.2,"sm_graph_ms":687.4,"sm_hex_ms":457.5,"sm_index_ms":2.6,"sm_listing_cold_ms":265.4,"sm_listing_warm_ms":288.8,"sm_nav_ms":303.2,"sm_palette_ms":0.3,"sm_render_ms":253.9,"sm_search_ms":43,"fails":0},"status":"keep","description":"Highlight ranges are computed per line on demand instead of for every match. Searching one character over bash matches 177k lines at 310k places, and all but the forty on screen were built and thrown away. _MatchRanges keeps the line SET eagerly and works out the offsets when a line is painted or the cursor lands on it; the blob scan now also skips to the next line after a hit.","timestamp":1786068588431,"segment":4,"confidence":4.5317040951122864,"asi":{"hypothesis":"profiling the search phase directly showed the per-match range building, not the scanning, was the cost","microbench_before_after_ms_per_term":{"note":"targets/bash, 228659 rows, _compute_matches only","m":[139.7,41.6],"mo":[153.9,49.2],"mov":[55.4,47],"c":[329.6,66.1],"ca":[51.5,47.3],"cal":[32.9,30],"call":[29.2,26.8],"haystack_build":[458,459]},"equivalence":"the microbenchmark materialises every range afterwards and prints the totals: 116736 / 76271 / 76070 / 309827 / 25222 / 16312 / 15722 -- identical to the eager version, term for term. Plus .auto/check_search.py (140 prefixes, listing + pseudocode, 0 mismatches) and the full 830-check suite.","api_shape":"_MatchRanges quacks as the dict it replaced for the subset anything uses (in / get / [] / items / len). tests/test_scenarios.py reads dis._ranges.get(cursor) and dis._ranges[cursor][0][0] directly, so that had to keep working -- it does, untouched.","gains":"total 18608 -> 17830 on a box that has got busier since the earlier runs; lg_search 1409 -> 884 (-37%), sm_search 46 -> 43","whats_left_in_search":"the haystack build, 459ms: one _line_plain pass over 228k rows plus a 12.3MB join and lower(). That is the floor unless the plain text is cached with the rows.","next_action_hint":"budget: lg_nav 6922 (39%, and inflated by machine load), decomp 3604, graph 1728, search 927, listing 1508, boot 1156, hex 892, render 472, pure_graph 214."}} {"run":27,"commit":"b2b59e0","metric":17700.4,"metrics":{"lg_boot_ms":776,"lg_decomp_ms":2690.1,"lg_graph_ms":888.1,"lg_hex_ms":449.1,"lg_index_ms":97.4,"lg_listing_cold_ms":439.4,"lg_listing_warm_ms":404.6,"lg_nav_ms":6762.9,"lg_palette_ms":4.6,"lg_render_ms":230.8,"lg_search_ms":762.9,"pure_graph_ms":217.1,"sm_boot_ms":452.6,"sm_decomp_ms":1270,"sm_graph_ms":698,"sm_hex_ms":433.2,"sm_index_ms":2.4,"sm_listing_cold_ms":259.1,"sm_listing_warm_ms":260.5,"sm_nav_ms":300.4,"sm_palette_ms":0.3,"sm_render_ms":258.3,"sm_search_ms":42.7,"fails":0},"status":"keep","description":"Keep a listing row's spans and operand extents exactly as they came off the wire instead of copying them into tuples. The copy re-proved types the worker's own tool guarantees, and it destroyed the object sharing the worker's line cache had created — 228k rows now reference 125k span lists, not 228k private tuples.","timestamp":1786068961469,"segment":4,"confidence":3.5430060816680973,"asi":{"hypothesis":"the tuple conversion in Head.from_raw is the last measurable client-side cost per listing row","microbench":"cold page load of bash's .text, broken into three: worker call 19.85 us/row, client _build_page 3.19 -> 2.51 us/row, index loop 0.37 us/row. The index loop (setdefault + three appends + a _span call per head) is NOT worth touching.","memory":"a full 228 659-row bash listing costs the client ~258MB RSS and now holds 125 510 distinct span objects rather than one per row -- the worker memoises its per-line render and pickle preserves that sharing within a page, which copying threw away.","safety_audit_done_first":"every reader of h.spans / h.ops only iterates or indexes (app.py _span_segments, _cursor_operand, GraphView._draw_node_row, Head.op_at, and four places in tests). Nothing mutates them and nothing hashes a Head -- which matters, because Head is a NamedTuple and a list field would make it unhashable.","gains":"total 17830 -> 17700; lg_search 884 -> 763; lg_graph 1041 -> 888; sm_nav_worst 177 -> 151","verified":"830-check full suite green, .auto/check_search.py 140 prefixes 0 mismatches","state":"the per-row cost is now 19.85us in the worker (of which generate_disasm_line is 5.9) and 2.9us in the client. There is no further client-side lever worth the risk."}} +{"run":28,"commit":"722025a","metric":17464.6,"metrics":{"lg_boot_ms":744.8,"lg_decomp_ms":2600.3,"lg_graph_ms":889.3,"lg_hex_ms":436.1,"lg_index_ms":95.5,"lg_listing_cold_ms":436.4,"lg_listing_warm_ms":407.8,"lg_nav_ms":6747.4,"lg_palette_ms":4.6,"lg_render_ms":218.9,"lg_search_ms":760.9,"pure_graph_ms":215.7,"sm_boot_ms":429.7,"sm_decomp_ms":1263.5,"sm_graph_ms":650.2,"sm_hex_ms":435.3,"sm_index_ms":2.3,"sm_listing_cold_ms":262.8,"sm_listing_warm_ms":264.7,"sm_nav_ms":305.4,"sm_palette_ms":0.3,"sm_render_ms":251.4,"sm_search_ms":41.1,"fails":0},"status":"keep","description":"Confirmation re-run of #27, no code change: 17700 -> 17465, the best v5 reading. Confirms the wire-shape change holds and that the run-to-run spread is ~250ms even on the now-busier box.","timestamp":1786069196018,"segment":4,"confidence":2.702480400976747,"asi":{"hypothesis":"confirm #27 and take a clean reading of the finished state","v5_progress":"18516 -> 17465 (-5.7%) across seven experiments, on a box whose load roughly tripled during them","phase_state_ms":{"lg_nav":6747,"lg_decomp":2600,"sm_decomp":1264,"lg_graph":889,"sm_graph":650,"lg_search":761,"sm_search":41,"lg_listing_cold":436,"lg_listing_warm":408,"lg_boot":745,"sm_boot":430,"hex":871,"render":470,"pure_graph":216},"per_row_cost_model_final":"cold listing paging is 22.7us/row: 19.85 in the worker (generate_disasm_line 5.9 of it, and the _idatui_line_parts cache absorbing the rest) + 2.5 building Heads + 0.37 indexing them. Before this session it was 116us/row.","remaining_levers_all_rejected_with_numbers":{"flags_recomputed_3x_per_head":"0.36us/head = 0.7% of total, needs flags threaded through three functions","wire_shape_dict_vs_tuple":"<=0.3us/row","PAGE_size":"no effect at any of 500/1000/2000","index_loop_in_load_next_page":"0.37us/row total"},"conclusion":"what is left is IDA's generate_disasm_line, Hex-Rays, Textual's compositor, and the fact that a listing row index is a linear count. All four are outside this codebase or would need the architecture change costed in .auto/ideas.md."}} +{"type":"config","name":"ida-tui performance (v6 bench: split view covered)","metricName":"total_ms","metricUnit":"ms","bestDirection":"lower"} diff --git a/.auto/wip-decompmap.patch b/.auto/wip-decompmap.patch new file mode 100644 index 0000000..a11db98 --- /dev/null +++ b/.auto/wip-decompmap.patch @@ -0,0 +1,49 @@ +diff --git a/server/patch_server.py b/server/patch_server.py +index fe16ede..b8f5453 100644 +--- a/server/patch_server.py ++++ b/server/patch_server.py +@@ -884,16 +884,39 @@ def decomp_map( + return {"error": f"decompile failed: {e}"} + if cfunc is None: + return {"error": "decompile failed"} ++ import ida_lines ++ # Three things this loop must not do, each measured on real functions (the 25 ++ # largest of bash went 68.3s -> 6.5s; echo's 60 largest 5.4s -> 0.6s, with ++ # byte-identical output): ++ # ++ # * allocate ctree_item_t's per COLUMN. They are SWIG objects and this is ++ # the innermost loop; one per call is enough, and head/tail are never ++ # read, so don't ask for them at all. ++ # * sweep the TAGGED length. ``x`` is a screen column but ``sl.line`` still ++ # carries IDA's colour tags, so a 23-column line was swept 124 times. ++ # * call dstr() per column. It formats a whole 'EA: description' string, and ++ # consecutive columns are nearly always the same ctree item -- so ask the ++ # item for its id first and only format when it changes. (The result is ++ # deduped by ``seen`` anyway, so skipping a repeat cannot change it.) ++ item = ida_hexrays.ctree_item_t() ++ tag_remove = ida_lines.tag_remove ++ get_line_item = cfunc.get_line_item + lines = [] + for sl in cfunc.get_pseudocode(): + line = sl.line + eas, seen = [], set() +- for x in range(len(line) + 1): +- head = ida_hexrays.ctree_item_t() +- item = ida_hexrays.ctree_item_t() +- tail = ida_hexrays.ctree_item_t() +- if not cfunc.get_line_item(line, x, False, head, item, tail): ++ prev_id = None ++ for x in range(len(tag_remove(line)) + 1): ++ if not get_line_item(line, x, False, None, item, None): + continue ++ it = item.it ++ if it is not None: ++ oid = it.obj_id ++ if oid == prev_id: ++ continue ++ prev_id = oid ++ else: ++ prev_id = None + # Match the /*ea*/ marker's source (decompile_function_safe): the + # item's dstr() is 'EA: description'; get_ea() reports a different ea. + dstr = item.dstr() |