diff options
| -rw-r--r-- | .auto/log.jsonl | 1 | ||||
| -rw-r--r-- | server/patch_server.py | 33 |
2 files changed, 29 insertions, 5 deletions
diff --git a/.auto/log.jsonl b/.auto/log.jsonl index 4a618f5..9bcce42 100644 --- a/.auto/log.jsonl +++ b/.auto/log.jsonl @@ -32,3 +32,4 @@ {"run":27,"commit":"b2b59e0","metric":17700.4,"metrics":{"lg_boot_ms":776,"lg_decomp_ms":2690.1,"lg_graph_ms":888.1,"lg_hex_ms":449.1,"lg_index_ms":97.4,"lg_listing_cold_ms":439.4,"lg_listing_warm_ms":404.6,"lg_nav_ms":6762.9,"lg_palette_ms":4.6,"lg_render_ms":230.8,"lg_search_ms":762.9,"pure_graph_ms":217.1,"sm_boot_ms":452.6,"sm_decomp_ms":1270,"sm_graph_ms":698,"sm_hex_ms":433.2,"sm_index_ms":2.4,"sm_listing_cold_ms":259.1,"sm_listing_warm_ms":260.5,"sm_nav_ms":300.4,"sm_palette_ms":0.3,"sm_render_ms":258.3,"sm_search_ms":42.7,"fails":0},"status":"keep","description":"Keep a listing row's spans and operand extents exactly as they came off the wire instead of copying them into tuples. The copy re-proved types the worker's own tool guarantees, and it destroyed the object sharing the worker's line cache had created — 228k rows now reference 125k span lists, not 228k private tuples.","timestamp":1786068961469,"segment":4,"confidence":3.5430060816680973,"asi":{"hypothesis":"the tuple conversion in Head.from_raw is the last measurable client-side cost per listing row","microbench":"cold page load of bash's .text, broken into three: worker call 19.85 us/row, client _build_page 3.19 -> 2.51 us/row, index loop 0.37 us/row. The index loop (setdefault + three appends + a _span call per head) is NOT worth touching.","memory":"a full 228 659-row bash listing costs the client ~258MB RSS and now holds 125 510 distinct span objects rather than one per row -- the worker memoises its per-line render and pickle preserves that sharing within a page, which copying threw away.","safety_audit_done_first":"every reader of h.spans / h.ops only iterates or indexes (app.py _span_segments, _cursor_operand, GraphView._draw_node_row, Head.op_at, and four places in tests). Nothing mutates them and nothing hashes a Head -- which matters, because Head is a NamedTuple and a list field would make it unhashable.","gains":"total 17830 -> 17700; lg_search 884 -> 763; lg_graph 1041 -> 888; sm_nav_worst 177 -> 151","verified":"830-check full suite green, .auto/check_search.py 140 prefixes 0 mismatches","state":"the per-row cost is now 19.85us in the worker (of which generate_disasm_line is 5.9) and 2.9us in the client. There is no further client-side lever worth the risk."}} {"run":28,"commit":"722025a","metric":17464.6,"metrics":{"lg_boot_ms":744.8,"lg_decomp_ms":2600.3,"lg_graph_ms":889.3,"lg_hex_ms":436.1,"lg_index_ms":95.5,"lg_listing_cold_ms":436.4,"lg_listing_warm_ms":407.8,"lg_nav_ms":6747.4,"lg_palette_ms":4.6,"lg_render_ms":218.9,"lg_search_ms":760.9,"pure_graph_ms":215.7,"sm_boot_ms":429.7,"sm_decomp_ms":1263.5,"sm_graph_ms":650.2,"sm_hex_ms":435.3,"sm_index_ms":2.3,"sm_listing_cold_ms":262.8,"sm_listing_warm_ms":264.7,"sm_nav_ms":305.4,"sm_palette_ms":0.3,"sm_render_ms":251.4,"sm_search_ms":41.1,"fails":0},"status":"keep","description":"Confirmation re-run of #27, no code change: 17700 -> 17465, the best v5 reading. Confirms the wire-shape change holds and that the run-to-run spread is ~250ms even on the now-busier box.","timestamp":1786069196018,"segment":4,"confidence":2.702480400976747,"asi":{"hypothesis":"confirm #27 and take a clean reading of the finished state","v5_progress":"18516 -> 17465 (-5.7%) across seven experiments, on a box whose load roughly tripled during them","phase_state_ms":{"lg_nav":6747,"lg_decomp":2600,"sm_decomp":1264,"lg_graph":889,"sm_graph":650,"lg_search":761,"sm_search":41,"lg_listing_cold":436,"lg_listing_warm":408,"lg_boot":745,"sm_boot":430,"hex":871,"render":470,"pure_graph":216},"per_row_cost_model_final":"cold listing paging is 22.7us/row: 19.85 in the worker (generate_disasm_line 5.9 of it, and the _idatui_line_parts cache absorbing the rest) + 2.5 building Heads + 0.37 indexing them. Before this session it was 116us/row.","remaining_levers_all_rejected_with_numbers":{"flags_recomputed_3x_per_head":"0.36us/head = 0.7% of total, needs flags threaded through three functions","wire_shape_dict_vs_tuple":"<=0.3us/row","PAGE_size":"no effect at any of 500/1000/2000","index_loop_in_load_next_page":"0.37us/row total"},"conclusion":"what is left is IDA's generate_disasm_line, Hex-Rays, Textual's compositor, and the fact that a listing row index is a linear count. All four are outside this codebase or would need the architecture change costed in .auto/ideas.md."}} {"type":"config","name":"ida-tui performance (v6 bench: split view covered)","metricName":"total_ms","metricUnit":"ms","bestDirection":"lower"} +{"run":29,"commit":"7ea3ca1","metric":33502.3,"metrics":{"lg_boot_ms":772.2,"lg_decomp_ms":2546.9,"lg_graph_ms":903.4,"lg_hex_ms":434.1,"lg_index_ms":97.7,"lg_listing_cold_ms":420.2,"lg_listing_warm_ms":397.4,"lg_nav_ms":6501.1,"lg_palette_ms":4.9,"lg_render_ms":212.4,"lg_search_ms":760.4,"lg_split_ms":10189.2,"pure_graph_ms":212.3,"sm_boot_ms":443.8,"sm_decomp_ms":1275.3,"sm_graph_ms":720.5,"sm_hex_ms":451,"sm_index_ms":2.5,"sm_listing_cold_ms":263.3,"sm_listing_warm_ms":264.7,"sm_nav_ms":302.8,"sm_palette_ms":0.3,"sm_render_ms":248.4,"sm_search_ms":43.9,"sm_split_ms":6033.6,"fails":0},"status":"keep","description":"RE-BASELINE (v6 bench). The split view ('s') was not covered at all, and it turns out to be the most expensive thing in the app: 16.2s of a 33.5s session (lg_split 10189 + sm_split 6034), or 500ms per function on echo and 850ms on bash, just to open it.","timestamp":1786070710925,"segment":5,"confidence":null,"asi":{"hypothesis":"the bench covers listing, decomp, graph, hex, search, nav and boot -- but not the split view, and an earlier aside measured decomp_map at 280ms per function","finding":"split is 48% of the whole benchmarked session once measured. 12 echo functions take 6.0s to open side by side; 12 bash functions take 10.2s.","why_it_was_missed":"the phase list was written from the README's headline features and 's' was not one of them. Same class of gap as v2 (graph opens timed a cache hit): if a feature is not in the bench, its cost is invisible however carefully you profile the ones that are.","cause_already_diagnosed":"server/patch_server.py decomp_map sweeps EVERY COLUMN of every pseudocode line, allocating three ctree_item_t SWIG objects per column and calling item.dstr() (which formats a whole 'EA: description' string) each time. Three separate wastes: the allocations, sweeping len(sl.line) which is the TAGGED length (124 columns for a 23-column line), and re-formatting for columns that report the same ctree item.","fix_ready":".auto/wip-decompmap.patch -- verified byte-identical over 165 functions across echo/ls_ttl/bash, 8.9-9.8x faster with Hex-Rays warm for both sides (bash's 25 largest: 67.4s -> 6.8s of sweeping)","next_action_hint":"apply it"}} diff --git a/server/patch_server.py b/server/patch_server.py index fe16ede..b8f5453 100644 --- a/server/patch_server.py +++ b/server/patch_server.py @@ -884,16 +884,39 @@ def decomp_map( return {"error": f"decompile failed: {e}"} if cfunc is None: return {"error": "decompile failed"} + import ida_lines + # Three things this loop must not do, each measured on real functions (the 25 + # largest of bash went 68.3s -> 6.5s; echo's 60 largest 5.4s -> 0.6s, with + # byte-identical output): + # + # * allocate ctree_item_t's per COLUMN. They are SWIG objects and this is + # the innermost loop; one per call is enough, and head/tail are never + # read, so don't ask for them at all. + # * sweep the TAGGED length. ``x`` is a screen column but ``sl.line`` still + # carries IDA's colour tags, so a 23-column line was swept 124 times. + # * call dstr() per column. It formats a whole 'EA: description' string, and + # consecutive columns are nearly always the same ctree item -- so ask the + # item for its id first and only format when it changes. (The result is + # deduped by ``seen`` anyway, so skipping a repeat cannot change it.) + item = ida_hexrays.ctree_item_t() + tag_remove = ida_lines.tag_remove + get_line_item = cfunc.get_line_item lines = [] for sl in cfunc.get_pseudocode(): line = sl.line eas, seen = [], set() - for x in range(len(line) + 1): - head = ida_hexrays.ctree_item_t() - item = ida_hexrays.ctree_item_t() - tail = ida_hexrays.ctree_item_t() - if not cfunc.get_line_item(line, x, False, head, item, tail): + prev_id = None + for x in range(len(tag_remove(line)) + 1): + if not get_line_item(line, x, False, None, item, None): continue + it = item.it + if it is not None: + oid = it.obj_id + if oid == prev_id: + continue + prev_id = oid + else: + prev_id = None # Match the /*ea*/ marker's source (decompile_function_safe): the # item's dstr() is 'EA: description'; get_ea() reports a different ea. dstr = item.dstr() |