diff options
| -rw-r--r-- | .auto/log.jsonl | 1 | ||||
| -rw-r--r-- | idatui/domain.py | 10 | ||||
| -rw-r--r-- | server/patch_server.py | 22 |
3 files changed, 27 insertions, 6 deletions
diff --git a/.auto/log.jsonl b/.auto/log.jsonl index 3960982..3f620f8 100644 --- a/.auto/log.jsonl +++ b/.auto/log.jsonl @@ -5,3 +5,4 @@ {"run":4,"commit":"b6a95ec","metric":26050.2,"metrics":{"lg_boot_ms":727.4,"lg_decomp_ms":2618.1,"lg_graph_ms":935.1,"lg_hex_ms":954.2,"lg_index_ms":67.4,"lg_listing_cold_ms":459.3,"lg_listing_warm_ms":545.7,"lg_nav_ms":9257.8,"lg_palette_ms":4.7,"lg_render_ms":238.8,"lg_search_ms":5472.5,"pure_graph_ms":530.7,"sm_boot_ms":534.6,"sm_decomp_ms":635.8,"sm_graph_ms":748.1,"sm_hex_ms":858.7,"sm_index_ms":0,"sm_listing_cold_ms":268.7,"sm_listing_warm_ms":269.2,"sm_nav_ms":441.6,"sm_palette_ms":0.3,"sm_render_ms":281.7,"sm_search_ms":200,"fails":0},"status":"keep","description":"_idatui_spans: one capturing re.split over the tag pairs instead of finditer+char-slicing, and collapse whitespace with ' '.join(txt.split()) instead of a regex sub. 13.15 -> 11.07 us/line (the previous finditer attempt was 14.4, i.e. SLOWER than the original char loop it replaced).","timestamp":1786059375617,"segment":0,"confidence":2.0769472107521656,"asi":{"hypothesis":"the span walker can beat the original char loop if the tokenisation is one C-level split and the whitespace collapse avoids re.sub","microbench_us_per_line":{"original_char_loop":13.15,"finditer_attempt":14.42,"re.split_version":11.07},"lesson":"re.finditer per tag is SLOWER than a plain character loop -- Match objects and .start() calls cost more than the ~54 trivial loop iterations they replace. A single capturing re.split that hands back [text, tag, text, ...] is what actually wins.","lesson2":"re.sub for whitespace collapse cost ~1us per call at ~6.5 calls/line; ' '.join(txt.split()) splits on exactly str.isspace() and is far cheaper. Leading/trailing space has to be re-attached by hand to keep cross-span runs collapsing the same way.","equivalence":"0 mismatches vs the pre-autoresearch implementation over 258k real disasm lines on echo/ls_ttl/bash/libcrypto (.auto/diff_spans.py --ref 2b0ae8d)","gains":"total 26924 -> 26050 (-3.2%); lg_nav 10598 -> 9258; lg_nav_worst 10170 -> 8869","work_unchanged":"every NOTES counter identical","next_action_hint":"lg_nav 9.3s and lg_search 5.5s are now the top two. For nav: ListingModel walks 500 heads/call at ~25us/row and the ROW TEXT is entirely wasted when the walk is only trying to reach an address -- a skeleton (ea,size,kind) mode on the heads tool would make ensure_ea nearly free. For search: _compute_matches/_line_plain over 224k rows."}} {"run":5,"commit":"b6a95ec","metric":23259.6,"metrics":{"lg_boot_ms":752.7,"lg_decomp_ms":2475.3,"lg_graph_ms":963,"lg_hex_ms":950.4,"lg_index_ms":70.8,"lg_listing_cold_ms":533.2,"lg_listing_warm_ms":406.1,"lg_nav_ms":6862.6,"lg_palette_ms":4.7,"lg_render_ms":228.9,"lg_search_ms":5401.5,"pure_graph_ms":510.5,"sm_boot_ms":538.8,"sm_decomp_ms":666.7,"sm_graph_ms":715.7,"sm_hex_ms":856.8,"sm_index_ms":0,"sm_listing_cold_ms":260.3,"sm_listing_warm_ms":262.5,"sm_nav_ms":335.5,"sm_palette_ms":0.3,"sm_render_ms":270.8,"sm_search_ms":192.5,"fails":0},"status":"checks_failed","description":"Memoise per-line rendering in the worker (lru_cache on a new _idatui_line_parts) + build listing Heads with their opcode bytes already attached instead of dataclasses.replace-ing them in. total 26050 -> 23260, lg_nav 9258 -> 6863. Reverted: 3 graph_minimap checks fail -- but the cause is a RACE IN THE SCENARIO that the speedup wins, not a functional regression (proved below).","timestamp":1786060327739,"segment":0,"confidence":5.600496684223448,"asi":{"hypothesis":"cache the per-line render (tagged line -> text/spans/ops) in the worker, and stop double-constructing Heads client-side","change_A":"server/patch_server.py: new _idatui_line_parts(line) = (text, spans, ops), functools.lru_cache(16384). bash: 196618 listing lines are only 53363 distinct, so hit rate is ~70% and cost falls 10.4 -> 3.9 us/line. Bonus: pickle memoises the shared span lists so pages serialise smaller.","change_B":"idatui/domain.py: ListingModel._build_page reads the code extent FIRST and passes raw into Head.from_raw, replacing _attach_opcode_bytes' dataclasses.replace (which re-ran __init__ per code head). from_raw now uses tuple(map(tuple,...)) instead of a coercing genexpr.","measured":"cold ListingModel paging 35.7 -> 25.7 us/row; PAGE size (500/1000/2000) makes NO difference, do not bother tuning it","failure_root_cause":"tests/test_scenarios.py graph_minimap. _open_graph() leaves _graph_sticky=True; the scenario then does c.open(big,'listing') and presses space expecting to ENTER the graph. With sticky on, the navigation itself schedules _load_graph, and if that async load lands before the space press then space LEAVES graph mode instead -> the following 60s wait times out (scenario 1.9s -> 65.5s) and every minimap click lands on a hidden widget.","proof":"/tmp/mmrace.py drives the same steps and prints _active right before the space press: NEW code 'after open(big): active=graph', OLD code 'active=listing'. Bisected: stashing idatui/domain.py alone still fails, stashing server/patch_server.py alone passes -> it is purely the speedup winning the race, no behaviour changed.","equivalence_evidence":"diff_spans.py now compares _idatui_head_row too (whole row dict, not just spans): 0 mismatches over 118k lines on bash/echo/ls_ttl vs pre-autoresearch HEAD 2b0ae8d","work_preserved":".auto/wip-headcache.patch holds the reverted diff","next_action_hint":"re-apply the patch and make the graph_minimap SETUP deterministic (clear _graph_sticky before the second navigation). Assertions untouched; graph_sticky scenario already covers sticky behaviour. Record the amended tests/ rule in .auto/prompt.md."}} {"run":6,"commit":"cf45e11","metric":22980.2,"metrics":{"lg_boot_ms":738.2,"lg_decomp_ms":2401.8,"lg_graph_ms":944.1,"lg_hex_ms":920.6,"lg_index_ms":75.2,"lg_listing_cold_ms":538.5,"lg_listing_warm_ms":411.1,"lg_nav_ms":6813.9,"lg_palette_ms":4.9,"lg_render_ms":221.8,"lg_search_ms":5630.1,"pure_graph_ms":240.7,"sm_boot_ms":537.5,"sm_decomp_ms":595.1,"sm_graph_ms":715.7,"sm_hex_ms":858.8,"sm_index_ms":0,"sm_listing_cold_ms":263.3,"sm_listing_warm_ms":265.3,"sm_nav_ms":335.2,"sm_palette_ms":0.3,"sm_render_ms":271.4,"sm_search_ms":196.5,"fails":0},"status":"keep","description":"Re-apply #5 (lru_cache on the per-line render + Heads built with their opcode bytes already attached) with the graph_minimap scenario's racy SETUP made deterministic: clear _graph_sticky before the second navigation so Space is known to be entering the graph, not leaving it. No assertion changed.","timestamp":1786060531752,"segment":0,"confidence":7.006489167396753,"asi":{"hypothesis":"the graph_minimap failure in #5 was a racy scenario setup, not lost functionality","proof_a_bisect":"stashing idatui/domain.py alone still failed; stashing server/patch_server.py alone passed -> the flip is caused purely by the backend getting faster","proof_b_race":"/tmp/mmrace.py replays the scenario's steps outside the suite and prints _active just before the Space press: NEW 'active=graph', OLD 'active=listing'. Same steps, two states. With sticky on, the navigation itself schedules _load_graph; whether it lands before the keypress decides whether Space enters or leaves graph mode.","proof_c_both_ways":"the repaired scenario passes on the fast code AND on the stashed slow code (graph_minimap + graph_sticky, 15 passed 0 failed)","test_edit_scope":"two setup lines (app._graph_sticky = False; wait for _active == listing). Every c.check is byte-identical. graph_sticky scenario still covers sticky navigation.","real_bug_noted":"there IS a genuine UX wart underneath: with sticky graph mode on, a keypress right after a navigation means something different depending on whether the async graph reload has landed. Out of scope for perf work -> .auto/ideas.md","gains":"total 26050 -> 22980 (-11.8%); lg_nav 9258 -> 6814; lg_nav_worst 8869 -> 6613; sm_nav_worst 213 -> 128; cumulative vs baseline -50.7%","work_unchanged":"every NOTES counter identical to baseline","next_action_hint":"lg_search_ms 5630 is now the largest single term after lg_nav 6814. Search runs _compute_matches + _line_plain over 224k rows client-side; profile SearchMixin._compute_matches / ListingView._line_plain next."}} +{"run":7,"commit":"5045ba1","metric":20835.7,"metrics":{"lg_boot_ms":754.4,"lg_decomp_ms":2593.7,"lg_graph_ms":929.5,"lg_hex_ms":1080.2,"lg_index_ms":76.1,"lg_listing_cold_ms":526.7,"lg_listing_warm_ms":410.1,"lg_nav_ms":6674.8,"lg_palette_ms":5,"lg_render_ms":225.2,"lg_search_ms":3310.3,"pure_graph_ms":239.3,"sm_boot_ms":537.9,"sm_decomp_ms":630.1,"sm_graph_ms":734.3,"sm_hex_ms":869.9,"sm_index_ms":0,"sm_listing_cold_ms":260.8,"sm_listing_warm_ms":262.3,"sm_nav_ms":334.8,"sm_palette_ms":0.3,"sm_render_ms":268.4,"sm_search_ms":111.6,"fails":0},"status":"keep","description":"Incremental search narrows instead of rescanning. Typing a character onto the term can only remove lines (a line holding \"mov\" holds \"mo\"), so _compute_matches rescans the previous hit list when the term grew and nothing else moved. Keyed on (term, case-fold, row count, line-source id) so a listing still streaming rows in behind the search falls back to a full scan.","timestamp":1786060806344,"segment":0,"confidence":8.383192182410422,"asi":{"hypothesis":"as-you-type search rescans every row per keystroke; the match set is monotonically shrinking so it need not","gains":"total 22980 -> 20836 (-9.3%); lg_search 5630 -> 3310 (-41%); sm_search 196 -> 112 (-43%)","results_identical":"search_hits 91783 (bash) / 2461 (echo) unchanged from baseline -- the same lines still match","equivalence_test":"/tmp/searcheq.py drives the real ListingView keystroke by keystroke over 9 terms (mov/call/rsp/Mov/1a/push/e/lea/sub_) and compares the narrowed _matches AND _ranges against a forced full rescan at every prefix: 0 mismatches","invalidation_traps":["case folding is per-term (_ci = term.islower()) and can FLIP as you type: '1' -> '1a' goes False -> True, which can ADD matches. The key stores _ci and only narrows when it is unchanged.","the listing streams rows in behind the search, so row count is in the key -- rows that arrived since the last pass have never been looked at.","action_toggle_opcodes changes the plain line's prefix, so it changes WHICH rows match (the opcode hex is searchable text), not just the highlight offsets. It now clears the key too.","every site that resets _matches/_ranges must reset _matched_key, or a cancelled search leaves a stale prefix and the next search narrows from an empty list. Six sites."],"next_action_hint":"lg_nav 6675 is again the biggest term, then lg_decomp 2594 and lg_search 3310. For search what is left is _line_plain: it rebuilds the whole formatted row (address gutter + opcode field + name prefix + text) per row on the first, unnarrowed pass. For nav see .auto/ideas.md (skeleton walk)."}} diff --git a/idatui/domain.py b/idatui/domain.py index 1c7883f..645d111 100644 --- a/idatui/domain.py +++ b/idatui/domain.py @@ -90,10 +90,16 @@ class Line: ) -@dataclass(frozen=True) +@dataclass(frozen=True, slots=True) class Head: """One flat-listing item (from the ``heads`` server tool): a code - instruction, a data item, or an undefined byte run.""" + instruction, a data item, or an undefined byte run. + + ``slots=True`` because this is the most-constructed object in the codebase: + a jump to an address near the end of a big binary builds one per listing row + it walks past, hundreds of thousands of them, and the slotted layout is ~20% + cheaper to build (and smaller to hold). + """ ea: int kind: str # 'code' | 'data' | 'unknown' | 'member' diff --git a/server/patch_server.py b/server/patch_server.py index a1ec1ef..fe16ede 100644 --- a/server/patch_server.py +++ b/server/patch_server.py @@ -391,6 +391,10 @@ def _idatui_tag_map(): _IDATUI_TAGS = None _IDATUI_OPND_TAGS = None _IDATUI_CTL = None # re: a tag = one of three control chars plus its argument +#: {tag character: (kind, operand index or None)} -- the two maps above merged, +#: because the span walker wants both for the same tag and a dict lookup per +#: tag per line is one of the few things it does often enough to matter. +_IDATUI_TAGINFO = None def _idatui_opnd_tag_map(): @@ -420,7 +424,7 @@ def _idatui_spans(line): Unknown tags become 'text' rather than being dropped: a processor module can emit a colour we don't classify, and losing the characters would corrupt the line.""" - global _IDATUI_TAGS, _IDATUI_OPND_TAGS, _IDATUI_CTL + global _IDATUI_TAGS, _IDATUI_OPND_TAGS, _IDATUI_CTL, _IDATUI_TAGINFO import ida_lines if _IDATUI_TAGS is None: _IDATUI_TAGS = _idatui_tag_map() @@ -434,7 +438,12 @@ def _idatui_spans(line): # characters but only ~13 tags -- everything between two tags is already # exactly one span's worth of text. _IDATUI_CTL = _re.compile("([\\x01\\x02\\x03](?s:.))") - tags, opnds = _IDATUI_TAGS, _IDATUI_OPND_TAGS + if _IDATUI_TAGINFO is None: + _IDATUI_TAGINFO = { + tag: (_IDATUI_TAGS.get(tag, "text"), _IDATUI_OPND_TAGS.get(tag)) + for tag in set(_IDATUI_TAGS) | set(_IDATUI_OPND_TAGS)} + taginfo = _IDATUI_TAGINFO + plain_tag = ("text", None) on, off, esc = "\x01", "\x02", "\x03" addr_tag = chr(getattr(ida_lines, "COLOR_ADDR", 0x28)) addr_len = int(getattr(ida_lines, "COLOR_ADDR_SIZE", 16)) @@ -478,8 +487,7 @@ def _idatui_spans(line): pend = "" if ch == on: stack.append((kind, opnd)) - kind = tags.get(tag, "text") - o = opnds.get(tag) + kind, o = taginfo.get(tag, plain_tag) if o is not None: opnd = o # operands nest: an inner colour keeps the operand elif stack: @@ -497,6 +505,12 @@ def _idatui_spans(line): out, prev_space = [], False for kind, txt, opnd in spans: core = " ".join(txt.split()) + if core == txt: + # Nothing to collapse and no edge whitespace -- which is the common + # case ("mov", "rax", ", ") and skips both isspace() probes below. + prev_space = False + out.append([kind, txt, opnd]) + continue if not core: # the span is nothing but padding if not prev_space: prev_space = True |