diff options
| author | user <user@clank> | 2026-08-07 03:05:33 +0200 |
|---|---|---|
| committer | user <user@clank> | 2026-08-07 03:05:33 +0200 |
| commit | ba96500478165710ab027522a685308357049175 (patch) | |
| tree | b647ea42df34e3dec2c6d2d4c38ff46c7a521e28 | |
| parent | Re-run of #16 (keep the joined search body across a cancelled search and acro... (diff) | |
| download | ida-tui-ba96500478165710ab027522a685308357049175.tar.gz ida-tui-ba96500478165710ab027522a685308357049175.tar.xz ida-tui-ba96500478165710ab027522a685308357049175.zip | |
Re-run of #18 (_CellRow slice assignment, one-string box borders, memoised Style sum), confirming it: 17590 -> 17501. lg_graph averages 951 over the two runs against 1136 before.
Result: {"status":"keep","total_ms":17501.4,"lg_boot_ms":711,"lg_decomp_ms":2484.8,"lg_graph_ms":1017.2,"lg_hex_ms":431.9,"lg_index_ms":98.7,"lg_listing_cold_ms":528,"lg_listing_warm_ms":409.2,"lg_nav_ms":6488.2,"lg_palette_ms":4.7,"lg_render_ms":214.8,"lg_search_ms":1473.5,"pure_graph_ms":244.3,"sm_boot_ms":451.4,"sm_decomp_ms":596,"sm_graph_ms":691.3,"sm_hex_ms":444.5,"sm_index_ms":0,"sm_listing_cold_ms":263,"sm_listing_warm_ms":284.2,"sm_nav_ms":374.7,"sm_palette_ms":0.3,"sm_render_ms":252.1,"sm_search_ms":37.5,"fails":0}
| -rw-r--r-- | .auto/log.jsonl | 2 | ||||
| -rw-r--r-- | .auto/wip-cellrow.patch | 94 | ||||
| -rw-r--r-- | idatui/app.py | 56 |
3 files changed, 138 insertions, 14 deletions
diff --git a/.auto/log.jsonl b/.auto/log.jsonl index 3aa42c4..8a3a0de 100644 --- a/.auto/log.jsonl +++ b/.auto/log.jsonl @@ -16,3 +16,5 @@ {"run":14,"commit":"2686901","metric":18618.9,"metrics":{"lg_boot_ms":689.6,"lg_decomp_ms":2560,"lg_graph_ms":1035.1,"lg_hex_ms":684.2,"lg_index_ms":100.4,"lg_listing_cold_ms":551.1,"lg_listing_warm_ms":397.8,"lg_nav_ms":6687.8,"lg_palette_ms":4.7,"lg_render_ms":211.9,"lg_search_ms":1858.2,"pure_graph_ms":238.5,"sm_boot_ms":431.4,"sm_decomp_ms":671.2,"sm_graph_ms":728.1,"sm_hex_ms":552,"sm_index_ms":0,"sm_listing_cold_ms":257.4,"sm_listing_warm_ms":281,"sm_nav_ms":365.9,"sm_palette_ms":0.3,"sm_render_ms":242.8,"sm_search_ms":69.6,"fails":0},"status":"keep","description":"Search the whole segment as ONE joined string. Every line is concatenated once (with a start-offset table) so finding a term is a C-level str.find walk instead of a python loop that rebuilds and case-folds 224k lines per keystroke. Falls back to the per-line loop if case-folding changes the string's length.","timestamp":1786063291908,"segment":1,"confidence":41.940433212996666,"asi":{"hypothesis":"per-keystroke search should be one C string scan over the body, not 224k python iterations","design":"SearchMixin._search_haystack builds (starts, blob, blob.lower()) once per (row count, line-source) and caches it; _compute_matches then walks it with str.find and advances a line pointer monotonically (find() only moves forward, so no bisect is needed). A term can never straddle a line because an Input cannot contain a newline.","safety":["str.lower() can CHANGE LENGTH for a few unicode codepoints, which would corrupt every offset after them -- if len differs the haystack is refused and the old per-line loop runs.","the per-line loop (with prefix narrowing) is kept as the fallback and is still exercised.","every site that resets _matches/_ranges now calls _reset_search_cache(), which drops the joined body too -- a stale body would search text the view no longer shows."],"equivalence_test":"/tmp/searcheq2.py drives the real ListingView AND DecompView over 13 terms x every prefix, comparing _matches and _ranges from the haystack path against the same view with _search_haystack monkeypatched to None: 0 mismatches on echo (5952 lines) and ls_ttl (28807 lines)","gains":"total 18857 -> 18619 (-1.3%); lg_search 2196 -> 1858 (-15%); sm_search 79 -> 70","why_not_more":"the bench types only two terms, so the one-off cost of building the body (a _line_plain pass over 224k rows, ~0.5s) is amortised over very little. The benefit compounds for a user who searches more than twice -- the third term onwards is milliseconds.","next_action_hint":"lg_nav 6688 is 36%. Client-side it is pickle.loads 2.4us + _build_page 4.5us per row, and it CANNOT be pipelined: idatui/worker.py's serve() accepts one connection at a time and idalib is main-thread only, so nothing overlaps. Try making Head a NamedTuple (tuple.__new__ vs a frozen dataclass __init__)."}} {"run":15,"commit":"625b067","metric":17784.1,"metrics":{"lg_boot_ms":680.8,"lg_decomp_ms":2344,"lg_graph_ms":1106.5,"lg_hex_ms":576.4,"lg_index_ms":96.5,"lg_listing_cold_ms":552.9,"lg_listing_warm_ms":431.3,"lg_nav_ms":6256.4,"lg_palette_ms":4.8,"lg_render_ms":224.1,"lg_search_ms":1916.5,"pure_graph_ms":241.3,"sm_boot_ms":442.6,"sm_decomp_ms":591.3,"sm_graph_ms":663.9,"sm_hex_ms":419.2,"sm_index_ms":0,"sm_listing_cold_ms":258.2,"sm_listing_warm_ms":281.8,"sm_nav_ms":374.4,"sm_palette_ms":0.3,"sm_render_ms":251.1,"sm_search_ms":69.8,"fails":0},"status":"keep","description":"HexView.render_line emits style RUNS instead of one Segment per byte cell (35 -> 7 segments per row), and Head is a NamedTuple rather than a frozen dataclass (tuple.__new__ 1.9us vs a dataclass __init__ 2.9us, and it is built once per listing row walked).","timestamp":1786063768655,"segment":1,"confidence":42.57587221521688,"asi":{"hypothesis":"a hex frame costs ~10ms of which only 1.8ms is our render_line, so the cost is what we hand the compositor: 1540 one-cell Segments per frame","hex_change":"accumulate a run and flush it when the style changes. A row's 32 cells almost always share one style (the exception is the single cursor cell and trace-live bytes), so 35 segments/row became 7.","hex_equivalence":"/tmp/hexeq.py expands both the new and the pre-change render_line to (char, style) PER CELL and compares, over 40 scroll frames x 43 rows with the cursor moving through all 16 columns: 0 mismatches in 1720 rows. Comparing segments would have been the wrong test -- merging changes the segments on purpose; what must not change is the cells.","head_change":"Head is a NamedTuple. Measured 2.93 -> 1.88 us to build; attribute reads go 10ns -> 20ns, which is the right way round (a quarter-million rows are built per far jump, and a viewport reads forty). _build_page 4.68 -> 4.05 us/row. Nothing used dataclasses.replace/asdict on it.","gains":"total 18619 -> 17784 (-4.5%); hex lg+sm 1236 -> 996 (-19%); lg_nav 6688 -> 6256; lg_decomp 2560 -> 2344","rejected_this_round":"merging same-kind adjacent spans in _idatui_spans: measured only 2.5% fewer spans on 20k real lines (6.43 -> 6.27 per line). Not worth changing the wire format for.","rejected_earlier":"deferring the 'decompiling...' loading cover until ~120ms (worth ~9ms per F5) -- tests/test_scenarios.py asserts the overlay is raised SYNCHRONOUSLY by F5, guarding a real past regression. That is an assertion, not setup, so it stands.","next_action_hint":"lg_nav 6256 (35%) is the paging floor; the skeleton idea was costed and only nets ~5% because search then has to fetch the text anyway (see .auto/ideas.md). Left: ListingView.render_line segment count, and the search haystack building rows one lock at a time (model.window would amortise it)."}} {"run":16,"commit":"625b067","metric":17821,"metrics":{"lg_boot_ms":743.4,"lg_decomp_ms":2349.8,"lg_graph_ms":1033.5,"lg_hex_ms":432.2,"lg_index_ms":97.4,"lg_listing_cold_ms":424.8,"lg_listing_warm_ms":508.5,"lg_nav_ms":6677.8,"lg_palette_ms":4.8,"lg_render_ms":223.1,"lg_search_ms":1374.9,"pure_graph_ms":523.7,"sm_boot_ms":438.6,"sm_decomp_ms":591.2,"sm_graph_ms":713.2,"sm_hex_ms":466.7,"sm_index_ms":0,"sm_listing_cold_ms":262.4,"sm_listing_warm_ms":287.1,"sm_nav_ms":376.6,"sm_palette_ms":0.3,"sm_render_ms":257.5,"sm_search_ms":33.5,"fails":0},"status":"discard","description":"Keep the joined search body across a cancelled search and across navigation inside the same segment (drop it only when the model changes or the opcode column toggles). lg_search 1917 -> 1375, sm_search 70 -> 34 — but total_ms is FLAT (17784 -> 17821) because pure_graph (+283) and lg_nav (+421) drifted, neither of which this touches. Re-running to separate the win from the drift.","timestamp":1786064057632,"segment":1,"confidence":16.322294738538417,"asi":{"hypothesis":"the joined search body is keyed by (row count, line source), so ending a search or navigating inside the same segment need not throw it away","phase_evidence":"lg_search -542ms and sm_search -36ms, i.e. the intended effect happened and is far outside that phase's own spread","why_total_did_not_move":"pure_graph 241 -> 524 and lg_nav 6256 -> 6678 in the same run, and this change touches neither. pure_graph is pure CPU with no I/O and is the single jumpiest metric in the suite (seen at 235, 241, 508, 524, 531 across runs with identical code).","work_preserved":".auto/wip-searchbody.patch","gate_strengthened":".auto/check_search.py added and wired into checks.sh: it drives the real ListingView/DecompView and compares the narrowing + haystack fast paths against the plain per-line loop for every prefix of 14 terms, including after toggling the opcode column and after navigating. 140 prefixes, 0 mismatches. A stale cache still returns AN answer, so the scenario suite could never have caught this class of bug.","invalidation_rules_now":["body dropped when ListingView.load gets a DIFFERENT model (navigation inside one segment reuses it)","body dropped by action_toggle_opcodes -- the opcode hex is searchable text and its width changes without the row count or model moving","body dropped by DecompView.show, explicitly rather than relying on id(self._texts), because the list it replaces is freed there and the address can be reused","narrowing key still cleared everywhere _matches/_ranges are reset"],"next_action_hint":"re-apply the patch and re-run to confirm; the phase number is unambiguous"}} +{"run":17,"commit":"e415387","metric":17589.8,"metrics":{"lg_boot_ms":710.6,"lg_decomp_ms":2404.9,"lg_graph_ms":1136.1,"lg_hex_ms":425.6,"lg_index_ms":103.4,"lg_listing_cold_ms":417.2,"lg_listing_warm_ms":508.1,"lg_nav_ms":6553.8,"lg_palette_ms":4.8,"lg_render_ms":215.7,"lg_search_ms":1400.3,"pure_graph_ms":239.3,"sm_boot_ms":443.2,"sm_decomp_ms":668.4,"sm_graph_ms":698.1,"sm_hex_ms":443.6,"sm_index_ms":0,"sm_listing_cold_ms":267.3,"sm_listing_warm_ms":286,"sm_nav_ms":356.6,"sm_palette_ms":0.3,"sm_render_ms":258.6,"sm_search_ms":47.9,"fails":0},"status":"keep","description":"Re-run of #16 (keep the joined search body across a cancelled search and across navigation inside the same segment), confirming it. total 17784 -> 17590; lg_search 1917 -> 1400, sm_search 70 -> 48. Also lands .auto/check_search.py in the checks gate: it compares both search fast paths against the plain per-line loop for every typed prefix.","timestamp":1786064181968,"segment":1,"confidence":12.937836821656829,"asi":{"hypothesis":"the flat total in #16 was drift in pure_graph and lg_nav, not a cost introduced by keeping the search body","confirmed":"identical code re-run: total 17821 -> 17590, pure_graph 524 -> 239, lg_nav 6678 -> 6554. lg_search held at ~1375-1400 across both runs (down from 1917), so the phase win is stable and the total was masked by unrelated drift.","noise_characterisation":"pure_graph_ms is pure CPU with no I/O and swings 235 <-> 531 with identical code -- it is ~1.5% of the total on its own. lg_nav swings ~+/-350ms (5%). Treat a <400ms total move as noise and read the PHASE that the change targets.","gate_strengthened":".auto/check_search.py now runs in checks.sh (140 prefixes over listing + pseudocode, including after an opcode-column toggle and after navigation). Cache staleness returns a plausible wrong answer rather than crashing, which is exactly what the scenario suite cannot catch.","gains":"total 17784 -> 17590 (-1.1%); lg_search 1917 -> 1400 (-27%); sm_search 70 -> 48 (-31%). Against the v2 baseline: -37.0%.","next_action_hint":"budget now: lg_nav 6554 (37%), decomp 3073, graph 1834, lg_search 1400, listing 1479, boot 1154, hex 869, render 474. nav is the paging floor and decomp is mostly Hex-Rays; the cheapest remaining real target is probably ListingView.render_line's segment count (the hex run-merge trick paid off there)."}} +{"run":18,"commit":"e415387","metric":17890.1,"metrics":{"lg_boot_ms":685.6,"lg_decomp_ms":2604.2,"lg_graph_ms":885.8,"lg_hex_ms":608,"lg_index_ms":96.4,"lg_listing_cold_ms":538,"lg_listing_warm_ms":415.7,"lg_nav_ms":6756.8,"lg_palette_ms":4.8,"lg_render_ms":211,"lg_search_ms":1370.1,"pure_graph_ms":247.9,"sm_boot_ms":431.2,"sm_decomp_ms":670.7,"sm_graph_ms":699.6,"sm_hex_ms":432.7,"sm_index_ms":0,"sm_listing_cold_ms":261.9,"sm_listing_warm_ms":287,"sm_nav_ms":378.1,"sm_palette_ms":0.3,"sm_render_ms":247.6,"sm_search_ms":56.9,"fails":0},"status":"discard","description":"_CellRow.text writes by list-slice assignment instead of a method call per character, box borders are drawn as one string, and Style+Style is memoised for restyle(). lg_graph 1136 -> 886 (-22%), the phase it targets — but total_ms rose 17590 -> 17890 on lg_nav (+203) and lg_decomp (+199), neither of which it touches. Re-running to tell the win from the drift.","timestamp":1786064561542,"segment":1,"confidence":19.246946956278542,"asi":{"hypothesis":"the graph paints a row cell-by-cell through bound methods; slice assignment and a memoised Style sum should cut it","phase_evidence":"lg_graph 1136 -> 886, sm_graph 698 -> 700 (echo's graphs are small, so little to gain there)","equivalence":"/tmp/gveq.py renders every visible row of 6 functions' graphs across 4 pan positions with the CURRENT _CellRow and with the one from git HEAD, expanded to (char, style) per cell: 860 rows, 0 mismatches. Graph scenarios: 50 passed 0 failed.","noise_problem":"the total's spread is now bigger than most single wins. Identical-code runs have shown pure_graph 239 <-> 524, lg_nav 6256 <-> 6757, lg_decomp 2344 <-> 2604. Call it +/-500ms on 17.6s (~3%); read the targeted PHASE, and confirm a keep with a second run.","work_preserved":".auto/wip-cellrow.patch","next_action_hint":"re-apply and re-run. Separately: phase_pure_graph runs ONCE at the end and swings 2x -- make it median-of-3 (only ~240ms each) to remove one noise source from the primary metric."}} diff --git a/.auto/wip-cellrow.patch b/.auto/wip-cellrow.patch new file mode 100644 index 0000000..cb46dae --- /dev/null +++ b/.auto/wip-cellrow.patch @@ -0,0 +1,94 @@ +diff --git a/idatui/app.py b/idatui/app.py +index 53dd79b..6c0a872 100644 +--- a/idatui/app.py ++++ b/idatui/app.py +@@ -2161,6 +2161,13 @@ _GPAD = 1 # columns of padding inside a box + _MINI_W, _MINI_H = 30, 14 + + ++#: Memo for ``Style + Style``. Rich rebuilds a whole Style on every ``+``, and ++#: the graph merges an overlay (cursor band, trail, highlight) over a run of ++#: cells that share only a handful of base styles -- so the same pair is ++#: recombined hundreds of times per frame. ++_STYLE_SUM: dict[tuple, Style] = {} ++ ++ + class _CellRow: + """A row of (char, style) cells that coalesces into a Strip. + +@@ -2183,13 +2190,39 @@ class _CellRow: + self.st[i] = style + + def text(self, i: int, s: str, style: Style) -> None: +- for k, c in enumerate(s): +- self.put(i + k, c, style) ++ """Write ``s`` at cell ``i``, clipped to the row. ++ ++ Slice assignment rather than a call per character: a graph row is drawn ++ from box borders, instruction text and the minimap, and doing it a cell ++ at a time made painting one frame thousands of bound-method calls. ++ Assigning a str to a list slice expands it to characters in C. ++ """ ++ if not s: ++ return ++ a = i if i > 0 else 0 ++ b = i + len(s) ++ if b > self.width: ++ b = self.width ++ if b <= a: ++ return ++ self.ch[a:b] = s[a - i:b - i] ++ self.st[a:b] = [style] * (b - a) + + def restyle(self, a: int, b: int, style: Style) -> None: + """Merge ``style`` over the cells in [a, b) (keeps the characters).""" +- for i in range(max(a, 0), min(b, self.width)): +- self.st[i] = self.st[i] + style ++ if a < 0: ++ a = 0 ++ if b > self.width: ++ b = self.width ++ st = self.st ++ combine = _STYLE_SUM ++ for i in range(a, b): ++ base = st[i] ++ key = (base, style) ++ got = combine.get(key) ++ if got is None: ++ got = combine[key] = base + style ++ st[i] = got + + def strip(self) -> Strip: + segs: list[Segment] = [] +@@ -2796,10 +2829,8 @@ class GraphView(NavMixin, ScrollView, can_focus=True): + label = f"loc_{n.block.start:X}" if n.block else "" + if b is not None and b.rows and b.rows[0].name: + label = b.rows[0].name +- out.put(left, graph.BOX["tl"], bs) +- for i in range(1, w - 1): +- out.put(left + i, graph.BOX["h"], bs) +- out.put(left + w - 1, graph.BOX["tr"], bs) ++ out.text(left, graph.BOX["tl"] + graph.BOX["h"] * (w - 2) ++ + graph.BOX["tr"], bs) + tag = f" {label} " + if len(tag) <= w - 4: + st = _S_GENTRY if (self.fc and n.id == self.fc.entry) else ( +@@ -2809,15 +2840,12 @@ class GraphView(NavMixin, ScrollView, can_focus=True): + out.put(left + w - 2, "↺", _S_EDGE[graph.E_BACK]) + return + if row == n.y + n.h - 1: +- out.put(left, graph.BOX["bl"], bs) +- for i in range(1, w - 1): +- out.put(left + i, graph.BOX["h"], bs) +- out.put(left + w - 1, graph.BOX["br"], bs) ++ out.text(left, graph.BOX["bl"] + graph.BOX["h"] * (w - 2) ++ + graph.BOX["br"], bs) + return + out.put(left, graph.BOX["v"], bs) + out.put(left + w - 1, graph.BOX["v"], bs) +- for i in range(1, w - 1): +- out.put(left + i, " ", _S_INSN) ++ out.text(left + 1, " " * (w - 2), _S_INSN) + i = row - n.y - 1 + rows = self._rows(n.id) + if not (0 <= i < len(rows)): diff --git a/idatui/app.py b/idatui/app.py index 53dd79b..6c0a872 100644 --- a/idatui/app.py +++ b/idatui/app.py @@ -2161,6 +2161,13 @@ _GPAD = 1 # columns of padding inside a box _MINI_W, _MINI_H = 30, 14 +#: Memo for ``Style + Style``. Rich rebuilds a whole Style on every ``+``, and +#: the graph merges an overlay (cursor band, trail, highlight) over a run of +#: cells that share only a handful of base styles -- so the same pair is +#: recombined hundreds of times per frame. +_STYLE_SUM: dict[tuple, Style] = {} + + class _CellRow: """A row of (char, style) cells that coalesces into a Strip. @@ -2183,13 +2190,39 @@ class _CellRow: self.st[i] = style def text(self, i: int, s: str, style: Style) -> None: - for k, c in enumerate(s): - self.put(i + k, c, style) + """Write ``s`` at cell ``i``, clipped to the row. + + Slice assignment rather than a call per character: a graph row is drawn + from box borders, instruction text and the minimap, and doing it a cell + at a time made painting one frame thousands of bound-method calls. + Assigning a str to a list slice expands it to characters in C. + """ + if not s: + return + a = i if i > 0 else 0 + b = i + len(s) + if b > self.width: + b = self.width + if b <= a: + return + self.ch[a:b] = s[a - i:b - i] + self.st[a:b] = [style] * (b - a) def restyle(self, a: int, b: int, style: Style) -> None: """Merge ``style`` over the cells in [a, b) (keeps the characters).""" - for i in range(max(a, 0), min(b, self.width)): - self.st[i] = self.st[i] + style + if a < 0: + a = 0 + if b > self.width: + b = self.width + st = self.st + combine = _STYLE_SUM + for i in range(a, b): + base = st[i] + key = (base, style) + got = combine.get(key) + if got is None: + got = combine[key] = base + style + st[i] = got def strip(self) -> Strip: segs: list[Segment] = [] @@ -2796,10 +2829,8 @@ class GraphView(NavMixin, ScrollView, can_focus=True): label = f"loc_{n.block.start:X}" if n.block else "" if b is not None and b.rows and b.rows[0].name: label = b.rows[0].name - out.put(left, graph.BOX["tl"], bs) - for i in range(1, w - 1): - out.put(left + i, graph.BOX["h"], bs) - out.put(left + w - 1, graph.BOX["tr"], bs) + out.text(left, graph.BOX["tl"] + graph.BOX["h"] * (w - 2) + + graph.BOX["tr"], bs) tag = f" {label} " if len(tag) <= w - 4: st = _S_GENTRY if (self.fc and n.id == self.fc.entry) else ( @@ -2809,15 +2840,12 @@ class GraphView(NavMixin, ScrollView, can_focus=True): out.put(left + w - 2, "↺", _S_EDGE[graph.E_BACK]) return if row == n.y + n.h - 1: - out.put(left, graph.BOX["bl"], bs) - for i in range(1, w - 1): - out.put(left + i, graph.BOX["h"], bs) - out.put(left + w - 1, graph.BOX["br"], bs) + out.text(left, graph.BOX["bl"] + graph.BOX["h"] * (w - 2) + + graph.BOX["br"], bs) return out.put(left, graph.BOX["v"], bs) out.put(left + w - 1, graph.BOX["v"], bs) - for i in range(1, w - 1): - out.put(left + i, " ", _S_INSN) + out.text(left + 1, " " * (w - 2), _S_INSN) i = row - n.y - 1 rows = self._rows(n.id) if not (0 <= i < len(rows)): |
