From d9e8fdb79136dfa32e01631aed8917c1cb9c1b54 Mon Sep 17 00:00:00 2001 From: user Date: Fri, 7 Aug 2026 06:45:52 +0200 Subject: heads(digest=True): the worker answers "does this page still render exactly as you hold it?" with a hash and a count instead of the page. After a rename nearly every page is unchanged, so the pickling, transfer, unpickling and Head rebuild are all skipped. Redone at PAGE granularity end to end, which fixes both bugs of the first attempt. lg_search 5628 -> 3959. Result: {"status":"keep","total_ms":26491.7,"lg_boot_ms":808.6,"lg_decomp_ms":2600.2,"lg_graph_ms":901.2,"lg_hex_ms":448.7,"lg_index_ms":70.3,"lg_listing_cold_ms":432.8,"lg_listing_warm_ms":445.5,"lg_nav_ms":7004.8,"lg_palette_ms":4.8,"lg_rename_ms":758.6,"lg_render_ms":231.3,"lg_search_ms":3959.2,"lg_split_ms":2659.2,"pure_graph_ms":214.7,"sm_boot_ms":432.6,"sm_decomp_ms":1305.1,"sm_graph_ms":758.3,"sm_hex_ms":431.8,"sm_index_ms":2.3,"sm_listing_cold_ms":275.4,"sm_listing_warm_ms":289.4,"sm_nav_ms":295.6,"sm_palette_ms":0.3,"sm_rename_ms":424.3,"sm_render_ms":263.6,"sm_search_ms":61.1,"sm_split_ms":1411.9,"fails":0} --- .auto/ideas.md | 15 ++--- .auto/log.jsonl | 2 + .auto/prompt.md | 11 +++- .auto/wip-digest.patch | 162 +++++++++++++++++++++++++++++++++++++++++++++++++ idatui/domain.py | 138 +++++++++++++++++++++++++---------------- server/patch_server.py | 35 ++++++++++- 6 files changed, 300 insertions(+), 63 deletions(-) create mode 100644 .auto/wip-digest.patch diff --git a/.auto/ideas.md b/.auto/ideas.md index 876ed8b..9226574 100644 --- a/.auto/ideas.md +++ b/.auto/ideas.md @@ -55,13 +55,14 @@ ## Added late in the session -- **Refresh only the rows a rename actually changed.** `bump_names` currently - stales every row's text; a wholesale re-read then costs about what a rebuild - costs (~10% more, measured). Only rows that *mention* the symbol change, and - `xrefs_to(renamed_ea)` plus the function's own extent would name most of them. - The catch: `bump_names()` is called from half a dozen places that don't have - an address (comment, retype, literal reformat), and a name can reach a row - through more than a direct xref. Would make a rename free everywhere. +- ~~**Refresh only the rows a rename actually changed** (xrefs-driven).~~ + **MEASURED AND REJECTED.** `/tmp/whatchanges.py` rebuilds the whole segment + before and after a rename and diffs every row. echo: all 19 changed rows over + 4 renames were covered by (function extent + `xrefs_to`). ls_ttl: 53 of 54 — + and the one that was not, `lea rcx, unk_1D7A0` → `byte_1D7A0`, **was not + caused by the rename at all**: IDA's own analysis defined that byte. Any + address-predicted invalidation leaves such a row stale for good. Superseded by + the digest scheme, which is exact because it looks at the rendered line. - **Features the bench still doesn't drive end to end**, in the order they seem worth probing: xrefs (`x`), the strings browser (`"`), literal formats (`o`), make-code/data/function edits, history (`back`), execution traces, the RPC diff --git a/.auto/log.jsonl b/.auto/log.jsonl index defe5a4..491fbf6 100644 --- a/.auto/log.jsonl +++ b/.auto/log.jsonl @@ -42,3 +42,5 @@ {"run":35,"commit":"6ec4bd8","metric":29392.6,"metrics":{"lg_boot_ms":711.9,"lg_decomp_ms":2406.4,"lg_graph_ms":1271.9,"lg_hex_ms":445.3,"lg_index_ms":71.9,"lg_listing_cold_ms":748.5,"lg_listing_warm_ms":403.8,"lg_nav_ms":6596.2,"lg_palette_ms":4.8,"lg_rename_ms":715.8,"lg_render_ms":219.3,"lg_search_ms":7055.2,"lg_split_ms":2629.8,"pure_graph_ms":212.8,"sm_boot_ms":445.6,"sm_decomp_ms":1281,"sm_graph_ms":775,"sm_hex_ms":442.3,"sm_index_ms":2.3,"sm_listing_cold_ms":265.6,"sm_listing_warm_ms":266.7,"sm_nav_ms":302.9,"sm_palette_ms":0.3,"sm_rename_ms":380.8,"sm_render_ms":255.1,"sm_search_ms":67,"sm_split_ms":1414.2,"fails":0},"status":"discard","description":"Documentation-only commit (playbook + ideas). Re-measured to confirm the post-fix state: 28651 -> 29393 is within the spread on this now-loaded box, and lg_search holds at ~7.0s, confirming that a whole-segment read after renames costs a whole-segment re-render.","timestamp":1786075083972,"segment":6,"confidence":7.108776152564333,"asi":{"state":"v7 baseline 33243 -> 29393 (-11.6%). rename 10055 -> 716 is the headline; the cost of re-rendering after a rename did not disappear, it moved to whoever reads the rows (lg_search 775 -> ~7000, because this bench reads the entire segment right after renaming). For a user reading a viewport the rename is 566x faster and nothing else changes.","why_there_is_no_more_free_win_here":"after a rename, the rows you read have to be re-rendered; re-rendering N heads costs what loading N heads costs. Refreshing only the rows that ACTUALLY changed would need xrefs_to(renamed_ea) to enumerate them, and a name can reach a row without a direct xref (a comment, a struct field in an operand) -- miss one and you are back to showing a stale name, which is the bug just fixed. Logged in .auto/ideas.md rather than attempted.","highest_yield_activity_this_session":"asking what the benchmark does NOT measure. Three of the five bench corrections found an invisible cost, and two of those (split view, rename) were among the biggest wins of the whole session.","next_action_hint":"keep probing uncovered features with the /tmp/featprobe.py shape: xrefs (x), the strings browser (\"), literal formats (o), make-code/data edits, history, traces, RPC. Domain-level probes already say xrefs/strings/structs/resolve are fast, so drive them through the UI instead."}} {"run":36,"commit":"1e9f47b","metric":28842.1,"metrics":{"lg_boot_ms":712.2,"lg_decomp_ms":2366.9,"lg_graph_ms":907.8,"lg_hex_ms":447.2,"lg_index_ms":72.2,"lg_listing_cold_ms":443.7,"lg_listing_warm_ms":453.7,"lg_nav_ms":6610.2,"lg_palette_ms":4.8,"lg_rename_ms":735.3,"lg_render_ms":228.4,"lg_search_ms":7122.8,"lg_split_ms":2609.7,"pure_graph_ms":212.3,"sm_boot_ms":438.7,"sm_decomp_ms":1279.7,"sm_graph_ms":783.2,"sm_hex_ms":447.3,"sm_index_ms":2.5,"sm_listing_cold_ms":267.7,"sm_listing_warm_ms":263.1,"sm_nav_ms":309.7,"sm_palette_ms":0.3,"sm_rename_ms":383.9,"sm_render_ms":255,"sm_search_ms":66.3,"sm_split_ms":1417.6,"fails":0},"status":"discard","description":"Tried and reverted: throttling the background listing streamer's UI reports by time, and adding a token so only the newest streamer survives a re-prime. Both measured WORSE on a direct responsiveness probe (xrefs dialog while streaming: 1314ms baseline, 1436ms throttled, 2038ms with the token). Reverted; the finding is recorded in .auto/ideas.md. This run confirms the committed state at 28842.","timestamp":1786075786983,"segment":6,"confidence":8.431402690090582,"asi":{"real_finding_worth_keeping":"the app is several times slower while a big segment streams in the background: an xrefs dialog is 691ms during the ~7s stream and 105ms after it. Two confirmed mechanisms -- Textual's @work(exclusive=True) does NOT stop a thread worker that is already running, and navigating inside the same segment re-primes against the SAME model, so every jump leaves another streamer behind; and each streamer reports growth to the UI every four pages (thread hop + virtual_size change + full repaint).","what_failed":"a 10/s time throttle made it worse (237 reports vs 112) because the count is per streamer and there are several. Adding a _grow_token so only the newest streamer survives did NOT reduce the report count either -- so the retirement is not happening where it looks like it should, and I do not understand the mechanism well enough to ship a fix.","why_i_stopped":"the probe navigates repeatedly, which is itself what spawns the extra streamers, so it measures the thing it perturbs. A clean probe would drive UI work from a SINGLE navigation while one streamer runs. Recorded in .auto/ideas.md with the reproduction.","also_found":"a PRE-EXISTING crash: StringsPalette.on_mount calls query_one(OptionList) before compose's children are mounted (NoMatches). Reproduces 3/3 on bash AND 3/3 on the pre-autoresearch commit 2b0ae8d, so it is not from this work. ProjectPalette has the same shape.","xrefs_is_fine":"the xrefs path is not a decomp_map-class problem: xrefs_to is 38ms for 12 functions, item building 2.6ms, and the whole 'x'-to-dialog cycle is ~105ms once the listing has finished streaming. Most of that is two modal screen mounts.","state":"v7 baseline 33243 -> 28842 (-13.2%)."}} {"run":37,"commit":"6882eea","metric":29365.3,"metrics":{"lg_boot_ms":738.3,"lg_decomp_ms":2438.1,"lg_graph_ms":1287.6,"lg_hex_ms":462.1,"lg_index_ms":72.1,"lg_listing_cold_ms":438.6,"lg_listing_warm_ms":460.3,"lg_nav_ms":6762.3,"lg_palette_ms":4.8,"lg_rename_ms":719.1,"lg_render_ms":224.7,"lg_search_ms":7055.7,"lg_split_ms":2672.5,"pure_graph_ms":217.4,"sm_boot_ms":435.3,"sm_decomp_ms":1269.6,"sm_graph_ms":721,"sm_hex_ms":437.5,"sm_index_ms":2.5,"sm_listing_cold_ms":281.1,"sm_listing_warm_ms":264.8,"sm_nav_ms":316.9,"sm_palette_ms":0.3,"sm_rename_ms":390,"sm_render_ms":257.4,"sm_search_ms":67.8,"sm_split_ms":1367.8,"fails":0},"status":"discard","description":"Documentation-only (trace scaling finding). Probed idatui/trace.py: loading is exactly linear, register_state is O(1), but Trace.memory() is linear in TRACE LENGTH per call. Recorded rather than fixed — the differential against Tenet's reference reader is skipped on this box, so the safety net for an indexing change is too thin. Run confirms the committed state.","timestamp":1786076140273,"segment":6,"confidence":13.948955495004537,"asi":{"trace_measurements":"load 36.6/73.3/143.9/280.7 ms for 20k/40k/80k/160k rows (x1.95 per doubling -- linear, correct). register_state x200: ~1.1ms at every size -- O(1). memory() x200: 16.3/31.7/63.4/125.8 ms -- LINEAR IN TRACE LENGTH per call.","why":"_mem_index sorts accesses by address and bisects to the query window, which is right, but then iterates every access in that window across all time and filters by t > idx. A stack slot written once per loop iteration accumulates one entry per iteration, so the stack pane's cost grows with how long the trace ran.","why_not_fixed":"tests/test_trace_vs_tenet.py -- the differential against Tenet's own reference reader -- is SKIPPED on this box, leaving 35 synthetic checks as the only guard on a subtle indexing change. Recorded in .auto/ideas.md with the probe (/tmp/traceprof.py) and the precondition.","session_state":"v7 baseline 33243 -> best 28651 kept. Working tree clean, 41 commits on autoresearch/perf-20260807, full 830-check gate plus two custom equivalence checks green.","next_action_hint":"remaining ideas, in .auto/ideas.md: the trace memory index (needs the Tenet differential first), why the _grow_token did not retire extra streamers, refreshing only the rows a rename actually changed, and the skeleton walk for ensure_ea (costed at ~5%)."}} +{"run":38,"commit":"df88ece","metric":27552.6,"metrics":{"lg_boot_ms":759.2,"lg_decomp_ms":2754.1,"lg_graph_ms":1207.2,"lg_hex_ms":448,"lg_index_ms":69.6,"lg_listing_cold_ms":434.5,"lg_listing_warm_ms":442.9,"lg_nav_ms":6642.3,"lg_palette_ms":4.7,"lg_rename_ms":730.6,"lg_render_ms":223.6,"lg_search_ms":5627.5,"lg_split_ms":2268,"pure_graph_ms":218.7,"sm_boot_ms":465,"sm_decomp_ms":1290.8,"sm_graph_ms":720.7,"sm_hex_ms":436.1,"sm_index_ms":2.4,"sm_listing_cold_ms":267.2,"sm_listing_warm_ms":266.5,"sm_nav_ms":292.9,"sm_palette_ms":0.3,"sm_rename_ms":380.1,"sm_render_ms":255.5,"sm_search_ms":68.1,"sm_split_ms":1276.3,"fails":0},"status":"keep","description":"Size the worker's per-line render cache to hold a segment's DISTINCT lines (16384 -> 65536, overridable with IDATUI_LINE_CACHE). This was a recorded dead end — it does nothing for a cold sweep — but the rename fix created a second-sweep workload, and re-rendering after a rename is now 21% cheaper. lg_search 7123 -> 5628.","timestamp":1786076605767,"segment":6,"confidence":8.431402690090582,"asi":{"hypothesis":"the biggest remaining term (lg_search 7.1s) is entirely the post-rename re-render, and a re-render is a SECOND sweep over the same lines -- which is exactly the case the line cache was measured to help and then filed as a dead end because nothing did it","why_it_stopped_being_a_dead_end":"experiment v1 #6 sized the cache at 16384 and noted that growing it 'does nothing for a cold sweep, only for a second sweep'. Nothing did a second sweep -- until v7 #2 made a rename keep the listing's walk and re-render its text on demand. The structural change created the workload the dead end was waiting for.","sizing_measurement":{"16384":[17.18,16.88,"+29MB"],"32768":[16.99,17.23,"+52MB"],"65536":[17.01,11.13,"+75MB"],"131072":[16.91,11.2,"+75MB"],"note":"bash .text, first sweep then second, in the worker"},"why_65536":"bash's .text is 228659 rows but only 53363 DISTINCT lines; 32768 still thrashes and 131072 buys nothing because the working set already fits. It is a bound, not a proportion -- a bigger binary fills it and stops, so the cost is capped at ~56MB whatever is open. IDATUI_LINE_CACHE lowers it for a pool of workers competing for memory.","direct_ab_on_the_real_path":"/tmp/refreshcost.py -- load the whole listing, rename, re-read every row through window(): 6445ms -> 5115ms (-21%), worker RSS 164MB -> 211MB. Initial load_all unchanged (5645 vs 5585).","gains":"total 28651 (previous best) -> 27553; lg_search 7123 -> 5628 (-21%); lg_split 2673 -> 2268","work_unchanged":"decomp_lines 3436, split_mapped 2070, search_hits 91783, graph_blocks 1000, rename_ok 6/6 -- all identical","verified":"830-check gate plus check_search and check_rename green; spans still byte-identical vs 2b0ae8d","lesson_recorded_in_prompt":"re-read the dead-end list after any structural change -- a rejected idea can become the right one when the workload around it moves","next_action_hint":"lg_nav 6642 is now the largest term again and is at the per-row floor. lg_search 5628 is still the post-rename re-render; the only way further down is to re-render fewer rows (see the xrefs-driven invalidation idea in .auto/ideas.md, which is risky because a name can reach a row without a direct xref)."}} +{"run":39,"commit":"df88ece","metric":28376.5,"metrics":{"lg_boot_ms":762.1,"lg_decomp_ms":2374.3,"lg_graph_ms":938.1,"lg_hex_ms":452.4,"lg_index_ms":70.2,"lg_listing_cold_ms":419.6,"lg_listing_warm_ms":464.1,"lg_nav_ms":6941.9,"lg_palette_ms":5,"lg_rename_ms":1973.2,"lg_render_ms":221.9,"lg_search_ms":4117.3,"lg_split_ms":2333.3,"pure_graph_ms":214.6,"sm_boot_ms":453.7,"sm_decomp_ms":1270.5,"sm_graph_ms":726.9,"sm_hex_ms":440.6,"sm_index_ms":2.4,"sm_listing_cold_ms":283,"sm_listing_warm_ms":284.3,"sm_nav_ms":310.4,"sm_palette_ms":0.3,"sm_rename_ms":1575.9,"sm_render_ms":257.6,"sm_search_ms":59.6,"sm_split_ms":1423.7,"fails":0},"status":"checks_failed","description":"Digest mode for `heads`: the worker answers \"does this page still render exactly as it did?\" for the cost of the render alone, so a post-rename refresh skips shipping, unpickling and rebuilding pages that did not change. lg_search 5628 -> 4117 — but it BREAKS 'O' cycles back, and it made rename 2-4x slower. Both causes understood.","timestamp":1786077531326,"segment":6,"confidence":10.35869418588966,"asi":{"hypothesis_first_tested_empirically":"before building anything I checked the parked xrefs-driven idea: /tmp/whatchanges.py rebuilds the whole segment before and after a rename and diffs every row. On echo, all 19 changed rows over 4 renames were covered by (function extent + xrefs_to). On ls_ttl, 53 of 54 were -- the one that was not is 'lea rcx, unk_1D7A0' -> 'byte_1D7A0', which the RENAME DID NOT CAUSE: IDA's own analysis defined that byte. An address-predicted invalidation would leave that row stale for good, so the idea is now measured-and-rejected rather than assumed-risky.","what_i_built_instead":"an exact check: `heads(..., digest=True)` builds the rows as usual but returns only hash+count instead of the rows. The client stores the digest each page came back with and asks 'still the same?' before re-fetching. Uses the interpreter's own hash deliberately -- it never has to mean anything outside the worker process, the client is only a courier.","measured_win":"post-rename whole-segment re-read 5115ms -> 3833ms (-25%); lg_search 5628 -> 4117","BUG_1_correctness":"the stored digest describes the page AS LOADED, not as the client currently holds it. After a full refetch the client's rows change but _page_digest is not updated -- so when a literal format cycles hex -> dec -> ... -> hex, the worker's digest matches the ORIGINAL stored one, the page is declared unchanged, and the row keeps the intermediate decimal text. That is exactly the failure: opfmt_listing ''O' cycles back' got 'sub rsp, 184' wanting 'sub rsp, 0B8h'.","BUG_2_performance":"_ensure_text_from tests page freshness with all(_head_gen[k] == gen for k in the page), and get() calls _ensure_text per ROW -- so every row scanned a whole page's gen array. rename went 380 -> 1576ms (sm) and 730 -> 1973ms (lg).","fix_for_both":"refresh at PAGE granularity end to end instead of the snapped TEXT_BLOCK: a page is exactly what the tool produced from (addr, count=PAGE), so refetching with the same parameters reproduces the same sequence with no snapping, the stored digest can be updated whenever the page's rows are replaced, and every head in a page shares one gen value so freshness is a single probe rather than a scan.","work_preserved":".auto/wip-digest.patch","gate_worked":"check_rename and check_search both passed -- neither exercises a format cycle. The scenario suite caught it. That is the third time a cache-shaped change failed in a way only one specific test could see."}} diff --git a/.auto/prompt.md b/.auto/prompt.md index 68bc46f..c57708b 100644 --- a/.auto/prompt.md +++ b/.auto/prompt.md @@ -154,7 +154,9 @@ v1 bench baseline `total_ms` ≈ 46 600 → 18 900 after thirteen experiments 7. *(v5 #6)* **Highlight ranges are computed per line on demand.** Searching one character over bash matches 177 k lines at 310 k places; all but the forty on screen were built and thrown away. -8. *(v1 #9/#10/#13, v2 #3, v5 #3)* Constants: `bytes.hex(" ")` for the opcode +8. *(v7 #10, −4%)* **`heads(digest=True)`** — ask whether a page still renders + as you hold it, rather than fetching it to find out. +9. *(v1 #9/#10/#13, v2 #3, v5 #3)* Constants: `bytes.hex(" ")` for the opcode column (12×), `bisect` imported at module scope, the deferred `refresh(layout=True)` only when a scroll actually clamped, a memoised pygments token→style lookup, `_CellRow` writing by slice, `HexView` emitting @@ -211,7 +213,12 @@ nobody reads. spends, and the walk is inherently linear. - `decomp` — the raw `decompile` tool is 1 711 ms cold for echo's twelve largest functions and 218 ms warm; pickling the result is 0.1 ms. It is Hex-Rays. -- `search` — after a rename, what you read has to be re-rendered. +- `search` — after a rename, what you read has to be re-rendered. The worker + still has to *render* a page to know it is unchanged (`generate_disasm_line` + is the floor), but since v7 #10 it no longer has to ship it: `heads(..., + digest=True)` returns hash+count, and the client keeps the page it already + has. That is ~40% of a page's cost, and after a rename nearly every page is + unchanged. - `listing`/`hex`/`graph`/`render` — mostly Textual's own compositing, ~6 ms per full-screen frame. Our `render_line` is ~1.8 ms of a ~10 ms hex frame. - `boot` — ~150 ms of it is the worker importing `idapro`. diff --git a/.auto/wip-digest.patch b/.auto/wip-digest.patch new file mode 100644 index 0000000..d4d23e8 --- /dev/null +++ b/.auto/wip-digest.patch @@ -0,0 +1,162 @@ +diff --git a/idatui/domain.py b/idatui/domain.py +index 1f8263d..9cf539f 100644 +--- a/idatui/domain.py ++++ b/idatui/domain.py +@@ -673,6 +673,14 @@ class ListingModel: + #: Whether a rename has ever staled this model. Until one has, every + #: read takes exactly the path it always did. + self._renamed = False ++ #: Where each loaded page starts, so a stale-text refresh can ask the ++ #: worker "is this page still what I have?" over exactly the extent the ++ #: worker itself produced. Parallel lists: first head index, the address ++ #: it was fetched from, the digest it came back with, and its row count. ++ self._page_head: list[int] = [] ++ self._page_addr: list[int] = [] ++ self._page_digest: list[object] = [] ++ self._page_rows: list[int] = [] + #: Set if a text refresh came back with a different head sequence, which + #: means something DID move the walk. Program.listing() throws the model + #: away when it sees this, so the next read rebuilds from scratch. +@@ -758,6 +766,11 @@ class ListingModel: + page = self._build_page(rows) + with self._lock: + gen = self._text_gen ++ self._page_head.append(len(self._heads)) ++ self._page_addr.append(frm) ++ self._page_digest.append(payload.get("digest") ++ if isinstance(payload, dict) else None) ++ self._page_rows.append(len(rows)) + for h in page: + # Banner/label rows (function headers, separators, code labels) + # are display-only; don't index them so navigation lands on the +@@ -924,13 +937,65 @@ class ListingModel: + search body asks for thousands at once) would come back short, fail the + sequence check, and condemn the model to a rebuild it did not need. + """ +- blk = self.TEXT_BLOCK + with self._lock: + n = len(self._heads) +- start = (max(j0, 0) // blk) * blk +- while start < min(j1, n): +- self._ensure_text_block(start, min(start + blk, n)) +- start += blk ++ j0 = max(j0, 0) ++ j1 = min(j1, n) ++ while j0 < j1: ++ j0 = self._ensure_text_from(j0, j1, n) ++ ++ def _ensure_text_from(self, j0: int, j1: int, n: int) -> int: ++ """Freshen from head ``j0`` and return where to carry on. ++ ++ Tries the page ``j0`` falls in first: the worker can say whether that ++ page still renders exactly as it did, for the cost of the render alone ++ -- no rows on the wire, none unpickled, no Heads rebuilt. After a rename ++ nearly every page comes back identical, and that is 40% of what asking ++ for it again would have cost. Falls back to the block refetch when the ++ page has genuinely changed (or predates the digest). ++ """ ++ with self._lock: ++ p = bisect.bisect_right(self._page_head, j0) - 1 ++ usable = (0 <= p < len(self._page_head) ++ and self._page_digest[p] is not None) ++ if usable: ++ p_lo = self._page_head[p] ++ p_hi = (self._page_head[p + 1] if p + 1 < len(self._page_head) ++ else len(self._heads)) ++ gen = self._text_gen ++ fresh = all(self._head_gen[k] == gen for k in range(p_lo, p_hi)) ++ addr, want_dig = self._page_addr[p], self._page_digest[p] ++ want_rows = self._page_rows[p] ++ if usable and fresh: ++ return p_hi ++ if usable and self._verify_page(p, p_lo, p_hi, addr, want_dig, ++ want_rows, gen): ++ return p_hi ++ blk = self.TEXT_BLOCK ++ end = min(j0 + blk, j1 if usable else n) ++ self._ensure_text_block(j0, max(end, j0 + 1)) ++ return max(end, j0 + 1) ++ ++ def _verify_page(self, p: int, p_lo: int, p_hi: int, addr: int, ++ want_dig: object, want_rows: int, gen: int) -> bool: ++ """Ask whether page ``p`` still renders as it did; mark it fresh if so.""" ++ try: ++ payload = self._prog.client.call( ++ "heads", addr=hex(addr), count=self.PAGE, annotate=True, ++ digest=True) ++ except Exception: # noqa: BLE001 -- an older worker has no digest mode ++ return False ++ if not isinstance(payload, dict): ++ return False ++ got = payload.get("digest") ++ if got is None or got != want_dig or payload.get("count") != want_rows: ++ return False ++ with self._lock: ++ if self._text_gen != gen or len(self._heads) < p_hi: ++ return False ++ for k in range(p_lo, p_hi): ++ self._head_gen[k] = gen ++ return True + + def _ensure_text_block(self, j0: int, j1: int) -> None: + """Re-render one block, snapped out to whole ADDRESS groups. +diff --git a/server/patch_server.py b/server/patch_server.py +index f3cbe35..a34a4a9 100644 +--- a/server/patch_server.py ++++ b/server/patch_server.py +@@ -576,6 +576,28 @@ def _idatui_spans(line): + return [[k, t] for k, t, _o in out], trimmed + + ++def _idatui_rows_digest(rows): ++ """A value that changes whenever any of ``rows`` would render differently. ++ ++ Covers everything a client keeps off a row: address, kind, size, the plain ++ text, the symbol name and the colour spans (which is what makes it exact ++ rather than a heuristic -- two lines can collapse to the same text and still ++ be coloured differently). ++ ++ Uses the interpreter's own ``hash``, deliberately. It never has to mean ++ anything outside this process: the client stores what a page hashed to when ++ it loaded it and hands the same number back to ask whether the page still ++ hashes to that. One worker, one process, one hash seed. ++ """ ++ acc = 0 ++ for r in rows: ++ sp = r.get("spans") ++ acc = hash((acc, r.get("ea"), r.get("kind"), r.get("size"), ++ r.get("text"), r.get("name"), ++ tuple(map(tuple, sp)) if sp else None)) ++ return acc ++ ++ + def _idatui_unknown_row(ea, size): + """One collapsed row for a run of ``size`` undefined bytes starting at + ``ea``. A single byte is rendered normally (shows its value); a longer run +@@ -661,6 +683,7 @@ def heads( + end: Annotated[str, "Optional exclusive end address; default = segment end"] = "", + back: Annotated[bool, "Walk backwards: return the count heads ENDING just before addr, in forward order"] = False, + annotate: Annotated[bool, "Emit IDA-style function boundary banner rows (kind sep/funchdr)"] = False, ++ digest: Annotated[bool, "Return only a digest+count of the rows, not the rows themselves"] = False, + ) -> dict: + """Walk item heads from ``addr`` as a flat listing: every head is rendered + (code OR data OR undefined) via generate_disasm_line and stepped with +@@ -767,7 +790,17 @@ def heads( + rows.extend(_rows_for(ea)) # a struct head expands into member rows + ea = _advance(ea) + cursor = {"next": hex(ea)} if more else {"done": True} +- return {"addr": str(addr), "heads": rows, "cursor": cursor} ++ out = {"addr": str(addr), "cursor": cursor, ++ "digest": _idatui_rows_digest(rows), "count": len(rows)} ++ # ``digest`` mode answers "is this page still exactly what you have?" without ++ # shipping it. The rows are built either way -- generate_disasm_line is the ++ # floor and there is no way to know a line is unchanged without rendering it ++ # -- but pickling several hundred rows with their colour spans, unpickling ++ # them and rebuilding Heads is about 40% of what a page costs, and after a ++ # rename almost every page comes back identical. ++ if not digest: ++ out["heads"] = rows ++ return out + + + @tool diff --git a/idatui/domain.py b/idatui/domain.py index 1f8263d..f4aa63c 100644 --- a/idatui/domain.py +++ b/idatui/domain.py @@ -642,11 +642,6 @@ class ListingModel: """ PAGE = 500 # heads per server call (well under the tool's 2000 cap) - #: Heads refreshed together when a rename makes their text stale. One server - #: call per block, so a viewport costs one round trip rather than forty -- - #: and the same size as a load page, so refreshing everything costs about - #: what rebuilding everything would have. - TEXT_BLOCK = 500 def __init__(self, program: "Program", seg_start: int, seg_end: int, name: str | None = None): @@ -673,6 +668,14 @@ class ListingModel: #: Whether a rename has ever staled this model. Until one has, every #: read takes exactly the path it always did. self._renamed = False + #: One entry per loaded PAGE: where its heads start, the address it was + #: fetched from, the digest it came back with, and how many rows it + #: held. A stale-text refresh re-asks for exactly that page, so it can + #: be told "still identical" for the price of the render alone. + self._page_head: list[int] = [] + self._page_addr: list[int] = [] + self._page_digest: list[object] = [] + self._page_rows: list[int] = [] #: Set if a text refresh came back with a different head sequence, which #: means something DID move the walk. Program.listing() throws the model #: away when it sees this, so the next read rebuilds from scratch. @@ -758,6 +761,11 @@ class ListingModel: page = self._build_page(rows) with self._lock: gen = self._text_gen + self._page_head.append(len(self._heads)) + self._page_addr.append(frm) + self._page_digest.append(payload.get("digest") + if isinstance(payload, dict) else None) + self._page_rows.append(len(rows)) for h in page: # Banner/label rows (function headers, separators, code labels) # are display-only; don't index them so navigation lands on the @@ -919,69 +927,93 @@ class ListingModel: def _ensure_text(self, j0: int, j1: int) -> None: """Re-render physical heads [j0, j1) if a rename staled them. - Done a block at a time. One call for the whole range would be simpler but - the ``heads`` tool caps a response at 2000 rows, so a wide request (the - search body asks for thousands at once) would come back short, fail the - sequence check, and condemn the model to a rebuild it did not need. + Works a PAGE at a time -- the same unit the loader fetched. A page is + exactly what ``heads(addr, count=PAGE)`` produced, so asking again with + the same arguments reproduces the same row sequence; nothing has to be + snapped out to whole address groups (a function start emits three banner + rows sharing one address, and an arbitrary boundary through those never + lines up again). It also means every head in a page can share one + generation marker, so "is this fresh?" is a single probe. """ - blk = self.TEXT_BLOCK with self._lock: n = len(self._heads) - start = (max(j0, 0) // blk) * blk - while start < min(j1, n): - self._ensure_text_block(start, min(start + blk, n)) - start += blk - - def _ensure_text_block(self, j0: int, j1: int) -> None: - """Re-render one block, snapped out to whole ADDRESS groups. - - A function start emits three banner rows and its code row at the same ea, - and a labelled instruction emits two -- so a boundary falling inside one - of those groups would refetch the whole group, never line up, and leave - the old names on screen for good. - """ + j1 = min(j1, n) + j0 = max(j0, 0) + if j1 <= j0: + return + p = max(bisect.bisect_right(self._page_head, j0) - 1, 0) + last = bisect.bisect_left(self._page_head, j1) + while p < last: + p = self._ensure_page(p) + + def _page_bounds(self, p: int) -> tuple[int, int]: + """[first, last) head index of page ``p`` (caller holds the lock).""" + lo = self._page_head[p] + hi = (self._page_head[p + 1] if p + 1 < len(self._page_head) + else len(self._heads)) + return lo, hi + + def _ensure_page(self, p: int) -> int: + """Freshen page ``p``; returns the next page to consider.""" with self._lock: + if not (0 <= p < len(self._page_head)): + return p + 1 gen = self._text_gen - n = len(self._heads) - a = max(j0, 0) - b = min(j1, n) - if b <= a: - return - head_gen = self._head_gen - if all(head_gen[j] == gen for j in range(a, b)): - return - eas = self._head_eas - while a > 0 and eas[a - 1] == eas[a]: - a -= 1 - while b < n and eas[b - 1] == eas[b]: - b += 1 - last = self._heads[b - 1] - lo = eas[a] - hi = last.ea + max(last.size, 1) - want = [(h.ea, h.kind) for h in self._heads[a:b]] + lo, hi = self._page_bounds(p) + if hi <= lo or self._head_gen[lo] == gen: + return p + 1 + addr = self._page_addr[p] + want_digest = self._page_digest[p] + want_rows = self._page_rows[p] + want = [(h.ea, h.kind) for h in self._heads[lo:hi]] + # Ask whether the page still renders as the client holds it. The worker + # builds the rows either way (there is no knowing a line is unchanged + # without rendering it), but skipping the pickling, the transfer, the + # unpickling and the Head rebuild is about 40% of what a page costs -- + # and after a rename almost every page is unchanged. + if want_digest is not None: + try: + probe = self._prog.client.call( + "heads", addr=hex(addr), count=self.PAGE, annotate=True, + digest=True) + except Exception: # noqa: BLE001 -- an older worker has no digest + probe = None + if (isinstance(probe, dict) and probe.get("digest") == want_digest + and probe.get("count") == want_rows): + with self._lock: + if self._text_gen == gen and len(self._heads) >= hi: + for k in range(lo, hi): + self._head_gen[k] = gen + return p + 1 try: payload = self._prog.client.call( - "heads", addr=hex(lo), end=hex(hi), - count=min(len(want) + 64, 2000), annotate=True) + "heads", addr=hex(addr), count=self.PAGE, annotate=True) except Exception: # noqa: BLE001 -- keep the old text rather than blank - return + return p + 1 rows = payload.get("heads", []) if isinstance(payload, dict) else [] - page = self._build_page(rows)[:len(want)] + page = self._build_page(rows) with self._lock: - if self._text_gen != gen or len(self._heads) < b: - return + if self._text_gen != gen or len(self._heads) < hi: + return p + 1 if [(h.ea, h.kind) for h in page] != want: # Something moved the walk, which a rename cannot do -- so this # was not one. Say so and let Program.listing() rebuild, rather - # than sit here re-fetching a block that will never line up (and + # than sit here re-fetching a page that will never line up (and # showing the old names while doing it). self.stale_structure = True - for j in range(a, b): - self._head_gen[j] = gen - return - self._heads[a:b] = page - for j in range(a, b): - self._head_gen[j] = gen + for k in range(lo, hi): + self._head_gen[k] = gen + return p + 1 + self._heads[lo:hi] = page + # The stored digest has to describe what the client now HOLDS, not + # what it once loaded. Leaving it stale is how a literal cycling + # hex -> dec -> hex ends up declared "unchanged" while the row still + # shows the decimal it was refetched with in between. + self._page_digest[p] = (payload.get("digest") + if isinstance(payload, dict) else None) + for k in range(lo, hi): + self._head_gen[k] = gen + return p + 1 def get(self, i: int) -> Head | None: with self._lock: diff --git a/server/patch_server.py b/server/patch_server.py index f3cbe35..a34a4a9 100644 --- a/server/patch_server.py +++ b/server/patch_server.py @@ -576,6 +576,28 @@ def _idatui_spans(line): return [[k, t] for k, t, _o in out], trimmed +def _idatui_rows_digest(rows): + """A value that changes whenever any of ``rows`` would render differently. + + Covers everything a client keeps off a row: address, kind, size, the plain + text, the symbol name and the colour spans (which is what makes it exact + rather than a heuristic -- two lines can collapse to the same text and still + be coloured differently). + + Uses the interpreter's own ``hash``, deliberately. It never has to mean + anything outside this process: the client stores what a page hashed to when + it loaded it and hands the same number back to ask whether the page still + hashes to that. One worker, one process, one hash seed. + """ + acc = 0 + for r in rows: + sp = r.get("spans") + acc = hash((acc, r.get("ea"), r.get("kind"), r.get("size"), + r.get("text"), r.get("name"), + tuple(map(tuple, sp)) if sp else None)) + return acc + + def _idatui_unknown_row(ea, size): """One collapsed row for a run of ``size`` undefined bytes starting at ``ea``. A single byte is rendered normally (shows its value); a longer run @@ -661,6 +683,7 @@ def heads( end: Annotated[str, "Optional exclusive end address; default = segment end"] = "", back: Annotated[bool, "Walk backwards: return the count heads ENDING just before addr, in forward order"] = False, annotate: Annotated[bool, "Emit IDA-style function boundary banner rows (kind sep/funchdr)"] = False, + digest: Annotated[bool, "Return only a digest+count of the rows, not the rows themselves"] = False, ) -> dict: """Walk item heads from ``addr`` as a flat listing: every head is rendered (code OR data OR undefined) via generate_disasm_line and stepped with @@ -767,7 +790,17 @@ def heads( rows.extend(_rows_for(ea)) # a struct head expands into member rows ea = _advance(ea) cursor = {"next": hex(ea)} if more else {"done": True} - return {"addr": str(addr), "heads": rows, "cursor": cursor} + out = {"addr": str(addr), "cursor": cursor, + "digest": _idatui_rows_digest(rows), "count": len(rows)} + # ``digest`` mode answers "is this page still exactly what you have?" without + # shipping it. The rows are built either way -- generate_disasm_line is the + # floor and there is no way to know a line is unchanged without rendering it + # -- but pickling several hundred rows with their colour spans, unpickling + # them and rebuilding Heads is about 40% of what a page costs, and after a + # rename almost every page comes back identical. + if not digest: + out["heads"] = rows + return out @tool -- cgit v1.3.1-sl0p