diff options
| author | blasty <peter@haxx.in> | 2026-08-10 01:42:14 +0200 |
|---|---|---|
| committer | blasty <peter@haxx.in> | 2026-08-10 01:42:14 +0200 |
| commit | 39165d016c059bdf73269a5e45df464f9075e579 (patch) | |
| tree | 4f24b2acfab32fc34cb80693abe532ef9bda8e83 /idatui/domain.py | |
| parent | segment_index: the row total in one call instead of 458 (diff) | |
| download | ida-tui-39165d016c059bdf73269a5e45df464f9075e579.tar.gz ida-tui-39165d016c059bdf73269a5e45df464f9075e579.tar.xz ida-tui-39165d016c059bdf73269a5e45df464f9075e579.zip | |
Build the listing index in one call: boot 9.3s -> 1.25s, 911 calls -> 4
The listing used to learn its shape by fetching it. Even after skeleton pages
that was 457 round trips and 227k rows for a 1.2MB bash, to end up knowing
how many rows there are and where each one is.
segment_index(detail=True) now returns exactly that -- every row's address,
kind and size as packed arrays, plus the page boundaries -- from one walk that
builds no rows and renders no text. ListingModel.build_from_index() decodes it
straight into _heads/_head_eas/_row_at/_by_ea/_page_*, marks every row
_SKELETON_GEN, and declares itself complete. _grow has nothing left to stream.
Nothing else in the model changed, because a row without text is a state it
already had: the FIRST read of a page materialises it through the same
_ensure_text/_ensure_page path a rename uses. That is why this is a ~90 line
change to a core view rather than a rewrite.
bash boot: 911 calls / 9.26s -> 4 calls / 1.25s 7.4x
whole census (boot + 9 UI actions): 933 calls -> 30
Two things had to be exactly right, and both are tested rather than argued:
* the ROW COUNT, or the scrollbar lies. Verified equal to a fully streamed
model, and every row's ea/kind/size equal too, 228,659 of them, zero
mismatches.
* the PAGE BOUNDARIES, or _ensure_page refetches a page that does not line
up, fails its structure check and triggers a full rebuild. heads() pages on
PHYSICAL rows; anchoring every N LOGICAL rows looks identical (the two only
diverge once a segment holds an undefined run) and would have been a
lurking bug on .bss. Anchors now carry [logical_row, ea, head_index] taken
at the real boundary, and are asserted equal to the streamer's own.
Transport note: the packed arrays are base64, not raw bytes. _PACK_EPILOGUE
serialises with json.dumps(default=str), which turns bytes into their repr --
2.97MB arrived as 11.26MB of unparseable text before that was spotted.
The new test builds both models back to back and compares every internal
array. An earlier version compared against the app's long-lived model and was
off by one row, because scenarios before it rename and define things: that
model describes the database at boot, not now.
Full gate: 1063 passed, twice.
Diffstat (limited to 'idatui/domain.py')
| -rw-r--r-- | idatui/domain.py | 78 |
1 files changed, 78 insertions, 0 deletions
diff --git a/idatui/domain.py b/idatui/domain.py index 60dbbbf..ecc6a80 100644 --- a/idatui/domain.py +++ b/idatui/domain.py @@ -17,9 +17,11 @@ Textual worker threads; the internal prefetch pool is separate and small. from __future__ import annotations +import array import bisect import re import threading +from base64 import b64decode from concurrent.futures import ThreadPoolExecutor from collections.abc import Sequence from dataclasses import dataclass, field, replace @@ -802,6 +804,82 @@ class ListingModel: with self._lock: return self._max_raw + def build_from_index(self) -> bool: + """Populate the whole row index from ONE call instead of streaming it. + + ``segment_index(detail=True)`` walks the segment and returns every row's + address, kind and size as packed arrays, plus the page boundaries a + refetch would use. That is everything this model needs to know how many + rows there are and where each one lives -- all that is missing is the + rendered text, which is exactly what a skeleton page is missing too. + + So the rows land marked ``_SKELETON_GEN`` and the FIRST read of any page + materialises it through the existing ``_ensure_text``/``_ensure_page`` + path, the same one a rename uses. Measured on bash: 594ms and one call, + against 1827ms and 458 for streaming the same thing. + + Returns False if the backend cannot supply it, in which case the caller + should stream as before -- this is an optimisation, not a new contract. + """ + try: + idx = self._prog.client.invoke( + "segment_index", addr=hex(self.seg_start), end=hex(self.seg_end), + page_rows=self.PAGE, detail=True) + except Exception: # noqa: BLE001 -- fall back to streaming + return False + if not isinstance(idx, dict) or idx.get("error") or "eas" not in idx: + return False + try: + eas = array.array("Q"); eas.frombytes(b64decode(idx["eas"])) + kinds = array.array("B"); kinds.frombytes(b64decode(idx["kinds"])) + sizes = array.array("I"); sizes.frombytes(b64decode(idx["sizes"])) + except Exception: # noqa: BLE001 + return False + names = idx.get("kind_names") or [] + anchors = idx.get("anchors") or [] + n = len(eas) + if not (n == len(kinds) == len(sizes)) or not anchors: + return False + + heads: list[Head] = [] + row_at: list[int] = [] + by_ea: dict[int, int] = {} + rows = 0 + ap = heads.append + rap = row_at.append + for i in range(n): + ea = eas[i] + kind = names[kinds[i]] if kinds[i] < len(names) else "unknown" + size = sizes[i] + ap(Head(ea, kind, size, "")) + rap(rows) + # Banner/label rows are display-only; navigation must land on the + # real head at that address. Same rule as the streaming loader. + if kind not in ("sep", "funchdr", "label"): + by_ea.setdefault(ea, rows) + rows += size if (kind == "unknown" and size > 1) else 1 + + with self._lock: + self._heads = heads + self._head_eas = list(eas) + self._head_gen = [self._SKELETON_GEN] * n + self._row_at = row_at + self._by_ea = by_ea + self._rows = rows + # Anchors are [logical_row, ea, head_index] at the exact boundaries + # heads(count=PAGE) pages on, so _ensure_page can refetch one page + # and have it line up head for head. + self._page_head = [a[2] for a in anchors] + self._page_addr = [_as_int(a[1]) for a in anchors] + self._page_digest = [None] * len(anchors) + self._page_rows = [ + (anchors[k + 1][2] if k + 1 < len(anchors) else n) - anchors[k][2] + for k in range(len(anchors))] + self._skeleton = True + self._done = True + self._next = None + return True + def load_next_page(self, text: bool = True) -> int: """Load one more page of heads; returns how many were added. |
