diff options
Diffstat (limited to '.auto/wip-headcache.patch')
| -rw-r--r-- | .auto/wip-headcache.patch | 256 |
1 files changed, 256 insertions, 0 deletions
diff --git a/.auto/wip-headcache.patch b/.auto/wip-headcache.patch new file mode 100644 index 0000000..d7d6ab6 --- /dev/null +++ b/.auto/wip-headcache.patch @@ -0,0 +1,256 @@ +diff --git a/.auto/diff_spans.py b/.auto/diff_spans.py +index 95721a5..aad4ebc 100644 +--- a/.auto/diff_spans.py ++++ b/.auto/diff_spans.py +@@ -39,11 +39,11 @@ def load_impl(path: str, name: str): + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) # IDA-free at import time + body = mod.BODY +- a = body.index("#: IDA colour tag -> the semantic kind") +- b = body.index("def _idatui_unknown_row") ++ a = body.index("def _idatui_head_row") ++ b = body.index("def _idatui_struct_member_rows") + g = {"__name__": name} + exec(compile(body[a:b], name, "exec"), g) # noqa: S102 +- return g["_idatui_spans"] ++ return g + + + def main() -> int: +@@ -59,8 +59,10 @@ def main() -> int: + fh.write(subprocess.run( + ["git", "-C", ROOT, "show", f"{a.ref}:server/patch_server.py"], + capture_output=True, text=True, check=True).stdout) +- new = load_impl(os.path.join(ROOT, "server", "patch_server.py"), "new") +- old = load_impl(old_path, "old") ++ gnew = load_impl(os.path.join(ROOT, "server", "patch_server.py"), "new") ++ gold = load_impl(old_path, "old") ++ new, old = gnew["_idatui_spans"], gold["_idatui_spans"] ++ new_row, old_row = gnew["_idatui_head_row"], gold["_idatui_head_row"] + + binary = os.path.join(ROOT, a.target) + tgt = os.path.join(d, os.path.basename(binary)) +@@ -89,7 +91,15 @@ def main() -> int: + if ra != rb: + bad += 1 + if bad <= 3: +- print(f"MISMATCH @ {ea:#x}\n line={line!r}\n" ++ print(f"SPAN MISMATCH @ {ea:#x}\n line={line!r}\n" ++ f" old={ra!r}\n new={rb!r}") ++ # The whole row, not just the spans: `text`, the spans/text ++ # agreement guard and the name all moved around too. ++ ra, rb = old_row(ea), new_row(ea) ++ if ra != rb: ++ bad += 1 ++ if bad <= 3: ++ print(f"ROW MISMATCH @ {ea:#x}\n" + f" old={ra!r}\n new={rb!r}") + nxt = ida_bytes.get_item_end(ea) + ea = nxt if nxt > ea else ea + 1 +diff --git a/idatui/domain.py b/idatui/domain.py +index b496638..1c7883f 100644 +--- a/idatui/domain.py ++++ b/idatui/domain.py +@@ -123,19 +123,22 @@ class Head: + return None + + @classmethod +- def from_raw(cls, d: dict) -> "Head": ++ def from_raw(cls, d: dict, raw: bytes | None = None) -> "Head": + sp = d.get("spans") + ops = d.get("ops") ++ # ``tuple(map(tuple, ...))`` rather than a per-item genexpr with str()/ ++ # int() coercion: this runs once per listing row (hundreds of thousands ++ # on a real binary) and the worker's own tool already emits [str, str] ++ # and [int, int, int]. The coercion was re-proving that on every row. + return cls( + ea=_as_int(d["ea"]), + kind=d.get("kind", "unknown"), + size=int(d.get("size", 0) or 0), + text=d.get("text", ""), + name=d.get("name"), +- spans=(tuple((str(k), str(t)) for k, t in sp) +- if isinstance(sp, list) and sp else None), +- ops=(tuple((int(a), int(b), int(n)) for a, b, n in ops) +- if isinstance(ops, list) and ops else None), ++ raw=raw, ++ spans=tuple(map(tuple, sp)) if sp else None, ++ ops=tuple(map(tuple, ops)) if ops else None, + ) + + +@@ -654,30 +657,48 @@ class ListingModel: + # containing a huge coalesced undefined run doesn't pull megabytes. + _OP_SPAN_CAP = 1 << 16 + +- def _attach_opcode_bytes(self, page: list[Head]) -> list[Head]: +- """Fill ``raw`` (opcode bytes) for the code heads in ``page`` via one +- bulk read over their extent (variable-length safe).""" +- code = [h for h in page if h.kind == "code" and h.size > 0] +- if not code: +- return page +- lo = code[0].ea +- hi = code[-1].ea + code[-1].size +- if hi - lo <= 0 or hi - lo > self._OP_SPAN_CAP: +- return page +- data = self._prog.read_bytes(lo, hi - lo) ++ def _build_page(self, rows: list) -> list[Head]: ++ """Turn the tool's raw rows into ``Head``s with their opcode bytes ++ already attached, via one bulk read over the code extent. ++ ++ The bytes are read BEFORE the Heads are built rather than patched in ++ afterwards: ``dataclasses.replace`` re-runs ``__init__`` with every ++ field, so filling ``raw`` after the fact meant constructing each code ++ head twice -- once per listing row, on the path a jump-to-address walks ++ hundreds of thousands of times. ++ """ ++ lo = hi = -1 ++ for r in rows: ++ if r.get("kind") == "code" and r.get("size"): ++ ea = _as_int(r["ea"]) ++ if lo < 0: ++ lo = ea ++ hi = ea + int(r["size"]) ++ data = None ++ if 0 <= lo < hi and hi - lo <= self._OP_SPAN_CAP: ++ try: ++ data = self._prog.read_bytes(lo, hi - lo) ++ except Exception: # noqa: BLE001 -- opcode bytes are decoration ++ data = None ++ page: list[Head] = [] + biggest = self._max_raw +- out = [] +- for h in page: +- if h.kind == "code" and h.size > 0: +- off = h.ea - lo +- b = bytes(data[off:off + h.size]) +- biggest = max(biggest, len(b)) +- out.append(replace(h, raw=b)) +- else: +- out.append(h) +- with self._lock: +- self._max_raw = biggest +- return out ++ for r in rows: ++ raw = None ++ if data is not None and r.get("kind") == "code": ++ size = int(r.get("size") or 0) ++ if size > 0: ++ off = _as_int(r["ea"]) - lo ++ raw = bytes(data[off:off + size]) ++ if len(raw) > biggest: ++ biggest = len(raw) ++ try: ++ page.append(Head.from_raw(r, raw)) ++ except (KeyError, ValueError, TypeError): ++ continue ++ if biggest != self._max_raw: ++ with self._lock: ++ self._max_raw = biggest ++ return page + + def max_raw_len(self) -> int: + with self._lock: +@@ -700,13 +721,7 @@ class ListingModel: + "heads", addr=hex(frm), count=self.PAGE, annotate=True) + rows = payload.get("heads", []) if isinstance(payload, dict) else [] + cur = payload.get("cursor", {}) if isinstance(payload, dict) else {} +- page = [] +- for r in rows: +- try: +- page.append(Head.from_raw(r)) +- except (KeyError, ValueError, TypeError): +- continue +- page = self._attach_opcode_bytes(page) ++ page = self._build_page(rows) + with self._lock: + for h in page: + # Banner/label rows (function headers, separators, code labels) +diff --git a/server/patch_server.py b/server/patch_server.py +index b75b120..a1ec1ef 100644 +--- a/server/patch_server.py ++++ b/server/patch_server.py +@@ -297,33 +297,59 @@ def _idatui_head_row(ea): + else: + kind = "unknown" + line = ida_lines.generate_disasm_line(ea, 0) +- text = ida_lines.tag_remove(line) if line else "" +- text = " ".join(text.split()) # collapse IDA's column padding ++ text, spans, ops = _idatui_line_parts(line) if line else ("", None, None) + row = { + "ea": hex(ea), + "kind": kind, + "size": int(ida_bytes.get_item_size(ea)), + "text": text, + } +- if line: +- # Keep IDA's own token classification for syntax highlighting. Built from +- # the SAME line as `text`, then whitespace-collapsed identically so the +- # two never disagree about what the row says. +- spans, ops = _idatui_spans(line) +- joined = "".join(t for _k, t in spans) +- if " ".join(joined.split()) == text: +- row["spans"] = spans +- # Where each operand sits in `text`. Comes out of the same tag walk +- # (free), and is what lets the client show WHICH literal a keypress +- # would reformat before you press it. +- if ops: +- row["ops"] = ops ++ if spans is not None: ++ row["spans"] = spans ++ # Where each operand sits in `text`. Comes out of the same tag walk ++ # (free), and is what lets the client show WHICH literal a keypress ++ # would reformat before you press it. ++ if ops: ++ row["ops"] = ops + nm = ida_name.get_ea_name(ea) + if nm: + row["name"] = nm + return row + + ++import functools as _idatui_functools ++ ++ ++@_idatui_functools.lru_cache(maxsize=16384) ++def _idatui_line_parts(line): ++ """``(text, spans, ops)`` for one tagged disassembly line -- memoised. ++ ++ A function of the tagged line and nothing else, so the same line always ++ gives the same answer: a rename changes the line, which changes the key. ++ And listings repeat themselves hard -- 196k lines of bash are 53k distinct ++ ones, so a 16k-entry cache serves ~70% of them and takes the per-line cost ++ from 10.4us to 3.9us. This is the most expensive thing the backend does per ++ listing row, and a jump to an address near the end of a big binary walks ++ hundreds of thousands of them. ++ ++ ``spans`` is None when the tag walk and the plain text disagree about what ++ the line says (then the text wins and the row renders unhighlighted). ++ ++ The returned lists are SHARED between every row that has the same line; ++ treat them as read-only. Pickle notices the sharing too, so a page of ++ repetitive disassembly also serialises smaller. ++ """ ++ import ida_lines ++ text = " ".join(ida_lines.tag_remove(line).split()) # collapse the padding ++ spans, ops = _idatui_spans(line) ++ # Built from the SAME line as `text`, then whitespace-collapsed identically, ++ # so the two can never disagree about what the row says. ++ joined = "".join([t for _k, t in spans]) ++ if " ".join(joined.split()) != text: ++ return (text, None, None) ++ return (text, spans, ops) ++ ++ + #: IDA colour tag -> the semantic kind the TUI styles. IDA already classifies + #: every token in a disassembly line, for every processor it supports, so there + #: is nothing to lex: generate_disasm_line emits \x01<tag>text\x02<tag> and the +@@ -407,7 +433,7 @@ def _idatui_spans(line): + # the most expensive thing the `heads` tool did, and a line is ~54 + # characters but only ~13 tags -- everything between two tags is already + # exactly one span's worth of text. +- _IDATUI_CTL = _re.compile("([\\x01\\x02\\x03][\\s\\S])") ++ _IDATUI_CTL = _re.compile("([\\x01\\x02\\x03](?s:.))") + tags, opnds = _IDATUI_TAGS, _IDATUI_OPND_TAGS + on, off, esc = "\x01", "\x02", "\x03" + addr_tag = chr(getattr(ida_lines, "COLOR_ADDR", 0x28)) |