diff --git a/.auto/diff_spans.py b/.auto/diff_spans.py index 95721a5..aad4ebc 100644 --- a/.auto/diff_spans.py +++ b/.auto/diff_spans.py @@ -39,11 +39,11 @@ def load_impl(path: str, name: str): mod = importlib.util.module_from_spec(spec) spec.loader.exec_module(mod) # IDA-free at import time body = mod.BODY - a = body.index("#: IDA colour tag -> the semantic kind") - b = body.index("def _idatui_unknown_row") + a = body.index("def _idatui_head_row") + b = body.index("def _idatui_struct_member_rows") g = {"__name__": name} exec(compile(body[a:b], name, "exec"), g) # noqa: S102 - return g["_idatui_spans"] + return g def main() -> int: @@ -59,8 +59,10 @@ def main() -> int: fh.write(subprocess.run( ["git", "-C", ROOT, "show", f"{a.ref}:server/patch_server.py"], capture_output=True, text=True, check=True).stdout) - new = load_impl(os.path.join(ROOT, "server", "patch_server.py"), "new") - old = load_impl(old_path, "old") + gnew = load_impl(os.path.join(ROOT, "server", "patch_server.py"), "new") + gold = load_impl(old_path, "old") + new, old = gnew["_idatui_spans"], gold["_idatui_spans"] + new_row, old_row = gnew["_idatui_head_row"], gold["_idatui_head_row"] binary = os.path.join(ROOT, a.target) tgt = os.path.join(d, os.path.basename(binary)) @@ -89,7 +91,15 @@ def main() -> int: if ra != rb: bad += 1 if bad <= 3: - print(f"MISMATCH @ {ea:#x}\n line={line!r}\n" + print(f"SPAN MISMATCH @ {ea:#x}\n line={line!r}\n" + f" old={ra!r}\n new={rb!r}") + # The whole row, not just the spans: `text`, the spans/text + # agreement guard and the name all moved around too. + ra, rb = old_row(ea), new_row(ea) + if ra != rb: + bad += 1 + if bad <= 3: + print(f"ROW MISMATCH @ {ea:#x}\n" f" old={ra!r}\n new={rb!r}") nxt = ida_bytes.get_item_end(ea) ea = nxt if nxt > ea else ea + 1 diff --git a/idatui/domain.py b/idatui/domain.py index b496638..1c7883f 100644 --- a/idatui/domain.py +++ b/idatui/domain.py @@ -123,19 +123,22 @@ class Head: return None @classmethod - def from_raw(cls, d: dict) -> "Head": + def from_raw(cls, d: dict, raw: bytes | None = None) -> "Head": sp = d.get("spans") ops = d.get("ops") + # ``tuple(map(tuple, ...))`` rather than a per-item genexpr with str()/ + # int() coercion: this runs once per listing row (hundreds of thousands + # on a real binary) and the worker's own tool already emits [str, str] + # and [int, int, int]. The coercion was re-proving that on every row. return cls( ea=_as_int(d["ea"]), kind=d.get("kind", "unknown"), size=int(d.get("size", 0) or 0), text=d.get("text", ""), name=d.get("name"), - spans=(tuple((str(k), str(t)) for k, t in sp) - if isinstance(sp, list) and sp else None), - ops=(tuple((int(a), int(b), int(n)) for a, b, n in ops) - if isinstance(ops, list) and ops else None), + raw=raw, + spans=tuple(map(tuple, sp)) if sp else None, + ops=tuple(map(tuple, ops)) if ops else None, ) @@ -654,30 +657,48 @@ class ListingModel: # containing a huge coalesced undefined run doesn't pull megabytes. _OP_SPAN_CAP = 1 << 16 - def _attach_opcode_bytes(self, page: list[Head]) -> list[Head]: - """Fill ``raw`` (opcode bytes) for the code heads in ``page`` via one - bulk read over their extent (variable-length safe).""" - code = [h for h in page if h.kind == "code" and h.size > 0] - if not code: - return page - lo = code[0].ea - hi = code[-1].ea + code[-1].size - if hi - lo <= 0 or hi - lo > self._OP_SPAN_CAP: - return page - data = self._prog.read_bytes(lo, hi - lo) + def _build_page(self, rows: list) -> list[Head]: + """Turn the tool's raw rows into ``Head``s with their opcode bytes + already attached, via one bulk read over the code extent. + + The bytes are read BEFORE the Heads are built rather than patched in + afterwards: ``dataclasses.replace`` re-runs ``__init__`` with every + field, so filling ``raw`` after the fact meant constructing each code + head twice -- once per listing row, on the path a jump-to-address walks + hundreds of thousands of times. + """ + lo = hi = -1 + for r in rows: + if r.get("kind") == "code" and r.get("size"): + ea = _as_int(r["ea"]) + if lo < 0: + lo = ea + hi = ea + int(r["size"]) + data = None + if 0 <= lo < hi and hi - lo <= self._OP_SPAN_CAP: + try: + data = self._prog.read_bytes(lo, hi - lo) + except Exception: # noqa: BLE001 -- opcode bytes are decoration + data = None + page: list[Head] = [] biggest = self._max_raw - out = [] - for h in page: - if h.kind == "code" and h.size > 0: - off = h.ea - lo - b = bytes(data[off:off + h.size]) - biggest = max(biggest, len(b)) - out.append(replace(h, raw=b)) - else: - out.append(h) - with self._lock: - self._max_raw = biggest - return out + for r in rows: + raw = None + if data is not None and r.get("kind") == "code": + size = int(r.get("size") or 0) + if size > 0: + off = _as_int(r["ea"]) - lo + raw = bytes(data[off:off + size]) + if len(raw) > biggest: + biggest = len(raw) + try: + page.append(Head.from_raw(r, raw)) + except (KeyError, ValueError, TypeError): + continue + if biggest != self._max_raw: + with self._lock: + self._max_raw = biggest + return page def max_raw_len(self) -> int: with self._lock: @@ -700,13 +721,7 @@ class ListingModel: "heads", addr=hex(frm), count=self.PAGE, annotate=True) rows = payload.get("heads", []) if isinstance(payload, dict) else [] cur = payload.get("cursor", {}) if isinstance(payload, dict) else {} - page = [] - for r in rows: - try: - page.append(Head.from_raw(r)) - except (KeyError, ValueError, TypeError): - continue - page = self._attach_opcode_bytes(page) + page = self._build_page(rows) with self._lock: for h in page: # Banner/label rows (function headers, separators, code labels) diff --git a/server/patch_server.py b/server/patch_server.py index b75b120..a1ec1ef 100644 --- a/server/patch_server.py +++ b/server/patch_server.py @@ -297,33 +297,59 @@ def _idatui_head_row(ea): else: kind = "unknown" line = ida_lines.generate_disasm_line(ea, 0) - text = ida_lines.tag_remove(line) if line else "" - text = " ".join(text.split()) # collapse IDA's column padding + text, spans, ops = _idatui_line_parts(line) if line else ("", None, None) row = { "ea": hex(ea), "kind": kind, "size": int(ida_bytes.get_item_size(ea)), "text": text, } - if line: - # Keep IDA's own token classification for syntax highlighting. Built from - # the SAME line as `text`, then whitespace-collapsed identically so the - # two never disagree about what the row says. - spans, ops = _idatui_spans(line) - joined = "".join(t for _k, t in spans) - if " ".join(joined.split()) == text: - row["spans"] = spans - # Where each operand sits in `text`. Comes out of the same tag walk - # (free), and is what lets the client show WHICH literal a keypress - # would reformat before you press it. - if ops: - row["ops"] = ops + if spans is not None: + row["spans"] = spans + # Where each operand sits in `text`. Comes out of the same tag walk + # (free), and is what lets the client show WHICH literal a keypress + # would reformat before you press it. + if ops: + row["ops"] = ops nm = ida_name.get_ea_name(ea) if nm: row["name"] = nm return row +import functools as _idatui_functools + + +@_idatui_functools.lru_cache(maxsize=16384) +def _idatui_line_parts(line): + """``(text, spans, ops)`` for one tagged disassembly line -- memoised. + + A function of the tagged line and nothing else, so the same line always + gives the same answer: a rename changes the line, which changes the key. + And listings repeat themselves hard -- 196k lines of bash are 53k distinct + ones, so a 16k-entry cache serves ~70% of them and takes the per-line cost + from 10.4us to 3.9us. This is the most expensive thing the backend does per + listing row, and a jump to an address near the end of a big binary walks + hundreds of thousands of them. + + ``spans`` is None when the tag walk and the plain text disagree about what + the line says (then the text wins and the row renders unhighlighted). + + The returned lists are SHARED between every row that has the same line; + treat them as read-only. Pickle notices the sharing too, so a page of + repetitive disassembly also serialises smaller. + """ + import ida_lines + text = " ".join(ida_lines.tag_remove(line).split()) # collapse the padding + spans, ops = _idatui_spans(line) + # Built from the SAME line as `text`, then whitespace-collapsed identically, + # so the two can never disagree about what the row says. + joined = "".join([t for _k, t in spans]) + if " ".join(joined.split()) != text: + return (text, None, None) + return (text, spans, ops) + + #: IDA colour tag -> the semantic kind the TUI styles. IDA already classifies #: every token in a disassembly line, for every processor it supports, so there #: is nothing to lex: generate_disasm_line emits \x01text\x02 and the @@ -407,7 +433,7 @@ def _idatui_spans(line): # the most expensive thing the `heads` tool did, and a line is ~54 # characters but only ~13 tags -- everything between two tags is already # exactly one span's worth of text. - _IDATUI_CTL = _re.compile("([\\x01\\x02\\x03][\\s\\S])") + _IDATUI_CTL = _re.compile("([\\x01\\x02\\x03](?s:.))") tags, opnds = _IDATUI_TAGS, _IDATUI_OPND_TAGS on, off, esc = "\x01", "\x02", "\x03" addr_tag = chr(getattr(ida_lines, "COLOR_ADDR", 0x28))