diff options
| -rw-r--r-- | README.md | 5 | ||||
| -rw-r--r-- | idatui/app.py | 87 | ||||
| -rw-r--r-- | idatui/trace.py | 115 | ||||
| -rw-r--r-- | tests/test_trace.py | 45 | ||||
| -rw-r--r-- | tests/test_trace_ui.py | 34 | ||||
| -rw-r--r-- | tests/test_trace_vs_tenet.py | 21 |
6 files changed, 299 insertions, 8 deletions
@@ -131,6 +131,11 @@ where you're about to go, and the instruction you're standing on. The pseudocode view is painted too — a trace records instructions, but `decomp_map` says which instructions each C line covers, so the same trail lands on the decompilation. +The dock also shows the **stack as of that instant**, read out of the trace. +Bytes the trace never observed print as `??` rather than zeros — a trace knows +what it saw and nothing else. The hex view (`\`) gets the same treatment: bytes +the trace saw at this timestamp are shown in green over the file's own contents. + Trace addresses are rebased onto the database automatically — a traced process is relocated, so nothing lines up until that's solved. diff --git a/idatui/app.py b/idatui/app.py index 9d99122..9661ebd 100644 --- a/idatui/app.py +++ b/idatui/app.py @@ -98,6 +98,11 @@ _S_CURSOR = Style(bgcolor="#2a313c") _S_TRAIL_NOW = Style(bgcolor="#3f3410") _S_TRAIL_PAST = Style(bgcolor="#2b1c17") # warm: behind you _S_TRAIL_FUTURE = Style(bgcolor="#152230") # cool: ahead of you +#: Hex with a trace loaded: bytes the trace SAW at this timestamp vs bytes we're +#: still showing from the file. The distinction matters more than the values — +#: one is evidence, the other is an assumption. +_S_HEX_LIVE = Style(color="#9ece6a") +_S_HEX_STALE = Style(color="#5e6875") _S_DIM = Style(color="#7c8b9e", italic=True) _S_MATCH = Style(bgcolor="#7a5c00") # all search matches _S_MATCH_CUR = Style(bgcolor="#d0a215", color="#12161c") # the current match @@ -1561,6 +1566,10 @@ class HexView(ScrollView, can_focus=True): super().__init__(id="hex") self.model = None self.total = 0 + #: Trace to read memory from, and the timestamp to read it at. When set, + #: the dump shows what memory HELD then rather than what the file holds. + self.trace = None + self.trace_idx = 0 self._internal_top: int | None = None # scroll target we set ourselves # -- public API -------------------------------------------------------- # @@ -1747,6 +1756,15 @@ class HexView(ScrollView, can_focus=True): return Strip([Segment("".ljust(width), _S_HEX)]) va, data = model.row(r) cur_row, cur_col = self.cursor // 16, self.cursor % 16 + # With a trace loaded the row shows what memory HELD at the current + # timestamp, not what the file contains. Only the bytes the trace + # actually saw are overlaid: the rest stay the database's, dimmed, so + # you can always tell evidence from the file's idea of the world. + tmem = tknown = None + if self.trace is not None and data is not None: + tmem, tknown = self.trace.memory(va, len(data), self.trace_idx) + if not any(tknown): + tmem = tknown = None fo = model.file_offset(va) fo_str = f"{fo:08X}" if fo is not None else "--------" segs: list[Segment] = [ @@ -1761,15 +1779,29 @@ class HexView(ScrollView, can_focus=True): if i == 8: segs.append(Segment(" ", _S_HEX)) if i < n: - st = _S_CELL if (r == cur_row and i == cur_col) else _S_HEX - segs.append(Segment(f"{data[i]:02X} ", st)) + live = tknown is not None and tknown[i] + val = tmem[i] if live else data[i] + if r == cur_row and i == cur_col: + st = _S_CELL + elif tknown is None: + st = _S_HEX + else: + st = _S_HEX_LIVE if live else _S_HEX_STALE + segs.append(Segment(f"{val:02X} ", st)) else: segs.append(Segment(" ", _S_HEX)) segs.append(Segment(" |", _S_DIM)) for i in range(16): if i < n: - ch = chr(data[i]) if 32 <= data[i] < 127 else "." - st = _S_CELL if (r == cur_row and i == cur_col) else _S_ASCII + live = tknown is not None and tknown[i] + val = tmem[i] if live else data[i] + ch = chr(val) if 32 <= val < 127 else "." + if r == cur_row and i == cur_col: + st = _S_CELL + elif tknown is None: + st = _S_ASCII + else: + st = _S_HEX_LIVE if live else _S_HEX_STALE else: ch, st = " ", _S_ASCII segs.append(Segment(ch, st)) @@ -2325,6 +2357,7 @@ class TraceDock(Vertical): def compose(self) -> ComposeResult: yield Static("", id="trace-head", markup=False) yield Static("", id="trace-regs", markup=False) + yield Static("", id="trace-stack", markup=False) yield TraceTimeline(id="trace-timeline") def show(self, trace, idx: int) -> None: @@ -2367,11 +2400,49 @@ class TraceDock(Vertical): body.append(f"{v:#018x}\n" if v > 0xFFFFFFFF else f"{v:#010x}\n", _S_DATA if hot else (_S_LABEL if name == pc else _S_INSN)) self.query_one("#trace-regs", Static).update(body) + self._render_stack(t) tl = self.query_one(TraceTimeline) tl.idx = self.idx tl.refresh() + STACK_WORDS = 8 + + def _render_stack(self, t) -> None: # type: ignore[no-untyped-def] + """The stack as of this instant, read out of the trace. + + This is where a trace's memory actually is: on two real traces, NONE of + the accesses fell inside the image — every one was stack or heap. A + memory view that could only address the image would have nothing to show. + + Bytes the trace never saw are printed as '??' rather than zeros. A trace + knows what it observed and nothing else, and quietly rendering unseen + memory as zero would invent facts. + """ + sp_name = next((r for r in ("rsp", "esp", "sp") if r in t.reg_at), "") + sp = t.register(sp_name, self.idx) if sp_name else None + out = Text() + if sp is None: + self.query_one("#trace-stack", Static).update(out) + return + width = 8 if (t.info and "64" in (t.info.arch or "")) else 4 + out.append(f" stack ({sp_name})\n", _S_DIM) + for k in range(self.STACK_WORDS): + a = sp + k * width + data, known = t.memory_raw(a, width, self.idx) + out.append(" \u25b8" if k == 0 else " ", _S_MNEM) + out.append(f"{a:012x} ", _S_ADDR) + if all(known): + v = int.from_bytes(data, "little") + out.append(f"{v:0{width * 2}x}\n", _S_DATA if k == 0 else _S_INSN) + elif any(known): + out.append("".join(f"{b:02x}" if known[i] else "??" + for i, b in enumerate(data)) + "\n", _S_INSN) + else: + out.append("?" * (width * 2) + "\n", _S_SEP) + self.query_one("#trace-stack", Static).update(out) + + class TraceTimeline(Static): """The trace as a vertical bar: where you are, and where you've been. @@ -3147,6 +3218,7 @@ class IdaTui(App): #trace-dock { dock: right; width: 34; background: $surface; border-left: solid $panel; } #trace-head { height: 2; padding: 0 1; background: $panel; } #trace-regs { height: auto; padding: 1 0 0 0; } + #trace-stack { height: auto; padding: 1 0 0 0; } #trace-timeline { height: 1fr; padding: 1 0 0 0; } #load-note { height: 2; padding: 1 1 0 1; color: $text-muted; } /* Cap the processor list so the ADDRESS FIELD is always on screen: with the @@ -5588,6 +5660,13 @@ class IdaTui(App): t = self._trace if t is None: return + try: + hx = self.query_one(HexView) + hx.trace, hx.trace_idx = t, self._t + if hx.display: + hx.refresh() + except Exception: # noqa: BLE001 -- not mounted yet + pass trail = t.trail(self._t) try: lst = self.query_one(ListingView) diff --git a/idatui/trace.py b/idatui/trace.py index 2afc8f6..931f918 100644 --- a/idatui/trace.py +++ b/idatui/trace.py @@ -76,7 +76,14 @@ class TraceInfo: @dataclass class MemOp: - """One memory access made by one instruction.""" + """One memory access made by one instruction. + + ``addr`` is the address the TRACE recorded — not slid onto the database. + Most accesses are stack or heap, which have no counterpart in the database + at all, and applying the image's relocation to a stack pointer produces a + nonsense address (it went negative in testing). Only addresses inside the + image can be translated, and the caller knows when that applies. + """ addr: int data: bytes @@ -116,6 +123,10 @@ class Trace: mem_row: array.array = field(default_factory=lambda: array.array("I")) #: applied so trace addresses line up with the database (see ``rebase``). slide: int = 0 + #: accesses ordered by address, built on first memory query. + _mem_order: list | None = None + _mem_starts: list = field(default_factory=list) + _mem_maxlen: int = 0 # -- construction ------------------------------------------------------ # @classmethod @@ -266,11 +277,111 @@ class Trace: out = [] for k in range(lo, hi): off, ln = self.mem_off[k], self.mem_len[k] - out.append(MemOp(addr=self.mem_addr[k] + self.slide, + out.append(MemOp(addr=self.mem_addr[k], data=bytes(self.mem_blob[off:off + ln]), write=bool(self.mem_write[k]))) return out + # -- memory state ------------------------------------------------------- # + def _mem_index(self) -> None: + """Order the accesses by address, once. + + Queries ask "what was at this window at time t", so the accesses that + matter are the few touching that window — not the tens of thousands in + the trace. Sorting by address makes those a bisect away; sorting by time + (the order they arrive in) would mean scanning everything per repaint. + """ + if self._mem_order is not None: + return + order = sorted(range(len(self.mem_addr)), key=lambda k: self.mem_addr[k]) + self._mem_order = order + self._mem_starts = [self.mem_addr[k] for k in order] + self._mem_maxlen = max(self.mem_len) if len(self.mem_len) else 0 + + def memory_raw(self, addr: int, length: int, + idx: int | None = None) -> tuple[bytes, bytes]: + """Memory at a TRACE address (no slide). + + The stack lives here. Measured on two real traces, 0% of memory accesses + fall inside the image — every one is stack or heap — so a query that + insists on database addresses can't answer the question anyone actually + has about memory in a trace. + """ + return self.memory(addr + self.slide, length, idx) + + def memory(self, addr: int, length: int, + idx: int | None = None) -> tuple[bytes, bytes]: + """``(data, known)`` for ``length`` bytes at ``addr`` as of ``idx``. + + ``known`` is a byte-per-byte mask: a trace only says what it saw, so a + byte nobody read or wrote is genuinely unknown and must not be drawn as + zero. That distinction is the whole value of reading memory from a trace + rather than from the database — the database has the file's bytes, the + trace has what was actually there at that instant. + + Reads count as evidence, not just writes: an instruction reading a byte + reveals what it held then. + """ + if idx is None: + idx = self.length - 1 + length = max(int(length), 0) + out, known = bytearray(length), bytearray(length) + if not length or not len(self.mem_idx): + return bytes(out), bytes(known) + self._mem_index() + raw = addr - self.slide + best = [-1] * length + import bisect as _b + lo = _b.bisect_left(self._mem_starts, raw - self._mem_maxlen) + hi = _b.bisect_right(self._mem_starts, raw + length - 1) + for pos in range(lo, hi): + k = self._mem_order[pos] + t = self.mem_idx[k] + if t > idx: + continue + a, ln = self.mem_addr[k], self.mem_len[k] + s, e = max(a, raw), min(a + ln, raw + length) + if s >= e: + continue + off = self.mem_off[k] + for b in range(s, e): + j = b - raw + # >= not >: several accesses can share a timestamp (an + # instruction that reads and writes), and the later entry on the + # line is the one that stands. + if t >= best[j]: + best[j] = t + out[j] = self.mem_blob[off + (b - a)] + known[j] = 1 + return bytes(out), bytes(known) + + def memory_writes(self, addr: int, length: int) -> list[int]: + """Timestamps that WROTE any byte of ``[addr, addr+length)``.""" + return self._mem_touch(addr, length, writes=True) + + def memory_accesses(self, addr: int, length: int) -> list[int]: + """Timestamps that read or wrote any byte of the range.""" + return self._mem_touch(addr, length, writes=False) + + def _mem_touch(self, addr: int, length: int, writes: bool) -> list[int]: + if not len(self.mem_idx) or length <= 0: + return [] + self._mem_index() + raw = addr - self.slide + import bisect as _b + lo = _b.bisect_left(self._mem_starts, raw - self._mem_maxlen) + hi = _b.bisect_right(self._mem_starts, raw + length - 1) + out = set() + for pos in range(lo, hi): + k = self._mem_order[pos] + a, ln = self.mem_addr[k], self.mem_len[k] + if a + ln <= raw or a >= raw + length: + continue + if writes and not self.mem_write[k]: + continue + out.add(self.mem_idx[k]) + return sorted(out) + # -- execution queries (what painting is built on) ---------------------- # def executions(self, ea: int) -> array.array: """Every timestamp that executed ``ea`` (database address).""" diff --git a/tests/test_trace.py b/tests/test_trace.py index 2803a11..58c9d81 100644 --- a/tests/test_trace.py +++ b/tests/test_trace.py @@ -87,6 +87,43 @@ def main() -> int: [o.write for o in t.memory_ops(4)] == [False]) check("an instruction with no memory has none", t.memory_ops(2) == []) + # -- memory state at a timestamp ------------------------------------ # + # The trace wrote 2a000000 at 0xff4 (t=3) and read it back (t=4); it + # pushed/popped 8 zero bytes at 0xff8 (t=1 write, t=5 read). + d, k = t.memory(0xff4, 4, 3) + check("memory reflects a write as of that timestamp", + d == bytes.fromhex("2a000000") and k == b"\x01" * 4, f"{d.hex()} {k.hex()}") + d, k = t.memory(0xff4, 4, 2) + check("and does NOT reflect it before the write happened", + k == b"\x00" * 4, f"{d.hex()} known={k.hex()}") + + # A trace only knows what it saw. A byte nobody touched is unknown, and + # must not be reported as zero — that distinction is the entire reason + # to read memory from a trace instead of from the database. + d, k = t.memory(0x5000, 4, t.length - 1) + check("untouched memory is unknown, not zero", + k == b"\x00" * 4 and d == b"\x00" * 4, f"known={k.hex()}") + # 0xff2..0xff3 was never touched; 0xff4..0xff7 came from the write at + # t=3 and 0xff8..0xff9 from the push at t=1 — a window can be knowable + # from several accesses at different times, which is what makes this + # worth a mask rather than a flag. + d, k = t.memory(0xff2, 8, 4) + check("a partially-covered window marks which bytes are known", + k == bytes([0, 0, 1, 1, 1, 1, 1, 1]), f"known={k.hex()}") + + # Reads are evidence too: an instruction reading a byte reveals what it + # held at that moment. + d, k = t.memory(0xff8, 8, 5) + check("a read reveals memory contents", + k == b"\x01" * 8, f"known={k.hex()}") + + check("memory_writes lists only the writers", + t.memory_writes(0xff4, 4) == [3], f"{t.memory_writes(0xff4, 4)}") + check("memory_accesses includes the readers", + t.memory_accesses(0xff4, 4) == [3, 4], f"{t.memory_accesses(0xff4, 4)}") + check("a never-touched range has no accesses", + t.memory_accesses(0x5000, 16) == []) + # -- execution queries: what painting is built on -------------------- # check("executions lists every timestamp for an address", list(t.executions(0x401000)) == [0, 6], f"{list(t.executions(0x401000))}") @@ -119,8 +156,12 @@ def main() -> int: check("addresses come back in database terms", t.ip(0) == 0x1000 and list(t.executions(0x1000)) == [0, 6], f"{t.ip(0):#x}") - check("memory addresses are slid too", - t.memory_ops(3)[0].addr == 0xff4 + slide) + # Memory addresses are NOT slid: the slide relocates the image, and + # these are overwhelmingly stack/heap addresses with no database + # counterpart — sliding a stack pointer by the image delta produced a + # negative address in testing. + check("memory op addresses stay in trace space", + t.memory_ops(3)[0].addr == 0xff4, f"{t.memory_ops(3)[0].addr:#x}") check("raw_ip still gives the traced address", t.raw_ip(0) == 0x401000) diff --git a/tests/test_trace_ui.py b/tests/test_trace_ui.py index 5d620a9..b672143 100644 --- a/tests/test_trace_ui.py +++ b/tests/test_trace_ui.py @@ -141,6 +141,40 @@ async def run() -> int: app._t == ret, f"{i} -> {app._t}, expected {ret}") check("which is further than a plain step", app._t > i + 1) + # -- memory at time T -------------------------------------------- # + # This is where a trace's memory actually lives: measured on two + # real traces, NONE of the accesses fell inside the image — every + # one was stack or heap. A memory view that could only address the + # image would have nothing to show. + dock = app.query_one(TraceDock) + app._seek(min(60, t.length - 1)) + await pilot.pause(0.8) + stack = str(dock.query_one("#trace-stack", Static).render()) + check("the dock shows the stack at this timestamp", + "stack (" in stack and len(stack.splitlines()) > 4, stack[:60]) + sp_name = next(r for r in ("rsp", "esp", "sp") if r in t.reg_at) + sp = t.register(sp_name, app._t) + check("anchored at the stack pointer", + f"{sp:012x}" in stack, f"sp={sp:#x} / {stack[:80]}") + + # A trace knows what it observed and nothing else. Unseen bytes are + # printed as '?', never as zeros — rendering them as zero would + # invent facts about memory nobody looked at. + data, known = t.memory_raw(sp, 8, app._t) + if not all(known): + check("memory the trace never saw is marked unknown", + "?" in stack, stack[:80]) + else: + check("known stack words are shown as values", + any(c in "0123456789abcdef" for c in stack), stack[:60]) + + # Stepping must move the memory view with time. + before = stack + app._seek(min(80, t.length - 1)) + await pilot.pause(0.8) + check("and it follows as you move through time", + str(dock.query_one("#trace-stack", Static).render()) != before) + # -- trails ------------------------------------------------------ # # Not "every address the trace ever touched": on a loop-heavy # program that's almost everything and says nothing. The last/next diff --git a/tests/test_trace_vs_tenet.py b/tests/test_trace_vs_tenet.py index 6c78c95..aa6002a 100644 --- a/tests/test_trace_vs_tenet.py +++ b/tests/test_trace_vs_tenet.py @@ -134,6 +134,27 @@ def compare(path, ref_cls, arch, dctx, samples=200): check(f"{name}: same execution timestamps for the hottest addresses", not ex_bad, f"{ex_bad[:3]}") + # Memory STATE at a timestamp — reconstructed from the deltas, which is the + # hard part and the whole point of reading memory from a trace. + mem_bad, mem_checked = [], 0 + for i in idxs[::4]: + for op in ours.memory_ops(i)[:2]: + n = min(len(op.data), 8) + mine, known = ours.memory(op.addr, n, i) + ref = theirs.get_memory(op.addr, n, i) + if ref is None: + continue + refb = bytes(ref.data) + # Compare only the bytes we claim to know; the reference reports its + # own coverage separately and a byte neither has seen is not a + # disagreement. + for j in range(n): + if known[j] and refb[j:j + 1] and mine[j] != refb[j]: + mem_bad.append((i, hex(op.addr + j), mine[j], refb[j])) + mem_checked += 1 + check(f"{name}: memory state at a timestamp matches the reference", + not mem_bad, f"{mem_bad[:3]}") + # Memory: the bytes an instruction touched, and which way. with_mem = [i for i in idxs if ours.memory_ops(i)][:40] mem_bad = [] |