diff options
Diffstat (limited to 'idatui')
| -rw-r--r-- | idatui/app.py | 81 | ||||
| -rw-r--r-- | idatui/codemode_client.py | 101 | ||||
| -rw-r--r-- | idatui/domain.py | 69 | ||||
| -rw-r--r-- | idatui/drive.py | 10 | ||||
| -rw-r--r-- | idatui/edit_ctl.py | 5 | ||||
| -rw-r--r-- | idatui/findings.py | 384 | ||||
| -rw-r--r-- | idatui/journal.py | 107 | ||||
| -rw-r--r-- | idatui/rpc.py | 22 |
8 files changed, 773 insertions, 6 deletions
diff --git a/idatui/app.py b/idatui/app.py index 608cc4d..4a603f3 100644 --- a/idatui/app.py +++ b/idatui/app.py @@ -50,7 +50,9 @@ from . import kittygfx from .edit_ctl import EditController from .prompt import PromptBar from .trace_ctl import TraceController +from . import findings from .highlight import CTextArea, highlight_c +from .journal import Journal from .errors import IDAToolError, IDAConnectionError from .codemode_client import CodeModeClient, registered_database @@ -3444,6 +3446,7 @@ _HELP = ( ("B", "cycle the opcode-bytes column"), ("Ctrl+B", "show/hide the names pane"), ("Ctrl+T", "structs / types editor"), + ("Ctrl+E", "export findings as markdown"), ("Ctrl+P", "command palette"), )), ("Move", ( @@ -4539,6 +4542,8 @@ class StructEditor(ModalScreen): self._set_status(f"save failed — {msg}", error=True) return self._loaded = name + if name: + self.app.journal.record("type", None, name) if formatted: # Reformat the editor to IDA's canonical layout (keeps cursor at top). ta = self.query_one("#se-edit", TextArea) @@ -4590,6 +4595,7 @@ class StructEditor(ModalScreen): return if getattr(self.app, "_dirty", None) is not None: self.app._dirty = True + self.app.journal.record("del_type", None, name) self._refresh() self._set_status(f"deleted {name}") @@ -4664,6 +4670,8 @@ class IdaCommands(Provider): app.action_strings), ("Switch binary…", "another binary in the project (Ctrl+O)", app.action_switch_binary), + ("Export findings…", "your comments, names and types as markdown " + "(Ctrl+E)", app.action_export), ("Keyboard shortcuts", "the key cheatsheet (F1 or H)", app.action_help), ("Follow symbol under cursor", "jump to the referenced symbol (Enter)", @@ -4774,6 +4782,10 @@ class IdaTui(App): height: 1; border: none; padding: 0 1; background: $primary-darken-3; color: $text; } + #export { + height: 1; border: none; padding: 0 1; + background: $success-darken-3; color: $text; + } #status { height: 1; background: $panel; color: $text; padding: 0 1; } .decomp-loading { width: 100%; height: 100%; content-align: center middle; @@ -4882,6 +4894,7 @@ class IdaTui(App): Binding("q", "quit", "Quit"), Binding("ctrl+n", "symbols", "Symbols"), Binding("ctrl+t", "structs", "Structs"), + Binding("ctrl+e", "export", "Export", show=False), Binding("backslash", "hex", "Hex"), Binding("s", "toggle_split", "Split", show=False), # IDA's own key for text/graph. Graph mode is opt-in and self-contained: @@ -4993,9 +5006,12 @@ class IdaTui(App): #: The one-line prompts above the footer. Each holds its own context #: for exactly as long as it is on screen; see idatui/prompt.py. self.prompts = PromptBar(self, "search", "rename", "comment", - "retype", "makedata", "goto") + "retype", "makedata", "goto", "export") #: Everything that writes to the database (idatui/edit_ctl.py). self.edits = EditController(self) + #: What those writes were, so the findings export can say which + #: comments and names are YOURS -- the database cannot (journal.py). + self.journal = Journal() self._xref_focus_name: str | None = None self._dirty = False @@ -5045,6 +5061,10 @@ class IdaTui(App): gi.display = False gi.can_focus = False yield gi + xi = Input(id="export") + xi.display = False + xi.can_focus = False + yield xi # markup=False: the status is plain text full of [listing]/[split]/[label] # markers and symbol names that may contain brackets. With Textual markup # on, a single-word marker parses as a style tag and is silently eaten — @@ -6304,6 +6324,50 @@ class IdaTui(App): inp.value = self._filter_term inp.focus() + def action_export(self) -> None: + """Ctrl+E: write what you have worked out to a markdown report. + + Prefilled with a path beside the binary, because the common case is + "just write it" and the rare case is worth one edit. + """ + if self.program is None: + self._status("no database open", priority=True) + return + inp = self.query_one("#export", Input) + inp.placeholder = "export findings to… (markdown) — Enter" + inp.can_focus = True + inp.display = True + inp.value = findings.default_path(self._open_path or "findings") + self.query_one("#status", Static).display = False + inp.focus() + + def _end_export(self) -> None: + inp = self.query_one("#export", Input) + inp.display = False + inp.can_focus = False + self.query_one("#status", Static).display = True + + def export_findings(self, path: str | None = None) -> None: + """Gather + write the report off the UI thread (it walks the database).""" + self._status("exporting findings…", priority=True) + self._export_worker(path) + + @work(thread=True, exclusive=True, group="export") + def _export_worker(self, path: str | None) -> None: + try: + self.journal.load(self.program) + self.journal.flush(self.program) + out, f = findings.export(self.program, self._open_path or "", path, + journal=self.journal) + except Exception as e: # noqa: BLE001 -- a bad path is a message, not a crash + self.call_from_thread(self._status, f"export failed: {e}", True) + return + n_named = len(findings._user_names(f)) + self.call_from_thread( + self._status, + f"exported {len(f.comments)} comments, {n_named} names, " + f"{len(f.types)} types → {out}", True) + def action_goto(self) -> None: inp = self.query_one("#goto", Input) inp.placeholder = ("hex goto: 0xADDR or name — Enter" if self.is_hex @@ -6892,6 +6956,7 @@ class IdaTui(App): def _save(self) -> None: assert self.program is not None try: + self.journal.flush(self.program) # ride along into the .i64 self.program.client.save_database() except Exception as e: # noqa: BLE001 self.app.call_from_thread(self._status, f"save failed: {e}") @@ -7191,8 +7256,8 @@ class IdaTui(App): event.prevent_default() if prompt.id == "search": self._end_search(cancel=True) - elif prompt.id == "goto": - self._end_goto() + elif prompt.id in ("goto", "export"): + self._end_goto() if prompt.id == "goto" else self._end_export() (self.query_one(HexView) if self.is_hex else (self._code_view() or self.query_one(ListingView))).focus() else: @@ -7240,6 +7305,13 @@ class IdaTui(App): if value: self._goto(value) return + if inp.id == "export": + self._end_export() + (self.query_one(HexView) if self.is_hex + else (self._code_view() or self.query_one(ListingView))).focus() + if value: + self.export_findings(value) + return # filter mode: already applied incrementally; Enter just confirms + closes. self._apply_filter(value) self.query_one("#func-table", DataTable).focus() @@ -7461,7 +7533,8 @@ class IdaTui(App): # Prompt overlays that own the keyboard while visible; a background # navigation must not yank focus out from under them (else typed keys leak # into a code view as destructive verbs — e.g. 'u' = undefine). - _PROMPT_IDS = ("search", "rename", "comment", "retype", "goto", "func-filter") + _PROMPT_IDS = ("search", "rename", "comment", "retype", "goto", "export", + "func-filter") def _prompt_active(self) -> bool: for iid in self._PROMPT_IDS: diff --git a/idatui/codemode_client.py b/idatui/codemode_client.py index 88d7b31..4c3ec99 100644 --- a/idatui/codemode_client.py +++ b/idatui/codemode_client.py @@ -501,6 +501,107 @@ for item in page: result = {"strings": rows, "total": len(items), "next_offset": offset + len(rows)} result ''', + # Everything a person ADDED to the database: comments, non-dummy names, and + # the prototypes they set. + # + # Names come from IDA's name list, which is already an index -- no scan at + # all. Comments have no index, so they need a walk, and the walk is over + # HEADS: `next_that`'s predicate is a *Python* callback (SWIG calls it with + # one argument, so `f_has_cmt` does not even fit), which would be one call + # per BYTE -- 400 million of them on a big image. `max_scan` bounds it and + # reports `truncated` rather than sitting there. + "list_annotations": r''' +import ida_bytes, ida_funcs, ida_lines, ida_nalt, ida_name +import ida_segment, ida_typeinf, idautils +limit = max(1, int(a.get("limit", 4000))) +max_scan = max(1000, int(a.get("max_scan", 2000000))) +comments, names = [], [] +scanned = 0 + +def _line(ea): + try: + txt = ida_lines.generate_disasm_line(ea, ida_lines.GENDSM_REMOVE_TAGS) + except Exception: + txt = "" + return " ".join((txt or "").split()) + +for ea, nm in idautils.Names(): + if len(names) >= limit: + break + if not nm or not ida_bytes.has_user_name(ida_bytes.get_flags(ea)): + continue + fn = ida_funcs.get_func(ea) + is_fn = fn is not None and int(fn.start_ea) == int(ea) + proto = None + if is_fn: + try: + ti = ida_typeinf.tinfo_t() + if ida_nalt.get_tinfo(ti, ea): + proto = str(ti) + except Exception: + proto = None + seg = ida_segment.getseg(ea) + names.append({"addr": hex(int(ea)), "name": nm, "func": is_fn, + "size": (int(fn.end_ea - fn.start_ea) if is_fn else 0), + "proto": proto, + "seg": (ida_segment.get_segm_name(seg) if seg else "")}) + +for i in range(ida_segment.get_segm_qty()): + seg = ida_segment.getnseg(i) + if seg is None or len(comments) >= limit or scanned >= max_scan: + continue + for ea in idautils.Heads(seg.start_ea, seg.end_ea): + scanned += 1 + if len(comments) >= limit or scanned >= max_scan: + break + if not ida_bytes.has_cmt(ida_bytes.get_flags(ea)): + continue + for rep in (False, True): + text = ida_bytes.get_cmt(ea, rep) + if text: + fn = ida_funcs.get_func(ea) + comments.append({ + "addr": hex(int(ea)), "text": text, "repeatable": rep, + "line": _line(ea), "seg": ida_segment.get_segm_name(seg), + "func": (ida_funcs.get_func_name(fn.start_ea) if fn else None), + "func_addr": (hex(int(fn.start_ea)) if fn else None)}) + +# Whole-function comments are not on the byte flags, so the scan cannot see them. +for fn_ea in idautils.Functions(): + fn = ida_funcs.get_func(fn_ea) + if fn is None or len(comments) >= limit: + continue + for rep in (False, True): + text = ida_funcs.get_func_cmt(fn, rep) + if text: + seg = ida_segment.getseg(fn_ea) + comments.append({"addr": hex(int(fn_ea)), "text": text, + "repeatable": rep, "line": "", "whole_func": True, + "seg": (ida_segment.get_segm_name(seg) if seg else ""), + "func": ida_funcs.get_func_name(fn_ea), + "func_addr": hex(int(fn_ea))}) +result = {"comments": comments, "names": names, "scanned": scanned, + "truncated": (len(comments) >= limit or len(names) >= limit + or scanned >= max_scan)} +result +''', + # The findings journal (idatui/journal.py). A netnode blob rides along in + # the .i64, so "what did I work out here" survives closing the database. + "journal_get": r''' +import ida_netnode +n = ida_netnode.netnode(a.get("node", "$ idatui.journal")) +blob = n.getblob(0, "I") if ida_netnode.exist(n) else None +result = {"data": blob.decode("utf-8", "replace") if blob else ""} +result +''', + "journal_put": r''' +import ida_netnode +n = ida_netnode.netnode(a.get("node", "$ idatui.journal"), 0, True) +payload = (a.get("data") or "").encode("utf-8") +n.setblob(payload, 0, "I") +result = {"ok": True, "bytes": len(payload)} +result +''', "list_linkage": r''' imports = [{"addr": hex(int(item.address)), "name": item.name, "module": item.module_name} for item in db.imports.get_all_imports() if item.name] diff --git a/idatui/domain.py b/idatui/domain.py index 6fe73c2..15f4ef4 100644 --- a/idatui/domain.py +++ b/idatui/domain.py @@ -252,6 +252,37 @@ def link_name(raw: str) -> str: @dataclass(frozen=True) +class Comment: + """One comment somebody wrote into the database. + + ``line`` is the disassembly the comment is attached to, carried along so a + report can show what was being commented ON without a second round trip. + ``whole_func`` marks a function comment rather than an instruction one. + """ + addr: int + text: str + repeatable: bool = False + whole_func: bool = False + line: str = "" + seg: str = "" + func: str | None = None + func_addr: int | None = None + + +@dataclass(frozen=True) +class NamedItem: + """An address carrying a real name -- one you typed, or one the file's own + symbols supplied. IDA records both as "user" names and does not remember + which was which, so a report must say so rather than claim authorship.""" + addr: int + name: str + is_func: bool = False + size: int = 0 + proto: str | None = None + seg: str = "" + + +@dataclass(frozen=True) class Linkage: """One import or export: a name this binary takes from, or offers to, other modules. ``module`` is set for imports (the library IDA attributes it to), @@ -1903,6 +1934,44 @@ class Program: self._linkage = out return out + def annotations(self, limit: int = 4000) -> tuple[list["Comment"], list["NamedItem"]]: + """``(comments, names)`` -- everything a person added to this database. + + Not cached: it is the *current* state of your work, and the one caller + (the findings export) asks for it once. ``([], [])`` if the backend has + no such operation, so an alternate client degrades instead of breaking. + """ + try: + payload = self.client.invoke("list_annotations", limit=int(limit)) + except IDAToolError: + return ([], []) + if not isinstance(payload, dict): + return ([], []) + comments = [ + Comment(addr=_as_int(r.get("addr", 0)), text=str(r.get("text", "")), + repeatable=bool(r.get("repeatable")), + whole_func=bool(r.get("whole_func")), + line=str(r.get("line", "") or ""), + seg=str(r.get("seg", "") or ""), + func=(r.get("func") or None), + func_addr=(_as_int(r["func_addr"]) if r.get("func_addr") else None)) + for r in payload.get("comments", []) if isinstance(r, dict) and r.get("text")] + names = [ + NamedItem(addr=_as_int(r.get("addr", 0)), name=str(r.get("name", "")), + is_func=bool(r.get("func")), size=int(r.get("size", 0) or 0), + proto=(r.get("proto") or None), seg=str(r.get("seg", "") or "")) + for r in payload.get("names", []) if isinstance(r, dict) and r.get("name")] + return (comments, names) + + + def journal_get(self) -> str: + """The findings journal blob stored in this database ('' if none).""" + payload = self.client.invoke("journal_get") + return str(payload.get("data", "")) if isinstance(payload, dict) else "" + + def journal_put(self, data: str) -> None: + self.client.invoke("journal_put", data=str(data)) + def decomp_map(self, ea: int) -> list[list[int]]: """Per-pseudocode-line instruction coverage for the split-view region highlight: a list aligned to the decompiled lines, each the EAs the diff --git a/idatui/drive.py b/idatui/drive.py index 825111c..6ba13c6 100644 --- a/idatui/drive.py +++ b/idatui/drive.py @@ -296,6 +296,14 @@ def cmd_save(c, args): return " saved" +def cmd_export(c, args): + """export [path] -- write the session's findings as markdown.""" + r = c.call("export", **({"path": args[0]} if args else {})) + return (f" {r.get('path')} ({r.get('bytes', 0)} bytes: " + f"{r.get('comments', 0)} comments, {r.get('names', 0)} names, " + f"{r.get('types', 0)} types)") + + def cmd_screen(c, args): return c.call("screen").get("text", "") @@ -315,7 +323,7 @@ COMMANDS = { "callees": cmd_callees, "callers": cmd_callers, "names": cmd_names, "rename": cmd_rename, "mv": cmd_mv, "note": cmd_note, "retype": cmd_retype, "save": cmd_save, "screen": cmd_screen, "raw": cmd_raw, "define": cmd_define, - "syms": cmd_syms, "fmt": cmd_fmt, + "syms": cmd_syms, "fmt": cmd_fmt, "export": cmd_export, "binaries": cmd_binaries, "switch": cmd_switch, } diff --git a/idatui/edit_ctl.py b/idatui/edit_ctl.py index 80e1109..52566ca 100644 --- a/idatui/edit_ctl.py +++ b/idatui/edit_ctl.py @@ -266,6 +266,7 @@ class EditController: if kind == "func" and addr is not None: self._rename_index(addr, new) app._dirty = True + app.journal.record("rename", addr, f"{old} → {new}", {"kind": kind}) app._status(f"renamed {old} → {new} (Ctrl+S to save)") def do_name_addr(self, addr: int, name: str) -> None: # worker context @@ -317,6 +318,7 @@ class EditController: self._rename_index(addr, name) app._open_at(addr, label, idx, False, -1, 0, True) app._dirty = True + app.journal.record("rename", addr, f"→ {name}") app._status(f"named {addr:#x} → {name} (Ctrl+S to save)") # -- comments (IDA ';') ------------------------------------------------- # @@ -382,6 +384,7 @@ class EditController: app.program.bump_names() self.reload_active_code() app._dirty = True + app.journal.record("comment" if text else "uncomment", ea, text) verb = "cleared comment" if not text else "commented" app._status(f"{verb} @ {ea:#x} (Ctrl+S to save)") @@ -470,6 +473,8 @@ class EditController: app.program.bump_names() self.reload_active_code() app._dirty = True + app.journal.record("retype", getattr(app._cur, "ea", None), word, + {"kind": kind}) what = "prototype" if kind == "func" else f"'{word}'" app._status(f"retyped {what} (Ctrl+S to save)") diff --git a/idatui/findings.py b/idatui/findings.py new file mode 100644 index 0000000..58a69f6 --- /dev/null +++ b/idatui/findings.py @@ -0,0 +1,384 @@ +"""Export a reverse-engineering session as a markdown report. + +The output of an RE session is not the database, it is what you *learned* -- +and that lives scattered across comments, names and types inside a `.i64` that +only IDA can read. This turns it into one document you can paste into an +advisory, a writeup or a ticket. + +Two halves, deliberately separated: + +* :func:`gather` talks to a :class:`~idatui.domain.Program` (the only part that + needs IDA) and returns a plain :class:`Findings`. +* :func:`render` turns a :class:`Findings` into markdown and knows nothing about + IDA, so the formatting -- grouping, sorting, escaping, the empty cases -- is + tested offline in ``tests/test_findings.py``. + +**On authorship.** IDA records "this address has a real name" but not *who* +named it, so a stripped binary's report is exactly your renames while a binary +with symbols also lists the ones it shipped with. The report says which case it +is rather than claiming credit; comments and types have no such ambiguity. +""" + +from __future__ import annotations + +import os +import re +import time +from dataclasses import dataclass, field + + +@dataclass +class Findings: + """Everything the report can show, already fetched. Plain data on purpose.""" + + binary: str = "" + path: str = "" + #: (start, end, name) segments, for the overview + sections: list[tuple[int, int, str]] = field(default_factory=list) + n_functions: int = 0 + #: idatui.domain.Comment + comments: list = field(default_factory=list) + #: idatui.domain.NamedItem + names: list = field(default_factory=list) + #: (idatui.domain.Struct, source or "") + types: list[tuple[object, str]] = field(default_factory=list) + #: names IDA supplied from imports/exports -- excluded from "named", since + #: they are the linker's work, not anyone's finding + linked: set[str] = field(default_factory=set) + stripped: bool = True + truncated: bool = False + #: annotations dropped as the loader's own work, reported as a count + skipped_loader: int = 0 + #: Addresses idatui recorded itself editing (idatui/journal.py). When this + #: is non-empty the report is EXACT -- it is what you did, not what the + #: database happens to contain. Empty means nobody journalled this database + #: (worked on in the IDA GUI, or before this feature), and the report falls + #: back to filtering by shape, which it says out loud. + recorded: set[int] = field(default_factory=set) + #: type names the journal saw declared, for the same reason + recorded_types: set[str] = field(default_factory=set) + n_recorded: int = 0 + generated_at: float = field(default_factory=time.time) + + +#: Segments the *loader* owns rather than the program: the ELF/PE header and +#: friends. IDA annotates those itself -- "File format: \x7FELF", "File class: +#: 64-bit", `elf_gnu_hash_nbuckets` -- through the very same set_cmt/set_name +#: calls a person uses, and the database does not record who called them. So a +#: report that trusted `has_user_name` alone opened with forty lines of ELF +#: header trivia. Anything here is the file describing itself; it is reported as +#: a count, never as a finding. +_LOADER_SEGS = frozenset({"LOAD", "HEADER", "MEMORY", "UNDEF", "abs", "extern"}) + +#: Same idea for names the loader derives from format structures. +_LOADER_NAME = re.compile(r"^(?:elf|pe|macho|coff|dos)_", re.I) + + +def from_loader(seg: str, name: str = "") -> bool: + """True if this annotation is the file format describing itself.""" + return (seg or "") in _LOADER_SEGS or bool(_LOADER_NAME.match(name or "")) + + +#: IDA's *analyzer* also writes comments, with `set_cmt`, and the database keeps +#: no record that they are its own -- verified: `get_cmt` at a switch returns +#: "switch jump" with exactly the flags a hand-written comment has. These are +#: its stereotyped shapes, which no one types by accident. +_ANALYZER = re.compile( + r"^(?:switch \d+ cases?|switch jump|jumptable [0-9A-Fa-f]+\b.*|" + r"indirect table for switch.*|jump table for switch.*)$", re.I) + +#: The other family is argument hints (`s1`, `locale`, `domainname`), which IDA +#: copies from the callee's prototype onto each argument-setup instruction. They +#: have no distinguishing shape -- but they REPEAT, once per call site, while a +#: note you wrote is yours alone. Three occurrences of one whitespace-free text +#: is the threshold; anything filtered is counted in the report, never dropped +#: in silence. +_HINT_REPEATS = 3 + + +def analyzer_texts(comments) -> set[str]: + """The comment texts in ``comments`` that look like IDA's own work.""" + counts: dict[str, int] = {} + for c in comments: + text = (c.text or "").strip() + if text and not text.split()[1:]: # a single whitespace-free token + counts[text] = counts.get(text, 0) + 1 + out = {t for t, n in counts.items() if n >= _HINT_REPEATS} + out |= {(c.text or "").strip() for c in comments + if _ANALYZER.match((c.text or "").strip())} + return out + + +#: Names IDA invents when nobody has said otherwise. An address carrying one of +#: these has not been understood by anybody, so it is not a finding. +_DUMMY = re.compile( + r"^(?:(?:sub|loc|locret|off|seg|asc|byte|word|dword|qword|xmmword|ymmword|" + r"flt|dbl|tbyte|stru|algn|unk|nullsub|def|jpt|jsub)_[0-9A-Fa-f]+" + # j_strlen: a thunk name IDA derives from its target, not from a person. + r"|j_\w+)$") + + +def is_dummy(name: str) -> bool: + """True for an IDA-generated placeholder name (``sub_1234``, ``loc_A0``…).""" + return bool(_DUMMY.match(name or "")) + + +def gather(program, path: str = "", *, limit: int = 4000, + types: bool = True, journal=None) -> Findings: + """Collect a :class:`Findings` from a live :class:`Program`. + + ``path`` is the binary the app opened -- ``Program`` speaks to a database + and does not know the file name the user would recognise. + + Every step is individually guarded: a report that is missing its types + section is worth far more than an exception at the end of a long session. + """ + out = Findings() + out.path = path or "" + out.binary = os.path.basename(out.path) if out.path else "" + if journal is not None: + try: + out.recorded = journal.addresses() + out.recorded_types = {e.get("d", "") for e in journal.entries + if e.get("k") == "type" and e.get("d")} + out.n_recorded = len(journal) + except Exception: # noqa: BLE001 + out.recorded, out.recorded_types, out.n_recorded = set(), set(), 0 + try: + out.sections = list(program.sections()) + except Exception: # noqa: BLE001 + out.sections = [] + try: + comments, names = program.annotations(limit=limit) + except Exception: # noqa: BLE001 + comments, names = [], [] + out.comments, out.names = list(comments), list(names) + try: + imports, exports = program.linkage() + out.linked = {i.name for i in imports} | {e.name for e in exports} + except Exception: # noqa: BLE001 + out.linked = set() + try: + idx = program.functions() + idx.load_all() + out.n_functions = len(idx) + # "Stripped" is a judgement about the report, not about the ELF: if + # almost every function is still sub_XXXX, a real name IS a finding. + named = sum(1 for f in idx.all_loaded() if not is_dummy(f.name)) + out.stripped = named <= max(4, out.n_functions // 20) + except Exception: # noqa: BLE001 + pass + if types: + try: + for st in program.list_structs(): + try: + src = program.struct_source(st.name) + except Exception: # noqa: BLE001 + src = "" + out.types.append((st, src)) + except Exception: # noqa: BLE001 + out.types = [] + return out + + +def _esc(text: str) -> str: + """Make one line safe inside a markdown TABLE cell.""" + return (text or "").replace("|", "\\|").replace("\n", " ").strip() + + +def _fence(text: str) -> str: + """Fence body text so a comment containing backticks cannot break out.""" + ticks = "`" * max(3, max((len(m) for m in re.findall(r"`+", text or "")), + default=0) + 1) + return f"{ticks}\n{(text or '').rstrip()}\n{ticks}" + + +def _user_names(f: Findings) -> list: + """The names worth reporting. + + With a journal, that is exactly the addresses we recorded renaming. Without + one, it is a judgement: a real name, not the linker's, not the loader's. + """ + names = [n for n in f.names + if not is_dummy(n.name) and n.name not in f.linked + and not from_loader(n.seg, n.name)] + if f.recorded: + return [n for n in names if n.addr in f.recorded] + return names + + +def _user_types(f: Findings) -> list: + """The types worth reporting. A database is seeded with the type libraries + IDA loaded, so with a journal we show only the ones declared here; without + one, all of them, newest ordinal first (yours are the newest).""" + if f.recorded or f.recorded_types: + return [t for t in f.types + if getattr(t[0], "name", "") in f.recorded_types] + return list(f.types) + + +def _user_comments(f: Findings) -> list: + """The comments a person wrote: not the loader's, not the analyzer's, and + not the same comment reported twice.""" + auto = analyzer_texts(f.comments) + out, seen = [], set() + for c in f.comments: + text = (c.text or "").strip() + if not text or from_loader(c.seg) or text in auto: + continue + if f.recorded and c.addr not in f.recorded: + continue + # A comment on a function's first instruction comes back BOTH as an + # instruction comment and as the function comment; report it once. + key = (c.addr, text) + if key in seen: + continue + seen.add(key) + out.append(c) + return out + + +def render(f: Findings) -> str: + """Render a :class:`Findings` as a markdown document.""" + when = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(f.generated_at)) + names = sorted(_user_names(f), key=lambda n: n.addr) + funcs = [n for n in names if n.is_func] + data = [n for n in names if not n.is_func] + comments = sorted(_user_comments(f), key=lambda c: (c.func_addr or c.addr, + c.addr)) + dropped = (len(f.comments) - len(comments)) + (len(f.names) - len(names)) + types = _user_types(f) + + L: list[str] = [] + title = f.binary or "database" + L.append(f"# Findings — {title}") + L.append("") + L.append(f"*{len(funcs)} named functions · {len(data)} named data · " + f"{len(comments)} comments · {len(types)} local types — " + f"exported {when} by idatui*") + L.append("") + if f.path: + L.append(f"- **binary**: `{f.path}`") + if f.n_functions: + L.append(f"- **functions**: {f.n_functions}") + if f.sections: + segs = ", ".join(f"`{nm}` {s:#x}–{e:#x}" for s, e, nm in f.sections[:8]) + more = f" (+{len(f.sections) - 8} more)" if len(f.sections) > 8 else "" + L.append(f"- **segments**: {segs}{more}") + if f.recorded or f.recorded_types: + n_at = len(f.recorded) + L.append(f"- **source**: idatui's edit journal — {f.n_recorded} recorded " + f"edits across {n_at} address{'' if n_at == 1 else 'es'}. " + "Everything below is work done here, not the analyzer's.") + else: + L.append("- **source**: a scan of the database. Nothing in a `.i64` " + "records *who* wrote a comment or a name — IDA's own analyzer " + "uses the same calls — so this is filtered by shape and may " + "include its work as well as yours.") + if not f.stripped: + L.append("- **note**: this binary has its own symbols, so the names " + "below include ones it shipped with.") + if dropped and (f.recorded or f.recorded_types): + L.append(f"- **note**: {dropped} other annotations in this database " + "were not made here (the analyzer's, the loader's, the " + "linker's) and are left out.") + elif dropped: + L.append(f"- **note**: {dropped} annotations left out as the loader's " + "own (file headers, dummy names, imports).") + if f.truncated: + L.append("- **note**: the scan hit its limit; this report is partial.") + L.append("") + + # -- comments: the actual reasoning, so they lead ----------------------- # + L.append("## Comments") + L.append("") + if not comments: + L.append("*None. (Comments are the part of a database nobody else can " + "reconstruct — they are worth writing.)*") + L.append("") + else: + by_func: dict[str, list] = {} + for c in comments: + by_func.setdefault(c.func or "", []).append(c) + for fn in sorted(by_func, key=lambda k: (k == "", k)): + rows = by_func[fn] + head = f"### `{fn}`" if fn else "### outside any function" + if fn and rows[0].func_addr is not None: + head += f" ({rows[0].func_addr:#x})" + L.append(head) + L.append("") + for c in rows: + if c.whole_func: + L.append(f"- **{c.addr:#x}** — *whole function*: " + f"{_esc(c.text)}") + elif c.line: + L.append(f"- **{c.addr:#x}** `{_esc(c.line)}` \n" + f" {_esc(c.text)}") + else: + L.append(f"- **{c.addr:#x}** — {_esc(c.text)}") + L.append("") + + # -- names -------------------------------------------------------------- # + L.append("## Named functions") + L.append("") + if not funcs: + L.append("*None.*") + L.append("") + else: + L.append("| address | name | size | prototype |") + L.append("|---|---|---|---|") + for n in funcs: + proto = f"`{_esc(n.proto)}`" if n.proto else "" + L.append(f"| `{n.addr:#x}` | `{_esc(n.name)}` | " + f"{n.size:#x} | {proto} |") + L.append("") + if data: + L.append("## Named data") + L.append("") + L.append("| address | name | segment |") + L.append("|---|---|---|") + for n in data: + L.append(f"| `{n.addr:#x}` | `{_esc(n.name)}` | {_esc(n.seg)} |") + L.append("") + + # -- types -------------------------------------------------------------- # + if types: + L.append("## Local types") + L.append("") + if not (f.recorded or f.recorded_types): + L.append("*Newest first. A database is seeded with types from the " + "libraries IDA loaded, so the ones you defined are the " + "ones with the highest ordinals — at the top of this " + "list.*") + L.append("") + ordered = sorted(types, key=lambda t: -getattr(t[0], "ordinal", 0)) + for st, src in ordered: + kw = "union" if getattr(st, "is_union", False) else "struct" + L.append(f"### `{kw} {st.name}` " + f"({getattr(st, 'size', 0):#x} bytes, " + f"{getattr(st, 'members', 0)} fields)") + L.append("") + if src: + L.append("```c") + L.append(src.rstrip()) + L.append("```") + L.append("") + return "\n".join(L).rstrip() + "\n" + + +def default_path(program_path: str) -> str: + """Where a report lands if nobody says otherwise: beside the binary.""" + base = program_path or "findings" + return f"{base}.findings.md" + + +def export(program, binary_path: str = "", out_path: str | None = None, *, + limit: int = 4000, types: bool = True, + journal=None) -> tuple[str, Findings]: + """Gather, render and WRITE the report. Returns ``(path, findings)``.""" + f = gather(program, binary_path, limit=limit, types=types, journal=journal) + out = out_path or default_path(f.path) + out = os.path.abspath(os.path.expanduser(out)) + with open(out, "w", encoding="utf-8") as fh: + fh.write(render(f)) + return out, f diff --git a/idatui/journal.py b/idatui/journal.py new file mode 100644 index 0000000..a5a07de --- /dev/null +++ b/idatui/journal.py @@ -0,0 +1,107 @@ +"""A record of the edits idatui makes, kept inside the database. + +**Why this has to exist.** A findings report wants to say "here is what *you* +worked out", and the database cannot answer that. IDA's own analyzer writes +comments with the same `set_cmt` a person uses (`; s1` on an argument setup, +`; switch 73 cases` on a jump table), and the loader writes both comments and +names for the file header. Four separate probes agree that nothing tells them +apart: the `FF_COMM` flag is identical, `get_cmt` returns them all, the colour +tag in `generate_disasm_line` is `COLOR_REGCMT` for every one of them, and they +survive with auto-comments switched off. So authorship is not recoverable after +the fact -- it has to be recorded as it happens, which is what this does. + +It lives in an IDA **netnode**, so it is saved into the `.i64` with everything +else and is still there next session. The entries are small and additive; the +journal is metadata *about* edits that themselves live in the database, so +losing it degrades the report to a heuristic rather than losing work. +""" + +from __future__ import annotations + +import json +import threading +import time + +#: Where the blob lives inside the database. +NODE = "$ idatui.journal" + +#: Cap: a long session is hundreds of edits, not hundreds of thousands, and the +#: blob is rewritten whole. Oldest entries fall off first. +MAX_ENTRIES = 20000 + + +class Journal: + """Append-only log of what was edited, with lazy load and explicit flush. + + Writing through to the database on every keystroke-sized edit would put a + round trip in the way of the user; the in-memory list is authoritative + during a session and :meth:`flush` persists it at the points that already + mean "keep this": saving, exporting, and quitting. + """ + + def __init__(self) -> None: + self.entries: list[dict] = [] + self._dirty = False + self._loaded = False + self._lock = threading.Lock() + + # -- recording ---------------------------------------------------------- # + def record(self, kind: str, ea: int | None = None, detail: str = "", + extra: dict | None = None) -> None: + """Note one edit: ``kind`` is 'rename' / 'comment' / 'retype' / …""" + entry = {"k": str(kind), "t": int(time.time())} + if ea is not None: + entry["ea"] = int(ea) + if detail: + entry["d"] = str(detail)[:400] + if extra: + entry.update(extra) + with self._lock: + self.entries.append(entry) + if len(self.entries) > MAX_ENTRIES: + del self.entries[:len(self.entries) - MAX_ENTRIES] + self._dirty = True + + def addresses(self, kinds: tuple[str, ...] | None = None) -> set[int]: + """Every address touched (optionally only by certain kinds of edit).""" + with self._lock: + return {e["ea"] for e in self.entries + if "ea" in e and (kinds is None or e.get("k") in kinds)} + + def __len__(self) -> int: + return len(self.entries) + + # -- persistence -------------------------------------------------------- # + def load(self, program) -> None: + """Read the journal out of the database, once. Never raises.""" + if self._loaded: + return + self._loaded = True + try: + raw = program.journal_get() + except Exception: # noqa: BLE001 -- an old database simply has none + return + if not raw: + return + try: + data = json.loads(raw) + except Exception: # noqa: BLE001 + return + if isinstance(data, list): + with self._lock: + # Prepend: what is already in memory happened later. + self.entries = [e for e in data if isinstance(e, dict)] + self.entries + + def flush(self, program) -> bool: + """Write the journal back if it changed. Returns whether it wrote.""" + with self._lock: + if not self._dirty: + return False + payload = json.dumps(self.entries, separators=(",", ":")) + try: + program.journal_put(payload) + except Exception: # noqa: BLE001 -- never let bookkeeping break an edit + return False + with self._lock: + self._dirty = False + return True diff --git a/idatui/rpc.py b/idatui/rpc.py index 2a63812..98ff90f 100644 --- a/idatui/rpc.py +++ b/idatui/rpc.py @@ -39,7 +39,7 @@ _PROGRAM_METHODS = { "goto", "open", "rename", "comment", "retype", "follow", "xrefs", "symbols", "structs", "search", "select", "save", "hex", "toggle_view", "pseudocode", "disassembly", "xrefs_to", "xrefs_from", "resolve", - "define", "rename_many", "opfmt", "graph", + "define", "rename_many", "opfmt", "graph", "export", } # Self-documenting method table (returned by the 'methods' verb). @@ -75,6 +75,8 @@ METHODS = { "xrefs": "open the xref picker", "symbols": "{query?} open the symbol palette", "structs": "open the struct editor", + "export": "{path?,types?=true} write the session's comments/names/types as " + "a markdown report -> {path,comments,names,types}", "search": "{term,direction?=1} incremental search in the code view", "select": "{index?} choose the highlighted/nth item in the open modal", "save": "persist the .i64 (Ctrl+S)", @@ -1173,6 +1175,24 @@ class RpcServer: return await self._press( ["ctrl+t"], lambda: type(app.screen).__name__ == "StructEditor", timeout, "structs") + if method == "export": + # Deliberately NOT driven through the prompt: this is the one verb + # whose whole point is the file it leaves behind, and a driver needs + # the path back, not a screenshot of a prompt closing. + from . import findings + path = params.get("path") + app.journal.load(app.program) + app.journal.flush(app.program) + out, f = await asyncio.to_thread( + findings.export, app.program, app._open_path or "", + str(path) if path else None, + types=bool(params.get("types", True)), journal=app.journal) + app._status(f"exported findings → {out}", priority=True) + await drain(app) + return {"path": out, "comments": len(f.comments), + "names": len(findings._user_names(f)), + "types": len(f.types), "functions": f.n_functions, + "bytes": os.path.getsize(out) if os.path.exists(out) else 0} if method == "close": return await self._press(["escape"], timeout=timeout) if method == "save": |