From f3715d8de0d255c8b14710acfa120ccb9ea953fd Mon Sep 17 00:00:00 2001 From: Duncan Ogilvie Date: Thu, 20 Aug 2026 23:42:42 +0200 Subject: Adopt idb_events and remote module features from ida-codemode --- idatui/codemode_client.py | 1371 +++++++-------------------------------------- 1 file changed, 214 insertions(+), 1157 deletions(-) (limited to 'idatui/codemode_client.py') diff --git a/idatui/codemode_client.py b/idatui/codemode_client.py index 8eb66b5..e95238f 100644 --- a/idatui/codemode_client.py +++ b/idatui/codemode_client.py @@ -5,24 +5,20 @@ GUI database, reuses a shared managed idalib worker, or starts one when needed. The TUI never owns or terminates an IDA process. Closing this client releases only its lease. -The Code Mode transport intentionally exposes one broad operation, -``execute_python``. ``CodeModeClient.invoke`` turns the small, address-centric -operations needed by the paging layer into self-contained snippets. The -snippets prefer the public ``ida-domain`` ``db`` object. A handful of features -that ida-domain does not currently expose (IDA-coloured listing rows, creating -instructions, ARM T-state, and detailed Hex-Rays line maps/failures) use the -IDAPython modules that Code Mode deliberately makes importable. +Remote operations are ordinary typed Python functions declared in +``idatui.remote_ops``. Code Mode installs their content-addressed modules once +per handle; subsequent calls send only encoded arguments. The optimized +IDAPython listing/decompiler implementation remains real source in +``idatui.remote_tools`` and is installed through the same module interface. """ + from __future__ import annotations -import hashlib -import json import os import shlex import threading import time -from pathlib import Path -from textwrap import dedent +from collections.abc import Callable from typing import Any from .errors import IDAConnectionError, IDATimeoutError, IDAToolError, Session @@ -68,7 +64,8 @@ def _require_codemode() -> None: "ida-codemode is not installed in this environment " f"({_CODEMODE_ERROR}). Install it (e.g. `uv sync`, or " "`pip install ida-codemode`) so ida-tui can lease a " - "database.") from _CODEMODE_ERROR + "database." + ) from _CODEMODE_ERROR def database_owner(idb_path: str, staged_path: str | None = None): @@ -155,1106 +152,119 @@ def _parse_load_args(value: str) -> tuple[str | None, int | None, str | None]: return processor, loading_address, file_type -#: Key of the pre-serialised payload envelope. See _script(). -_PACKED = "__idatui_json__" - -#: Serialise the answer INSIDE the database process and hand back one string. -#: -#: Written when Code Mode ran to_jsonable() over every snippet result, walking -#: the whole structure in Python to make it JSON-safe: a 200-row listing page is -#: ~10k small objects, which cost 66ms to walk -- 72% of the page's total cost, -#: and 114x what json.dumps of the same data cost (0.58ms). -#: -#: ida-codemode 0.3.2 removed that reason: serialization.dumps_json now hands -#: the structure straight to the C encoder and only falls back to the walker for -#: values json.dumps rejects. Re-measured against 0.3.2, packing buys 0.97x on -#: that same page (experiments/bench_pack_trace.py) -- i.e. nothing, because the -#: dodged walk is replaced by a double encode. -#: -#: It is kept anyway, on correctness rather than speed: packing pins OUR encoder -#: settings (compact separators, default=str) inside the database process, so an -#: un-encodable IDA object degrades to repr() at a point we control instead of -#: depending on the runtime's fallback. Delete it if that stops being worth a -#: protocol step -- it is no longer load-bearing for performance. -_PACK_EPILOGUE = ( - '\n{"' + _PACKED + '": json.dumps(result, separators=(",", ":"), default=str)}\n' -) - +class IDBEventListener: + """Debounced, closeable delivery of another client's IDB changes. -def _script(args: dict[str, Any], body: str) -> str: - """Bind JSON arguments without interpolating user text into Python code. - - This used to also run the body with Code Mode's trace hook detached - (sys.settrace(None) + restore), because the runtime wrapped every - execute_python in a trace function that returned ITSELF -- enabling line - tracing in every frame it saw, so every line of every function we called - paid a Python-level callback (ida_bytes.get_flags: 0.106us -> 5.49us, 52x). - - ida-codemode 0.3.2 deleted that hook; cancellation is now a C-level thread - interrupt (runtime._interrupt_thread) that costs nothing while idle. The - workaround measured 0.99x on a 200-row listing page against 0.3.2 -- pure - noise -- so it is gone, and with it the caveat that a pure-Python loop in a - snippet escaped its deadline. See experiments/bench_pack_trace.py. + Code Mode's subscription is a blocking iterator, so one daemon thread reads + it and a second waits for a quiet period before handing a batch to the UI. + Keeping the debounce here avoids a permanent Textual worker (which would + make the app's worker-idle contract impossible) and bounds refresh work to + one pass per edit burst. """ - encoded = json.dumps(args, ensure_ascii=False, separators=(",", ":")) - head = f"import json\na = json.loads({encoded!r})\n" - return f"{head}{dedent(body).strip()}\n{_PACK_EPILOGUE}" - -_OPERATIONS: dict[str, str] = { - "list_funcs": r''' -import fnmatch -queries = a.get("queries") or [{}] -q = queries[0] -offset, count = max(0, int(q.get("offset", 0))), max(1, int(q.get("count", 500))) -pattern = str(q.get("filter") or "").lower() -if pattern and not any(ch in pattern for ch in "*?["): pattern = "*" + pattern + "*" -rows = [] -for fn in db.functions.get_all(): - name = db.functions.get_name(fn) or f"sub_{int(fn.start_ea):X}" - if pattern and not fnmatch.fnmatchcase(name.lower(), pattern): continue - rows.append({"addr": hex(int(fn.start_ea)), "name": name, - "size": int(fn.end_ea) - int(fn.start_ea)}) -page = rows[offset:offset + count] -result = {"result": [{"data": page, "next_offset": offset + len(page), "total": len(rows)}]} -result -''', - "disasm": r''' -ea = int(str(a["addr"]), 16) -fn = db.functions.get_at(ea) -if fn is None: - result = {"instructions": [], "total_instructions": 0, "instruction_count": 0} -else: - instructions = list(db.functions.get_instructions(fn)) - limit = max(1, int(a.get("max_instructions", len(instructions) or 1))) - rows = [{"addr": hex(int(insn.ea)), "instruction": db.instructions.get_disassembly(insn)} - for insn in instructions[:limit]] - result = {"instructions": rows, "total_instructions": len(instructions), - "instruction_count": len(instructions)} -result -''', - "file_regions": r''' -import idaapi -rows = [] -for seg in db.segments.get_all(): - try: file_off = int(idaapi.get_fileregion_offset(seg.start_ea)) - except Exception: file_off = -1 - if file_off < 0 or file_off >= (1 << 48): file_off = -1 - rows.append({"start": hex(int(seg.start_ea)), "end": hex(int(seg.end_ea)), - "file_off": file_off, "name": db.segments.get_name(seg) or ""}) -result = {"regions": rows} -result -''', - "read_raw": r''' -import ida_bytes -ea, size = int(str(a["addr"]), 16), max(0, int(a["size"])) -raw = ida_bytes.get_bytes(ea, size) or b"" -raw = raw[:size] + b"\xff" * max(0, size - len(raw)) -data = bytearray(raw) -for index, value in enumerate(data): - if value == 0xFF and not ida_bytes.is_loaded(ea + index): data[index] = 0 -result = {"addr": a["addr"], "hex": bytes(data).hex(), "n": len(data)} -result -''', - "get_bytes": r''' -rows = [] -for region in a.get("regions", []): - ea, size = int(str(region["addr"]), 16), int(region["size"]) - raw = db.bytes.get_bytes_at(ea, size) or b"" - rows.append({"addr": region["addr"], "data": " ".join(f"{b:02x}" for b in raw)}) -result = {"result": rows} -result -''', - "search_structs": r''' -needle = str(a.get("filter") or "").lower() -rows = [] -for tif in db.types.get_all(): - name = tif.get_type_name() or "" - if not name or needle not in name.lower() or not tif.is_udt(): continue - members = list(db.types.get_udt_members(tif)) - rows.append({"name": name, "size": int(tif.get_size()), "is_union": bool(tif.is_union()), - "cardinality": len(members), "ordinal": int(tif.get_ordinal())}) -result = {"result": rows} -result -''', - "type_inspect": r''' -rows = [] -for query in a.get("queries", []): - name = str(query.get("name") or "") - tif = db.types.get_by_name(name) - if tif is None: - rows.append({"name": name, "error": "type not found"}); continue - members = [{"name": m.name, "type": m.type.dstr() or str(m.type), - "offset": int(m.offset), "size": int(m.size)} - for m in db.types.get_udt_members(tif)] if tif.is_udt() else [] - rows.append({"name": name, "size": int(tif.get_size()), "is_union": bool(tif.is_union()), - "members": members}) -result = {"result": rows} -result -''', - "declare_type": r''' -import ida_typeinf -decls = a.get("decls", "") -if isinstance(decls, str): decls = [decls] -rows = [] -for declaration in decls: - try: - errors = int(db.types.parse_declarations(ida_typeinf.get_idati(), declaration)) - rows.append({"ok": errors == 0, **({} if errors == 0 else {"error": f"{errors} parse error(s)"})}) - except Exception as exc: - rows.append({"ok": False, "error": str(exc)}) -result = {"result": rows} -result -''', - "del_type": r''' -import ida_typeinf -name = str(a["name"]) -ok = bool(ida_typeinf.del_named_type(ida_typeinf.get_idati(), name, ida_typeinf.NTF_TYPE)) -result = {"name": name, "deleted": ok, **({} if ok else {"error": f"Type {name!r} not found or could not be deleted"})} -result -''', - "func_types": r''' -import ida_typeinf -ea = int(str(a["addr"]), 16) -fn = db.functions.get_at(ea) -if fn is None: - result = {"addr": a["addr"], "error": "no function at address"} -else: - pseudo = db.pseudocode.decompile(fn) - name = db.functions.get_name(fn) or "" - tif = pseudo.get_func_type() - try: prototype = ida_typeinf.print_tinfo("", 0, 0, ida_typeinf.PRTYPE_1LINE, tif, name, "") if tif else "" - except Exception: prototype = tif.dstr() if tif else "" - lvars = [{"name": var.name, "type": var.type_info.dstr() if var.type_info else "", - "is_arg": bool(var.is_arg)} for var in pseudo.local_variables] - result = {"addr": hex(int(fn.start_ea)), "name": name, - "prototype": (prototype or "").strip(), "lvars": lvars} -result -''', - "set_lvar_type": r''' -import ida_typeinf -ea, variable, declaration = int(str(a["addr"]), 16), str(a["variable"]), str(a["type"]) -fn = db.functions.get_at(ea) -if fn is None: - result = {"error": "no function at address"} -else: - pseudo = db.pseudocode.decompile(fn) - var = pseudo.find_local_variable(variable) - if var is None: - result = {"error": f"local variable {variable!r} not found"} - else: + def __init__( + self, + client: "CodeModeClient", + callback: Callable[[tuple[dict[str, Any], ...]], None], + *, + on_error: Callable[[BaseException], None] | None = None, + debounce: float = 0.2, + ) -> None: + self._client = client + self._callback = callback + self._on_error = on_error + self._debounce = max(float(debounce), 0.0) + self._condition = threading.Condition() + self._closed = False + self._subscription = None + self._pending: list[dict[str, Any]] = [] + self._deadline = 0.0 + self._reader = threading.Thread( + target=self._read, name="idatui-idb-events", daemon=True + ) + self._deliverer = threading.Thread( + target=self._deliver, name="idatui-idb-refresh", daemon=True + ) + self._deliverer.start() + self._reader.start() + + def _report(self, error: BaseException) -> None: + disconnected = DatabaseDisconnectedError + if isinstance(disconnected, type) and isinstance(error, disconnected): + error = self._client._connection_error(error) + with self._condition: + closed = self._closed + if not closed and self._on_error is not None: + self._on_error(error) + + def _read(self) -> None: try: - tif = db.types.parse_one_declaration(ida_typeinf.get_idati(), declaration) - accepted = bool(var.set_type(tif)) - saved = bool(pseudo.save_local_variable_info(var, save_type=True)) if accepted else False - result = {"addr": hex(int(fn.start_ea)), "variable": variable, - "type": declaration, "ok": accepted and saved} - except Exception as exc: - result = {"error": f"bad type {declaration!r}: {exc}"} -result -''', - "set_type": r''' -from ida_domain.types import TypeApplyFlags -rows = [] -for edit in a.get("edits", []): - ea = int(str(edit["addr"]), 16) - declaration = str(edit.get("signature") or edit.get("type") or "") - try: - ok = bool(db.types.apply_declaration_at(ea, declaration, TypeApplyFlags.DEFINITE)) - rows.append({"addr": hex(ea), "ok": ok, **({} if ok else {"error": "IDA rejected the type"})}) - except Exception as exc: - rows.append({"addr": hex(ea), "ok": False, "error": str(exc)}) -result = {"result": rows} -result -''', - "data_type": r''' -ea = int(str(a["addr"]), 16) -try: - tif = db.types.get_at(ea) - fn = db.functions.get_at(ea) - result = {"addr": hex(ea), "name": db.names.get_at(ea) or "", - "type": tif.dstr() if tif else "", "size": int(db.heads.size(ea)) if db.heads.is_head(ea) else 0, - "is_func": bool(fn)} -except Exception as exc: - result = {"addr": hex(ea), "error": str(exc)} -result -''', - "force_recompile": r''' -import ida_hexrays -rows = [] -for item in a.get("items", []): - ea = int(str(item["addr"]), 16) - ida_hexrays.mark_cfunc_dirty(ea, False) - rows.append({"addr": hex(ea), "ok": True}) -result = {"result": rows} -result -''', - "undefine": r''' -import ida_bytes -rows = [] -for item in a.get("items", []): - ea = int(str(item["addr"]), 16) - size = max(1, int(item.get("size") or ida_bytes.get_item_size(ea) or 1)) - ok = bool(ida_bytes.del_items(ea, ida_bytes.DELIT_SIMPLE, size)) - rows.append({"addr": hex(ea), "ok": ok, **({} if ok else {"error": "delete items failed"})}) -result = {"result": rows} -result -''', - "define_code": r''' -import ida_ua -rows = [] -for item in a.get("items", []): - ea = int(str(item["addr"]), 16); size = int(ida_ua.create_insn(ea)) - rows.append({"addr": hex(ea), "ok": size > 0, "size": size, - **({} if size > 0 else {"error": "instruction did not decode"})}) -result = {"result": rows} -result -''', - "define_func": r''' -rows = [] -for item in a.get("items", []): - ea = int(str(item["addr"]), 16); ok = bool(db.functions.create(ea)) - rows.append({"addr": hex(ea), "ok": ok, **({} if ok else {"error": "IDA refused the function"})}) -result = {"result": rows} -result -''', - "make_data": r''' -import ida_bytes, ida_idaapi, ida_typeinf -from ida_domain.types import TypeApplyFlags -rows = [] -for item in a.get("items", []): - ea, declaration = int(str(item["addr"]), 16), str(item["type"]) - try: - tif = db.types.parse_one_declaration(ida_typeinf.get_idati(), declaration) - size = max(1, int(tif.get_size())) - saved_names = [(addr, name) for addr, name in db.names.get_all() - if ea <= int(addr) < ea + size] - ida_bytes.del_items(ea, ida_bytes.DELIT_EXPAND | ida_bytes.DELIT_DELNAMES, - max(size, int(ida_bytes.get_item_size(ea) or 1))) - created = bool(ida_bytes.create_data(ea, ida_bytes.FF_BYTE, size, ida_idaapi.BADADDR)) - ok = created and bool(db.types.apply_at(tif, ea, TypeApplyFlags.DEFINITE)) - for address, name in saved_names: - db.names.set_name(int(address), name) - if ok and item.get("name"): ok = bool(db.names.set_name(ea, str(item["name"]))) - rows.append({"addr": hex(ea), "ok": ok, "size": size, - **({} if ok else {"error": "IDA rejected the data type"})}) - except Exception as exc: - rows.append({"addr": hex(ea), "ok": False, "error": str(exc)}) -result = {"result": rows} -result -''', - "make_string": r''' -from ida_domain.strings import StringType -ea, length = int(str(a["addr"]), 16), max(0, int(a.get("length", 0))) -kind = {"c": StringType.C, "c16": StringType.C_16, "c32": StringType.C_32, - "pascal": StringType.PASCAL}.get(str(a.get("kind", "c")).lower(), StringType.C) -import ida_bytes -try: - ida_bytes.del_items(ea, ida_bytes.DELIT_SIMPLE, length if length > 0 else 1) -except Exception: - pass -try: - ok = bool(db.bytes.create_string_at(ea, length or None, kind)) - text = db.bytes.get_string_at(ea) or "" if ok else "" - result = {"addr": hex(ea), "ok": ok, "size": int(db.heads.size(ea)) if ok else 0, "text": text} -except Exception as exc: - result = {"addr": hex(ea), "ok": False, "error": str(exc)} -result -''', - "list_strings": r''' -from ida_domain.strings import StringListConfig -offset, count, min_len = max(0, int(a.get("offset", 0))), max(1, int(a.get("count", 2000))), max(1, int(a.get("min_len", 4))) -if offset == 0 or a.get("refresh"): - from ida_domain.strings import StringType - db.strings.rebuild(StringListConfig(string_types=list(StringType), min_len=min_len, - only_ascii_7bit=False)) -items = list(db.strings.get_all()) -page = items[offset:offset + count] -rows = [] -for item in page: - try: text = str(item) - except Exception: text = item.contents.decode("utf-8", "replace") if item.contents else "" - rows.append({"addr": hex(int(item.address)), "text": text, "len": int(item.length), "type": item.type.name}) -result = {"strings": rows, "total": len(items), "next_offset": offset + len(rows)} -result -''', - # Everything a person ADDED to the database: comments, non-dummy names, and - # the prototypes they set. - # - # Names come from IDA's name list, which is already an index -- no scan at - # all. Comments have no index, so they need a walk, and the walk is over - # HEADS: `next_that`'s predicate is a *Python* callback (SWIG calls it with - # one argument, so `f_has_cmt` does not even fit), which would be one call - # per BYTE -- 400 million of them on a big image. `max_scan` bounds it and - # reports `truncated` rather than sitting there. - "list_annotations": r''' -import ida_bytes, ida_funcs, ida_lines, ida_nalt, ida_name -import ida_segment, ida_typeinf, idautils -limit = max(1, int(a.get("limit", 4000))) -max_scan = max(1000, int(a.get("max_scan", 2000000))) -comments, names = [], [] -scanned = 0 - -def _line(ea): - try: - txt = ida_lines.generate_disasm_line(ea, ida_lines.GENDSM_REMOVE_TAGS) - except Exception: - txt = "" - return " ".join((txt or "").split()) - -for ea, nm in idautils.Names(): - if len(names) >= limit: - break - if not nm or not ida_bytes.has_user_name(ida_bytes.get_flags(ea)): - continue - fn = ida_funcs.get_func(ea) - is_fn = fn is not None and int(fn.start_ea) == int(ea) - proto = None - if is_fn: + subscription = self._client.subscribe_idb_events() + except Exception as exc: # noqa: BLE001 -- surfaced through on_error + self._report(exc) + with self._condition: + self._closed = True + self._pending.clear() + self._condition.notify_all() + return + with self._condition: + if self._closed: + subscription.close() + return + self._subscription = subscription try: - ti = ida_typeinf.tinfo_t() - if ida_nalt.get_tinfo(ti, ea): - proto = str(ti) - except Exception: - proto = None - seg = ida_segment.getseg(ea) - names.append({"addr": hex(int(ea)), "name": nm, "func": is_fn, - "size": (int(fn.end_ea - fn.start_ea) if is_fn else 0), - "proto": proto, - "seg": (ida_segment.get_segm_name(seg) if seg else "")}) - -for i in range(ida_segment.get_segm_qty()): - seg = ida_segment.getnseg(i) - if seg is None or len(comments) >= limit or scanned >= max_scan: - continue - for ea in idautils.Heads(seg.start_ea, seg.end_ea): - scanned += 1 - if len(comments) >= limit or scanned >= max_scan: - break - if not ida_bytes.has_cmt(ida_bytes.get_flags(ea)): - continue - for rep in (False, True): - text = ida_bytes.get_cmt(ea, rep) - if text: - fn = ida_funcs.get_func(ea) - comments.append({ - "addr": hex(int(ea)), "text": text, "repeatable": rep, - "line": _line(ea), "seg": ida_segment.get_segm_name(seg), - "func": (ida_funcs.get_func_name(fn.start_ea) if fn else None), - "func_addr": (hex(int(fn.start_ea)) if fn else None)}) - -# Whole-function comments are not on the byte flags, so the scan cannot see them. -for fn_ea in idautils.Functions(): - fn = ida_funcs.get_func(fn_ea) - if fn is None or len(comments) >= limit: - continue - for rep in (False, True): - text = ida_funcs.get_func_cmt(fn, rep) - if text: - seg = ida_segment.getseg(fn_ea) - comments.append({"addr": hex(int(fn_ea)), "text": text, - "repeatable": rep, "line": "", "whole_func": True, - "seg": (ida_segment.get_segm_name(seg) if seg else ""), - "func": ida_funcs.get_func_name(fn_ea), - "func_addr": hex(int(fn_ea))}) -result = {"comments": comments, "names": names, "scanned": scanned, - "truncated": (len(comments) >= limit or len(names) >= limit - or scanned >= max_scan)} -result -''', - # The findings journal (idatui/journal.py). A netnode blob rides along in - # the .i64, so "what did I work out here" survives closing the database. - "journal_get": r''' -import ida_netnode -n = ida_netnode.netnode(a.get("node", "$ idatui.journal")) -blob = n.getblob(0, "I") if ida_netnode.exist(n) else None -result = {"data": blob.decode("utf-8", "replace") if blob else ""} -result -''', - "journal_put": r''' -import ida_netnode -n = ida_netnode.netnode(a.get("node", "$ idatui.journal"), 0, True) -payload = (a.get("data") or "").encode("utf-8") -n.setblob(payload, 0, "I") -result = {"ok": True, "bytes": len(payload)} -result -''', - # Database-wide search (Ctrl+F), two kinds. - # - # BYTES uses IDA's own `find_bytes`, which already understands the pattern - # language people expect -- "B8 ? ? ? ? 90", nibble wildcards ("48 8? ??") - # and quoted literals -- so we neither parse nor match anything ourselves. - # Iterating is match+1, per its documented contract. - "search_bytes": r''' -import ida_bytes, ida_funcs, ida_idaapi, ida_lines, ida_segment -pat = str(a.get("pattern", "")).strip() -limit = max(1, int(a.get("limit", 500))) -lo = int(a.get("start", 0)) -hi = int(a.get("end", 0)) or ida_idaapi.BADADDR -flags = ida_bytes.BIN_SEARCH_FORWARD | ida_bytes.BIN_SEARCH_NOSHOW -if a.get("case"): - flags |= ida_bytes.BIN_SEARCH_CASE -rows, err, ea = [], None, lo -while len(rows) < limit: - try: - hit = ida_bytes.find_bytes(pat, range_start=ea, range_end=hi, flags=flags) - except Exception as exc: - err = str(exc) or exc.__class__.__name__ - break - if hit is None or hit == ida_idaapi.BADADDR: - break - head = ida_bytes.get_item_head(hit) - fn = ida_funcs.get_func(hit) - seg = ida_segment.getseg(hit) - try: - line = ida_lines.generate_disasm_line(head, ida_lines.GENDSM_REMOVE_TAGS) or "" - except Exception: - line = "" - rows.append({"addr": hex(int(hit)), "head": hex(int(head)), - "line": " ".join(line.split()), - "func": (ida_funcs.get_func_name(fn.start_ea) if fn else None), - "func_addr": (hex(int(fn.start_ea)) if fn else None), - "seg": (ida_segment.get_segm_name(seg) if seg else "")}) - ea = int(hit) + 1 -result = {"hits": rows, "error": err, "truncated": len(rows) >= limit} -result -''', - # TEXT walks the listing the way a person reads it: every head's rendered - # disassembly line, which is why it finds "call cs:__isoc99_scanf" and - # "0deadbeefh" alike. Bounded by max_scan, so a 400MB image reports partial - # results instead of stalling. - "search_text": r''' -import ida_lines, ida_funcs, ida_segment, idautils -import re as _re -q = str(a.get("query", "")) -limit = max(1, int(a.get("limit", 500))) -max_scan = max(1000, int(a.get("max_scan", 3000000))) -ci = (not a.get("case")) and q.islower() # smartcase, like the in-view search -rx, err = None, None -if a.get("regex"): - try: - rx = _re.compile(q, _re.I if ci else 0) - except Exception as exc: - err = "bad regex: " + str(exc) -needle = q.lower() if ci else q -rows, scanned = [], 0 -if err is None and q: - for i in range(ida_segment.get_segm_qty()): - seg = ida_segment.getnseg(i) - if seg is None or len(rows) >= limit or scanned >= max_scan: - continue - for ea in idautils.Heads(seg.start_ea, seg.end_ea): - scanned += 1 - if len(rows) >= limit or scanned >= max_scan: - break + for event in subscription: + with self._condition: + if self._closed: + break + if self._client.owns_event(event): + continue + with self._condition: + if self._closed: + break + self._pending.append(event) + self._deadline = time.monotonic() + self._debounce + self._condition.notify_all() + except Exception as exc: # noqa: BLE001 -- stream failures are recoverable + self._report(exc) + finally: + subscription.close() + with self._condition: + if self._subscription is subscription: + self._subscription = None + self._closed = True + self._pending.clear() + self._condition.notify_all() + + def _deliver(self) -> None: + while True: + with self._condition: + while not self._closed and not self._pending: + self._condition.wait() + if self._closed: + return + remaining = self._deadline - time.monotonic() + if remaining > 0: + self._condition.wait(remaining) + continue + batch = tuple(self._pending) + self._pending.clear() try: - line = ida_lines.generate_disasm_line(ea, ida_lines.GENDSM_REMOVE_TAGS) or "" - except Exception: - continue - # Match what the user SEES, not IDA's column padding: nobody types - # "call" + four spaces + "cs:getenv_ptr". - line = " ".join(line.split()) - hay = line.lower() if ci else line - if (rx.search(line) if rx is not None else (needle in hay)): - fn = ida_funcs.get_func(ea) - rows.append({"addr": hex(int(ea)), "head": hex(int(ea)), - "line": line, - "func": (ida_funcs.get_func_name(fn.start_ea) if fn else None), - "func_addr": (hex(int(fn.start_ea)) if fn else None), - "seg": ida_segment.get_segm_name(seg)}) -result = {"hits": rows, "error": err, "scanned": scanned, - "truncated": len(rows) >= limit or scanned >= max_scan} -result -''', - "list_linkage": r''' -imports = [{"addr": hex(int(item.address)), "name": item.name, "module": item.module_name} - for item in db.imports.get_all_imports() if item.name] -exports = [{"addr": hex(int(item.address)), "name": item.name, "ordinal": int(item.ordinal)} - for item in db.entries.get_all() if item.name] -result = {"imports": imports, "exports": exports, - "n_imports": len(imports), "n_exports": len(exports)} -result -''', - "lookup_funcs": r''' -rows = [] -for query in a.get("queries", []): - raw = str(query) - try: ea = int(raw, 16) - except ValueError: - fn = db.functions.get_by_name(raw); ea = int(fn.start_ea) if fn else None - else: fn = db.functions.get_at(ea) - if fn is None: - rows.append({"query": raw, "fn": None}) - else: - rows.append({"query": raw, "fn": {"addr": hex(int(fn.start_ea)), - "name": db.functions.get_name(fn) or f"sub_{int(fn.start_ea):X}", - "size": int(fn.end_ea) - int(fn.start_ea)}}) -result = {"result": rows} -result -''', - "resolve_names": r''' -import ida_idaapi, ida_name -rows = [] -for query in a.get("queries", []): - name = str(query).strip(); ea = ida_name.get_name_ea(ida_idaapi.BADADDR, name) - rows.append({"query": name, "ea": hex(int(ea)) if ea != ida_idaapi.BADADDR else None}) -result = {"result": rows} -result -''', - # Ours: the coarse code/data type plus a fine `kind` (call/jump/flow, - # read/write/offset/text/info) that the xref dialog draws its badges from. - # Deliberately NOT sorted -- the dialog lists xrefs in IDA's own order. - "xref_types": r''' -import idaapi, idautils, ida_bytes, ida_funcs, ida_xref -code_kind = {ida_xref.fl_CF: "call", ida_xref.fl_CN: "call", ida_xref.fl_JF: "jump", - ida_xref.fl_JN: "jump", ida_xref.fl_F: "flow"} -data_kind = {ida_xref.dr_O: "offset", ida_xref.dr_W: "write", ida_xref.dr_R: "read", - ida_xref.dr_T: "text", ida_xref.dr_I: "info"} -def _kind(xr): - return (code_kind if xr.iscode else data_kind).get(xr.type, "code" if xr.iscode else "data") -def _fn(ea): - f = ida_funcs.get_func(ea) - return {"addr": hex(int(f.start_ea)), "name": ida_funcs.get_func_name(f.start_ea) or ""} if f else None -queries = a.get("queries") or [] -all_results = [] -for query in queries: - query = query if isinstance(query, dict) else {"addr": query} - raw = str(query.get("addr", "")).strip() - direction = str(query.get("direction", "to") or "to").lower() - include_fn = bool(query.get("include_fn", True)) - dedup = bool(query.get("dedup", True)) - try: count = int(query.get("count", 2000) or 2000) - except (TypeError, ValueError): count = 2000 - try: target = int(raw, 16) - except ValueError: target = idaapi.get_name_ea(idaapi.BADADDR, raw) - rows = [] - if target is not None and target != idaapi.BADADDR and ida_bytes.is_mapped(target): - if direction in ("to", "both"): - for xr in idautils.XrefsTo(target, 0): - row = {"direction": "to", "addr": hex(int(xr.frm)), "from": hex(int(xr.frm)), - "to": hex(int(target)), "type": "code" if xr.iscode else "data", "kind": _kind(xr)} - if include_fn: row["fn"] = _fn(xr.frm) - rows.append(row) - if direction in ("from", "both"): - for xr in idautils.XrefsFrom(target, 0): - row = {"direction": "from", "addr": hex(int(xr.to)), "from": hex(int(target)), - "to": hex(int(xr.to)), "type": "code" if xr.iscode else "data", "kind": _kind(xr)} - if include_fn: row["fn"] = _fn(xr.to) - rows.append(row) - if dedup: - seen, deduped = set(), [] - for r in rows: - k = (r["direction"], r["from"], r["to"], r["kind"]) - if k in seen: continue - seen.add(k); deduped.append(r) - rows = deduped - rows = rows[:count] - all_results.append({"query": raw, "data": rows, "next_offset": None}) -result = {"result": all_results} -result -''', - # Mirrors the tool ida-tui was written against, ORDER INCLUDED. The rows are - # sorted by the far-end address and deduped by default, and the pseudocode - # follow's address fallback silently depends on it: at a call site the raw - # IDA order yields the ordinary-flow xref (the next instruction) first, so an - # unsorted result makes "follow the call" land on the following line instead. - "xref_query": r''' -import idaapi, idautils, ida_bytes, ida_funcs -def _fn(ea): - f = ida_funcs.get_func(ea) - return {"addr": hex(int(f.start_ea)), "name": ida_funcs.get_func_name(f.start_ea) or ""} if f else None -queries = a.get("queries") or [] -all_results = [] -for query in queries: - raw = str(query.get("addr", "")).strip() - direction = str(query.get("direction", "both") or "both").lower() - if direction not in ("to", "from", "both"): direction = "both" - xref_type = str(query.get("xref_type", "any") or "any").lower() - if xref_type not in ("any", "code", "data"): xref_type = "any" - include_fn = bool(query.get("include_fn", True)) - dedup = bool(query.get("dedup", True)) - sort_by = str(query.get("sort_by", "addr") or "addr") - descending = bool(query.get("descending", False)) - try: offset = max(0, int(query.get("offset", 0) or 0)) - except (TypeError, ValueError): offset = 0 - try: count = max(0, min(int(query.get("count", 200) or 200), 5000)) - except (TypeError, ValueError): count = 200 - try: - try: target = int(raw, 16) - except ValueError: - target = idaapi.get_name_ea(idaapi.BADADDR, raw) - if target == idaapi.BADADDR: raise ValueError(f"Failed to resolve address/name: {raw}") - if not ida_bytes.is_mapped(target): raise ValueError(f"Address not mapped: {raw}") - rows = [] - if direction in ("to", "both"): - for xr in idautils.XrefsTo(target, 0): - kind = "code" if xr.iscode else "data" - if xref_type != "any" and kind != xref_type: continue - row = {"direction": "to", "addr": hex(int(xr.frm)), "from": hex(int(xr.frm)), - "to": hex(int(target)), "type": kind} - if include_fn: row["fn"] = _fn(xr.frm) - rows.append(row) - if direction in ("from", "both"): - for xr in idautils.XrefsFrom(target, 0): - kind = "code" if xr.iscode else "data" - if xref_type != "any" and kind != xref_type: continue - row = {"direction": "from", "addr": hex(int(xr.to)), "from": hex(int(target)), - "to": hex(int(xr.to)), "type": kind} - if include_fn: row["fn"] = _fn(xr.to) - rows.append(row) - if dedup: - seen, deduped = set(), [] - for row in rows: - key = (row["direction"], row["from"], row["to"], row["type"]) - if key in seen: continue - seen.add(key); deduped.append(row) - rows = deduped - if sort_by == "type": - rows.sort(key=lambda r: (str(r.get("type", "")), int(str(r["addr"]), 16)), reverse=descending) - else: - rows.sort(key=lambda r: int(str(r["addr"]), 16), reverse=descending) - page = rows[offset:offset + count] if count else rows[offset:] - nxt = offset + len(page) - all_results.append({"target": raw, "resolved_addr": hex(int(target)), "direction": direction, - "xref_type": xref_type, "data": page, - "next_offset": nxt if nxt < len(rows) else None, - "total": len(rows), "error": None}) - except Exception as exc: - all_results.append({"target": raw, "resolved_addr": None, "direction": direction, - "xref_type": xref_type, "data": [], "next_offset": None, - "total": 0, "error": str(exc)}) -result = {"result": all_results} -result -''', - # A comment must land in BOTH views, and the pseudocode half is not a - # simple set: db.comments.set_at() alone leaves the pseudocode unchanged. - # Hex-Rays comments are anchored to a ctree location (treeloc_t), and an - # anchor the ctree does not actually own is dropped as an "orphan" -- so the - # itp slot has to be searched until one sticks, exactly as IDA's own UI does. - # Without it a comment silently never appears in the decompilation. - "set_comments": r''' -import idaapi, idc, ida_hexrays -rows = [] -for item in a.get("items", []): - addr_s = str(item.get("addr", "")) - text = str(item.get("comment") or "") - try: - ea = int(addr_s, 16) - if not idaapi.set_cmt(ea, text, False): - rows.append({"addr": addr_s, - "error": f"Failed to set disassembly comment at {hex(ea)}"}) - continue - if not ida_hexrays.init_hexrays_plugin(): - rows.append({"addr": addr_s}); continue - try: - cfunc = ida_hexrays.decompile(ea) - except Exception: - cfunc = None - if cfunc is None: - rows.append({"addr": addr_s}); continue - if ea == cfunc.entry_ea: - # The signature line carries no ctree item: it is a function comment. - idc.set_func_cmt(ea, text, True) - cfunc.refresh_func_ctext() - rows.append({"addr": addr_s}); continue - eamap = cfunc.get_eamap() - if ea not in eamap: - rows.append({"addr": addr_s, - "error": f"Failed to set decompiler comment at {hex(ea)}"}) - continue - nearest_ea = eamap[ea][0].ea - if cfunc.has_orphan_cmts(): - cfunc.del_orphan_cmts(); cfunc.save_user_cmts() - tl = idaapi.treeloc_t(); tl.ea = nearest_ea - placed = False - for itp in range(idaapi.ITP_SEMI, idaapi.ITP_COLON): - tl.itp = itp - cfunc.set_user_cmt(tl, text) - cfunc.save_user_cmts() - cfunc.refresh_func_ctext() - if not cfunc.has_orphan_cmts(): - placed = True; break - cfunc.del_orphan_cmts(); cfunc.save_user_cmts() - rows.append({"addr": addr_s} if placed else - {"addr": addr_s, - "error": f"Failed to set decompiler comment at {hex(ea)}"}) - except Exception as exc: - rows.append({"addr": addr_s, "error": str(exc)}) -result = {"result": rows} -result -''', - # Every category takes EITHER one edit or a LIST of them, and the answer is - # one row per edit. The port accepted only a single dict, so any batch path - # (rpc rename_many applying a whole symbol file, which is the entire point of - # that verb) died with "list indices must be integers or slices, not str" and - # reported the failure against addr=null. Mirrors the real tool: conflict - # detection before the write, dry_run/allow_overwrite/stop_on_error, per-row - # addr/old/name, and a summary counting EDITS rather than categories. - "rename": r''' -import idaapi, ida_hexrays, ida_name -batch = a.get("batch") or {} -dry_run = bool(batch.get("dry_run", False)) -allow_overwrite = bool(batch.get("allow_overwrite", False)) -stop_on_error = bool(batch.get("stop_on_error", False)) - -def _items(value): - if value is None: return [] - if isinstance(value, dict): return [value] - if isinstance(value, list): return [i for i in value if isinstance(i, dict)] - return [] - -def _set_name_checked(ea, new): - conflict = idaapi.get_name_ea(idaapi.BADADDR, new) - if conflict != idaapi.BADADDR and conflict != ea and not allow_overwrite: - return False, f"can't rename at {hex(ea)} as {new!r}: name already used at {hex(conflict)}" - if dry_run: - return True, None - flags = idaapi.SN_CHECK - if allow_overwrite: flags |= int(getattr(idaapi, "SN_FORCE", 0)) - if not idaapi.set_name(ea, new, flags): - return False, (f"Rename failed at {hex(ea)}: IDA rejected name {new!r} " - "(invalid identifier or internal conflict)") - return True, None - -def _refresh_ctext(fn_addr): - # A renamed function must invalidate Hex-Rays' cache, which is per function - # and persisted in the .i64: without this the pseudocode keeps calling the - # old name forever while every other readback reports the new one. - if not ida_hexrays.init_hexrays_plugin(): return - failure = ida_hexrays.hexrays_failure_t() - cfunc = ida_hexrays.decompile_func(fn_addr, failure, ida_hexrays.DECOMP_WARNINGS) - if cfunc: cfunc.refresh_func_ctext() - -out = {}; ok_count = failed = 0; halted = False -for category in ("func", "data", "local", "stack"): - if category not in batch: continue - rows = [] - for edit in _items(batch.get(category)): - try: - if category == "func": - addr_text = edit.get("addr") or edit.get("func_addr") or edit.get("func") - new = edit.get("name") or edit.get("new") or edit.get("new_name") - if not addr_text or not new: - row = {"addr": addr_text, "name": new, - "error": "Function rename requires addr + name"} - else: - ea = int(str(addr_text), 16) - fn = idaapi.get_func(ea) - if fn is None: - row = {"addr": addr_text, "name": new, "error": "Function not found"} - else: - old = idaapi.get_name(fn.start_ea) or None - ok, err = _set_name_checked(fn.start_ea, str(new)) - row = {"addr": addr_text, "old": old, "name": str(new)} - if err: row["error"] = err - if dry_run: row["dry_run"] = True - if ok and not dry_run: _refresh_ctext(fn.start_ea) - elif category == "data": - addr_text = edit.get("addr") - old = edit.get("old") or edit.get("old_name") - new = edit.get("new") or edit.get("new_name") or edit.get("name") - if not new and new != "": - row = {"old": old, "new": None, - "error": "Global rename requires target and new name"} - else: - if addr_text is not None: - ea = int(str(addr_text), 16) - old = old or (idaapi.get_name(ea) or None) - else: - ea = idaapi.get_name_ea(idaapi.BADADDR, str(old or "")) - if ea == idaapi.BADADDR: - row = {"old": old, "new": str(new), "error": f"Global {old!r} not found"} - else: - # An empty new name CLEARS the label; that is a real - # request (tests revert with it), not a missing argument. - if str(new) == "": - ok = bool(ida_name.set_name(ea, "", idaapi.SN_CHECK)) - err = None if ok else f"Failed to clear the name at {hex(ea)}" - else: - ok, err = _set_name_checked(ea, str(new)) - row = {"addr": hex(ea), "old": old, "new": str(new)} - if err: row["error"] = err - if dry_run: row["dry_run"] = True - else: - fa, old, new = edit.get("func_addr"), edit.get("old"), edit.get("new") - if not fa or not old or not new: - row = {"old": old, "new": new, - "error": f"{category} rename requires func_addr + old + new"} - else: - ea = int(str(fa), 16) - pseudo = db.pseudocode.decompile(ea) - var = pseudo.find_local_variable(str(old)) - if var is None: - row = {"func_addr": fa, "old": old, "new": new, - "error": f"no local {old!r} in that function"} - elif dry_run: - row = {"func_addr": fa, "old": old, "new": new, "dry_run": True} - else: - var.set_user_name(str(new)) - ok = bool(pseudo.save_local_variable_info(var, save_name=True)) - row = {"func_addr": fa, "old": old, "new": new} - if not ok: row["error"] = "IDA rejected the local variable name" - except Exception as exc: - row = {"addr": edit.get("addr"), "error": str(exc)} - rows.append(row) - if row.get("error"): failed += 1 - else: ok_count += 1 - if row.get("error") and stop_on_error: - halted = True; break - out[category] = rows - if halted: break -out["summary"] = {"ok": ok_count, "failed": failed} -if dry_run: out["summary"]["dry_run"] = True -if halted: out["summary"]["halted"] = True -result = out -result -''', -} - - -_OPERATIONS["define_code_run"] = r''' -import ida_bytes, ida_idp, ida_segment, ida_ua, idaapi -ea, limit = int(str(a["addr"]), 16), max(1, min(int(a.get("limit", 20000)), 200000)) -seg = ida_segment.getseg(ea) -if seg is None: - result = {"addr": a["addr"], "error": "no segment", "count": 0} -else: - start, count, stopped, hi = ea, 0, "limit", int(seg.end_ea) - while count < limit: - if ea >= hi: stopped = "segment"; break - flags = ida_bytes.get_flags(ea) - if ida_bytes.is_code(flags) or ida_bytes.is_data(flags): stopped = "defined"; break - size = int(ida_ua.create_insn(ea)) - if size <= 0: stopped = "undecodable"; break - count += 1 - insn = ida_ua.insn_t() - if ida_ua.decode_insn(insn, ea) > 0: - try: is_ret = bool(ida_idp.is_ret_insn(insn)) - except Exception: is_ret = False - if is_ret or (insn.get_canon_feature() & idaapi.CF_STOP): - ea += size; stopped = "flow"; break - ea += size - result = {"start": hex(start), "end": hex(ea), "count": count, "stopped": stopped} -result -''' - - -_OPERATIONS["define_func_run"] = r''' -import ida_bytes, ida_funcs, ida_segment -ea = int(str(a["addr"]), 16) -fn = db.functions.get_at(ea) -if fn is not None and int(fn.start_ea) == ea: - result = {"addr": hex(ea), "ok": True, "start": hex(ea), "end": hex(int(fn.end_ea)), "how": "existed"} -else: - automatic = bool(db.functions.create(ea)) - if not automatic: - seg = db.segments.get_at(ea); end = ea; hi = int(seg.end_ea) if seg else ea - while end < hi and ida_bytes.is_code(ida_bytes.get_flags(end)): - nxt = int(ida_bytes.get_item_end(end)) - if nxt <= end: break - end = nxt - ok = bool(end > ea and ida_funcs.add_func(ea, end)) - else: ok = True - fn = db.functions.get_at(ea) - result = ({"addr": hex(ea), "ok": True, "start": hex(int(fn.start_ea)), - "end": hex(int(fn.end_ea)), "how": "auto" if automatic else "explicit-end"} - if ok and fn is not None else - {"addr": hex(ea), "ok": False, "error": f"IDA refused a function at {ea:#x}"}) -result -''' - - -_OPERATIONS["set_thumb"] = r''' -import ida_bytes, ida_ida, ida_idp, ida_segment, ida_segregs -ea = int(str(a["addr"]), 16); treg = ida_idp.str2reg("T") -seg = ida_segment.getseg(ea) -if treg is None or treg < 0: - result = {"addr": hex(ea), "error": "no T register (not an ARM database)"} -elif seg is None: - result = {"addr": hex(ea), "error": "no segment"} -else: - current = ida_segregs.get_sreg(ea, treg) - current = 0 if current in (None, 0xFFFFFFFF, -1) else int(current) - want = {"on": 1, "off": 0}.get(str(a.get("mode", "toggle")).lower(), 0 if current else 1) - changed = False - if want and seg.bitness != 1: - ida_segment.set_segm_addressing(seg, 1); changed = True - size = max(int(ida_bytes.get_item_size(ea)), 2) - ida_bytes.del_items(ea, 0, size) - ok = bool(ida_segregs.split_sreg_range(ea, treg, want, ida_segregs.SR_user)) - now = ida_segregs.get_sreg(ea, treg) - result = {"addr": hex(ea), "thumb": bool(now), "was": bool(current), "ok": ok, - "bitness": ida_segment.getseg(ea).bitness, "forced_32bit": changed, - "db_64bit": bool(ida_ida.inf_get_app_bitness() == 64 and want)} -result -''' - - -_OPERATIONS["thumb_scan"] = r''' -import ida_bytes, ida_funcs, ida_idp, ida_segment, ida_segregs, ida_ua -lo, hi = int(str(a["start"]), 16), int(str(a["end"]), 16) -apply, limit = bool(a.get("apply", True)), int(a.get("limit", 512)) -treg = ida_idp.str2reg("T"); found = []; applied = 0; cursor = lo -while cursor + 4 <= hi and len(found) < limit: - at = cursor; value = int(ida_bytes.get_dword(cursor)); cursor += 4 - if not value & 1: continue - target = value & ~1; seg = ida_segment.getseg(target) - if seg is None or not (seg.perm & ida_segment.SEGPERM_EXEC or seg.perm == 0): continue - flags = ida_bytes.get_flags(target) - if ida_bytes.is_data(flags): continue - item = {"at": hex(at), "value": hex(value), "target": hex(target), - "was_code": bool(ida_bytes.is_code(flags))}; found.append(item) - if not apply: continue - if treg is not None and treg >= 0: ida_segregs.split_sreg_range(target, treg, 1, ida_segregs.SR_user) - if not ida_bytes.is_code(ida_bytes.get_flags(target)): - ida_bytes.del_items(target, 0, 2) - if ida_ua.create_insn(target) <= 0: item["decoded"] = False; continue - item["decoded"] = True; item["function"] = bool(db.functions.get_at(target) or db.functions.create(target)); applied += 1 -result = {"start": hex(lo), "end": hex(hi), "found": found, "applied": applied, "n": len(found)} -result -''' - - -_OPERATIONS["decomp_error"] = r''' -import ida_hexrays, ida_ida -ea = int(str(a["addr"]), 16); fn = db.functions.get_at(ea) -result = {"addr": hex(ea), "bitness": ida_ida.inf_get_app_bitness()} -if fn is None: - result["reason"] = "no function here" -else: - try: - failure = ida_hexrays.hexrays_failure_t(); cfunc = ida_hexrays.decompile_func(fn, failure) - if cfunc is not None: result["reason"] = "" - else: - result.update({"reason": failure.desc() or f"error {failure.code}", - "code": int(failure.code), "errea": hex(int(failure.errea))}) - except Exception as exc: result["reason"] = f"{type(exc).__name__}: {exc}" -result -''' - -# `heads` and the operand-format tools are the port's IDAPython island: the -# continuous listing's presentation model (undefined runs, colour spans, operand -# extents, banners, struct members, the digest protocol) and IDA/Hex-Rays number -# formats have no ida-domain surface. Rather than paraphrase ~1100 lines of -# performance-tuned, behaviour-sensitive code into string literals, they stay -# real, diffable source in idatui/remote_tools.py and are shipped to the database -# process as text. Read once at import; the file ships beside this module. -_REMOTE_LIB = (Path(__file__).with_name("remote_tools.py")).read_text(encoding="utf-8") - -#: Versioned by content, so editing remote_tools.py re-installs it instead of -#: silently running the copy a long-lived worker already has. -_REMOTE_MODULE = "_idatui_remote_" + hashlib.sha1( - _REMOTE_LIB.encode("utf-8")).hexdigest()[:12] - -#: Sent back when the database process has not got the library yet; the client -#: installs it and retries once. Amortised, a worker receives it exactly once. -_NEED_LIB = "__idatui_needs_remote_lib__" - -#: Installs the library as a real module in the database process. Persisting it -#: in sys.modules is what makes the module-level caches (the tag maps, and the -#: line-render lru_cache the listing's throughput depends on) survive between -#: calls -- execute_python builds a fresh namespace every time, so a library -#: exec'd inline is rebuilt, and its caches thrown away, on every single call. -_INSTALL_LIB = f''' -import sys, types -_m = types.ModuleType({_REMOTE_MODULE!r}) -exec(compile(a["source"], {_REMOTE_MODULE!r}, "exec"), _m.__dict__) -sys.modules[{_REMOTE_MODULE!r}] = _m -result = True -result -''' - - -def _remote_op(call: str) -> str: - """A snippet that calls one of the carried-over tools by its real signature. - - Costs one short request: the library is imported from the database process's - own sys.modules, not shipped again. - """ - return (f"import sys\n" - f"_m = sys.modules.get({_REMOTE_MODULE!r})\n" - f"result = {{{_NEED_LIB!r}: True}} if _m is None else _m.{call}\n" - f"result\n") - - -_OPERATIONS["op_format"] = _remote_op( - 'op_format(addr=a["addr"], mode=a.get("mode", "cycle"),' - ' col=int(a.get("col", -1)), n=int(a.get("n", -1)))') -_OPERATIONS["pc_nums"] = _remote_op('pc_nums(addr=a["addr"])') -_OPERATIONS["decompile"] = _remote_op( - 'decompile(addr=a["addr"],' - ' include_addresses=bool(a.get("include_addresses", True)))') -_OPERATIONS["decomp_map"] = _remote_op('decomp_map(addr=a["addr"])') -_OPERATIONS["pc_num_format"] = _remote_op( - 'pc_num_format(addr=a["addr"], mode=a.get("mode", "cycle"),' - ' line=int(a.get("line", -1)), col=int(a.get("col", -1)),' - ' ea=a.get("ea", ""), opnum=int(a.get("opnum", -1)))') - -# The listing walker itself. Replaces the port's re-implementation, which -# rendered no per-operand extents (so no keypress could say which literal it -# would reformat) and had no digest/expect support (so every page was re-sent -# after any edit), and whose span walk was the per-character loop our own -# version had already been rewritten to avoid. -#: Row count + seek anchors for a whole segment, in ONE call. See -#: remote_tools.segment_index: the alternative is fetching every row. -_OPERATIONS["segment_index"] = _remote_op( - 'segment_index(addr=a["addr"], end=a.get("end", ""),' - ' page_rows=int(a.get("page_rows", 500)), detail=bool(a.get("detail", False)))') - -_HEADS = _remote_op( - 'heads(addr=a["addr"], count=int(a.get("count", 200)),' - ' offset=int(a.get("offset", 0)), end=a.get("end", ""),' - ' back=bool(a.get("back", False)), annotate=bool(a.get("annotate", False)),' - ' expect=a.get("expect", ""), text=bool(a.get("text", True)))') - - -# The graph view's only backend call. Blocks are address RANGES, never text: -# the client re-renders them with `heads`, so boxes reuse the exact listing rows -# (colours, operand marks, trail painting) instead of growing a second renderer. -# -# ida-domain exposes no basic-block/edge-kind surface, so this stays on ida_gdl. -_OPERATIONS["flowchart"] = r''' -import ida_funcs, ida_gdl -ea = int(str(a["addr"]), 16) -fn = ida_funcs.get_func(ea) -if fn is None: - result = {"addr": hex(ea), "error": "no function at that address", "blocks": []} -else: - fc = ida_gdl.FlowChart(fn, flags=ida_gdl.FC_PREDS) - index, order = {}, [] - for bb in fc: - index[bb.start_ea] = len(order) - order.append(bb) - blocks = [] - for bb in order: - sl = [s for s in bb.succs() if s.start_ea in index] - succs = [] - for s in sl: - # Edge kind is what the graph view colours by: an n-way dispatch is - # "switch", a successor that is literally the next address falls - # through, anything else is a taken branch. - if len(sl) > 2: kind = "switch" - elif s.start_ea == bb.end_ea: kind = "fall" - else: kind = "jump" - succs.append([index[s.start_ea], kind]) - blocks.append({"id": index[bb.start_ea], "start": hex(int(bb.start_ea)), - "end": hex(int(bb.end_ea)), "succs": succs}) - result = {"addr": hex(ea), - "func": {"addr": hex(int(fn.start_ea)), "end": hex(int(fn.end_ea)), - "name": ida_funcs.get_func_name(fn.start_ea) or ""}, - "entry": index.get(fn.start_ea, 0), "blocks": blocks} -result -''' - -# Only ever reached as domain.py's fallback when file_regions yields nothing. -_OPERATIONS["survey_binary"] = r''' -segments = [] -for seg in db.segments.get_all(): - segments.append({"start": hex(int(seg.start_ea)), "end": hex(int(seg.end_ea)), - "name": db.segments.get_name(seg) or ""}) -result = {"segments": segments} -result -''' + self._callback(batch) + except Exception as exc: # noqa: BLE001 -- keep the stream alive + self._report(exc) + + def close(self) -> None: + """Stop delivery and unblock the subscription reader.""" + with self._condition: + if self._closed: + return + self._closed = True + self._pending.clear() + subscription = self._subscription + self._condition.notify_all() + if subscription is not None: + subscription.close() class CodeModeClient: @@ -1277,7 +287,9 @@ class CodeModeClient: self._path = os.path.abspath(os.path.expanduser(binary_path)) parsed_processor, parsed_address, parsed_file_type = _parse_load_args(load_args) self._processor = processor or parsed_processor - self._loading_address = loading_address if loading_address is not None else parsed_address + self._loading_address = ( + loading_address if loading_address is not None else parsed_address + ) self._file_type = file_type or parsed_file_type self._output_database = output_database self._spawn = spawn @@ -1289,10 +301,17 @@ class CodeModeClient: def connect(self, timeout: float = 1800.0, progress=None) -> "CodeModeClient": _require_codemode() with self._connect_lock: - if self._handle is not None and self._handle.connected: - return self + handle = self._handle + if handle is not None: + if handle.connected: + return self + raise IDAConnectionError( + "Code Mode database disconnected; explicit rediscovery required" + ) if progress: - progress(f"discovering Code Mode database for {os.path.basename(self._path)}…") + progress( + f"discovering Code Mode database for {os.path.basename(self._path)}…" + ) try: # A Ctrl+L reload releases its current managed-worker lease, but # that worker remains registered during Code Mode's final-lease @@ -1321,7 +340,9 @@ class CodeModeClient: if not self._new_database or time.monotonic() >= deadline: raise if progress: - progress("waiting for the previous Code Mode lease to close…") + progress( + "waiting for the previous Code Mode lease to close…" + ) owner = find_database_owner( self._path, output_database=self._output_database, @@ -1336,7 +357,9 @@ class CodeModeClient: time.sleep(0.2) if progress: backend = handle.instance.backend - progress(f"attached to {backend} database; waiting for auto-analysis…") + progress( + f"attached to {backend} database; waiting for auto-analysis…" + ) handle.wait_autoanalysis(timeout=timeout) except Exception as exc: # normalize the dependency's transport errors raise self._connection_error(exc) from exc @@ -1360,61 +383,60 @@ class CodeModeClient: def backend(self) -> str | None: return self._handle.instance.backend if self._handle is not None else None - def execute_python(self, code: str, *, timeout: float | None = None) -> Any: + def owns_event(self, event: dict[str, Any]) -> bool: + """Whether ``event`` was produced through this client's handle.""" + handle = self._handle + return handle is not None and handle.owns_event(event) + + def subscribe_idb_events(self): + """Open Code Mode's closeable IDB-change iterator.""" + if not self.connected: + self.connect() + handle = self._handle + if handle is None: + raise IDAConnectionError("Code Mode database is not connected") + try: + return handle.subscribe_idb_events() + except (DatabaseDisconnectedError, CodeModeConnectionError) as exc: + raise self._connection_error(exc) from exc + + def watch_idb_events( + self, + callback: Callable[[tuple[dict[str, Any], ...]], None], + *, + on_error: Callable[[BaseException], None] | None = None, + debounce: float = 0.2, + ) -> IDBEventListener: + """Deliver external IDB changes in debounced batches.""" + return IDBEventListener(self, callback, on_error=on_error, debounce=debounce) + + def call(self, operation: Callable[..., Any], /, **args) -> Any: + """Execute one source-backed remote declaration through this client.""" + name = getattr(operation, "__name__", "remote operation") + try: + from .remote_ops import bind + + remote = bind(operation) + except KeyError as exc: + raise IDAToolError( + name, f"remote operation {name!r} is not registered" + ) from exc if not self.connected: self.connect() handle = self._handle if handle is None: raise IDAConnectionError("Code Mode database is not connected") try: - response = handle.execute_python(code, timeout=timeout) + return remote(handle, **args) except RemoteError as exc: - details = exc.details or {} message = str(exc) - if details.get("traceback"): - message += f"\n{details['traceback']}" + if exc.details.get("traceback"): + message += f"\n{exc.details['traceback']}" if exc.code == "operation_timeout": raise IDATimeoutError(message) from exc - raise IDAToolError("execute_python", message) from exc + raise IDAToolError(name, message) from exc except (DatabaseDisconnectedError, CodeModeConnectionError) as exc: raise self._connection_error(exc) from exc - if not isinstance(response, dict) or "result" not in response: - raise IDAToolError("execute_python", "Code Mode returned an invalid execution result") - return response["result"] - - @staticmethod - def _unpack(answer: Any) -> Any: - """Undo _PACK_EPILOGUE. Anything else passes through untouched.""" - if isinstance(answer, dict) and _PACKED in answer: - return json.loads(answer[_PACKED]) - return answer - - def invoke(self, operation: str, *, timeout: float | None = None, **args) -> Any: - """Execute one TUI domain operation through Code Mode.""" - if operation in ("idb_save", "save"): - return self.save_database() - if operation in ("server_health", "ping", "health", "state"): - return self.health() - body = _HEADS if operation == "heads" else _OPERATIONS.get(operation) - if body is None: - raise IDAToolError(operation, f"unknown ida-tui Code Mode operation: {operation}") - try: - answer = self._unpack(self.execute_python(_script(args, body), timeout=timeout)) - if isinstance(answer, dict) and answer.get(_NEED_LIB): - # First call against this database process (or a restarted one). - self.execute_python(_script({"source": _REMOTE_LIB}, _INSTALL_LIB), - timeout=timeout) - answer = self._unpack( - self.execute_python(_script(args, body), timeout=timeout)) - return answer - except IDAToolError as exc: - if exc.tool == "execute_python": - raise IDAToolError(operation, exc.message) from exc - raise - - # Temporary source compatibility for external drivers/tests that used the - # old WorkerClient. Application code uses the accurately named invoke(). - call = invoke def save_database(self) -> dict[str, Any]: if not self.connected: @@ -1429,6 +451,35 @@ class CodeModeClient: except (DatabaseDisconnectedError, CodeModeConnectionError) as exc: raise self._connection_error(exc) from exc + def discard_database(self, timeout: float = 5.0) -> bool: + """Discard a final managed-worker lease; otherwise transfer finalization. + + ``False`` is an expected ownership result: a GUI owns its session, or + another lease still shares the managed worker. A busy final worker is + retried briefly so background reads finishing during quit do not turn a + real discard into an implicit save. + """ + handle = self._handle + if handle is None or not handle.connected: + return False + entry = handle.instance + if entry.backend != "idalib" or not getattr(entry, "managed", False): + return False + deadline = time.monotonic() + max(float(timeout), 0.0) + while True: + try: + handle.shutdown_database(save=False) + return True + except RemoteError as exc: + if exc.code in ("instance_shared", "shutdown_not_supported"): + return False + if exc.code == "instance_busy" and time.monotonic() < deadline: + time.sleep(0.05) + continue + raise IDAToolError("shutdown_database", str(exc)) from exc + except (DatabaseDisconnectedError, CodeModeConnectionError) as exc: + raise self._connection_error(exc) from exc + def health(self) -> dict[str, Any]: if not self.connected: self.connect() @@ -1463,8 +514,14 @@ class CodeModeClient: assert self._handle is not None entry = self._handle.instance path = entry.exe_path or entry.idb_path or self._path - return [Session(session_id=entry.record_id, filename=os.path.basename(path), - input_path=path, is_active=True)] + return [ + Session( + session_id=entry.record_id, + filename=os.path.basename(path), + input_path=path, + is_active=True, + ) + ] def close(self, grace: float = 0.0) -> None: del grace -- cgit v1.3.1-sl0p From f4c1d9b5497fd0d38137b6b345e8171b307c1117 Mon Sep 17 00:00:00 2001 From: Duncan Ogilvie Date: Thu, 20 Aug 2026 23:55:52 +0200 Subject: Switch to ida-nexus --- CONTRIBUTING.md | 8 +- README.md | 18 +- TODO | 4 +- docs/CODEMODE_UPSTREAM.md | 385 ---------------------------- docs/GRAPH_VIEW.md | 2 +- docs/NEXUS_UPSTREAM.md | 385 ++++++++++++++++++++++++++++ docs/PAGING_FINDINGS.md | 14 +- docs/PROJECTS.md | 10 +- docs/SPLIT_VIEW.md | 8 +- experiments/bench_ops.py | 12 +- experiments/bench_pack_trace.py | 4 +- experiments/call_census.py | 10 +- experiments/profile_client.py | 4 +- experiments/profile_remote.py | 4 +- experiments/worker_smoke.py | 10 +- ida-tui | 2 +- idatui/__init__.py | 6 +- idatui/app.py | 42 +-- idatui/codemode_client.py | 551 ---------------------------------------- idatui/domain.py | 28 +- idatui/errors.py | 2 +- idatui/launch.py | 16 +- idatui/nexus_client.py | 551 ++++++++++++++++++++++++++++++++++++++++ idatui/pane.py | 18 +- idatui/pool.py | 14 +- idatui/project.py | 14 +- idatui/remote_ops.py | 4 +- idatui/remote_tools.py | 12 +- pyproject.toml | 7 +- tests/_fixtures.py | 2 +- tests/run.py | 4 +- tests/test_codemode_client.py | 440 -------------------------------- tests/test_kittygfx.py | 2 +- tests/test_launch.py | 6 +- tests/test_nexus_client.py | 440 ++++++++++++++++++++++++++++++++ tests/test_pool.py | 4 +- tests/test_project.py | 2 +- tests/test_scenarios.py | 2 +- tests/test_thumb_ui.py | 2 +- tests/test_trace_ui.py | 2 +- uv.lock | 28 +- 41 files changed, 1539 insertions(+), 1540 deletions(-) delete mode 100644 docs/CODEMODE_UPSTREAM.md create mode 100644 docs/NEXUS_UPSTREAM.md delete mode 100644 idatui/codemode_client.py create mode 100644 idatui/nexus_client.py delete mode 100644 tests/test_codemode_client.py create mode 100644 tests/test_nexus_client.py (limited to 'idatui/codemode_client.py') diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8efdfe7..b66df1c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -18,12 +18,12 @@ without a licence. uv sync ``` -That pulls [ida-codemode](https://github.com/HexRaysSA/ida-codemode) from PyPI, +That pulls [ida-nexus](https://github.com/HexRaysSA/ida-nexus) from PyPI, which is how ida-tui talks to IDA. To also attach to databases open in the IDA GUI: ```sh -uvx ida-hcli plugin install ida-codemode +uvx ida-hcli plugin install ida-nexus ``` ## Running the tests @@ -39,7 +39,7 @@ python3 tests/run.py # everything (needs IDA) ``` The IDA-backed suites need an interpreter that has `textual`, `idapro` and -`ida_codemode` on it: +`ida_nexus` on it: ```sh tests/test_scenarios.py /path/to/binary --only rename @@ -47,7 +47,7 @@ The IDA-backed suites need an interpreter that has `textual`, `idapro` and ``` **House rule:** a suite marked `pure` must keep running under a plain system -`python3`. This is why `idatui/codemode_client.py` defers its `ida_codemode` +`python3`. This is why `idatui/nexus_client.py` defers its `ida_nexus` import instead of doing it at module top. Please don't break that — it's what keeps the fast gate fast and lets people without IDA contribute at all. diff --git a/README.md b/README.md index 5cfc59e..9a751a1 100644 --- a/README.md +++ b/README.md @@ -27,18 +27,18 @@ Needs **Python ≥ 3.11** and **IDA Pro 9.4+ with idalib**. uv sync ``` -That pulls [ida-codemode](https://github.com/HexRaysSA/ida-codemode) from PyPI, +That pulls [ida-nexus](https://github.com/HexRaysSA/ida-nexus) from PyPI, which is how ida-tui talks to IDA. To also attach to databases you have open in the IDA GUI, install its plugin: ```sh -uvx ida-hcli plugin install ida-codemode +uvx ida-hcli plugin install ida-nexus ``` -Hacking on ida-codemode itself? Point at a checkout instead: +Hacking on ida-nexus itself? Point at a checkout instead: ```sh -uv add --editable ../ida-codemode +uv add --editable ../ida-nexus ``` ## Run @@ -49,9 +49,9 @@ uv add --editable ../ida-codemode ``` ida-tui never owns an IDA process — it takes a **lease**. A matching database open -in the IDA GUI is reused, otherwise Code Mode starts or shares a managed idalib +in the IDA GUI is reused, otherwise IDA Nexus starts or shares a managed idalib worker. Quitting drops the lease and leaves everyone else alone. -Changes made in the GUI or another client arrive over Code Mode's IDB event +Changes made in the GUI or another client arrive over IDA Nexus's IDB event stream; ida-tui debounces bursts and refreshes its cached views automatically. On quit with unsaved changes, a final managed-worker lease can discard the session without saving; GUI-backed or still-shared sessions leave that final @@ -59,9 +59,9 @@ decision with their owner or remaining clients. If the owning GUI or worker closes, ida-tui never replaces it by spawning a headless worker implicitly. It keeps the cached view disconnected until a matching owner is reopened and an attach-only rediscovery succeeds. -Remote operations are typed, source-backed Python functions. ida-codemode -installs content-addressed modules once per handle, so ida-tui keeps normal -refactorable source without paying to resend hot listing/decompiler code. +Remote operations are typed, source-backed Python functions. ida-nexus installs +their content-addressed modules once per IDA Python interpreter, so ida-tui keeps +normal refactorable source without paying to resend hot listing/decompiler code. Operation attribution is also a per-call provider rather than a fixed string, so it can evolve from `IDA TUI` to labels such as `IDA TUI: alice`. diff --git a/TODO b/TODO index 726e2b1..b497c89 100644 --- a/TODO +++ b/TODO @@ -4,12 +4,12 @@ TODO: [x] PORT TO ida-codemode-mcp as a library dependency [x] DatabaseHandle discovery prefers registered GUI sessions [x] shared managed idalib workers + SSE lease lifecycle - [x] domain operations execute against ida-domain through Code Mode + [x] domain operations execute against ida-domain through IDA Nexus [x] delete the private pickle worker and ida-pro-mcp patch injection [x] stop sweeping/reaping resources that may belong to another client [ ] run the full live Pilot suite against both GUI and managed backends [ ] add database revision/change notifications for cross-client cache invalidation - [ ] decide how "discard changes" should work (Code Mode final workers save) + [ ] decide how "discard changes" should work (IDA Nexus final workers save) - [x] add support for toggling literal types, ala `o` in IDA. (decimal to hex to reference etc.) diff --git a/docs/CODEMODE_UPSTREAM.md b/docs/CODEMODE_UPSTREAM.md deleted file mode 100644 index 7dff13e..0000000 --- a/docs/CODEMODE_UPSTREAM.md +++ /dev/null @@ -1,385 +0,0 @@ -# Findings from porting a real client to IDA Code Mode - -Notes for the `ida-codemode` maintainers, gathered while porting **ida-tui** (a -Textual TUI frontend for IDA) from a private idalib worker to -`ida_codemode.DatabaseHandle`. - -Everything below is measured, not inferred. Where we worked around something, the -workaround is named so you can judge whether the library should make it -unnecessary. - -**Environment:** ida-codemode 0.3.1, IDA 9.4 (idalib), Linux, single managed -worker backend, quiet box. Target for timings: `targets/echo` unless stated. - -> **Status against the protocol-6 event-stream development tree, based on 0.6.1 -> (upstream `439289f`) — every item re-checked.** -> -> | item | verdict | -> |---|---| -> | 1 `timeout_trace` line tracing | ✅ **fixed in 0.3.2** — no `settrace` in the runtime at all | -> | 2 `to_jsonable` on large results | ✅ **fixed in 0.3.2** — `dumps_json` C fast path | -> | 3 2 ms `execute_sync` floor | ✅ **fixed in 0.3.2, 7.0x** — 2.055 ms → 0.294 ms | -> | 4 loader switches fatal on reopen | **partial** — normal reopen fixed in 0.5.x; direct `.i64` paths remain [issue #36](https://github.com/HexRaysSA/ida-codemode/issues/36) | -> | 5 IDB replaced under a live lease | **stale** — out-of-band replacement is outside the supported lifecycle, as it is for the IDA GUI | -> | 6 close without save | **fixed in protocol 6** — the final managed-worker lease can choose `shutdown_database(save=False)` | -> | 7 no change notification | ✅ **fixed in protocol 6** — `DatabaseHandle.subscribe_idb_events()` streams revisioned, operation-attributed IDB changes | -> | 8 package exports | ✅ **fixed in 0.5.x** — a real `__all__` on the package root | -> | 9 no `py.typed` / handle Protocol | ✅ **fixed in 0.5.x** — `ida_codemode/py.typed` ships | -> -> **0.5.x restructured the package**, which is why the old "these files are -> byte-identical" re-check recipe no longer works: `client.py` → `handle.py`, -> `registry.py` → `_registry.py` + `instances.py`, `resolver.py` → `_resolver.py`, -> and the loader options moved into a frozen `DatabaseOpenOptions` dataclass. -> Everything private is now underscore-prefixed, so the cheap re-check after an -> upstream pull is simply: does anything we import still appear in -> `ida_codemode.__all__`? -> -> 0.5.3 → 0.6.1 changed **nothing** we depend on: `__init__.py`, `handle.py`, -> `instances.py`, `options.py`, `errors.py` and `models.py` are byte-identical -> between those two releases. 0.6.1 only collapses the six console scripts into a -> single `ida-codemode` command. -> -> Both client-side workarounds re-measured at **0.99x and 0.97x** on 0.3.2 — -> i.e. nothing — and are deleted. Remote code is now ordinary typed Python, -> installed as content-addressed modules by ida-codemode. Harness: -> `experiments/bench_pack_trace.py`. - -**What the client does**, for scale: it renders a continuous disassembly listing, -pseudocode, a CFG graph view and a hex view, paging over the database as the user -scrolls. It is latency-sensitive in a way an agent-driven MCP client is not — a -keypress must repaint. It issues ~1–8 operations per user action. - ---- - -## 1. `timeout_trace` enables line tracing in every frame — 52x on IDA calls - -**Highest-impact item by a wide margin.** — ✅ **FIXED in 0.3.2.** The runtime no -longer installs a trace hook at all; cancellation is a C-level thread interrupt. -Our `sys.settrace(None)` workaround is deleted as of `a5137fe`. - -`runtime.py` wraps every `execute_python` in `sys.settrace(timeout_trace)` to -enforce the deadline. `timeout_trace` ends with `return timeout_trace`, and -returning a trace function from a `'call'` event asks CPython to trace **every -line of that frame**. So every line of every function the snippet touches pays a -Python-level callback, and the specialising interpreter is disabled throughout. - -Measured inside the worker, same process, same database: - -| | traced (stock) | untraced | native idalib | -|---|---|---|---| -| `ida_bytes.get_flags(ea)` | 5.49 µs | 0.106 µs | 0.119 µs | -| our 200-row listing page | 20.2 ms | 2.0 ms | — | - -Untraced matches a plain idalib process, so the trace hook accounts for -essentially all of it. For us this was the single largest cost in the port — -larger than HTTP, serialisation and IDA itself combined. - -Reproduce inside any `execute_python`: - -```python -import sys, time, ida_bytes -def bench(): - t = time.perf_counter() - for _ in range(20000): ida_bytes.get_flags(0x1000) - return (time.perf_counter() - t) / 20000 * 1e6 -traced = bench() -old = sys.gettrace(); sys.settrace(None) -try: untraced = bench() -finally: sys.settrace(old) -result = {"traced_us": traced, "untraced_us": untraced} -``` - -**Suggested fixes, cheapest first** - -1. `return None` from `timeout_trace` instead of itself. You keep `'call'`-event - deadline checks — which is enough to interrupt anything that calls a function - — and drop per-line tracing entirely. -2. On 3.12+, use `sys.monitoring` with only the events you need; it is designed - for exactly this and is far cheaper than `settrace`. -3. Or drop the trace and rely on the `threading.Timer` → - `ida_kernwin.set_cancelled()` path you already have, accepting that a - pure-Python loop with no calls in it cannot be interrupted. - -**Our workaround** (we would rather not ship it): the snippet detaches the trace -and restores it in a `finally`. That gives up deadline enforcement for -pure-Python loops inside our own code; your native cancel timer is unaffected and -still fires. Every client that does real work per call will eventually find this -and do the same, which is an argument for fixing it in the runtime. - ---- - -## 2. `to_jsonable` dominates any large result - -**FIXED in 0.3.2**, via the first suggested fix below: -`serialization.dumps_json` calls `json.dumps(value, default=to_jsonable)`, so a -JSON-safe result never enters the Python walker. Our packing workaround measured -0.97x and has been deleted. - -`execute_python` runs `to_jsonable()` over whatever the snippet returns. Our -answers are already JSON-safe and they are big — a 200-row listing page is -roughly 10k small objects. - -| | cost | -|---|---| -| `to_jsonable(page)` | 66.2 ms | -| `json.dumps(page, separators=(",",":"))` — same data | 0.58 ms | -| serialised size | 34.9 KB | - -That is 114x, and it was 72% of the page's total cost before we changed it. - -**Suggested fixes** - -- Fast-path values that are already JSON-safe (a cheap recursive type check that - bails to the original object beats rebuilding it), or -- let a snippet opt out by returning an already-serialised payload — a documented - envelope such as `{"__json__": "<...>"}`, or simply passing `str`/`bytes` - through untouched. - -**Retired workaround:** snippets used to `json.dumps` inside the database process -and return one string, which the client parsed. The typed remote API now owns -strict argument/result encoding, and ida-tui contains no generated script -strings or packing envelope. - ---- - -## 3. The per-operation floor is `execute_sync`, not HTTP - -✅ **FIXED in 0.3.2 — 7.0x.** Re-measured as a same-box A/B by checking the -installed editable checkout back to `4195f21` and forward again, 200 iterations -each, `targets/echo`: - -| | 0.3.1 | 0.3.2 | | -|---|---|---|---| -| `GET /health` | 0.497 ms | 0.318 ms | 1.6x | -| `execute_python("result = 1")` | **2.055 ms** | **0.294 ms** | **7.0x** | - -The 0.3.1 column reproduces the original 2.025 ms measurement below almost -exactly, which is what makes the 0.3.2 column believable. `execute_python` now -costs about the same as a bare HTTP GET, so the `execute_sync` marshalling that -was ~93% of the floor is essentially gone. The design advice below — "a client -that makes one call per row will be 20–100x slower than an in-process one" — is -correspondingly much weaker now. - -Original 0.3.1 measurement, same worker, same connection, 200 iterations: - -| | cost | -|---|---| -| `GET /health` (no `execute_sync`) | **0.165 ms** | -| `execute_python("result = 1")` | **2.025 ms** | - -HTTP framing is ~7% of the floor; marshalling the operation onto IDA's main -thread is the other ~93%. The worker runs IDA's own `kernwin.serve()`, so this is -plausibly IDA's dispatch latency rather than anything you control — but it is -worth **documenting**, because it sets a hard 2 ms per-operation budget that -shapes how a client must be designed. - -It did not hurt us (our call volume is 1–8 per user action; 4 calls to build a -1060-block graph), but a client that makes one call per row or per symbol will be -20–100x slower than an in-process one and the authors will not know why. - -**Suggested fixes:** document the floor; and consider a batch endpoint — accept -`[{op, args}, ...]` and dispatch them within a single `execute_sync` — which -would let chatty clients amortise it without redesigning around it. - ---- - -## 4. Loader switches on an existing database are a FATAL, not an error — one edge remains - -Opening a target that already has an `.i64`, while passing spawn-only options, -kills the worker: - -``` -FATAL ERROR: @0:636[] -Switch '-b400' can be used only when loading a new file -``` - -The client sees only: - -``` -IDAConnectionError: idalib worker launcher exited with status 1 -``` - -This is easy to hit and hard to diagnose: it is the natural second run of -anything that opens a raw blob (`processor=`/`image_base=`/`file_type=` are -recorded in the database the first run produced). Our test suite hit it as a -crash five minutes into a run. - -**Suggested fixes** - -- In `DatabaseHandle.open()`, when the resolved IDB already exists and - `new_database` is not set, either ignore the spawn-only options or raise a - typed error naming them — before handing them to IDA. -- Propagate the worker's fatal text into the client exception. The message - already exists on the worker's stderr; losing it turns a one-line fix into a - bisect. - -**Our workaround:** the client checked whether the expected IDB exists and -dropped `processor`/`image_base`/`file_type` when it did. - -**FIXED in 0.5.x**, with exactly this fix, in `_resolver._build_worker_command`: - -```python -if input_path == expected_idb and input_path != source: - # Loader/import switches are baked into an existing IDB... - options = WorkerLaunchOptions() -``` - -Our workaround is therefore deleted. **One narrow case remains**: the strip needs -`input_path != source`, so passing an `.i64` path *directly* together with load -options (`ida-tui foo.i64 --processor arm`) still forwards the switches and still -fatals. Our old guard keyed on "the target IDB exists" and so covered it. It is a -nonsense invocation and no ida-tui code path generates it — the project layer -always passes `output_database`, and `_needs_load_options` bails when an `.i64` -exists — but the library boundary should still reject or normalize it rather -than launch a known-fatal IDA command. Tracked upstream as -[issue #36](https://github.com/HexRaysSA/ida-codemode/issues/36). - ---- - -## 5. Deleting or replacing an IDB under a live lease — STALE - -The original suite deleted an `.i64` while a private worker still had it open, -then immediately reopened the same path. That ownership model no longer applies: -Code Mode databases are shared resources, and the IDA GUI itself does not survive -out-of-band replacement of its open database. Detecting arbitrary filesystem -replacement is therefore not part of the supported lifecycle. - -The actionable lifecycle gaps that originally forced private-registry access are -fixed. `find_database_owner()` and `wait_database_released()` are public exports; -`DatabaseHandle.close(wait_for_database=True)` can wait for a final managed close; -a draining owner remains registered until the IDB is actually closed; and -`new_database=True` refuses to replace a live owner. - -ida-tui now uses the public owner/release API while recreating a database and no -longer reaches into registry locks. Owner loss is attach-only: ida-tui will -rediscover a replacement GUI or worker, but will never turn a -user-closing-the-GUI action into an implicit headless reopen. There is no -remaining upstream request in this section. - ---- - -## 6. Close without save — FIXED in protocol 6 - -`DatabaseHandle.shutdown_database(save=False)` can discard a managed idalib -worker when the requesting handle is its only active lease and no other operation -is running. The server rejects GUI databases and shared workers. - -The coherent ownership model is the **final lease**, not necessarily the lease -that spawned the worker. Releasing a non-final lease makes no whole-database save -decision; responsibility transfers to the leases that remain. The final client -can save or discard the shared session. A client that needs its work to survive -regardless of that later decision must call `save_database()` before releasing -its lease. - -This does not claim to provide per-client rollback. Discard applies to all -changes since the last database save, and attempting it while another lease is -active is correctly rejected. That is the same ref-counted lifetime model used -by other shared resources and requires no separate starter capability. - -The upstream gap is therefore closed. ida-tui now routes its discard action -through `shutdown_database(save=False)`: a final managed-worker lease discards, -while GUI-backed and still-shared sessions transfer finalization to their owner -or remaining leases. - ---- - -## 7. No change notification for shared databases — FIXED in protocol 6 - -`DatabaseHandle.subscribe_idb_events()` now returns a closeable iterator over -structured IDB changes. Each event carries a monotonic revision plus -`operation_id`/`operation_label` attribution and an opaque `origin_id`. -`DatabaseHandle.owns_event()` compares that origin with the handle's lease, so a -caching client does not need to generate, retain, or race operation IDs itself. - -ida-tui keeps one subscription for its active database, asks the handle to drop -its own events, and batches peer events behind a 200 ms quiet period. One batch -invalidates the function, listing, decompiler, graph, strings, linkage, segment -and byte caches, then reloads the visible view in place. Closing or switching -databases closes the subscription, so the blocking event reader does not leak. - ---- - -## 8. Package exports and API surface stability — FIXED in 0.5.x - -`ida_codemode/__init__.py` used to export nothing, so a library consumer had to -import from submodules, including things that were clearly internals (`FileLock`, -`REGISTRY_DIR`, `canonical_path`, `idb_key`, `scan_instances`) that we only -touched because no public equivalent existed. - -**Suggested fix was:** export `DatabaseHandle` and the public exception types from -the package root, and mark the intended-public registry helpers explicitly. - -**That is what 0.5.x did.** Everything we need is now on the package root, and -the internals moved behind an underscore: - -```python -from ida_codemode import DatabaseHandle, DatabaseOpenOptions, DatabaseInstance -from ida_codemode import RemoteError, DatabaseBusyError, DatabaseDisconnectedError -from ida_codemode import discover_databases, find_database_owner, wait_database_released -``` - -The two lock-poking helpers we had reimplemented client-side -(`_wait_for_entry_release`) are now `wait_database_released()`, and our -registry-scanning ownership check is now `find_database_owner()`. Both are -deleted from our tree. Note `find_database_owner()` *raises* -`AmbiguousDatabaseError` where our scan silently took the first match — a -behaviour improvement, but callers need a handler. - ---- - -## 9. A testing note: `DatabaseHandle.open()`'s 30 keyword-only options — FIXED in 0.5.x - -The port we started from called `open(..., loading_address=...)`. The real -parameter is `image_base`. Every `connect()` would have raised `TypeError` on the -first call, and its contract tests passed anyway, because a hand-written fake -handle accepts `**kwargs`. - -Not a library bug — but with 30 keyword-only options it is a very easy mistake, -and it is invisible to exactly the offline tests people write. - -**Suggested fix:** ship `py.typed` and/or a `Protocol` for the handle, so a fake -can be checked against the real signature and a typo is caught statically. (We -added a test asserting our kwargs are a subset of -`inspect.signature(DatabaseHandle.open).parameters`, which is a poor substitute.) - -**0.5.x ships `ida_codemode/py.typed`**, and the 30 keyword-only options became a -frozen `DatabaseOpenOptions` dataclass — which is strictly better, because an -invented option name is now a `TypeError` at construction rather than something a -`**kwargs` fake swallows. Our subset test survives in two halves -(`_open_kwargs_are_real` for `open()`, `_option_fields_are_real` for the -dataclass fields), because the offline contract suite must keep running with no -`ida_codemode` installed at all and therefore still fakes both. - ---- - -## Priority, from a client author's view - -| # | item | impact | fixable by you? | -|---|---|---|---| -| ~~1~~ | ~~`timeout_trace` line tracing~~ | ~~52x on IDA calls~~ | ✅ fixed in 0.3.2 | -| ~~2~~ | ~~`to_jsonable` on large results~~ | ~~114x on serialisation~~ | ✅ fixed in 0.3.2 | -| ~~3~~ | ~~2 ms `execute_sync` floor~~ | ~~shapes client design~~ | ✅ fixed in 0.3.2, 7.0x | -| ~~7~~ | ~~no change/revision counter~~ | ~~correctness for shared editing~~ | ✅ fixed in protocol 6 | -| 4 | direct `.i64` forwards loader-only options | fatal worker startup | [issue #36](https://github.com/HexRaysSA/ida-codemode/issues/36) | -| ~~5~~ | ~~replaced/deleted IDB under lease~~ | ~~out-of-contract filesystem mutation~~ | **stale** | -| ~~6~~ | ~~no close without save~~ | ~~could not discard a managed session~~ | **fixed in protocol 6: final lease decides** | -| ~~8~~ | ~~package exports~~ | ~~forces internal imports~~ | ✅ fixed in 0.5.x | -| ~~9~~ | ~~typed handle for fakes~~ | ~~catches a whole bug class~~ | ✅ fixed in 0.5.x (`py.typed` + options dataclass) | - -Items 1 and 2 together were the difference between "the port is 35x slower than -the private worker it replaced" and "the port is within 2x, and faster on several -operations". Both are in the runtime, not in client code — which is why they are -worth fixing centrally rather than leaving each client to rediscover. - -**Both landed in 0.3.2**, along with item 3 — all three performance items are now -fixed upstream, and both client-side workarounds could be measured at parity and -retired. That is the outcome this document was written for. - -**What is left is entirely non-performance.** Items 6 through 9 are fixed, and -item 5 is stale because out-of-band replacement is not a supported lifecycle for -either Code Mode or the IDA GUI. One narrow piece remains: **4**, normalize or -reject loader-only options when the source is itself an existing `.i64` -([issue #36](https://github.com/HexRaysSA/ida-codemode/issues/36)). - -Happy to supply the benchmark harness (it is backend-agnostic and runs against -both our old worker and Code Mode), or to test a patch. diff --git a/docs/GRAPH_VIEW.md b/docs/GRAPH_VIEW.md index 3aced1f..0b24aa5 100644 --- a/docs/GRAPH_VIEW.md +++ b/docs/GRAPH_VIEW.md @@ -54,7 +54,7 @@ Growing a second disassembly renderer for graph mode would have been the real cost. The backend adds exactly one operation, `flowchart(addr)` in -`idatui/codemode_client.py`, which returns block ranges and typed edges — **not** +`idatui/nexus_client.py`, which returns block ranges and typed edges — **not** text. ## Two layout engines diff --git a/docs/NEXUS_UPSTREAM.md b/docs/NEXUS_UPSTREAM.md new file mode 100644 index 0000000..f4dcf91 --- /dev/null +++ b/docs/NEXUS_UPSTREAM.md @@ -0,0 +1,385 @@ +# Findings from porting a real client to IDA Nexus + +Notes for the `ida-nexus` maintainers, gathered while porting **ida-tui** (a +Textual TUI frontend for IDA) from a private idalib worker to +`ida_nexus.DatabaseHandle`. + +Everything below is measured, not inferred. Where we worked around something, the +workaround is named so you can judge whether the library should make it +unnecessary. + +**Environment:** ida-nexus 0.3.1, IDA 9.4 (idalib), Linux, single managed +worker backend, quiet box. Target for timings: `targets/echo` unless stated. + +> **Status against the protocol-6 event-stream development tree, based on 0.6.1 +> (upstream `439289f`) — every item re-checked.** +> +> | item | verdict | +> |---|---| +> | 1 `timeout_trace` line tracing | ✅ **fixed in 0.3.2** — no `settrace` in the runtime at all | +> | 2 `to_jsonable` on large results | ✅ **fixed in 0.3.2** — `dumps_json` C fast path | +> | 3 2 ms `execute_sync` floor | ✅ **fixed in 0.3.2, 7.0x** — 2.055 ms → 0.294 ms | +> | 4 loader switches fatal on reopen | **partial** — normal reopen fixed in 0.5.x; direct `.i64` paths remain [issue #36](https://github.com/HexRaysSA/ida-nexus/issues/36) | +> | 5 IDB replaced under a live lease | **stale** — out-of-band replacement is outside the supported lifecycle, as it is for the IDA GUI | +> | 6 close without save | **fixed in protocol 6** — the final managed-worker lease can choose `shutdown_database(save=False)` | +> | 7 no change notification | ✅ **fixed in protocol 6** — `DatabaseHandle.subscribe_idb_events()` streams revisioned, operation-attributed IDB changes | +> | 8 package exports | ✅ **fixed in 0.5.x** — a real `__all__` on the package root | +> | 9 no `py.typed` / handle Protocol | ✅ **fixed in 0.5.x** — `ida_nexus/py.typed` ships | +> +> **0.5.x restructured the package**, which is why the old "these files are +> byte-identical" re-check recipe no longer works: `client.py` → `handle.py`, +> `registry.py` → `_registry.py` + `instances.py`, `resolver.py` → `_resolver.py`, +> and the loader options moved into a frozen `DatabaseOpenOptions` dataclass. +> Everything private is now underscore-prefixed, so the cheap re-check after an +> upstream pull is simply: does anything we import still appear in +> `ida_nexus.__all__`? +> +> 0.5.3 → 0.6.1 changed **nothing** we depend on: `__init__.py`, `handle.py`, +> `instances.py`, `options.py`, `errors.py` and `models.py` are byte-identical +> between those two releases. 0.6.1 only collapses the six console scripts into a +> single `ida-nexus` command. +> +> Both client-side workarounds re-measured at **0.99x and 0.97x** on 0.3.2 — +> i.e. nothing — and are deleted. Remote code is now ordinary typed Python, +> installed as content-addressed modules by ida-nexus. Harness: +> `experiments/bench_pack_trace.py`. + +**What the client does**, for scale: it renders a continuous disassembly listing, +pseudocode, a CFG graph view and a hex view, paging over the database as the user +scrolls. It is latency-sensitive in a way an agent-driven MCP client is not — a +keypress must repaint. It issues ~1–8 operations per user action. + +--- + +## 1. `timeout_trace` enables line tracing in every frame — 52x on IDA calls + +**Highest-impact item by a wide margin.** — ✅ **FIXED in 0.3.2.** The runtime no +longer installs a trace hook at all; cancellation is a C-level thread interrupt. +Our `sys.settrace(None)` workaround is deleted as of `a5137fe`. + +`runtime.py` wraps every `execute_python` in `sys.settrace(timeout_trace)` to +enforce the deadline. `timeout_trace` ends with `return timeout_trace`, and +returning a trace function from a `'call'` event asks CPython to trace **every +line of that frame**. So every line of every function the snippet touches pays a +Python-level callback, and the specialising interpreter is disabled throughout. + +Measured inside the worker, same process, same database: + +| | traced (stock) | untraced | native idalib | +|---|---|---|---| +| `ida_bytes.get_flags(ea)` | 5.49 µs | 0.106 µs | 0.119 µs | +| our 200-row listing page | 20.2 ms | 2.0 ms | — | + +Untraced matches a plain idalib process, so the trace hook accounts for +essentially all of it. For us this was the single largest cost in the port — +larger than HTTP, serialisation and IDA itself combined. + +Reproduce inside any `execute_python`: + +```python +import sys, time, ida_bytes +def bench(): + t = time.perf_counter() + for _ in range(20000): ida_bytes.get_flags(0x1000) + return (time.perf_counter() - t) / 20000 * 1e6 +traced = bench() +old = sys.gettrace(); sys.settrace(None) +try: untraced = bench() +finally: sys.settrace(old) +result = {"traced_us": traced, "untraced_us": untraced} +``` + +**Suggested fixes, cheapest first** + +1. `return None` from `timeout_trace` instead of itself. You keep `'call'`-event + deadline checks — which is enough to interrupt anything that calls a function + — and drop per-line tracing entirely. +2. On 3.12+, use `sys.monitoring` with only the events you need; it is designed + for exactly this and is far cheaper than `settrace`. +3. Or drop the trace and rely on the `threading.Timer` → + `ida_kernwin.set_cancelled()` path you already have, accepting that a + pure-Python loop with no calls in it cannot be interrupted. + +**Our workaround** (we would rather not ship it): the snippet detaches the trace +and restores it in a `finally`. That gives up deadline enforcement for +pure-Python loops inside our own code; your native cancel timer is unaffected and +still fires. Every client that does real work per call will eventually find this +and do the same, which is an argument for fixing it in the runtime. + +--- + +## 2. `to_jsonable` dominates any large result + +**FIXED in 0.3.2**, via the first suggested fix below: +`serialization.dumps_json` calls `json.dumps(value, default=to_jsonable)`, so a +JSON-safe result never enters the Python walker. Our packing workaround measured +0.97x and has been deleted. + +`execute_python` runs `to_jsonable()` over whatever the snippet returns. Our +answers are already JSON-safe and they are big — a 200-row listing page is +roughly 10k small objects. + +| | cost | +|---|---| +| `to_jsonable(page)` | 66.2 ms | +| `json.dumps(page, separators=(",",":"))` — same data | 0.58 ms | +| serialised size | 34.9 KB | + +That is 114x, and it was 72% of the page's total cost before we changed it. + +**Suggested fixes** + +- Fast-path values that are already JSON-safe (a cheap recursive type check that + bails to the original object beats rebuilding it), or +- let a snippet opt out by returning an already-serialised payload — a documented + envelope such as `{"__json__": "<...>"}`, or simply passing `str`/`bytes` + through untouched. + +**Retired workaround:** snippets used to `json.dumps` inside the database process +and return one string, which the client parsed. The typed remote API now owns +strict argument/result encoding, and ida-tui contains no generated script +strings or packing envelope. + +--- + +## 3. The per-operation floor is `execute_sync`, not HTTP + +✅ **FIXED in 0.3.2 — 7.0x.** Re-measured as a same-box A/B by checking the +installed editable checkout back to `4195f21` and forward again, 200 iterations +each, `targets/echo`: + +| | 0.3.1 | 0.3.2 | | +|---|---|---|---| +| `GET /health` | 0.497 ms | 0.318 ms | 1.6x | +| `execute_python("result = 1")` | **2.055 ms** | **0.294 ms** | **7.0x** | + +The 0.3.1 column reproduces the original 2.025 ms measurement below almost +exactly, which is what makes the 0.3.2 column believable. `execute_python` now +costs about the same as a bare HTTP GET, so the `execute_sync` marshalling that +was ~93% of the floor is essentially gone. The design advice below — "a client +that makes one call per row will be 20–100x slower than an in-process one" — is +correspondingly much weaker now. + +Original 0.3.1 measurement, same worker, same connection, 200 iterations: + +| | cost | +|---|---| +| `GET /health` (no `execute_sync`) | **0.165 ms** | +| `execute_python("result = 1")` | **2.025 ms** | + +HTTP framing is ~7% of the floor; marshalling the operation onto IDA's main +thread is the other ~93%. The worker runs IDA's own `kernwin.serve()`, so this is +plausibly IDA's dispatch latency rather than anything you control — but it is +worth **documenting**, because it sets a hard 2 ms per-operation budget that +shapes how a client must be designed. + +It did not hurt us (our call volume is 1–8 per user action; 4 calls to build a +1060-block graph), but a client that makes one call per row or per symbol will be +20–100x slower than an in-process one and the authors will not know why. + +**Suggested fixes:** document the floor; and consider a batch endpoint — accept +`[{op, args}, ...]` and dispatch them within a single `execute_sync` — which +would let chatty clients amortise it without redesigning around it. + +--- + +## 4. Loader switches on an existing database are a FATAL, not an error — one edge remains + +Opening a target that already has an `.i64`, while passing spawn-only options, +kills the worker: + +``` +FATAL ERROR: @0:636[] +Switch '-b400' can be used only when loading a new file +``` + +The client sees only: + +``` +IDAConnectionError: idalib worker launcher exited with status 1 +``` + +This is easy to hit and hard to diagnose: it is the natural second run of +anything that opens a raw blob (`processor=`/`image_base=`/`file_type=` are +recorded in the database the first run produced). Our test suite hit it as a +crash five minutes into a run. + +**Suggested fixes** + +- In `DatabaseHandle.open()`, when the resolved IDB already exists and + `new_database` is not set, either ignore the spawn-only options or raise a + typed error naming them — before handing them to IDA. +- Propagate the worker's fatal text into the client exception. The message + already exists on the worker's stderr; losing it turns a one-line fix into a + bisect. + +**Our workaround:** the client checked whether the expected IDB exists and +dropped `processor`/`image_base`/`file_type` when it did. + +**FIXED in 0.5.x**, with exactly this fix, in `_resolver._build_worker_command`: + +```python +if input_path == expected_idb and input_path != source: + # Loader/import switches are baked into an existing IDB... + options = WorkerLaunchOptions() +``` + +Our workaround is therefore deleted. **One narrow case remains**: the strip needs +`input_path != source`, so passing an `.i64` path *directly* together with load +options (`ida-tui foo.i64 --processor arm`) still forwards the switches and still +fatals. Our old guard keyed on "the target IDB exists" and so covered it. It is a +nonsense invocation and no ida-tui code path generates it — the project layer +always passes `output_database`, and `_needs_load_options` bails when an `.i64` +exists — but the library boundary should still reject or normalize it rather +than launch a known-fatal IDA command. Tracked upstream as +[issue #36](https://github.com/HexRaysSA/ida-nexus/issues/36). + +--- + +## 5. Deleting or replacing an IDB under a live lease — STALE + +The original suite deleted an `.i64` while a private worker still had it open, +then immediately reopened the same path. That ownership model no longer applies: +IDA Nexus databases are shared resources, and the IDA GUI itself does not survive +out-of-band replacement of its open database. Detecting arbitrary filesystem +replacement is therefore not part of the supported lifecycle. + +The actionable lifecycle gaps that originally forced private-registry access are +fixed. `find_database_owner()` and `wait_database_released()` are public exports; +`DatabaseHandle.close(wait_for_database=True)` can wait for a final managed close; +a draining owner remains registered until the IDB is actually closed; and +`new_database=True` refuses to replace a live owner. + +ida-tui now uses the public owner/release API while recreating a database and no +longer reaches into registry locks. Owner loss is attach-only: ida-tui will +rediscover a replacement GUI or worker, but will never turn a +user-closing-the-GUI action into an implicit headless reopen. There is no +remaining upstream request in this section. + +--- + +## 6. Close without save — FIXED in protocol 6 + +`DatabaseHandle.shutdown_database(save=False)` can discard a managed idalib +worker when the requesting handle is its only active lease and no other operation +is running. The server rejects GUI databases and shared workers. + +The coherent ownership model is the **final lease**, not necessarily the lease +that spawned the worker. Releasing a non-final lease makes no whole-database save +decision; responsibility transfers to the leases that remain. The final client +can save or discard the shared session. A client that needs its work to survive +regardless of that later decision must call `save_database()` before releasing +its lease. + +This does not claim to provide per-client rollback. Discard applies to all +changes since the last database save, and attempting it while another lease is +active is correctly rejected. That is the same ref-counted lifetime model used +by other shared resources and requires no separate starter capability. + +The upstream gap is therefore closed. ida-tui now routes its discard action +through `shutdown_database(save=False)`: a final managed-worker lease discards, +while GUI-backed and still-shared sessions transfer finalization to their owner +or remaining leases. + +--- + +## 7. No change notification for shared databases — FIXED in protocol 6 + +`DatabaseHandle.subscribe_idb_events()` now returns a closeable iterator over +structured IDB changes. Each event carries a monotonic revision plus +`operation_id`/`operation_label` attribution and an opaque `origin_id`. +`DatabaseHandle.owns_event()` compares that origin with the handle's lease, so a +caching client does not need to generate, retain, or race operation IDs itself. + +ida-tui keeps one subscription for its active database, asks the handle to drop +its own events, and batches peer events behind a 200 ms quiet period. One batch +invalidates the function, listing, decompiler, graph, strings, linkage, segment +and byte caches, then reloads the visible view in place. Closing or switching +databases closes the subscription, so the blocking event reader does not leak. + +--- + +## 8. Package exports and API surface stability — FIXED in 0.5.x + +`ida_nexus/__init__.py` used to export nothing, so a library consumer had to +import from submodules, including things that were clearly internals (`FileLock`, +`REGISTRY_DIR`, `canonical_path`, `idb_key`, `scan_instances`) that we only +touched because no public equivalent existed. + +**Suggested fix was:** export `DatabaseHandle` and the public exception types from +the package root, and mark the intended-public registry helpers explicitly. + +**That is what 0.5.x did.** Everything we need is now on the package root, and +the internals moved behind an underscore: + +```python +from ida_nexus import DatabaseHandle, DatabaseOpenOptions, DatabaseInstance +from ida_nexus import RemoteError, DatabaseBusyError, DatabaseDisconnectedError +from ida_nexus import discover_databases, find_database_owner, wait_database_released +``` + +The two lock-poking helpers we had reimplemented client-side +(`_wait_for_entry_release`) are now `wait_database_released()`, and our +registry-scanning ownership check is now `find_database_owner()`. Both are +deleted from our tree. Note `find_database_owner()` *raises* +`AmbiguousDatabaseError` where our scan silently took the first match — a +behaviour improvement, but callers need a handler. + +--- + +## 9. A testing note: `DatabaseHandle.open()`'s 30 keyword-only options — FIXED in 0.5.x + +The port we started from called `open(..., loading_address=...)`. The real +parameter is `image_base`. Every `connect()` would have raised `TypeError` on the +first call, and its contract tests passed anyway, because a hand-written fake +handle accepts `**kwargs`. + +Not a library bug — but with 30 keyword-only options it is a very easy mistake, +and it is invisible to exactly the offline tests people write. + +**Suggested fix:** ship `py.typed` and/or a `Protocol` for the handle, so a fake +can be checked against the real signature and a typo is caught statically. (We +added a test asserting our kwargs are a subset of +`inspect.signature(DatabaseHandle.open).parameters`, which is a poor substitute.) + +**0.5.x ships `ida_nexus/py.typed`**, and the 30 keyword-only options became a +frozen `DatabaseOpenOptions` dataclass — which is strictly better, because an +invented option name is now a `TypeError` at construction rather than something a +`**kwargs` fake swallows. Our subset test survives in two halves +(`_open_kwargs_are_real` for `open()`, `_option_fields_are_real` for the +dataclass fields), because the offline contract suite must keep running with no +`ida_nexus` installed at all and therefore still fakes both. + +--- + +## Priority, from a client author's view + +| # | item | impact | fixable by you? | +|---|---|---|---| +| ~~1~~ | ~~`timeout_trace` line tracing~~ | ~~52x on IDA calls~~ | ✅ fixed in 0.3.2 | +| ~~2~~ | ~~`to_jsonable` on large results~~ | ~~114x on serialisation~~ | ✅ fixed in 0.3.2 | +| ~~3~~ | ~~2 ms `execute_sync` floor~~ | ~~shapes client design~~ | ✅ fixed in 0.3.2, 7.0x | +| ~~7~~ | ~~no change/revision counter~~ | ~~correctness for shared editing~~ | ✅ fixed in protocol 6 | +| 4 | direct `.i64` forwards loader-only options | fatal worker startup | [issue #36](https://github.com/HexRaysSA/ida-nexus/issues/36) | +| ~~5~~ | ~~replaced/deleted IDB under lease~~ | ~~out-of-contract filesystem mutation~~ | **stale** | +| ~~6~~ | ~~no close without save~~ | ~~could not discard a managed session~~ | **fixed in protocol 6: final lease decides** | +| ~~8~~ | ~~package exports~~ | ~~forces internal imports~~ | ✅ fixed in 0.5.x | +| ~~9~~ | ~~typed handle for fakes~~ | ~~catches a whole bug class~~ | ✅ fixed in 0.5.x (`py.typed` + options dataclass) | + +Items 1 and 2 together were the difference between "the port is 35x slower than +the private worker it replaced" and "the port is within 2x, and faster on several +operations". Both are in the runtime, not in client code — which is why they are +worth fixing centrally rather than leaving each client to rediscover. + +**Both landed in 0.3.2**, along with item 3 — all three performance items are now +fixed upstream, and both client-side workarounds could be measured at parity and +retired. That is the outcome this document was written for. + +**What is left is entirely non-performance.** Items 6 through 9 are fixed, and +item 5 is stale because out-of-band replacement is not a supported lifecycle for +either IDA Nexus or the IDA GUI. One narrow piece remains: **4**, normalize or +reject loader-only options when the source is itself an existing `.i64` +([issue #36](https://github.com/HexRaysSA/ida-nexus/issues/36)). + +Happy to supply the benchmark harness (it is backend-agnostic and runs against +both our old worker and IDA Nexus), or to test a patch. diff --git a/docs/PAGING_FINDINGS.md b/docs/PAGING_FINDINGS.md index bd6c38f..f343973 100644 --- a/docs/PAGING_FINDINGS.md +++ b/docs/PAGING_FINDINGS.md @@ -3,9 +3,9 @@ Measured against a real target: `libcrypto.so.3` (5.7 MB, **10,092 functions**, biggest function **52,120 instructions**). These constraints drive the domain / paging layer. The measurements below came from the former ida-pro-mcp tool -backend. The Code Mode port preserves the adapter response shapes and conservative +backend. The IDA Nexus port preserves the adapter response shapes and conservative page sizes, but executes enumeration through ida-domain; old server caps and RTT -numbers are historical rather than Code Mode constraints. +numbers are historical rather than IDA Nexus constraints. ## Response shape (list_* / *_query tools) @@ -93,17 +93,17 @@ disasm totals are **top-level** fields, not under `asm`: (correct). The pseudocode view must handle "decompilation failed" gracefully — fall back to the disassembly view or show an error panel. -Code Mode returns the complete execution result directly; ida-tui no longer +IDA Nexus returns the complete execution result directly; ida-tui no longer needs MCP structured-content/download-URL recovery for large pseudocode bodies. -## Code Mode lifecycle +## IDA Nexus lifecycle -`CodeModeClient` owns an authenticated SSE lease on a registered database: +`NexusClient` owns an authenticated SSE lease on a registered database: * A matching GUI is preferred and remains open when the TUI exits. -* Otherwise Code Mode reuses or starts a shared managed idalib worker. +* Otherwise IDA Nexus reuses or starts a shared managed idalib worker. * Releasing one lease never terminates another client's session. A managed - worker saves and exits after its final lease under Code Mode's grace policy. + worker saves and exits after its final lease under IDA Nexus's grace policy. * Lease loss surfaces as `IDAConnectionError`; reconnect performs discovery again and may bind a newly-created instance. It does not silently swap the handle underneath an operation. diff --git a/docs/PROJECTS.md b/docs/PROJECTS.md index 0efee31..52dbf83 100644 --- a/docs/PROJECTS.md +++ b/docs/PROJECTS.md @@ -7,7 +7,7 @@ search across all of them, and (later) follow calls from one into another. ## The constraint that shapes everything -IDA still exposes one active database per GUI/idalib process. Code Mode makes +IDA still exposes one active database per GUI/idalib process. IDA Nexus makes those instances discoverable and shareable: each project entry retains one `DatabaseHandle` lease, which may target a registered GUI or a managed idalib worker. N resident project databases can therefore mean up to N processes, but @@ -32,13 +32,13 @@ crypto library. Two capabilities that feel like one, but aren't: -1. **Switching** to a binary needs a *live Code Mode lease*. +1. **Switching** to a binary needs a *live IDA Nexus lease*. 2. **Searching across** binaries does *not* — if a per-binary index (functions, strings, imports/exports) is cached on disk. That split is the unlock: project-wide search stays instant across every binary, including ones never opened this session, and only *jumping* to a hit costs a -Code Mode attach/open. +IDA Nexus attach/open. ## Layout @@ -85,11 +85,11 @@ basename and must be unique (it names the staged file). ## Runtime -- **`DatabasePool`** — one `CodeModeClient` lease per resident binary, attached +- **`DatabasePool`** — one `NexusClient` lease per resident binary, attached lazily on first switch and LRU-released when the advisory memory budget is exceeded. Eviction explicitly saves managed IDBs but never implicitly saves a GUI. Closing a lease never kills a GUI or another client's managed worker; - Code Mode owns final worker shutdown. + IDA Nexus owns final worker shutdown. - **`BinaryState`** — per binary: `client, program, nav, cur, func_index, pref/active/split, filter`. Switching snapshots the current state and restores the target's. `_after_reconnect` provides the client/program swap seam. diff --git a/docs/SPLIT_VIEW.md b/docs/SPLIT_VIEW.md index c421656..6e9b938 100644 --- a/docs/SPLIT_VIEW.md +++ b/docs/SPLIT_VIEW.md @@ -32,7 +32,7 @@ known technique: The old ida-pro-mcp backend derived the per-line marker via `cfunc.get_line_item(line, col=0, …).get_ea()`. To get the **full set**, sweep every column of the line (`get_line_item(line, x, …).get_ea()` for `x` in -`0..len`) and collect distinct non-`BADADDR` EAs. The Code Mode adapter's +`0..len`) and collect distinct non-`BADADDR` EAs. The IDA Nexus adapter's `decomp_map(ea)` operation returns `[{line, primary_ea, eas:[…]}, …]`; invert for `ea → line`. @@ -78,14 +78,14 @@ decomp→listing uses `ListingModel.ensure_ea`. Tab re-links from the new driver Still single-ea per line (one instruction highlighted); the region comes in phase 3. -**Phase 3 — rich highlight. DONE.** The Code Mode `decomp_map` operation -(`idatui/codemode_client.py`) sweeps `cfunc.get_line_item` across every column of +**Phase 3 — rich highlight. DONE.** The IDA Nexus `decomp_map` operation +(`idatui/nexus_client.py`) sweeps `cfunc.get_line_item` across every column of each pseudocode line and collects the EAs from each item's `dstr()` (`'EA: desc'` — the same source as the `/*ea*/` marker, so it aligns). `Program.decomp_map(ea)` returns the per-line ea lists (cached by name-gen); the app loads it async into `_split_eamap` / `_split_ea2line` and `_sync_split` bands the **whole** instruction region of a C line (and uses the exact ea→line inverse for the reverse). Falls -back to the single marker until the map lands. Verified on a real Code Mode database +back to the single marker until the map lands. Verified on a real IDA Nexus database (alignment + multi-instruction region band). **Phase 4 — polish. DONE.** diff --git a/experiments/bench_ops.py b/experiments/bench_ops.py index 9cc51fa..1c0ad69 100644 --- a/experiments/bench_ops.py +++ b/experiments/bench_ops.py @@ -1,4 +1,4 @@ -"""Time a realistic idatui operation mix against whatever ida-codemode is installed. +"""Time a realistic idatui operation mix against whatever ida-nexus is installed. The companion to `bench_pack_trace.py`: that one isolates a single workaround, this one answers "how much faster is the whole client, on real operations". @@ -14,16 +14,16 @@ replace or delete it:: PYTHONPATH=. ~/ida-venv/bin/python /tmp/bench_ops.py # B: current client against the OLD library (shows what the workarounds were for) - git -C ~/dev/ida-codemode checkout 4195f21 + git -C ~/dev/ida-nexus checkout 4195f21 PYTHONPATH=. ~/ida-venv/bin/python /tmp/bench_ops.py # A: the client as it SHIPPED on the old library, workarounds and all git checkout 8550474 # the commit before the workaround removal PYTHONPATH=. ~/ida-venv/bin/python /tmp/bench_ops.py - git checkout main && git -C ~/dev/ida-codemode checkout main # ALWAYS restore + git checkout main && git -C ~/dev/ida-nexus checkout main # ALWAYS restore -ida-codemode is installed **editable** into both venvs, so checking that repo out +ida-nexus is installed **editable** into both venvs, so checking that repo out swaps the backend under the TUI with no reinstall -- which is what makes this A/B cheap. """ @@ -36,7 +36,7 @@ import statistics import time from idatui import remote_ops -from idatui.codemode_client import CodeModeClient +from idatui.nexus_client import NexusClient def bench(fn, reps: int) -> tuple[float, float]: @@ -55,7 +55,7 @@ def main() -> int: ap.add_argument("--reps", type=int, default=20) args = ap.parse_args() - client = CodeModeClient(os.path.abspath(args.target)) + client = NexusClient(os.path.abspath(args.target)) client.connect() handle = client._handle diff --git a/experiments/bench_pack_trace.py b/experiments/bench_pack_trace.py index edbb9f7..2b53afd 100644 --- a/experiments/bench_pack_trace.py +++ b/experiments/bench_pack_trace.py @@ -17,7 +17,7 @@ import statistics import time from idatui import remote_ops -from idatui.codemode_client import CodeModeClient +from idatui.nexus_client import NexusClient def timed(function, reps: int = 1) -> tuple[object, float]: @@ -37,7 +37,7 @@ def main() -> int: parser.add_argument("--rows", type=int, default=200) args = parser.parse_args() - client = CodeModeClient(os.path.abspath(args.target)).connect() + client = NexusClient(os.path.abspath(args.target)).connect() index, operations_cold = timed( lambda: client.call(remote_ops.list_funcs, queries=[{"offset": 0, "count": 40}]) diff --git a/experiments/call_census.py b/experiments/call_census.py index 1680691..17a5f94 100644 --- a/experiments/call_census.py +++ b/experiments/call_census.py @@ -1,7 +1,7 @@ """Count backend round-trips per user action. Answers "are we batching, or paying a round-trip per item?" with numbers rather -than intent. Wraps ``CodeModeClient.invoke`` on the live app, drives a headless +than intent. Wraps ``NexusClient.invoke`` on the live app, drives a headless Pilot through realistic actions, and reports calls + wall time + which operations were used for each. @@ -25,7 +25,7 @@ from _fixtures import fast_keys, staged # noqa: E402 fast_keys() from idatui.app import IdaTui, ListingView # noqa: E402 -from idatui.codemode_client import CodeModeClient # noqa: E402 +from idatui.nexus_client import NexusClient # noqa: E402 class Census: @@ -34,18 +34,18 @@ class Census: def __init__(self) -> None: self.ops: collections.Counter = collections.Counter() self.n = 0 - original = CodeModeClient.invoke + original = NexusClient.invoke def counting(client, operation, *a, **kw): self.n += 1 self.ops[operation] += 1 return original(client, operation, *a, **kw) - CodeModeClient.invoke = counting + NexusClient.invoke = counting self._original = original def restore(self) -> None: - CodeModeClient.invoke = self._original + NexusClient.invoke = self._original def span(self, label: str): return _Span(self, label) diff --git a/experiments/profile_client.py b/experiments/profile_client.py index bac130e..84abd8a 100644 --- a/experiments/profile_client.py +++ b/experiments/profile_client.py @@ -21,7 +21,7 @@ import pstats import time from idatui import remote_ops -from idatui.codemode_client import CodeModeClient +from idatui.nexus_client import NexusClient from idatui.domain import Program @@ -35,7 +35,7 @@ def main() -> int: ) args = ap.parse_args() - client = CodeModeClient(os.path.abspath(args.binary)) + client = NexusClient(os.path.abspath(args.binary)) client.connect() program = Program(client) regions = client.call(remote_ops.file_regions) diff --git a/experiments/profile_remote.py b/experiments/profile_remote.py index 97f4fb1..fbf17dc 100644 --- a/experiments/profile_remote.py +++ b/experiments/profile_remote.py @@ -15,7 +15,7 @@ import argparse import os from idatui import remote_ops -from idatui.codemode_client import CodeModeClient +from idatui.nexus_client import NexusClient CALLS = { "heads": ("heads", {"count": 500, "annotate": True}), @@ -36,7 +36,7 @@ def main() -> int: parser.add_argument("--addr", default=None, help="default: the .text start") args = parser.parse_args() - client = CodeModeClient(os.path.abspath(args.binary)).connect() + client = NexusClient(os.path.abspath(args.binary)).connect() addr = args.addr if addr is None: regions = client.call(remote_ops.file_regions) diff --git a/experiments/worker_smoke.py b/experiments/worker_smoke.py index 9b55138..1c2125f 100644 --- a/experiments/worker_smoke.py +++ b/experiments/worker_smoke.py @@ -1,6 +1,6 @@ -"""Exercise the real domain.Program through an IDA Code Mode lease. +"""Exercise the real domain.Program through an IDA Nexus lease. -A matching registered GUI is reused; otherwise Code Mode starts a managed +A matching registered GUI is reused; otherwise IDA Nexus starts a managed idalib worker. Usage: ``uv run python experiments/worker_smoke.py FILE``. """ from __future__ import annotations @@ -9,15 +9,15 @@ import os import sys import time -from idatui.codemode_client import CodeModeClient +from idatui.nexus_client import NexusClient from idatui.domain import Program def main() -> int: target = os.path.abspath(sys.argv[1] if len(sys.argv) > 1 else "experiments/fibonacci.elf") - print(f"attaching Code Mode to {target}…", flush=True) + print(f"attaching IDA Nexus to {target}…", flush=True) started = time.time() - client = CodeModeClient(target) + client = NexusClient(target) client.connect(progress=lambda message: print(f" {message}", flush=True)) print( f" ready in {time.time() - started:.2f}s; backend={client.backend}; " diff --git a/ida-tui b/ida-tui index a488cac..de60a45 100755 --- a/ida-tui +++ b/ida-tui @@ -4,7 +4,7 @@ # ./ida-tui foo.elf # open a binary and drive it — that's it # # The launcher leases a registered IDA GUI or shared managed idalib worker -# through ida_codemode. The selected Python must have ida-tui's dependencies; +# through ida_nexus. The selected Python must have ida-tui's dependencies; # override it with $IDATUI_PYTHON. set -eu diff --git a/idatui/__init__.py b/idatui/__init__.py index 7da28b3..e89d8f6 100644 --- a/idatui/__init__.py +++ b/idatui/__init__.py @@ -1,4 +1,4 @@ -"""idatui — a keyboard-first TUI using shared IDA Code Mode databases.""" +"""idatui — a keyboard-first TUI using shared IDA Nexus databases.""" from .errors import ( IDAError, @@ -10,7 +10,7 @@ from .errors import ( IDASessionError, Session, ) -from .codemode_client import CodeModeClient +from .nexus_client import NexusClient from .domain import ( Program, FunctionIndex, @@ -25,7 +25,7 @@ from .domain import ( ) __all__ = [ - "CodeModeClient", + "NexusClient", "Program", "FunctionIndex", "DisasmModel", diff --git a/idatui/app.py b/idatui/app.py index e4c2936..05b428a 100644 --- a/idatui/app.py +++ b/idatui/app.py @@ -10,7 +10,7 @@ Design notes: without ever materializing 52k lines in a widget. * All network/domain work runs in Textual worker threads; the UI never blocks. * An address-history stack backs Enter (follow) / Esc (back), IDA-style. -* Database lifecycle is lease-based through ida_codemode: matching GUI sessions +* Database lifecycle is lease-based through ida_nexus: matching GUI sessions are reused, otherwise a shared managed idalib worker is opened on demand. """ @@ -55,7 +55,7 @@ from .highlight import CTextArea, highlight_c from .journal import Journal from .errors import IDAConnectionError -from .codemode_client import CodeModeClient, registered_database +from .nexus_client import NexusClient, registered_database from .domain import Func, Head, ListingModel, Program, Struct # Styles for the disassembly listing. @@ -1087,7 +1087,7 @@ class ListingView(SearchMixin, NavMixin, ColumnCursor, ScrollView, can_focus=Tru def _span_segments(h: Head, fallback: Style): """Segments for a row's disassembly text. - Uses IDA's own token classification when Code Mode supplies it; falls + Uses IDA's own token classification when IDA Nexus supplies it; falls back to the mnemonic/rest split when spans are absent or disagree with the plain text. """ @@ -5238,7 +5238,7 @@ class IdaTui(App): self._open_path = open_path self._ttl = ttl self._load_args = load_args or "" # first-open options for a headerless blob - self._new_database = False # Ctrl+L asks Code Mode for a fresh IDB + self._new_database = False # Ctrl+L asks IDA Nexus for a fresh IDB self._title = (os.path.basename(open_path) if open_path else "") #: Where we are in the execution trace, and everything that moves us. #: Owns the trace state; the _trace/_t/_trail_* properties below @@ -5247,7 +5247,7 @@ class IdaTui(App): self._do_keepalive = keepalive self._rpc_path = rpc_path self._rpc = None - self.client: CodeModeClient | None = None + self.client: NexusClient | None = None self.program: Program | None = None self._loading_screen: LoadingScreen | None = None self._ka = None @@ -5288,7 +5288,7 @@ class IdaTui(App): self.journal = Journal() self._xref_focus_name: str | None = None self._dirty = False - # One subscription for the active database. CodeModeClient debounces + # One subscription for the active database. NexusClient debounces # bursts off the Textual worker pool; the callback re-enters here on the # UI thread to invalidate and reload the visible models. self._idb_event_watch = None @@ -5362,7 +5362,7 @@ class IdaTui(App): if self._rpc_path: self._start_rpc() # A file no loader recognises has to be described before it can be - # opened, so ask BEFORE Code Mode creates it — once IDA has made a database + # opened, so ask BEFORE IDA Nexus creates it — once IDA has made a database # the answer is baked in and changing it requires a fresh-IDB reopen. if self._project is not None: ref = self._pending_load_ref() @@ -5435,7 +5435,7 @@ class IdaTui(App): ref = self._project.by_label(self._binary) if ref is not None: path, label = ref.source, ref.label - # Release our lease first. Code Mode waits for a managed worker's final + # Release our lease first. IDA Nexus waits for a managed worker's final # lease grace, then creates the replacement IDB atomically. A GUI-backed # database is rejected by _can_reload(): the TUI must never close it. self._release_database() @@ -5634,7 +5634,7 @@ class IdaTui(App): return len(text) # -- live refresh from shared IDB changes ----------------------------- # - def _start_idb_event_watch(self, client: CodeModeClient) -> None: + def _start_idb_event_watch(self, client: NexusClient) -> None: self._stop_idb_event_watch() watch = getattr(client, "watch_idb_events", None) if watch is None: # IDA-free test doubles and pre-event adapters @@ -5661,7 +5661,7 @@ class IdaTui(App): watcher.close() def _idb_event_watch_failed( - self, client: CodeModeClient, error: BaseException + self, client: NexusClient, error: BaseException ) -> None: if client is not self.client: return @@ -5685,7 +5685,7 @@ class IdaTui(App): return anchor def _refresh_idb_events( - self, client: CodeModeClient, events: tuple[dict, ...] + self, client: NexusClient, events: tuple[dict, ...] ) -> None: """Invalidate once per external edit burst and reload the active surface.""" program = self.program @@ -5768,7 +5768,7 @@ class IdaTui(App): # -- connection loss / recovery --------------------------------------- # def _handle_exception(self, error: BaseException) -> None: - """Intercept a lost Code Mode lease so the app can rediscover the DB. + """Intercept a lost IDA Nexus lease so the app can rediscover the DB. Everything unrelated to database connectivity crashes as usual. """ @@ -5812,11 +5812,11 @@ class IdaTui(App): return if self._project is not None and self._binary is not None: ref = self._project.by_label(self._binary) - client = CodeModeClient( + client = NexusClient( ref.staged, ttl=self._ttl, load_args=ref.load_args, output_database=ref.db, spawn=False) else: - client = CodeModeClient( + client = NexusClient( self._open_path, ttl=self._ttl, load_args=self._load_args, spawn=False) client.connect(progress=lambda m: self.app.call_from_thread( @@ -5826,7 +5826,7 @@ class IdaTui(App): return self.app.call_from_thread(self._after_reconnect, client, Program(client)) - def _after_reconnect(self, client: "CodeModeClient", program: "Program") -> None: + def _after_reconnect(self, client: "NexusClient", program: "Program") -> None: old_client, old_program = self.client, self.program self._stop_idb_event_watch() if old_program is not None: @@ -5886,7 +5886,7 @@ class IdaTui(App): self._load_functions() def _open_database_client(self): # type: ignore[no-untyped-def] - """Attach through Code Mode, reusing a GUI or managed idalib database.""" + """Attach through IDA Nexus, reusing a GUI or managed idalib database.""" if self._pool is not None: # project mode: the pool owns the leases label = self._binary or self._project.refs[0].label client = self._pool.get(label, progress=lambda m: @@ -5898,13 +5898,13 @@ class IdaTui(App): return client if not self._open_path: self.app.call_from_thread( - self._status, "Code Mode needs a database or executable path") + self._status, "IDA Nexus needs a database or executable path") self.app.call_from_thread(self._dismiss_loading) return None base = os.path.basename(self._open_path) self.app.call_from_thread( - self._status, f"discovering Code Mode database for {base}…") - client = CodeModeClient(self._open_path, ttl=self._ttl, + self._status, f"discovering IDA Nexus database for {base}…") + client = NexusClient(self._open_path, ttl=self._ttl, load_args=self._load_args, new_database=self._new_database) client.connect(progress=lambda m: self.app.call_from_thread( @@ -5978,7 +5978,7 @@ class IdaTui(App): @work(thread=True, exclusive=True, group="index") def _index_binary(self) -> None: """Fold this binary's symbols + strings into the project index, so it can - be searched later even when its Code Mode lease is gone.""" + be searched later even when its IDA Nexus lease is gone.""" if self._index is None or self._project is None or self._binary is None: return ref = self._project.by_label(self._binary) @@ -6081,7 +6081,7 @@ class IdaTui(App): cursor=0, push=True, is_region=True) def _can_reload(self) -> bool: - """Whether Code Mode can replace this IDB with different options. + """Whether IDA Nexus can replace this IDB with different options. A GUI database is owned by the user and has no remote close/rollback route. Managed idalib databases can be released and reopened fresh. diff --git a/idatui/codemode_client.py b/idatui/codemode_client.py deleted file mode 100644 index e95238f..0000000 --- a/idatui/codemode_client.py +++ /dev/null @@ -1,551 +0,0 @@ -"""Client adapter from ida-tui's domain operations to IDA Code Mode. - -``DatabaseHandle`` is the lifecycle boundary: it discovers an already-registered -GUI database, reuses a shared managed idalib worker, or starts one when needed. -The TUI never owns or terminates an IDA process. Closing this client releases -only its lease. - -Remote operations are ordinary typed Python functions declared in -``idatui.remote_ops``. Code Mode installs their content-addressed modules once -per handle; subsequent calls send only encoded arguments. The optimized -IDAPython listing/decompiler implementation remains real source in -``idatui.remote_tools`` and is installed through the same module interface. -""" - -from __future__ import annotations - -import os -import shlex -import threading -import time -from collections.abc import Callable -from typing import Any - -from .errors import IDAConnectionError, IDATimeoutError, IDAToolError, Session - -# ida_codemode is imported EAGERLY-IF-PRESENT but never at hard import cost. -# -# The paging/graph/trace layers and their offline test suites must keep importing -# `idatui` on a machine with no IDA and no Code Mode installed -- that is the -# house rule the stdlib-only worker client used to satisfy for free, and -# `tests/run.py --fast` (380 checks, any python3) depends on it. A hard top-level -# import here makes the whole package unimportable, so the failure is deferred to -# the first operation that genuinely needs the library. -_CODEMODE_ERROR: Exception | None = None -try: - from ida_codemode import ( - CodeModeConnectionError, - DatabaseBusyError, - DatabaseDisconnectedError, - DatabaseHandle, - DatabaseInstance, - DatabaseOpenOptions, - RemoteError, - find_database_owner, - wait_database_released, - ) -except ImportError as _exc: # library absent: usable only for offline layers - _CODEMODE_ERROR = _exc - # Bound to None rather than left undefined so the names stay patchable: the - # offline contract tests inject a fake DatabaseHandle here. - CodeModeConnectionError = DatabaseDisconnectedError = RemoteError = None # type: ignore[assignment,misc] - DatabaseBusyError = DatabaseHandle = DatabaseInstance = None # type: ignore[assignment,misc] - DatabaseOpenOptions = find_database_owner = wait_database_released = None # type: ignore[assignment] - - -def _require_codemode() -> None: - """Raise an actionable error when the Code Mode library is missing. - - Gated on the binding, not on the original import result, so a test that - injects a fake ``DatabaseHandle`` exercises the real adapter logic. - """ - if DatabaseHandle is None: - raise IDAConnectionError( - "ida-codemode is not installed in this environment " - f"({_CODEMODE_ERROR}). Install it (e.g. `uv sync`, or " - "`pip install ida-codemode`) so ida-tui can lease a " - "database." - ) from _CODEMODE_ERROR - - -def database_owner(idb_path: str, staged_path: str | None = None): - """The Code Mode instance that owns ``idb_path``/``staged_path``, else None. - - Returns None when the Code Mode library is absent: with no library there is - no client in this environment that could be holding the database, and the - IDA-free layers (project staging) must keep working. Discovery errors with - the library installed still propagate because unknown ownership is unsafe. - """ - if DatabaseHandle is None: - return None - if staged_path: - owner = find_database_owner( - staged_path, - output_database=idb_path, - timeout=0.5, - ) - return owner or find_database_owner(staged_path, timeout=0.5) - return find_database_owner(idb_path, timeout=0.5) - - -def registered_database(path: str, output_database: str | None = None) -> bool: - """Whether a live/lock-held Code Mode instance owns this target.""" - _require_codemode() - return ( - find_database_owner( - path, - output_database=output_database, - timeout=0.5, - ) - is not None - ) - - -class _NoopKeepAlive: - """Compatibility shim: the DatabaseHandle's SSE lease is the heartbeat.""" - - def __init__(self) -> None: - self.beats = self.failures = 0 - - def start(self) -> "_NoopKeepAlive": - return self - - def stop(self) -> None: - pass - - -def _parse_load_args(value: str) -> tuple[str | None, int | None, str | None]: - """Translate ida-tui's legacy first-open switches to Code Mode options. - - Code Mode has typed options for processor, natural loading address and file - type. It deliberately has no arbitrary command-line escape hatch; reject - switches we cannot represent instead of silently loading a blob wrongly. - """ - processor: str | None = None - loading_address: int | None = None - file_type: str | None = None - unsupported: list[str] = [] - try: - words = shlex.split(value or "", posix=os.name != "nt") - except ValueError as exc: - raise ValueError(f"invalid IDA load options: {exc}") from exc - for word in words: - if word.startswith("-p") and len(word) > 2: - processor = word[2:] - elif word.startswith("-b") and len(word) > 2: - try: - # IDA's -b is in 16-byte paragraphs. DatabaseHandle expects the - # natural address, which is the safer public API. - loading_address = int(word[2:], 16) << 4 - except ValueError as exc: - raise ValueError(f"invalid IDA loading address: {word!r}") from exc - elif word.startswith("-T") and len(word) > 2: - file_type = word[2:] - else: - unsupported.append(word) - if unsupported: - joined = " ".join(unsupported) - raise ValueError( - "ida-codemode cannot represent arbitrary IDA load options: " - f"{joined!r}; use processor/base/file type options instead" - ) - return processor, loading_address, file_type - - -class IDBEventListener: - """Debounced, closeable delivery of another client's IDB changes. - - Code Mode's subscription is a blocking iterator, so one daemon thread reads - it and a second waits for a quiet period before handing a batch to the UI. - Keeping the debounce here avoids a permanent Textual worker (which would - make the app's worker-idle contract impossible) and bounds refresh work to - one pass per edit burst. - """ - - def __init__( - self, - client: "CodeModeClient", - callback: Callable[[tuple[dict[str, Any], ...]], None], - *, - on_error: Callable[[BaseException], None] | None = None, - debounce: float = 0.2, - ) -> None: - self._client = client - self._callback = callback - self._on_error = on_error - self._debounce = max(float(debounce), 0.0) - self._condition = threading.Condition() - self._closed = False - self._subscription = None - self._pending: list[dict[str, Any]] = [] - self._deadline = 0.0 - self._reader = threading.Thread( - target=self._read, name="idatui-idb-events", daemon=True - ) - self._deliverer = threading.Thread( - target=self._deliver, name="idatui-idb-refresh", daemon=True - ) - self._deliverer.start() - self._reader.start() - - def _report(self, error: BaseException) -> None: - disconnected = DatabaseDisconnectedError - if isinstance(disconnected, type) and isinstance(error, disconnected): - error = self._client._connection_error(error) - with self._condition: - closed = self._closed - if not closed and self._on_error is not None: - self._on_error(error) - - def _read(self) -> None: - try: - subscription = self._client.subscribe_idb_events() - except Exception as exc: # noqa: BLE001 -- surfaced through on_error - self._report(exc) - with self._condition: - self._closed = True - self._pending.clear() - self._condition.notify_all() - return - with self._condition: - if self._closed: - subscription.close() - return - self._subscription = subscription - try: - for event in subscription: - with self._condition: - if self._closed: - break - if self._client.owns_event(event): - continue - with self._condition: - if self._closed: - break - self._pending.append(event) - self._deadline = time.monotonic() + self._debounce - self._condition.notify_all() - except Exception as exc: # noqa: BLE001 -- stream failures are recoverable - self._report(exc) - finally: - subscription.close() - with self._condition: - if self._subscription is subscription: - self._subscription = None - self._closed = True - self._pending.clear() - self._condition.notify_all() - - def _deliver(self) -> None: - while True: - with self._condition: - while not self._closed and not self._pending: - self._condition.wait() - if self._closed: - return - remaining = self._deadline - time.monotonic() - if remaining > 0: - self._condition.wait(remaining) - continue - batch = tuple(self._pending) - self._pending.clear() - try: - self._callback(batch) - except Exception as exc: # noqa: BLE001 -- keep the stream alive - self._report(exc) - - def close(self) -> None: - """Stop delivery and unblock the subscription reader.""" - with self._condition: - if self._closed: - return - self._closed = True - self._pending.clear() - subscription = self._subscription - self._condition.notify_all() - if subscription is not None: - subscription.close() - - -class CodeModeClient: - """A leased GUI/idalib database accessed through ``ida_codemode``.""" - - def __init__( - self, - binary_path: str, - *, - ttl: int = 0, - load_args: str = "", - processor: str | None = None, - loading_address: int | None = None, - file_type: str | None = None, - output_database: str | None = None, - spawn: bool = True, - new_database: bool = False, - ) -> None: - del ttl # managed-worker lifetime is lease-based, not idle-TTL based - self._path = os.path.abspath(os.path.expanduser(binary_path)) - parsed_processor, parsed_address, parsed_file_type = _parse_load_args(load_args) - self._processor = processor or parsed_processor - self._loading_address = ( - loading_address if loading_address is not None else parsed_address - ) - self._file_type = file_type or parsed_file_type - self._output_database = output_database - self._spawn = spawn - self._new_database = new_database - self._handle: DatabaseHandle | None = None - self._last_instance: DatabaseInstance | None = None - self._connect_lock = threading.Lock() - - def connect(self, timeout: float = 1800.0, progress=None) -> "CodeModeClient": - _require_codemode() - with self._connect_lock: - handle = self._handle - if handle is not None: - if handle.connected: - return self - raise IDAConnectionError( - "Code Mode database disconnected; explicit rediscovery required" - ) - if progress: - progress( - f"discovering Code Mode database for {os.path.basename(self._path)}…" - ) - try: - # A Ctrl+L reload releases its current managed-worker lease, but - # that worker remains registered during Code Mode's final-lease - # grace period. Retry only that known handoff window. A GUI or - # another long-lived client remains busy and yields a clear - # failure rather than being modified underneath its owner. - deadline = time.monotonic() + min(timeout, 60.0) - while True: - try: - handle = DatabaseHandle.open( - self._path, - options=DatabaseOpenOptions( - spawn=self._spawn, - startup_timeout=max(0.1, timeout), - output_database=self._output_database, - processor=self._processor, - # The natural byte address is converted to IDA's - # paragraph-based -b value by Code Mode. - image_base=self._loading_address, - file_type=self._file_type, - new_database=self._new_database, - ), - ) - break - except DatabaseBusyError: - if not self._new_database or time.monotonic() >= deadline: - raise - if progress: - progress( - "waiting for the previous Code Mode lease to close…" - ) - owner = find_database_owner( - self._path, - output_database=self._output_database, - timeout=0.5, - ) - if owner is not None: - wait_database_released( - owner, - max(0.0, deadline - time.monotonic()), - ) - else: - time.sleep(0.2) - if progress: - backend = handle.instance.backend - progress( - f"attached to {backend} database; waiting for auto-analysis…" - ) - handle.wait_autoanalysis(timeout=timeout) - except Exception as exc: # normalize the dependency's transport errors - raise self._connection_error(exc) from exc - self._handle = handle - self._last_instance = handle.instance - return self - - @staticmethod - def _connection_error(exc: BaseException) -> IDAConnectionError: - return IDAConnectionError(str(exc) or type(exc).__name__) - - @property - def connected(self) -> bool: - return self._handle is not None and self._handle.connected - - @property - def pid(self) -> int | None: - return self._handle.instance.pid if self._handle is not None else None - - @property - def backend(self) -> str | None: - return self._handle.instance.backend if self._handle is not None else None - - def owns_event(self, event: dict[str, Any]) -> bool: - """Whether ``event`` was produced through this client's handle.""" - handle = self._handle - return handle is not None and handle.owns_event(event) - - def subscribe_idb_events(self): - """Open Code Mode's closeable IDB-change iterator.""" - if not self.connected: - self.connect() - handle = self._handle - if handle is None: - raise IDAConnectionError("Code Mode database is not connected") - try: - return handle.subscribe_idb_events() - except (DatabaseDisconnectedError, CodeModeConnectionError) as exc: - raise self._connection_error(exc) from exc - - def watch_idb_events( - self, - callback: Callable[[tuple[dict[str, Any], ...]], None], - *, - on_error: Callable[[BaseException], None] | None = None, - debounce: float = 0.2, - ) -> IDBEventListener: - """Deliver external IDB changes in debounced batches.""" - return IDBEventListener(self, callback, on_error=on_error, debounce=debounce) - - def call(self, operation: Callable[..., Any], /, **args) -> Any: - """Execute one source-backed remote declaration through this client.""" - name = getattr(operation, "__name__", "remote operation") - try: - from .remote_ops import bind - - remote = bind(operation) - except KeyError as exc: - raise IDAToolError( - name, f"remote operation {name!r} is not registered" - ) from exc - if not self.connected: - self.connect() - handle = self._handle - if handle is None: - raise IDAConnectionError("Code Mode database is not connected") - try: - return remote(handle, **args) - except RemoteError as exc: - message = str(exc) - if exc.details.get("traceback"): - message += f"\n{exc.details['traceback']}" - if exc.code == "operation_timeout": - raise IDATimeoutError(message) from exc - raise IDAToolError(name, message) from exc - except (DatabaseDisconnectedError, CodeModeConnectionError) as exc: - raise self._connection_error(exc) from exc - - def save_database(self) -> dict[str, Any]: - if not self.connected: - self.connect() - handle = self._handle - if handle is None: - raise IDAConnectionError("Code Mode database is not connected") - try: - return handle.save_database() - except RemoteError as exc: - raise IDAToolError("save_database", str(exc)) from exc - except (DatabaseDisconnectedError, CodeModeConnectionError) as exc: - raise self._connection_error(exc) from exc - - def discard_database(self, timeout: float = 5.0) -> bool: - """Discard a final managed-worker lease; otherwise transfer finalization. - - ``False`` is an expected ownership result: a GUI owns its session, or - another lease still shares the managed worker. A busy final worker is - retried briefly so background reads finishing during quit do not turn a - real discard into an implicit save. - """ - handle = self._handle - if handle is None or not handle.connected: - return False - entry = handle.instance - if entry.backend != "idalib" or not getattr(entry, "managed", False): - return False - deadline = time.monotonic() + max(float(timeout), 0.0) - while True: - try: - handle.shutdown_database(save=False) - return True - except RemoteError as exc: - if exc.code in ("instance_shared", "shutdown_not_supported"): - return False - if exc.code == "instance_busy" and time.monotonic() < deadline: - time.sleep(0.05) - continue - raise IDAToolError("shutdown_database", str(exc)) from exc - except (DatabaseDisconnectedError, CodeModeConnectionError) as exc: - raise self._connection_error(exc) from exc - - def health(self) -> dict[str, Any]: - if not self.connected: - self.connect() - assert self._handle is not None - entry = self._handle.instance - module = os.path.basename(entry.exe_path or entry.idb_path or self._path) - return { - "ok": self._handle.connected, - "module": module, - "backend": entry.backend, - "record_id": entry.record_id, - "input_path": entry.exe_path, - "idb_path": entry.idb_path, - } - - def keepalive(self, interval: float = 120.0) -> _NoopKeepAlive: - del interval - return _NoopKeepAlive() - - def resolve_db(self) -> str: - if not self.connected: - self.connect() - assert self._handle is not None - return self._handle.instance.record_id - - def set_db(self, db: str | None) -> None: - del db # one handle is permanently bound to one registered database - - def list_sessions(self) -> list[Session]: - if not self.connected: - self.connect() - assert self._handle is not None - entry = self._handle.instance - path = entry.exe_path or entry.idb_path or self._path - return [ - Session( - session_id=entry.record_id, - filename=os.path.basename(path), - input_path=path, - is_active=True, - ) - ] - - def close(self, grace: float = 0.0) -> None: - del grace - with self._connect_lock: - handle, self._handle = self._handle, None - if handle is not None: - self._last_instance = handle.instance - handle.close() # release our lease; never close a GUI/other client's DB - - def wait_released(self, timeout: float = 45.0) -> bool: - """Wait until a managed instance releases its lifetime lock. - - Normal application shutdown must not wait: another client may retain the - worker. This is an explicit test/maintenance helper for deleting a - temporary IDB safely after this client closes. GUI instances return - ``False`` immediately because clients never own their lifetime. - """ - instance = self._last_instance - if instance is None or instance.backend != "idalib": - return False - return wait_database_released(instance, timeout) - - def __enter__(self) -> "CodeModeClient": - return self.connect() - - def __exit__(self, *exc) -> None: - self.close() diff --git a/idatui/domain.py b/idatui/domain.py index b042332..1e5a863 100644 --- a/idatui/domain.py +++ b/idatui/domain.py @@ -1,4 +1,4 @@ -"""Domain / paging layer: address-centric models over IDA Code Mode. +"""Domain / paging layer: address-centric models over IDA Nexus. This is where the "millions of lines" problem is solved, so the TUI widgets only ever see a viewport-sized slice. Every hard-won constraint from @@ -7,7 +7,7 @@ ever see a viewport-sized slice. Every hard-won constraint from * Page sizes remain bounded so remote execution returns viewport-scale JSON. * Pagination advances by the number of rows actually returned. * Deep head walks are block-cached (revisits are free) and neighboring blocks - prefetch through the thread-safe Code Mode client. + prefetch through the thread-safe IDA Nexus client. * Expensive function totals are fetched once and cached. * Decompilation failures are surfaced as data, not application crashes. @@ -31,7 +31,7 @@ from . import remote_ops from .errors import IDAToolError if TYPE_CHECKING: # type hint only - from .codemode_client import CodeModeClient + from .nexus_client import NexusClient # Clamps derived from measured caps (list ~700, disasm ~500). Margin included. LIST_PAGE = 500 @@ -90,7 +90,7 @@ class Line: class Head(NamedTuple): - """One flat-listing item (from the Code Mode ``heads`` operation): a code + """One flat-listing item (from the IDA Nexus ``heads`` operation): a code instruction, a data item, or an undefined byte run. A ``NamedTuple`` rather than a dataclass because this is by far the @@ -111,7 +111,7 @@ class Head(NamedTuple): name: str | None = None raw: bytes | None = None # opcode/item bytes (filled in for code by the model) #: [(kind, text)] from IDA's own colour tags — mnem/reg/num/name/str/punct/… - #: None when Code Mode didn't provide them (or the spans + #: None when IDA Nexus didn't provide them (or the spans #: disagreed with the plain text, in which case the text wins). #: #: Held exactly as it came off the wire, and **read-only**. The worker @@ -693,7 +693,7 @@ class ListingModel: """A flat, IDA-style disassembly *listing* over one segment: code, data and undefined heads interleaved, unlike ``DisasmModel`` (one function, code only). - Backed by the Code Mode adapter's ``heads`` operation, which walks item heads + Backed by the IDA Nexus adapter's ``heads`` operation, which walks item heads and renders each via ``generate_disasm_line``. The segment is walked lazily in forward pages (``FunctionIndex`` style); line index == position in the walked head list. Random access to an address is O(distance-from-seg-start) the @@ -701,7 +701,7 @@ class ListingModel: on demand as the viewport scrolls. Synchronous + thread-safe. """ - PAGE = 500 # viewport-scale heads per Code Mode execution + PAGE = 500 # viewport-scale heads per IDA Nexus execution #: Generation marker for a skeleton (text-less) page. Never equals a real #: _text_gen, which counts up from 0, so such a page always reads as stale. _SKELETON_GEN = -1 @@ -1434,7 +1434,7 @@ class HexModel: class Program: """The bound analysis session: models, caches, and a small prefetch pool.""" - def __init__(self, client: "CodeModeClient", prefetch_workers: int = 2): + def __init__(self, client: "NexusClient", prefetch_workers: int = 2): self.client = client self._pool = ThreadPoolExecutor( max_workers=prefetch_workers, thread_name_prefix="idatui-prefetch" @@ -1485,7 +1485,7 @@ class Program: """Sorted raw segment map [(start, end, file_off, name)] — the single source for sections()/file_regions()/image_range. Cached. - Uses the Code Mode adapter's ``file_regions`` operation (a plain segment + Uses the IDA Nexus adapter's ``file_regions`` operation (a plain segment walk, ~ms), avoiding broad binary surveys on the hex-pane open path. """ if self._segments_cache is not None: @@ -1573,7 +1573,7 @@ class Program: def read_bytes(self, ea: int, n: int) -> bytes: """Raw bytes [ea, ea+n) from IDA (gaps read as zero). - The Code Mode adapter returns one contiguous hex string (C-speed in IDA). + The IDA Nexus adapter returns one contiguous hex string (C-speed in IDA). A legacy ``get_bytes`` decoding fallback remains for alternate clients. """ if n <= 0: @@ -1778,7 +1778,7 @@ class Program: except IDAToolError as e: msg = e.message if "not found" in msg.lower() and "del_type" in msg: - return "the connected Code Mode runtime cannot delete local types" + return "the connected IDA Nexus runtime cannot delete local types" return msg # -- disassembly ------------------------------------------------------- # @@ -1795,7 +1795,7 @@ class Program: """Drop local and Hex-Rays caches before an explicit view refresh. Normal edit paths use generation-based invalidation. Ctrl+R is also for - changes made by another Code Mode/IDA client, for which this Program has + changes made by another IDA Nexus/IDA client, for which this Program has seen no generation bump, so it must explicitly ask Hex-Rays to discard its cached cfunc. """ @@ -1809,7 +1809,7 @@ class Program: pass def decompile(self, ea: int, refresh: bool = False) -> Decompilation: - """Full pseudocode for a function, returned directly by Code Mode.""" + """Full pseudocode for a function, returned directly by IDA Nexus.""" if not refresh: with self._lock: hit = self._decomp.get(ea) @@ -2011,7 +2011,7 @@ class Program: def define_func(self, ea: int) -> dict: """Create a function starting at ``ea`` (IDA's 'p'). - Prefers the Code Mode operation, which works out the end when IDA can't; + Prefers the IDA Nexus operation, which works out the end when IDA can't; falls back to a plain create for alternate clients. """ try: diff --git a/idatui/errors.py b/idatui/errors.py index 7632ade..ab1365a 100644 --- a/idatui/errors.py +++ b/idatui/errors.py @@ -1,6 +1,6 @@ """TUI-facing error hierarchy and lightweight database session model. -The Code Mode adapter normalizes ``ida_codemode`` transport and execution +The IDA Nexus adapter normalizes ``ida_nexus`` transport and execution errors into these types so the domain and Textual layers do not depend on HTTP or registry implementation details. """ diff --git a/idatui/launch.py b/idatui/launch.py index e3cb091..a0201a7 100644 --- a/idatui/launch.py +++ b/idatui/launch.py @@ -1,6 +1,6 @@ -"""One-shot launcher for the IDA Code Mode-backed TUI. +"""One-shot launcher for the IDA Nexus-backed TUI. -A path first resolves to a registered GUI database; when none matches, Code Mode +A path first resolves to a registered GUI database; when none matches, IDA Nexus reuses or starts a managed idalib worker. With no path, a single registered database is selected automatically. @@ -34,9 +34,9 @@ def _log(msg: str) -> None: def _registered_databases() -> tuple[list[dict], list[dict]]: - """Ready and blocked Code Mode registrations, with normalized errors.""" + """Ready and blocked IDA Nexus registrations, with normalized errors.""" try: - from ida_codemode import InstanceState, discover_databases + from ida_nexus import InstanceState, discover_databases ready: list[dict] = [] blocked: list[dict] = [] @@ -70,7 +70,7 @@ def main(argv: list[str] | None = None) -> int: help="open a multi-binary project (created from the given " "binaries if FILE doesn't exist)") p.add_argument("--ttl", type=int, default=1800, - help="deprecated compatibility option (Code Mode uses leases)") + help="deprecated compatibility option (IDA Nexus uses leases)") p.add_argument("--no-keepalive", action="store_true", help="deprecated compatibility option (the lease is the heartbeat)") p.add_argument("--rpc", metavar="PATH", @@ -87,7 +87,7 @@ def main(argv: list[str] | None = None) -> int: g.add_argument("--base", metavar="ADDR", help="load address, e.g. 0x8000000 (any base; NOT paragraphs)") g.add_argument("--ida-args", metavar="STR", dest="ida_args", - help="legacy switches; only Code Mode-representable -p/-b/-T are accepted") + help="legacy switches; only IDA Nexus-representable -p/-b/-T are accepted") args = p.parse_args(argv) load: dict = {} @@ -166,10 +166,10 @@ def main(argv: list[str] | None = None) -> int: _log(f"attaching to registered {item.get('backend')} database: {binary}") elif not ready: detail = f" ({blocked[0].get('error')})" if blocked else "" - _log(f"no registered Code Mode database; pass a binary path{detail}") + _log(f"no registered IDA Nexus database; pass a binary path{detail}") return 2 else: - _log("several Code Mode databases are registered; pass one of these paths:") + _log("several IDA Nexus databases are registered; pass one of these paths:") for item in ready: _log(f" {item.get('exe_path') or item.get('idb_path')} " f"[{item.get('backend')}, {item.get('record_id')}]") diff --git a/idatui/nexus_client.py b/idatui/nexus_client.py new file mode 100644 index 0000000..44eb1b1 --- /dev/null +++ b/idatui/nexus_client.py @@ -0,0 +1,551 @@ +"""Client adapter from ida-tui's domain operations to IDA Nexus. + +``DatabaseHandle`` is the lifecycle boundary: it discovers an already-registered +GUI database, reuses a shared managed idalib worker, or starts one when needed. +The TUI never owns or terminates an IDA process. Closing this client releases +only its lease. + +Remote operations are ordinary typed Python functions declared in +``idatui.remote_ops``. IDA Nexus installs their content-addressed modules once +per IDA Python interpreter; subsequent calls send only encoded arguments. The +optimized IDAPython listing/decompiler implementation remains real source in +``idatui.remote_tools`` and is installed through the same module interface. +""" + +from __future__ import annotations + +import os +import shlex +import threading +import time +from collections.abc import Callable +from typing import Any + +from .errors import IDAConnectionError, IDATimeoutError, IDAToolError, Session + +# ida_nexus is imported EAGERLY-IF-PRESENT but never at hard import cost. +# +# The paging/graph/trace layers and their offline test suites must keep importing +# `idatui` on a machine with no IDA and no IDA Nexus installed -- that is the +# house rule the stdlib-only worker client used to satisfy for free, and +# `tests/run.py --fast` (380 checks, any python3) depends on it. A hard top-level +# import here makes the whole package unimportable, so the failure is deferred to +# the first operation that genuinely needs the library. +_NEXUS_ERROR: Exception | None = None +try: + from ida_nexus import ( + DatabaseBusyError, + DatabaseDisconnectedError, + DatabaseHandle, + DatabaseInstance, + DatabaseOpenOptions, + NexusConnectionError, + RemoteError, + find_database_owner, + wait_database_released, + ) +except ImportError as _exc: # library absent: usable only for offline layers + _NEXUS_ERROR = _exc + # Bound to None rather than left undefined so the names stay patchable: the + # offline contract tests inject a fake DatabaseHandle here. + NexusConnectionError = DatabaseDisconnectedError = RemoteError = None # type: ignore[assignment,misc] + DatabaseBusyError = DatabaseHandle = DatabaseInstance = None # type: ignore[assignment,misc] + DatabaseOpenOptions = find_database_owner = wait_database_released = None # type: ignore[assignment] + + +def _require_nexus() -> None: + """Raise an actionable error when the IDA Nexus library is missing. + + Gated on the binding, not on the original import result, so a test that + injects a fake ``DatabaseHandle`` exercises the real adapter logic. + """ + if DatabaseHandle is None: + raise IDAConnectionError( + "ida-nexus is not installed in this environment " + f"({_NEXUS_ERROR}). Install it (e.g. `uv sync`, or " + "`pip install ida-nexus`) so ida-tui can lease a " + "database." + ) from _NEXUS_ERROR + + +def database_owner(idb_path: str, staged_path: str | None = None): + """The IDA Nexus instance that owns ``idb_path``/``staged_path``, else None. + + Returns None when the IDA Nexus library is absent: with no library there is + no client in this environment that could be holding the database, and the + IDA-free layers (project staging) must keep working. Discovery errors with + the library installed still propagate because unknown ownership is unsafe. + """ + if DatabaseHandle is None: + return None + if staged_path: + owner = find_database_owner( + staged_path, + output_database=idb_path, + timeout=0.5, + ) + return owner or find_database_owner(staged_path, timeout=0.5) + return find_database_owner(idb_path, timeout=0.5) + + +def registered_database(path: str, output_database: str | None = None) -> bool: + """Whether a live/lock-held IDA Nexus instance owns this target.""" + _require_nexus() + return ( + find_database_owner( + path, + output_database=output_database, + timeout=0.5, + ) + is not None + ) + + +class _NoopKeepAlive: + """Compatibility shim: the DatabaseHandle's SSE lease is the heartbeat.""" + + def __init__(self) -> None: + self.beats = self.failures = 0 + + def start(self) -> "_NoopKeepAlive": + return self + + def stop(self) -> None: + pass + + +def _parse_load_args(value: str) -> tuple[str | None, int | None, str | None]: + """Translate ida-tui's legacy first-open switches to IDA Nexus options. + + IDA Nexus has typed options for processor, natural loading address and file + type. It deliberately has no arbitrary command-line escape hatch; reject + switches we cannot represent instead of silently loading a blob wrongly. + """ + processor: str | None = None + loading_address: int | None = None + file_type: str | None = None + unsupported: list[str] = [] + try: + words = shlex.split(value or "", posix=os.name != "nt") + except ValueError as exc: + raise ValueError(f"invalid IDA load options: {exc}") from exc + for word in words: + if word.startswith("-p") and len(word) > 2: + processor = word[2:] + elif word.startswith("-b") and len(word) > 2: + try: + # IDA's -b is in 16-byte paragraphs. DatabaseHandle expects the + # natural address, which is the safer public API. + loading_address = int(word[2:], 16) << 4 + except ValueError as exc: + raise ValueError(f"invalid IDA loading address: {word!r}") from exc + elif word.startswith("-T") and len(word) > 2: + file_type = word[2:] + else: + unsupported.append(word) + if unsupported: + joined = " ".join(unsupported) + raise ValueError( + "ida-nexus cannot represent arbitrary IDA load options: " + f"{joined!r}; use processor/base/file type options instead" + ) + return processor, loading_address, file_type + + +class IDBEventListener: + """Debounced, closeable delivery of another client's IDB changes. + + IDA Nexus's subscription is a blocking iterator, so one daemon thread reads + it and a second waits for a quiet period before handing a batch to the UI. + Keeping the debounce here avoids a permanent Textual worker (which would + make the app's worker-idle contract impossible) and bounds refresh work to + one pass per edit burst. + """ + + def __init__( + self, + client: "NexusClient", + callback: Callable[[tuple[dict[str, Any], ...]], None], + *, + on_error: Callable[[BaseException], None] | None = None, + debounce: float = 0.2, + ) -> None: + self._client = client + self._callback = callback + self._on_error = on_error + self._debounce = max(float(debounce), 0.0) + self._condition = threading.Condition() + self._closed = False + self._subscription = None + self._pending: list[dict[str, Any]] = [] + self._deadline = 0.0 + self._reader = threading.Thread( + target=self._read, name="idatui-idb-events", daemon=True + ) + self._deliverer = threading.Thread( + target=self._deliver, name="idatui-idb-refresh", daemon=True + ) + self._deliverer.start() + self._reader.start() + + def _report(self, error: BaseException) -> None: + disconnected = DatabaseDisconnectedError + if isinstance(disconnected, type) and isinstance(error, disconnected): + error = self._client._connection_error(error) + with self._condition: + closed = self._closed + if not closed and self._on_error is not None: + self._on_error(error) + + def _read(self) -> None: + try: + subscription = self._client.subscribe_idb_events() + except Exception as exc: # noqa: BLE001 -- surfaced through on_error + self._report(exc) + with self._condition: + self._closed = True + self._pending.clear() + self._condition.notify_all() + return + with self._condition: + if self._closed: + subscription.close() + return + self._subscription = subscription + try: + for event in subscription: + with self._condition: + if self._closed: + break + if self._client.owns_event(event): + continue + with self._condition: + if self._closed: + break + self._pending.append(event) + self._deadline = time.monotonic() + self._debounce + self._condition.notify_all() + except Exception as exc: # noqa: BLE001 -- stream failures are recoverable + self._report(exc) + finally: + subscription.close() + with self._condition: + if self._subscription is subscription: + self._subscription = None + self._closed = True + self._pending.clear() + self._condition.notify_all() + + def _deliver(self) -> None: + while True: + with self._condition: + while not self._closed and not self._pending: + self._condition.wait() + if self._closed: + return + remaining = self._deadline - time.monotonic() + if remaining > 0: + self._condition.wait(remaining) + continue + batch = tuple(self._pending) + self._pending.clear() + try: + self._callback(batch) + except Exception as exc: # noqa: BLE001 -- keep the stream alive + self._report(exc) + + def close(self) -> None: + """Stop delivery and unblock the subscription reader.""" + with self._condition: + if self._closed: + return + self._closed = True + self._pending.clear() + subscription = self._subscription + self._condition.notify_all() + if subscription is not None: + subscription.close() + + +class NexusClient: + """A leased GUI/idalib database accessed through ``ida_nexus``.""" + + def __init__( + self, + binary_path: str, + *, + ttl: int = 0, + load_args: str = "", + processor: str | None = None, + loading_address: int | None = None, + file_type: str | None = None, + output_database: str | None = None, + spawn: bool = True, + new_database: bool = False, + ) -> None: + del ttl # managed-worker lifetime is lease-based, not idle-TTL based + self._path = os.path.abspath(os.path.expanduser(binary_path)) + parsed_processor, parsed_address, parsed_file_type = _parse_load_args(load_args) + self._processor = processor or parsed_processor + self._loading_address = ( + loading_address if loading_address is not None else parsed_address + ) + self._file_type = file_type or parsed_file_type + self._output_database = output_database + self._spawn = spawn + self._new_database = new_database + self._handle: DatabaseHandle | None = None + self._last_instance: DatabaseInstance | None = None + self._connect_lock = threading.Lock() + + def connect(self, timeout: float = 1800.0, progress=None) -> "NexusClient": + _require_nexus() + with self._connect_lock: + handle = self._handle + if handle is not None: + if handle.connected: + return self + raise IDAConnectionError( + "IDA Nexus database disconnected; explicit rediscovery required" + ) + if progress: + progress( + f"discovering IDA Nexus database for {os.path.basename(self._path)}…" + ) + try: + # A Ctrl+L reload releases its current managed-worker lease, but + # that worker remains registered during IDA Nexus's final-lease + # grace period. Retry only that known handoff window. A GUI or + # another long-lived client remains busy and yields a clear + # failure rather than being modified underneath its owner. + deadline = time.monotonic() + min(timeout, 60.0) + while True: + try: + handle = DatabaseHandle.open( + self._path, + options=DatabaseOpenOptions( + spawn=self._spawn, + startup_timeout=max(0.1, timeout), + output_database=self._output_database, + processor=self._processor, + # The natural byte address is converted to IDA's + # paragraph-based -b value by IDA Nexus. + image_base=self._loading_address, + file_type=self._file_type, + new_database=self._new_database, + ), + ) + break + except DatabaseBusyError: + if not self._new_database or time.monotonic() >= deadline: + raise + if progress: + progress( + "waiting for the previous IDA Nexus lease to close…" + ) + owner = find_database_owner( + self._path, + output_database=self._output_database, + timeout=0.5, + ) + if owner is not None: + wait_database_released( + owner, + max(0.0, deadline - time.monotonic()), + ) + else: + time.sleep(0.2) + if progress: + backend = handle.instance.backend + progress( + f"attached to {backend} database; waiting for auto-analysis…" + ) + handle.wait_autoanalysis(timeout=timeout) + except Exception as exc: # normalize the dependency's transport errors + raise self._connection_error(exc) from exc + self._handle = handle + self._last_instance = handle.instance + return self + + @staticmethod + def _connection_error(exc: BaseException) -> IDAConnectionError: + return IDAConnectionError(str(exc) or type(exc).__name__) + + @property + def connected(self) -> bool: + return self._handle is not None and self._handle.connected + + @property + def pid(self) -> int | None: + return self._handle.instance.pid if self._handle is not None else None + + @property + def backend(self) -> str | None: + return self._handle.instance.backend if self._handle is not None else None + + def owns_event(self, event: dict[str, Any]) -> bool: + """Whether ``event`` was produced through this client's handle.""" + handle = self._handle + return handle is not None and handle.owns_event(event) + + def subscribe_idb_events(self): + """Open IDA Nexus's closeable IDB-change iterator.""" + if not self.connected: + self.connect() + handle = self._handle + if handle is None: + raise IDAConnectionError("IDA Nexus database is not connected") + try: + return handle.subscribe_idb_events() + except (DatabaseDisconnectedError, NexusConnectionError) as exc: + raise self._connection_error(exc) from exc + + def watch_idb_events( + self, + callback: Callable[[tuple[dict[str, Any], ...]], None], + *, + on_error: Callable[[BaseException], None] | None = None, + debounce: float = 0.2, + ) -> IDBEventListener: + """Deliver external IDB changes in debounced batches.""" + return IDBEventListener(self, callback, on_error=on_error, debounce=debounce) + + def call(self, operation: Callable[..., Any], /, **args) -> Any: + """Execute one source-backed remote declaration through this client.""" + name = getattr(operation, "__name__", "remote operation") + try: + from .remote_ops import bind + + remote = bind(operation) + except KeyError as exc: + raise IDAToolError( + name, f"remote operation {name!r} is not registered" + ) from exc + if not self.connected: + self.connect() + handle = self._handle + if handle is None: + raise IDAConnectionError("IDA Nexus database is not connected") + try: + return remote(handle, **args) + except RemoteError as exc: + message = str(exc) + if exc.details.get("traceback"): + message += f"\n{exc.details['traceback']}" + if exc.code == "operation_timeout": + raise IDATimeoutError(message) from exc + raise IDAToolError(name, message) from exc + except (DatabaseDisconnectedError, NexusConnectionError) as exc: + raise self._connection_error(exc) from exc + + def save_database(self) -> dict[str, Any]: + if not self.connected: + self.connect() + handle = self._handle + if handle is None: + raise IDAConnectionError("IDA Nexus database is not connected") + try: + return handle.save_database() + except RemoteError as exc: + raise IDAToolError("save_database", str(exc)) from exc + except (DatabaseDisconnectedError, NexusConnectionError) as exc: + raise self._connection_error(exc) from exc + + def discard_database(self, timeout: float = 5.0) -> bool: + """Discard a final managed-worker lease; otherwise transfer finalization. + + ``False`` is an expected ownership result: a GUI owns its session, or + another lease still shares the managed worker. A busy final worker is + retried briefly so background reads finishing during quit do not turn a + real discard into an implicit save. + """ + handle = self._handle + if handle is None or not handle.connected: + return False + entry = handle.instance + if entry.backend != "idalib" or not getattr(entry, "managed", False): + return False + deadline = time.monotonic() + max(float(timeout), 0.0) + while True: + try: + handle.shutdown_database(save=False) + return True + except RemoteError as exc: + if exc.code in ("instance_shared", "shutdown_not_supported"): + return False + if exc.code == "instance_busy" and time.monotonic() < deadline: + time.sleep(0.05) + continue + raise IDAToolError("shutdown_database", str(exc)) from exc + except (DatabaseDisconnectedError, NexusConnectionError) as exc: + raise self._connection_error(exc) from exc + + def health(self) -> dict[str, Any]: + if not self.connected: + self.connect() + assert self._handle is not None + entry = self._handle.instance + module = os.path.basename(entry.exe_path or entry.idb_path or self._path) + return { + "ok": self._handle.connected, + "module": module, + "backend": entry.backend, + "record_id": entry.record_id, + "input_path": entry.exe_path, + "idb_path": entry.idb_path, + } + + def keepalive(self, interval: float = 120.0) -> _NoopKeepAlive: + del interval + return _NoopKeepAlive() + + def resolve_db(self) -> str: + if not self.connected: + self.connect() + assert self._handle is not None + return self._handle.instance.record_id + + def set_db(self, db: str | None) -> None: + del db # one handle is permanently bound to one registered database + + def list_sessions(self) -> list[Session]: + if not self.connected: + self.connect() + assert self._handle is not None + entry = self._handle.instance + path = entry.exe_path or entry.idb_path or self._path + return [ + Session( + session_id=entry.record_id, + filename=os.path.basename(path), + input_path=path, + is_active=True, + ) + ] + + def close(self, grace: float = 0.0) -> None: + del grace + with self._connect_lock: + handle, self._handle = self._handle, None + if handle is not None: + self._last_instance = handle.instance + handle.close() # release our lease; never close a GUI/other client's DB + + def wait_released(self, timeout: float = 45.0) -> bool: + """Wait until a managed instance releases its lifetime lock. + + Normal application shutdown must not wait: another client may retain the + worker. This is an explicit test/maintenance helper for deleting a + temporary IDB safely after this client closes. GUI instances return + ``False`` immediately because clients never own their lifetime. + """ + instance = self._last_instance + if instance is None or instance.backend != "idalib": + return False + return wait_database_released(instance, timeout) + + def __enter__(self) -> "NexusClient": + return self.connect() + + def __exit__(self, *exc) -> None: + self.close() diff --git a/idatui/pane.py b/idatui/pane.py index c31a93c..1eee847 100644 --- a/idatui/pane.py +++ b/idatui/pane.py @@ -24,7 +24,7 @@ per pane in the registry, so stop/list/capture/keys keep working across both python -m idatui.pane keys --pane Escape Requires: running inside tmux or zellij. Each pane leases a registered GUI or -shared managed idalib database through Code Mode. Uses ~/ida-venv/bin/python for +shared managed idalib database through IDA Nexus. Uses ~/ida-venv/bin/python for the TUI (needs textual) unless --python / IDATUI_PYTHON says otherwise. """ from __future__ import annotations @@ -250,8 +250,8 @@ def _pane_keys(pane: str, keys: list[str], mux: str | None = None) -> None: subprocess.run(["tmux", "send-keys", "-t", pane, *keys], check=True) -# Code Mode owns database process lifetime: a closed pane drops its lease at the -# socket/kernel boundary and Code Mode decides whether a managed worker still +# IDA Nexus owns database process lifetime: a closed pane drops its lease at the +# socket/kernel boundary and IDA Nexus decides whether a managed worker still # has clients. There is nothing for the pane layer to reap. @@ -261,7 +261,7 @@ def _count_live_panes() -> int: def _reap_orphan_workers(force: bool = False) -> int: - """Compatibility no-op: Code Mode workers are shared and lease-managed.""" + """Compatibility no-op: IDA Nexus workers are shared and lease-managed.""" del force return 0 @@ -294,7 +294,7 @@ def spawn(args) -> int: print(f"error: no such project: {project}", file=sys.stderr) return 2 - # The pane owns only the TUI. Code Mode's lease cleanup handles crashes; + # The pane owns only the TUI. IDA Nexus's lease cleanup handles crashes; # kill-pane must never reap a shared GUI/idalib database. if project is not None: # launch takes: --project FILE [binaries...]; extra binaries are added to @@ -347,7 +347,7 @@ def _wait_ready(sock: str, timeout: float, pane: str, stuck_after: float = 45.0, mux: str | None = None) -> dict[str, Any]: """Poll the socket + ping until the TUI reports ready (or timeout). - Emits a one-time hint if Code Mode discovery/opening is still not ready after + Emits a one-time hint if IDA Nexus discovery/opening is still not ready after ``stuck_after`` seconds. """ start = time.time() @@ -370,7 +370,7 @@ def _wait_ready(sock: str, timeout: float, pane: str, why = ("RPC socket not created yet" if not os.path.exists(sock) else "TUI up but analysis not ready") print(f"still waiting ({int(time.time() - start)}s): {why}. " - f"Check Code Mode registrations and worker logs.", file=sys.stderr) + f"Check IDA Nexus registrations and worker logs.", file=sys.stderr) time.sleep(0.4) last = dict(last) last["ready"] = False @@ -467,7 +467,7 @@ def list_panes(args) -> int: def reap(args) -> int: - """Deprecated no-op; shared Code Mode workers are managed by leases.""" + """Deprecated no-op; shared IDA Nexus workers are managed by leases.""" print(json.dumps({"reaped_workers": 0, "live_panes": _count_live_panes(), "forced": args.force, "deprecated": True})) return 0 @@ -572,7 +572,7 @@ def main(argv: list[str]) -> int: ls.add_argument("--prune", action="store_true", help="drop dead panes (and their sockets)") ls.set_defaults(fn=list_panes) - rp = sub.add_parser("reap", help="deprecated no-op (Code Mode uses shared leases)") + rp = sub.add_parser("reap", help="deprecated no-op (IDA Nexus uses shared leases)") rp.add_argument("--force", action="store_true", help=argparse.SUPPRESS) rp.set_defaults(fn=reap) diff --git a/idatui/pool.py b/idatui/pool.py index 573aa75..2407b44 100644 --- a/idatui/pool.py +++ b/idatui/pool.py @@ -1,6 +1,6 @@ -"""DatabasePool — LRU leases on Code Mode databases for a project. +"""DatabasePool — LRU leases on IDA Nexus databases for a project. -Code Mode may bind a lease to an existing IDA GUI or to a shared managed idalib +IDA Nexus may bind a lease to an existing IDA GUI or to a shared managed idalib worker. The pool therefore owns *client interest*, never an IDA process. Releasing an LRU entry persists managed IDBs but does not implicitly save a GUI, then closes only this TUI's lease; other clients and GUI sessions remain alive. Managed workers exit themselves after their final lease. @@ -48,8 +48,8 @@ def _pss_mb(pid: int | None) -> int: def _default_spawn(ref: BinaryRef, ttl: int, *, new_database: bool = False): # pragma: no cover - needs IDA - from .codemode_client import CodeModeClient - return CodeModeClient( + from .nexus_client import NexusClient + return NexusClient( ref.staged, ttl=ttl, load_args=ref.load_args, @@ -59,7 +59,7 @@ def _default_spawn(ref: BinaryRef, ttl: int, *, new_database: bool = False): # class DatabasePool: - """Live Code Mode database leases, keyed by project label.""" + """Live IDA Nexus database leases, keyed by project label.""" def __init__(self, project: Project, *, budget_mb: int | None = None, ttl: int = 1800, spawn=None, mem_fn=None) -> None: @@ -101,7 +101,7 @@ class DatabasePool: """A live client for ``label``, attaching or spawning as needed. Do not sweep IDA scratch files here: a registered GUI or another Code - Mode client may own the database. Code Mode's registry locks and health + Mode client may own the database. IDA Nexus's registry locks and health probes are the authority for safe discovery and stale-record cleanup. """ client = self._clients.get(label) @@ -269,5 +269,3 @@ class DatabasePool: f"{self.memory_mb()}/{self.budget_mb}MB active={self.active}>") -# Source compatibility for callers that imported the pre-Code-Mode name. -WorkerPool = DatabasePool diff --git a/idatui/project.py b/idatui/project.py index 53f3b0a..e681dfc 100644 --- a/idatui/project.py +++ b/idatui/project.py @@ -23,7 +23,7 @@ firmware image, a cleaned build tree). A source whose size/mtime no longer matches the staged copy is re-staged, and its now-stale database is dropped (the DB describes the old bytes). -The model has no IDA imports. Staging consults ida_codemode's registry before +The model has no IDA imports. Staging consults ida_nexus's registry before replacing files so it never mutates a database owned by a GUI/shared worker. """ from __future__ import annotations @@ -57,7 +57,7 @@ class BinaryRef: #: it is, a raw firmware image doesn't, and IDA defaults to metapc at 0. processor: str = "" # IDA processor name: arm, armb, mipsb, metapc, … base: int = 0 # load address (natural, e.g. 0x8000000) - ida_args: str = "" # legacy -p/-b/-T switches accepted by Code Mode adapter + ida_args: str = "" # legacy -p/-b/-T switches accepted by IDA Nexus adapter @property def db(self) -> str: @@ -311,7 +311,7 @@ class Project: """Ensure ``ref`` is staged in the sidecar; returns the staged path. Re-staging a changed source drops its database: the DB describes the old - bytes. Refuse while Code Mode reports a GUI/idalib owner; replacing a + bytes. Refuse while IDA Nexus reports a GUI/idalib owner; replacing a staged executable or IDB underneath a shared live instance is corruption. """ if not os.path.isfile(ref.source): @@ -319,15 +319,15 @@ class Project: if not self.is_stale(ref): return ref.staged try: - from .codemode_client import database_owner + from .nexus_client import database_owner owner = database_owner(ref.db, ref.staged) except Exception as exc: raise ProjectError( - f"cannot verify Code Mode ownership before staging {ref.label}: {exc}" + f"cannot verify IDA Nexus ownership before staging {ref.label}: {exc}" ) from exc if owner is not None: raise ProjectError( - f"cannot restage {ref.label}: Code Mode instance {owner.record_id} " + f"cannot restage {ref.label}: IDA Nexus instance {owner.record_id} " f"still owns {owner.idb_path}; close/release it first" ) os.makedirs(self.bin_dir, exist_ok=True) @@ -353,7 +353,7 @@ class Project: def sweep_scratch(self, ref: BinaryRef) -> int: """Delete unpacked working files (never the ``.i64``) for maintenance. - Runtime paths no longer call this: Code Mode instances are shared, so a + Runtime paths no longer call this: IDA Nexus instances are shared, so a registry owner may still be using these files. Callers must independently prove that no GUI/idalib instance owns the database. """ diff --git a/idatui/remote_ops.py b/idatui/remote_ops.py index 91ffeb7..a67b2b7 100644 --- a/idatui/remote_ops.py +++ b/idatui/remote_ops.py @@ -1,4 +1,4 @@ -"""Typed remote operations executed through ida-codemode.""" +"""Typed remote operations executed through ida-nexus.""" from __future__ import annotations # ruff: noqa @@ -1655,7 +1655,7 @@ def _bindings() -> dict[Callable[..., Any], Any]: with _BIND_LOCK: if _BOUND is not None: return _BOUND - from ida_codemode import RemoteModule + from ida_nexus import RemoteModule operations_module = RemoteModule( Path(__file__), operation_label=operation_label, codec="json" diff --git a/idatui/remote_tools.py b/idatui/remote_tools.py index db04244..6681379 100644 --- a/idatui/remote_tools.py +++ b/idatui/remote_tools.py @@ -1,8 +1,8 @@ -"""The IDAPython ida-tui runs inside the Code Mode sandbox. +"""The IDAPython ida-tui runs inside the IDA Nexus sandbox. Two features have no ida-domain surface at all and are carried over VERBATIM from the tools ida-tui was developed against (`server/patch_server.py`'s -injected BODY, which the Code Mode port deletes): +injected BODY, which the IDA Nexus port deletes): * `heads` -- the continuous listing. ida-domain enumerates defined heads and renders plain disassembly; the listing also needs coalesced undefined runs, @@ -20,8 +20,8 @@ what you see). A re-implementation drifts from it silently. This file is SOURCE SHIPPED AS TEXT to the database process; it is never imported here, because the ida_* modules do not exist in the TUI's interpreter. -`codemode_client` reads it and prepends it to the relevant snippets. Keep it -self-contained: no relative imports, nothing beyond what Code Mode provides. +`nexus_client` reads it and prepends it to the relevant snippets. Keep it +self-contained: no relative imports, nothing beyond what IDA Nexus provides. """ # ruff: noqa @@ -29,7 +29,7 @@ import re as _re # IDAPython, imported ONCE at module scope. # -# This file is never imported by the client -- codemode_client reads it as +# This file is never imported by the client -- nexus_client reads it as # TEXT and installs it as a module inside the database process -- so the # no-IDA house rule that keeps idatui importable without IDA does not apply # here, and these need not be function-local. @@ -1740,7 +1740,7 @@ def decompile(addr, include_addresses=True): Faithful to the tool ida-tui was written against, and in particular to its COST: the per-line address anchor comes from ONE ``get_line_item`` at column - 0 per line. The Code Mode port asked for the full per-column line map (what + 0 per line. The IDA Nexus port asked for the full per-column line map (what ``decomp_map`` is for) purely to fill in that anchor, which is thousands of ``get_line_item``+``dstr()`` calls per function instead of one per line, and made every pseudocode open cost the same as opening the split view. diff --git a/pyproject.toml b/pyproject.toml index a46dd45..dc68d12 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,12 +1,12 @@ [project] name = "idatui" version = "0.0.1" -description = "A keyboard-first TUI frontend for shared IDA Code Mode databases." +description = "A keyboard-first TUI frontend for shared IDA Nexus databases." requires-python = ">=3.11" -# ida-codemode supplies GUI discovery, shared idalib workers, leases, and the +# ida-nexus supplies GUI discovery, shared idalib workers, leases, and the # execute_python/ida-domain database surface. dependencies = [ - "ida-codemode>=0.5.3", + "ida-nexus>=0.7.0", "textual>=8", "pygments>=2", # Used directly for pseudocode highlighting. ] @@ -29,3 +29,4 @@ build-backend = "hatchling.build" [tool.hatch.build.targets.wheel] packages = ["idatui"] + diff --git a/tests/_fixtures.py b/tests/_fixtures.py index 16192fa..bc10235 100644 --- a/tests/_fixtures.py +++ b/tests/_fixtures.py @@ -133,7 +133,7 @@ async def build_pristine(binary: str, cache: str, app_factory) -> None: break app.program.client.save_database() # Textual's headless run_test context does not reliably emit App.Unmount on - # every platform/version; release the Code Mode lease explicitly. + # every platform/version; release the IDA Nexus lease explicitly. if app.program is not None: app.program.close() if app.client is not None: diff --git a/tests/run.py b/tests/run.py index b38a707..5d552bb 100755 --- a/tests/run.py +++ b/tests/run.py @@ -51,8 +51,8 @@ import time ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) TESTS = os.path.join(ROOT, "tests") -#: The IDA-capable interpreter. The pilot tests need textual AND the Code Mode -#: library in one python; the database process is Code Mode's to place. +#: The IDA-capable interpreter. The pilot tests need textual AND the IDA Nexus +#: library in one python; the database process is IDA Nexus's to place. DEFAULT_PY = os.path.expanduser("~/ida-venv/bin/python") #: Both shapes the suites print: "N passed, M failed" and "N checks, M failed". diff --git a/tests/test_codemode_client.py b/tests/test_codemode_client.py deleted file mode 100644 index dd0ae08..0000000 --- a/tests/test_codemode_client.py +++ /dev/null @@ -1,440 +0,0 @@ -"""IDA-free contract tests for the Code Mode client adapter.""" - -from __future__ import annotations - -import os -import queue -import sys -import threading -import time -import tempfile -from dataclasses import dataclass - -sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from idatui.errors import IDAConnectionError, IDAToolError # noqa: E402 -from idatui import remote_ops # noqa: E402 -import idatui.codemode_client as module # noqa: E402 -from idatui.codemode_client import CodeModeClient, _parse_load_args # noqa: E402 - -#: Pure: fakes the DatabaseHandle, never touches IDA or the Code Mode library. -NEEDS_IDA = False - -PASS = FAIL = 0 - - -def check(name: str, condition: bool, detail="") -> None: - global PASS, FAIL - if condition: - PASS += 1 - print(f" ok {name}") - else: - FAIL += 1 - print(f" FAIL {name} {detail}") - - -@dataclass(frozen=True) -class FakeEntry: - pid: int = 123 - backend: str = "gui" - record_id: str = "123-abcdef" - exe_path: str = "" - idb_path: str = "" - managed: bool = False - - -_CLOSED = object() - - -class FakeSubscription: - def __init__(self) -> None: - self._queue: queue.Queue = queue.Queue() - self.closed = False - - def __iter__(self): - return self - - def __next__(self): - item = self._queue.get() - if isinstance(item, BaseException): - raise item - if item is _CLOSED: - raise StopIteration - return item - - def emit(self, event: dict) -> None: - self._queue.put(event) - - def close(self) -> None: - if not self.closed: - self.closed = True - self._queue.put(_CLOSED) - - -class FakeHandle: - def __init__(self, path: str) -> None: - self.connected = True - self.instance = FakeEntry(exe_path=path, idb_path=path + ".i64") - self.waited = None - self.saved = 0 - self.closed = False - self.code = "" - self.codes = [] - self.code_timeout = None - self.operation_label = None - self.event_origin_id = "fake-handle-origin" - self.owns_checks = 0 - self.subscription = FakeSubscription() - self.shutdown_calls = [] - self.shutdown_error = None - - def wait_autoanalysis(self, timeout=None): - self.waited = timeout - return {"complete": True, "status": "complete"} - - def execute_python( - self, - code, - timeout=None, - *, - operation_id=None, - operation_label=None, - persist_globals=False, - filename=None, - ): - self.code = code - self.codes.append(code) - self.code_timeout = timeout - self.operation_label = operation_label - result = ( - { - "__remote_ida_status__": "ok", - "__remote_ida_value__": {"sentinel": 7}, - } - if ".modules.get(" in code - else True - ) - return {"result": result, "stdout": "", "stderr": ""} - - def subscribe_idb_events(self): - return self.subscription - - def owns_event(self, event): - self.owns_checks += 1 - return event.get("origin_id") == self.event_origin_id - - def save_database(self): - self.saved += 1 - return {"saved": True, "idb_path": self.instance.idb_path} - - def shutdown_database(self, *, save=True): - self.shutdown_calls.append(save) - if self.shutdown_error is not None: - raise module.RemoteError(self.shutdown_error, self.shutdown_error, 409) - return {"shutting_down": True, "save": save} - - def close(self): - self.subscription.close() - self.connected = False - self.closed = True - - -class FakeDatabaseHandle: - opened = None - opens = 0 - kwargs = None - - @classmethod - def open(cls, path, **kwargs): - cls.opens += 1 - cls.opened = path - cls.kwargs = kwargs - return FakeHandle(path) - - -@dataclass(frozen=True) -class FakeOpenOptions: - """Stand-in for DatabaseOpenOptions when the library is not installed. - - Deliberately STRICT (no **kwargs): an option the adapter invents would - raise here, and `_option_fields_are_real` checks the surviving names - against the real dataclass wherever it is importable. - """ - - spawn: bool = True - startup_timeout: float = 120.0 - output_database: str | None = None - processor: str | None = None - image_base: int | None = None - file_type: str | None = None - new_database: bool = False - - -class FakeBusy(Exception): - """Stand-in for DatabaseBusyError: `except None` is a TypeError.""" - - -class FakeDisconnected(Exception): - """Stand-in for DatabaseDisconnectedError in stdlib-only runs.""" - - -def _open_kwargs_are_real(sent: dict): - """(ok, detail) for the kwargs the adapter passes to DatabaseHandle.open. - - Skips (passes) when ida_codemode is not installed, so the file stays pure. - """ - try: - import inspect - from ida_codemode import DatabaseHandle as Real - except ImportError: - return True, "ida_codemode not installed - signature not checked" - accepted = set(inspect.signature(Real.open).parameters) - unknown = sorted(set(sent) - accepted) - return not unknown, f"open() rejects {unknown}" - - -def _option_fields_are_real(options): - """(ok, detail) for the option names the adapter fills in. - - The open() signature no longer names the loader options -- they moved - inside DatabaseOpenOptions -- so the `loading_address` class of bug now - hides there instead. Check it in the same way. - """ - try: - import dataclasses - from ida_codemode import DatabaseOpenOptions as Real - except ImportError: - return True, "ida_codemode not installed - fields not checked" - accepted = {field.name for field in dataclasses.fields(Real)} - unknown = sorted({f.name for f in dataclasses.fields(options)} - accepted) - return not unknown, f"DatabaseOpenOptions rejects {unknown}" - - -def main() -> int: - proc, base, file_type = _parse_load_args("-parm:ARMv7-M -b800000 -TRaw") - check( - "legacy switches map to typed Code Mode options", - (proc, base, file_type) == ("arm:ARMv7-M", 0x8000000, "Raw"), - (proc, base, file_type), - ) - try: - _parse_load_args("-parm -zcustom") - except ValueError as exc: - check("arbitrary IDA switches fail loudly", "cannot represent" in str(exc), exc) - else: - check("arbitrary IDA switches fail loudly", False) - - original = module.DatabaseHandle - module.DatabaseHandle = FakeDatabaseHandle - # The library's own names when it is installed; strict fakes when it is not - # (this file must keep running under a stdlib-only python3). - original_options = module.DatabaseOpenOptions - original_busy = module.DatabaseBusyError - original_disconnected = module.DatabaseDisconnectedError - module.DatabaseOpenOptions = original_options or FakeOpenOptions - module.DatabaseBusyError = original_busy or FakeBusy - module.DatabaseDisconnectedError = original_disconnected or FakeDisconnected - try: - with tempfile.TemporaryDirectory() as tmp: - path = os.path.join(tmp, "sample.bin") - with open(path, "wb") as file: - file.write(b"sample") - client = CodeModeClient(path, load_args="-parm:ARMv7-A -b100") - notes = [] - client.connect(timeout=42, progress=notes.append) - handle = client._handle - check( - "connect delegates database discovery to DatabaseHandle.open", - FakeDatabaseHandle.opened == path and handle is not None, - ) - options = FakeDatabaseHandle.kwargs["options"] - check( - "typed loader options cross the dependency boundary", - options.processor == "arm:ARMv7-A" and options.image_base == 0x1000, - options, - ) - check( - "every open option exists in the real library", - *_option_fields_are_real(options), - ) - # A fake that swallows **kwargs cannot catch a keyword the real - # library does not have -- which is exactly how this port shipped - # `loading_address` (the real name is `image_base`) and would have - # raised TypeError on the very first connect. Check the names we - # send against the real signature whenever it is importable. - check( - "every open() keyword exists in the real library", - *_open_kwargs_are_real(FakeDatabaseHandle.kwargs), - ) - check( - "connect waits for Code Mode autoanalysis", - handle.waited == 42, - getattr(handle, "waited", None), - ) - check( - "progress distinguishes discovery and backend attachment", - len(notes) == 2 and "gui" in notes[-1], - notes, - ) - result = client.call( - remote_ops.list_funcs, queries=[{"offset": 0, "count": 2}] - ) - check( - "remote operation returns its JSON result", - result == {"sentinel": 7}, - result, - ) - check( - "operation source is real Python installed through ida-domain", - any("db.functions.get_all()" in code for code in handle.codes), - handle.codes[0][:200], - ) - check( - "remote operations attribute IDB events to IDA TUI", - handle.operation_label == "IDA TUI", - handle.operation_label, - ) - batches = [] - delivered = threading.Event() - - def changed(batch): - batches.append(batch) - delivered.set() - - watcher = client.watch_idb_events(changed, debounce=0.05) - handle.subscription.emit({"event_name": "renamed", "origin_id": "peer-1"}) - handle.subscription.emit( - {"event_name": "cmt_changed", "origin_id": "peer-2"} - ) - check( - "event bursts produce one debounced refresh", - delivered.wait(1) and len(batches) == 1 and len(batches[0]) == 2, - batches, - ) - delivered.clear() - handle.subscription.emit( - {"event_name": "renamed", "origin_id": handle.event_origin_id} - ) - time.sleep(0.1) - check( - "the listener uses handle ownership to ignore its own events", - not delivered.is_set() - and len(batches) == 1 - and handle.owns_checks >= 3, - (batches, handle.owns_checks), - ) - handle.subscription.emit( - {"event_name": "byte_patched", "origin_id": "peer-3"} - ) - watcher.close() - time.sleep(0.1) - check( - "closing drops a pending debounced refresh", - not delivered.is_set() and len(batches) == 1, - batches, - ) - check( - "health exposes registry identity", - client.health()["record_id"] == "123-abcdef", - ) - client.save_database() - check("save uses the public Code Mode save route", handle.saved == 1) - check( - "GUI leases transfer rather than claiming discard", - client.discard_database() is False and handle.shutdown_calls == [], - handle.shutdown_calls, - ) - handle.instance = FakeEntry( - backend="idalib", managed=True, exe_path=path, idb_path=path + ".i64" - ) - check( - "a final managed lease discards without saving", - client.discard_database() is True and handle.shutdown_calls == [False], - handle.shutdown_calls, - ) - handle.shutdown_error = "instance_shared" - check( - "a shared managed lease transfers finalization", - client.discard_database() is False, - handle.shutdown_calls, - ) - handle.shutdown_error = "instance_busy" - try: - client.discard_database(timeout=0) - except IDAToolError as exc: - check( - "a busy final lease never silently saves", - exc.tool == "shutdown_database", - exc, - ) - else: - check("a busy final lease never silently saves", False) - handle.shutdown_error = None - handle.instance = FakeEntry(exe_path=path, idb_path=path + ".i64") - opens = FakeDatabaseHandle.opens - handle.connected = False - try: - client.health() - except IDAConnectionError as exc: - check( - "a disconnected handle requires explicit rediscovery", - "explicit rediscovery" in str(exc) - and FakeDatabaseHandle.opens == opens, - (exc, FakeDatabaseHandle.opens, opens), - ) - else: - check("a disconnected handle requires explicit rediscovery", False) - client.close() - check("close releases only the handle lease", handle.closed) - check( - "GUI lifetime is never claimed by the client", - client.wait_released(0) is False, - ) - disconnected = CodeModeClient(path).connect() - stream_errors = [] - stream_failed = threading.Event() - - def failed(error): - stream_errors.append(error) - stream_failed.set() - - stream_watch = disconnected.watch_idb_events( - lambda _batch: None, on_error=failed, debounce=0 - ) - disconnected._handle.subscription.emit( - module.DatabaseDisconnectedError("GUI database closed") - ) - check( - "stream disconnects become application connection errors", - stream_failed.wait(1) - and isinstance(stream_errors[0], IDAConnectionError), - stream_errors, - ) - stream_watch.close() - disconnected.close() - finally: - module.DatabaseHandle = original - module.DatabaseOpenOptions = original_options - module.DatabaseBusyError = original_busy - module.DatabaseDisconnectedError = original_disconnected - - client = CodeModeClient(__file__) - - def unknown_operation(): - pass - - try: - client.call(unknown_operation) - except IDAToolError as exc: - check( - "unknown adapter operations are explicit", - exc.tool == "unknown_operation", - ) - else: - check("unknown adapter operations are explicit", False) - - print(f"\n{PASS} passed, {FAIL} failed") - return 1 if FAIL else 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/tests/test_kittygfx.py b/tests/test_kittygfx.py index 74b6bb7..cfc6512 100644 --- a/tests/test_kittygfx.py +++ b/tests/test_kittygfx.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """Cross-platform checks for the optional kitty-graphics startup splash. -Pure: stdlib only, no terminal, Textual, Code Mode, or IDA. +Pure: stdlib only, no terminal, Textual, IDA Nexus, or IDA. """ from __future__ import annotations diff --git a/tests/test_launch.py b/tests/test_launch.py index d57ee5f..35a0b8d 100644 --- a/tests/test_launch.py +++ b/tests/test_launch.py @@ -3,12 +3,12 @@ The old `_sweep_locks` deleted `.id0/.id1/.id2/.nam/.til` next to the user's binary when a database failed to open. That was only defensible while the TUI -exclusively owned a private worker; under Code Mode a GUI or another client may +exclusively owned a private worker; under IDA Nexus a GUI or another client may own the database, so the sweep is gone. Its tests are replaced by one that keeps it gone -- deleting a shared database's working files is unrecoverable, and this is the cheapest guard against someone reintroducing the "helpful" cleanup. -Pure: no IDA, no Code Mode library, no Textual. +Pure: no IDA, no IDA Nexus library, no Textual. """ from __future__ import annotations @@ -47,7 +47,7 @@ def touch(*paths): def t_no_lock_sweeping(): """The launcher must not delete database working files any more. - Code Mode's registry locks, health probes and IDA itself arbitrate database + IDA Nexus's registry locks, health probes and IDA itself arbitrate database ownership now. A sweep here would delete files out from under a live GUI. """ check("_sweep_locks is gone", not hasattr(launch, "_sweep_locks")) diff --git a/tests/test_nexus_client.py b/tests/test_nexus_client.py new file mode 100644 index 0000000..4ebf753 --- /dev/null +++ b/tests/test_nexus_client.py @@ -0,0 +1,440 @@ +"""IDA-free contract tests for the IDA Nexus client adapter.""" + +from __future__ import annotations + +import os +import queue +import sys +import threading +import time +import tempfile +from dataclasses import dataclass + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +from idatui.errors import IDAConnectionError, IDAToolError # noqa: E402 +from idatui import remote_ops # noqa: E402 +import idatui.nexus_client as module # noqa: E402 +from idatui.nexus_client import NexusClient, _parse_load_args # noqa: E402 + +#: Pure: fakes the DatabaseHandle, never touches IDA or the IDA Nexus library. +NEEDS_IDA = False + +PASS = FAIL = 0 + + +def check(name: str, condition: bool, detail="") -> None: + global PASS, FAIL + if condition: + PASS += 1 + print(f" ok {name}") + else: + FAIL += 1 + print(f" FAIL {name} {detail}") + + +@dataclass(frozen=True) +class FakeEntry: + pid: int = 123 + backend: str = "gui" + record_id: str = "123-abcdef" + exe_path: str = "" + idb_path: str = "" + managed: bool = False + + +_CLOSED = object() + + +class FakeSubscription: + def __init__(self) -> None: + self._queue: queue.Queue = queue.Queue() + self.closed = False + + def __iter__(self): + return self + + def __next__(self): + item = self._queue.get() + if isinstance(item, BaseException): + raise item + if item is _CLOSED: + raise StopIteration + return item + + def emit(self, event: dict) -> None: + self._queue.put(event) + + def close(self) -> None: + if not self.closed: + self.closed = True + self._queue.put(_CLOSED) + + +class FakeHandle: + def __init__(self, path: str) -> None: + self.connected = True + self.instance = FakeEntry(exe_path=path, idb_path=path + ".i64") + self.waited = None + self.saved = 0 + self.closed = False + self.code = "" + self.codes = [] + self.code_timeout = None + self.operation_label = None + self.event_origin_id = "fake-handle-origin" + self.owns_checks = 0 + self.subscription = FakeSubscription() + self.shutdown_calls = [] + self.shutdown_error = None + + def wait_autoanalysis(self, timeout=None): + self.waited = timeout + return {"complete": True, "status": "complete"} + + def execute_python( + self, + code, + timeout=None, + *, + operation_id=None, + operation_label=None, + persist_globals=False, + filename=None, + ): + self.code = code + self.codes.append(code) + self.code_timeout = timeout + self.operation_label = operation_label + result = ( + { + "__remote_ida_status__": "ok", + "__remote_ida_value__": {"sentinel": 7}, + } + if ".modules.get(" in code + else True + ) + return {"result": result, "stdout": "", "stderr": ""} + + def subscribe_idb_events(self): + return self.subscription + + def owns_event(self, event): + self.owns_checks += 1 + return event.get("origin_id") == self.event_origin_id + + def save_database(self): + self.saved += 1 + return {"saved": True, "idb_path": self.instance.idb_path} + + def shutdown_database(self, *, save=True): + self.shutdown_calls.append(save) + if self.shutdown_error is not None: + raise module.RemoteError(self.shutdown_error, self.shutdown_error, 409) + return {"shutting_down": True, "save": save} + + def close(self): + self.subscription.close() + self.connected = False + self.closed = True + + +class FakeDatabaseHandle: + opened = None + opens = 0 + kwargs = None + + @classmethod + def open(cls, path, **kwargs): + cls.opens += 1 + cls.opened = path + cls.kwargs = kwargs + return FakeHandle(path) + + +@dataclass(frozen=True) +class FakeOpenOptions: + """Stand-in for DatabaseOpenOptions when the library is not installed. + + Deliberately STRICT (no **kwargs): an option the adapter invents would + raise here, and `_option_fields_are_real` checks the surviving names + against the real dataclass wherever it is importable. + """ + + spawn: bool = True + startup_timeout: float = 120.0 + output_database: str | None = None + processor: str | None = None + image_base: int | None = None + file_type: str | None = None + new_database: bool = False + + +class FakeBusy(Exception): + """Stand-in for DatabaseBusyError: `except None` is a TypeError.""" + + +class FakeDisconnected(Exception): + """Stand-in for DatabaseDisconnectedError in stdlib-only runs.""" + + +def _open_kwargs_are_real(sent: dict): + """(ok, detail) for the kwargs the adapter passes to DatabaseHandle.open. + + Skips (passes) when ida_nexus is not installed, so the file stays pure. + """ + try: + import inspect + from ida_nexus import DatabaseHandle as Real + except ImportError: + return True, "ida_nexus not installed - signature not checked" + accepted = set(inspect.signature(Real.open).parameters) + unknown = sorted(set(sent) - accepted) + return not unknown, f"open() rejects {unknown}" + + +def _option_fields_are_real(options): + """(ok, detail) for the option names the adapter fills in. + + The open() signature no longer names the loader options -- they moved + inside DatabaseOpenOptions -- so the `loading_address` class of bug now + hides there instead. Check it in the same way. + """ + try: + import dataclasses + from ida_nexus import DatabaseOpenOptions as Real + except ImportError: + return True, "ida_nexus not installed - fields not checked" + accepted = {field.name for field in dataclasses.fields(Real)} + unknown = sorted({f.name for f in dataclasses.fields(options)} - accepted) + return not unknown, f"DatabaseOpenOptions rejects {unknown}" + + +def main() -> int: + proc, base, file_type = _parse_load_args("-parm:ARMv7-M -b800000 -TRaw") + check( + "legacy switches map to typed IDA Nexus options", + (proc, base, file_type) == ("arm:ARMv7-M", 0x8000000, "Raw"), + (proc, base, file_type), + ) + try: + _parse_load_args("-parm -zcustom") + except ValueError as exc: + check("arbitrary IDA switches fail loudly", "cannot represent" in str(exc), exc) + else: + check("arbitrary IDA switches fail loudly", False) + + original = module.DatabaseHandle + module.DatabaseHandle = FakeDatabaseHandle + # The library's own names when it is installed; strict fakes when it is not + # (this file must keep running under a stdlib-only python3). + original_options = module.DatabaseOpenOptions + original_busy = module.DatabaseBusyError + original_disconnected = module.DatabaseDisconnectedError + module.DatabaseOpenOptions = original_options or FakeOpenOptions + module.DatabaseBusyError = original_busy or FakeBusy + module.DatabaseDisconnectedError = original_disconnected or FakeDisconnected + try: + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, "sample.bin") + with open(path, "wb") as file: + file.write(b"sample") + client = NexusClient(path, load_args="-parm:ARMv7-A -b100") + notes = [] + client.connect(timeout=42, progress=notes.append) + handle = client._handle + check( + "connect delegates database discovery to DatabaseHandle.open", + FakeDatabaseHandle.opened == path and handle is not None, + ) + options = FakeDatabaseHandle.kwargs["options"] + check( + "typed loader options cross the dependency boundary", + options.processor == "arm:ARMv7-A" and options.image_base == 0x1000, + options, + ) + check( + "every open option exists in the real library", + *_option_fields_are_real(options), + ) + # A fake that swallows **kwargs cannot catch a keyword the real + # library does not have -- which is exactly how this port shipped + # `loading_address` (the real name is `image_base`) and would have + # raised TypeError on the very first connect. Check the names we + # send against the real signature whenever it is importable. + check( + "every open() keyword exists in the real library", + *_open_kwargs_are_real(FakeDatabaseHandle.kwargs), + ) + check( + "connect waits for IDA Nexus autoanalysis", + handle.waited == 42, + getattr(handle, "waited", None), + ) + check( + "progress distinguishes discovery and backend attachment", + len(notes) == 2 and "gui" in notes[-1], + notes, + ) + result = client.call( + remote_ops.list_funcs, queries=[{"offset": 0, "count": 2}] + ) + check( + "remote operation returns its JSON result", + result == {"sentinel": 7}, + result, + ) + check( + "operation source is real Python installed through ida-domain", + any("db.functions.get_all()" in code for code in handle.codes), + handle.codes[0][:200], + ) + check( + "remote operations attribute IDB events to IDA TUI", + handle.operation_label == "IDA TUI", + handle.operation_label, + ) + batches = [] + delivered = threading.Event() + + def changed(batch): + batches.append(batch) + delivered.set() + + watcher = client.watch_idb_events(changed, debounce=0.05) + handle.subscription.emit({"event_name": "renamed", "origin_id": "peer-1"}) + handle.subscription.emit( + {"event_name": "cmt_changed", "origin_id": "peer-2"} + ) + check( + "event bursts produce one debounced refresh", + delivered.wait(1) and len(batches) == 1 and len(batches[0]) == 2, + batches, + ) + delivered.clear() + handle.subscription.emit( + {"event_name": "renamed", "origin_id": handle.event_origin_id} + ) + time.sleep(0.1) + check( + "the listener uses handle ownership to ignore its own events", + not delivered.is_set() + and len(batches) == 1 + and handle.owns_checks >= 3, + (batches, handle.owns_checks), + ) + handle.subscription.emit( + {"event_name": "byte_patched", "origin_id": "peer-3"} + ) + watcher.close() + time.sleep(0.1) + check( + "closing drops a pending debounced refresh", + not delivered.is_set() and len(batches) == 1, + batches, + ) + check( + "health exposes registry identity", + client.health()["record_id"] == "123-abcdef", + ) + client.save_database() + check("save uses the public IDA Nexus save route", handle.saved == 1) + check( + "GUI leases transfer rather than claiming discard", + client.discard_database() is False and handle.shutdown_calls == [], + handle.shutdown_calls, + ) + handle.instance = FakeEntry( + backend="idalib", managed=True, exe_path=path, idb_path=path + ".i64" + ) + check( + "a final managed lease discards without saving", + client.discard_database() is True and handle.shutdown_calls == [False], + handle.shutdown_calls, + ) + handle.shutdown_error = "instance_shared" + check( + "a shared managed lease transfers finalization", + client.discard_database() is False, + handle.shutdown_calls, + ) + handle.shutdown_error = "instance_busy" + try: + client.discard_database(timeout=0) + except IDAToolError as exc: + check( + "a busy final lease never silently saves", + exc.tool == "shutdown_database", + exc, + ) + else: + check("a busy final lease never silently saves", False) + handle.shutdown_error = None + handle.instance = FakeEntry(exe_path=path, idb_path=path + ".i64") + opens = FakeDatabaseHandle.opens + handle.connected = False + try: + client.health() + except IDAConnectionError as exc: + check( + "a disconnected handle requires explicit rediscovery", + "explicit rediscovery" in str(exc) + and FakeDatabaseHandle.opens == opens, + (exc, FakeDatabaseHandle.opens, opens), + ) + else: + check("a disconnected handle requires explicit rediscovery", False) + client.close() + check("close releases only the handle lease", handle.closed) + check( + "GUI lifetime is never claimed by the client", + client.wait_released(0) is False, + ) + disconnected = NexusClient(path).connect() + stream_errors = [] + stream_failed = threading.Event() + + def failed(error): + stream_errors.append(error) + stream_failed.set() + + stream_watch = disconnected.watch_idb_events( + lambda _batch: None, on_error=failed, debounce=0 + ) + disconnected._handle.subscription.emit( + module.DatabaseDisconnectedError("GUI database closed") + ) + check( + "stream disconnects become application connection errors", + stream_failed.wait(1) + and isinstance(stream_errors[0], IDAConnectionError), + stream_errors, + ) + stream_watch.close() + disconnected.close() + finally: + module.DatabaseHandle = original + module.DatabaseOpenOptions = original_options + module.DatabaseBusyError = original_busy + module.DatabaseDisconnectedError = original_disconnected + + client = NexusClient(__file__) + + def unknown_operation(): + pass + + try: + client.call(unknown_operation) + except IDAToolError as exc: + check( + "unknown adapter operations are explicit", + exc.tool == "unknown_operation", + ) + else: + check("unknown adapter operations are explicit", False) + + print(f"\n{PASS} passed, {FAIL} failed") + return 1 if FAIL else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_pool.py b/tests/test_pool.py index 6309e13..37058bf 100644 --- a/tests/test_pool.py +++ b/tests/test_pool.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Unit tests for idatui.pool (Code Mode lease residency and LRU budget). +"""Unit tests for idatui.pool (IDA Nexus lease residency and LRU budget). A fake client keeps the policy testable without IDA or Textual. @@ -31,7 +31,7 @@ def check(name, cond, detail=""): class FakeClient: - """Stands in for a CodeModeClient lease and records saves/closes.""" + """Stands in for a NexusClient lease and records saves/closes.""" def __init__(self, ref, mem=100, backend="idalib", discardable=True): diff --git a/tests/test_project.py b/tests/test_project.py index 690f360..bae0802 100644 --- a/tests/test_project.py +++ b/tests/test_project.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """Unit tests for idatui.project (the multi-binary project model + staging). -IDA-free: exercises staging plus Code Mode ownership checks without opening a database. +IDA-free: exercises staging plus IDA Nexus ownership checks without opening a database. python tests/test_project.py """ diff --git a/tests/test_scenarios.py b/tests/test_scenarios.py index 45ed51b..190777c 100644 --- a/tests/test_scenarios.py +++ b/tests/test_scenarios.py @@ -5272,7 +5272,7 @@ async def run(binary, only=None): async def _run_on(binary, only=None): - # Code Mode attaches a registered GUI or starts/reuses a managed worker. + # IDA Nexus attaches a registered GUI or starts/reuses a managed worker. app = IdaTui(open_path=binary, keepalive=False) async with app.run_test(size=(140, 44)) as pilot: c = Ctx(app, pilot) diff --git a/tests/test_thumb_ui.py b/tests/test_thumb_ui.py index f12fbc4..1b4c438 100644 --- a/tests/test_thumb_ui.py +++ b/tests/test_thumb_ui.py @@ -50,7 +50,7 @@ def check(name, ok, detail=""): #: #: This suite used to delete .i64 and reopen the SAME path for each phase. #: That was safe when the TUI owned a private worker that died with it; under -#: Code Mode the database is leased and the previous phase's worker can still +#: IDA Nexus the database is leased and the previous phase's worker can still #: hold it through its lease grace, so the delete raced a live owner and the #: next open never produced a listing (the crash this fixed). Separate paths #: cannot collide, and nothing has to wait for anyone else to let go. diff --git a/tests/test_trace_ui.py b/tests/test_trace_ui.py index 855ce4d..586ce2e 100644 --- a/tests/test_trace_ui.py +++ b/tests/test_trace_ui.py @@ -115,7 +115,7 @@ async def run() -> int: # `app._t` is assigned the moment the key is handled, so it is NOT a # signal that the VIEW has followed -- the navigation it kicks off # runs in a worker. Waiting on it and then reading the cursor was a - # race that the (slower) Code Mode backend loses. Gate on the thing + # race that the (slower) IDA Nexus backend loses. Gate on the thing # the check is about. await settle(app, lambda: app._t == 1 and lst._cursor_ea() == t.ip(1), timeout=20) diff --git a/uv.lock b/uv.lock index 7cbfa25..42e544a 100644 --- a/uv.lock +++ b/uv.lock @@ -12,31 +12,31 @@ wheels = [ ] [[package]] -name = "ida-codemode" -version = "0.6.1" +name = "ida-domain" +version = "0.5.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "ida-domain" }, + { name = "idapro" }, { name = "packaging" }, - { name = "zeromcp" }, + { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/5d/55/b9b72626e371bb36b712659b567ad1d890947c237bfbcaeccfb30f12fe80/ida_codemode-0.6.1.tar.gz", hash = "sha256:44df2a986d24a7e35e64b92c910a21eb485dc4c47d1e9db1e572e8fc1aa08a44", size = 188081, upload-time = "2026-08-13T16:46:35.504Z" } +sdist = { url = "https://files.pythonhosted.org/packages/80/34/be087d3ea1c3a6573e0660cb5b40f0c4ade9ae5772cf1c5d98d52472d28b/ida_domain-0.5.1.tar.gz", hash = "sha256:c49f2c417047d882e954f651b50a709a3f27903b33ba533b794aa54d6536d16f", size = 396413, upload-time = "2026-08-10T13:32:48.45Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/05/20/2397e9b34cefe7ce01945cc2299a01459b9371c48027f61bfdbd4bfcf677/ida_codemode-0.6.1-py3-none-any.whl", hash = "sha256:69a39e25f7441aab794f6737133f8c6155fd794c77924e46e83cb938acf8944f", size = 104728, upload-time = "2026-08-13T16:46:34.265Z" }, + { url = "https://files.pythonhosted.org/packages/27/78/9c698d818b0fddc6648f703a0821edeeb65b18404b43f556d249c5446c96/ida_domain-0.5.1-py3-none-any.whl", hash = "sha256:bfbb17c7d0cb2ed7d3f21342e1c8787f9018d5c2a94cdfd29e06537dd026a06d", size = 201275, upload-time = "2026-08-10T13:32:46.955Z" }, ] [[package]] -name = "ida-domain" -version = "0.5.1" +name = "ida-nexus" +version = "0.7.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "idapro" }, + { name = "ida-domain" }, { name = "packaging" }, - { name = "typing-extensions" }, + { name = "zeromcp" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/80/34/be087d3ea1c3a6573e0660cb5b40f0c4ade9ae5772cf1c5d98d52472d28b/ida_domain-0.5.1.tar.gz", hash = "sha256:c49f2c417047d882e954f651b50a709a3f27903b33ba533b794aa54d6536d16f", size = 396413, upload-time = "2026-08-10T13:32:48.45Z" } +sdist = { url = "https://files.pythonhosted.org/packages/b6/93/2f87cbd64ffc45e181542f133ba8db101c9049155b5f015d9d318f32dcfb/ida_nexus-0.7.0.tar.gz", hash = "sha256:838698c6a2456d474da833b2f4955da9f1fca4a488c95a157539a3c663a919ba", size = 220190, upload-time = "2026-08-20T21:52:18.228Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/27/78/9c698d818b0fddc6648f703a0821edeeb65b18404b43f556d249c5446c96/ida_domain-0.5.1-py3-none-any.whl", hash = "sha256:bfbb17c7d0cb2ed7d3f21342e1c8787f9018d5c2a94cdfd29e06537dd026a06d", size = 201275, upload-time = "2026-08-10T13:32:46.955Z" }, + { url = "https://files.pythonhosted.org/packages/a5/c2/58704fc74618c7867cf7542d3150a75a7678aae2fff7960fa8e5cc67d934/ida_nexus-0.7.0-py3-none-any.whl", hash = "sha256:a5006c7170a0a758a598b864d6fa248bf4eccbea848a30fc6a3eeeadf638d07a", size = 127656, upload-time = "2026-08-20T21:52:19.395Z" }, ] [[package]] @@ -53,7 +53,7 @@ name = "idatui" version = "0.0.1" source = { editable = "." } dependencies = [ - { name = "ida-codemode" }, + { name = "ida-nexus" }, { name = "pygments" }, { name = "textual" }, ] @@ -65,7 +65,7 @@ dev = [ [package.metadata] requires-dist = [ - { name = "ida-codemode", specifier = ">=0.5.3" }, + { name = "ida-nexus", specifier = ">=0.7.0" }, { name = "pygments", specifier = ">=2" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=8" }, { name = "textual", specifier = ">=8" }, -- cgit v1.3.1-sl0p