From f3715d8de0d255c8b14710acfa120ccb9ea953fd Mon Sep 17 00:00:00 2001 From: Duncan Ogilvie Date: Thu, 20 Aug 2026 23:42:42 +0200 Subject: Adopt idb_events and remote module features from ida-codemode --- experiments/bench_ops.py | 33 ++++++++---- experiments/bench_pack_trace.py | 104 ++++++++++++++++--------------------- experiments/profile_client.py | 15 ++++-- experiments/profile_remote.py | 111 ++++++++++++++++------------------------ 4 files changed, 120 insertions(+), 143 deletions(-) (limited to 'experiments') diff --git a/experiments/bench_ops.py b/experiments/bench_ops.py index 178b4bc..9cc51fa 100644 --- a/experiments/bench_ops.py +++ b/experiments/bench_ops.py @@ -27,6 +27,7 @@ ida-codemode is installed **editable** into both venvs, so checking that repo ou swaps the backend under the TUI with no reinstall -- which is what makes this A/B cheap. """ + from __future__ import annotations import argparse @@ -34,6 +35,7 @@ import os import statistics import time +from idatui import remote_ops from idatui.codemode_client import CodeModeClient @@ -59,7 +61,7 @@ def main() -> int: # Work on the biggest function we can find, so the payload-heavy operations # are actually payload-heavy. - index = client.invoke("list_funcs", queries=[{"offset": 0, "count": 60}]) + index = client.call(remote_ops.list_funcs, queries=[{"offset": 0, "count": 60}]) funcs = (index.get("result") or [{}])[0].get("data") or [] if not funcs: print("VERDICT: FAIL - no functions") @@ -71,19 +73,30 @@ def main() -> int: # Synthetic: isolates the per-operation floor (execute_sync marshalling). ("empty round trip", lambda: handle.execute_python("result = 1")), # Payload-dominated: what _PACK_EPILOGUE was written for. - ("list_funcs 500", lambda: client.invoke( - "list_funcs", queries=[{"offset": 0, "count": 500}])), - ("heads 200 (listing page)", lambda: client.invoke( - "heads", addr=ea, count=200, annotate=True)), + ( + "list_funcs 500", + lambda: client.call( + remote_ops.list_funcs, queries=[{"offset": 0, "count": 500}] + ), + ), + ( + "heads 200 (listing page)", + lambda: client.call(remote_ops.heads, addr=ea, count=200, annotate=True), + ), # IDA-work-dominated: Hex-Rays, nothing upstream can move. - ("decompile (warm)", lambda: client.invoke("decompile", addr=ea)), - ("flowchart (graph)", lambda: client.invoke("flowchart", addr=ea)), + ("decompile (warm)", lambda: client.call(remote_ops.decompile, addr=ea)), + ("flowchart (graph)", lambda: client.call(remote_ops.flowchart, addr=ea)), # Round-trip-dominated: small payload, so only the floor matters. - ("xrefs_to", lambda: client.invoke("xref_query", direction="to", addr=ea)), + ( + "xrefs_to", + lambda: client.call(remote_ops.xref_query, direction="to", addr=ea), + ), ] - print(f"# target={os.path.basename(args.target)} func={ea} reps={args.reps} " - f"backend={client.backend}") + print( + f"# target={os.path.basename(args.target)} func={ea} reps={args.reps} " + f"backend={client.backend}" + ) results = {} for name, fn in ops: try: diff --git a/experiments/bench_pack_trace.py b/experiments/bench_pack_trace.py index c6ca980..edbb9f7 100644 --- a/experiments/bench_pack_trace.py +++ b/experiments/bench_pack_trace.py @@ -1,59 +1,49 @@ -"""Measure ``_PACK_EPILOGUE`` against the live ida-codemode runtime. +"""Measure cold installation versus warm calls for typed remote modules. -Our snippets return one pre-serialised JSON STRING instead of a structure, to -dodge to_jsonable()'s Python-level walk of the result (a 200-row listing page -is ~10k small objects). ida-codemode 0.3.2 gave that path a C fast path -- -``serialization.dumps_json`` hands the structure straight to ``json.dumps`` and -only falls back to the walker for values the encoder rejects -- so the packing -now costs a double encode (escaping the whole payload as a string literal) to -avoid a walk that may no longer happen. - -This script answers whether packing still pays. Its sibling question, the -``sys.settrace`` strip, is settled: 0.3.2 deleted the trace hook, the workaround -measured 0.99x, and it has been removed. +Historical note: this file used to benchmark ``_PACK_EPILOGUE``. Application +scripts are no longer strings and packing is gone; the relevant design cost is +now the one-time content-addressed module installation versus steady-state calls. Usage:: - PYTHONPATH=. ~/ida-venv/bin/python experiments/bench_pack_trace.py [FILE] + PYTHONPATH=. python experiments/bench_pack_trace.py [FILE] """ + from __future__ import annotations import argparse -import json import os import statistics import time -from idatui import codemode_client as cc +from idatui import remote_ops from idatui.codemode_client import CodeModeClient -def _time(fn, reps: int) -> tuple[float, float]: - """Best-of and median wall time in ms; best-of resists co-tenant noise.""" +def timed(function, reps: int = 1) -> tuple[object, float]: samples = [] + result = None for _ in range(reps): started = time.perf_counter() - fn() + result = function() samples.append((time.perf_counter() - started) * 1000.0) - return min(samples), statistics.median(samples) + return result, statistics.median(samples) def main() -> int: - ap = argparse.ArgumentParser() - ap.add_argument("target", nargs="?", default="targets/bash") - ap.add_argument("--reps", type=int, default=25) - ap.add_argument("--rows", type=int, default=200) - args = ap.parse_args() - - target = os.path.abspath(args.target) - client = CodeModeClient(target) - client.connect() - - # A real listing page: the flow the workaround was tuned for. - # list_funcs answers {"result": [{"data": [...], "total": N}]}. - index = client.invoke("list_funcs", queries=[{"offset": 0, "count": 40}]) + parser = argparse.ArgumentParser() + parser.add_argument("target", nargs="?", default="targets/bash") + parser.add_argument("--reps", type=int, default=25) + parser.add_argument("--rows", type=int, default=200) + args = parser.parse_args() + + client = CodeModeClient(os.path.abspath(args.target)).connect() + + index, operations_cold = timed( + lambda: client.call(remote_ops.list_funcs, queries=[{"offset": 0, "count": 40}]) + ) funcs = (index.get("result") or [{}])[0].get("data") or [] - biggest = max(funcs, key=lambda f: f.get("size") or 0, default=None) + biggest = max(funcs, key=lambda function: function.get("size") or 0, default=None) if not biggest: print("VERDICT: FAIL - no functions") return 1 @@ -61,34 +51,26 @@ def main() -> int: if isinstance(addr, int): addr = hex(addr) - page = lambda: client.invoke( # noqa: E731 - "heads", addr=addr, count=args.rows, annotate=True) - - # _script() reads _PACK_EPILOGUE at CALL time, so both variants share one - # process -- one lease, one warm database, one fair baseline. _unpack() - # passes an unpacked answer through untouched, so plain `result` works. - packed_epilogue = cc._PACK_EPILOGUE - results = {} - for packing in (True, False): - cc._PACK_EPILOGUE = packed_epilogue if packing else "\nresult\n" - payload = page() - rows = len(payload.get("heads", [])) - for _ in range(3): # warm caches; the first sample is always an outlier - page() - results[packing] = (rows, *_time(page, args.reps)) - cc._PACK_EPILOGUE = packed_epilogue - - size = len(json.dumps(payload, separators=(",", ":"), default=str)) / 1024 - print(f"target {os.path.basename(target)} backend={client.backend}") - print(f"listing page func {addr}, {results[True][0]} rows, " - f"{size:.1f} KiB of JSON, {args.reps} reps") - for packing, label in ((True, "packed string (current)"), - (False, "plain structure")): - rows, best, med = results[packing] - print(f" {label:<26} best {best:7.2f}ms median {med:7.2f}ms" - f" rows={rows}") - print(f" packing buys " - f"{results[False][1] / results[True][1]:.2f}x") + _, operations_warm = timed( + lambda: client.call( + remote_ops.list_funcs, queries=[{"offset": 0, "count": 40}] + ), + args.reps, + ) + page = lambda: client.call( # noqa: E731 + remote_ops.heads, addr=addr, count=args.rows, annotate=True + ) + payload, tools_cold = timed(page) + _, tools_warm = timed(page, args.reps) + + print(f"target {os.path.basename(args.target)} backend={client.backend}") + print(f"function {addr}") + print(f"listing rows {len(payload.get('heads', []))}") + print( + f"operations.py cold {operations_cold:8.3f}ms warm {operations_warm:8.3f}ms" + ) + print(f"remote_tools.py cold {tools_cold:8.3f}ms warm {tools_warm:8.3f}ms") + print(f"tools install overhead {tools_cold / max(tools_warm, 0.001):.2f}x one time") client.close() return 0 diff --git a/experiments/profile_client.py b/experiments/profile_client.py index 77f16e7..bac130e 100644 --- a/experiments/profile_client.py +++ b/experiments/profile_client.py @@ -10,6 +10,7 @@ nothing else here can see it. Time spent in `invoke` is the backend + transport; everything below it in the `tottime` list is ours and is what this file is for. """ + from __future__ import annotations import argparse @@ -19,6 +20,7 @@ import os import pstats import time +from idatui import remote_ops from idatui.codemode_client import CodeModeClient from idatui.domain import Program @@ -28,14 +30,15 @@ def main() -> int: ap.add_argument("binary", nargs="?", default="targets/bash") ap.add_argument("--pages", type=int, default=60) ap.add_argument("--lines", type=int, default=16) - ap.add_argument("--text", action="store_true", - help="load full pages instead of skeletons") + ap.add_argument( + "--text", action="store_true", help="load full pages instead of skeletons" + ) args = ap.parse_args() client = CodeModeClient(os.path.abspath(args.binary)) client.connect() program = Program(client) - regions = client.invoke("file_regions") + regions = client.call(remote_ops.file_regions) rows = regions.get("regions") or regions.get("result") or [] text_seg = next((r for r in rows if ".text" in str(r.get("name", ""))), rows[0]) model = program.listing(int(str(text_seg["start"]), 16)) @@ -57,8 +60,10 @@ def main() -> int: pr.disable() wall = (time.perf_counter() - started) * 1000 - print(f"# {os.path.basename(args.binary)} pages={loaded} " - f"text={want_text} {wall:.0f}ms ({wall/max(loaded,1):.2f}ms/page)") + print( + f"# {os.path.basename(args.binary)} pages={loaded} " + f"text={want_text} {wall:.0f}ms ({wall / max(loaded, 1):.2f}ms/page)" + ) buf = io.StringIO() pstats.Stats(pr, stream=buf).sort_stats("tottime").print_stats(args.lines) print(buf.getvalue()) diff --git a/experiments/profile_remote.py b/experiments/profile_remote.py index ca673db..97f4fb1 100644 --- a/experiments/profile_remote.py +++ b/experiments/profile_remote.py @@ -1,94 +1,71 @@ -"""Profile an operation INSIDE the database process. +"""Profile persistent ida-tui operations inside the IDA process. -`bench_ops.py` says how long an operation takes; this says where that time -goes. The snippet ships cProfile into the Code Mode sandbox, runs the real -remote-library function there in a loop, and returns the stats as text -- so -the split between IDA's own calls and OUR python in `remote_tools.py` is -visible, which no client-side timer can see. +The profiler itself is a typed ``RemoteModule`` function in ``remote_tools.py``; +this file contains no generated Python source or knowledge of remote module names. - PYTHONPATH=. ~/ida-venv/bin/python experiments/profile_remote.py [BINARY] - PYTHONPATH=. ~/ida-venv/bin/python experiments/profile_remote.py --op decompile +Usage:: -Read the `tottime` column: time in that function excluding subcalls. IDA -builtins (generate_disasm_line, get_flags, next_head...) are the floor; a -python frame from ida_tui_remote near the top is ours, and ours is fixable. + PYTHONPATH=. python experiments/profile_remote.py [BINARY] + PYTHONPATH=. python experiments/profile_remote.py --op decompile """ + from __future__ import annotations import argparse import os -import sys - -from idatui.codemode_client import CodeModeClient, _REMOTE_MODULE, _script -# Runs in the database process. `a` is the bound argument dict. -PROFILE = ''' -import cProfile, pstats, io, sys -_m = sys.modules.get(%(mod)r) -if _m is None: - result = {"error": "remote lib not installed yet"} -else: - call = a["call"] - reps = int(a["reps"]) - ns = {"_m": _m, "a": a} - src = "for _ in range(%%d):\\n _m.%%s" %% (reps, call) - code = compile(src, "", "exec") - pr = cProfile.Profile() - pr.enable() - exec(code, ns) - pr.disable() - buf = io.StringIO() - st = pstats.Stats(pr, stream=buf).sort_stats("tottime") - st.print_stats(int(a["lines"])) - result = {"stats": buf.getvalue(), "total": st.total_tt, "reps": reps} -''' % {"mod": _REMOTE_MODULE} +from idatui import remote_ops +from idatui.codemode_client import CodeModeClient CALLS = { - # One full listing page, exactly as the background grower asks for it. - "heads": 'heads(addr=a["addr"], count=500, annotate=True)', - "heads_plain": 'heads(addr=a["addr"], count=500, annotate=False)', - "heads_skeleton": 'heads(addr=a["addr"], count=500, annotate=True, text=False)', - "decompile": 'decompile(a["addr"])', - "disasm": 'disasm(a["addr"], 500)', + "heads": ("heads", {"count": 500, "annotate": True}), + "heads_plain": ("heads", {"count": 500, "annotate": False}), + "heads_skeleton": ( + "heads", + {"count": 500, "annotate": True, "text": False}, + ), + "decompile": ("decompile", {}), } def main() -> int: - ap = argparse.ArgumentParser() - ap.add_argument("binary", nargs="?", default="targets/bash") - ap.add_argument("--op", default="heads", choices=sorted(CALLS)) - ap.add_argument("--reps", type=int, default=20) - ap.add_argument("--lines", type=int, default=18) - ap.add_argument("--addr", default=None, help="default: the .text start") - args = ap.parse_args() - - client = CodeModeClient(os.path.abspath(args.binary)) - client.connect() + parser = argparse.ArgumentParser() + parser.add_argument("binary", nargs="?", default="targets/bash") + parser.add_argument("--op", default="heads", choices=sorted(CALLS)) + parser.add_argument("--reps", type=int, default=20) + parser.add_argument("--addr", default=None, help="default: the .text start") + args = parser.parse_args() + client = CodeModeClient(os.path.abspath(args.binary)).connect() addr = args.addr if addr is None: - regions = client.invoke("file_regions") + regions = client.call(remote_ops.file_regions) rows = regions.get("regions") or regions.get("result") or [] - text = next((r for r in rows if ".text" in str(r.get("name", ""))), None) + text = next( + (row for row in rows if ".text" in str(row.get("name", ""))), + None, + ) addr = (text or rows[0])["start"] if rows else "0x0" - print(f"# {os.path.basename(args.binary)} op={args.op} addr={addr} reps={args.reps}") - # Prime: the remote lib installs lazily, and its lru_caches must be warm or - # the profile measures cache misses that the real workload never pays. - client.invoke("heads", addr=addr, count=500, annotate=True) + operation, call_args = CALLS[args.op] + call_args = {"addr": addr, **call_args} + print( + f"# {os.path.basename(args.binary)} op={args.op} " + f"addr={addr} reps={args.reps}" + ) - # _script binds the args as JSON and adds the pack epilogue, exactly as a - # real operation is shipped -- so this measures the same path, not a - # special one. - out = client._unpack(client.execute_python(_script( - {"call": CALLS[args.op], "addr": addr, - "reps": args.reps, "lines": args.lines}, PROFILE), timeout=600)) - if "error" in out: - print("FAILED:", out["error"]) - return 1 + # Install the persistent tool module and warm its caches before profiling. + client.call(remote_ops.heads, addr=addr, count=500, annotate=True) + out = client.call( + remote_ops.profile_remote, + operation=operation, + args=call_args, + reps=args.reps, + ) per = out["total"] / out["reps"] * 1000 - print(f"# {out['total']*1000:.0f}ms total, {per:.1f}ms per call\n") + print(f"# {out['total'] * 1000:.0f}ms total, {per:.1f}ms per call\n") print(out["stats"]) + client.close() return 0 -- cgit v1.3.1-sl0p From f4c1d9b5497fd0d38137b6b345e8171b307c1117 Mon Sep 17 00:00:00 2001 From: Duncan Ogilvie Date: Thu, 20 Aug 2026 23:55:52 +0200 Subject: Switch to ida-nexus --- CONTRIBUTING.md | 8 +- README.md | 18 +- TODO | 4 +- docs/CODEMODE_UPSTREAM.md | 385 ---------------------------- docs/GRAPH_VIEW.md | 2 +- docs/NEXUS_UPSTREAM.md | 385 ++++++++++++++++++++++++++++ docs/PAGING_FINDINGS.md | 14 +- docs/PROJECTS.md | 10 +- docs/SPLIT_VIEW.md | 8 +- experiments/bench_ops.py | 12 +- experiments/bench_pack_trace.py | 4 +- experiments/call_census.py | 10 +- experiments/profile_client.py | 4 +- experiments/profile_remote.py | 4 +- experiments/worker_smoke.py | 10 +- ida-tui | 2 +- idatui/__init__.py | 6 +- idatui/app.py | 42 +-- idatui/codemode_client.py | 551 ---------------------------------------- idatui/domain.py | 28 +- idatui/errors.py | 2 +- idatui/launch.py | 16 +- idatui/nexus_client.py | 551 ++++++++++++++++++++++++++++++++++++++++ idatui/pane.py | 18 +- idatui/pool.py | 14 +- idatui/project.py | 14 +- idatui/remote_ops.py | 4 +- idatui/remote_tools.py | 12 +- pyproject.toml | 7 +- tests/_fixtures.py | 2 +- tests/run.py | 4 +- tests/test_codemode_client.py | 440 -------------------------------- tests/test_kittygfx.py | 2 +- tests/test_launch.py | 6 +- tests/test_nexus_client.py | 440 ++++++++++++++++++++++++++++++++ tests/test_pool.py | 4 +- tests/test_project.py | 2 +- tests/test_scenarios.py | 2 +- tests/test_thumb_ui.py | 2 +- tests/test_trace_ui.py | 2 +- uv.lock | 28 +- 41 files changed, 1539 insertions(+), 1540 deletions(-) delete mode 100644 docs/CODEMODE_UPSTREAM.md create mode 100644 docs/NEXUS_UPSTREAM.md delete mode 100644 idatui/codemode_client.py create mode 100644 idatui/nexus_client.py delete mode 100644 tests/test_codemode_client.py create mode 100644 tests/test_nexus_client.py (limited to 'experiments') diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8efdfe7..b66df1c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -18,12 +18,12 @@ without a licence. uv sync ``` -That pulls [ida-codemode](https://github.com/HexRaysSA/ida-codemode) from PyPI, +That pulls [ida-nexus](https://github.com/HexRaysSA/ida-nexus) from PyPI, which is how ida-tui talks to IDA. To also attach to databases open in the IDA GUI: ```sh -uvx ida-hcli plugin install ida-codemode +uvx ida-hcli plugin install ida-nexus ``` ## Running the tests @@ -39,7 +39,7 @@ python3 tests/run.py # everything (needs IDA) ``` The IDA-backed suites need an interpreter that has `textual`, `idapro` and -`ida_codemode` on it: +`ida_nexus` on it: ```sh tests/test_scenarios.py /path/to/binary --only rename @@ -47,7 +47,7 @@ The IDA-backed suites need an interpreter that has `textual`, `idapro` and ``` **House rule:** a suite marked `pure` must keep running under a plain system -`python3`. This is why `idatui/codemode_client.py` defers its `ida_codemode` +`python3`. This is why `idatui/nexus_client.py` defers its `ida_nexus` import instead of doing it at module top. Please don't break that — it's what keeps the fast gate fast and lets people without IDA contribute at all. diff --git a/README.md b/README.md index 5cfc59e..9a751a1 100644 --- a/README.md +++ b/README.md @@ -27,18 +27,18 @@ Needs **Python ≥ 3.11** and **IDA Pro 9.4+ with idalib**. uv sync ``` -That pulls [ida-codemode](https://github.com/HexRaysSA/ida-codemode) from PyPI, +That pulls [ida-nexus](https://github.com/HexRaysSA/ida-nexus) from PyPI, which is how ida-tui talks to IDA. To also attach to databases you have open in the IDA GUI, install its plugin: ```sh -uvx ida-hcli plugin install ida-codemode +uvx ida-hcli plugin install ida-nexus ``` -Hacking on ida-codemode itself? Point at a checkout instead: +Hacking on ida-nexus itself? Point at a checkout instead: ```sh -uv add --editable ../ida-codemode +uv add --editable ../ida-nexus ``` ## Run @@ -49,9 +49,9 @@ uv add --editable ../ida-codemode ``` ida-tui never owns an IDA process — it takes a **lease**. A matching database open -in the IDA GUI is reused, otherwise Code Mode starts or shares a managed idalib +in the IDA GUI is reused, otherwise IDA Nexus starts or shares a managed idalib worker. Quitting drops the lease and leaves everyone else alone. -Changes made in the GUI or another client arrive over Code Mode's IDB event +Changes made in the GUI or another client arrive over IDA Nexus's IDB event stream; ida-tui debounces bursts and refreshes its cached views automatically. On quit with unsaved changes, a final managed-worker lease can discard the session without saving; GUI-backed or still-shared sessions leave that final @@ -59,9 +59,9 @@ decision with their owner or remaining clients. If the owning GUI or worker closes, ida-tui never replaces it by spawning a headless worker implicitly. It keeps the cached view disconnected until a matching owner is reopened and an attach-only rediscovery succeeds. -Remote operations are typed, source-backed Python functions. ida-codemode -installs content-addressed modules once per handle, so ida-tui keeps normal -refactorable source without paying to resend hot listing/decompiler code. +Remote operations are typed, source-backed Python functions. ida-nexus installs +their content-addressed modules once per IDA Python interpreter, so ida-tui keeps +normal refactorable source without paying to resend hot listing/decompiler code. Operation attribution is also a per-call provider rather than a fixed string, so it can evolve from `IDA TUI` to labels such as `IDA TUI: alice`. diff --git a/TODO b/TODO index 726e2b1..b497c89 100644 --- a/TODO +++ b/TODO @@ -4,12 +4,12 @@ TODO: [x] PORT TO ida-codemode-mcp as a library dependency [x] DatabaseHandle discovery prefers registered GUI sessions [x] shared managed idalib workers + SSE lease lifecycle - [x] domain operations execute against ida-domain through Code Mode + [x] domain operations execute against ida-domain through IDA Nexus [x] delete the private pickle worker and ida-pro-mcp patch injection [x] stop sweeping/reaping resources that may belong to another client [ ] run the full live Pilot suite against both GUI and managed backends [ ] add database revision/change notifications for cross-client cache invalidation - [ ] decide how "discard changes" should work (Code Mode final workers save) + [ ] decide how "discard changes" should work (IDA Nexus final workers save) - [x] add support for toggling literal types, ala `o` in IDA. (decimal to hex to reference etc.) diff --git a/docs/CODEMODE_UPSTREAM.md b/docs/CODEMODE_UPSTREAM.md deleted file mode 100644 index 7dff13e..0000000 --- a/docs/CODEMODE_UPSTREAM.md +++ /dev/null @@ -1,385 +0,0 @@ -# Findings from porting a real client to IDA Code Mode - -Notes for the `ida-codemode` maintainers, gathered while porting **ida-tui** (a -Textual TUI frontend for IDA) from a private idalib worker to -`ida_codemode.DatabaseHandle`. - -Everything below is measured, not inferred. Where we worked around something, the -workaround is named so you can judge whether the library should make it -unnecessary. - -**Environment:** ida-codemode 0.3.1, IDA 9.4 (idalib), Linux, single managed -worker backend, quiet box. Target for timings: `targets/echo` unless stated. - -> **Status against the protocol-6 event-stream development tree, based on 0.6.1 -> (upstream `439289f`) — every item re-checked.** -> -> | item | verdict | -> |---|---| -> | 1 `timeout_trace` line tracing | ✅ **fixed in 0.3.2** — no `settrace` in the runtime at all | -> | 2 `to_jsonable` on large results | ✅ **fixed in 0.3.2** — `dumps_json` C fast path | -> | 3 2 ms `execute_sync` floor | ✅ **fixed in 0.3.2, 7.0x** — 2.055 ms → 0.294 ms | -> | 4 loader switches fatal on reopen | **partial** — normal reopen fixed in 0.5.x; direct `.i64` paths remain [issue #36](https://github.com/HexRaysSA/ida-codemode/issues/36) | -> | 5 IDB replaced under a live lease | **stale** — out-of-band replacement is outside the supported lifecycle, as it is for the IDA GUI | -> | 6 close without save | **fixed in protocol 6** — the final managed-worker lease can choose `shutdown_database(save=False)` | -> | 7 no change notification | ✅ **fixed in protocol 6** — `DatabaseHandle.subscribe_idb_events()` streams revisioned, operation-attributed IDB changes | -> | 8 package exports | ✅ **fixed in 0.5.x** — a real `__all__` on the package root | -> | 9 no `py.typed` / handle Protocol | ✅ **fixed in 0.5.x** — `ida_codemode/py.typed` ships | -> -> **0.5.x restructured the package**, which is why the old "these files are -> byte-identical" re-check recipe no longer works: `client.py` → `handle.py`, -> `registry.py` → `_registry.py` + `instances.py`, `resolver.py` → `_resolver.py`, -> and the loader options moved into a frozen `DatabaseOpenOptions` dataclass. -> Everything private is now underscore-prefixed, so the cheap re-check after an -> upstream pull is simply: does anything we import still appear in -> `ida_codemode.__all__`? -> -> 0.5.3 → 0.6.1 changed **nothing** we depend on: `__init__.py`, `handle.py`, -> `instances.py`, `options.py`, `errors.py` and `models.py` are byte-identical -> between those two releases. 0.6.1 only collapses the six console scripts into a -> single `ida-codemode` command. -> -> Both client-side workarounds re-measured at **0.99x and 0.97x** on 0.3.2 — -> i.e. nothing — and are deleted. Remote code is now ordinary typed Python, -> installed as content-addressed modules by ida-codemode. Harness: -> `experiments/bench_pack_trace.py`. - -**What the client does**, for scale: it renders a continuous disassembly listing, -pseudocode, a CFG graph view and a hex view, paging over the database as the user -scrolls. It is latency-sensitive in a way an agent-driven MCP client is not — a -keypress must repaint. It issues ~1–8 operations per user action. - ---- - -## 1. `timeout_trace` enables line tracing in every frame — 52x on IDA calls - -**Highest-impact item by a wide margin.** — ✅ **FIXED in 0.3.2.** The runtime no -longer installs a trace hook at all; cancellation is a C-level thread interrupt. -Our `sys.settrace(None)` workaround is deleted as of `a5137fe`. - -`runtime.py` wraps every `execute_python` in `sys.settrace(timeout_trace)` to -enforce the deadline. `timeout_trace` ends with `return timeout_trace`, and -returning a trace function from a `'call'` event asks CPython to trace **every -line of that frame**. So every line of every function the snippet touches pays a -Python-level callback, and the specialising interpreter is disabled throughout. - -Measured inside the worker, same process, same database: - -| | traced (stock) | untraced | native idalib | -|---|---|---|---| -| `ida_bytes.get_flags(ea)` | 5.49 µs | 0.106 µs | 0.119 µs | -| our 200-row listing page | 20.2 ms | 2.0 ms | — | - -Untraced matches a plain idalib process, so the trace hook accounts for -essentially all of it. For us this was the single largest cost in the port — -larger than HTTP, serialisation and IDA itself combined. - -Reproduce inside any `execute_python`: - -```python -import sys, time, ida_bytes -def bench(): - t = time.perf_counter() - for _ in range(20000): ida_bytes.get_flags(0x1000) - return (time.perf_counter() - t) / 20000 * 1e6 -traced = bench() -old = sys.gettrace(); sys.settrace(None) -try: untraced = bench() -finally: sys.settrace(old) -result = {"traced_us": traced, "untraced_us": untraced} -``` - -**Suggested fixes, cheapest first** - -1. `return None` from `timeout_trace` instead of itself. You keep `'call'`-event - deadline checks — which is enough to interrupt anything that calls a function - — and drop per-line tracing entirely. -2. On 3.12+, use `sys.monitoring` with only the events you need; it is designed - for exactly this and is far cheaper than `settrace`. -3. Or drop the trace and rely on the `threading.Timer` → - `ida_kernwin.set_cancelled()` path you already have, accepting that a - pure-Python loop with no calls in it cannot be interrupted. - -**Our workaround** (we would rather not ship it): the snippet detaches the trace -and restores it in a `finally`. That gives up deadline enforcement for -pure-Python loops inside our own code; your native cancel timer is unaffected and -still fires. Every client that does real work per call will eventually find this -and do the same, which is an argument for fixing it in the runtime. - ---- - -## 2. `to_jsonable` dominates any large result - -**FIXED in 0.3.2**, via the first suggested fix below: -`serialization.dumps_json` calls `json.dumps(value, default=to_jsonable)`, so a -JSON-safe result never enters the Python walker. Our packing workaround measured -0.97x and has been deleted. - -`execute_python` runs `to_jsonable()` over whatever the snippet returns. Our -answers are already JSON-safe and they are big — a 200-row listing page is -roughly 10k small objects. - -| | cost | -|---|---| -| `to_jsonable(page)` | 66.2 ms | -| `json.dumps(page, separators=(",",":"))` — same data | 0.58 ms | -| serialised size | 34.9 KB | - -That is 114x, and it was 72% of the page's total cost before we changed it. - -**Suggested fixes** - -- Fast-path values that are already JSON-safe (a cheap recursive type check that - bails to the original object beats rebuilding it), or -- let a snippet opt out by returning an already-serialised payload — a documented - envelope such as `{"__json__": "<...>"}`, or simply passing `str`/`bytes` - through untouched. - -**Retired workaround:** snippets used to `json.dumps` inside the database process -and return one string, which the client parsed. The typed remote API now owns -strict argument/result encoding, and ida-tui contains no generated script -strings or packing envelope. - ---- - -## 3. The per-operation floor is `execute_sync`, not HTTP - -✅ **FIXED in 0.3.2 — 7.0x.** Re-measured as a same-box A/B by checking the -installed editable checkout back to `4195f21` and forward again, 200 iterations -each, `targets/echo`: - -| | 0.3.1 | 0.3.2 | | -|---|---|---|---| -| `GET /health` | 0.497 ms | 0.318 ms | 1.6x | -| `execute_python("result = 1")` | **2.055 ms** | **0.294 ms** | **7.0x** | - -The 0.3.1 column reproduces the original 2.025 ms measurement below almost -exactly, which is what makes the 0.3.2 column believable. `execute_python` now -costs about the same as a bare HTTP GET, so the `execute_sync` marshalling that -was ~93% of the floor is essentially gone. The design advice below — "a client -that makes one call per row will be 20–100x slower than an in-process one" — is -correspondingly much weaker now. - -Original 0.3.1 measurement, same worker, same connection, 200 iterations: - -| | cost | -|---|---| -| `GET /health` (no `execute_sync`) | **0.165 ms** | -| `execute_python("result = 1")` | **2.025 ms** | - -HTTP framing is ~7% of the floor; marshalling the operation onto IDA's main -thread is the other ~93%. The worker runs IDA's own `kernwin.serve()`, so this is -plausibly IDA's dispatch latency rather than anything you control — but it is -worth **documenting**, because it sets a hard 2 ms per-operation budget that -shapes how a client must be designed. - -It did not hurt us (our call volume is 1–8 per user action; 4 calls to build a -1060-block graph), but a client that makes one call per row or per symbol will be -20–100x slower than an in-process one and the authors will not know why. - -**Suggested fixes:** document the floor; and consider a batch endpoint — accept -`[{op, args}, ...]` and dispatch them within a single `execute_sync` — which -would let chatty clients amortise it without redesigning around it. - ---- - -## 4. Loader switches on an existing database are a FATAL, not an error — one edge remains - -Opening a target that already has an `.i64`, while passing spawn-only options, -kills the worker: - -``` -FATAL ERROR: @0:636[] -Switch '-b400' can be used only when loading a new file -``` - -The client sees only: - -``` -IDAConnectionError: idalib worker launcher exited with status 1 -``` - -This is easy to hit and hard to diagnose: it is the natural second run of -anything that opens a raw blob (`processor=`/`image_base=`/`file_type=` are -recorded in the database the first run produced). Our test suite hit it as a -crash five minutes into a run. - -**Suggested fixes** - -- In `DatabaseHandle.open()`, when the resolved IDB already exists and - `new_database` is not set, either ignore the spawn-only options or raise a - typed error naming them — before handing them to IDA. -- Propagate the worker's fatal text into the client exception. The message - already exists on the worker's stderr; losing it turns a one-line fix into a - bisect. - -**Our workaround:** the client checked whether the expected IDB exists and -dropped `processor`/`image_base`/`file_type` when it did. - -**FIXED in 0.5.x**, with exactly this fix, in `_resolver._build_worker_command`: - -```python -if input_path == expected_idb and input_path != source: - # Loader/import switches are baked into an existing IDB... - options = WorkerLaunchOptions() -``` - -Our workaround is therefore deleted. **One narrow case remains**: the strip needs -`input_path != source`, so passing an `.i64` path *directly* together with load -options (`ida-tui foo.i64 --processor arm`) still forwards the switches and still -fatals. Our old guard keyed on "the target IDB exists" and so covered it. It is a -nonsense invocation and no ida-tui code path generates it — the project layer -always passes `output_database`, and `_needs_load_options` bails when an `.i64` -exists — but the library boundary should still reject or normalize it rather -than launch a known-fatal IDA command. Tracked upstream as -[issue #36](https://github.com/HexRaysSA/ida-codemode/issues/36). - ---- - -## 5. Deleting or replacing an IDB under a live lease — STALE - -The original suite deleted an `.i64` while a private worker still had it open, -then immediately reopened the same path. That ownership model no longer applies: -Code Mode databases are shared resources, and the IDA GUI itself does not survive -out-of-band replacement of its open database. Detecting arbitrary filesystem -replacement is therefore not part of the supported lifecycle. - -The actionable lifecycle gaps that originally forced private-registry access are -fixed. `find_database_owner()` and `wait_database_released()` are public exports; -`DatabaseHandle.close(wait_for_database=True)` can wait for a final managed close; -a draining owner remains registered until the IDB is actually closed; and -`new_database=True` refuses to replace a live owner. - -ida-tui now uses the public owner/release API while recreating a database and no -longer reaches into registry locks. Owner loss is attach-only: ida-tui will -rediscover a replacement GUI or worker, but will never turn a -user-closing-the-GUI action into an implicit headless reopen. There is no -remaining upstream request in this section. - ---- - -## 6. Close without save — FIXED in protocol 6 - -`DatabaseHandle.shutdown_database(save=False)` can discard a managed idalib -worker when the requesting handle is its only active lease and no other operation -is running. The server rejects GUI databases and shared workers. - -The coherent ownership model is the **final lease**, not necessarily the lease -that spawned the worker. Releasing a non-final lease makes no whole-database save -decision; responsibility transfers to the leases that remain. The final client -can save or discard the shared session. A client that needs its work to survive -regardless of that later decision must call `save_database()` before releasing -its lease. - -This does not claim to provide per-client rollback. Discard applies to all -changes since the last database save, and attempting it while another lease is -active is correctly rejected. That is the same ref-counted lifetime model used -by other shared resources and requires no separate starter capability. - -The upstream gap is therefore closed. ida-tui now routes its discard action -through `shutdown_database(save=False)`: a final managed-worker lease discards, -while GUI-backed and still-shared sessions transfer finalization to their owner -or remaining leases. - ---- - -## 7. No change notification for shared databases — FIXED in protocol 6 - -`DatabaseHandle.subscribe_idb_events()` now returns a closeable iterator over -structured IDB changes. Each event carries a monotonic revision plus -`operation_id`/`operation_label` attribution and an opaque `origin_id`. -`DatabaseHandle.owns_event()` compares that origin with the handle's lease, so a -caching client does not need to generate, retain, or race operation IDs itself. - -ida-tui keeps one subscription for its active database, asks the handle to drop -its own events, and batches peer events behind a 200 ms quiet period. One batch -invalidates the function, listing, decompiler, graph, strings, linkage, segment -and byte caches, then reloads the visible view in place. Closing or switching -databases closes the subscription, so the blocking event reader does not leak. - ---- - -## 8. Package exports and API surface stability — FIXED in 0.5.x - -`ida_codemode/__init__.py` used to export nothing, so a library consumer had to -import from submodules, including things that were clearly internals (`FileLock`, -`REGISTRY_DIR`, `canonical_path`, `idb_key`, `scan_instances`) that we only -touched because no public equivalent existed. - -**Suggested fix was:** export `DatabaseHandle` and the public exception types from -the package root, and mark the intended-public registry helpers explicitly. - -**That is what 0.5.x did.** Everything we need is now on the package root, and -the internals moved behind an underscore: - -```python -from ida_codemode import DatabaseHandle, DatabaseOpenOptions, DatabaseInstance -from ida_codemode import RemoteError, DatabaseBusyError, DatabaseDisconnectedError -from ida_codemode import discover_databases, find_database_owner, wait_database_released -``` - -The two lock-poking helpers we had reimplemented client-side -(`_wait_for_entry_release`) are now `wait_database_released()`, and our -registry-scanning ownership check is now `find_database_owner()`. Both are -deleted from our tree. Note `find_database_owner()` *raises* -`AmbiguousDatabaseError` where our scan silently took the first match — a -behaviour improvement, but callers need a handler. - ---- - -## 9. A testing note: `DatabaseHandle.open()`'s 30 keyword-only options — FIXED in 0.5.x - -The port we started from called `open(..., loading_address=...)`. The real -parameter is `image_base`. Every `connect()` would have raised `TypeError` on the -first call, and its contract tests passed anyway, because a hand-written fake -handle accepts `**kwargs`. - -Not a library bug — but with 30 keyword-only options it is a very easy mistake, -and it is invisible to exactly the offline tests people write. - -**Suggested fix:** ship `py.typed` and/or a `Protocol` for the handle, so a fake -can be checked against the real signature and a typo is caught statically. (We -added a test asserting our kwargs are a subset of -`inspect.signature(DatabaseHandle.open).parameters`, which is a poor substitute.) - -**0.5.x ships `ida_codemode/py.typed`**, and the 30 keyword-only options became a -frozen `DatabaseOpenOptions` dataclass — which is strictly better, because an -invented option name is now a `TypeError` at construction rather than something a -`**kwargs` fake swallows. Our subset test survives in two halves -(`_open_kwargs_are_real` for `open()`, `_option_fields_are_real` for the -dataclass fields), because the offline contract suite must keep running with no -`ida_codemode` installed at all and therefore still fakes both. - ---- - -## Priority, from a client author's view - -| # | item | impact | fixable by you? | -|---|---|---|---| -| ~~1~~ | ~~`timeout_trace` line tracing~~ | ~~52x on IDA calls~~ | ✅ fixed in 0.3.2 | -| ~~2~~ | ~~`to_jsonable` on large results~~ | ~~114x on serialisation~~ | ✅ fixed in 0.3.2 | -| ~~3~~ | ~~2 ms `execute_sync` floor~~ | ~~shapes client design~~ | ✅ fixed in 0.3.2, 7.0x | -| ~~7~~ | ~~no change/revision counter~~ | ~~correctness for shared editing~~ | ✅ fixed in protocol 6 | -| 4 | direct `.i64` forwards loader-only options | fatal worker startup | [issue #36](https://github.com/HexRaysSA/ida-codemode/issues/36) | -| ~~5~~ | ~~replaced/deleted IDB under lease~~ | ~~out-of-contract filesystem mutation~~ | **stale** | -| ~~6~~ | ~~no close without save~~ | ~~could not discard a managed session~~ | **fixed in protocol 6: final lease decides** | -| ~~8~~ | ~~package exports~~ | ~~forces internal imports~~ | ✅ fixed in 0.5.x | -| ~~9~~ | ~~typed handle for fakes~~ | ~~catches a whole bug class~~ | ✅ fixed in 0.5.x (`py.typed` + options dataclass) | - -Items 1 and 2 together were the difference between "the port is 35x slower than -the private worker it replaced" and "the port is within 2x, and faster on several -operations". Both are in the runtime, not in client code — which is why they are -worth fixing centrally rather than leaving each client to rediscover. - -**Both landed in 0.3.2**, along with item 3 — all three performance items are now -fixed upstream, and both client-side workarounds could be measured at parity and -retired. That is the outcome this document was written for. - -**What is left is entirely non-performance.** Items 6 through 9 are fixed, and -item 5 is stale because out-of-band replacement is not a supported lifecycle for -either Code Mode or the IDA GUI. One narrow piece remains: **4**, normalize or -reject loader-only options when the source is itself an existing `.i64` -([issue #36](https://github.com/HexRaysSA/ida-codemode/issues/36)). - -Happy to supply the benchmark harness (it is backend-agnostic and runs against -both our old worker and Code Mode), or to test a patch. diff --git a/docs/GRAPH_VIEW.md b/docs/GRAPH_VIEW.md index 3aced1f..0b24aa5 100644 --- a/docs/GRAPH_VIEW.md +++ b/docs/GRAPH_VIEW.md @@ -54,7 +54,7 @@ Growing a second disassembly renderer for graph mode would have been the real cost. The backend adds exactly one operation, `flowchart(addr)` in -`idatui/codemode_client.py`, which returns block ranges and typed edges — **not** +`idatui/nexus_client.py`, which returns block ranges and typed edges — **not** text. ## Two layout engines diff --git a/docs/NEXUS_UPSTREAM.md b/docs/NEXUS_UPSTREAM.md new file mode 100644 index 0000000..f4dcf91 --- /dev/null +++ b/docs/NEXUS_UPSTREAM.md @@ -0,0 +1,385 @@ +# Findings from porting a real client to IDA Nexus + +Notes for the `ida-nexus` maintainers, gathered while porting **ida-tui** (a +Textual TUI frontend for IDA) from a private idalib worker to +`ida_nexus.DatabaseHandle`. + +Everything below is measured, not inferred. Where we worked around something, the +workaround is named so you can judge whether the library should make it +unnecessary. + +**Environment:** ida-nexus 0.3.1, IDA 9.4 (idalib), Linux, single managed +worker backend, quiet box. Target for timings: `targets/echo` unless stated. + +> **Status against the protocol-6 event-stream development tree, based on 0.6.1 +> (upstream `439289f`) — every item re-checked.** +> +> | item | verdict | +> |---|---| +> | 1 `timeout_trace` line tracing | ✅ **fixed in 0.3.2** — no `settrace` in the runtime at all | +> | 2 `to_jsonable` on large results | ✅ **fixed in 0.3.2** — `dumps_json` C fast path | +> | 3 2 ms `execute_sync` floor | ✅ **fixed in 0.3.2, 7.0x** — 2.055 ms → 0.294 ms | +> | 4 loader switches fatal on reopen | **partial** — normal reopen fixed in 0.5.x; direct `.i64` paths remain [issue #36](https://github.com/HexRaysSA/ida-nexus/issues/36) | +> | 5 IDB replaced under a live lease | **stale** — out-of-band replacement is outside the supported lifecycle, as it is for the IDA GUI | +> | 6 close without save | **fixed in protocol 6** — the final managed-worker lease can choose `shutdown_database(save=False)` | +> | 7 no change notification | ✅ **fixed in protocol 6** — `DatabaseHandle.subscribe_idb_events()` streams revisioned, operation-attributed IDB changes | +> | 8 package exports | ✅ **fixed in 0.5.x** — a real `__all__` on the package root | +> | 9 no `py.typed` / handle Protocol | ✅ **fixed in 0.5.x** — `ida_nexus/py.typed` ships | +> +> **0.5.x restructured the package**, which is why the old "these files are +> byte-identical" re-check recipe no longer works: `client.py` → `handle.py`, +> `registry.py` → `_registry.py` + `instances.py`, `resolver.py` → `_resolver.py`, +> and the loader options moved into a frozen `DatabaseOpenOptions` dataclass. +> Everything private is now underscore-prefixed, so the cheap re-check after an +> upstream pull is simply: does anything we import still appear in +> `ida_nexus.__all__`? +> +> 0.5.3 → 0.6.1 changed **nothing** we depend on: `__init__.py`, `handle.py`, +> `instances.py`, `options.py`, `errors.py` and `models.py` are byte-identical +> between those two releases. 0.6.1 only collapses the six console scripts into a +> single `ida-nexus` command. +> +> Both client-side workarounds re-measured at **0.99x and 0.97x** on 0.3.2 — +> i.e. nothing — and are deleted. Remote code is now ordinary typed Python, +> installed as content-addressed modules by ida-nexus. Harness: +> `experiments/bench_pack_trace.py`. + +**What the client does**, for scale: it renders a continuous disassembly listing, +pseudocode, a CFG graph view and a hex view, paging over the database as the user +scrolls. It is latency-sensitive in a way an agent-driven MCP client is not — a +keypress must repaint. It issues ~1–8 operations per user action. + +--- + +## 1. `timeout_trace` enables line tracing in every frame — 52x on IDA calls + +**Highest-impact item by a wide margin.** — ✅ **FIXED in 0.3.2.** The runtime no +longer installs a trace hook at all; cancellation is a C-level thread interrupt. +Our `sys.settrace(None)` workaround is deleted as of `a5137fe`. + +`runtime.py` wraps every `execute_python` in `sys.settrace(timeout_trace)` to +enforce the deadline. `timeout_trace` ends with `return timeout_trace`, and +returning a trace function from a `'call'` event asks CPython to trace **every +line of that frame**. So every line of every function the snippet touches pays a +Python-level callback, and the specialising interpreter is disabled throughout. + +Measured inside the worker, same process, same database: + +| | traced (stock) | untraced | native idalib | +|---|---|---|---| +| `ida_bytes.get_flags(ea)` | 5.49 µs | 0.106 µs | 0.119 µs | +| our 200-row listing page | 20.2 ms | 2.0 ms | — | + +Untraced matches a plain idalib process, so the trace hook accounts for +essentially all of it. For us this was the single largest cost in the port — +larger than HTTP, serialisation and IDA itself combined. + +Reproduce inside any `execute_python`: + +```python +import sys, time, ida_bytes +def bench(): + t = time.perf_counter() + for _ in range(20000): ida_bytes.get_flags(0x1000) + return (time.perf_counter() - t) / 20000 * 1e6 +traced = bench() +old = sys.gettrace(); sys.settrace(None) +try: untraced = bench() +finally: sys.settrace(old) +result = {"traced_us": traced, "untraced_us": untraced} +``` + +**Suggested fixes, cheapest first** + +1. `return None` from `timeout_trace` instead of itself. You keep `'call'`-event + deadline checks — which is enough to interrupt anything that calls a function + — and drop per-line tracing entirely. +2. On 3.12+, use `sys.monitoring` with only the events you need; it is designed + for exactly this and is far cheaper than `settrace`. +3. Or drop the trace and rely on the `threading.Timer` → + `ida_kernwin.set_cancelled()` path you already have, accepting that a + pure-Python loop with no calls in it cannot be interrupted. + +**Our workaround** (we would rather not ship it): the snippet detaches the trace +and restores it in a `finally`. That gives up deadline enforcement for +pure-Python loops inside our own code; your native cancel timer is unaffected and +still fires. Every client that does real work per call will eventually find this +and do the same, which is an argument for fixing it in the runtime. + +--- + +## 2. `to_jsonable` dominates any large result + +**FIXED in 0.3.2**, via the first suggested fix below: +`serialization.dumps_json` calls `json.dumps(value, default=to_jsonable)`, so a +JSON-safe result never enters the Python walker. Our packing workaround measured +0.97x and has been deleted. + +`execute_python` runs `to_jsonable()` over whatever the snippet returns. Our +answers are already JSON-safe and they are big — a 200-row listing page is +roughly 10k small objects. + +| | cost | +|---|---| +| `to_jsonable(page)` | 66.2 ms | +| `json.dumps(page, separators=(",",":"))` — same data | 0.58 ms | +| serialised size | 34.9 KB | + +That is 114x, and it was 72% of the page's total cost before we changed it. + +**Suggested fixes** + +- Fast-path values that are already JSON-safe (a cheap recursive type check that + bails to the original object beats rebuilding it), or +- let a snippet opt out by returning an already-serialised payload — a documented + envelope such as `{"__json__": "<...>"}`, or simply passing `str`/`bytes` + through untouched. + +**Retired workaround:** snippets used to `json.dumps` inside the database process +and return one string, which the client parsed. The typed remote API now owns +strict argument/result encoding, and ida-tui contains no generated script +strings or packing envelope. + +--- + +## 3. The per-operation floor is `execute_sync`, not HTTP + +✅ **FIXED in 0.3.2 — 7.0x.** Re-measured as a same-box A/B by checking the +installed editable checkout back to `4195f21` and forward again, 200 iterations +each, `targets/echo`: + +| | 0.3.1 | 0.3.2 | | +|---|---|---|---| +| `GET /health` | 0.497 ms | 0.318 ms | 1.6x | +| `execute_python("result = 1")` | **2.055 ms** | **0.294 ms** | **7.0x** | + +The 0.3.1 column reproduces the original 2.025 ms measurement below almost +exactly, which is what makes the 0.3.2 column believable. `execute_python` now +costs about the same as a bare HTTP GET, so the `execute_sync` marshalling that +was ~93% of the floor is essentially gone. The design advice below — "a client +that makes one call per row will be 20–100x slower than an in-process one" — is +correspondingly much weaker now. + +Original 0.3.1 measurement, same worker, same connection, 200 iterations: + +| | cost | +|---|---| +| `GET /health` (no `execute_sync`) | **0.165 ms** | +| `execute_python("result = 1")` | **2.025 ms** | + +HTTP framing is ~7% of the floor; marshalling the operation onto IDA's main +thread is the other ~93%. The worker runs IDA's own `kernwin.serve()`, so this is +plausibly IDA's dispatch latency rather than anything you control — but it is +worth **documenting**, because it sets a hard 2 ms per-operation budget that +shapes how a client must be designed. + +It did not hurt us (our call volume is 1–8 per user action; 4 calls to build a +1060-block graph), but a client that makes one call per row or per symbol will be +20–100x slower than an in-process one and the authors will not know why. + +**Suggested fixes:** document the floor; and consider a batch endpoint — accept +`[{op, args}, ...]` and dispatch them within a single `execute_sync` — which +would let chatty clients amortise it without redesigning around it. + +--- + +## 4. Loader switches on an existing database are a FATAL, not an error — one edge remains + +Opening a target that already has an `.i64`, while passing spawn-only options, +kills the worker: + +``` +FATAL ERROR: @0:636[] +Switch '-b400' can be used only when loading a new file +``` + +The client sees only: + +``` +IDAConnectionError: idalib worker launcher exited with status 1 +``` + +This is easy to hit and hard to diagnose: it is the natural second run of +anything that opens a raw blob (`processor=`/`image_base=`/`file_type=` are +recorded in the database the first run produced). Our test suite hit it as a +crash five minutes into a run. + +**Suggested fixes** + +- In `DatabaseHandle.open()`, when the resolved IDB already exists and + `new_database` is not set, either ignore the spawn-only options or raise a + typed error naming them — before handing them to IDA. +- Propagate the worker's fatal text into the client exception. The message + already exists on the worker's stderr; losing it turns a one-line fix into a + bisect. + +**Our workaround:** the client checked whether the expected IDB exists and +dropped `processor`/`image_base`/`file_type` when it did. + +**FIXED in 0.5.x**, with exactly this fix, in `_resolver._build_worker_command`: + +```python +if input_path == expected_idb and input_path != source: + # Loader/import switches are baked into an existing IDB... + options = WorkerLaunchOptions() +``` + +Our workaround is therefore deleted. **One narrow case remains**: the strip needs +`input_path != source`, so passing an `.i64` path *directly* together with load +options (`ida-tui foo.i64 --processor arm`) still forwards the switches and still +fatals. Our old guard keyed on "the target IDB exists" and so covered it. It is a +nonsense invocation and no ida-tui code path generates it — the project layer +always passes `output_database`, and `_needs_load_options` bails when an `.i64` +exists — but the library boundary should still reject or normalize it rather +than launch a known-fatal IDA command. Tracked upstream as +[issue #36](https://github.com/HexRaysSA/ida-nexus/issues/36). + +--- + +## 5. Deleting or replacing an IDB under a live lease — STALE + +The original suite deleted an `.i64` while a private worker still had it open, +then immediately reopened the same path. That ownership model no longer applies: +IDA Nexus databases are shared resources, and the IDA GUI itself does not survive +out-of-band replacement of its open database. Detecting arbitrary filesystem +replacement is therefore not part of the supported lifecycle. + +The actionable lifecycle gaps that originally forced private-registry access are +fixed. `find_database_owner()` and `wait_database_released()` are public exports; +`DatabaseHandle.close(wait_for_database=True)` can wait for a final managed close; +a draining owner remains registered until the IDB is actually closed; and +`new_database=True` refuses to replace a live owner. + +ida-tui now uses the public owner/release API while recreating a database and no +longer reaches into registry locks. Owner loss is attach-only: ida-tui will +rediscover a replacement GUI or worker, but will never turn a +user-closing-the-GUI action into an implicit headless reopen. There is no +remaining upstream request in this section. + +--- + +## 6. Close without save — FIXED in protocol 6 + +`DatabaseHandle.shutdown_database(save=False)` can discard a managed idalib +worker when the requesting handle is its only active lease and no other operation +is running. The server rejects GUI databases and shared workers. + +The coherent ownership model is the **final lease**, not necessarily the lease +that spawned the worker. Releasing a non-final lease makes no whole-database save +decision; responsibility transfers to the leases that remain. The final client +can save or discard the shared session. A client that needs its work to survive +regardless of that later decision must call `save_database()` before releasing +its lease. + +This does not claim to provide per-client rollback. Discard applies to all +changes since the last database save, and attempting it while another lease is +active is correctly rejected. That is the same ref-counted lifetime model used +by other shared resources and requires no separate starter capability. + +The upstream gap is therefore closed. ida-tui now routes its discard action +through `shutdown_database(save=False)`: a final managed-worker lease discards, +while GUI-backed and still-shared sessions transfer finalization to their owner +or remaining leases. + +--- + +## 7. No change notification for shared databases — FIXED in protocol 6 + +`DatabaseHandle.subscribe_idb_events()` now returns a closeable iterator over +structured IDB changes. Each event carries a monotonic revision plus +`operation_id`/`operation_label` attribution and an opaque `origin_id`. +`DatabaseHandle.owns_event()` compares that origin with the handle's lease, so a +caching client does not need to generate, retain, or race operation IDs itself. + +ida-tui keeps one subscription for its active database, asks the handle to drop +its own events, and batches peer events behind a 200 ms quiet period. One batch +invalidates the function, listing, decompiler, graph, strings, linkage, segment +and byte caches, then reloads the visible view in place. Closing or switching +databases closes the subscription, so the blocking event reader does not leak. + +--- + +## 8. Package exports and API surface stability — FIXED in 0.5.x + +`ida_nexus/__init__.py` used to export nothing, so a library consumer had to +import from submodules, including things that were clearly internals (`FileLock`, +`REGISTRY_DIR`, `canonical_path`, `idb_key`, `scan_instances`) that we only +touched because no public equivalent existed. + +**Suggested fix was:** export `DatabaseHandle` and the public exception types from +the package root, and mark the intended-public registry helpers explicitly. + +**That is what 0.5.x did.** Everything we need is now on the package root, and +the internals moved behind an underscore: + +```python +from ida_nexus import DatabaseHandle, DatabaseOpenOptions, DatabaseInstance +from ida_nexus import RemoteError, DatabaseBusyError, DatabaseDisconnectedError +from ida_nexus import discover_databases, find_database_owner, wait_database_released +``` + +The two lock-poking helpers we had reimplemented client-side +(`_wait_for_entry_release`) are now `wait_database_released()`, and our +registry-scanning ownership check is now `find_database_owner()`. Both are +deleted from our tree. Note `find_database_owner()` *raises* +`AmbiguousDatabaseError` where our scan silently took the first match — a +behaviour improvement, but callers need a handler. + +--- + +## 9. A testing note: `DatabaseHandle.open()`'s 30 keyword-only options — FIXED in 0.5.x + +The port we started from called `open(..., loading_address=...)`. The real +parameter is `image_base`. Every `connect()` would have raised `TypeError` on the +first call, and its contract tests passed anyway, because a hand-written fake +handle accepts `**kwargs`. + +Not a library bug — but with 30 keyword-only options it is a very easy mistake, +and it is invisible to exactly the offline tests people write. + +**Suggested fix:** ship `py.typed` and/or a `Protocol` for the handle, so a fake +can be checked against the real signature and a typo is caught statically. (We +added a test asserting our kwargs are a subset of +`inspect.signature(DatabaseHandle.open).parameters`, which is a poor substitute.) + +**0.5.x ships `ida_nexus/py.typed`**, and the 30 keyword-only options became a +frozen `DatabaseOpenOptions` dataclass — which is strictly better, because an +invented option name is now a `TypeError` at construction rather than something a +`**kwargs` fake swallows. Our subset test survives in two halves +(`_open_kwargs_are_real` for `open()`, `_option_fields_are_real` for the +dataclass fields), because the offline contract suite must keep running with no +`ida_nexus` installed at all and therefore still fakes both. + +--- + +## Priority, from a client author's view + +| # | item | impact | fixable by you? | +|---|---|---|---| +| ~~1~~ | ~~`timeout_trace` line tracing~~ | ~~52x on IDA calls~~ | ✅ fixed in 0.3.2 | +| ~~2~~ | ~~`to_jsonable` on large results~~ | ~~114x on serialisation~~ | ✅ fixed in 0.3.2 | +| ~~3~~ | ~~2 ms `execute_sync` floor~~ | ~~shapes client design~~ | ✅ fixed in 0.3.2, 7.0x | +| ~~7~~ | ~~no change/revision counter~~ | ~~correctness for shared editing~~ | ✅ fixed in protocol 6 | +| 4 | direct `.i64` forwards loader-only options | fatal worker startup | [issue #36](https://github.com/HexRaysSA/ida-nexus/issues/36) | +| ~~5~~ | ~~replaced/deleted IDB under lease~~ | ~~out-of-contract filesystem mutation~~ | **stale** | +| ~~6~~ | ~~no close without save~~ | ~~could not discard a managed session~~ | **fixed in protocol 6: final lease decides** | +| ~~8~~ | ~~package exports~~ | ~~forces internal imports~~ | ✅ fixed in 0.5.x | +| ~~9~~ | ~~typed handle for fakes~~ | ~~catches a whole bug class~~ | ✅ fixed in 0.5.x (`py.typed` + options dataclass) | + +Items 1 and 2 together were the difference between "the port is 35x slower than +the private worker it replaced" and "the port is within 2x, and faster on several +operations". Both are in the runtime, not in client code — which is why they are +worth fixing centrally rather than leaving each client to rediscover. + +**Both landed in 0.3.2**, along with item 3 — all three performance items are now +fixed upstream, and both client-side workarounds could be measured at parity and +retired. That is the outcome this document was written for. + +**What is left is entirely non-performance.** Items 6 through 9 are fixed, and +item 5 is stale because out-of-band replacement is not a supported lifecycle for +either IDA Nexus or the IDA GUI. One narrow piece remains: **4**, normalize or +reject loader-only options when the source is itself an existing `.i64` +([issue #36](https://github.com/HexRaysSA/ida-nexus/issues/36)). + +Happy to supply the benchmark harness (it is backend-agnostic and runs against +both our old worker and IDA Nexus), or to test a patch. diff --git a/docs/PAGING_FINDINGS.md b/docs/PAGING_FINDINGS.md index bd6c38f..f343973 100644 --- a/docs/PAGING_FINDINGS.md +++ b/docs/PAGING_FINDINGS.md @@ -3,9 +3,9 @@ Measured against a real target: `libcrypto.so.3` (5.7 MB, **10,092 functions**, biggest function **52,120 instructions**). These constraints drive the domain / paging layer. The measurements below came from the former ida-pro-mcp tool -backend. The Code Mode port preserves the adapter response shapes and conservative +backend. The IDA Nexus port preserves the adapter response shapes and conservative page sizes, but executes enumeration through ida-domain; old server caps and RTT -numbers are historical rather than Code Mode constraints. +numbers are historical rather than IDA Nexus constraints. ## Response shape (list_* / *_query tools) @@ -93,17 +93,17 @@ disasm totals are **top-level** fields, not under `asm`: (correct). The pseudocode view must handle "decompilation failed" gracefully — fall back to the disassembly view or show an error panel. -Code Mode returns the complete execution result directly; ida-tui no longer +IDA Nexus returns the complete execution result directly; ida-tui no longer needs MCP structured-content/download-URL recovery for large pseudocode bodies. -## Code Mode lifecycle +## IDA Nexus lifecycle -`CodeModeClient` owns an authenticated SSE lease on a registered database: +`NexusClient` owns an authenticated SSE lease on a registered database: * A matching GUI is preferred and remains open when the TUI exits. -* Otherwise Code Mode reuses or starts a shared managed idalib worker. +* Otherwise IDA Nexus reuses or starts a shared managed idalib worker. * Releasing one lease never terminates another client's session. A managed - worker saves and exits after its final lease under Code Mode's grace policy. + worker saves and exits after its final lease under IDA Nexus's grace policy. * Lease loss surfaces as `IDAConnectionError`; reconnect performs discovery again and may bind a newly-created instance. It does not silently swap the handle underneath an operation. diff --git a/docs/PROJECTS.md b/docs/PROJECTS.md index 0efee31..52dbf83 100644 --- a/docs/PROJECTS.md +++ b/docs/PROJECTS.md @@ -7,7 +7,7 @@ search across all of them, and (later) follow calls from one into another. ## The constraint that shapes everything -IDA still exposes one active database per GUI/idalib process. Code Mode makes +IDA still exposes one active database per GUI/idalib process. IDA Nexus makes those instances discoverable and shareable: each project entry retains one `DatabaseHandle` lease, which may target a registered GUI or a managed idalib worker. N resident project databases can therefore mean up to N processes, but @@ -32,13 +32,13 @@ crypto library. Two capabilities that feel like one, but aren't: -1. **Switching** to a binary needs a *live Code Mode lease*. +1. **Switching** to a binary needs a *live IDA Nexus lease*. 2. **Searching across** binaries does *not* — if a per-binary index (functions, strings, imports/exports) is cached on disk. That split is the unlock: project-wide search stays instant across every binary, including ones never opened this session, and only *jumping* to a hit costs a -Code Mode attach/open. +IDA Nexus attach/open. ## Layout @@ -85,11 +85,11 @@ basename and must be unique (it names the staged file). ## Runtime -- **`DatabasePool`** — one `CodeModeClient` lease per resident binary, attached +- **`DatabasePool`** — one `NexusClient` lease per resident binary, attached lazily on first switch and LRU-released when the advisory memory budget is exceeded. Eviction explicitly saves managed IDBs but never implicitly saves a GUI. Closing a lease never kills a GUI or another client's managed worker; - Code Mode owns final worker shutdown. + IDA Nexus owns final worker shutdown. - **`BinaryState`** — per binary: `client, program, nav, cur, func_index, pref/active/split, filter`. Switching snapshots the current state and restores the target's. `_after_reconnect` provides the client/program swap seam. diff --git a/docs/SPLIT_VIEW.md b/docs/SPLIT_VIEW.md index c421656..6e9b938 100644 --- a/docs/SPLIT_VIEW.md +++ b/docs/SPLIT_VIEW.md @@ -32,7 +32,7 @@ known technique: The old ida-pro-mcp backend derived the per-line marker via `cfunc.get_line_item(line, col=0, …).get_ea()`. To get the **full set**, sweep every column of the line (`get_line_item(line, x, …).get_ea()` for `x` in -`0..len`) and collect distinct non-`BADADDR` EAs. The Code Mode adapter's +`0..len`) and collect distinct non-`BADADDR` EAs. The IDA Nexus adapter's `decomp_map(ea)` operation returns `[{line, primary_ea, eas:[…]}, …]`; invert for `ea → line`. @@ -78,14 +78,14 @@ decomp→listing uses `ListingModel.ensure_ea`. Tab re-links from the new driver Still single-ea per line (one instruction highlighted); the region comes in phase 3. -**Phase 3 — rich highlight. DONE.** The Code Mode `decomp_map` operation -(`idatui/codemode_client.py`) sweeps `cfunc.get_line_item` across every column of +**Phase 3 — rich highlight. DONE.** The IDA Nexus `decomp_map` operation +(`idatui/nexus_client.py`) sweeps `cfunc.get_line_item` across every column of each pseudocode line and collects the EAs from each item's `dstr()` (`'EA: desc'` — the same source as the `/*ea*/` marker, so it aligns). `Program.decomp_map(ea)` returns the per-line ea lists (cached by name-gen); the app loads it async into `_split_eamap` / `_split_ea2line` and `_sync_split` bands the **whole** instruction region of a C line (and uses the exact ea→line inverse for the reverse). Falls -back to the single marker until the map lands. Verified on a real Code Mode database +back to the single marker until the map lands. Verified on a real IDA Nexus database (alignment + multi-instruction region band). **Phase 4 — polish. DONE.** diff --git a/experiments/bench_ops.py b/experiments/bench_ops.py index 9cc51fa..1c0ad69 100644 --- a/experiments/bench_ops.py +++ b/experiments/bench_ops.py @@ -1,4 +1,4 @@ -"""Time a realistic idatui operation mix against whatever ida-codemode is installed. +"""Time a realistic idatui operation mix against whatever ida-nexus is installed. The companion to `bench_pack_trace.py`: that one isolates a single workaround, this one answers "how much faster is the whole client, on real operations". @@ -14,16 +14,16 @@ replace or delete it:: PYTHONPATH=. ~/ida-venv/bin/python /tmp/bench_ops.py # B: current client against the OLD library (shows what the workarounds were for) - git -C ~/dev/ida-codemode checkout 4195f21 + git -C ~/dev/ida-nexus checkout 4195f21 PYTHONPATH=. ~/ida-venv/bin/python /tmp/bench_ops.py # A: the client as it SHIPPED on the old library, workarounds and all git checkout 8550474 # the commit before the workaround removal PYTHONPATH=. ~/ida-venv/bin/python /tmp/bench_ops.py - git checkout main && git -C ~/dev/ida-codemode checkout main # ALWAYS restore + git checkout main && git -C ~/dev/ida-nexus checkout main # ALWAYS restore -ida-codemode is installed **editable** into both venvs, so checking that repo out +ida-nexus is installed **editable** into both venvs, so checking that repo out swaps the backend under the TUI with no reinstall -- which is what makes this A/B cheap. """ @@ -36,7 +36,7 @@ import statistics import time from idatui import remote_ops -from idatui.codemode_client import CodeModeClient +from idatui.nexus_client import NexusClient def bench(fn, reps: int) -> tuple[float, float]: @@ -55,7 +55,7 @@ def main() -> int: ap.add_argument("--reps", type=int, default=20) args = ap.parse_args() - client = CodeModeClient(os.path.abspath(args.target)) + client = NexusClient(os.path.abspath(args.target)) client.connect() handle = client._handle diff --git a/experiments/bench_pack_trace.py b/experiments/bench_pack_trace.py index edbb9f7..2b53afd 100644 --- a/experiments/bench_pack_trace.py +++ b/experiments/bench_pack_trace.py @@ -17,7 +17,7 @@ import statistics import time from idatui import remote_ops -from idatui.codemode_client import CodeModeClient +from idatui.nexus_client import NexusClient def timed(function, reps: int = 1) -> tuple[object, float]: @@ -37,7 +37,7 @@ def main() -> int: parser.add_argument("--rows", type=int, default=200) args = parser.parse_args() - client = CodeModeClient(os.path.abspath(args.target)).connect() + client = NexusClient(os.path.abspath(args.target)).connect() index, operations_cold = timed( lambda: client.call(remote_ops.list_funcs, queries=[{"offset": 0, "count": 40}]) diff --git a/experiments/call_census.py b/experiments/call_census.py index 1680691..17a5f94 100644 --- a/experiments/call_census.py +++ b/experiments/call_census.py @@ -1,7 +1,7 @@ """Count backend round-trips per user action. Answers "are we batching, or paying a round-trip per item?" with numbers rather -than intent. Wraps ``CodeModeClient.invoke`` on the live app, drives a headless +than intent. Wraps ``NexusClient.invoke`` on the live app, drives a headless Pilot through realistic actions, and reports calls + wall time + which operations were used for each. @@ -25,7 +25,7 @@ from _fixtures import fast_keys, staged # noqa: E402 fast_keys() from idatui.app import IdaTui, ListingView # noqa: E402 -from idatui.codemode_client import CodeModeClient # noqa: E402 +from idatui.nexus_client import NexusClient # noqa: E402 class Census: @@ -34,18 +34,18 @@ class Census: def __init__(self) -> None: self.ops: collections.Counter = collections.Counter() self.n = 0 - original = CodeModeClient.invoke + original = NexusClient.invoke def counting(client, operation, *a, **kw): self.n += 1 self.ops[operation] += 1 return original(client, operation, *a, **kw) - CodeModeClient.invoke = counting + NexusClient.invoke = counting self._original = original def restore(self) -> None: - CodeModeClient.invoke = self._original + NexusClient.invoke = self._original def span(self, label: str): return _Span(self, label) diff --git a/experiments/profile_client.py b/experiments/profile_client.py index bac130e..84abd8a 100644 --- a/experiments/profile_client.py +++ b/experiments/profile_client.py @@ -21,7 +21,7 @@ import pstats import time from idatui import remote_ops -from idatui.codemode_client import CodeModeClient +from idatui.nexus_client import NexusClient from idatui.domain import Program @@ -35,7 +35,7 @@ def main() -> int: ) args = ap.parse_args() - client = CodeModeClient(os.path.abspath(args.binary)) + client = NexusClient(os.path.abspath(args.binary)) client.connect() program = Program(client) regions = client.call(remote_ops.file_regions) diff --git a/experiments/profile_remote.py b/experiments/profile_remote.py index 97f4fb1..fbf17dc 100644 --- a/experiments/profile_remote.py +++ b/experiments/profile_remote.py @@ -15,7 +15,7 @@ import argparse import os from idatui import remote_ops -from idatui.codemode_client import CodeModeClient +from idatui.nexus_client import NexusClient CALLS = { "heads": ("heads", {"count": 500, "annotate": True}), @@ -36,7 +36,7 @@ def main() -> int: parser.add_argument("--addr", default=None, help="default: the .text start") args = parser.parse_args() - client = CodeModeClient(os.path.abspath(args.binary)).connect() + client = NexusClient(os.path.abspath(args.binary)).connect() addr = args.addr if addr is None: regions = client.call(remote_ops.file_regions) diff --git a/experiments/worker_smoke.py b/experiments/worker_smoke.py index 9b55138..1c2125f 100644 --- a/experiments/worker_smoke.py +++ b/experiments/worker_smoke.py @@ -1,6 +1,6 @@ -"""Exercise the real domain.Program through an IDA Code Mode lease. +"""Exercise the real domain.Program through an IDA Nexus lease. -A matching registered GUI is reused; otherwise Code Mode starts a managed +A matching registered GUI is reused; otherwise IDA Nexus starts a managed idalib worker. Usage: ``uv run python experiments/worker_smoke.py FILE``. """ from __future__ import annotations @@ -9,15 +9,15 @@ import os import sys import time -from idatui.codemode_client import CodeModeClient +from idatui.nexus_client import NexusClient from idatui.domain import Program def main() -> int: target = os.path.abspath(sys.argv[1] if len(sys.argv) > 1 else "experiments/fibonacci.elf") - print(f"attaching Code Mode to {target}…", flush=True) + print(f"attaching IDA Nexus to {target}…", flush=True) started = time.time() - client = CodeModeClient(target) + client = NexusClient(target) client.connect(progress=lambda message: print(f" {message}", flush=True)) print( f" ready in {time.time() - started:.2f}s; backend={client.backend}; " diff --git a/ida-tui b/ida-tui index a488cac..de60a45 100755 --- a/ida-tui +++ b/ida-tui @@ -4,7 +4,7 @@ # ./ida-tui foo.elf # open a binary and drive it — that's it # # The launcher leases a registered IDA GUI or shared managed idalib worker -# through ida_codemode. The selected Python must have ida-tui's dependencies; +# through ida_nexus. The selected Python must have ida-tui's dependencies; # override it with $IDATUI_PYTHON. set -eu diff --git a/idatui/__init__.py b/idatui/__init__.py index 7da28b3..e89d8f6 100644 --- a/idatui/__init__.py +++ b/idatui/__init__.py @@ -1,4 +1,4 @@ -"""idatui — a keyboard-first TUI using shared IDA Code Mode databases.""" +"""idatui — a keyboard-first TUI using shared IDA Nexus databases.""" from .errors import ( IDAError, @@ -10,7 +10,7 @@ from .errors import ( IDASessionError, Session, ) -from .codemode_client import CodeModeClient +from .nexus_client import NexusClient from .domain import ( Program, FunctionIndex, @@ -25,7 +25,7 @@ from .domain import ( ) __all__ = [ - "CodeModeClient", + "NexusClient", "Program", "FunctionIndex", "DisasmModel", diff --git a/idatui/app.py b/idatui/app.py index e4c2936..05b428a 100644 --- a/idatui/app.py +++ b/idatui/app.py @@ -10,7 +10,7 @@ Design notes: without ever materializing 52k lines in a widget. * All network/domain work runs in Textual worker threads; the UI never blocks. * An address-history stack backs Enter (follow) / Esc (back), IDA-style. -* Database lifecycle is lease-based through ida_codemode: matching GUI sessions +* Database lifecycle is lease-based through ida_nexus: matching GUI sessions are reused, otherwise a shared managed idalib worker is opened on demand. """ @@ -55,7 +55,7 @@ from .highlight import CTextArea, highlight_c from .journal import Journal from .errors import IDAConnectionError -from .codemode_client import CodeModeClient, registered_database +from .nexus_client import NexusClient, registered_database from .domain import Func, Head, ListingModel, Program, Struct # Styles for the disassembly listing. @@ -1087,7 +1087,7 @@ class ListingView(SearchMixin, NavMixin, ColumnCursor, ScrollView, can_focus=Tru def _span_segments(h: Head, fallback: Style): """Segments for a row's disassembly text. - Uses IDA's own token classification when Code Mode supplies it; falls + Uses IDA's own token classification when IDA Nexus supplies it; falls back to the mnemonic/rest split when spans are absent or disagree with the plain text. """ @@ -5238,7 +5238,7 @@ class IdaTui(App): self._open_path = open_path self._ttl = ttl self._load_args = load_args or "" # first-open options for a headerless blob - self._new_database = False # Ctrl+L asks Code Mode for a fresh IDB + self._new_database = False # Ctrl+L asks IDA Nexus for a fresh IDB self._title = (os.path.basename(open_path) if open_path else "") #: Where we are in the execution trace, and everything that moves us. #: Owns the trace state; the _trace/_t/_trail_* properties below @@ -5247,7 +5247,7 @@ class IdaTui(App): self._do_keepalive = keepalive self._rpc_path = rpc_path self._rpc = None - self.client: CodeModeClient | None = None + self.client: NexusClient | None = None self.program: Program | None = None self._loading_screen: LoadingScreen | None = None self._ka = None @@ -5288,7 +5288,7 @@ class IdaTui(App): self.journal = Journal() self._xref_focus_name: str | None = None self._dirty = False - # One subscription for the active database. CodeModeClient debounces + # One subscription for the active database. NexusClient debounces # bursts off the Textual worker pool; the callback re-enters here on the # UI thread to invalidate and reload the visible models. self._idb_event_watch = None @@ -5362,7 +5362,7 @@ class IdaTui(App): if self._rpc_path: self._start_rpc() # A file no loader recognises has to be described before it can be - # opened, so ask BEFORE Code Mode creates it — once IDA has made a database + # opened, so ask BEFORE IDA Nexus creates it — once IDA has made a database # the answer is baked in and changing it requires a fresh-IDB reopen. if self._project is not None: ref = self._pending_load_ref() @@ -5435,7 +5435,7 @@ class IdaTui(App): ref = self._project.by_label(self._binary) if ref is not None: path, label = ref.source, ref.label - # Release our lease first. Code Mode waits for a managed worker's final + # Release our lease first. IDA Nexus waits for a managed worker's final # lease grace, then creates the replacement IDB atomically. A GUI-backed # database is rejected by _can_reload(): the TUI must never close it. self._release_database() @@ -5634,7 +5634,7 @@ class IdaTui(App): return len(text) # -- live refresh from shared IDB changes ----------------------------- # - def _start_idb_event_watch(self, client: CodeModeClient) -> None: + def _start_idb_event_watch(self, client: NexusClient) -> None: self._stop_idb_event_watch() watch = getattr(client, "watch_idb_events", None) if watch is None: # IDA-free test doubles and pre-event adapters @@ -5661,7 +5661,7 @@ class IdaTui(App): watcher.close() def _idb_event_watch_failed( - self, client: CodeModeClient, error: BaseException + self, client: NexusClient, error: BaseException ) -> None: if client is not self.client: return @@ -5685,7 +5685,7 @@ class IdaTui(App): return anchor def _refresh_idb_events( - self, client: CodeModeClient, events: tuple[dict, ...] + self, client: NexusClient, events: tuple[dict, ...] ) -> None: """Invalidate once per external edit burst and reload the active surface.""" program = self.program @@ -5768,7 +5768,7 @@ class IdaTui(App): # -- connection loss / recovery --------------------------------------- # def _handle_exception(self, error: BaseException) -> None: - """Intercept a lost Code Mode lease so the app can rediscover the DB. + """Intercept a lost IDA Nexus lease so the app can rediscover the DB. Everything unrelated to database connectivity crashes as usual. """ @@ -5812,11 +5812,11 @@ class IdaTui(App): return if self._project is not None and self._binary is not None: ref = self._project.by_label(self._binary) - client = CodeModeClient( + client = NexusClient( ref.staged, ttl=self._ttl, load_args=ref.load_args, output_database=ref.db, spawn=False) else: - client = CodeModeClient( + client = NexusClient( self._open_path, ttl=self._ttl, load_args=self._load_args, spawn=False) client.connect(progress=lambda m: self.app.call_from_thread( @@ -5826,7 +5826,7 @@ class IdaTui(App): return self.app.call_from_thread(self._after_reconnect, client, Program(client)) - def _after_reconnect(self, client: "CodeModeClient", program: "Program") -> None: + def _after_reconnect(self, client: "NexusClient", program: "Program") -> None: old_client, old_program = self.client, self.program self._stop_idb_event_watch() if old_program is not None: @@ -5886,7 +5886,7 @@ class IdaTui(App): self._load_functions() def _open_database_client(self): # type: ignore[no-untyped-def] - """Attach through Code Mode, reusing a GUI or managed idalib database.""" + """Attach through IDA Nexus, reusing a GUI or managed idalib database.""" if self._pool is not None: # project mode: the pool owns the leases label = self._binary or self._project.refs[0].label client = self._pool.get(label, progress=lambda m: @@ -5898,13 +5898,13 @@ class IdaTui(App): return client if not self._open_path: self.app.call_from_thread( - self._status, "Code Mode needs a database or executable path") + self._status, "IDA Nexus needs a database or executable path") self.app.call_from_thread(self._dismiss_loading) return None base = os.path.basename(self._open_path) self.app.call_from_thread( - self._status, f"discovering Code Mode database for {base}…") - client = CodeModeClient(self._open_path, ttl=self._ttl, + self._status, f"discovering IDA Nexus database for {base}…") + client = NexusClient(self._open_path, ttl=self._ttl, load_args=self._load_args, new_database=self._new_database) client.connect(progress=lambda m: self.app.call_from_thread( @@ -5978,7 +5978,7 @@ class IdaTui(App): @work(thread=True, exclusive=True, group="index") def _index_binary(self) -> None: """Fold this binary's symbols + strings into the project index, so it can - be searched later even when its Code Mode lease is gone.""" + be searched later even when its IDA Nexus lease is gone.""" if self._index is None or self._project is None or self._binary is None: return ref = self._project.by_label(self._binary) @@ -6081,7 +6081,7 @@ class IdaTui(App): cursor=0, push=True, is_region=True) def _can_reload(self) -> bool: - """Whether Code Mode can replace this IDB with different options. + """Whether IDA Nexus can replace this IDB with different options. A GUI database is owned by the user and has no remote close/rollback route. Managed idalib databases can be released and reopened fresh. diff --git a/idatui/codemode_client.py b/idatui/codemode_client.py deleted file mode 100644 index e95238f..0000000 --- a/idatui/codemode_client.py +++ /dev/null @@ -1,551 +0,0 @@ -"""Client adapter from ida-tui's domain operations to IDA Code Mode. - -``DatabaseHandle`` is the lifecycle boundary: it discovers an already-registered -GUI database, reuses a shared managed idalib worker, or starts one when needed. -The TUI never owns or terminates an IDA process. Closing this client releases -only its lease. - -Remote operations are ordinary typed Python functions declared in -``idatui.remote_ops``. Code Mode installs their content-addressed modules once -per handle; subsequent calls send only encoded arguments. The optimized -IDAPython listing/decompiler implementation remains real source in -``idatui.remote_tools`` and is installed through the same module interface. -""" - -from __future__ import annotations - -import os -import shlex -import threading -import time -from collections.abc import Callable -from typing import Any - -from .errors import IDAConnectionError, IDATimeoutError, IDAToolError, Session - -# ida_codemode is imported EAGERLY-IF-PRESENT but never at hard import cost. -# -# The paging/graph/trace layers and their offline test suites must keep importing -# `idatui` on a machine with no IDA and no Code Mode installed -- that is the -# house rule the stdlib-only worker client used to satisfy for free, and -# `tests/run.py --fast` (380 checks, any python3) depends on it. A hard top-level -# import here makes the whole package unimportable, so the failure is deferred to -# the first operation that genuinely needs the library. -_CODEMODE_ERROR: Exception | None = None -try: - from ida_codemode import ( - CodeModeConnectionError, - DatabaseBusyError, - DatabaseDisconnectedError, - DatabaseHandle, - DatabaseInstance, - DatabaseOpenOptions, - RemoteError, - find_database_owner, - wait_database_released, - ) -except ImportError as _exc: # library absent: usable only for offline layers - _CODEMODE_ERROR = _exc - # Bound to None rather than left undefined so the names stay patchable: the - # offline contract tests inject a fake DatabaseHandle here. - CodeModeConnectionError = DatabaseDisconnectedError = RemoteError = None # type: ignore[assignment,misc] - DatabaseBusyError = DatabaseHandle = DatabaseInstance = None # type: ignore[assignment,misc] - DatabaseOpenOptions = find_database_owner = wait_database_released = None # type: ignore[assignment] - - -def _require_codemode() -> None: - """Raise an actionable error when the Code Mode library is missing. - - Gated on the binding, not on the original import result, so a test that - injects a fake ``DatabaseHandle`` exercises the real adapter logic. - """ - if DatabaseHandle is None: - raise IDAConnectionError( - "ida-codemode is not installed in this environment " - f"({_CODEMODE_ERROR}). Install it (e.g. `uv sync`, or " - "`pip install ida-codemode`) so ida-tui can lease a " - "database." - ) from _CODEMODE_ERROR - - -def database_owner(idb_path: str, staged_path: str | None = None): - """The Code Mode instance that owns ``idb_path``/``staged_path``, else None. - - Returns None when the Code Mode library is absent: with no library there is - no client in this environment that could be holding the database, and the - IDA-free layers (project staging) must keep working. Discovery errors with - the library installed still propagate because unknown ownership is unsafe. - """ - if DatabaseHandle is None: - return None - if staged_path: - owner = find_database_owner( - staged_path, - output_database=idb_path, - timeout=0.5, - ) - return owner or find_database_owner(staged_path, timeout=0.5) - return find_database_owner(idb_path, timeout=0.5) - - -def registered_database(path: str, output_database: str | None = None) -> bool: - """Whether a live/lock-held Code Mode instance owns this target.""" - _require_codemode() - return ( - find_database_owner( - path, - output_database=output_database, - timeout=0.5, - ) - is not None - ) - - -class _NoopKeepAlive: - """Compatibility shim: the DatabaseHandle's SSE lease is the heartbeat.""" - - def __init__(self) -> None: - self.beats = self.failures = 0 - - def start(self) -> "_NoopKeepAlive": - return self - - def stop(self) -> None: - pass - - -def _parse_load_args(value: str) -> tuple[str | None, int | None, str | None]: - """Translate ida-tui's legacy first-open switches to Code Mode options. - - Code Mode has typed options for processor, natural loading address and file - type. It deliberately has no arbitrary command-line escape hatch; reject - switches we cannot represent instead of silently loading a blob wrongly. - """ - processor: str | None = None - loading_address: int | None = None - file_type: str | None = None - unsupported: list[str] = [] - try: - words = shlex.split(value or "", posix=os.name != "nt") - except ValueError as exc: - raise ValueError(f"invalid IDA load options: {exc}") from exc - for word in words: - if word.startswith("-p") and len(word) > 2: - processor = word[2:] - elif word.startswith("-b") and len(word) > 2: - try: - # IDA's -b is in 16-byte paragraphs. DatabaseHandle expects the - # natural address, which is the safer public API. - loading_address = int(word[2:], 16) << 4 - except ValueError as exc: - raise ValueError(f"invalid IDA loading address: {word!r}") from exc - elif word.startswith("-T") and len(word) > 2: - file_type = word[2:] - else: - unsupported.append(word) - if unsupported: - joined = " ".join(unsupported) - raise ValueError( - "ida-codemode cannot represent arbitrary IDA load options: " - f"{joined!r}; use processor/base/file type options instead" - ) - return processor, loading_address, file_type - - -class IDBEventListener: - """Debounced, closeable delivery of another client's IDB changes. - - Code Mode's subscription is a blocking iterator, so one daemon thread reads - it and a second waits for a quiet period before handing a batch to the UI. - Keeping the debounce here avoids a permanent Textual worker (which would - make the app's worker-idle contract impossible) and bounds refresh work to - one pass per edit burst. - """ - - def __init__( - self, - client: "CodeModeClient", - callback: Callable[[tuple[dict[str, Any], ...]], None], - *, - on_error: Callable[[BaseException], None] | None = None, - debounce: float = 0.2, - ) -> None: - self._client = client - self._callback = callback - self._on_error = on_error - self._debounce = max(float(debounce), 0.0) - self._condition = threading.Condition() - self._closed = False - self._subscription = None - self._pending: list[dict[str, Any]] = [] - self._deadline = 0.0 - self._reader = threading.Thread( - target=self._read, name="idatui-idb-events", daemon=True - ) - self._deliverer = threading.Thread( - target=self._deliver, name="idatui-idb-refresh", daemon=True - ) - self._deliverer.start() - self._reader.start() - - def _report(self, error: BaseException) -> None: - disconnected = DatabaseDisconnectedError - if isinstance(disconnected, type) and isinstance(error, disconnected): - error = self._client._connection_error(error) - with self._condition: - closed = self._closed - if not closed and self._on_error is not None: - self._on_error(error) - - def _read(self) -> None: - try: - subscription = self._client.subscribe_idb_events() - except Exception as exc: # noqa: BLE001 -- surfaced through on_error - self._report(exc) - with self._condition: - self._closed = True - self._pending.clear() - self._condition.notify_all() - return - with self._condition: - if self._closed: - subscription.close() - return - self._subscription = subscription - try: - for event in subscription: - with self._condition: - if self._closed: - break - if self._client.owns_event(event): - continue - with self._condition: - if self._closed: - break - self._pending.append(event) - self._deadline = time.monotonic() + self._debounce - self._condition.notify_all() - except Exception as exc: # noqa: BLE001 -- stream failures are recoverable - self._report(exc) - finally: - subscription.close() - with self._condition: - if self._subscription is subscription: - self._subscription = None - self._closed = True - self._pending.clear() - self._condition.notify_all() - - def _deliver(self) -> None: - while True: - with self._condition: - while not self._closed and not self._pending: - self._condition.wait() - if self._closed: - return - remaining = self._deadline - time.monotonic() - if remaining > 0: - self._condition.wait(remaining) - continue - batch = tuple(self._pending) - self._pending.clear() - try: - self._callback(batch) - except Exception as exc: # noqa: BLE001 -- keep the stream alive - self._report(exc) - - def close(self) -> None: - """Stop delivery and unblock the subscription reader.""" - with self._condition: - if self._closed: - return - self._closed = True - self._pending.clear() - subscription = self._subscription - self._condition.notify_all() - if subscription is not None: - subscription.close() - - -class CodeModeClient: - """A leased GUI/idalib database accessed through ``ida_codemode``.""" - - def __init__( - self, - binary_path: str, - *, - ttl: int = 0, - load_args: str = "", - processor: str | None = None, - loading_address: int | None = None, - file_type: str | None = None, - output_database: str | None = None, - spawn: bool = True, - new_database: bool = False, - ) -> None: - del ttl # managed-worker lifetime is lease-based, not idle-TTL based - self._path = os.path.abspath(os.path.expanduser(binary_path)) - parsed_processor, parsed_address, parsed_file_type = _parse_load_args(load_args) - self._processor = processor or parsed_processor - self._loading_address = ( - loading_address if loading_address is not None else parsed_address - ) - self._file_type = file_type or parsed_file_type - self._output_database = output_database - self._spawn = spawn - self._new_database = new_database - self._handle: DatabaseHandle | None = None - self._last_instance: DatabaseInstance | None = None - self._connect_lock = threading.Lock() - - def connect(self, timeout: float = 1800.0, progress=None) -> "CodeModeClient": - _require_codemode() - with self._connect_lock: - handle = self._handle - if handle is not None: - if handle.connected: - return self - raise IDAConnectionError( - "Code Mode database disconnected; explicit rediscovery required" - ) - if progress: - progress( - f"discovering Code Mode database for {os.path.basename(self._path)}…" - ) - try: - # A Ctrl+L reload releases its current managed-worker lease, but - # that worker remains registered during Code Mode's final-lease - # grace period. Retry only that known handoff window. A GUI or - # another long-lived client remains busy and yields a clear - # failure rather than being modified underneath its owner. - deadline = time.monotonic() + min(timeout, 60.0) - while True: - try: - handle = DatabaseHandle.open( - self._path, - options=DatabaseOpenOptions( - spawn=self._spawn, - startup_timeout=max(0.1, timeout), - output_database=self._output_database, - processor=self._processor, - # The natural byte address is converted to IDA's - # paragraph-based -b value by Code Mode. - image_base=self._loading_address, - file_type=self._file_type, - new_database=self._new_database, - ), - ) - break - except DatabaseBusyError: - if not self._new_database or time.monotonic() >= deadline: - raise - if progress: - progress( - "waiting for the previous Code Mode lease to close…" - ) - owner = find_database_owner( - self._path, - output_database=self._output_database, - timeout=0.5, - ) - if owner is not None: - wait_database_released( - owner, - max(0.0, deadline - time.monotonic()), - ) - else: - time.sleep(0.2) - if progress: - backend = handle.instance.backend - progress( - f"attached to {backend} database; waiting for auto-analysis…" - ) - handle.wait_autoanalysis(timeout=timeout) - except Exception as exc: # normalize the dependency's transport errors - raise self._connection_error(exc) from exc - self._handle = handle - self._last_instance = handle.instance - return self - - @staticmethod - def _connection_error(exc: BaseException) -> IDAConnectionError: - return IDAConnectionError(str(exc) or type(exc).__name__) - - @property - def connected(self) -> bool: - return self._handle is not None and self._handle.connected - - @property - def pid(self) -> int | None: - return self._handle.instance.pid if self._handle is not None else None - - @property - def backend(self) -> str | None: - return self._handle.instance.backend if self._handle is not None else None - - def owns_event(self, event: dict[str, Any]) -> bool: - """Whether ``event`` was produced through this client's handle.""" - handle = self._handle - return handle is not None and handle.owns_event(event) - - def subscribe_idb_events(self): - """Open Code Mode's closeable IDB-change iterator.""" - if not self.connected: - self.connect() - handle = self._handle - if handle is None: - raise IDAConnectionError("Code Mode database is not connected") - try: - return handle.subscribe_idb_events() - except (DatabaseDisconnectedError, CodeModeConnectionError) as exc: - raise self._connection_error(exc) from exc - - def watch_idb_events( - self, - callback: Callable[[tuple[dict[str, Any], ...]], None], - *, - on_error: Callable[[BaseException], None] | None = None, - debounce: float = 0.2, - ) -> IDBEventListener: - """Deliver external IDB changes in debounced batches.""" - return IDBEventListener(self, callback, on_error=on_error, debounce=debounce) - - def call(self, operation: Callable[..., Any], /, **args) -> Any: - """Execute one source-backed remote declaration through this client.""" - name = getattr(operation, "__name__", "remote operation") - try: - from .remote_ops import bind - - remote = bind(operation) - except KeyError as exc: - raise IDAToolError( - name, f"remote operation {name!r} is not registered" - ) from exc - if not self.connected: - self.connect() - handle = self._handle - if handle is None: - raise IDAConnectionError("Code Mode database is not connected") - try: - return remote(handle, **args) - except RemoteError as exc: - message = str(exc) - if exc.details.get("traceback"): - message += f"\n{exc.details['traceback']}" - if exc.code == "operation_timeout": - raise IDATimeoutError(message) from exc - raise IDAToolError(name, message) from exc - except (DatabaseDisconnectedError, CodeModeConnectionError) as exc: - raise self._connection_error(exc) from exc - - def save_database(self) -> dict[str, Any]: - if not self.connected: - self.connect() - handle = self._handle - if handle is None: - raise IDAConnectionError("Code Mode database is not connected") - try: - return handle.save_database() - except RemoteError as exc: - raise IDAToolError("save_database", str(exc)) from exc - except (DatabaseDisconnectedError, CodeModeConnectionError) as exc: - raise self._connection_error(exc) from exc - - def discard_database(self, timeout: float = 5.0) -> bool: - """Discard a final managed-worker lease; otherwise transfer finalization. - - ``False`` is an expected ownership result: a GUI owns its session, or - another lease still shares the managed worker. A busy final worker is - retried briefly so background reads finishing during quit do not turn a - real discard into an implicit save. - """ - handle = self._handle - if handle is None or not handle.connected: - return False - entry = handle.instance - if entry.backend != "idalib" or not getattr(entry, "managed", False): - return False - deadline = time.monotonic() + max(float(timeout), 0.0) - while True: - try: - handle.shutdown_database(save=False) - return True - except RemoteError as exc: - if exc.code in ("instance_shared", "shutdown_not_supported"): - return False - if exc.code == "instance_busy" and time.monotonic() < deadline: - time.sleep(0.05) - continue - raise IDAToolError("shutdown_database", str(exc)) from exc - except (DatabaseDisconnectedError, CodeModeConnectionError) as exc: - raise self._connection_error(exc) from exc - - def health(self) -> dict[str, Any]: - if not self.connected: - self.connect() - assert self._handle is not None - entry = self._handle.instance - module = os.path.basename(entry.exe_path or entry.idb_path or self._path) - return { - "ok": self._handle.connected, - "module": module, - "backend": entry.backend, - "record_id": entry.record_id, - "input_path": entry.exe_path, - "idb_path": entry.idb_path, - } - - def keepalive(self, interval: float = 120.0) -> _NoopKeepAlive: - del interval - return _NoopKeepAlive() - - def resolve_db(self) -> str: - if not self.connected: - self.connect() - assert self._handle is not None - return self._handle.instance.record_id - - def set_db(self, db: str | None) -> None: - del db # one handle is permanently bound to one registered database - - def list_sessions(self) -> list[Session]: - if not self.connected: - self.connect() - assert self._handle is not None - entry = self._handle.instance - path = entry.exe_path or entry.idb_path or self._path - return [ - Session( - session_id=entry.record_id, - filename=os.path.basename(path), - input_path=path, - is_active=True, - ) - ] - - def close(self, grace: float = 0.0) -> None: - del grace - with self._connect_lock: - handle, self._handle = self._handle, None - if handle is not None: - self._last_instance = handle.instance - handle.close() # release our lease; never close a GUI/other client's DB - - def wait_released(self, timeout: float = 45.0) -> bool: - """Wait until a managed instance releases its lifetime lock. - - Normal application shutdown must not wait: another client may retain the - worker. This is an explicit test/maintenance helper for deleting a - temporary IDB safely after this client closes. GUI instances return - ``False`` immediately because clients never own their lifetime. - """ - instance = self._last_instance - if instance is None or instance.backend != "idalib": - return False - return wait_database_released(instance, timeout) - - def __enter__(self) -> "CodeModeClient": - return self.connect() - - def __exit__(self, *exc) -> None: - self.close() diff --git a/idatui/domain.py b/idatui/domain.py index b042332..1e5a863 100644 --- a/idatui/domain.py +++ b/idatui/domain.py @@ -1,4 +1,4 @@ -"""Domain / paging layer: address-centric models over IDA Code Mode. +"""Domain / paging layer: address-centric models over IDA Nexus. This is where the "millions of lines" problem is solved, so the TUI widgets only ever see a viewport-sized slice. Every hard-won constraint from @@ -7,7 +7,7 @@ ever see a viewport-sized slice. Every hard-won constraint from * Page sizes remain bounded so remote execution returns viewport-scale JSON. * Pagination advances by the number of rows actually returned. * Deep head walks are block-cached (revisits are free) and neighboring blocks - prefetch through the thread-safe Code Mode client. + prefetch through the thread-safe IDA Nexus client. * Expensive function totals are fetched once and cached. * Decompilation failures are surfaced as data, not application crashes. @@ -31,7 +31,7 @@ from . import remote_ops from .errors import IDAToolError if TYPE_CHECKING: # type hint only - from .codemode_client import CodeModeClient + from .nexus_client import NexusClient # Clamps derived from measured caps (list ~700, disasm ~500). Margin included. LIST_PAGE = 500 @@ -90,7 +90,7 @@ class Line: class Head(NamedTuple): - """One flat-listing item (from the Code Mode ``heads`` operation): a code + """One flat-listing item (from the IDA Nexus ``heads`` operation): a code instruction, a data item, or an undefined byte run. A ``NamedTuple`` rather than a dataclass because this is by far the @@ -111,7 +111,7 @@ class Head(NamedTuple): name: str | None = None raw: bytes | None = None # opcode/item bytes (filled in for code by the model) #: [(kind, text)] from IDA's own colour tags — mnem/reg/num/name/str/punct/… - #: None when Code Mode didn't provide them (or the spans + #: None when IDA Nexus didn't provide them (or the spans #: disagreed with the plain text, in which case the text wins). #: #: Held exactly as it came off the wire, and **read-only**. The worker @@ -693,7 +693,7 @@ class ListingModel: """A flat, IDA-style disassembly *listing* over one segment: code, data and undefined heads interleaved, unlike ``DisasmModel`` (one function, code only). - Backed by the Code Mode adapter's ``heads`` operation, which walks item heads + Backed by the IDA Nexus adapter's ``heads`` operation, which walks item heads and renders each via ``generate_disasm_line``. The segment is walked lazily in forward pages (``FunctionIndex`` style); line index == position in the walked head list. Random access to an address is O(distance-from-seg-start) the @@ -701,7 +701,7 @@ class ListingModel: on demand as the viewport scrolls. Synchronous + thread-safe. """ - PAGE = 500 # viewport-scale heads per Code Mode execution + PAGE = 500 # viewport-scale heads per IDA Nexus execution #: Generation marker for a skeleton (text-less) page. Never equals a real #: _text_gen, which counts up from 0, so such a page always reads as stale. _SKELETON_GEN = -1 @@ -1434,7 +1434,7 @@ class HexModel: class Program: """The bound analysis session: models, caches, and a small prefetch pool.""" - def __init__(self, client: "CodeModeClient", prefetch_workers: int = 2): + def __init__(self, client: "NexusClient", prefetch_workers: int = 2): self.client = client self._pool = ThreadPoolExecutor( max_workers=prefetch_workers, thread_name_prefix="idatui-prefetch" @@ -1485,7 +1485,7 @@ class Program: """Sorted raw segment map [(start, end, file_off, name)] — the single source for sections()/file_regions()/image_range. Cached. - Uses the Code Mode adapter's ``file_regions`` operation (a plain segment + Uses the IDA Nexus adapter's ``file_regions`` operation (a plain segment walk, ~ms), avoiding broad binary surveys on the hex-pane open path. """ if self._segments_cache is not None: @@ -1573,7 +1573,7 @@ class Program: def read_bytes(self, ea: int, n: int) -> bytes: """Raw bytes [ea, ea+n) from IDA (gaps read as zero). - The Code Mode adapter returns one contiguous hex string (C-speed in IDA). + The IDA Nexus adapter returns one contiguous hex string (C-speed in IDA). A legacy ``get_bytes`` decoding fallback remains for alternate clients. """ if n <= 0: @@ -1778,7 +1778,7 @@ class Program: except IDAToolError as e: msg = e.message if "not found" in msg.lower() and "del_type" in msg: - return "the connected Code Mode runtime cannot delete local types" + return "the connected IDA Nexus runtime cannot delete local types" return msg # -- disassembly ------------------------------------------------------- # @@ -1795,7 +1795,7 @@ class Program: """Drop local and Hex-Rays caches before an explicit view refresh. Normal edit paths use generation-based invalidation. Ctrl+R is also for - changes made by another Code Mode/IDA client, for which this Program has + changes made by another IDA Nexus/IDA client, for which this Program has seen no generation bump, so it must explicitly ask Hex-Rays to discard its cached cfunc. """ @@ -1809,7 +1809,7 @@ class Program: pass def decompile(self, ea: int, refresh: bool = False) -> Decompilation: - """Full pseudocode for a function, returned directly by Code Mode.""" + """Full pseudocode for a function, returned directly by IDA Nexus.""" if not refresh: with self._lock: hit = self._decomp.get(ea) @@ -2011,7 +2011,7 @@ class Program: def define_func(self, ea: int) -> dict: """Create a function starting at ``ea`` (IDA's 'p'). - Prefers the Code Mode operation, which works out the end when IDA can't; + Prefers the IDA Nexus operation, which works out the end when IDA can't; falls back to a plain create for alternate clients. """ try: diff --git a/idatui/errors.py b/idatui/errors.py index 7632ade..ab1365a 100644 --- a/idatui/errors.py +++ b/idatui/errors.py @@ -1,6 +1,6 @@ """TUI-facing error hierarchy and lightweight database session model. -The Code Mode adapter normalizes ``ida_codemode`` transport and execution +The IDA Nexus adapter normalizes ``ida_nexus`` transport and execution errors into these types so the domain and Textual layers do not depend on HTTP or registry implementation details. """ diff --git a/idatui/launch.py b/idatui/launch.py index e3cb091..a0201a7 100644 --- a/idatui/launch.py +++ b/idatui/launch.py @@ -1,6 +1,6 @@ -"""One-shot launcher for the IDA Code Mode-backed TUI. +"""One-shot launcher for the IDA Nexus-backed TUI. -A path first resolves to a registered GUI database; when none matches, Code Mode +A path first resolves to a registered GUI database; when none matches, IDA Nexus reuses or starts a managed idalib worker. With no path, a single registered database is selected automatically. @@ -34,9 +34,9 @@ def _log(msg: str) -> None: def _registered_databases() -> tuple[list[dict], list[dict]]: - """Ready and blocked Code Mode registrations, with normalized errors.""" + """Ready and blocked IDA Nexus registrations, with normalized errors.""" try: - from ida_codemode import InstanceState, discover_databases + from ida_nexus import InstanceState, discover_databases ready: list[dict] = [] blocked: list[dict] = [] @@ -70,7 +70,7 @@ def main(argv: list[str] | None = None) -> int: help="open a multi-binary project (created from the given " "binaries if FILE doesn't exist)") p.add_argument("--ttl", type=int, default=1800, - help="deprecated compatibility option (Code Mode uses leases)") + help="deprecated compatibility option (IDA Nexus uses leases)") p.add_argument("--no-keepalive", action="store_true", help="deprecated compatibility option (the lease is the heartbeat)") p.add_argument("--rpc", metavar="PATH", @@ -87,7 +87,7 @@ def main(argv: list[str] | None = None) -> int: g.add_argument("--base", metavar="ADDR", help="load address, e.g. 0x8000000 (any base; NOT paragraphs)") g.add_argument("--ida-args", metavar="STR", dest="ida_args", - help="legacy switches; only Code Mode-representable -p/-b/-T are accepted") + help="legacy switches; only IDA Nexus-representable -p/-b/-T are accepted") args = p.parse_args(argv) load: dict = {} @@ -166,10 +166,10 @@ def main(argv: list[str] | None = None) -> int: _log(f"attaching to registered {item.get('backend')} database: {binary}") elif not ready: detail = f" ({blocked[0].get('error')})" if blocked else "" - _log(f"no registered Code Mode database; pass a binary path{detail}") + _log(f"no registered IDA Nexus database; pass a binary path{detail}") return 2 else: - _log("several Code Mode databases are registered; pass one of these paths:") + _log("several IDA Nexus databases are registered; pass one of these paths:") for item in ready: _log(f" {item.get('exe_path') or item.get('idb_path')} " f"[{item.get('backend')}, {item.get('record_id')}]") diff --git a/idatui/nexus_client.py b/idatui/nexus_client.py new file mode 100644 index 0000000..44eb1b1 --- /dev/null +++ b/idatui/nexus_client.py @@ -0,0 +1,551 @@ +"""Client adapter from ida-tui's domain operations to IDA Nexus. + +``DatabaseHandle`` is the lifecycle boundary: it discovers an already-registered +GUI database, reuses a shared managed idalib worker, or starts one when needed. +The TUI never owns or terminates an IDA process. Closing this client releases +only its lease. + +Remote operations are ordinary typed Python functions declared in +``idatui.remote_ops``. IDA Nexus installs their content-addressed modules once +per IDA Python interpreter; subsequent calls send only encoded arguments. The +optimized IDAPython listing/decompiler implementation remains real source in +``idatui.remote_tools`` and is installed through the same module interface. +""" + +from __future__ import annotations + +import os +import shlex +import threading +import time +from collections.abc import Callable +from typing import Any + +from .errors import IDAConnectionError, IDATimeoutError, IDAToolError, Session + +# ida_nexus is imported EAGERLY-IF-PRESENT but never at hard import cost. +# +# The paging/graph/trace layers and their offline test suites must keep importing +# `idatui` on a machine with no IDA and no IDA Nexus installed -- that is the +# house rule the stdlib-only worker client used to satisfy for free, and +# `tests/run.py --fast` (380 checks, any python3) depends on it. A hard top-level +# import here makes the whole package unimportable, so the failure is deferred to +# the first operation that genuinely needs the library. +_NEXUS_ERROR: Exception | None = None +try: + from ida_nexus import ( + DatabaseBusyError, + DatabaseDisconnectedError, + DatabaseHandle, + DatabaseInstance, + DatabaseOpenOptions, + NexusConnectionError, + RemoteError, + find_database_owner, + wait_database_released, + ) +except ImportError as _exc: # library absent: usable only for offline layers + _NEXUS_ERROR = _exc + # Bound to None rather than left undefined so the names stay patchable: the + # offline contract tests inject a fake DatabaseHandle here. + NexusConnectionError = DatabaseDisconnectedError = RemoteError = None # type: ignore[assignment,misc] + DatabaseBusyError = DatabaseHandle = DatabaseInstance = None # type: ignore[assignment,misc] + DatabaseOpenOptions = find_database_owner = wait_database_released = None # type: ignore[assignment] + + +def _require_nexus() -> None: + """Raise an actionable error when the IDA Nexus library is missing. + + Gated on the binding, not on the original import result, so a test that + injects a fake ``DatabaseHandle`` exercises the real adapter logic. + """ + if DatabaseHandle is None: + raise IDAConnectionError( + "ida-nexus is not installed in this environment " + f"({_NEXUS_ERROR}). Install it (e.g. `uv sync`, or " + "`pip install ida-nexus`) so ida-tui can lease a " + "database." + ) from _NEXUS_ERROR + + +def database_owner(idb_path: str, staged_path: str | None = None): + """The IDA Nexus instance that owns ``idb_path``/``staged_path``, else None. + + Returns None when the IDA Nexus library is absent: with no library there is + no client in this environment that could be holding the database, and the + IDA-free layers (project staging) must keep working. Discovery errors with + the library installed still propagate because unknown ownership is unsafe. + """ + if DatabaseHandle is None: + return None + if staged_path: + owner = find_database_owner( + staged_path, + output_database=idb_path, + timeout=0.5, + ) + return owner or find_database_owner(staged_path, timeout=0.5) + return find_database_owner(idb_path, timeout=0.5) + + +def registered_database(path: str, output_database: str | None = None) -> bool: + """Whether a live/lock-held IDA Nexus instance owns this target.""" + _require_nexus() + return ( + find_database_owner( + path, + output_database=output_database, + timeout=0.5, + ) + is not None + ) + + +class _NoopKeepAlive: + """Compatibility shim: the DatabaseHandle's SSE lease is the heartbeat.""" + + def __init__(self) -> None: + self.beats = self.failures = 0 + + def start(self) -> "_NoopKeepAlive": + return self + + def stop(self) -> None: + pass + + +def _parse_load_args(value: str) -> tuple[str | None, int | None, str | None]: + """Translate ida-tui's legacy first-open switches to IDA Nexus options. + + IDA Nexus has typed options for processor, natural loading address and file + type. It deliberately has no arbitrary command-line escape hatch; reject + switches we cannot represent instead of silently loading a blob wrongly. + """ + processor: str | None = None + loading_address: int | None = None + file_type: str | None = None + unsupported: list[str] = [] + try: + words = shlex.split(value or "", posix=os.name != "nt") + except ValueError as exc: + raise ValueError(f"invalid IDA load options: {exc}") from exc + for word in words: + if word.startswith("-p") and len(word) > 2: + processor = word[2:] + elif word.startswith("-b") and len(word) > 2: + try: + # IDA's -b is in 16-byte paragraphs. DatabaseHandle expects the + # natural address, which is the safer public API. + loading_address = int(word[2:], 16) << 4 + except ValueError as exc: + raise ValueError(f"invalid IDA loading address: {word!r}") from exc + elif word.startswith("-T") and len(word) > 2: + file_type = word[2:] + else: + unsupported.append(word) + if unsupported: + joined = " ".join(unsupported) + raise ValueError( + "ida-nexus cannot represent arbitrary IDA load options: " + f"{joined!r}; use processor/base/file type options instead" + ) + return processor, loading_address, file_type + + +class IDBEventListener: + """Debounced, closeable delivery of another client's IDB changes. + + IDA Nexus's subscription is a blocking iterator, so one daemon thread reads + it and a second waits for a quiet period before handing a batch to the UI. + Keeping the debounce here avoids a permanent Textual worker (which would + make the app's worker-idle contract impossible) and bounds refresh work to + one pass per edit burst. + """ + + def __init__( + self, + client: "NexusClient", + callback: Callable[[tuple[dict[str, Any], ...]], None], + *, + on_error: Callable[[BaseException], None] | None = None, + debounce: float = 0.2, + ) -> None: + self._client = client + self._callback = callback + self._on_error = on_error + self._debounce = max(float(debounce), 0.0) + self._condition = threading.Condition() + self._closed = False + self._subscription = None + self._pending: list[dict[str, Any]] = [] + self._deadline = 0.0 + self._reader = threading.Thread( + target=self._read, name="idatui-idb-events", daemon=True + ) + self._deliverer = threading.Thread( + target=self._deliver, name="idatui-idb-refresh", daemon=True + ) + self._deliverer.start() + self._reader.start() + + def _report(self, error: BaseException) -> None: + disconnected = DatabaseDisconnectedError + if isinstance(disconnected, type) and isinstance(error, disconnected): + error = self._client._connection_error(error) + with self._condition: + closed = self._closed + if not closed and self._on_error is not None: + self._on_error(error) + + def _read(self) -> None: + try: + subscription = self._client.subscribe_idb_events() + except Exception as exc: # noqa: BLE001 -- surfaced through on_error + self._report(exc) + with self._condition: + self._closed = True + self._pending.clear() + self._condition.notify_all() + return + with self._condition: + if self._closed: + subscription.close() + return + self._subscription = subscription + try: + for event in subscription: + with self._condition: + if self._closed: + break + if self._client.owns_event(event): + continue + with self._condition: + if self._closed: + break + self._pending.append(event) + self._deadline = time.monotonic() + self._debounce + self._condition.notify_all() + except Exception as exc: # noqa: BLE001 -- stream failures are recoverable + self._report(exc) + finally: + subscription.close() + with self._condition: + if self._subscription is subscription: + self._subscription = None + self._closed = True + self._pending.clear() + self._condition.notify_all() + + def _deliver(self) -> None: + while True: + with self._condition: + while not self._closed and not self._pending: + self._condition.wait() + if self._closed: + return + remaining = self._deadline - time.monotonic() + if remaining > 0: + self._condition.wait(remaining) + continue + batch = tuple(self._pending) + self._pending.clear() + try: + self._callback(batch) + except Exception as exc: # noqa: BLE001 -- keep the stream alive + self._report(exc) + + def close(self) -> None: + """Stop delivery and unblock the subscription reader.""" + with self._condition: + if self._closed: + return + self._closed = True + self._pending.clear() + subscription = self._subscription + self._condition.notify_all() + if subscription is not None: + subscription.close() + + +class NexusClient: + """A leased GUI/idalib database accessed through ``ida_nexus``.""" + + def __init__( + self, + binary_path: str, + *, + ttl: int = 0, + load_args: str = "", + processor: str | None = None, + loading_address: int | None = None, + file_type: str | None = None, + output_database: str | None = None, + spawn: bool = True, + new_database: bool = False, + ) -> None: + del ttl # managed-worker lifetime is lease-based, not idle-TTL based + self._path = os.path.abspath(os.path.expanduser(binary_path)) + parsed_processor, parsed_address, parsed_file_type = _parse_load_args(load_args) + self._processor = processor or parsed_processor + self._loading_address = ( + loading_address if loading_address is not None else parsed_address + ) + self._file_type = file_type or parsed_file_type + self._output_database = output_database + self._spawn = spawn + self._new_database = new_database + self._handle: DatabaseHandle | None = None + self._last_instance: DatabaseInstance | None = None + self._connect_lock = threading.Lock() + + def connect(self, timeout: float = 1800.0, progress=None) -> "NexusClient": + _require_nexus() + with self._connect_lock: + handle = self._handle + if handle is not None: + if handle.connected: + return self + raise IDAConnectionError( + "IDA Nexus database disconnected; explicit rediscovery required" + ) + if progress: + progress( + f"discovering IDA Nexus database for {os.path.basename(self._path)}…" + ) + try: + # A Ctrl+L reload releases its current managed-worker lease, but + # that worker remains registered during IDA Nexus's final-lease + # grace period. Retry only that known handoff window. A GUI or + # another long-lived client remains busy and yields a clear + # failure rather than being modified underneath its owner. + deadline = time.monotonic() + min(timeout, 60.0) + while True: + try: + handle = DatabaseHandle.open( + self._path, + options=DatabaseOpenOptions( + spawn=self._spawn, + startup_timeout=max(0.1, timeout), + output_database=self._output_database, + processor=self._processor, + # The natural byte address is converted to IDA's + # paragraph-based -b value by IDA Nexus. + image_base=self._loading_address, + file_type=self._file_type, + new_database=self._new_database, + ), + ) + break + except DatabaseBusyError: + if not self._new_database or time.monotonic() >= deadline: + raise + if progress: + progress( + "waiting for the previous IDA Nexus lease to close…" + ) + owner = find_database_owner( + self._path, + output_database=self._output_database, + timeout=0.5, + ) + if owner is not None: + wait_database_released( + owner, + max(0.0, deadline - time.monotonic()), + ) + else: + time.sleep(0.2) + if progress: + backend = handle.instance.backend + progress( + f"attached to {backend} database; waiting for auto-analysis…" + ) + handle.wait_autoanalysis(timeout=timeout) + except Exception as exc: # normalize the dependency's transport errors + raise self._connection_error(exc) from exc + self._handle = handle + self._last_instance = handle.instance + return self + + @staticmethod + def _connection_error(exc: BaseException) -> IDAConnectionError: + return IDAConnectionError(str(exc) or type(exc).__name__) + + @property + def connected(self) -> bool: + return self._handle is not None and self._handle.connected + + @property + def pid(self) -> int | None: + return self._handle.instance.pid if self._handle is not None else None + + @property + def backend(self) -> str | None: + return self._handle.instance.backend if self._handle is not None else None + + def owns_event(self, event: dict[str, Any]) -> bool: + """Whether ``event`` was produced through this client's handle.""" + handle = self._handle + return handle is not None and handle.owns_event(event) + + def subscribe_idb_events(self): + """Open IDA Nexus's closeable IDB-change iterator.""" + if not self.connected: + self.connect() + handle = self._handle + if handle is None: + raise IDAConnectionError("IDA Nexus database is not connected") + try: + return handle.subscribe_idb_events() + except (DatabaseDisconnectedError, NexusConnectionError) as exc: + raise self._connection_error(exc) from exc + + def watch_idb_events( + self, + callback: Callable[[tuple[dict[str, Any], ...]], None], + *, + on_error: Callable[[BaseException], None] | None = None, + debounce: float = 0.2, + ) -> IDBEventListener: + """Deliver external IDB changes in debounced batches.""" + return IDBEventListener(self, callback, on_error=on_error, debounce=debounce) + + def call(self, operation: Callable[..., Any], /, **args) -> Any: + """Execute one source-backed remote declaration through this client.""" + name = getattr(operation, "__name__", "remote operation") + try: + from .remote_ops import bind + + remote = bind(operation) + except KeyError as exc: + raise IDAToolError( + name, f"remote operation {name!r} is not registered" + ) from exc + if not self.connected: + self.connect() + handle = self._handle + if handle is None: + raise IDAConnectionError("IDA Nexus database is not connected") + try: + return remote(handle, **args) + except RemoteError as exc: + message = str(exc) + if exc.details.get("traceback"): + message += f"\n{exc.details['traceback']}" + if exc.code == "operation_timeout": + raise IDATimeoutError(message) from exc + raise IDAToolError(name, message) from exc + except (DatabaseDisconnectedError, NexusConnectionError) as exc: + raise self._connection_error(exc) from exc + + def save_database(self) -> dict[str, Any]: + if not self.connected: + self.connect() + handle = self._handle + if handle is None: + raise IDAConnectionError("IDA Nexus database is not connected") + try: + return handle.save_database() + except RemoteError as exc: + raise IDAToolError("save_database", str(exc)) from exc + except (DatabaseDisconnectedError, NexusConnectionError) as exc: + raise self._connection_error(exc) from exc + + def discard_database(self, timeout: float = 5.0) -> bool: + """Discard a final managed-worker lease; otherwise transfer finalization. + + ``False`` is an expected ownership result: a GUI owns its session, or + another lease still shares the managed worker. A busy final worker is + retried briefly so background reads finishing during quit do not turn a + real discard into an implicit save. + """ + handle = self._handle + if handle is None or not handle.connected: + return False + entry = handle.instance + if entry.backend != "idalib" or not getattr(entry, "managed", False): + return False + deadline = time.monotonic() + max(float(timeout), 0.0) + while True: + try: + handle.shutdown_database(save=False) + return True + except RemoteError as exc: + if exc.code in ("instance_shared", "shutdown_not_supported"): + return False + if exc.code == "instance_busy" and time.monotonic() < deadline: + time.sleep(0.05) + continue + raise IDAToolError("shutdown_database", str(exc)) from exc + except (DatabaseDisconnectedError, NexusConnectionError) as exc: + raise self._connection_error(exc) from exc + + def health(self) -> dict[str, Any]: + if not self.connected: + self.connect() + assert self._handle is not None + entry = self._handle.instance + module = os.path.basename(entry.exe_path or entry.idb_path or self._path) + return { + "ok": self._handle.connected, + "module": module, + "backend": entry.backend, + "record_id": entry.record_id, + "input_path": entry.exe_path, + "idb_path": entry.idb_path, + } + + def keepalive(self, interval: float = 120.0) -> _NoopKeepAlive: + del interval + return _NoopKeepAlive() + + def resolve_db(self) -> str: + if not self.connected: + self.connect() + assert self._handle is not None + return self._handle.instance.record_id + + def set_db(self, db: str | None) -> None: + del db # one handle is permanently bound to one registered database + + def list_sessions(self) -> list[Session]: + if not self.connected: + self.connect() + assert self._handle is not None + entry = self._handle.instance + path = entry.exe_path or entry.idb_path or self._path + return [ + Session( + session_id=entry.record_id, + filename=os.path.basename(path), + input_path=path, + is_active=True, + ) + ] + + def close(self, grace: float = 0.0) -> None: + del grace + with self._connect_lock: + handle, self._handle = self._handle, None + if handle is not None: + self._last_instance = handle.instance + handle.close() # release our lease; never close a GUI/other client's DB + + def wait_released(self, timeout: float = 45.0) -> bool: + """Wait until a managed instance releases its lifetime lock. + + Normal application shutdown must not wait: another client may retain the + worker. This is an explicit test/maintenance helper for deleting a + temporary IDB safely after this client closes. GUI instances return + ``False`` immediately because clients never own their lifetime. + """ + instance = self._last_instance + if instance is None or instance.backend != "idalib": + return False + return wait_database_released(instance, timeout) + + def __enter__(self) -> "NexusClient": + return self.connect() + + def __exit__(self, *exc) -> None: + self.close() diff --git a/idatui/pane.py b/idatui/pane.py index c31a93c..1eee847 100644 --- a/idatui/pane.py +++ b/idatui/pane.py @@ -24,7 +24,7 @@ per pane in the registry, so stop/list/capture/keys keep working across both python -m idatui.pane keys --pane Escape Requires: running inside tmux or zellij. Each pane leases a registered GUI or -shared managed idalib database through Code Mode. Uses ~/ida-venv/bin/python for +shared managed idalib database through IDA Nexus. Uses ~/ida-venv/bin/python for the TUI (needs textual) unless --python / IDATUI_PYTHON says otherwise. """ from __future__ import annotations @@ -250,8 +250,8 @@ def _pane_keys(pane: str, keys: list[str], mux: str | None = None) -> None: subprocess.run(["tmux", "send-keys", "-t", pane, *keys], check=True) -# Code Mode owns database process lifetime: a closed pane drops its lease at the -# socket/kernel boundary and Code Mode decides whether a managed worker still +# IDA Nexus owns database process lifetime: a closed pane drops its lease at the +# socket/kernel boundary and IDA Nexus decides whether a managed worker still # has clients. There is nothing for the pane layer to reap. @@ -261,7 +261,7 @@ def _count_live_panes() -> int: def _reap_orphan_workers(force: bool = False) -> int: - """Compatibility no-op: Code Mode workers are shared and lease-managed.""" + """Compatibility no-op: IDA Nexus workers are shared and lease-managed.""" del force return 0 @@ -294,7 +294,7 @@ def spawn(args) -> int: print(f"error: no such project: {project}", file=sys.stderr) return 2 - # The pane owns only the TUI. Code Mode's lease cleanup handles crashes; + # The pane owns only the TUI. IDA Nexus's lease cleanup handles crashes; # kill-pane must never reap a shared GUI/idalib database. if project is not None: # launch takes: --project FILE [binaries...]; extra binaries are added to @@ -347,7 +347,7 @@ def _wait_ready(sock: str, timeout: float, pane: str, stuck_after: float = 45.0, mux: str | None = None) -> dict[str, Any]: """Poll the socket + ping until the TUI reports ready (or timeout). - Emits a one-time hint if Code Mode discovery/opening is still not ready after + Emits a one-time hint if IDA Nexus discovery/opening is still not ready after ``stuck_after`` seconds. """ start = time.time() @@ -370,7 +370,7 @@ def _wait_ready(sock: str, timeout: float, pane: str, why = ("RPC socket not created yet" if not os.path.exists(sock) else "TUI up but analysis not ready") print(f"still waiting ({int(time.time() - start)}s): {why}. " - f"Check Code Mode registrations and worker logs.", file=sys.stderr) + f"Check IDA Nexus registrations and worker logs.", file=sys.stderr) time.sleep(0.4) last = dict(last) last["ready"] = False @@ -467,7 +467,7 @@ def list_panes(args) -> int: def reap(args) -> int: - """Deprecated no-op; shared Code Mode workers are managed by leases.""" + """Deprecated no-op; shared IDA Nexus workers are managed by leases.""" print(json.dumps({"reaped_workers": 0, "live_panes": _count_live_panes(), "forced": args.force, "deprecated": True})) return 0 @@ -572,7 +572,7 @@ def main(argv: list[str]) -> int: ls.add_argument("--prune", action="store_true", help="drop dead panes (and their sockets)") ls.set_defaults(fn=list_panes) - rp = sub.add_parser("reap", help="deprecated no-op (Code Mode uses shared leases)") + rp = sub.add_parser("reap", help="deprecated no-op (IDA Nexus uses shared leases)") rp.add_argument("--force", action="store_true", help=argparse.SUPPRESS) rp.set_defaults(fn=reap) diff --git a/idatui/pool.py b/idatui/pool.py index 573aa75..2407b44 100644 --- a/idatui/pool.py +++ b/idatui/pool.py @@ -1,6 +1,6 @@ -"""DatabasePool — LRU leases on Code Mode databases for a project. +"""DatabasePool — LRU leases on IDA Nexus databases for a project. -Code Mode may bind a lease to an existing IDA GUI or to a shared managed idalib +IDA Nexus may bind a lease to an existing IDA GUI or to a shared managed idalib worker. The pool therefore owns *client interest*, never an IDA process. Releasing an LRU entry persists managed IDBs but does not implicitly save a GUI, then closes only this TUI's lease; other clients and GUI sessions remain alive. Managed workers exit themselves after their final lease. @@ -48,8 +48,8 @@ def _pss_mb(pid: int | None) -> int: def _default_spawn(ref: BinaryRef, ttl: int, *, new_database: bool = False): # pragma: no cover - needs IDA - from .codemode_client import CodeModeClient - return CodeModeClient( + from .nexus_client import NexusClient + return NexusClient( ref.staged, ttl=ttl, load_args=ref.load_args, @@ -59,7 +59,7 @@ def _default_spawn(ref: BinaryRef, ttl: int, *, new_database: bool = False): # class DatabasePool: - """Live Code Mode database leases, keyed by project label.""" + """Live IDA Nexus database leases, keyed by project label.""" def __init__(self, project: Project, *, budget_mb: int | None = None, ttl: int = 1800, spawn=None, mem_fn=None) -> None: @@ -101,7 +101,7 @@ class DatabasePool: """A live client for ``label``, attaching or spawning as needed. Do not sweep IDA scratch files here: a registered GUI or another Code - Mode client may own the database. Code Mode's registry locks and health + Mode client may own the database. IDA Nexus's registry locks and health probes are the authority for safe discovery and stale-record cleanup. """ client = self._clients.get(label) @@ -269,5 +269,3 @@ class DatabasePool: f"{self.memory_mb()}/{self.budget_mb}MB active={self.active}>") -# Source compatibility for callers that imported the pre-Code-Mode name. -WorkerPool = DatabasePool diff --git a/idatui/project.py b/idatui/project.py index 53f3b0a..e681dfc 100644 --- a/idatui/project.py +++ b/idatui/project.py @@ -23,7 +23,7 @@ firmware image, a cleaned build tree). A source whose size/mtime no longer matches the staged copy is re-staged, and its now-stale database is dropped (the DB describes the old bytes). -The model has no IDA imports. Staging consults ida_codemode's registry before +The model has no IDA imports. Staging consults ida_nexus's registry before replacing files so it never mutates a database owned by a GUI/shared worker. """ from __future__ import annotations @@ -57,7 +57,7 @@ class BinaryRef: #: it is, a raw firmware image doesn't, and IDA defaults to metapc at 0. processor: str = "" # IDA processor name: arm, armb, mipsb, metapc, … base: int = 0 # load address (natural, e.g. 0x8000000) - ida_args: str = "" # legacy -p/-b/-T switches accepted by Code Mode adapter + ida_args: str = "" # legacy -p/-b/-T switches accepted by IDA Nexus adapter @property def db(self) -> str: @@ -311,7 +311,7 @@ class Project: """Ensure ``ref`` is staged in the sidecar; returns the staged path. Re-staging a changed source drops its database: the DB describes the old - bytes. Refuse while Code Mode reports a GUI/idalib owner; replacing a + bytes. Refuse while IDA Nexus reports a GUI/idalib owner; replacing a staged executable or IDB underneath a shared live instance is corruption. """ if not os.path.isfile(ref.source): @@ -319,15 +319,15 @@ class Project: if not self.is_stale(ref): return ref.staged try: - from .codemode_client import database_owner + from .nexus_client import database_owner owner = database_owner(ref.db, ref.staged) except Exception as exc: raise ProjectError( - f"cannot verify Code Mode ownership before staging {ref.label}: {exc}" + f"cannot verify IDA Nexus ownership before staging {ref.label}: {exc}" ) from exc if owner is not None: raise ProjectError( - f"cannot restage {ref.label}: Code Mode instance {owner.record_id} " + f"cannot restage {ref.label}: IDA Nexus instance {owner.record_id} " f"still owns {owner.idb_path}; close/release it first" ) os.makedirs(self.bin_dir, exist_ok=True) @@ -353,7 +353,7 @@ class Project: def sweep_scratch(self, ref: BinaryRef) -> int: """Delete unpacked working files (never the ``.i64``) for maintenance. - Runtime paths no longer call this: Code Mode instances are shared, so a + Runtime paths no longer call this: IDA Nexus instances are shared, so a registry owner may still be using these files. Callers must independently prove that no GUI/idalib instance owns the database. """ diff --git a/idatui/remote_ops.py b/idatui/remote_ops.py index 91ffeb7..a67b2b7 100644 --- a/idatui/remote_ops.py +++ b/idatui/remote_ops.py @@ -1,4 +1,4 @@ -"""Typed remote operations executed through ida-codemode.""" +"""Typed remote operations executed through ida-nexus.""" from __future__ import annotations # ruff: noqa @@ -1655,7 +1655,7 @@ def _bindings() -> dict[Callable[..., Any], Any]: with _BIND_LOCK: if _BOUND is not None: return _BOUND - from ida_codemode import RemoteModule + from ida_nexus import RemoteModule operations_module = RemoteModule( Path(__file__), operation_label=operation_label, codec="json" diff --git a/idatui/remote_tools.py b/idatui/remote_tools.py index db04244..6681379 100644 --- a/idatui/remote_tools.py +++ b/idatui/remote_tools.py @@ -1,8 +1,8 @@ -"""The IDAPython ida-tui runs inside the Code Mode sandbox. +"""The IDAPython ida-tui runs inside the IDA Nexus sandbox. Two features have no ida-domain surface at all and are carried over VERBATIM from the tools ida-tui was developed against (`server/patch_server.py`'s -injected BODY, which the Code Mode port deletes): +injected BODY, which the IDA Nexus port deletes): * `heads` -- the continuous listing. ida-domain enumerates defined heads and renders plain disassembly; the listing also needs coalesced undefined runs, @@ -20,8 +20,8 @@ what you see). A re-implementation drifts from it silently. This file is SOURCE SHIPPED AS TEXT to the database process; it is never imported here, because the ida_* modules do not exist in the TUI's interpreter. -`codemode_client` reads it and prepends it to the relevant snippets. Keep it -self-contained: no relative imports, nothing beyond what Code Mode provides. +`nexus_client` reads it and prepends it to the relevant snippets. Keep it +self-contained: no relative imports, nothing beyond what IDA Nexus provides. """ # ruff: noqa @@ -29,7 +29,7 @@ import re as _re # IDAPython, imported ONCE at module scope. # -# This file is never imported by the client -- codemode_client reads it as +# This file is never imported by the client -- nexus_client reads it as # TEXT and installs it as a module inside the database process -- so the # no-IDA house rule that keeps idatui importable without IDA does not apply # here, and these need not be function-local. @@ -1740,7 +1740,7 @@ def decompile(addr, include_addresses=True): Faithful to the tool ida-tui was written against, and in particular to its COST: the per-line address anchor comes from ONE ``get_line_item`` at column - 0 per line. The Code Mode port asked for the full per-column line map (what + 0 per line. The IDA Nexus port asked for the full per-column line map (what ``decomp_map`` is for) purely to fill in that anchor, which is thousands of ``get_line_item``+``dstr()`` calls per function instead of one per line, and made every pseudocode open cost the same as opening the split view. diff --git a/pyproject.toml b/pyproject.toml index a46dd45..dc68d12 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,12 +1,12 @@ [project] name = "idatui" version = "0.0.1" -description = "A keyboard-first TUI frontend for shared IDA Code Mode databases." +description = "A keyboard-first TUI frontend for shared IDA Nexus databases." requires-python = ">=3.11" -# ida-codemode supplies GUI discovery, shared idalib workers, leases, and the +# ida-nexus supplies GUI discovery, shared idalib workers, leases, and the # execute_python/ida-domain database surface. dependencies = [ - "ida-codemode>=0.5.3", + "ida-nexus>=0.7.0", "textual>=8", "pygments>=2", # Used directly for pseudocode highlighting. ] @@ -29,3 +29,4 @@ build-backend = "hatchling.build" [tool.hatch.build.targets.wheel] packages = ["idatui"] + diff --git a/tests/_fixtures.py b/tests/_fixtures.py index 16192fa..bc10235 100644 --- a/tests/_fixtures.py +++ b/tests/_fixtures.py @@ -133,7 +133,7 @@ async def build_pristine(binary: str, cache: str, app_factory) -> None: break app.program.client.save_database() # Textual's headless run_test context does not reliably emit App.Unmount on - # every platform/version; release the Code Mode lease explicitly. + # every platform/version; release the IDA Nexus lease explicitly. if app.program is not None: app.program.close() if app.client is not None: diff --git a/tests/run.py b/tests/run.py index b38a707..5d552bb 100755 --- a/tests/run.py +++ b/tests/run.py @@ -51,8 +51,8 @@ import time ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) TESTS = os.path.join(ROOT, "tests") -#: The IDA-capable interpreter. The pilot tests need textual AND the Code Mode -#: library in one python; the database process is Code Mode's to place. +#: The IDA-capable interpreter. The pilot tests need textual AND the IDA Nexus +#: library in one python; the database process is IDA Nexus's to place. DEFAULT_PY = os.path.expanduser("~/ida-venv/bin/python") #: Both shapes the suites print: "N passed, M failed" and "N checks, M failed". diff --git a/tests/test_codemode_client.py b/tests/test_codemode_client.py deleted file mode 100644 index dd0ae08..0000000 --- a/tests/test_codemode_client.py +++ /dev/null @@ -1,440 +0,0 @@ -"""IDA-free contract tests for the Code Mode client adapter.""" - -from __future__ import annotations - -import os -import queue -import sys -import threading -import time -import tempfile -from dataclasses import dataclass - -sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from idatui.errors import IDAConnectionError, IDAToolError # noqa: E402 -from idatui import remote_ops # noqa: E402 -import idatui.codemode_client as module # noqa: E402 -from idatui.codemode_client import CodeModeClient, _parse_load_args # noqa: E402 - -#: Pure: fakes the DatabaseHandle, never touches IDA or the Code Mode library. -NEEDS_IDA = False - -PASS = FAIL = 0 - - -def check(name: str, condition: bool, detail="") -> None: - global PASS, FAIL - if condition: - PASS += 1 - print(f" ok {name}") - else: - FAIL += 1 - print(f" FAIL {name} {detail}") - - -@dataclass(frozen=True) -class FakeEntry: - pid: int = 123 - backend: str = "gui" - record_id: str = "123-abcdef" - exe_path: str = "" - idb_path: str = "" - managed: bool = False - - -_CLOSED = object() - - -class FakeSubscription: - def __init__(self) -> None: - self._queue: queue.Queue = queue.Queue() - self.closed = False - - def __iter__(self): - return self - - def __next__(self): - item = self._queue.get() - if isinstance(item, BaseException): - raise item - if item is _CLOSED: - raise StopIteration - return item - - def emit(self, event: dict) -> None: - self._queue.put(event) - - def close(self) -> None: - if not self.closed: - self.closed = True - self._queue.put(_CLOSED) - - -class FakeHandle: - def __init__(self, path: str) -> None: - self.connected = True - self.instance = FakeEntry(exe_path=path, idb_path=path + ".i64") - self.waited = None - self.saved = 0 - self.closed = False - self.code = "" - self.codes = [] - self.code_timeout = None - self.operation_label = None - self.event_origin_id = "fake-handle-origin" - self.owns_checks = 0 - self.subscription = FakeSubscription() - self.shutdown_calls = [] - self.shutdown_error = None - - def wait_autoanalysis(self, timeout=None): - self.waited = timeout - return {"complete": True, "status": "complete"} - - def execute_python( - self, - code, - timeout=None, - *, - operation_id=None, - operation_label=None, - persist_globals=False, - filename=None, - ): - self.code = code - self.codes.append(code) - self.code_timeout = timeout - self.operation_label = operation_label - result = ( - { - "__remote_ida_status__": "ok", - "__remote_ida_value__": {"sentinel": 7}, - } - if ".modules.get(" in code - else True - ) - return {"result": result, "stdout": "", "stderr": ""} - - def subscribe_idb_events(self): - return self.subscription - - def owns_event(self, event): - self.owns_checks += 1 - return event.get("origin_id") == self.event_origin_id - - def save_database(self): - self.saved += 1 - return {"saved": True, "idb_path": self.instance.idb_path} - - def shutdown_database(self, *, save=True): - self.shutdown_calls.append(save) - if self.shutdown_error is not None: - raise module.RemoteError(self.shutdown_error, self.shutdown_error, 409) - return {"shutting_down": True, "save": save} - - def close(self): - self.subscription.close() - self.connected = False - self.closed = True - - -class FakeDatabaseHandle: - opened = None - opens = 0 - kwargs = None - - @classmethod - def open(cls, path, **kwargs): - cls.opens += 1 - cls.opened = path - cls.kwargs = kwargs - return FakeHandle(path) - - -@dataclass(frozen=True) -class FakeOpenOptions: - """Stand-in for DatabaseOpenOptions when the library is not installed. - - Deliberately STRICT (no **kwargs): an option the adapter invents would - raise here, and `_option_fields_are_real` checks the surviving names - against the real dataclass wherever it is importable. - """ - - spawn: bool = True - startup_timeout: float = 120.0 - output_database: str | None = None - processor: str | None = None - image_base: int | None = None - file_type: str | None = None - new_database: bool = False - - -class FakeBusy(Exception): - """Stand-in for DatabaseBusyError: `except None` is a TypeError.""" - - -class FakeDisconnected(Exception): - """Stand-in for DatabaseDisconnectedError in stdlib-only runs.""" - - -def _open_kwargs_are_real(sent: dict): - """(ok, detail) for the kwargs the adapter passes to DatabaseHandle.open. - - Skips (passes) when ida_codemode is not installed, so the file stays pure. - """ - try: - import inspect - from ida_codemode import DatabaseHandle as Real - except ImportError: - return True, "ida_codemode not installed - signature not checked" - accepted = set(inspect.signature(Real.open).parameters) - unknown = sorted(set(sent) - accepted) - return not unknown, f"open() rejects {unknown}" - - -def _option_fields_are_real(options): - """(ok, detail) for the option names the adapter fills in. - - The open() signature no longer names the loader options -- they moved - inside DatabaseOpenOptions -- so the `loading_address` class of bug now - hides there instead. Check it in the same way. - """ - try: - import dataclasses - from ida_codemode import DatabaseOpenOptions as Real - except ImportError: - return True, "ida_codemode not installed - fields not checked" - accepted = {field.name for field in dataclasses.fields(Real)} - unknown = sorted({f.name for f in dataclasses.fields(options)} - accepted) - return not unknown, f"DatabaseOpenOptions rejects {unknown}" - - -def main() -> int: - proc, base, file_type = _parse_load_args("-parm:ARMv7-M -b800000 -TRaw") - check( - "legacy switches map to typed Code Mode options", - (proc, base, file_type) == ("arm:ARMv7-M", 0x8000000, "Raw"), - (proc, base, file_type), - ) - try: - _parse_load_args("-parm -zcustom") - except ValueError as exc: - check("arbitrary IDA switches fail loudly", "cannot represent" in str(exc), exc) - else: - check("arbitrary IDA switches fail loudly", False) - - original = module.DatabaseHandle - module.DatabaseHandle = FakeDatabaseHandle - # The library's own names when it is installed; strict fakes when it is not - # (this file must keep running under a stdlib-only python3). - original_options = module.DatabaseOpenOptions - original_busy = module.DatabaseBusyError - original_disconnected = module.DatabaseDisconnectedError - module.DatabaseOpenOptions = original_options or FakeOpenOptions - module.DatabaseBusyError = original_busy or FakeBusy - module.DatabaseDisconnectedError = original_disconnected or FakeDisconnected - try: - with tempfile.TemporaryDirectory() as tmp: - path = os.path.join(tmp, "sample.bin") - with open(path, "wb") as file: - file.write(b"sample") - client = CodeModeClient(path, load_args="-parm:ARMv7-A -b100") - notes = [] - client.connect(timeout=42, progress=notes.append) - handle = client._handle - check( - "connect delegates database discovery to DatabaseHandle.open", - FakeDatabaseHandle.opened == path and handle is not None, - ) - options = FakeDatabaseHandle.kwargs["options"] - check( - "typed loader options cross the dependency boundary", - options.processor == "arm:ARMv7-A" and options.image_base == 0x1000, - options, - ) - check( - "every open option exists in the real library", - *_option_fields_are_real(options), - ) - # A fake that swallows **kwargs cannot catch a keyword the real - # library does not have -- which is exactly how this port shipped - # `loading_address` (the real name is `image_base`) and would have - # raised TypeError on the very first connect. Check the names we - # send against the real signature whenever it is importable. - check( - "every open() keyword exists in the real library", - *_open_kwargs_are_real(FakeDatabaseHandle.kwargs), - ) - check( - "connect waits for Code Mode autoanalysis", - handle.waited == 42, - getattr(handle, "waited", None), - ) - check( - "progress distinguishes discovery and backend attachment", - len(notes) == 2 and "gui" in notes[-1], - notes, - ) - result = client.call( - remote_ops.list_funcs, queries=[{"offset": 0, "count": 2}] - ) - check( - "remote operation returns its JSON result", - result == {"sentinel": 7}, - result, - ) - check( - "operation source is real Python installed through ida-domain", - any("db.functions.get_all()" in code for code in handle.codes), - handle.codes[0][:200], - ) - check( - "remote operations attribute IDB events to IDA TUI", - handle.operation_label == "IDA TUI", - handle.operation_label, - ) - batches = [] - delivered = threading.Event() - - def changed(batch): - batches.append(batch) - delivered.set() - - watcher = client.watch_idb_events(changed, debounce=0.05) - handle.subscription.emit({"event_name": "renamed", "origin_id": "peer-1"}) - handle.subscription.emit( - {"event_name": "cmt_changed", "origin_id": "peer-2"} - ) - check( - "event bursts produce one debounced refresh", - delivered.wait(1) and len(batches) == 1 and len(batches[0]) == 2, - batches, - ) - delivered.clear() - handle.subscription.emit( - {"event_name": "renamed", "origin_id": handle.event_origin_id} - ) - time.sleep(0.1) - check( - "the listener uses handle ownership to ignore its own events", - not delivered.is_set() - and len(batches) == 1 - and handle.owns_checks >= 3, - (batches, handle.owns_checks), - ) - handle.subscription.emit( - {"event_name": "byte_patched", "origin_id": "peer-3"} - ) - watcher.close() - time.sleep(0.1) - check( - "closing drops a pending debounced refresh", - not delivered.is_set() and len(batches) == 1, - batches, - ) - check( - "health exposes registry identity", - client.health()["record_id"] == "123-abcdef", - ) - client.save_database() - check("save uses the public Code Mode save route", handle.saved == 1) - check( - "GUI leases transfer rather than claiming discard", - client.discard_database() is False and handle.shutdown_calls == [], - handle.shutdown_calls, - ) - handle.instance = FakeEntry( - backend="idalib", managed=True, exe_path=path, idb_path=path + ".i64" - ) - check( - "a final managed lease discards without saving", - client.discard_database() is True and handle.shutdown_calls == [False], - handle.shutdown_calls, - ) - handle.shutdown_error = "instance_shared" - check( - "a shared managed lease transfers finalization", - client.discard_database() is False, - handle.shutdown_calls, - ) - handle.shutdown_error = "instance_busy" - try: - client.discard_database(timeout=0) - except IDAToolError as exc: - check( - "a busy final lease never silently saves", - exc.tool == "shutdown_database", - exc, - ) - else: - check("a busy final lease never silently saves", False) - handle.shutdown_error = None - handle.instance = FakeEntry(exe_path=path, idb_path=path + ".i64") - opens = FakeDatabaseHandle.opens - handle.connected = False - try: - client.health() - except IDAConnectionError as exc: - check( - "a disconnected handle requires explicit rediscovery", - "explicit rediscovery" in str(exc) - and FakeDatabaseHandle.opens == opens, - (exc, FakeDatabaseHandle.opens, opens), - ) - else: - check("a disconnected handle requires explicit rediscovery", False) - client.close() - check("close releases only the handle lease", handle.closed) - check( - "GUI lifetime is never claimed by the client", - client.wait_released(0) is False, - ) - disconnected = CodeModeClient(path).connect() - stream_errors = [] - stream_failed = threading.Event() - - def failed(error): - stream_errors.append(error) - stream_failed.set() - - stream_watch = disconnected.watch_idb_events( - lambda _batch: None, on_error=failed, debounce=0 - ) - disconnected._handle.subscription.emit( - module.DatabaseDisconnectedError("GUI database closed") - ) - check( - "stream disconnects become application connection errors", - stream_failed.wait(1) - and isinstance(stream_errors[0], IDAConnectionError), - stream_errors, - ) - stream_watch.close() - disconnected.close() - finally: - module.DatabaseHandle = original - module.DatabaseOpenOptions = original_options - module.DatabaseBusyError = original_busy - module.DatabaseDisconnectedError = original_disconnected - - client = CodeModeClient(__file__) - - def unknown_operation(): - pass - - try: - client.call(unknown_operation) - except IDAToolError as exc: - check( - "unknown adapter operations are explicit", - exc.tool == "unknown_operation", - ) - else: - check("unknown adapter operations are explicit", False) - - print(f"\n{PASS} passed, {FAIL} failed") - return 1 if FAIL else 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/tests/test_kittygfx.py b/tests/test_kittygfx.py index 74b6bb7..cfc6512 100644 --- a/tests/test_kittygfx.py +++ b/tests/test_kittygfx.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """Cross-platform checks for the optional kitty-graphics startup splash. -Pure: stdlib only, no terminal, Textual, Code Mode, or IDA. +Pure: stdlib only, no terminal, Textual, IDA Nexus, or IDA. """ from __future__ import annotations diff --git a/tests/test_launch.py b/tests/test_launch.py index d57ee5f..35a0b8d 100644 --- a/tests/test_launch.py +++ b/tests/test_launch.py @@ -3,12 +3,12 @@ The old `_sweep_locks` deleted `.id0/.id1/.id2/.nam/.til` next to the user's binary when a database failed to open. That was only defensible while the TUI -exclusively owned a private worker; under Code Mode a GUI or another client may +exclusively owned a private worker; under IDA Nexus a GUI or another client may own the database, so the sweep is gone. Its tests are replaced by one that keeps it gone -- deleting a shared database's working files is unrecoverable, and this is the cheapest guard against someone reintroducing the "helpful" cleanup. -Pure: no IDA, no Code Mode library, no Textual. +Pure: no IDA, no IDA Nexus library, no Textual. """ from __future__ import annotations @@ -47,7 +47,7 @@ def touch(*paths): def t_no_lock_sweeping(): """The launcher must not delete database working files any more. - Code Mode's registry locks, health probes and IDA itself arbitrate database + IDA Nexus's registry locks, health probes and IDA itself arbitrate database ownership now. A sweep here would delete files out from under a live GUI. """ check("_sweep_locks is gone", not hasattr(launch, "_sweep_locks")) diff --git a/tests/test_nexus_client.py b/tests/test_nexus_client.py new file mode 100644 index 0000000..4ebf753 --- /dev/null +++ b/tests/test_nexus_client.py @@ -0,0 +1,440 @@ +"""IDA-free contract tests for the IDA Nexus client adapter.""" + +from __future__ import annotations + +import os +import queue +import sys +import threading +import time +import tempfile +from dataclasses import dataclass + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +from idatui.errors import IDAConnectionError, IDAToolError # noqa: E402 +from idatui import remote_ops # noqa: E402 +import idatui.nexus_client as module # noqa: E402 +from idatui.nexus_client import NexusClient, _parse_load_args # noqa: E402 + +#: Pure: fakes the DatabaseHandle, never touches IDA or the IDA Nexus library. +NEEDS_IDA = False + +PASS = FAIL = 0 + + +def check(name: str, condition: bool, detail="") -> None: + global PASS, FAIL + if condition: + PASS += 1 + print(f" ok {name}") + else: + FAIL += 1 + print(f" FAIL {name} {detail}") + + +@dataclass(frozen=True) +class FakeEntry: + pid: int = 123 + backend: str = "gui" + record_id: str = "123-abcdef" + exe_path: str = "" + idb_path: str = "" + managed: bool = False + + +_CLOSED = object() + + +class FakeSubscription: + def __init__(self) -> None: + self._queue: queue.Queue = queue.Queue() + self.closed = False + + def __iter__(self): + return self + + def __next__(self): + item = self._queue.get() + if isinstance(item, BaseException): + raise item + if item is _CLOSED: + raise StopIteration + return item + + def emit(self, event: dict) -> None: + self._queue.put(event) + + def close(self) -> None: + if not self.closed: + self.closed = True + self._queue.put(_CLOSED) + + +class FakeHandle: + def __init__(self, path: str) -> None: + self.connected = True + self.instance = FakeEntry(exe_path=path, idb_path=path + ".i64") + self.waited = None + self.saved = 0 + self.closed = False + self.code = "" + self.codes = [] + self.code_timeout = None + self.operation_label = None + self.event_origin_id = "fake-handle-origin" + self.owns_checks = 0 + self.subscription = FakeSubscription() + self.shutdown_calls = [] + self.shutdown_error = None + + def wait_autoanalysis(self, timeout=None): + self.waited = timeout + return {"complete": True, "status": "complete"} + + def execute_python( + self, + code, + timeout=None, + *, + operation_id=None, + operation_label=None, + persist_globals=False, + filename=None, + ): + self.code = code + self.codes.append(code) + self.code_timeout = timeout + self.operation_label = operation_label + result = ( + { + "__remote_ida_status__": "ok", + "__remote_ida_value__": {"sentinel": 7}, + } + if ".modules.get(" in code + else True + ) + return {"result": result, "stdout": "", "stderr": ""} + + def subscribe_idb_events(self): + return self.subscription + + def owns_event(self, event): + self.owns_checks += 1 + return event.get("origin_id") == self.event_origin_id + + def save_database(self): + self.saved += 1 + return {"saved": True, "idb_path": self.instance.idb_path} + + def shutdown_database(self, *, save=True): + self.shutdown_calls.append(save) + if self.shutdown_error is not None: + raise module.RemoteError(self.shutdown_error, self.shutdown_error, 409) + return {"shutting_down": True, "save": save} + + def close(self): + self.subscription.close() + self.connected = False + self.closed = True + + +class FakeDatabaseHandle: + opened = None + opens = 0 + kwargs = None + + @classmethod + def open(cls, path, **kwargs): + cls.opens += 1 + cls.opened = path + cls.kwargs = kwargs + return FakeHandle(path) + + +@dataclass(frozen=True) +class FakeOpenOptions: + """Stand-in for DatabaseOpenOptions when the library is not installed. + + Deliberately STRICT (no **kwargs): an option the adapter invents would + raise here, and `_option_fields_are_real` checks the surviving names + against the real dataclass wherever it is importable. + """ + + spawn: bool = True + startup_timeout: float = 120.0 + output_database: str | None = None + processor: str | None = None + image_base: int | None = None + file_type: str | None = None + new_database: bool = False + + +class FakeBusy(Exception): + """Stand-in for DatabaseBusyError: `except None` is a TypeError.""" + + +class FakeDisconnected(Exception): + """Stand-in for DatabaseDisconnectedError in stdlib-only runs.""" + + +def _open_kwargs_are_real(sent: dict): + """(ok, detail) for the kwargs the adapter passes to DatabaseHandle.open. + + Skips (passes) when ida_nexus is not installed, so the file stays pure. + """ + try: + import inspect + from ida_nexus import DatabaseHandle as Real + except ImportError: + return True, "ida_nexus not installed - signature not checked" + accepted = set(inspect.signature(Real.open).parameters) + unknown = sorted(set(sent) - accepted) + return not unknown, f"open() rejects {unknown}" + + +def _option_fields_are_real(options): + """(ok, detail) for the option names the adapter fills in. + + The open() signature no longer names the loader options -- they moved + inside DatabaseOpenOptions -- so the `loading_address` class of bug now + hides there instead. Check it in the same way. + """ + try: + import dataclasses + from ida_nexus import DatabaseOpenOptions as Real + except ImportError: + return True, "ida_nexus not installed - fields not checked" + accepted = {field.name for field in dataclasses.fields(Real)} + unknown = sorted({f.name for f in dataclasses.fields(options)} - accepted) + return not unknown, f"DatabaseOpenOptions rejects {unknown}" + + +def main() -> int: + proc, base, file_type = _parse_load_args("-parm:ARMv7-M -b800000 -TRaw") + check( + "legacy switches map to typed IDA Nexus options", + (proc, base, file_type) == ("arm:ARMv7-M", 0x8000000, "Raw"), + (proc, base, file_type), + ) + try: + _parse_load_args("-parm -zcustom") + except ValueError as exc: + check("arbitrary IDA switches fail loudly", "cannot represent" in str(exc), exc) + else: + check("arbitrary IDA switches fail loudly", False) + + original = module.DatabaseHandle + module.DatabaseHandle = FakeDatabaseHandle + # The library's own names when it is installed; strict fakes when it is not + # (this file must keep running under a stdlib-only python3). + original_options = module.DatabaseOpenOptions + original_busy = module.DatabaseBusyError + original_disconnected = module.DatabaseDisconnectedError + module.DatabaseOpenOptions = original_options or FakeOpenOptions + module.DatabaseBusyError = original_busy or FakeBusy + module.DatabaseDisconnectedError = original_disconnected or FakeDisconnected + try: + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, "sample.bin") + with open(path, "wb") as file: + file.write(b"sample") + client = NexusClient(path, load_args="-parm:ARMv7-A -b100") + notes = [] + client.connect(timeout=42, progress=notes.append) + handle = client._handle + check( + "connect delegates database discovery to DatabaseHandle.open", + FakeDatabaseHandle.opened == path and handle is not None, + ) + options = FakeDatabaseHandle.kwargs["options"] + check( + "typed loader options cross the dependency boundary", + options.processor == "arm:ARMv7-A" and options.image_base == 0x1000, + options, + ) + check( + "every open option exists in the real library", + *_option_fields_are_real(options), + ) + # A fake that swallows **kwargs cannot catch a keyword the real + # library does not have -- which is exactly how this port shipped + # `loading_address` (the real name is `image_base`) and would have + # raised TypeError on the very first connect. Check the names we + # send against the real signature whenever it is importable. + check( + "every open() keyword exists in the real library", + *_open_kwargs_are_real(FakeDatabaseHandle.kwargs), + ) + check( + "connect waits for IDA Nexus autoanalysis", + handle.waited == 42, + getattr(handle, "waited", None), + ) + check( + "progress distinguishes discovery and backend attachment", + len(notes) == 2 and "gui" in notes[-1], + notes, + ) + result = client.call( + remote_ops.list_funcs, queries=[{"offset": 0, "count": 2}] + ) + check( + "remote operation returns its JSON result", + result == {"sentinel": 7}, + result, + ) + check( + "operation source is real Python installed through ida-domain", + any("db.functions.get_all()" in code for code in handle.codes), + handle.codes[0][:200], + ) + check( + "remote operations attribute IDB events to IDA TUI", + handle.operation_label == "IDA TUI", + handle.operation_label, + ) + batches = [] + delivered = threading.Event() + + def changed(batch): + batches.append(batch) + delivered.set() + + watcher = client.watch_idb_events(changed, debounce=0.05) + handle.subscription.emit({"event_name": "renamed", "origin_id": "peer-1"}) + handle.subscription.emit( + {"event_name": "cmt_changed", "origin_id": "peer-2"} + ) + check( + "event bursts produce one debounced refresh", + delivered.wait(1) and len(batches) == 1 and len(batches[0]) == 2, + batches, + ) + delivered.clear() + handle.subscription.emit( + {"event_name": "renamed", "origin_id": handle.event_origin_id} + ) + time.sleep(0.1) + check( + "the listener uses handle ownership to ignore its own events", + not delivered.is_set() + and len(batches) == 1 + and handle.owns_checks >= 3, + (batches, handle.owns_checks), + ) + handle.subscription.emit( + {"event_name": "byte_patched", "origin_id": "peer-3"} + ) + watcher.close() + time.sleep(0.1) + check( + "closing drops a pending debounced refresh", + not delivered.is_set() and len(batches) == 1, + batches, + ) + check( + "health exposes registry identity", + client.health()["record_id"] == "123-abcdef", + ) + client.save_database() + check("save uses the public IDA Nexus save route", handle.saved == 1) + check( + "GUI leases transfer rather than claiming discard", + client.discard_database() is False and handle.shutdown_calls == [], + handle.shutdown_calls, + ) + handle.instance = FakeEntry( + backend="idalib", managed=True, exe_path=path, idb_path=path + ".i64" + ) + check( + "a final managed lease discards without saving", + client.discard_database() is True and handle.shutdown_calls == [False], + handle.shutdown_calls, + ) + handle.shutdown_error = "instance_shared" + check( + "a shared managed lease transfers finalization", + client.discard_database() is False, + handle.shutdown_calls, + ) + handle.shutdown_error = "instance_busy" + try: + client.discard_database(timeout=0) + except IDAToolError as exc: + check( + "a busy final lease never silently saves", + exc.tool == "shutdown_database", + exc, + ) + else: + check("a busy final lease never silently saves", False) + handle.shutdown_error = None + handle.instance = FakeEntry(exe_path=path, idb_path=path + ".i64") + opens = FakeDatabaseHandle.opens + handle.connected = False + try: + client.health() + except IDAConnectionError as exc: + check( + "a disconnected handle requires explicit rediscovery", + "explicit rediscovery" in str(exc) + and FakeDatabaseHandle.opens == opens, + (exc, FakeDatabaseHandle.opens, opens), + ) + else: + check("a disconnected handle requires explicit rediscovery", False) + client.close() + check("close releases only the handle lease", handle.closed) + check( + "GUI lifetime is never claimed by the client", + client.wait_released(0) is False, + ) + disconnected = NexusClient(path).connect() + stream_errors = [] + stream_failed = threading.Event() + + def failed(error): + stream_errors.append(error) + stream_failed.set() + + stream_watch = disconnected.watch_idb_events( + lambda _batch: None, on_error=failed, debounce=0 + ) + disconnected._handle.subscription.emit( + module.DatabaseDisconnectedError("GUI database closed") + ) + check( + "stream disconnects become application connection errors", + stream_failed.wait(1) + and isinstance(stream_errors[0], IDAConnectionError), + stream_errors, + ) + stream_watch.close() + disconnected.close() + finally: + module.DatabaseHandle = original + module.DatabaseOpenOptions = original_options + module.DatabaseBusyError = original_busy + module.DatabaseDisconnectedError = original_disconnected + + client = NexusClient(__file__) + + def unknown_operation(): + pass + + try: + client.call(unknown_operation) + except IDAToolError as exc: + check( + "unknown adapter operations are explicit", + exc.tool == "unknown_operation", + ) + else: + check("unknown adapter operations are explicit", False) + + print(f"\n{PASS} passed, {FAIL} failed") + return 1 if FAIL else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_pool.py b/tests/test_pool.py index 6309e13..37058bf 100644 --- a/tests/test_pool.py +++ b/tests/test_pool.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Unit tests for idatui.pool (Code Mode lease residency and LRU budget). +"""Unit tests for idatui.pool (IDA Nexus lease residency and LRU budget). A fake client keeps the policy testable without IDA or Textual. @@ -31,7 +31,7 @@ def check(name, cond, detail=""): class FakeClient: - """Stands in for a CodeModeClient lease and records saves/closes.""" + """Stands in for a NexusClient lease and records saves/closes.""" def __init__(self, ref, mem=100, backend="idalib", discardable=True): diff --git a/tests/test_project.py b/tests/test_project.py index 690f360..bae0802 100644 --- a/tests/test_project.py +++ b/tests/test_project.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """Unit tests for idatui.project (the multi-binary project model + staging). -IDA-free: exercises staging plus Code Mode ownership checks without opening a database. +IDA-free: exercises staging plus IDA Nexus ownership checks without opening a database. python tests/test_project.py """ diff --git a/tests/test_scenarios.py b/tests/test_scenarios.py index 45ed51b..190777c 100644 --- a/tests/test_scenarios.py +++ b/tests/test_scenarios.py @@ -5272,7 +5272,7 @@ async def run(binary, only=None): async def _run_on(binary, only=None): - # Code Mode attaches a registered GUI or starts/reuses a managed worker. + # IDA Nexus attaches a registered GUI or starts/reuses a managed worker. app = IdaTui(open_path=binary, keepalive=False) async with app.run_test(size=(140, 44)) as pilot: c = Ctx(app, pilot) diff --git a/tests/test_thumb_ui.py b/tests/test_thumb_ui.py index f12fbc4..1b4c438 100644 --- a/tests/test_thumb_ui.py +++ b/tests/test_thumb_ui.py @@ -50,7 +50,7 @@ def check(name, ok, detail=""): #: #: This suite used to delete .i64 and reopen the SAME path for each phase. #: That was safe when the TUI owned a private worker that died with it; under -#: Code Mode the database is leased and the previous phase's worker can still +#: IDA Nexus the database is leased and the previous phase's worker can still #: hold it through its lease grace, so the delete raced a live owner and the #: next open never produced a listing (the crash this fixed). Separate paths #: cannot collide, and nothing has to wait for anyone else to let go. diff --git a/tests/test_trace_ui.py b/tests/test_trace_ui.py index 855ce4d..586ce2e 100644 --- a/tests/test_trace_ui.py +++ b/tests/test_trace_ui.py @@ -115,7 +115,7 @@ async def run() -> int: # `app._t` is assigned the moment the key is handled, so it is NOT a # signal that the VIEW has followed -- the navigation it kicks off # runs in a worker. Waiting on it and then reading the cursor was a - # race that the (slower) Code Mode backend loses. Gate on the thing + # race that the (slower) IDA Nexus backend loses. Gate on the thing # the check is about. await settle(app, lambda: app._t == 1 and lst._cursor_ea() == t.ip(1), timeout=20) diff --git a/uv.lock b/uv.lock index 7cbfa25..42e544a 100644 --- a/uv.lock +++ b/uv.lock @@ -12,31 +12,31 @@ wheels = [ ] [[package]] -name = "ida-codemode" -version = "0.6.1" +name = "ida-domain" +version = "0.5.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "ida-domain" }, + { name = "idapro" }, { name = "packaging" }, - { name = "zeromcp" }, + { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/5d/55/b9b72626e371bb36b712659b567ad1d890947c237bfbcaeccfb30f12fe80/ida_codemode-0.6.1.tar.gz", hash = "sha256:44df2a986d24a7e35e64b92c910a21eb485dc4c47d1e9db1e572e8fc1aa08a44", size = 188081, upload-time = "2026-08-13T16:46:35.504Z" } +sdist = { url = "https://files.pythonhosted.org/packages/80/34/be087d3ea1c3a6573e0660cb5b40f0c4ade9ae5772cf1c5d98d52472d28b/ida_domain-0.5.1.tar.gz", hash = "sha256:c49f2c417047d882e954f651b50a709a3f27903b33ba533b794aa54d6536d16f", size = 396413, upload-time = "2026-08-10T13:32:48.45Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/05/20/2397e9b34cefe7ce01945cc2299a01459b9371c48027f61bfdbd4bfcf677/ida_codemode-0.6.1-py3-none-any.whl", hash = "sha256:69a39e25f7441aab794f6737133f8c6155fd794c77924e46e83cb938acf8944f", size = 104728, upload-time = "2026-08-13T16:46:34.265Z" }, + { url = "https://files.pythonhosted.org/packages/27/78/9c698d818b0fddc6648f703a0821edeeb65b18404b43f556d249c5446c96/ida_domain-0.5.1-py3-none-any.whl", hash = "sha256:bfbb17c7d0cb2ed7d3f21342e1c8787f9018d5c2a94cdfd29e06537dd026a06d", size = 201275, upload-time = "2026-08-10T13:32:46.955Z" }, ] [[package]] -name = "ida-domain" -version = "0.5.1" +name = "ida-nexus" +version = "0.7.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "idapro" }, + { name = "ida-domain" }, { name = "packaging" }, - { name = "typing-extensions" }, + { name = "zeromcp" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/80/34/be087d3ea1c3a6573e0660cb5b40f0c4ade9ae5772cf1c5d98d52472d28b/ida_domain-0.5.1.tar.gz", hash = "sha256:c49f2c417047d882e954f651b50a709a3f27903b33ba533b794aa54d6536d16f", size = 396413, upload-time = "2026-08-10T13:32:48.45Z" } +sdist = { url = "https://files.pythonhosted.org/packages/b6/93/2f87cbd64ffc45e181542f133ba8db101c9049155b5f015d9d318f32dcfb/ida_nexus-0.7.0.tar.gz", hash = "sha256:838698c6a2456d474da833b2f4955da9f1fca4a488c95a157539a3c663a919ba", size = 220190, upload-time = "2026-08-20T21:52:18.228Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/27/78/9c698d818b0fddc6648f703a0821edeeb65b18404b43f556d249c5446c96/ida_domain-0.5.1-py3-none-any.whl", hash = "sha256:bfbb17c7d0cb2ed7d3f21342e1c8787f9018d5c2a94cdfd29e06537dd026a06d", size = 201275, upload-time = "2026-08-10T13:32:46.955Z" }, + { url = "https://files.pythonhosted.org/packages/a5/c2/58704fc74618c7867cf7542d3150a75a7678aae2fff7960fa8e5cc67d934/ida_nexus-0.7.0-py3-none-any.whl", hash = "sha256:a5006c7170a0a758a598b864d6fa248bf4eccbea848a30fc6a3eeeadf638d07a", size = 127656, upload-time = "2026-08-20T21:52:19.395Z" }, ] [[package]] @@ -53,7 +53,7 @@ name = "idatui" version = "0.0.1" source = { editable = "." } dependencies = [ - { name = "ida-codemode" }, + { name = "ida-nexus" }, { name = "pygments" }, { name = "textual" }, ] @@ -65,7 +65,7 @@ dev = [ [package.metadata] requires-dist = [ - { name = "ida-codemode", specifier = ">=0.5.3" }, + { name = "ida-nexus", specifier = ">=0.7.0" }, { name = "pygments", specifier = ">=2" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=8" }, { name = "textual", specifier = ">=8" }, -- cgit v1.3.1-sl0p