diff options
| author | blasty <blasty@local> | 2026-08-09 22:50:31 +0200 |
|---|---|---|
| committer | blasty <blasty@local> | 2026-08-09 22:50:31 +0200 |
| commit | 4d490052f7d4dc0fccb3e718c801ef99e8646e22 (patch) | |
| tree | 5b4cb17c9ac59978f14fb7351b229a19fb60d0b4 /experiments/bench_ops.py | |
| parent | Docs: re-check ALL nine upstream findings against 0.3.2 (diff) | |
| download | ida-tui-4d490052f7d4dc0fccb3e718c801ef99e8646e22.tar.gz ida-tui-4d490052f7d4dc0fccb3e718c801ef99e8646e22.tar.xz ida-tui-4d490052f7d4dc0fccb3e718c801ef99e8646e22.zip | |
SPEED: replace the historical backend table with a real 0.3.1 vs 0.3.2 A/B
The worker-vs-Code-Mode table was measured before 0.3.2 and with both
workarounds active, so it answered a question nobody asks any more. Replaced
with three configurations measured on the same box, rolling both checkouts
back and forward:
A old client WITH workarounds on 0.3.1 -- what shipped
B current client on 0.3.1 -- what the workarounds were for
C current client on 0.3.2 -- now
Headline: the real-world gain is ~1.4x geomean, NOT the 6.9x the empty round
trip advertises, and the doc says so in those words -- because the tempting
number to quote is the wrong one. The A->C vs B->C gap is the actual story:
stock 0.3.1 was 5.4x slower, so the workarounds had already recovered nearly
everything and upstream mostly bought us the right to delete them.
Also records the three cost classes (payload- / round-trip- / IDA-dominated)
so the next person optimising here knows which lever moves which op, and the
~10% run-to-run spread so a sub-1.2x 'regression' doesn't start a hunt.
Old worker table kept below, labelled historical. experiments/bench_ops.py is
the harness, with the copy-to-/tmp-before-checkout trick documented in it.
Diffstat (limited to 'experiments/bench_ops.py')
| -rw-r--r-- | experiments/bench_ops.py | 103 |
1 files changed, 103 insertions, 0 deletions
diff --git a/experiments/bench_ops.py b/experiments/bench_ops.py new file mode 100644 index 0000000..6ed64e0 --- /dev/null +++ b/experiments/bench_ops.py @@ -0,0 +1,103 @@ +"""Time a realistic idatui operation mix against whatever ida-codemode is installed. + +The companion to `bench_pack_trace.py`: that one isolates a single workaround, +this one answers "how much faster is the whole client, on real operations". + +**It deliberately does not import anything version-specific**, so the SAME file +can measure an OLD idatui checkout (with its `sys.settrace` strip and packing +workarounds) and the current one. To compare across versions, copy it somewhere +outside the repo first -- `git checkout` of an older commit would otherwise +replace or delete it:: + + cp experiments/bench_ops.py /tmp/ + # C: current client, current library + PYTHONPATH=. ~/ida-venv/bin/python /tmp/bench_ops.py + + # B: current client against the OLD library (shows what the workarounds were for) + git -C ~/dev/ida-codemode checkout 4195f21 + PYTHONPATH=. ~/ida-venv/bin/python /tmp/bench_ops.py + + # A: the client as it SHIPPED on the old library, workarounds and all + git checkout d74b6f5 # the commit before the workaround removal + PYTHONPATH=. ~/ida-venv/bin/python /tmp/bench_ops.py + + git checkout master && git -C ~/dev/ida-codemode checkout main # ALWAYS restore + +ida-codemode is installed **editable** into both venvs, so checking that repo out +swaps the backend under the TUI with no reinstall -- which is what makes this A/B +cheap. Results for 0.3.1 vs 0.3.2 are in `.fastfeedback/SPEED.md`. +""" +from __future__ import annotations + +import argparse +import os +import statistics +import time + +from idatui.codemode_client import CodeModeClient + + +def bench(fn, reps: int) -> tuple[float, float]: + """Best-of and median wall time in ms; best-of resists co-tenant noise.""" + samples = [] + for _ in range(reps): + started = time.perf_counter() + fn() + samples.append((time.perf_counter() - started) * 1000.0) + return min(samples), statistics.median(samples) + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("target", nargs="?", default="targets/bash") + ap.add_argument("--reps", type=int, default=20) + args = ap.parse_args() + + client = CodeModeClient(os.path.abspath(args.target)) + client.connect() + handle = client._handle + + # Work on the biggest function we can find, so the payload-heavy operations + # are actually payload-heavy. + index = client.invoke("list_funcs", queries=[{"offset": 0, "count": 60}]) + funcs = (index.get("result") or [{}])[0].get("data") or [] + if not funcs: + print("VERDICT: FAIL - no functions") + return 1 + big = max(funcs, key=lambda f: f.get("size") or 0) + ea = big["addr"] if isinstance(big["addr"], str) else hex(big["addr"]) + + ops = [ + # Synthetic: isolates the per-operation floor (execute_sync marshalling). + ("empty round trip", lambda: handle.execute_python("result = 1")), + # Payload-dominated: what _PACK_EPILOGUE was written for. + ("list_funcs 500", lambda: client.invoke( + "list_funcs", queries=[{"offset": 0, "count": 500}])), + ("heads 200 (listing page)", lambda: client.invoke( + "heads", addr=ea, count=200, annotate=True)), + # IDA-work-dominated: Hex-Rays, nothing upstream can move. + ("decompile (warm)", lambda: client.invoke("decompile", addr=ea)), + ("flowchart (graph)", lambda: client.invoke("flowchart", addr=ea)), + # Round-trip-dominated: small payload, so only the floor matters. + ("xrefs_to", lambda: client.invoke("xref_query", direction="to", addr=ea)), + ] + + print(f"# target={os.path.basename(args.target)} func={ea} reps={args.reps} " + f"backend={client.backend}") + results = {} + for name, fn in ops: + try: + for _ in range(3): # warm caches; the first sample is always an outlier + fn() + best, med = bench(fn, args.reps) + results[name] = med + print(f"{name:28} best {best:8.3f}ms median {med:8.3f}ms") + except Exception as exc: # one broken op must not lose the other five + print(f"{name:28} FAILED: {type(exc).__name__}: {str(exc)[:60]}") + client.close() + print("RESULT " + ";".join(f"{k}={v:.3f}" for k, v in results.items())) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) |
