aboutsummaryrefslogtreecommitdiffstats
path: root/experiments/bench_ops.py
diff options
context:
space:
mode:
authorblasty <blasty@local>2026-08-09 22:50:31 +0200
committerblasty <blasty@local>2026-08-09 22:50:31 +0200
commit4d490052f7d4dc0fccb3e718c801ef99e8646e22 (patch)
tree5b4cb17c9ac59978f14fb7351b229a19fb60d0b4 /experiments/bench_ops.py
parentDocs: re-check ALL nine upstream findings against 0.3.2 (diff)
downloadida-tui-4d490052f7d4dc0fccb3e718c801ef99e8646e22.tar.gz
ida-tui-4d490052f7d4dc0fccb3e718c801ef99e8646e22.tar.xz
ida-tui-4d490052f7d4dc0fccb3e718c801ef99e8646e22.zip
SPEED: replace the historical backend table with a real 0.3.1 vs 0.3.2 A/B
The worker-vs-Code-Mode table was measured before 0.3.2 and with both workarounds active, so it answered a question nobody asks any more. Replaced with three configurations measured on the same box, rolling both checkouts back and forward: A old client WITH workarounds on 0.3.1 -- what shipped B current client on 0.3.1 -- what the workarounds were for C current client on 0.3.2 -- now Headline: the real-world gain is ~1.4x geomean, NOT the 6.9x the empty round trip advertises, and the doc says so in those words -- because the tempting number to quote is the wrong one. The A->C vs B->C gap is the actual story: stock 0.3.1 was 5.4x slower, so the workarounds had already recovered nearly everything and upstream mostly bought us the right to delete them. Also records the three cost classes (payload- / round-trip- / IDA-dominated) so the next person optimising here knows which lever moves which op, and the ~10% run-to-run spread so a sub-1.2x 'regression' doesn't start a hunt. Old worker table kept below, labelled historical. experiments/bench_ops.py is the harness, with the copy-to-/tmp-before-checkout trick documented in it.
Diffstat (limited to 'experiments/bench_ops.py')
-rw-r--r--experiments/bench_ops.py103
1 files changed, 103 insertions, 0 deletions
diff --git a/experiments/bench_ops.py b/experiments/bench_ops.py
new file mode 100644
index 0000000..6ed64e0
--- /dev/null
+++ b/experiments/bench_ops.py
@@ -0,0 +1,103 @@
+"""Time a realistic idatui operation mix against whatever ida-codemode is installed.
+
+The companion to `bench_pack_trace.py`: that one isolates a single workaround,
+this one answers "how much faster is the whole client, on real operations".
+
+**It deliberately does not import anything version-specific**, so the SAME file
+can measure an OLD idatui checkout (with its `sys.settrace` strip and packing
+workarounds) and the current one. To compare across versions, copy it somewhere
+outside the repo first -- `git checkout` of an older commit would otherwise
+replace or delete it::
+
+ cp experiments/bench_ops.py /tmp/
+ # C: current client, current library
+ PYTHONPATH=. ~/ida-venv/bin/python /tmp/bench_ops.py
+
+ # B: current client against the OLD library (shows what the workarounds were for)
+ git -C ~/dev/ida-codemode checkout 4195f21
+ PYTHONPATH=. ~/ida-venv/bin/python /tmp/bench_ops.py
+
+ # A: the client as it SHIPPED on the old library, workarounds and all
+ git checkout d74b6f5 # the commit before the workaround removal
+ PYTHONPATH=. ~/ida-venv/bin/python /tmp/bench_ops.py
+
+ git checkout master && git -C ~/dev/ida-codemode checkout main # ALWAYS restore
+
+ida-codemode is installed **editable** into both venvs, so checking that repo out
+swaps the backend under the TUI with no reinstall -- which is what makes this A/B
+cheap. Results for 0.3.1 vs 0.3.2 are in `.fastfeedback/SPEED.md`.
+"""
+from __future__ import annotations
+
+import argparse
+import os
+import statistics
+import time
+
+from idatui.codemode_client import CodeModeClient
+
+
+def bench(fn, reps: int) -> tuple[float, float]:
+ """Best-of and median wall time in ms; best-of resists co-tenant noise."""
+ samples = []
+ for _ in range(reps):
+ started = time.perf_counter()
+ fn()
+ samples.append((time.perf_counter() - started) * 1000.0)
+ return min(samples), statistics.median(samples)
+
+
+def main() -> int:
+ ap = argparse.ArgumentParser()
+ ap.add_argument("target", nargs="?", default="targets/bash")
+ ap.add_argument("--reps", type=int, default=20)
+ args = ap.parse_args()
+
+ client = CodeModeClient(os.path.abspath(args.target))
+ client.connect()
+ handle = client._handle
+
+ # Work on the biggest function we can find, so the payload-heavy operations
+ # are actually payload-heavy.
+ index = client.invoke("list_funcs", queries=[{"offset": 0, "count": 60}])
+ funcs = (index.get("result") or [{}])[0].get("data") or []
+ if not funcs:
+ print("VERDICT: FAIL - no functions")
+ return 1
+ big = max(funcs, key=lambda f: f.get("size") or 0)
+ ea = big["addr"] if isinstance(big["addr"], str) else hex(big["addr"])
+
+ ops = [
+ # Synthetic: isolates the per-operation floor (execute_sync marshalling).
+ ("empty round trip", lambda: handle.execute_python("result = 1")),
+ # Payload-dominated: what _PACK_EPILOGUE was written for.
+ ("list_funcs 500", lambda: client.invoke(
+ "list_funcs", queries=[{"offset": 0, "count": 500}])),
+ ("heads 200 (listing page)", lambda: client.invoke(
+ "heads", addr=ea, count=200, annotate=True)),
+ # IDA-work-dominated: Hex-Rays, nothing upstream can move.
+ ("decompile (warm)", lambda: client.invoke("decompile", addr=ea)),
+ ("flowchart (graph)", lambda: client.invoke("flowchart", addr=ea)),
+ # Round-trip-dominated: small payload, so only the floor matters.
+ ("xrefs_to", lambda: client.invoke("xref_query", direction="to", addr=ea)),
+ ]
+
+ print(f"# target={os.path.basename(args.target)} func={ea} reps={args.reps} "
+ f"backend={client.backend}")
+ results = {}
+ for name, fn in ops:
+ try:
+ for _ in range(3): # warm caches; the first sample is always an outlier
+ fn()
+ best, med = bench(fn, args.reps)
+ results[name] = med
+ print(f"{name:28} best {best:8.3f}ms median {med:8.3f}ms")
+ except Exception as exc: # one broken op must not lose the other five
+ print(f"{name:28} FAILED: {type(exc).__name__}: {str(exc)[:60]}")
+ client.close()
+ print("RESULT " + ";".join(f"{k}={v:.3f}" for k, v in results.items()))
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())