15 files changed, 306 insertions, 0 deletions
diff --git a/tests/run.py b/tests/run.py
new file mode 100755
index 0000000..ca0a319
--- /dev/null
+++ b/tests/run.py
@@ -0,0 +1,250 @@
+#!/usr/bin/env python3
+"""The front door for the test suite.
+
+There are two kinds of test file here and the difference matters a lot:
+
+  * **pure** — stdlib (sometimes + textual), no IDA, no worker, no binary.
+    Runs anywhere in about a second. ``tests/run.py --fast`` is exactly this
+    set, and it's what you run between edits.
+  * **IDA** — spawns a real idalib worker on a real target and drives the
+    Textual pilot against it. Minutes, needs a licensed IDA, and gets CPU
+    starved on a loaded box (a SIGKILLed worker looks like a hang, not a bug —
+    check ``uptime`` before believing a regression).
+
+Rather than keep that classification in a table here, where it would rot the
+first time someone adds a test, **each test file declares it**::
+
+    NEEDS_IDA = True   # or False
+
+``run.py`` reads that marker with :mod:`ast` (it never imports the file — these
+modules run their suite at import time). A test file with no marker is a hard
+error, so a new test can't quietly join the fast set and start needing IDA.
+
+Usage::
+
+    python3 tests/run.py              # everything
+    python3 tests/run.py --fast       # only the no-IDA files (seconds)
+    python3 tests/run.py --list       # what would run, and why
+    python3 tests/run.py trace graph  # only files matching these substrings
+    python3 tests/run.py --fast -x    # stop at the first failing file
+
+Exit code is 0 only if every file selected ran and passed.
+"""
+from __future__ import annotations
+
+import argparse
+import ast
+import os
+import re
+import subprocess
+import sys
+import time
+
+ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+TESTS = os.path.join(ROOT, "tests")
+
+#: The IDA-capable interpreter. The pilot tests need textual AND idapro in one
+#: python; the worker python is auto-detected separately by WorkerClient.
+DEFAULT_PY = os.path.expanduser("~/ida-venv/bin/python")
+
+#: Both shapes the suites print: "N passed, M failed" and "N checks, M failed".
+_TALLY = re.compile(r"^\s*(\d+)\s+(?:passed|checks),\s*(\d+)\s+failed\s*$", re.M)
+_SKIP = re.compile(r"^\s*skip(?:ped|ping)?\b[:.]", re.M | re.I)
+
+
+class Marker(Exception):
+    """A test file didn't declare NEEDS_IDA."""
+
+
+def needs_ida(path: str) -> bool:
+    """Read the module-level ``NEEDS_IDA`` without importing the module.
+
+    Importing is not an option: every one of these files runs its whole suite
+    under ``if __name__ == "__main__"`` but also defines its checks at module
+    scope, and some spawn workers on import of a helper. AST it is.
+    """
+    with open(path, encoding="utf-8") as fh:
+        tree = ast.parse(fh.read(), filename=path)
+    for node in tree.body:
+        if not isinstance(node, ast.Assign):
+            continue
+        for target in node.targets:
+            if isinstance(target, ast.Name) and target.id == "NEEDS_IDA":
+                value = ast.literal_eval(node.value)
+                if not isinstance(value, bool):
+                    raise Marker(f"{os.path.basename(path)}: "
+                                 f"NEEDS_IDA must be a bool, got {value!r}")
+                return value
+    raise Marker(
+        f"{os.path.basename(path)}: no NEEDS_IDA marker.\n"
+        f"  Add `NEEDS_IDA = True` (spawns a worker / drives the pilot) or\n"
+        f"  `NEEDS_IDA = False` (pure: stdlib, no IDA, runs anywhere) at module\n"
+        f"  scope, so tests/run.py --fast knows whether it can run you.")
+
+
+def discover() -> list[tuple[str, bool]]:
+    """Every ``tests/test_*.py``, with its IDA requirement. Sorted: fast first,
+    so ``run.py`` fails on a cheap file before burning minutes on a pilot."""
+    out = []
+    problems = []
+    for name in sorted(os.listdir(TESTS)):
+        if not (name.startswith("test_") and name.endswith(".py")):
+            continue
+        path = os.path.join(TESTS, name)
+        try:
+            out.append((path, needs_ida(path)))
+        except Marker as exc:
+            problems.append(str(exc))
+    if problems:
+        raise Marker("\n".join(problems))
+    out.sort(key=lambda p: (p[1], p[0]))
+    return out
+
+
+def tally(output: str) -> tuple[int, int] | None:
+    """The last ``N passed, M failed`` line a suite printed.
+
+    The last one, not the first: test_graph prints a per-section tally on the
+    way through and the total at the end.
+    """
+    found = _TALLY.findall(output)
+    if not found:
+        return None
+    passed, failed = found[-1]
+    return int(passed), int(failed)
+
+
+def run_one(path: str, python: str, extra: list[str], echo: bool) -> dict:
+    """Run one test file as a subprocess and summarise it."""
+    name = os.path.basename(path)[len("test_"):-len(".py")]
+    started = time.time()
+    proc = subprocess.run([python, path, *extra], cwd=ROOT,
+                          capture_output=not echo, text=True)
+    took = time.time() - started
+    out = "" if echo else (proc.stdout or "") + (proc.stderr or "")
+    counts = tally(out)
+    skipped = bool(_SKIP.search(out)) and (counts is None or counts == (0, 0))
+    return {
+        "name": name, "path": path, "code": proc.returncode, "took": took,
+        "passed": counts[0] if counts else 0,
+        "failed": counts[1] if counts else 0,
+        "counted": counts is not None,
+        "skipped": skipped, "output": out,
+    }
+
+
+def main(argv: list[str]) -> int:
+    ap = argparse.ArgumentParser(
+        prog="tests/run.py", description=__doc__,
+        formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("only", nargs="*", metavar="SUBSTR",
+                    help="only run test files whose name contains one of these")
+    ap.add_argument("--fast", action="store_true",
+                    help="skip every file that needs IDA (seconds, runs anywhere)")
+    ap.add_argument("--ida-only", action="store_true",
+                    help="only the files that need IDA")
+    ap.add_argument("--list", action="store_true",
+                    help="show what would run, and whether it needs IDA")
+    ap.add_argument("-x", "--exitfirst", action="store_true",
+                    help="stop after the first failing file")
+    ap.add_argument("-v", "--verbose", action="store_true",
+                    help="stream each suite's output instead of capturing it")
+    ap.add_argument("--python", default=os.environ.get("IDATUI_PYTHON", DEFAULT_PY),
+                    help=f"interpreter for the IDA suites (default {DEFAULT_PY})")
+    args, extra = ap.parse_known_args(argv)
+
+    try:
+        files = discover()
+    except Marker as exc:
+        print(f"error: {exc}", file=sys.stderr)
+        return 2
+
+    selected = []
+    for path, ida in files:
+        if args.fast and ida:
+            continue
+        if args.ida_only and not ida:
+            continue
+        if args.only and not any(s in os.path.basename(path) for s in args.only):
+            continue
+        selected.append((path, ida))
+
+    if not selected:
+        print("nothing selected", file=sys.stderr)
+        return 2
+
+    if args.list:
+        for path, ida in selected:
+            print(f"{'ida ' if ida else 'pure'}  {os.path.basename(path)}")
+        return 0
+
+    # A pure file runs under whatever python invoked us (it needs nothing);
+    # an IDA file needs the interpreter that has textual + idapro.
+    pure_py = sys.executable
+    if any(ida for _, ida in selected) and not os.path.exists(args.python):
+        print(f"error: {args.python} not found — the IDA suites need an "
+              f"interpreter with textual + idapro.\n"
+              f"       Pass --python, set $IDATUI_PYTHON, or use --fast.",
+              file=sys.stderr)
+        return 2
+
+    results = []
+    started = time.time()
+    for path, ida in selected:
+        label = os.path.basename(path)
+        print(f"\033[1m>> {label}\033[0m{' (ida)' if ida else ''}", flush=True)
+        res = run_one(path, args.python if ida else pure_py, extra, args.verbose)
+        results.append(res)
+        bad = res["code"] != 0 or res["failed"]
+        if bad and not args.verbose:
+            print(res["output"].rstrip())
+        elif res["skipped"]:
+            print("   skipped")
+        else:
+            print(f"   {res['passed']} passed  ({res['took']:.1f}s)")
+        if bad and args.exitfirst:
+            print("\nstopping at the first failure (-x)", file=sys.stderr)
+            break
+
+    total = time.time() - started
+    print("\n" + "=" * 62)
+    width = max(len(r["name"]) for r in results)
+    passed = failed = 0
+    for r in results:
+        passed += r["passed"]
+        failed += r["failed"]
+        if r["skipped"]:
+            state = "\033[33mSKIP\033[0m"
+        elif r["code"] != 0 or r["failed"]:
+            state = "\033[31mFAIL\033[0m"
+        elif not r["counted"]:
+            state = "\033[33m ?  \033[0m"   # exit 0 but printed no tally
+        else:
+            state = "\033[32m ok \033[0m"
+        detail = f"{r['passed']:4d} passed"
+        if r["failed"]:
+            detail += f", \033[31m{r['failed']} failed\033[0m"
+        if r["code"] != 0 and not r["failed"]:
+            detail += f", \033[31mexit {r['code']}\033[0m"
+        print(f" {state}  {r['name']:<{width}}  {detail}   {r['took']:6.1f}s")
+    print("=" * 62)
+
+    ran, total_files = len(results), len(selected)
+    skipped = sum(1 for r in results if r["skipped"])
+    hurt = [r["name"] for r in results if r["code"] != 0 or r["failed"]]
+    summary = f"{passed} passed"
+    if failed:
+        summary += f", {failed} failed"
+    if skipped:
+        summary += f", {skipped} file(s) skipped"
+    if ran != total_files:
+        summary += f", {total_files - ran} file(s) not reached"
+    print(f"{summary}   [{ran} file(s), {total:.1f}s]")
+    if hurt:
+        print(f"\033[31mfailing files: {', '.join(hurt)}\033[0m")
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main(sys.argv[1:]))
diff --git a/tests/test_blob_ui.py b/tests/test_blob_ui.py
index b30bba5..f60e914 100644
--- a/tests/test_blob_ui.py
+++ b/tests/test_blob_ui.py
@@ -7,6 +7,10 @@ which was a lie: loading had finished, there was simply nothing to land on.
 
 Needs IDA (spawns a real worker). ~40s.
 """
+
+#: spawns a real worker on a raw blob and drives the pilot.
+#: Read by tests/run.py (--fast skips every NEEDS_IDA file).
+NEEDS_IDA = True
 import asyncio
 import os
 import sys
diff --git a/tests/test_formats.py b/tests/test_formats.py
index 1045703..c0ad542 100644
--- a/tests/test_formats.py
+++ b/tests/test_formats.py
@@ -1,5 +1,9 @@
 #!/usr/bin/env python3
 """Format sniffing + load-switch construction (pure stdlib, no IDA)."""
+
+#: format sniffing + load switches, pure stdlib.
+#: Read by tests/run.py (--fast skips every NEEDS_IDA file).
+NEEDS_IDA = False
 import os
 import sys
 import tempfile
diff --git a/tests/test_graph.py b/tests/test_graph.py
index 9c0e792..146092d 100644
--- a/tests/test_graph.py
+++ b/tests/test_graph.py
@@ -11,6 +11,10 @@ monsters) actually live.
 """
 from __future__ import annotations
 
+#: the layout engine is pure: no IDA, no Textual.
+#: Read by tests/run.py (--fast skips every NEEDS_IDA file).
+NEEDS_IDA = False
+
 import json
 import os
 import sys
diff --git a/tests/test_index.py b/tests/test_index.py
index e14f715..f6761b1 100644
--- a/tests/test_index.py
+++ b/tests/test_index.py
@@ -5,6 +5,10 @@ Pure stdlib: no IDA, no textual, no worker.
 
     python tests/test_index.py
 """
+
+#: the symbol/string index, pure stdlib.
+#: Read by tests/run.py (--fast skips every NEEDS_IDA file).
+NEEDS_IDA = False
 import os
 import sys
 import tempfile
diff --git a/tests/test_pool.py b/tests/test_pool.py
index 5c6e2c4..94e197b 100644
--- a/tests/test_pool.py
+++ b/tests/test_pool.py
@@ -6,6 +6,10 @@ without spawning real idalib workers.
 
     python tests/test_pool.py
 """
+
+#: worker residency policy, with a fake client injected.
+#: Read by tests/run.py (--fast skips every NEEDS_IDA file).
+NEEDS_IDA = False
 import os
 import sys
 import tempfile
diff --git a/tests/test_project.py b/tests/test_project.py
index 91fd250..822002f 100644
--- a/tests/test_project.py
+++ b/tests/test_project.py
@@ -5,6 +5,10 @@ Pure stdlib: no IDA, no textual, no worker — runs anywhere in under a second.
 
     python tests/test_project.py
 """
+
+#: the project model + staging, pure stdlib.
+#: Read by tests/run.py (--fast skips every NEEDS_IDA file).
+NEEDS_IDA = False
 import json
 import os
 import sys
diff --git a/tests/test_project_ui.py b/tests/test_project_ui.py
index 54c06bd..753717a 100644
--- a/tests/test_project_ui.py
+++ b/tests/test_project_ui.py
@@ -8,6 +8,10 @@ Needs idalib (it spawns real workers, one per binary) and textual:
 Defaults to targets/echo + targets/cat. The binaries are copied into a temp
 source dir first, so the "source tree stays pristine" promise is checkable.
 """
+
+#: spawns one real worker per binary and drives the pilot.
+#: Read by tests/run.py (--fast skips every NEEDS_IDA file).
+NEEDS_IDA = True
 import asyncio
 import os
 import shutil
diff --git a/tests/test_rawimage_rpc.py b/tests/test_rawimage_rpc.py
index df1166d..d39ad35 100644
--- a/tests/test_rawimage_rpc.py
+++ b/tests/test_rawimage_rpc.py
@@ -20,6 +20,10 @@ Requires: tmux or zellij, IDA (idalib). ~2min.
 
     ~/ida-venv/bin/python tests/test_rawimage_rpc.py
 """
+
+#: spawns a real mux pane on a firmware image.
+#: Read by tests/run.py (--fast skips every NEEDS_IDA file).
+NEEDS_IDA = True
 import json
 import os
 import re
diff --git a/tests/test_scenarios.py b/tests/test_scenarios.py
index e138946..a6e7753 100644
--- a/tests/test_scenarios.py
+++ b/tests/test_scenarios.py
@@ -16,6 +16,10 @@ Uses ~/ida-venv python (has textual).
 --stop-after <substr>     stop once a check whose name contains <substr> ran
 --list                    print scenario names and exit
 """
+
+#: the pilot suite: one real worker, 56 scenarios.
+#: Read by tests/run.py (--fast skips every NEEDS_IDA file).
+NEEDS_IDA = True
 import asyncio
 import fnmatch
 import os
diff --git a/tests/test_thumb_ui.py b/tests/test_thumb_ui.py
index b0ca00b..92a19fb 100644
--- a/tests/test_thumb_ui.py
+++ b/tests/test_thumb_ui.py
@@ -9,6 +9,10 @@ garbage. `t` switches the mode.
 Uses experiments/fibonacci.bin (real Thumb), so the encodings are not a guess.
 Needs IDA. ~40s.
 """
+
+#: spawns a real worker on Thumb code and drives the pilot.
+#: Read by tests/run.py (--fast skips every NEEDS_IDA file).
+NEEDS_IDA = True
 import asyncio
 import os
 import sys
diff --git a/tests/test_trace.py b/tests/test_trace.py
index 58c9d81..8803cda 100644
--- a/tests/test_trace.py
+++ b/tests/test_trace.py
@@ -6,6 +6,10 @@ Pure stdlib — no IDA, no trace files needed. The differential test
 this one covers what the reference can't arbitrate: the set-shaped queries
 painting needs, and rebasing.
 """
+
+#: the trace model, pure stdlib and no trace files.
+#: Read by tests/run.py (--fast skips every NEEDS_IDA file).
+NEEDS_IDA = False
 import os
 import sys
 import tempfile
diff --git a/tests/test_trace_rpc.py b/tests/test_trace_rpc.py
index b5a77a4..015ce8b 100644
--- a/tests/test_trace_rpc.py
+++ b/tests/test_trace_rpc.py
@@ -13,6 +13,10 @@ message otherwise.
 
     ~/ida-venv/bin/python tests/test_trace_rpc.py
 """
+
+#: spawns a real mux pane with --trace.
+#: Read by tests/run.py (--fast skips every NEEDS_IDA file).
+NEEDS_IDA = True
 import json
 import os
 import shutil
diff --git a/tests/test_trace_ui.py b/tests/test_trace_ui.py
index 32977ad..d222eab 100644
--- a/tests/test_trace_ui.py
+++ b/tests/test_trace_ui.py
@@ -5,6 +5,10 @@ Needs IDA and a trace. Generates its own trace with the QEMU tracer if the
 tracer is built; skips with a message otherwise, since neither the emulator nor
 the trace is part of this repo.
 """
+
+#: needs IDA and a trace; skips if the tracer isn't built.
+#: Read by tests/run.py (--fast skips every NEEDS_IDA file).
+NEEDS_IDA = True
 import asyncio
 import os
 import shutil
diff --git a/tests/test_trace_vs_tenet.py b/tests/test_trace_vs_tenet.py
index aa6002a..30d74b0 100644
--- a/tests/test_trace_vs_tenet.py
+++ b/tests/test_trace_vs_tenet.py
@@ -12,6 +12,10 @@ this repo.
 
     python3 tests/test_trace_vs_tenet.py [trace.0.log ...]
 """
+
+#: differential vs Tenet's reader; skips without traces.
+#: Read by tests/run.py (--fast skips every NEEDS_IDA file).
+NEEDS_IDA = False
 import os
 import random
 import sys