diff options
| -rwxr-xr-x | .auto/checks.sh | 77 | ||||
| -rw-r--r-- | idatui/worker_client.py | 25 | ||||
| -rw-r--r-- | tests/test_project_ui.py | 29 |
3 files changed, 74 insertions, 57 deletions
diff --git a/.auto/checks.sh b/.auto/checks.sh index 6cf1102..5323fce 100755 --- a/.auto/checks.sh +++ b/.auto/checks.sh @@ -1,28 +1,26 @@ #!/bin/bash # Correctness gate: no perf win is allowed to cost functionality. # -# 1. the pure suites (344 checks, ~3.5s, stdlib only) -# 2. the pilot scenario suite (301 checks, ~80s) -- a real idalib worker -# driving the real Textual app. This is the project's source of truth for -# behaviour; a rendering/paging/nav optimisation that breaks the UI shows -# up here and nowhere else. -# 3. .auto/check_search.py -- the search fast paths against the plain loop. -# Caches that go stale still return an answer, so the scenario suite -# cannot see them; this compares fast and slow directly. +# 1. .auto/check_search.py -- the search fast paths against the plain loop. +# Caches that go stale still return AN answer, so no scenario test can see +# them; this compares fast and slow directly, for every typed prefix. +# 2. tests/run.py -- the project's own front door: every suite, pure and IDA, +# ~185s. It used to be only the scenario suite here, and that gap cost a +# real regression: a faster worker connect left the loading overlay up a +# moment longer relative to the index finishing, and project mode's first +# keypress landed on the overlay. Only test_project_ui.py covers that, and +# it was not being run. # -# Only failures reach stdout: the agent sees the last 80 lines on failure, and -# a wall of "ok" would push the actual break out of view. A failing scenario is -# re-run ALONE before it counts -- the suite shares one warm session, and a -# scenario that timed out under load and then failed a check trivially is a -# flake, not a regression (see the note in tests/run.py). +# Only failures reach stdout: the agent sees the last 80 lines on failure, and a +# wall of "ok" would push the actual break out of view. A file that fails is +# re-run ALONE before it counts -- the IDA suites share a loaded box, and a +# worker that got CPU-starved mid-analysis reads as a failure but is not one +# (see the note in tests/run.py). set -euo pipefail cd "$(dirname "$0")/.." PY="${IDATUI_PYTHON:-$HOME/ida-venv/bin/python}" -pure=$(python3 tests/run.py --fast 2>&1) || { echo "$pure" | tail -40; exit 1; } -echo "$pure" | tail -2 - search=$("$PY" .auto/check_search.py targets/echo 2>&1) || { echo "--- search fast paths disagree with the plain loop ---" echo "$search" | tail -20 @@ -30,34 +28,31 @@ search=$("$PY" .auto/check_search.py targets/echo 2>&1) || { } echo "$search" | tail -1 -run_suite() { # $1 = optional --only filter - if [ -n "${1:-}" ]; then - "$PY" tests/test_scenarios.py targets/echo --only "$1" 2>&1 - else - "$PY" tests/test_scenarios.py targets/echo 2>&1 - fi -} - -out=$(run_suite || true) -echo "$out" | grep -E "^[0-9]+ passed, [0-9]+ failed" | tail -1 +out=$(python3 tests/run.py 2>&1) || true +echo "$out" | tail -2 -fails=$(echo "$out" | grep -E "^ FAIL" || true) -if [ -z "$fails" ]; then +# Files run.py marked bad, e.g. " FAIL scenarios 301 passed, 2 failed" +# run.py's summary says "N passed[, M failed]..."; no "failed" clause == green. +if ! echo "$out" | tail -3 | grep -q "failed"; then exit 0 fi echo "--- first pass failures ---" -echo "$fails" | head -20 - -# Retry just the scenarios that failed, alone, on a fresh session. -names=$(echo "$fails" | sed -E 's/^ FAIL \[([^]]+)\].*/\1/' | sort -u | paste -sd, -) -echo "--- retrying alone: $names ---" -retry=$(run_suite "$names" || true) -again=$(echo "$retry" | grep -E "^ FAIL" || true) -if [ -z "$again" ]; then - echo "flake: all of [$names] pass in isolation" - exit 0 +echo "$out" | grep -E "^ FAIL" | head -20 +# run.py names them itself: "failing files: formats scenarios" (colourised). +files=$(echo "$out" | sed -e 's/\x1b\[[0-9;]*m//g' \ + | sed -n 's/^failing files: *//p' | tr '\n' ' ') +if [ -z "$files" ]; then + echo "--- could not identify the failing file; full tail ---" + echo "$out" | tail -30 + exit 1 +fi +echo "--- retrying alone: $files ---" +retry=$(python3 tests/run.py $files 2>&1) || true +if echo "$retry" | tail -3 | grep -q "failed"; then + echo "--- REAL regression ---" + echo "$retry" | grep -E "^ FAIL" | head -30 + echo "$retry" | tail -3 + exit 1 fi -echo "--- REAL regression ---" -echo "$again" | head -30 -exit 1 +echo "flake: [$files] pass in isolation" diff --git a/idatui/worker_client.py b/idatui/worker_client.py index 4f90700..173cd9d 100644 --- a/idatui/worker_client.py +++ b/idatui/worker_client.py @@ -117,16 +117,20 @@ class WorkerClient: # (a small binary, or a seeded .i64), which is most of the time in # the tests and noticeable on a re-open. # - # Backing off geometrically from the first probe was still too eager: - # a seeded database is ready at ~250ms, by which point the delay has - # grown to 134ms, so every open waited ~350ms whatever the binary -- - # the same number for a 47KB `echo` and a 1.2MB `bash`, which is what - # gives a polling artefact away. Hold the fast rate for the first few - # seconds (a connect attempt on an absent socket is microseconds) and - # only slow down for a genuine cold auto-analysis, which runs for - # minutes and does not care about 200ms. + # Backing off all the way to 0.2s was too eager: a seeded database + # is ready at ~250ms, by which point the delay has grown to 134ms, so + # every open waited ~350ms whatever the binary -- the same number for + # a 47KB `echo` and a 1.2MB `bash`, which is what gives a polling + # artefact away. Cap the backoff at 25ms instead: the overshoot on a + # fast open is bounded by that, and 40 probes a second is nothing + # next to an auto-analysis that runs for minutes. + # + # Do NOT be tempted to hold the 5ms rate instead. This poll runs on a + # background thread while the UI thread is drawing, and 200 wakeups a + # second through a cold analysis cost enough GIL time to delay the + # app's own startup -- it left the loading overlay up long enough for + # project mode's first keypress to land on it. delay = 0.005 - fast_until = t0 + 5.0 while time.time() < deadline: try: s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) @@ -142,8 +146,7 @@ class WorkerClient: progress(f"auto-analyzing {os.path.basename(self._bin)}… " f"({int(time.time() - t0)}s)") time.sleep(delay) - if time.time() > fast_until: - delay = min(delay * 1.6, 0.2) + delay = min(delay * 1.6, 0.025) raise IDAConnectionError("worker did not become ready in time") @property diff --git a/tests/test_project_ui.py b/tests/test_project_ui.py index 753717a..7e93c81 100644 --- a/tests/test_project_ui.py +++ b/tests/test_project_ui.py @@ -56,10 +56,27 @@ async def run(bins): async def settle(pred, t=180.0): return await wait_for(pred, pilot.pause, t, 0.05) + async def usable(t=180.0): + """Loaded AND drivable. + + An index that has finished streaming does not mean the app is + taking keys yet: the loading overlay is a ModalScreen and it is + dismissed a moment later, by auto-land. Between those two the + app looks ready and swallows every keypress -- so a test that + waits only for the index presses Ctrl+O into the overlay and + sees no switcher. Which side of that gap the poll lands on is + decided by how fast the backend happens to be, so it has to be + waited for explicitly, not hoped for. + """ + return await settle( + lambda: app.program is not None + and app._func_index is not None + and app._func_index.complete + and app._loading_screen is None + and len(app.screen_stack) == 1, t) + # -- boots on the project's first binary ----------------------- # - ok = await settle(lambda: app.program is not None - and app._func_index is not None - and app._func_index.complete) + ok = await usable() check("project mode boots on the first binary", ok, f"binary={app._binary}") check("the active binary is the first one", app._binary == first, @@ -100,7 +117,8 @@ async def run(bins): await pilot.press("enter") switched = await settle( lambda: app._binary == second and app.program is not None - and app._func_index is not None and app._func_index.complete) + and app._func_index is not None and app._func_index.complete + and len(app.screen_stack) == 1) check("switching opens the other binary", switched, f"binary={app._binary}") check("the second binary has its own function index", @@ -127,7 +145,8 @@ async def run(bins): await pilot.press("enter") back = await settle(lambda: app._binary == first and app._func_index is not None - and app._func_index.complete, 120) + and app._func_index.complete + and len(app.screen_stack) == 1, 120) check("switching back returns to the first binary", back, f"binary={app._binary}") check("its function index came back intact", |