autoresearch: checks gate names the failing scenario and retries it alone

1 files changed, 34 insertions, 11 deletions

diff --git a/.auto/checks.sh b/.auto/checks.sh
index c555988..323f0fd 100755
--- a/.auto/checks.sh
+++ b/.auto/checks.sh
@@ -7,8 +7,11 @@
 #      behaviour; a rendering/paging/nav optimisation that breaks the UI shows
 #      up here and nowhere else.
 #
-# Only failures reach stdout: the agent sees the last 80 lines on failure and
-# a wall of "ok" would push the actual break out of view.
+# Only failures reach stdout: the agent sees the last 80 lines on failure, and
+# a wall of "ok" would push the actual break out of view. A failing scenario is
+# re-run ALONE before it counts -- the suite shares one warm session, and a
+# scenario that timed out under load and then failed a check trivially is a
+# flake, not a regression (see the note in tests/run.py).
 set -euo pipefail
 cd "$(dirname "$0")/.."
 
@@ -17,14 +20,34 @@ PY="${IDATUI_PYTHON:-$HOME/ida-venv/bin/python}"
 pure=$(python3 tests/run.py --fast 2>&1) || { echo "$pure" | tail -40; exit 1; }
 echo "$pure" | tail -2
 
-out=$("$PY" tests/test_scenarios.py targets/echo 2>&1) || {
-    echo "--- scenario suite crashed ---"
-    echo "$out" | tail -40
-    exit 1
+run_suite() {  # $1 = optional --only filter
+    if [ -n "${1:-}" ]; then
+        "$PY" tests/test_scenarios.py targets/echo --only "$1" 2>&1
+    else
+        "$PY" tests/test_scenarios.py targets/echo 2>&1
+    fi
 }
-echo "$out" | grep -E "^  FAIL|failed" | tail -40
-if ! echo "$out" | tail -3 | grep -qE "^[0-9]+ passed, 0 failed"; then
-    echo "--- scenario suite regressions ---"
-    echo "$out" | grep -E "^  FAIL" | head -30
-    exit 1
+
+out=$(run_suite || true)
+echo "$out" | grep -E "^[0-9]+ passed, [0-9]+ failed" | tail -1
+
+fails=$(echo "$out" | grep -E "^  FAIL" || true)
+if [ -z "$fails" ]; then
+    exit 0
+fi
+
+echo "--- first pass failures ---"
+echo "$fails" | head -20
+
+# Retry just the scenarios that failed, alone, on a fresh session.
+names=$(echo "$fails" | sed -E 's/^  FAIL \[([^]]+)\].*/\1/' | sort -u | paste -sd, -)
+echo "--- retrying alone: $names ---"
+retry=$(run_suite "$names" || true)
+again=$(echo "$retry" | grep -E "^  FAIL" || true)
+if [ -z "$again" ]; then
+    echo "flake: all of [$names] pass in isolation"
+    exit 0
 fi
+echo "--- REAL regression ---"
+echo "$again" | head -30
+exit 1