From 2d342a92f5c9995c7014b1699eaf1180aadcef26 Mon Sep 17 00:00:00 2001 From: blasty Date: Sat, 25 Jul 2026 11:29:36 +0200 Subject: projects: worker pool with memory-budgeted residency (phase 1b) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit idatui/pool.py — WorkerPool keeps one live worker per project binary: * lazy spawn on first use; staging + a scratch sweep happen first, so a database wedged by a previously hard-killed worker reopens instead of crash-looping. * residency is bounded by a MEMORY BUDGET (default project.memory_pct of RAM), not a worker count — a count is the wrong knob when one project holds a 50KB helper and a 6MB crypto lib. Cost is measured per worker from /proc//smaps_rollup (PSS, which splits shared pages so summing means something). * over budget -> evict least-recently-used, never the active or pinned binary, and give up rather than thrash when nothing is evictable. Eviction calls idb_save first, so returning to a binary is a DB load, not a re-analysis. * status() feeds the switcher UI (resident/pinned/active/analysed/memory). worker_client: fix the wedge-file bug this depends on. close() sent __shutdown__ and then IMMEDIATELY SIGTERMed, killing the worker mid close_database() — which is what leaves the unpacked .id0/.id1/... behind and makes the .i64 refuse to reopen. Now it waits out a grace period (the worker returns from serve() on __shutdown__ and closes the DB in its finally) and only escalates if it's genuinely stuck. Also expose .pid for memory accounting. Verified: a pilot run that used to leave echo.id0/.id1/.id2/.nam/.til now leaves only echo.i64, and teardown is no slower (14 checks in 5s). tests/test_pool.py: 22 checks with an injected fake client — LRU order, budget eviction, active/pinned protection, save-before-close, thrash avoidance, status(), teardown. Pure stdlib, no idalib. --- idatui/worker_client.py | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) (limited to 'idatui/worker_client.py') diff --git a/idatui/worker_client.py b/idatui/worker_client.py index 27fa660..586ff21 100644 --- a/idatui/worker_client.py +++ b/idatui/worker_client.py @@ -117,7 +117,19 @@ class WorkerClient: time.sleep(0.2) raise IDAConnectionError("worker did not become ready in time") - def close(self) -> None: + @property + def pid(self) -> int | None: + """The worker process id (for memory accounting), or None if not spawned.""" + return self._proc.pid if self._proc is not None else None + + def close(self, grace: float = 20.0) -> None: + """Shut the worker down cleanly. + + After ``__shutdown__`` the worker still has to ``close_database()``, which + re-packs the ``.i64`` and removes the unpacked ``.id0/.id1/...`` scratch. + Signalling it before that finishes is what leaves databases wedged, so + wait out the grace period first and only escalate if it really is stuck. + """ with self._lock: s = self._sock self._sock = None @@ -132,13 +144,16 @@ class WorkerClient: pass if self._proc is not None: try: - self._proc.terminate() - self._proc.wait(timeout=5) - except Exception: # noqa: BLE001 + self._proc.wait(timeout=grace) # let it close the DB properly + except Exception: # noqa: BLE001 -- TimeoutExpired: it's stuck try: - self._proc.kill() + self._proc.terminate() + self._proc.wait(timeout=5) except Exception: # noqa: BLE001 - pass + try: + self._proc.kill() + except Exception: # noqa: BLE001 + pass # -- the call surface -------------------------------------------------- # def call(self, tool: str, *, timeout: float | None = None, **args) -> Any: -- cgit v1.3.1-sl0p