From 4c1b63e645a95b88c5dd87cdcfbbbec28bd2c464 Mon Sep 17 00:00:00 2001 From: wenil Date: Sat, 20 Jun 2026 18:39:58 +0300 Subject: [PATCH] daemon: survive BLE adapter removal and auto-restart on crash MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pulling the USB Bluetooth dongle made BleakScanner raise from inside the main loop; the exception propagated out of daemon_main, the bg thread died, and the tray froze on its last state with no recovery (reported: "daemon crashed and doesn't come back"). Two layers of hardening: 1. main() loop now wraps scan+connect in try/except — adapter-gone errors are logged, the tray shows Scanning, and it backs off and retries, so replugging the adapter recovers automatically. CancelledError is re-raised. 2. The tray supervises the asyncio loop: an unexpected crash is logged and the loop is RESTARTED after a backoff, gated by a quit_evt so Quit still stops cleanly (and call_soon_threadsafe is guarded against a closed loop). Co-Authored-By: Claude Opus 4.8 --- daemon/claude_usage_daemon_windows.py | 59 +++++++++++++++++---------- daemon/tray_windows.py | 44 ++++++++++++++------ 2 files changed, 69 insertions(+), 34 deletions(-) diff --git a/daemon/claude_usage_daemon_windows.py b/daemon/claude_usage_daemon_windows.py index ba6bfe8..a645fe5 100644 --- a/daemon/claude_usage_daemon_windows.py +++ b/daemon/claude_usage_daemon_windows.py @@ -596,34 +596,51 @@ async def main(tray_state=None) -> None: search_backoff = 1 # caps at 60s — gentle, for a device that is genuinely absent/off reconnect_backoff = 1 # caps at RECONNECT_BACKOFF_CAP — fast, to clear the 120s SLA after a drop while not stop_event.is_set(): - device = await scan_for_device() - if not device: - # Slow-search regime: device was not found by scan — back off gently + try: + device = await scan_for_device() + if not device: + # Slow-search regime: device was not found by scan — back off gently + if tray_state: + tray_state.set_scanning() + log(f"Device not found, retrying in {search_backoff}s...") + try: + await asyncio.wait_for(stop_event.wait(), timeout=search_backoff) + except asyncio.TimeoutError: + pass + search_backoff = _next_backoff(search_backoff, 60) + continue + + ok = await connect_and_run(device, stop_event, tray_state) + if not ok: + # Fast-reconnect regime: had/attempted a link that dropped — retry quickly + if tray_state: + tray_state.set_scanning() + log(f"Connection lost, reconnecting in {reconnect_backoff}s...") + try: + await asyncio.wait_for(stop_event.wait(), timeout=reconnect_backoff) + except asyncio.TimeoutError: + pass + reconnect_backoff = _next_backoff(reconnect_backoff, RECONNECT_BACKOFF_CAP) + else: + # Successful session — reset reconnect counter to floor; search_backoff also reset + reconnect_backoff = 1 + search_backoff = 1 + except asyncio.CancelledError: + raise + except Exception as e: + # The whole BLE adapter can disappear (USB dongle unplugged) — bleak/ + # WinRT then raises from the scan or the connect. Never let that kill + # the loop: log, show Scanning, back off, and keep retrying so + # replugging the adapter recovers on its own, no manual restart + # (field SC: pulled the dongle -> daemon crashed and stayed down). if tray_state: tray_state.set_scanning() - log(f"Device not found, retrying in {search_backoff}s...") + log(f"BLE error ({type(e).__name__}: {e}); retrying in {search_backoff}s...") try: await asyncio.wait_for(stop_event.wait(), timeout=search_backoff) except asyncio.TimeoutError: pass search_backoff = _next_backoff(search_backoff, 60) - continue - - ok = await connect_and_run(device, stop_event, tray_state) - if not ok: - # Fast-reconnect regime: had/attempted a link that dropped — retry quickly - if tray_state: - tray_state.set_scanning() - log(f"Connection lost, reconnecting in {reconnect_backoff}s...") - try: - await asyncio.wait_for(stop_event.wait(), timeout=reconnect_backoff) - except asyncio.TimeoutError: - pass - reconnect_backoff = _next_backoff(reconnect_backoff, RECONNECT_BACKOFF_CAP) - else: - # Successful session — reset reconnect counter to floor; search_backoff also reset - reconnect_backoff = 1 - search_backoff = 1 if __name__ == "__main__": diff --git a/daemon/tray_windows.py b/daemon/tray_windows.py index dac3195..58da9f0 100644 --- a/daemon/tray_windows.py +++ b/daemon/tray_windows.py @@ -190,19 +190,33 @@ def main() -> None: ts = TrayState() icon = pystray.Icon("Clawdmeter", images["scanning"], "Clawdmeter") - # --- background thread: asyncio loop --- + # quit_evt lets the supervisor below tell a clean shutdown (Quit) apart from + # an unexpected crash: restart the loop on a crash, but NOT after Quit. + quit_evt = threading.Event() + + # --- background thread: supervised asyncio loop --- def _run_daemon() -> None: - # daemon=True thread: an unhandled exception here would vanish silently - # and freeze the tray on its last state forever (the field "frozen tray" - # failure mode). Surface it instead — log the traceback to the rotating - # file and flip the tray to an actionable error state. - try: - _asyncio.run(daemon_main(tray_state=ts)) - except Exception as e: # last-resort thread guard - import traceback - daemon_log(f"Daemon thread crashed: {e!r}") - daemon_log(traceback.format_exc()) - ts.set_error(f"daemon crashed: {type(e).__name__}") + # daemon=True thread. The inner loop (daemon_main) is now resilient to the + # BLE adapter vanishing, but as a last resort we also supervise the whole + # asyncio.run: any unexpected crash is logged AND the loop is restarted + # after a backoff, instead of the thread dying and freezing the tray + # forever (field SC: pulling the BT dongle killed the daemon and it never + # came back). A clean return means stop_event was set (Quit) — stop then. + backoff = 1 + while not quit_evt.is_set(): + try: + _asyncio.run(daemon_main(tray_state=ts)) + break # clean return == Quit requested + except Exception as e: # last-resort thread guard + import traceback + daemon_log(f"Daemon thread crashed: {e!r}") + daemon_log(traceback.format_exc()) + ts.set_error(f"daemon crashed: {type(e).__name__}") + if quit_evt.is_set(): + break + daemon_log(f"Restarting daemon loop in {backoff}s") + time.sleep(backoff) + backoff = min(backoff * 2, 30) daemon_thread = threading.Thread(target=_run_daemon, daemon=True) daemon_thread.start() @@ -219,8 +233,12 @@ def main() -> None: # the device sits frozen on stale data instead of returning to its waiting # screen (SC#3 field report). The timeout caps the block so Quit can never # hang if a WinRT disconnect wedges (rare) — we exit anyway as a fallback. + quit_evt.set() # tell the supervisor this is a clean stop, not a crash if ts.loop is not None and ts.stop_event is not None: - ts.loop.call_soon_threadsafe(ts.stop_event.set) + try: + ts.loop.call_soon_threadsafe(ts.stop_event.set) + except RuntimeError: + pass # loop already closed (e.g. mid-restart) — quit_evt handles it daemon_thread.join(timeout=6.0) icon_ref.stop()