daemon: survive BLE adapter removal and auto-restart on crash
Pulling the USB Bluetooth dongle made BleakScanner raise from inside the main loop; the exception propagated out of daemon_main, the bg thread died, and the tray froze on its last state with no recovery (reported: "daemon crashed and doesn't come back"). Two layers of hardening: 1. main() loop now wraps scan+connect in try/except — adapter-gone errors are logged, the tray shows Scanning, and it backs off and retries, so replugging the adapter recovers automatically. CancelledError is re-raised. 2. The tray supervises the asyncio loop: an unexpected crash is logged and the loop is RESTARTED after a backoff, gated by a quit_evt so Quit still stops cleanly (and call_soon_threadsafe is guarded against a closed loop). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -596,6 +596,7 @@ async def main(tray_state=None) -> None:
|
|||||||
search_backoff = 1 # caps at 60s — gentle, for a device that is genuinely absent/off
|
search_backoff = 1 # caps at 60s — gentle, for a device that is genuinely absent/off
|
||||||
reconnect_backoff = 1 # caps at RECONNECT_BACKOFF_CAP — fast, to clear the 120s SLA after a drop
|
reconnect_backoff = 1 # caps at RECONNECT_BACKOFF_CAP — fast, to clear the 120s SLA after a drop
|
||||||
while not stop_event.is_set():
|
while not stop_event.is_set():
|
||||||
|
try:
|
||||||
device = await scan_for_device()
|
device = await scan_for_device()
|
||||||
if not device:
|
if not device:
|
||||||
# Slow-search regime: device was not found by scan — back off gently
|
# Slow-search regime: device was not found by scan — back off gently
|
||||||
@@ -624,6 +625,22 @@ async def main(tray_state=None) -> None:
|
|||||||
# Successful session — reset reconnect counter to floor; search_backoff also reset
|
# Successful session — reset reconnect counter to floor; search_backoff also reset
|
||||||
reconnect_backoff = 1
|
reconnect_backoff = 1
|
||||||
search_backoff = 1
|
search_backoff = 1
|
||||||
|
except asyncio.CancelledError:
|
||||||
|
raise
|
||||||
|
except Exception as e:
|
||||||
|
# The whole BLE adapter can disappear (USB dongle unplugged) — bleak/
|
||||||
|
# WinRT then raises from the scan or the connect. Never let that kill
|
||||||
|
# the loop: log, show Scanning, back off, and keep retrying so
|
||||||
|
# replugging the adapter recovers on its own, no manual restart
|
||||||
|
# (field SC: pulled the dongle -> daemon crashed and stayed down).
|
||||||
|
if tray_state:
|
||||||
|
tray_state.set_scanning()
|
||||||
|
log(f"BLE error ({type(e).__name__}: {e}); retrying in {search_backoff}s...")
|
||||||
|
try:
|
||||||
|
await asyncio.wait_for(stop_event.wait(), timeout=search_backoff)
|
||||||
|
except asyncio.TimeoutError:
|
||||||
|
pass
|
||||||
|
search_backoff = _next_backoff(search_backoff, 60)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|||||||
+23
-5
@@ -190,19 +190,33 @@ def main() -> None:
|
|||||||
ts = TrayState()
|
ts = TrayState()
|
||||||
icon = pystray.Icon("Clawdmeter", images["scanning"], "Clawdmeter")
|
icon = pystray.Icon("Clawdmeter", images["scanning"], "Clawdmeter")
|
||||||
|
|
||||||
# --- background thread: asyncio loop ---
|
# quit_evt lets the supervisor below tell a clean shutdown (Quit) apart from
|
||||||
|
# an unexpected crash: restart the loop on a crash, but NOT after Quit.
|
||||||
|
quit_evt = threading.Event()
|
||||||
|
|
||||||
|
# --- background thread: supervised asyncio loop ---
|
||||||
def _run_daemon() -> None:
|
def _run_daemon() -> None:
|
||||||
# daemon=True thread: an unhandled exception here would vanish silently
|
# daemon=True thread. The inner loop (daemon_main) is now resilient to the
|
||||||
# and freeze the tray on its last state forever (the field "frozen tray"
|
# BLE adapter vanishing, but as a last resort we also supervise the whole
|
||||||
# failure mode). Surface it instead — log the traceback to the rotating
|
# asyncio.run: any unexpected crash is logged AND the loop is restarted
|
||||||
# file and flip the tray to an actionable error state.
|
# after a backoff, instead of the thread dying and freezing the tray
|
||||||
|
# forever (field SC: pulling the BT dongle killed the daemon and it never
|
||||||
|
# came back). A clean return means stop_event was set (Quit) — stop then.
|
||||||
|
backoff = 1
|
||||||
|
while not quit_evt.is_set():
|
||||||
try:
|
try:
|
||||||
_asyncio.run(daemon_main(tray_state=ts))
|
_asyncio.run(daemon_main(tray_state=ts))
|
||||||
|
break # clean return == Quit requested
|
||||||
except Exception as e: # last-resort thread guard
|
except Exception as e: # last-resort thread guard
|
||||||
import traceback
|
import traceback
|
||||||
daemon_log(f"Daemon thread crashed: {e!r}")
|
daemon_log(f"Daemon thread crashed: {e!r}")
|
||||||
daemon_log(traceback.format_exc())
|
daemon_log(traceback.format_exc())
|
||||||
ts.set_error(f"daemon crashed: {type(e).__name__}")
|
ts.set_error(f"daemon crashed: {type(e).__name__}")
|
||||||
|
if quit_evt.is_set():
|
||||||
|
break
|
||||||
|
daemon_log(f"Restarting daemon loop in {backoff}s")
|
||||||
|
time.sleep(backoff)
|
||||||
|
backoff = min(backoff * 2, 30)
|
||||||
|
|
||||||
daemon_thread = threading.Thread(target=_run_daemon, daemon=True)
|
daemon_thread = threading.Thread(target=_run_daemon, daemon=True)
|
||||||
daemon_thread.start()
|
daemon_thread.start()
|
||||||
@@ -219,8 +233,12 @@ def main() -> None:
|
|||||||
# the device sits frozen on stale data instead of returning to its waiting
|
# the device sits frozen on stale data instead of returning to its waiting
|
||||||
# screen (SC#3 field report). The timeout caps the block so Quit can never
|
# screen (SC#3 field report). The timeout caps the block so Quit can never
|
||||||
# hang if a WinRT disconnect wedges (rare) — we exit anyway as a fallback.
|
# hang if a WinRT disconnect wedges (rare) — we exit anyway as a fallback.
|
||||||
|
quit_evt.set() # tell the supervisor this is a clean stop, not a crash
|
||||||
if ts.loop is not None and ts.stop_event is not None:
|
if ts.loop is not None and ts.stop_event is not None:
|
||||||
|
try:
|
||||||
ts.loop.call_soon_threadsafe(ts.stop_event.set)
|
ts.loop.call_soon_threadsafe(ts.stop_event.set)
|
||||||
|
except RuntimeError:
|
||||||
|
pass # loop already closed (e.g. mid-restart) — quit_evt handles it
|
||||||
daemon_thread.join(timeout=6.0)
|
daemon_thread.join(timeout=6.0)
|
||||||
icon_ref.stop()
|
icon_ref.stop()
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user