daemon: survive BLE adapter removal and auto-restart on crash
Pulling the USB Bluetooth dongle made BleakScanner raise from inside the main loop; the exception propagated out of daemon_main, the bg thread died, and the tray froze on its last state with no recovery (reported: "daemon crashed and doesn't come back"). Two layers of hardening: 1. main() loop now wraps scan+connect in try/except — adapter-gone errors are logged, the tray shows Scanning, and it backs off and retries, so replugging the adapter recovers automatically. CancelledError is re-raised. 2. The tray supervises the asyncio loop: an unexpected crash is logged and the loop is RESTARTED after a backoff, gated by a quit_evt so Quit still stops cleanly (and call_soon_threadsafe is guarded against a closed loop). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -596,34 +596,51 @@ async def main(tray_state=None) -> None:
|
||||
search_backoff = 1 # caps at 60s — gentle, for a device that is genuinely absent/off
|
||||
reconnect_backoff = 1 # caps at RECONNECT_BACKOFF_CAP — fast, to clear the 120s SLA after a drop
|
||||
while not stop_event.is_set():
|
||||
device = await scan_for_device()
|
||||
if not device:
|
||||
# Slow-search regime: device was not found by scan — back off gently
|
||||
try:
|
||||
device = await scan_for_device()
|
||||
if not device:
|
||||
# Slow-search regime: device was not found by scan — back off gently
|
||||
if tray_state:
|
||||
tray_state.set_scanning()
|
||||
log(f"Device not found, retrying in {search_backoff}s...")
|
||||
try:
|
||||
await asyncio.wait_for(stop_event.wait(), timeout=search_backoff)
|
||||
except asyncio.TimeoutError:
|
||||
pass
|
||||
search_backoff = _next_backoff(search_backoff, 60)
|
||||
continue
|
||||
|
||||
ok = await connect_and_run(device, stop_event, tray_state)
|
||||
if not ok:
|
||||
# Fast-reconnect regime: had/attempted a link that dropped — retry quickly
|
||||
if tray_state:
|
||||
tray_state.set_scanning()
|
||||
log(f"Connection lost, reconnecting in {reconnect_backoff}s...")
|
||||
try:
|
||||
await asyncio.wait_for(stop_event.wait(), timeout=reconnect_backoff)
|
||||
except asyncio.TimeoutError:
|
||||
pass
|
||||
reconnect_backoff = _next_backoff(reconnect_backoff, RECONNECT_BACKOFF_CAP)
|
||||
else:
|
||||
# Successful session — reset reconnect counter to floor; search_backoff also reset
|
||||
reconnect_backoff = 1
|
||||
search_backoff = 1
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as e:
|
||||
# The whole BLE adapter can disappear (USB dongle unplugged) — bleak/
|
||||
# WinRT then raises from the scan or the connect. Never let that kill
|
||||
# the loop: log, show Scanning, back off, and keep retrying so
|
||||
# replugging the adapter recovers on its own, no manual restart
|
||||
# (field SC: pulled the dongle -> daemon crashed and stayed down).
|
||||
if tray_state:
|
||||
tray_state.set_scanning()
|
||||
log(f"Device not found, retrying in {search_backoff}s...")
|
||||
log(f"BLE error ({type(e).__name__}: {e}); retrying in {search_backoff}s...")
|
||||
try:
|
||||
await asyncio.wait_for(stop_event.wait(), timeout=search_backoff)
|
||||
except asyncio.TimeoutError:
|
||||
pass
|
||||
search_backoff = _next_backoff(search_backoff, 60)
|
||||
continue
|
||||
|
||||
ok = await connect_and_run(device, stop_event, tray_state)
|
||||
if not ok:
|
||||
# Fast-reconnect regime: had/attempted a link that dropped — retry quickly
|
||||
if tray_state:
|
||||
tray_state.set_scanning()
|
||||
log(f"Connection lost, reconnecting in {reconnect_backoff}s...")
|
||||
try:
|
||||
await asyncio.wait_for(stop_event.wait(), timeout=reconnect_backoff)
|
||||
except asyncio.TimeoutError:
|
||||
pass
|
||||
reconnect_backoff = _next_backoff(reconnect_backoff, RECONNECT_BACKOFF_CAP)
|
||||
else:
|
||||
# Successful session — reset reconnect counter to floor; search_backoff also reset
|
||||
reconnect_backoff = 1
|
||||
search_backoff = 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
+31
-13
@@ -190,19 +190,33 @@ def main() -> None:
|
||||
ts = TrayState()
|
||||
icon = pystray.Icon("Clawdmeter", images["scanning"], "Clawdmeter")
|
||||
|
||||
# --- background thread: asyncio loop ---
|
||||
# quit_evt lets the supervisor below tell a clean shutdown (Quit) apart from
|
||||
# an unexpected crash: restart the loop on a crash, but NOT after Quit.
|
||||
quit_evt = threading.Event()
|
||||
|
||||
# --- background thread: supervised asyncio loop ---
|
||||
def _run_daemon() -> None:
|
||||
# daemon=True thread: an unhandled exception here would vanish silently
|
||||
# and freeze the tray on its last state forever (the field "frozen tray"
|
||||
# failure mode). Surface it instead — log the traceback to the rotating
|
||||
# file and flip the tray to an actionable error state.
|
||||
try:
|
||||
_asyncio.run(daemon_main(tray_state=ts))
|
||||
except Exception as e: # last-resort thread guard
|
||||
import traceback
|
||||
daemon_log(f"Daemon thread crashed: {e!r}")
|
||||
daemon_log(traceback.format_exc())
|
||||
ts.set_error(f"daemon crashed: {type(e).__name__}")
|
||||
# daemon=True thread. The inner loop (daemon_main) is now resilient to the
|
||||
# BLE adapter vanishing, but as a last resort we also supervise the whole
|
||||
# asyncio.run: any unexpected crash is logged AND the loop is restarted
|
||||
# after a backoff, instead of the thread dying and freezing the tray
|
||||
# forever (field SC: pulling the BT dongle killed the daemon and it never
|
||||
# came back). A clean return means stop_event was set (Quit) — stop then.
|
||||
backoff = 1
|
||||
while not quit_evt.is_set():
|
||||
try:
|
||||
_asyncio.run(daemon_main(tray_state=ts))
|
||||
break # clean return == Quit requested
|
||||
except Exception as e: # last-resort thread guard
|
||||
import traceback
|
||||
daemon_log(f"Daemon thread crashed: {e!r}")
|
||||
daemon_log(traceback.format_exc())
|
||||
ts.set_error(f"daemon crashed: {type(e).__name__}")
|
||||
if quit_evt.is_set():
|
||||
break
|
||||
daemon_log(f"Restarting daemon loop in {backoff}s")
|
||||
time.sleep(backoff)
|
||||
backoff = min(backoff * 2, 30)
|
||||
|
||||
daemon_thread = threading.Thread(target=_run_daemon, daemon=True)
|
||||
daemon_thread.start()
|
||||
@@ -219,8 +233,12 @@ def main() -> None:
|
||||
# the device sits frozen on stale data instead of returning to its waiting
|
||||
# screen (SC#3 field report). The timeout caps the block so Quit can never
|
||||
# hang if a WinRT disconnect wedges (rare) — we exit anyway as a fallback.
|
||||
quit_evt.set() # tell the supervisor this is a clean stop, not a crash
|
||||
if ts.loop is not None and ts.stop_event is not None:
|
||||
ts.loop.call_soon_threadsafe(ts.stop_event.set)
|
||||
try:
|
||||
ts.loop.call_soon_threadsafe(ts.stop_event.set)
|
||||
except RuntimeError:
|
||||
pass # loop already closed (e.g. mid-restart) — quit_evt handles it
|
||||
daemon_thread.join(timeout=6.0)
|
||||
icon_ref.stop()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user