From 94d55a88d216d2bc3e414b3abdf8f4a0a1221cd3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hermann=20Bj=C3=B6rgvin=20Haraldsson?= Date: Wed, 1 Apr 2026 17:29:06 +0000 Subject: [PATCH] Add USB HID keyboard input via touch gestures The SC01 Plus now acts as a composite USB device (CDC serial + HID keyboard). Touch gestures on the screen send keystrokes to the computer: - Swipe up/down: arrow keys - Swipe right: Enter - Swipe left: Shift+Tab (change mode in Claude Code) - Double-tap logo: Ctrl+Space (voice mode) - Tap and hold: Space (held while touching) Gesture-to-key mapping is defined as a simple config table in hid.cpp. Touch detection uses a state machine with configurable thresholds in touch.cpp. Switches to TinyUSB composite mode (ARDUINO_USB_MODE=0). Flashing now requires flash.sh which handles the DTR/RTS bootloader reset. Co-Authored-By: Claude Opus 4.6 (1M context) --- firmware/platformio.ini | 1 + firmware/src/hid.cpp | 51 +++++++++++++++++ firmware/src/hid.h | 17 ++++++ firmware/src/main.cpp | 11 +++- firmware/src/touch.cpp | 124 ++++++++++++++++++++++++++++++++++++++++ firmware/src/touch.h | 5 ++ flash.sh | 44 ++++++++++++++ 7 files changed, 252 insertions(+), 1 deletion(-) create mode 100644 firmware/src/hid.cpp create mode 100644 firmware/src/hid.h create mode 100644 firmware/src/touch.cpp create mode 100644 firmware/src/touch.h create mode 100755 flash.sh diff --git a/firmware/platformio.ini b/firmware/platformio.ini index ab1430c..e007926 100644 --- a/firmware/platformio.ini +++ b/firmware/platformio.ini @@ -7,6 +7,7 @@ monitor_speed = 115200 build_flags = -DARDUINO_USB_CDC_ON_BOOT=1 + -UARDUINO_USB_MODE -DARDUINO_USB_MODE=0 -DBOARD_HAS_PSRAM ; LVGL config via build flags -DLV_CONF_SKIP diff --git a/firmware/src/hid.cpp b/firmware/src/hid.cpp new file mode 100644 index 0000000..3fd840a --- /dev/null +++ b/firmware/src/hid.cpp @@ -0,0 +1,51 @@ +#include "hid.h" +#include "USB.h" +#include "USBHIDKeyboard.h" + +static USBHIDKeyboard keyboard; + +// Gesture-to-key mapping table. Edit this to change controls. +struct gesture_mapping_t { + gesture_t gesture; + uint8_t key; // HID_KEY_* constant (for pressRaw/releaseRaw) + uint8_t modifier; // KEY_LEFT_CTRL, KEY_LEFT_SHIFT, etc. (0 = none) +}; + +static const gesture_mapping_t mappings[] = { + { GESTURE_SWIPE_UP, HID_KEY_ARROW_UP, 0 }, + { GESTURE_SWIPE_DOWN, HID_KEY_ARROW_DOWN, 0 }, + { GESTURE_SWIPE_LEFT, HID_KEY_TAB, KEY_LEFT_SHIFT }, + { GESTURE_SWIPE_RIGHT, HID_KEY_ENTER, 0 }, + { GESTURE_DOUBLE_TAP_LOGO, HID_KEY_SPACE, KEY_LEFT_CTRL }, +}; +#define MAPPING_COUNT (sizeof(mappings) / sizeof(mappings[0])) + +// Hold gesture uses space key (press on start, release on end) +#define HOLD_KEY HID_KEY_SPACE + +void hid_init(void) { + USB.begin(); + keyboard.begin(); +} + +void hid_on_gesture(gesture_t gesture) { + if (gesture == GESTURE_HOLD_START) { + keyboard.pressRaw(HOLD_KEY); + return; + } + if (gesture == GESTURE_HOLD_END) { + keyboard.releaseRaw(HOLD_KEY); + return; + } + + for (int i = 0; i < MAPPING_COUNT; i++) { + if (mappings[i].gesture == gesture) { + if (mappings[i].modifier) keyboard.press(mappings[i].modifier); + keyboard.pressRaw(mappings[i].key); + delay(20); + keyboard.releaseRaw(mappings[i].key); + if (mappings[i].modifier) keyboard.release(mappings[i].modifier); + return; + } + } +} diff --git a/firmware/src/hid.h b/firmware/src/hid.h new file mode 100644 index 0000000..ca011c6 --- /dev/null +++ b/firmware/src/hid.h @@ -0,0 +1,17 @@ +#pragma once +#include + +// Gesture types +enum gesture_t { + GESTURE_NONE, + GESTURE_SWIPE_UP, + GESTURE_SWIPE_DOWN, + GESTURE_SWIPE_LEFT, + GESTURE_SWIPE_RIGHT, + GESTURE_DOUBLE_TAP_LOGO, + GESTURE_HOLD_START, + GESTURE_HOLD_END, +}; + +void hid_init(void); +void hid_on_gesture(gesture_t gesture); diff --git a/firmware/src/main.cpp b/firmware/src/main.cpp index 5bd0ced..04a97ec 100644 --- a/firmware/src/main.cpp +++ b/firmware/src/main.cpp @@ -4,8 +4,10 @@ #include "display_cfg.h" #include "data.h" #include "ui.h" +#include "hid.h" +#include "touch.h" -static LGFX lcd; +LGFX lcd; // global - used by touch.cpp static UsageData usage = {}; // LVGL draw buffers @@ -105,6 +107,12 @@ void setup() { lv_indev_set_type(indev, LV_INDEV_TYPE_POINTER); lv_indev_set_read_cb(indev, my_touch_cb); + // Init USB HID keyboard + hid_init(); + + // Init touch gesture detection + touch_init(); + // Build dashboard ui_init(); @@ -114,6 +122,7 @@ void setup() { void loop() { lv_timer_handler(); ui_tick_anim(); + touch_tick(); if (read_serial_line()) { if (parse_json(serial_buf, &usage)) { diff --git a/firmware/src/touch.cpp b/firmware/src/touch.cpp new file mode 100644 index 0000000..ae74e9d --- /dev/null +++ b/firmware/src/touch.cpp @@ -0,0 +1,124 @@ +#include "touch.h" +#include +#include "display_cfg.h" + +// Thresholds (easy to tune) +#define SWIPE_MIN_PX 50 // minimum distance to count as swipe +#define SWIPE_MAX_CROSS_PX 30 // max perpendicular movement for a clean swipe +#define HOLD_TIME_MS 500 // time before touch becomes a hold +#define DOUBLE_TAP_MS 300 // max gap between taps for double-tap +#define DOUBLE_TAP_PX 20 // max distance between taps +#define LOGO_X_MAX 60 // logo hit region (upper-left) +#define LOGO_Y_MAX 60 + +extern LGFX lcd; // from main.cpp + +enum touch_state_t { + TS_IDLE, + TS_DOWN, // finger just touched, waiting to classify + TS_HOLDING, // classified as hold, space key is pressed + TS_SWIPING, // movement exceeded threshold, will be swipe on release +}; + +static touch_state_t state = TS_IDLE; +static uint16_t start_x, start_y; +static uint16_t last_x, last_y; // track last known position for swipe classification +static uint32_t down_time; + +// Double-tap tracking +static uint32_t last_tap_time = 0; +static uint16_t last_tap_x, last_tap_y; + +static gesture_t classify_swipe(int dx, int dy) { + int ax = abs(dx); + int ay = abs(dy); + + if (ax > ay && ax >= SWIPE_MIN_PX && ay <= SWIPE_MAX_CROSS_PX) { + return dx > 0 ? GESTURE_SWIPE_RIGHT : GESTURE_SWIPE_LEFT; + } + if (ay > ax && ay >= SWIPE_MIN_PX && ax <= SWIPE_MAX_CROSS_PX) { + return dy > 0 ? GESTURE_SWIPE_DOWN : GESTURE_SWIPE_UP; + } + return GESTURE_NONE; +} + +static bool in_logo_region(uint16_t x, uint16_t y) { + return x < LOGO_X_MAX && y < LOGO_Y_MAX; +} + +void touch_init(void) { + state = TS_IDLE; +} + +void touch_tick(void) { + uint16_t x, y; + bool touching = lcd.getTouch(&x, &y); + uint32_t now = millis(); + + switch (state) { + case TS_IDLE: + if (touching) { + start_x = x; + start_y = y; + down_time = now; + state = TS_DOWN; + } + break; + + case TS_DOWN: + if (!touching) { + // Released quickly - this is a tap (check for double-tap on logo) + if (in_logo_region(start_x, start_y) && + (now - last_tap_time) < DOUBLE_TAP_MS && + abs((int)start_x - (int)last_tap_x) < DOUBLE_TAP_PX && + abs((int)start_y - (int)last_tap_y) < DOUBLE_TAP_PX) { + hid_on_gesture(GESTURE_DOUBLE_TAP_LOGO); + last_tap_time = 0; // reset so triple-tap doesn't re-trigger + } else { + last_tap_time = now; + last_tap_x = start_x; + last_tap_y = start_y; + } + state = TS_IDLE; + } else { + last_x = x; + last_y = y; + int dx = (int)x - (int)start_x; + int dy = (int)y - (int)start_y; + + // Check if movement qualifies as a swipe + if (abs(dx) >= SWIPE_MIN_PX || abs(dy) >= SWIPE_MIN_PX) { + state = TS_SWIPING; + } + // Check if held long enough for a hold gesture + else if ((now - down_time) >= HOLD_TIME_MS) { + hid_on_gesture(GESTURE_HOLD_START); + state = TS_HOLDING; + } + } + break; + + case TS_HOLDING: + if (!touching) { + hid_on_gesture(GESTURE_HOLD_END); + state = TS_IDLE; + } + break; + + case TS_SWIPING: + if (touching) { + last_x = x; + last_y = y; + } else { + // Use last known position since getTouch returns garbage on release + int dx = (int)last_x - (int)start_x; + int dy = (int)last_y - (int)start_y; + gesture_t g = classify_swipe(dx, dy); + if (g != GESTURE_NONE) { + hid_on_gesture(g); + } + state = TS_IDLE; + } + break; + } +} diff --git a/firmware/src/touch.h b/firmware/src/touch.h new file mode 100644 index 0000000..32fd4a6 --- /dev/null +++ b/firmware/src/touch.h @@ -0,0 +1,5 @@ +#pragma once +#include "hid.h" + +void touch_init(void); +void touch_tick(void); diff --git a/flash.sh b/flash.sh new file mode 100755 index 0000000..cc3517b --- /dev/null +++ b/flash.sh @@ -0,0 +1,44 @@ +#!/bin/bash +# Flash helper for TinyUSB mode firmware. +# The built-in JTAG bootloader doesn't auto-reset with TinyUSB, so we +# trigger a reset via DTR/RTS on the CDC serial port before flashing. +set -e + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +PORT="${1:-/dev/ttyACM0}" + +echo "=== Flashing Claude Usage Tracker ===" +echo "Port: $PORT" +echo "" + +# Build first +echo "[1/3] Building firmware..." +cd "$SCRIPT_DIR/firmware" +~/.platformio/penv/bin/pio run + +# Reset the device into bootloader via DTR/RTS +echo "[2/3] Resetting device into bootloader..." +python3 -c " +import serial, time +port = serial.Serial('$PORT', 115200) +port.dtr = False; port.rts = True; time.sleep(0.1) +port.dtr = True; port.rts = False; time.sleep(0.05) +port.dtr = False; time.sleep(0.1); port.close() +" +sleep 0.5 + +# Flash with no_reset (device is already in bootloader) +echo "[3/3] Flashing..." +python3 ~/.platformio/packages/tool-esptoolpy/esptool.py \ + --chip esp32s3 \ + --port "$PORT" \ + --before no_reset \ + --baud 921600 \ + write_flash -z \ + 0x0 .pio/build/sc01plus/bootloader.bin \ + 0x8000 .pio/build/sc01plus/partitions.bin \ + 0xe000 ~/.platformio/packages/framework-arduinoespressif32/tools/partitions/boot_app0.bin \ + 0x10000 .pio/build/sc01plus/firmware.bin + +echo "" +echo "=== Done! Device will reboot automatically. ==="