feat: human-readable action logs for GUI harnesses

CHANGELOG.md

@@ -4,6 +4,20 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
+## [1.2.0] - 2026-07-19
+
+### Added
+
+- **Human-readable action logs for GUI harnesses.** Every routed press logs the physical control and its action — `△ → push-to-talk`, `right stick flick right → thinking depth up`, `R3 → send ctrl+shift+m` — replacing the raw osascript payload dump. Face buttons show PlayStation glyphs (`✕ ○ □ △`) on DualSense/DS4 and A/B/X/Y on Xbox/GameSir; the session-cycle button shows `▭` (DualSense touchpad) or `⌂` (Guide/home).
+
+### Changed
+
+- **Codex app: dictation is now true hold-to-talk.** Hold `△` to dictate, release to insert the transcript — replacing the press-once-to-start / press-again-to-stop toggle, whose internal state could desync and leave the chord held.
+
+### Fixed
+
+- **Codex app: synthetic keystrokes could leave modifiers stuck machine-wide** (Ctrl stuck → left click acts as right click, keyboard unusable until reboot). osascript chords now run strictly serialized with a timeout, a failed key-down chord immediately releases every key the harness can hold, and quitting OpenMicro (Ctrl+C/SIGTERM) releases all held keys instead of doing nothing.
+
 ## [1.1.0] - 2026-07-19
 
 ### Added

package.json

@@ -1,6 +1,6 @@
 {
   "name": "openmicro",
-  "version": "1.1.0",
+  "version": "1.2.0",
   "description": "Open-source replica of Work Louder's Codex Micro using a consumer gamepad as the physical AI-agent controller.",
   "license": "MIT",
   "author": "stephenleo",

src/cli.ts

@@ -23,7 +23,7 @@ import {
   releaseAgent,
   reportAgentState,
 } from './herdr.js'
-import type { Harness } from './harness/types.js'
+import type { Action, Harness } from './harness/types.js'
 import { parseInvocation, USAGE } from './invocation.js'
 import { loadConfig } from './layers.js'
 import type { OpenMicroConfig } from './layers.js'
@@ -37,7 +37,8 @@ import { LayerRouter } from './router.js'
 import { HostServer } from './server.js'
 import { nextFocus } from './state.js'
 import type { Aggregate } from './state.js'
-import type { ButtonId, ControllerEvent } from './types.js'
+import { actionLabel, controlLabel } from './labels.js'
+import type { ButtonId, ControllerEvent, ControllerType } from './types.js'
 
 const DEFAULT_THINKING_LEVEL = 2 // 'high' — level 2 of Claude's 5 effort steps
 const FEEDBACK_DEBOUNCE_MS = 50
@@ -151,11 +152,11 @@ const agent: Pick<AgentPty, 'write' | 'dispose'> = usesPty
     )
   : {
       write: (bytes: string) => {
-        const sep = bytes.indexOf(':')
-        if (sep > 0) guiStatus(`→ ${bytes.slice(sep + 1) || bytes.slice(0, sep)}`, 35)
         harness.execute?.(bytes)
       },
-      dispose: () => {},
+      // Exit must never strand synthetic keys held down (dictation chord) —
+      // a stuck Ctrl turns every click into a right-click machine-wide.
+      dispose: () => harness.dispose?.(),
     }
 
 if (!usesPty) {
@@ -492,9 +493,17 @@ if (!isHost) {
     scheduleFeedback()
   })
 
+  let padType: ControllerType = 'generic-hid'
+  // One status line per routed action: "triangle → push-to-talk" (physical
+  // name per pad family), replacing the raw keystroke payload dump.
+  const announce = (action: Action): void => {
+    const control = router.lastControl
+    if (control) guiStatus(`${controlLabel(control, padType)} → ${actionLabel(action)}`, 35)
+  }
   hid.on('data', (e: ControllerEvent) => {
     try {
       if (e.kind === 'connected') {
+        padType = e.controllerType
         logger.info(`Controller connected: ${e.controllerType}`)
         guiStatus(`controller connected (${e.controllerType}) — buttons now drive the app`, 32)
         scheduleFeedback()
@@ -510,11 +519,21 @@ if (!isHost) {
 
       // Held d-pad arrows auto-repeat; every other control fires once on press.
       if (e.kind === 'button' && REPEATING.has(e.button) && action.type === 'keys') {
-        if (e.pressed) repeater.press(e.button, () => dispatchAction(action, deps))
-        else repeater.release(e.button)
+        if (e.pressed) {
+          announce(action)
+          repeater.press(e.button, () => dispatchAction(action, deps))
+        } else repeater.release(e.button)
+        return
+      }
+      // GUI dictation is hold-to-talk: forward both button edges so the chord
+      // stays down exactly while the button is held (pty voice stays a toggle).
+      if (!usesPty && action.type === 'push_to_talk' && e.kind === 'button') {
+        if (e.pressed) announce(action)
+        dispatchAction({ ...action, pressed: e.pressed }, deps)
         return
       }
       if (e.kind === 'button' && !e.pressed) return // press-only for non-repeating buttons
+      announce(action)
       if (action.type === 'push_to_talk') retargetVoice()
       dispatchAction(action, deps)
     } catch (err) {

src/harness/codex-app.ts

@@ -3,7 +3,7 @@
 // resolve to tagged strings that execute() turns into `open` deep links or
 // System Events keystrokes into the frontmost Codex window.
 
-import { execFile } from 'node:child_process'
+import { execFile, execFileSync } from 'node:child_process'
 import { createRequire } from 'node:module'
 import os from 'node:os'
 import path from 'node:path'
@@ -30,8 +30,59 @@ const KEY_EQUIVALENTS: Record<string, string> = {
   '\x1b[109;6u': 'key down control\nkey down shift\nkey code 46\nkey up shift\nkey up control',
 }
 
-// Whether the dictation key chord is currently held down (see push_to_talk).
-let dictationHeld = false
+// Synthetic `key down` events are system-wide: a chord whose `key up` never
+// lands (osascript error/timeout, process death, interleaved chords) leaves
+// modifiers stuck for the whole machine — Ctrl turns clicks into right-clicks
+// until the user reboots. Every key this harness ever holds down is listed
+// here; `key up` on an already-up key is a no-op, so releasing all is safe.
+const RELEASE_ALL: string[] = [
+  'key up control',
+  'key up option',
+  'key up shift',
+  'key up command',
+  'key up "d"',
+]
+
+// Chords must run strictly one at a time — two concurrent osascripts can
+// interleave their key down/up steps and strand a modifier. execFile timeout
+// bounds a hung osascript so the queue always drains.
+let osascriptQueue: Promise<void> = Promise.resolve()
+function enqueueOsascript(
+  args: string[],
+  done: (err: Error | null, stderr?: string) => void,
+): void {
+  osascriptQueue = osascriptQueue.then(
+    () =>
+      new Promise<void>((resolve) => {
+        execFile('osascript', args, { timeout: 5000 }, (err, _stdout, stderr) => {
+          done(err, stderr)
+          resolve()
+        })
+      }),
+  )
+}
+
+/**
+ * Synchronously release every key this harness can hold down.
+ *
+ * Args:
+ *     None.
+ *
+ * Returns:
+ *     None.
+ */
+function releaseAllKeys(): void {
+  try {
+    execFileSync(
+      'osascript',
+      RELEASE_ALL.flatMap((step) => ['-e', `tell application "System Events" to ${step}`]),
+      { timeout: 3000 },
+    )
+  } catch {
+    // Best-effort: no Accessibility permission (nothing was ever held) or a
+    // hung System Events — nothing more we can do from here.
+  }
+}
 
 // ── Desktop thread/project cycling ──────────────────────────────────────────
 // The app has Next/Previous Chat shortcuts but they stop at the list ends and
@@ -198,18 +249,13 @@ export const codexAppHarness: Harness = {
         return { bytes: 'osascript:keystroke return' }
       case 'push_to_talk':
         // Ctrl+Shift+D = the app's composer.startDictation binding, and it is
-        // hold-to-dictate: dictation runs only while the keys stay down, so an
-        // instantaneous press+release starts and stops it in the same moment.
-        // Emulate the hold — first press holds the keys down, second press
-        // releases them (verified live: mic engages while held, transcript
-        // inserts on release).
-        // ponytail: module state; if the process dies mid-hold the keys stay
-        // down until the user taps them physically. Upgrade path: plumb
-        // controller press/release through dispatch for true push-to-talk.
-        dictationHeld = !dictationHeld
-        return dictationHeld
-          ? { bytes: 'osascript:key down control\nkey down shift\nkey down "d"' }
-          : { bytes: 'osascript:key up "d"\nkey up shift\nkey up control' }
+        // hold-to-dictate: mic engages while the keys stay down, transcript
+        // inserts on release. The cli forwards the controller edge in
+        // action.pressed — button down holds the chord, button up releases it
+        // — so dictation runs exactly while the button is held.
+        return action.pressed === false
+          ? { bytes: 'osascript:key up "d"\nkey up shift\nkey up control' }
+          : { bytes: 'osascript:key down control\nkey down shift\nkey down "d"' }
       case 'new_chat':
         // codex://threads/new, not codex://new — the app's deep-link parser
         // drops a bare codex://new (it requires a prompt/path/originUrl query
@@ -284,11 +330,20 @@ export const codexAppHarness: Harness = {
         .flatMap((step) => ['-e', `tell application "System Events" to ${step}`])
       // The short delay lets activation land before the keystroke — without it
       // a keypress sent while Codex is still coming frontmost is dropped.
-      execFile(
-        'osascript',
+      enqueueOsascript(
         ['-e', 'tell application "Codex" to activate', '-e', 'delay 0.15', ...steps],
-        (err, _stdout, stderr) => report(err, stderr),
+        (err, stderr) => {
+          report(err, stderr)
+          // A failed/timed-out chord may have landed its `key down` steps but
+          // not the matching `key up` — release everything rather than leave
+          // the whole machine with a stuck modifier.
+          if (err && payload.includes('key down')) releaseAllKeys()
+        },
       )
     }
   },
+
+  dispose(): void {
+    releaseAllKeys()
+  },
 }

src/harness/types.ts

@@ -9,7 +9,7 @@ export type AgentState = 'executing' | 'waiting' | 'idle' | 'complete' | 'error'
 export type Action =
   | { type: 'accept' }
   | { type: 'reject' }
-  | { type: 'push_to_talk' }
+  | { type: 'push_to_talk'; pressed?: boolean } // pressed set by the cli for GUI hold-to-talk (true = button down, false = up); absent for pty toggle harnesses
   | { type: 'new_chat' }
   | { type: 'thinking_depth'; delta: 1 | -1 }
   | { type: 'workflow'; presetId: string } // core resolves presetId → text via config, then calls resolveAction({type:'prompt', text})
@@ -40,4 +40,6 @@ export interface Harness {
   ): { bytes: string; thinkingLevel?: number } | null
   /** Side-effect runner for usesPty:false harnesses — the cli calls it with resolveAction's bytes in place of a pty write. */
   execute?(bytes: string): void
+  /** Shutdown cleanup for usesPty:false harnesses — must undo any lingering system side effects (e.g. release synthetic keys still held down). */
+  dispose?(): void
 }