Feat/screenpipe wizard

AGENTS.md

@@ -602,11 +602,21 @@ For detailed technical documentation, see:
 - **[@docs/init-system.md](docs/init-system.md)**: Initialization system and service management
 - **[@docs/ssl-certificates.md](docs/ssl-certificates.md)**: HTTPS/SSL setup details
 - **[@docs/podman.md](docs/podman.md)**: Running with Podman instead of Docker (engine selection, rootless/GPU setup)
+- **[@docs/screenpipe.md](docs/screenpipe.md)**: ScreenPipe capture-node architecture, services, desktop controls, and troubleshooting
 - **[@docs/audio-pipeline-architecture.md](docs/audio-pipeline-architecture.md)**: Audio pipeline design
 - **[@docs/backend/auth.md](docs/backend/auth.md)**: Authentication architecture
 - **[@docs/backend/memories.md](docs/backend/memories.md)**: Memory system documentation
 - **[@docs/backend/plugin-development-guide.md](docs/backend/plugin-development-guide.md)**: Plugin development guide
 
+### ScreenPipe Capture Nodes
+
+Before changing ScreenPipe ingestion, the desktop tray, or capture-node services, read
+[@docs/screenpipe.md](docs/screenpipe.md). ScreenPipe owns the high-volume local capture
+store; Chronicle's companion sends compact activity metadata and serves bounded
+snapshot/OCR requests. The desktop entry point is shared across macOS and Linux, with
+platform UI adapters over common state, logging, and vault-sync code. The ScreenPipe UI
+is an optional on-demand viewer and must not be required for background capture.
+
 ## Robot Framework Testing
 
 **IMPORTANT: When writing or modifying Robot Framework tests, you MUST follow the testing guidelines.**

backends/advanced/src/advanced_omi_backend/services/device_audio_ingest.py

@@ -48,7 +48,18 @@ def group_audio_sessions(items: list[DeviceInputItem]) -> list[list[DeviceInputI
     return sessions
 
 
-async def _mix_session(items: list[DeviceInputItem], workspace: Path, output: Path) -> None:
+def audio_stream_key(item: DeviceInputItem) -> tuple[str, str, str]:
+    """Keep microphone and system output in independent processing streams."""
+    return (
+        item.user_id,
+        item.source_id,
+        str(item.metadata.get("direction", "unknown")),
+    )
+
+
+async def _mix_session(
+    items: list[DeviceInputItem], workspace: Path, output: Path
+) -> None:
     start = min(_as_utc(item.captured_at) for item in items)
     command = ["ffmpeg", "-hide_banner", "-loglevel", "error", "-y"]
     valid = [item for item in items if item.media_data]
@@ -107,11 +118,11 @@ async def process_device_audio() -> dict[str, Any]:
         .sort([("source_id", 1), ("captured_at", 1)])
         .to_list()
     )
-    by_source: dict[tuple[str, str], list[DeviceInputItem]] = {}
+    by_source: dict[tuple[str, str, str], list[DeviceInputItem]] = {}
     for item in pending:
-        by_source.setdefault((item.user_id, item.source_id), []).append(item)
+        by_source.setdefault(audio_stream_key(item), []).append(item)
     processed = 0
-    for (user_id, source_id), source_items in by_source.items():
+    for (user_id, source_id, direction), source_items in by_source.items():
         try:
             user = await User.get(PydanticObjectId(user_id))
         except Exception:
@@ -136,7 +147,7 @@ async def process_device_audio() -> dict[str, Any]:
                     result = await upload_and_process_audio_files(
                         user,
                         [UploadFile(file=handle, filename=output.name)],
-                        device_name=source_id,
+                        device_name=f"{source_id}-{direction}",
                         source="screenpipe",
                     )
             if (
@@ -153,7 +164,7 @@ async def process_device_audio() -> dict[str, Any]:
                 conversation.created_at = min(
                     _as_utc(item.captured_at) for item in session
                 )
-                conversation.external_source_id = f"screenpipe:{source_id}:{session[0].source_item_id}-{session[-1].source_item_id}"
+                conversation.external_source_id = f"screenpipe:{source_id}:{direction}:{session[0].source_item_id}-{session[-1].source_item_id}"
                 conversation.external_source_type = "screenpipe"
                 await conversation.save()
             for item in session:

backends/advanced/tests/test_device_audio_ingest.py

@@ -1,7 +1,10 @@
 from datetime import datetime, timedelta, timezone
 from types import SimpleNamespace
 
-from advanced_omi_backend.services.device_audio_ingest import group_audio_sessions
+from advanced_omi_backend.services.device_audio_ingest import (
+    audio_stream_key,
+    group_audio_sessions,
+)
 
 
 def item(identifier: str, start: datetime, duration: float = 30):
@@ -12,6 +15,17 @@ def item(identifier: str, start: datetime, duration: float = 30):
     )
 
 
+def test_audio_stream_key_separates_microphone_from_system_output():
+    microphone = SimpleNamespace(
+        user_id="user", source_id="rainbow", metadata={"direction": "input"}
+    )
+    system = SimpleNamespace(
+        user_id="user", source_id="rainbow", metadata={"direction": "output"}
+    )
+
+    assert audio_stream_key(microphone) != audio_stream_key(system)
+
+
 def test_audio_chunks_group_across_input_and_output_devices():
     start = datetime(2026, 7, 22, tzinfo=timezone.utc)
     sessions = group_audio_sessions(

docs/README.md

@@ -10,6 +10,7 @@ day-to-day operation, and use [AGENTS.md](../AGENTS.md) for development conventi
 - [Testing and coverage](testing.md): fast Python lanes, coverage reports, and integration tests
 - [Audio pipeline](audio-pipeline-architecture.md): session, transcription, and memory flow
 - [Initialization system](init-system.md): setup wizard and service orchestration
+- [ScreenPipe capture nodes](screenpipe.md): local capture, Chronicle ingestion, desktop controls, and logs
 - [Podman](podman.md): rootless containers, GPU access, and engine migration
 - [SSL certificates](ssl-certificates.md): HTTPS setup and certificate trust
 

docs/screenpipe.md

@@ -0,0 +1,73 @@
+# ScreenPipe capture nodes
+
+ScreenPipe is Chronicle's local, cross-platform screen/activity capture interface. It
+owns the high-volume frame, OCR, accessibility, and optional audio store on each
+computer. Chronicle does not mirror that entire store: the companion collector sends
+compact application/window transitions and can fulfill bounded snapshot/OCR requests
+for the timeline.
+
+## Component boundaries
+
+- `screenpipe record` captures and retains source data locally.
+- `extras/screenpipe-collector/` pairs the node with Chronicle, uploads activity
+  metadata, checkpoints progress, and answers bounded media requests.
+- Chronicle's backend stores timeline activity and only the media selected for a
+  durable Chronicle view or memory.
+- `extras/vault-sync/` is the single Chronicle desktop entry point. `main.py` selects
+  the macOS menu bar or Linux system tray adapter; `desktop_core.py` contains their
+  shared state, logging, and vault synchronization.
+- ScreenPipe's own desktop UI is an optional, on-demand local timeline viewer. It is
+  not the recorder and should not autostart or launch a second recording process.
+
+Do not scan or upload the complete frame stream, and do not copy ScreenPipe's SQLite
+database into Chronicle as a second source of truth.
+
+## Capture mode
+
+Microphone and system-output capture are separate sources. A capture node may record
+neither, system output only, microphone only, or both. Use
+`--audio-transcription-engine disabled` in every enabled mode because Chronicle owns
+speech-to-text. ScreenPipe's `--use-system-default-audio true` follows and enrolls both
+the default input and output; system-only or microphone-only modes therefore require
+an explicit `--audio-device "... (output)"` or `--audio-device "... (input)"` with
+`--use-system-default-audio false`. The collector preserves the source direction when
+it sends audio to Chronicle.
+
+Local capture and Chronicle forwarding are independent policies. The collector's
+`forward_audio` setting accepts `none`, `output`, `input`, or `both`; excluded chunks
+remain in ScreenPipe's local store and are checkpointed without upload. Chronicle also
+processes input and output as separate sessions so microphone and system audio are not
+mixed together before transcription.
+
+The setup wizard's capture-node option delegates to
+`extras/screenpipe-collector/init.py`. Pairing and standalone commands are documented
+in the [companion README](../extras/screenpipe-collector/README.md).
+
+## Desktop controls and logs
+
+On Linux the Chronicle tray controls `screenpipe.service` and
+`chronicle-screenpipe.service`, shows local frame/storage statistics, toggles capture
+modes, and offers timed pauses. Both Linux and macOS expose **View Logs** for the
+current Chronicle desktop process. System-service history remains in the native
+service manager:
+
+```bash
+systemctl --user status screenpipe.service chronicle-screenpipe.service chronicle-desktop.service
+journalctl --user -u screenpipe.service -u chronicle-screenpipe.service -u chronicle-desktop.service
+```
+
+On macOS, use `extras/vault-sync/start.sh logs` for the installed desktop service.
+
+The Linux ScreenPipe UI launcher may set rendering/onboarding environment required by
+the locally installed build, but its desktop autostart entry should remain disabled.
+Open that UI manually only when the full local ScreenPipe timeline is needed; the
+recorder and Chronicle collector remain independently managed background services.
+
+## Implementation map
+
+- `extras/screenpipe-collector/`: collector CLI, local client, checkpoints, and service installation
+- `backends/advanced/src/routers/device_activity.py`: capture-node ingestion API
+- `backends/advanced/src/services/device_activity_service.py`: activity persistence and media requests
+- `extras/vault-sync/main.py`: cross-platform desktop entry point
+- `extras/vault-sync/desktop_core.py`: shared desktop state, logs, and vault sync
+- `extras/vault-sync/menu_linux.py` and `menu_vault.py`: platform-specific UI adapters

extras/screenpipe-collector/README.md

@@ -1,6 +1,9 @@
 # Chronicle ScreenPipe companion
 
-This service keeps ScreenPipe as the local capture store while forwarding completed audio chunks and compact application/window transitions to Chronicle. Screen pixels and OCR are retrieved only for bounded jobs requested by Chronicle.
+This service keeps ScreenPipe as the local capture store while forwarding compact
+application/window transitions and, only when enabled, completed audio chunks to
+Chronicle. Screen pixels and OCR are retrieved only for bounded jobs requested by
+Chronicle. See the full [capture-node architecture](../../docs/screenpipe.md).
 
 ## Pair and run
 
@@ -13,7 +16,9 @@ This service keeps ScreenPipe as the local capture store while forwarding comple
      --code PAIRING_CODE
    ```
 
-3. Start ScreenPipe with Chronicle's privacy-oriented defaults:
+3. Start ScreenPipe with Chronicle's privacy-oriented defaults. The example records
+   both the default microphone and system output while leaving transcription to
+   Chronicle:
 
    ```bash
    screenpipe record --audio-transcription-engine disabled \
@@ -30,6 +35,16 @@ This service keeps ScreenPipe as the local capture store while forwarding comple
    Set `SCREENPIPE_API_KEY` for both ScreenPipe and the pairing command so the
    companion can authenticate bounded local OCR queries.
 
+   For system audio without the microphone, replace
+   `--use-system-default-audio true` with
+   `--use-system-default-audio false --audio-device "DEVICE (output)"`. Discover the
+   exact platform device names with `screenpipe audio list --output json`. Use
+   `--disable-audio` only for screen-only capture.
+
+   Pair with `--forward-audio none|output|input|both` to independently control which
+   locally recorded sources are uploaded. The guided setup asks for both the local
+   capture mode and forwarding mode.
+
 4. Run the companion:
 
    ```bash

extras/screenpipe-collector/chronicle_screenpipe/collector.py

@@ -25,6 +25,7 @@ class Config:
     screenpipe_dir: Path
     screenpipe_url: str = "http://127.0.0.1:3030"
     screenpipe_token: str | None = None
+    forward_audio: str = "both"
     poll_seconds: float = 5.0
     activity_debounce_seconds: float = 10.0
 
@@ -245,6 +246,13 @@ def collect_audio(self, connection: sqlite3.Connection) -> int:
         sent = 0
         for row in rows:
             path = Path(row["file_path"])
+            direction = infer_audio_direction(str(path))
+            if self.config.forward_audio == "none" or (
+                self.config.forward_audio != "both"
+                and direction != self.config.forward_audio
+            ):
+                self.checkpoints.set("audio", row["id"])
+                continue
             if not path.is_file():
                 captured = timestamp_seconds(iso_timestamp(row["timestamp"]))
                 if time.time() - captured < 120:
@@ -284,7 +292,7 @@ def collect_audio(self, connection: sqlite3.Connection) -> int:
                         "captured_at": iso_timestamp(row["timestamp"]),
                         "duration_seconds": str(audio_duration(path)),
                         "device_name": path.stem,
-                        "direction": infer_audio_direction(str(path)),
+                        "direction": direction,
                         "content_hash": digest,
                     },
                     files={"file": (path.name, handle, content_type)},

extras/screenpipe-collector/chronicle_screenpipe/main.py

@@ -54,6 +54,7 @@ def pair(args: argparse.Namespace) -> None:
                 "screenpipe_dir": str(Path(args.screenpipe_dir).expanduser()),
                 "screenpipe_url": args.screenpipe_url,
                 "screenpipe_token": args.screenpipe_token,
+                "forward_audio": args.forward_audio,
             },
             indent=2,
         ),
@@ -99,6 +100,12 @@ def main() -> None:
         default=os.getenv("SCREENPIPE_API_KEY"),
         help="token used by ScreenPipe's authenticated local API",
     )
+    pair_parser.add_argument(
+        "--forward-audio",
+        choices=("none", "output", "input", "both"),
+        default="both",
+        help="which locally captured ScreenPipe audio sources Chronicle receives",
+    )
     sub.add_parser("run")
     sub.add_parser("install-service")
     args = parser.parse_args()

extras/screenpipe-collector/init.py

@@ -4,8 +4,10 @@
 from __future__ import annotations
 
 import argparse
+import json
 import os
 import secrets
+import shlex
 import shutil
 import subprocess
 from pathlib import Path
@@ -23,14 +25,37 @@ def screenpipe_command() -> str | None:
     return shutil.which("screenpipe")
 
 
-def write_screenpipe_unit(binary: str, api_key: str) -> Path:
+def list_audio_devices(binary: str) -> list[str]:
+    result = subprocess.run(
+        [binary, "audio", "list", "--output", "json"],
+        capture_output=True,
+        text=True,
+        check=True,
+    )
+    return [entry["name"] for entry in json.loads(result.stdout)["data"]]
+
+
+def audio_arguments(mode: str, devices: list[str]) -> list[str]:
+    if mode == "off":
+        return ["--disable-audio"]
+    if mode == "both":
+        return ["--use-system-default-audio", "true"]
+    suffix = "(output)" if mode == "system" else "(input)"
+    matches = [device for device in devices if device.lower().endswith(suffix)]
+    if not matches:
+        raise ValueError(f"no {mode} audio device is available")
+    return ["--use-system-default-audio", "false", "--audio-device", matches[0]]
+
+
+def write_screenpipe_unit(
+    binary: str, api_key: str, audio_mode: str = "both", devices: list[str] | None = None
+) -> Path:
     SYSTEMD_USER_DIR.mkdir(parents=True, exist_ok=True)
     path = SYSTEMD_USER_DIR / "screenpipe.service"
     args = [
         binary,
         "record",
         "--audio-transcription-engine", "disabled",
-        "--use-system-default-audio", "true",
         "--use-all-monitors", "true",
         "--use-pii-removal", "true",
         "--disable-keyboard-capture",
@@ -44,11 +69,12 @@ def write_screenpipe_unit(binary: str, api_key: str) -> Path:
         "--retention-mode", "media",
         "--api-auth", "true",
     ]
+    args.extend(audio_arguments(audio_mode, devices or []))
     path.write_text(
         "[Unit]\nDescription=ScreenPipe local recorder for Chronicle\n"
         "After=graphical-session.target\n\n[Service]\nType=simple\n"
         f"Environment=SCREENPIPE_API_KEY={api_key}\n"
-        f"ExecStart={' '.join(args)}\nRestart=on-failure\nRestartSec=5\n\n"
+        f"ExecStart={shlex.join(args)}\nRestart=on-failure\nRestartSec=5\n\n"
         "[Install]\nWantedBy=default.target\n",
         encoding="utf-8",
     )
@@ -76,6 +102,24 @@ def main() -> None:
     console.print(f"[green]✅[/green] ScreenPipe detected: [cyan]{binary}[/cyan]")
 
     backend = Prompt.ask("Chronicle backend URL", default=args.backend or "http://127.0.0.1:8000")
+    devices = list_audio_devices(binary)
+    audio_mode = Prompt.ask(
+        "Local audio capture",
+        choices=("off", "system", "mic", "both"),
+        default="system",
+    )
+    forward_default = (
+        "none"
+        if audio_mode == "off"
+        else "output"
+        if audio_mode == "system"
+        else audio_mode.replace("mic", "input")
+    )
+    forward_audio = Prompt.ask(
+        "Audio sent to Chronicle",
+        choices=("none", "output", "input", "both"),
+        default=forward_default,
+    )
     console.print(
         "Open Chronicle → Timeline → Sources and create a pairing code. "
         "The code expires after 10 minutes."
@@ -92,8 +136,9 @@ def main() -> None:
         "--screenpipe-dir", str(Path.home() / ".screenpipe"),
         "--screenpipe-url", "http://127.0.0.1:3030",
         "--screenpipe-token", api_key,
+        "--forward-audio", forward_audio,
     )
-    write_screenpipe_unit(binary, api_key)
+    write_screenpipe_unit(binary, api_key, audio_mode, devices)
     run("uv", "run", "--project", str(PROJECT), "chronicle-screenpipe", "install-service")
     run("systemctl", "--user", "daemon-reload")
     run("systemctl", "--user", "enable", "--now", "screenpipe.service")

extras/screenpipe-collector/pyproject.toml

@@ -3,7 +3,7 @@ name = "chronicle-screenpipe-collector"
 version = "0.1.0"
 description = "Chronicle companion for a local ScreenPipe recorder"
 requires-python = ">=3.11"
-dependencies = ["httpx>=0.28,<1"]
+dependencies = ["httpx>=0.28,<1", "rich>=13,<15"]
 
 [project.scripts]
 chronicle-screenpipe = "chronicle_screenpipe.main:main"