diff --git a/.agents/skills b/.agents/skills new file mode 120000 index 00000000..42c5394a --- /dev/null +++ b/.agents/skills @@ -0,0 +1 @@ +../skills \ No newline at end of file diff --git a/.assets/menu-bar-client.png b/.assets/menu-bar-client.png new file mode 100644 index 00000000..1cf2d4c0 Binary files /dev/null and b/.assets/menu-bar-client.png differ diff --git a/.claude/skills b/.claude/skills new file mode 120000 index 00000000..42c5394a --- /dev/null +++ b/.claude/skills @@ -0,0 +1 @@ +../skills \ No newline at end of file diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 00000000..5b30b17e --- /dev/null +++ b/.coveragerc @@ -0,0 +1,22 @@ +[run] +branch = True +relative_files = True +source = + config_manager + discovery + services + setup_utils + status + updates + wizard + +[report] +precision = 1 +show_missing = True +skip_empty = True + +[xml] +output = coverage-reports/coverage.xml + +[html] +directory = coverage-reports/html diff --git a/.easignore b/.easignore new file mode 100644 index 00000000..a5360919 --- /dev/null +++ b/.easignore @@ -0,0 +1,73 @@ +# .easignore at repo root — used by EAS because this is a monorepo and +# the git root is the upload root. Must cover both the monorepo top-level +# and the app/ subdir (EAS ignores .gitignore when this file exists). + +# --- Sibling projects not needed for the Expo build --- +backends/ +extras/ +tests/ +docs/ +sdk/ +untracked/ + +# --- Root-level junk / non-code artifacts --- +*.log +*.env* +asc-api-key.p8 +**/asc-api-key.p8 +*.m4a +*.wav +plan.md +sample-voice-response.json +deepgram_response.json +init-feedback +*.ipa +*.apk +*.aab +connection-logging-*.md +memory-service-settings.md +docs-consolidation-analysis.md +BLE_OPTIMIZATION.md +.github/ +.cursor/ +.claude/ + +# --- Inside app/ (the EAS project) --- +# Dependencies — reinstalled on EAS build server +app/node_modules/ +# Expo / Metro +app/.expo/ +app/dist/ +app/web-build/ +app/.metro-health-check* +# Native build outputs — regenerated on EAS +app/android/.gradle/ +app/android/build/ +app/android/app/build/ +app/android/app/.cxx/ +app/ios/build/ +app/ios/Pods/ +app/ios/DerivedData/ +app/ios/*.xcworkspace/xcuserdata/ +app/ios/*.xcodeproj/xcuserdata/ +app/ios/*.xcodeproj/project.xcworkspace/xcuserdata/ +# Local artifacts +app/*.ipa +app/*.apk +app/*.aab +app/build-*.ipa +# Credentials (EAS uses server-side credentials) +app/*.jks +app/*.p8 +app/*.p12 +app/*.key +app/*.mobileprovision +app/*.pem +# Logs +app/npm-debug.* +app/yarn-debug.* +app/yarn-error.* + +# --- OS / editor --- +.DS_Store +**/.DS_Store diff --git a/.env.template b/.env.template index 388edbf5..5f313dc3 100644 --- a/.env.template +++ b/.env.template @@ -32,7 +32,6 @@ BACKEND_PORT=8000 WEBUI_PORT=5173 SPEAKER_PORT=8085 MONGODB_PORT=27017 -QDRANT_PORT=6333 NGROK_PORT=4040 # Kubernetes node ports (for LoadBalancer services) @@ -63,8 +62,18 @@ AUTH_SECRET_KEY=your-super-secret-jwt-key-here-make-it-random-and-long ADMIN_EMAIL=admin@example.com ADMIN_PASSWORD=secure-admin-password +# Native client login (vault sync and other client-node integrations). +# These may use a non-admin Chronicle account. +AUTH_USERNAME=admin@example.com +AUTH_PASSWORD= + +# Vault sync client settings +LOCAL_VAULT_DIR=~/ChronicleVault +# DEVICE_NAME=my-macbook +# VAULT_SYNC_GUI_PORT=8385 + # CORS origins (auto-generated based on DOMAIN and ports) -CORS_ORIGINS=http://${DOMAIN}:${WEBUI_PORT},http://${DOMAIN}:3000,http://localhost:${WEBUI_PORT},http://localhost:3000 +CORS_ORIGINS=http://${DOMAIN}:${WEBUI_PORT},http://localhost:${WEBUI_PORT} # ======================================== # LLM CONFIGURATION @@ -107,28 +116,13 @@ PARAKEET_ASR_URL=http://host.docker.internal:8767 MONGODB_URI=mongodb://mongo:${MONGODB_PORT} MONGODB_K8S_URI=mongodb://mongodb.${INFRASTRUCTURE_NAMESPACE}.svc.cluster.local:27017/chronicle -# Qdrant configuration -QDRANT_BASE_URL=qdrant -QDRANT_K8S_URL=qdrant.${INFRASTRUCTURE_NAMESPACE}.svc.cluster.local - -# Neo4j configuration (optional) -NEO4J_HOST=neo4j-mem0 -NEO4J_USER=neo4j -NEO4J_PASSWORD=neo4j-password - # ======================================== # MEMORY PROVIDER CONFIGURATION # ======================================== -# Memory Provider: chronicle or openmemory_mcp +# Memory Provider: chronicle (agentic Markdown vault) — the only provider MEMORY_PROVIDER=chronicle -# OpenMemory MCP configuration (when MEMORY_PROVIDER=openmemory_mcp) -OPENMEMORY_MCP_URL=http://host.docker.internal:8765 -OPENMEMORY_CLIENT_NAME=chronicle -OPENMEMORY_USER_ID=openmemory -OPENMEMORY_TIMEOUT=30 - # ======================================== # SPEAKER RECOGNITION CONFIGURATION # ======================================== @@ -218,4 +212,4 @@ WEBUI_MEMORY_REQUEST=128Mi SPEAKER_CPU_LIMIT=2000m SPEAKER_MEMORY_LIMIT=4Gi SPEAKER_CPU_REQUEST=500m -SPEAKER_MEMORY_REQUEST=2Gi \ No newline at end of file +SPEAKER_MEMORY_REQUEST=2Gi diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 0b8987c5..a9bbc1a9 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -6,11 +6,12 @@ Documentation for CI/CD workflows and test automation. Chronicle uses **three separate test workflows** to balance fast PR feedback with comprehensive testing: -| Workflow | Trigger | Test Coverage | API Keys | Purpose | +| Workflow | Trigger | Test selection | API Keys | Purpose | |----------|---------|---------------|----------|---------| -| `robot-tests.yml` | All PRs | ~70% (no-API tests) | ❌ Not required | Fast PR validation | -| `full-tests-with-api.yml` | Push to dev/main | 100% (full suite) | ✅ Required | Comprehensive validation | -| `pr-tests-with-api.yml` | PR label trigger | 100% (full suite) | ✅ Required | Pre-merge API testing | +| `python-tests.yml` | Relevant PRs, dev/main | Root, backend, and ASR pytest lanes | Not required | Unit tests and branch coverage reports | +| `robot-tests.yml` | All PRs | No-API Robot subset | Not required | Fast PR validation | +| `full-tests-with-api.yml` | Push to dev/main | Full Robot selection | Required | Comprehensive validation | +| `pr-tests-with-api.yml` | PR label trigger | Full Robot selection | Required | Pre-merge API testing | ## Workflow Details @@ -32,10 +33,10 @@ on: - **No secrets required** - Works for external contributors - **Excludes**: Tests tagged with `requires-api-keys` - **Config**: `tests/configs/mock-services.yml` -- **Test Script**: `./run-no-api-tests.sh` +- **Makefile Target**: `make test-no-api OUTPUTDIR=results-no-api` - **Results**: `results-no-api/` - **Time**: ~10-15 minutes -- **Coverage**: ~70% of test suite +- **Selection**: Robot cases that do not require secrets, GPUs, or excluded slow/SDK environments **Benefits**: - Fast feedback on PRs @@ -134,7 +135,7 @@ if: contains(github.event.pull_request.labels.*.name, 'test-with-api-keys') **Normal PR Workflow**: 1. Push your branch 2. Create PR -3. `robot-tests.yml` runs automatically (~70% coverage) +3. `robot-tests.yml` runs the no-API Robot selection automatically 4. Fix any failures 5. Merge when tests pass @@ -142,7 +143,7 @@ if: contains(github.event.pull_request.labels.*.name, 'test-with-api-keys') 1. Push your branch 2. Create PR 3. Ask maintainer to add `test-with-api-keys` label -4. `pr-tests-with-api.yml` runs (100% coverage) +4. `pr-tests-with-api.yml` runs the full Robot selection 5. Fix any failures 6. Merge when tests pass @@ -312,7 +313,7 @@ runs-on: ubuntu-latest CLEANUP_CONTAINERS: "false" # Handled by workflow # API keys if needed run: | - ./run-{no-api|robot}-tests.sh + make test-no-api # or ./run-robot-tests.sh for full suite TEST_EXIT_CODE=$? echo "test_exit_code=$TEST_EXIT_CODE" >> $GITHUB_ENV exit 0 # Don't fail yet diff --git a/.github/workflows/advanced-docker-compose-build.yml b/.github/workflows/advanced-docker-compose-build.yml index 93e72d68..f04088d5 100644 --- a/.github/workflows/advanced-docker-compose-build.yml +++ b/.github/workflows/advanced-docker-compose-build.yml @@ -13,7 +13,6 @@ on: - "backends/advanced/**" - "extras/asr-services/**" - "extras/speaker-recognition/**" - - "extras/openmemory-mcp/**" - ".github/workflows/advanced-docker-compose-build.yml" release: types: [ published ] @@ -28,27 +27,55 @@ env: REGISTRY: ghcr.io jobs: - build-default: + build-images: + name: Build ${{ matrix.image }} runs-on: ubuntu-latest timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + include: + - image: chronicle-backend + service: chronicle-backend + source-image: chronicle-backend:latest + directory: backends/advanced + variant: "" + - image: chronicle-asr-nemo-cu126 + service: parakeet-asr + source-image: parakeet-asr:latest + directory: extras/asr-services + variant: cu126 + - image: chronicle-asr-nemo-cu128 + service: parakeet-asr + source-image: parakeet-asr:latest + directory: extras/asr-services + variant: cu128 + - image: chronicle-speaker-cpu + service: speaker-service + source-image: chronicle-speaker:latest + directory: extras/speaker-recognition + variant: cpu + - image: chronicle-speaker-cu126 + service: speaker-service + source-image: chronicle-speaker:latest + directory: extras/speaker-recognition + variant: cu126 + - image: chronicle-speaker-cu128 + service: speaker-service + source-image: chronicle-speaker:latest + directory: extras/speaker-recognition + variant: cu128 env: ADVANCED_ENV: ${{ secrets.ADVANCED_ENV }} - RUNNER_FLAVOUR: ubuntu-latest - defaults: - run: - shell: bash - working-directory: backends/advanced steps: - - name: Show selected runner - run: echo "Workflow running on ${RUNNER_FLAVOUR} runner" - working-directory: . - - name: Checkout uses: actions/checkout@v4 - name: Print commit details + working-directory: ${{ matrix.directory }} run: | + echo "Image: ${{ matrix.image }}" echo "Event: ${{ github.event_name }}" echo "Ref: $GITHUB_REF" echo "Ref name: ${{ github.ref_name }}" @@ -69,44 +96,25 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Copy .env.template to .env + - name: Copy .env template + working-directory: ${{ matrix.directory }} run: | set -euo pipefail - copy_env() { - local dir="$1" - local template="${dir}/.env.template" - local target="${dir}/.env" - if [ -f "$template" ]; then - echo "Copying $template to $target" - cp "$template" "$target" - else - echo "$template not found; skipping" - fi - } - - copy_env . - copy_env ../../extras/asr-services - copy_env ../../extras/speaker-recognition - copy_env ../../extras/openmemory-mcp + if [ -f .env.template ]; then + cp .env.template .env + else + touch .env + fi - name: Create .env from secret (if provided) - if: env.ADVANCED_ENV != '' + if: matrix.image == 'chronicle-backend' && env.ADVANCED_ENV != '' + working-directory: ${{ matrix.directory }} run: | echo "Writing .env from ADVANCED_ENV secret" printf "%s\n" "${ADVANCED_ENV}" > .env - - name: Source .env (if present) - run: | - if [ -f .env ]; then - set -a - # shellcheck disable=SC1091 - source .env - set +a - else - echo ".env not found; continuing" - fi - - name: Free Disk Space + working-directory: ${{ matrix.directory }} run: | echo "Freeing disk space..." df -h @@ -119,6 +127,7 @@ jobs: - name: Determine version id: version + working-directory: ${{ matrix.directory }} run: | if [ -n "${{ github.event.inputs.version }}" ]; then VERSION="${{ github.event.inputs.version }}" @@ -132,151 +141,40 @@ jobs: echo "VERSION=$VERSION" >> "$GITHUB_OUTPUT" - - name: Build, tag, and push services sequentially with version + - name: Build, tag, and push ${{ matrix.image }} + working-directory: ${{ matrix.directory }} env: OWNER: ${{ github.repository_owner }} VERSION: ${{ steps.version.outputs.VERSION }} + CHRONICLE_BUILD_VERSION: ${{ steps.version.outputs.VERSION }} + PYTORCH_CUDA_VERSION: ${{ matrix.variant }} run: | set -euo pipefail docker compose version OWNER_LC=$(echo "$OWNER" | tr '[:upper:]' '[:lower:]') - - # CUDA variants from pyproject.toml - CUDA_VARIANTS=("cpu" "cu121" "cu126" "cu128") - - # Base services (no CUDA variants, no profiles) - base_service_specs=( - "chronicle-backend|advanced-chronicle-backend|docker-compose.yml|." - "workers|advanced-workers|docker-compose.yml|." - "webui|advanced-webui|docker-compose.yml|." - "openmemory-mcp|openmemory-mcp|../../extras/openmemory-mcp/docker-compose.yml|../../extras/openmemory-mcp" - ) - - # Build and push base services - for spec in "${base_service_specs[@]}"; do - IFS='|' read -r svc svc_repo compose_file project_dir <<< "$spec" - - echo "::group::Building and pushing $svc_repo" - if [ "$compose_file" = "docker-compose.yml" ] && [ "$project_dir" = "." ]; then - docker compose build --pull "$svc" - else - docker compose -f "$compose_file" --project-directory "$project_dir" build "$svc" - fi - # Resolve the built image ID via compose (avoids name mismatches) - if [ "$compose_file" = "docker-compose.yml" ] && [ "$project_dir" = "." ]; then - img_id=$(docker compose images -q "$svc" | head -n1) - else - img_id=$(docker compose -f "$compose_file" --project-directory "$project_dir" images -q "$svc" | head -n1) - fi - if [ -z "${img_id:-}" ]; then - echo "Skipping $svc_repo (no built image found after build)" - echo "::endgroup::" - continue - fi - - # Tag and push with version - target_image="$REGISTRY/$OWNER_LC/$svc_repo:$VERSION" - latest_image="$REGISTRY/$OWNER_LC/$svc_repo:latest" - echo "Tagging $img_id as $target_image" - docker tag "$img_id" "$target_image" - echo "Tagging $img_id as $latest_image" - docker tag "$img_id" "$latest_image" - - echo "Pushing $target_image" - docker push "$target_image" - echo "Pushing $latest_image" - docker push "$latest_image" - - # Clean up local tags - docker image rm -f "$target_image" || true - docker image rm -f "$latest_image" || true - echo "::endgroup::" - - # Aggressive cleanup to save space - docker system prune -af || true - done - - # Build and push parakeet-asr with CUDA variants (cu121, cu126, cu128) - echo "::group::Building and pushing parakeet-asr CUDA variants" - cd ../../extras/asr-services - for cuda_variant in cu121 cu126 cu128; do - echo "Building parakeet-asr-${cuda_variant}" - export PYTORCH_CUDA_VERSION="${cuda_variant}" - docker compose build parakeet-asr - - img_id=$(docker compose images -q parakeet-asr | head -n1) - if [ -n "${img_id:-}" ]; then - target_image="$REGISTRY/$OWNER_LC/parakeet-asr-${cuda_variant}:$VERSION" - latest_image="$REGISTRY/$OWNER_LC/parakeet-asr-${cuda_variant}:latest" - echo "Tagging $img_id as $target_image" - docker tag "$img_id" "$target_image" - echo "Tagging $img_id as $latest_image" - docker tag "$img_id" "$latest_image" - - echo "Pushing $target_image" - docker push "$target_image" - echo "Pushing $latest_image" - docker push "$latest_image" - - # Clean up local tags - docker image rm -f "$target_image" || true - docker image rm -f "$latest_image" || true - fi - - # Aggressive cleanup to save space - docker system prune -af || true - done - cd - > /dev/null - echo "::endgroup::" - - # Build and push speaker-recognition with all CUDA variants (including CPU) - # Note: speaker-service has profiles, but we can build it directly by setting PYTORCH_CUDA_VERSION - echo "::group::Building and pushing speaker-recognition variants" - cd ../../extras/speaker-recognition - for cuda_variant in "${CUDA_VARIANTS[@]}"; do - echo "Building speaker-recognition-${cuda_variant}" - export PYTORCH_CUDA_VERSION="${cuda_variant}" - # Build speaker-service directly (profiles only affect 'up', not 'build') - docker compose build speaker-service - - img_id=$(docker compose images -q speaker-service | head -n1) - if [ -n "${img_id:-}" ]; then - target_image="$REGISTRY/$OWNER_LC/speaker-recognition-${cuda_variant}:$VERSION" - latest_image="$REGISTRY/$OWNER_LC/speaker-recognition-${cuda_variant}:latest" - echo "Tagging $img_id as $target_image" - docker tag "$img_id" "$target_image" - echo "Tagging $img_id as $latest_image" - docker tag "$img_id" "$latest_image" - - echo "Pushing $target_image" - docker push "$target_image" - echo "Pushing $latest_image" - docker push "$latest_image" - - # Clean up local tags - docker image rm -f "$target_image" || true - docker image rm -f "$latest_image" || true - fi - - # Aggressive cleanup to save space - docker system prune -af || true - done - cd - > /dev/null - echo "::endgroup::" - - # Summary - echo "::group::Build Summary" - echo "Built and pushed images with version tag: ${VERSION}" - echo "Images pushed to: $REGISTRY/$OWNER_LC/" - echo "::endgroup::" + docker compose build --pull "${{ matrix.service }}" + img_id=$(docker image inspect "${{ matrix.source-image }}" --format '{{.Id}}') + + target_image="$REGISTRY/$OWNER_LC/${{ matrix.image }}:$VERSION" + latest_image="$REGISTRY/$OWNER_LC/${{ matrix.image }}:latest" + docker tag "$img_id" "$target_image" + docker tag "$img_id" "$latest_image" + docker push "$target_image" + docker push "$latest_image" + + update-release: + name: Update release notes + needs: build-images + if: github.event_name == 'release' || inputs.version != '' + runs-on: ubuntu-latest + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + OWNER: ${{ github.repository_owner }} + VERSION: ${{ github.event.release.tag_name || inputs.version }} + TAG_NAME: ${{ github.event.release.tag_name || inputs.version }} + steps: - name: Update release notes with Docker images - if: github.event_name == 'release' - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - OWNER: ${{ github.repository_owner }} - VERSION: ${{ steps.version.outputs.VERSION }} - TAG_NAME: ${{ github.event.release.tag_name }} run: | set -euo pipefail OWNER_LC=$(echo "$OWNER" | tr '[:upper:]' '[:lower:]') @@ -291,31 +189,26 @@ jobs: ### Core Services \`\`\`bash - docker pull ghcr.io/${OWNER_LC}/advanced-chronicle-backend:${VERSION} - docker pull ghcr.io/${OWNER_LC}/advanced-workers:${VERSION} - docker pull ghcr.io/${OWNER_LC}/advanced-webui:${VERSION} - docker pull ghcr.io/${OWNER_LC}/openmemory-mcp:${VERSION} + docker pull ghcr.io/${OWNER_LC}/chronicle-backend:${VERSION} \`\`\` ### Parakeet ASR (pick your CUDA version) \`\`\`bash - docker pull ghcr.io/${OWNER_LC}/parakeet-asr-cu121:${VERSION} - docker pull ghcr.io/${OWNER_LC}/parakeet-asr-cu126:${VERSION} - docker pull ghcr.io/${OWNER_LC}/parakeet-asr-cu128:${VERSION} + docker pull ghcr.io/${OWNER_LC}/chronicle-asr-nemo-cu126:${VERSION} + docker pull ghcr.io/${OWNER_LC}/chronicle-asr-nemo-cu128:${VERSION} \`\`\` ### Speaker Recognition (pick your variant) \`\`\`bash - docker pull ghcr.io/${OWNER_LC}/speaker-recognition-cpu:${VERSION} - docker pull ghcr.io/${OWNER_LC}/speaker-recognition-cu121:${VERSION} - docker pull ghcr.io/${OWNER_LC}/speaker-recognition-cu126:${VERSION} - docker pull ghcr.io/${OWNER_LC}/speaker-recognition-cu128:${VERSION} + docker pull ghcr.io/${OWNER_LC}/chronicle-speaker-cpu:${VERSION} + docker pull ghcr.io/${OWNER_LC}/chronicle-speaker-cu126:${VERSION} + docker pull ghcr.io/${OWNER_LC}/chronicle-speaker-cu128:${VERSION} \`\`\` EOF ) EXISTING_BODY=$(gh release view "$TAG_NAME" --json body -q '.body' --repo "$GITHUB_REPOSITORY") + EXISTING_BODY="${EXISTING_BODY%%$'\n---\n\n## Docker Images'*}" UPDATED_BODY="${EXISTING_BODY}${DOCKER_SECTION}" gh release edit "$TAG_NAME" --notes "$UPDATED_BODY" --repo "$GITHUB_REPOSITORY" echo "Release notes updated with Docker image info" - working-directory: . diff --git a/.github/workflows/android-apk-build.yml b/.github/workflows/android-apk-build.yml index 4434eb13..756da707 100644 --- a/.github/workflows/android-apk-build.yml +++ b/.github/workflows/android-apk-build.yml @@ -5,10 +5,10 @@ permissions: on: push: - branches: [main, develop] + branches: [main, dev] paths: ['app/**'] pull_request: - branches: [main] + branches: [main, dev] paths: ['app/**'] workflow_dispatch: @@ -18,11 +18,11 @@ jobs: defaults: run: working-directory: ./app - + steps: - name: Setup repo uses: actions/checkout@v4 - + - name: Setup node uses: actions/setup-node@v4.0.2 with: @@ -54,7 +54,7 @@ jobs: - name: Build Android APK run: eas build --platform android --profile local --local --output ${{ github.workspace }}/app-release.apk --non-interactive - + - name: Generate release tag id: tag run: | @@ -73,11 +73,11 @@ jobs: release_name: ${{ steps.tag.outputs.RELEASE_NAME }} body: | ## 📱 Android APK Build - + **Built from commit:** ${{ github.sha }} **Branch:** ${{ github.ref_name }} **Build time:** ${{ steps.tag.outputs.BUILD_TIME }} - + Ready to install on Android devices! draft: false prerelease: true @@ -90,4 +90,4 @@ jobs: upload_url: ${{ steps.create_release.outputs.upload_url }} asset_path: ${{ github.workspace }}/app-release.apk asset_name: friend-lite-android.apk - asset_content_type: application/vnd.android.package-archive \ No newline at end of file + asset_content_type: application/vnd.android.package-archive diff --git a/.github/workflows/build-all-platforms.yml b/.github/workflows/build-all-platforms.yml index e73e6147..9be5cb25 100644 --- a/.github/workflows/build-all-platforms.yml +++ b/.github/workflows/build-all-platforms.yml @@ -26,11 +26,11 @@ jobs: defaults: run: working-directory: ./app - + steps: - name: Setup repo uses: actions/checkout@v4 - + - name: Setup node uses: actions/setup-node@v4.0.2 with: @@ -80,14 +80,14 @@ jobs: release_name: ${{ steps.tag.outputs.RELEASE_NAME }} body: | ## 🚀 Automated Build - + **Built from commit:** ${{ github.sha }} **Branch:** ${{ github.ref_name }} **Build time:** ${{ steps.tag.outputs.BUILD_TIME }} - + ### 📱 Downloads - **Android APK**: Ready for installation on Android devices - + ### 🔧 Build Info - Built with GitHub Actions - Debug build (unsigned) @@ -112,11 +112,11 @@ jobs: defaults: run: working-directory: ./app - + steps: - name: Setup repo uses: actions/checkout@v4 - + - name: Setup node uses: actions/setup-node@v4.0.2 with: @@ -173,11 +173,11 @@ jobs: release_name: ${{ steps.ios_tag.outputs.IOS_RELEASE_NAME }} body: | ## 🍎 iOS IPA Build - + **Built from commit:** ${{ github.sha }} **Branch:** ${{ github.ref_name }} **Build time:** ${{ steps.ios_tag.outputs.IOS_BUILD_TIME }} - + For iOS Simulator testing! draft: false prerelease: true @@ -191,4 +191,4 @@ jobs: upload_url: ${{ steps.create_ios_release.outputs.upload_url }} asset_path: ${{ github.workspace }}/friend-lite-ios.ipa asset_name: friend-lite-ios.ipa - asset_content_type: application/octet-stream \ No newline at end of file + asset_content_type: application/octet-stream diff --git a/.github/workflows/claude.yml b/.github/workflows/claude.yml index ae36c007..3cf327b9 100644 --- a/.github/workflows/claude.yml +++ b/.github/workflows/claude.yml @@ -35,7 +35,7 @@ jobs: uses: anthropics/claude-code-action@v1 with: claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} - + # This is an optional setting that allows Claude to read CI results on PRs additional_permissions: | actions: read @@ -47,4 +47,3 @@ jobs: # See https://github.com/anthropics/claude-code-action/blob/main/docs/usage.md # or https://docs.anthropic.com/en/docs/claude-code/sdk#command-line for available options # claude_args: '--model claude-opus-4-1-20250805 --allowed-tools Bash(gh pr:*)' - diff --git a/.github/workflows/full-tests-with-api.yml b/.github/workflows/full-tests-with-api.yml index b5881fcd..80fbe663 100644 --- a/.github/workflows/full-tests-with-api.yml +++ b/.github/workflows/full-tests-with-api.yml @@ -111,9 +111,9 @@ jobs: CLEANUP_CONTAINERS: "false" # Don't cleanup in CI - handled by workflow run: | # Use the full test script (includes all tests with API keys) - ./run-robot-tests.sh - TEST_EXIT_CODE=$? - echo "test_exit_code=$TEST_EXIT_CODE" >> $GITHUB_ENV + TEST_EXIT_CODE=0 + ./run-robot-tests.sh || TEST_EXIT_CODE=$? + echo "test_exit_code=$TEST_EXIT_CODE" >> "$GITHUB_ENV" exit 0 # Don't fail here, we'll fail at the end after uploading artifacts - name: Save service logs to files @@ -129,7 +129,6 @@ jobs: docker compose -f docker-compose-test.yml logs workers-test > logs/workers.log 2>&1 || true docker compose -f docker-compose-test.yml logs mongo-test > logs/mongo.log 2>&1 || true docker compose -f docker-compose-test.yml logs redis-test > logs/redis.log 2>&1 || true - docker compose -f docker-compose-test.yml logs qdrant-test > logs/qdrant.log 2>&1 || true docker compose -f docker-compose-test.yml logs speaker-service-test > logs/speaker.log 2>&1 || true echo "✓ Logs saved to backends/advanced/logs/" ls -lh logs/ @@ -256,8 +255,12 @@ jobs: - name: Fail workflow if tests failed if: always() run: | - if [ "${{ env.test_exit_code }}" != "0" ]; then - echo "❌ Tests failed with exit code ${{ env.test_exit_code }}" + TEST_EXIT_CODE="${{ env.test_exit_code }}" + if [ -z "$TEST_EXIT_CODE" ]; then + echo "❌ Test step did not record an exit code; check earlier setup/test steps" + exit 1 + elif [ "$TEST_EXIT_CODE" != "0" ]; then + echo "❌ Tests failed with exit code $TEST_EXIT_CODE" exit 1 else echo "✅ All tests passed" diff --git a/.github/workflows/ios-ipa-build.yml b/.github/workflows/ios-ipa-build.yml index dbd0c5bb..28d987f7 100644 --- a/.github/workflows/ios-ipa-build.yml +++ b/.github/workflows/ios-ipa-build.yml @@ -5,10 +5,10 @@ permissions: on: push: - branches: [main, develop] + branches: [main, dev] paths: ['app/**'] pull_request: - branches: [main] + branches: [main, dev] paths: ['app/**'] workflow_dispatch: @@ -18,11 +18,11 @@ jobs: defaults: run: working-directory: ./app - + steps: - name: Setup repo uses: actions/checkout@v4 - + - name: Setup node uses: actions/setup-node@v4.0.2 with: @@ -66,11 +66,11 @@ jobs: release_name: ${{ steps.tag.outputs.RELEASE_NAME }} body: | ## 🍎 iOS IPA Build - + **Built from commit:** ${{ github.sha }} **Branch:** ${{ github.ref_name }} **Build time:** ${{ steps.tag.outputs.BUILD_TIME }} - + For iOS Simulator testing! draft: false prerelease: true @@ -83,4 +83,4 @@ jobs: upload_url: ${{ steps.create_release.outputs.upload_url }} asset_path: ${{ github.workspace }}/app-release.ipa asset_name: friend-lite-ios.ipa - asset_content_type: application/octet-stream \ No newline at end of file + asset_content_type: application/octet-stream diff --git a/.github/workflows/ios-testflight.yml b/.github/workflows/ios-testflight.yml new file mode 100644 index 00000000..8c468e35 --- /dev/null +++ b/.github/workflows/ios-testflight.yml @@ -0,0 +1,44 @@ +name: iOS TestFlight Deploy + +permissions: + contents: read + +on: + push: + branches: [main] + paths: ['app/**'] + workflow_dispatch: + +jobs: + build-and-submit: + runs-on: ubuntu-latest + defaults: + run: + working-directory: ./app + + steps: + - name: Setup repo + uses: actions/checkout@v4 + + - name: Setup node + uses: actions/setup-node@v4.0.2 + with: + node-version: 20.x + cache: 'npm' + cache-dependency-path: ./app/package-lock.json + + - name: Setup Expo + uses: expo/expo-github-action@v8 + with: + expo-version: latest + eas-version: latest + token: ${{ secrets.EXPO_TOKEN }} + + - name: Install dependencies + run: npm ci + + - name: Write ASC API Key + run: echo "${{ secrets.ASC_API_KEY_P8 }}" > asc-api-key.p8 + + - name: Build and submit to TestFlight + run: eas build --platform ios --profile testflight --auto-submit --non-interactive diff --git a/.github/workflows/pr-tests-with-api.yml b/.github/workflows/pr-tests-with-api.yml index aeb45b1c..8e4b1cd1 100644 --- a/.github/workflows/pr-tests-with-api.yml +++ b/.github/workflows/pr-tests-with-api.yml @@ -105,9 +105,9 @@ jobs: CLEANUP_CONTAINERS: "false" # Don't cleanup in CI - handled by workflow run: | # Use the full test script (includes all tests with API keys) - ./run-robot-tests.sh - TEST_EXIT_CODE=$? - echo "test_exit_code=$TEST_EXIT_CODE" >> $GITHUB_ENV + TEST_EXIT_CODE=0 + ./run-robot-tests.sh || TEST_EXIT_CODE=$? + echo "test_exit_code=$TEST_EXIT_CODE" >> "$GITHUB_ENV" exit 0 # Don't fail here, we'll fail at the end after uploading artifacts - name: Save service logs to files @@ -123,7 +123,6 @@ jobs: docker compose -f docker-compose-test.yml logs workers-test > logs/workers.log 2>&1 || true docker compose -f docker-compose-test.yml logs mongo-test > logs/mongo.log 2>&1 || true docker compose -f docker-compose-test.yml logs redis-test > logs/redis.log 2>&1 || true - docker compose -f docker-compose-test.yml logs qdrant-test > logs/qdrant.log 2>&1 || true docker compose -f docker-compose-test.yml logs speaker-service-test > logs/speaker.log 2>&1 || true echo "✓ Logs saved to backends/advanced/logs/" ls -lh logs/ @@ -295,8 +294,12 @@ jobs: - name: Fail workflow if tests failed if: always() run: | - if [ "${{ env.test_exit_code }}" != "0" ]; then - echo "❌ Tests failed with exit code ${{ env.test_exit_code }}" + TEST_EXIT_CODE="${{ env.test_exit_code }}" + if [ -z "$TEST_EXIT_CODE" ]; then + echo "❌ Test step did not record an exit code; check earlier setup/test steps" + exit 1 + elif [ "$TEST_EXIT_CODE" != "0" ]; then + echo "❌ Tests failed with exit code $TEST_EXIT_CODE" exit 1 else echo "✅ All tests passed" diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml new file mode 100644 index 00000000..155be63d --- /dev/null +++ b/.github/workflows/python-tests.yml @@ -0,0 +1,173 @@ +name: Python Tests + +on: + pull_request: + paths: + - "*.py" + - "setup-requirements.txt" + - ".coveragerc" + - "tests/unit/**" + - "backends/advanced/src/**" + - "backends/advanced/tests/**" + - "backends/advanced/pyproject.toml" + - "backends/advanced/uv.lock" + - "extras/asr-services/common/**" + - "extras/asr-services/providers/**" + - "extras/asr-services/tests/**" + - "extras/asr-services/pyproject.toml" + - "extras/asr-services/uv.lock" + - ".github/workflows/python-tests.yml" + push: + branches: [dev, main] + workflow_dispatch: + +permissions: + contents: read + +jobs: + root-unit: + name: Root tooling unit tests + runs-on: ubuntu-latest + timeout-minutes: 10 + env: + PYTHONPATH: ${{ github.workspace }}/backends/advanced/src + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install uv + uses: astral-sh/setup-uv@v4 + with: + version: "latest" + + - name: Run unit tests with coverage + run: >- + uv run + --with-requirements setup-requirements.txt + --with pytest + --with pytest-cov + pytest tests/unit + --cov + --cov-config=.coveragerc + --cov-report=term-missing + --cov-report=xml + --cov-report=html + + - name: Upload coverage report + if: always() + uses: actions/upload-artifact@v4 + with: + name: root-unit-coverage + path: coverage-reports/ + if-no-files-found: warn + retention-days: 14 + + advanced-backend-unit: + name: Advanced backend unit tests + runs-on: ubuntu-latest + timeout-minutes: 15 + defaults: + run: + working-directory: backends/advanced + services: + # Some backend tests hit real local services (vault locks are Redis-backed + # and fail closed; test_leading_silence_trim_db uses Mongo). Match the + # defaults the code assumes: redis://localhost:6379, mongodb://localhost:27018. + redis: + image: redis:7-alpine + ports: + - 6379:6379 + mongo: + image: mongo:8 + ports: + - 27018:27017 + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install uv + uses: astral-sh/setup-uv@v4 + with: + version: "latest" + + - name: Install system dependencies + # libopus0: native library behind opuslib (services/device_audio.py), + # imported transitively during test collection. + run: sudo apt-get update && sudo apt-get install -y --no-install-recommends libopus0 + + - name: Install test dependencies + run: uv sync --locked --group test + + - name: Run unit tests with coverage + run: >- + uv run --group test pytest + --ignore=tests/test_audio_persistence_mongodb.py + --cov=advanced_omi_backend + --cov-report=term-missing + --cov-report=xml + --cov-report=html + + - name: Upload coverage report + if: always() + uses: actions/upload-artifact@v4 + with: + name: advanced-backend-unit-coverage + path: backends/advanced/coverage-reports/ + if-no-files-found: warn + retention-days: 14 + + asr-unit: + name: ASR unit tests + runs-on: ubuntu-latest + timeout-minutes: 10 + defaults: + run: + working-directory: extras/asr-services + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install uv + uses: astral-sh/setup-uv@v4 + with: + version: "latest" + + - name: Install test dependencies + run: uv sync --locked --group test + + - name: Run unit tests with coverage + run: >- + uv run --group test pytest + --ignore=tests/test_parakeet_service.py + --cov=common + --cov=providers + --cov-report=term-missing + --cov-report=xml + --cov-report=html + + - name: Upload coverage report + if: always() + uses: actions/upload-artifact@v4 + with: + name: asr-unit-coverage + path: extras/asr-services/coverage-reports/ + if-no-files-found: warn + retention-days: 14 diff --git a/.github/workflows/robot-tests.yml b/.github/workflows/robot-tests.yml index 35e4dffa..9eb22692 100644 --- a/.github/workflows/robot-tests.yml +++ b/.github/workflows/robot-tests.yml @@ -76,13 +76,11 @@ jobs: - name: Run Robot Framework tests (No API Keys) working-directory: tests - env: - CLEANUP_CONTAINERS: "false" # Don't cleanup in CI - handled by workflow run: | - # Use the no-API test script (excludes tests tagged with requires-api-keys) - ./run-no-api-tests.sh - TEST_EXIT_CODE=$? - echo "test_exit_code=$TEST_EXIT_CODE" >> $GITHUB_ENV + # Use Makefile target (starts containers with mock config, excludes api-keys/slow/sdk/gpu tests) + TEST_EXIT_CODE=0 + make test-no-api OUTPUTDIR=results-no-api || TEST_EXIT_CODE=$? + echo "test_exit_code=$TEST_EXIT_CODE" >> "$GITHUB_ENV" exit 0 # Don't fail here, we'll fail at the end after uploading artifacts - name: Save service logs to files @@ -98,11 +96,19 @@ jobs: docker compose -f docker-compose-test.yml logs workers-test > logs/workers.log 2>&1 || true docker compose -f docker-compose-test.yml logs mongo-test > logs/mongo.log 2>&1 || true docker compose -f docker-compose-test.yml logs redis-test > logs/redis.log 2>&1 || true - docker compose -f docker-compose-test.yml logs qdrant-test > logs/qdrant.log 2>&1 || true docker compose -f docker-compose-test.yml logs speaker-service-test > logs/speaker.log 2>&1 || true echo "✓ Logs saved to backends/advanced/logs/" ls -lh logs/ + - name: Upload service logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: robot-service-logs + path: backends/advanced/logs/ + if-no-files-found: warn + retention-days: 14 + - name: Check if test results exist if: always() id: check_results @@ -273,8 +279,12 @@ jobs: - name: Fail workflow if tests failed if: always() run: | - if [ "${{ env.test_exit_code }}" != "0" ]; then - echo "❌ Tests failed with exit code ${{ env.test_exit_code }}" + TEST_EXIT_CODE="${{ env.test_exit_code }}" + if [ -z "$TEST_EXIT_CODE" ]; then + echo "❌ Test step did not record an exit code; check earlier setup/test steps" + exit 1 + elif [ "$TEST_EXIT_CODE" != "0" ]; then + echo "❌ Tests failed with exit code $TEST_EXIT_CODE" exit 1 else echo "✅ All tests passed" diff --git a/.github/workflows/speaker-recognition-tests.yml b/.github/workflows/speaker-recognition-tests.yml index 5768ada7..0225fb47 100644 --- a/.github/workflows/speaker-recognition-tests.yml +++ b/.github/workflows/speaker-recognition-tests.yml @@ -2,7 +2,7 @@ name: Speaker Recognition Tests on: push: - branches: [ main, develop ] + branches: [ main, dev ] paths: - 'extras/speaker-recognition/src/**' - 'extras/speaker-recognition/tests/**' @@ -13,7 +13,7 @@ on: - 'extras/speaker-recognition/run-test.sh' - '.github/workflows/speaker-recognition-tests.yml' pull_request: - branches: [ main, develop ] + branches: [ main, dev ] paths: - 'extras/speaker-recognition/src/**' - 'extras/speaker-recognition/tests/**' @@ -28,7 +28,7 @@ jobs: speaker-recognition-tests: runs-on: ubuntu-latest timeout-minutes: 30 - + steps: - name: Checkout code uses: actions/checkout@v4 @@ -53,17 +53,17 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - + - name: Install uv uses: astral-sh/setup-uv@v4 with: version: "latest" - + - name: Set up Python uses: actions/setup-python@v5 with: python-version: "3.12" - + - name: Run Speaker Recognition Integration Tests env: HF_TOKEN: ${{ secrets.HF_TOKEN }} @@ -71,7 +71,7 @@ jobs: run: | cd extras/speaker-recognition ./run-test.sh - + - name: Debug Docker build failure if: failure() run: | @@ -84,7 +84,7 @@ jobs: docker compose -f docker-compose-test.yml logs || true echo "=== Docker system info ===" docker system df || true - + - name: Upload test logs on failure if: failure() uses: actions/upload-artifact@v4 @@ -93,4 +93,4 @@ jobs: path: | extras/speaker-recognition/docker-compose-test.yml extras/speaker-recognition/.env - retention-days: 7 \ No newline at end of file + retention-days: 7 diff --git a/.gitignore b/.gitignore index 4b5c84d3..90da9be1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,12 @@ **/__pycache__ +.coverage +.coverage.* +**/coverage-reports/ +**/htmlcov/ *.wav +# Wake-word notification tones are bundled assets, not user audio — keep them tracked +# so they're baked into the wakeword-service image (and served to all clients). +!extras/wakeword-service/tones/*.wav **/*.env !**/.env.template **/memory_config.yaml @@ -15,6 +22,10 @@ config/config.yml config/plugins.yml !config/plugins.yml.template +# Advertised services — per-machine runtime state, regenerated by the node agent on +# every start (services.py / edge/service_manager.py). No defaults, so no template. +config/advertised-services.json + # Individual plugin configs (may contain user-specific settings) backends/advanced/src/advanced_omi_backend/plugins/*/config.yml !backends/advanced/src/advanced_omi_backend/plugins/*/config.yml.template @@ -22,11 +33,12 @@ backends/advanced/src/advanced_omi_backend/plugins/*/config.yml # Config backups config/*.backup.* config/*.backup* +plugins/*/config.yml.backup example/* **/node_modules/* **/ollama-data/* -**/qdrant_data/* +**/model_cache **/model_cache/* .vscode/* **/audio_chunks/* @@ -53,6 +65,7 @@ untracked/* backends/advanced/data/* backends/advanced/diarization_config.json extras/local-wearable-client/devices.yml +extras/llm-services/cache/** !extras/local-wearable-client/devices.yml.template extras/havpe-relay/firmware/secrets.yaml extras/test-audios/* @@ -64,28 +77,45 @@ extras/test-audios/* extras/speaker-omni-experimental/data/* extras/speaker-omni-experimental/cache/* -# AI Stuff -.claude +# Discovery agent PID file +edge/.discovery-agent.pid + +# AI Stuff: keep machine-local Claude state ignored, but track the shared skill link. +.claude/* +!.claude/skills + +# SSL / TLS certificates (centralized) +certs/*.crt +certs/*.key +certs/*.pem -# SSL +# Legacy per-service SSL (kept for cleanup) extras/speaker-recognition/ssl/* backends/advanced/ssl/* -# nginx +# Generated reverse proxy configs extras/speaker-recognition/nginx.conf +extras/speaker-recognition/Caddyfile + +# Generated compose overrides (e.g. Tailscale socket mount for Caddy-managed certs) +backends/advanced/docker-compose.override.yml +extras/speaker-recognition/docker-compose.override.yml # Cache extras/speaker-recognition/cache/* extras/speaker-recognition/outputs/* +**/model_cache_strix_test/* # my backup backends/advanced/src/_webui_original/* -backends/advanced-backend/data/neo4j_data/* +backends/advanced-backend/data/falkordb_data/* backends/advanced-backend/data/speaker_model_cache/ *.bin *.sqlite3 *checkpoints +# Experimental provider-local model fine-tuning workspaces +extras/asr-services/providers/*/finetune/ # k8s config @@ -106,3 +136,5 @@ report.html .secrets sdk/ + +edge/.discovery-agent.pid diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6ebb6573..859eca3c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,42 +1,22 @@ repos: - # Local hooks (project-specific checks) - - repo: local - hooks: - # Run Robot Framework endpoint tests before push - - id: robot-framework-tests - name: Robot Framework Tests (Endpoints) - entry: bash -c 'cd tests && make endpoints OUTPUTDIR=.pre-commit-results' - language: system - pass_filenames: false - stages: [push] - verbose: true - - # Clean up test results after hook runs - - id: cleanup-test-results - name: Cleanup Test Results - entry: bash -c 'cd tests && rm -rf .pre-commit-results' - language: system - pass_filenames: false - stages: [push] - always_run: true - # Code formatting - repo: https://github.com/psf/black rev: 24.4.2 hooks: - id: black - files: ^backends/advanced-backend/src/.*\.py$ + exclude: \.venv/ - repo: https://github.com/PyCQA/isort rev: 5.13.2 hooks: - id: isort - files: ^backends/advanced-backend/src/.*\.py$ + args: ["--profile", "black"] + exclude: \.venv/ # File hygiene - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.5.0 hooks: - id: trailing-whitespace - files: ^backends/advanced-backend/src/.* + exclude: \.venv/ - id: end-of-file-fixer - files: ^backends/advanced-backend/src/.* \ No newline at end of file + exclude: \.venv/ diff --git a/CLAUDE.md b/AGENTS.md similarity index 68% rename from CLAUDE.md rename to AGENTS.md index fc3d8818..3bad60f8 100644 --- a/CLAUDE.md +++ b/AGENTS.md @@ -1,6 +1,6 @@ -# CLAUDE.md +# AGENTS.md -This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. +This file provides guidance to coding agents working in this repository. ## Project Overview @@ -20,7 +20,7 @@ Chronicle includes an **interactive setup wizard** for easy configuration. The w - Authentication setup (admin account, JWT secrets) - Transcription provider configuration (Deepgram or offline ASR) - LLM provider setup (OpenAI or Ollama) -- Memory provider selection (Chronicle Native with Qdrant or OpenMemory MCP) +- Memory configuration (agentic Markdown vault — Chronicle's single memory provider) - Network configuration and HTTPS setup - Optional services (speaker recognition, Parakeet ASR) @@ -37,17 +37,19 @@ uv run --with-requirements setup-requirements.txt python wizard.py **Note on Convenience Scripts**: Chronicle provides wrapper scripts (`./wizard.sh`, `./start.sh`, `./restart.sh`, `./stop.sh`, `./status.sh`) that simplify the longer `uv run --with-requirements setup-requirements.txt python` commands. Use these for everyday operations. +**Temporary Tooling Rule**: When a Python tool or dependency is needed only for a one-off task, run it ephemerally with `uv run --with ...`; do not install it into a project environment or add it to project dependencies. For a package-owned CLI, use `uvx --from `. This applies to browser automation, screenshots, data inspection, and other temporary utilities. + ### Setup Documentation For detailed setup instructions and troubleshooting, see: - **[@quickstart.md](quickstart.md)**: Beginner-friendly step-by-step setup guide -- **[@Docs/init-system.md](Docs/init-system.md)**: Complete initialization system architecture and design +- **[@docs/init-system.md](docs/init-system.md)**: Complete initialization system architecture and design ### Wizard Architecture The initialization system uses a **root orchestrator pattern**: - **`wizard.py`**: Root setup orchestrator for service selection and delegation - **`backends/advanced/init.py`**: Backend configuration wizard - **`extras/speaker-recognition/init.py`**: Speaker recognition setup -- **Service setup scripts**: Individual setup for ASR services and OpenMemory MCP +- **Service setup scripts**: Individual setup for ASR services Key features: - Interactive prompts with validation @@ -99,13 +101,13 @@ make test # Start containers + run all tests # Or step by step make start # Start test containers (with health checks) -make test-all # Run all test suites +make all # Run all test suites make stop # Stop containers (preserves volumes) # Run specific test suites -make test-endpoints # API endpoint tests (~40 tests, fast) -make test-integration # End-to-end workflows (~15 tests, slower) -make test-infra # Infrastructure resilience (~5 tests) +make endpoints # API endpoint tests +make integration # End-to-end workflows +make infra # Infrastructure resilience # Quick iteration (reuse existing containers) make test-quick # Run tests without restarting containers @@ -129,7 +131,7 @@ make logs SERVICE= # View specific service logs #### Test Environment Test services use isolated ports and database: -- **Ports:** Backend (8001), MongoDB (27018), Redis (6380), Qdrant (6337/6338) +- **Ports:** Backend (8001), MongoDB (27018), Redis (6380) - **Database:** `test_db` (separate from production) - **Credentials:** `test-admin@example.com` / `test-admin-password-123` @@ -162,6 +164,13 @@ docker compose up --build # HAVPE Relay (ESP32 bridge) cd extras/havpe-relay docker compose up --build + +# TTS Services (text-to-speech, run ONE provider at a time on port 8770) +cd extras/tts +docker compose up tada-tts -d --build # HumeAI TADA (GPU, voice cloning) +docker compose up fish-tts -d --build # Fish Speech (GPU, 50+ langs, emotion tags) +docker compose up kittentts-tts -d --build # KittenTTS (~25MB CPU ONNX, no GPU) +docker compose up kokoro-tts -d --build # Kokoro-82M (<~1GB VRAM GPU/CPU, preset voices) ``` ## Architecture Overview @@ -173,10 +182,10 @@ docker compose up --build - **Job Tracker**: Tracks pipeline jobs with stage events (audio → transcription → memory) and completion status - **Task Management**: BackgroundTaskManager tracks all async tasks to prevent orphaned processes - **Unified Transcription**: Deepgram transcription with fallback to offline ASR services -- **Memory System**: Pluggable providers (Chronicle native or OpenMemory MCP) +- **Memory System**: Single agentic Markdown vault — a tool-calling memory agent records conversations and surgically edits Obsidian-style People/Topic/Category notes; a read-only retrieval agent drives ripgrep over the vault to answer queries - **Authentication**: Email-based login with MongoDB ObjectId user system - **Client Management**: Auto-generated client IDs as `{user_id_suffix}-{device_name}`, centralized ClientManager -- **Data Storage**: MongoDB (`audio_chunks` collection for conversations), vector storage (Qdrant or OpenMemory) +- **Data Storage**: MongoDB (conversations, `audio_chunks`, chat, annotations), disk WAV files, and the Markdown vault (`data/conversation_docs//`) as the memory source of truth - **Web Interface**: React-based web dashboard with authentication and real-time monitoring ### Service Dependencies @@ -184,9 +193,8 @@ docker compose up --build Required: - MongoDB: User data and conversations - Redis: Job queues (RQ workers) and session state - - Qdrant: Vector storage for memory search - FastAPI Backend: Core audio processing - - LLM Service: Memory extraction and action items (OpenAI or Ollama) + - LLM Service: Memory agent (vault read/write) and action items (OpenAI or Ollama) Recommended: - Transcription: Deepgram or offline ASR services @@ -195,7 +203,6 @@ Optional: - Parakeet ASR: Offline transcription service - Speaker Recognition: Voice identification service - Caddy: HTTPS reverse proxy (auto-configured when HTTPS enabled) - - OpenMemory MCP: For cross-client memory compatibility ``` ## Data Flow Architecture @@ -206,8 +213,8 @@ Optional: 4. **Speech-Driven Conversation Creation**: User-facing conversations only created when speech is detected 5. **Dual Storage System**: Audio sessions always stored in `audio_chunks`, conversations created in `conversations` collection only with speech 6. **Versioned Processing**: Transcript and memory versions tracked with active version pointers -7. **Memory Processing**: Pluggable providers (Chronicle native with individual facts or OpenMemory MCP delegation) -8. **Memory Storage**: Direct Qdrant (Chronicle) or OpenMemory server (MCP provider) +7. **Memory Processing**: A tool-calling memory agent records each conversation and surgically edits People/Topic/Category notes in the Markdown vault +8. **Memory Storage**: Obsidian-style Markdown vault at `data/conversation_docs//` — the single source of truth, searched by a read-only retrieval agent via ripgrep 9. **Audio Optimization**: Speech segment extraction removes silence automatically 10. **Task Tracking**: BackgroundTaskManager ensures proper cleanup of all async operations @@ -256,51 +263,31 @@ DEEPGRAM_API_KEY=your-deepgram-key-here # Optional: TRANSCRIPTION_PROVIDER=deepgram # Memory Provider -MEMORY_PROVIDER=chronicle # or openmemory_mcp +MEMORY_PROVIDER=chronicle # agentic Markdown vault (only valid value) # Database MONGODB_URI=mongodb://mongo:27017 # Database name: chronicle -QDRANT_BASE_URL=qdrant # Network Configuration HOST_IP=localhost BACKEND_PUBLIC_PORT=8000 -WEBUI_PORT=3010 # Production port (5173 is Vite dev server only) -CORS_ORIGINS=http://localhost:3010,http://localhost:8000 +WEBUI_PORT=5173 # Vite dev server (the only webui; fronted by Caddy for HTTPS) +CORS_ORIGINS=http://localhost:5173,http://localhost:8000 ``` ### Memory Provider Configuration -Chronicle supports two pluggable memory backends: +Chronicle has a single memory provider, `chronicle`: an **agentic Markdown vault**. The vault (Obsidian-style notes at `data/conversation_docs//`) is the single source of truth. A tool-calling memory agent records each conversation and surgically edits People/Topic/Category notes; a read-only retrieval agent drives ripgrep over the vault to synthesize answers. There is no provider choice to configure — only an LLM for the agents to use. -#### Chronicle Memory Provider (Default) ```bash -# Use Chronicle memory provider (default) +# Memory provider (only valid value) MEMORY_PROVIDER=chronicle -# LLM Configuration for memory extraction +# LLM Configuration for the memory agent LLM_PROVIDER=openai OPENAI_API_KEY=your-openai-key-here OPENAI_MODEL=gpt-4o-mini - -# Vector Storage -QDRANT_BASE_URL=qdrant -``` - -#### OpenMemory MCP Provider -```bash -# Use OpenMemory MCP provider -MEMORY_PROVIDER=openmemory_mcp - -# OpenMemory MCP Server Configuration -OPENMEMORY_MCP_URL=http://host.docker.internal:8765 -OPENMEMORY_CLIENT_NAME=chronicle -OPENMEMORY_USER_ID=openmemory -OPENMEMORY_TIMEOUT=30 - -# OpenAI key for OpenMemory server -OPENAI_API_KEY=your-openai-key-here ``` ### Transcription Provider Configuration @@ -357,7 +344,7 @@ SPEAKER_SERVICE_URL=http://speaker-recognition:8085 - **GET /readiness**: Service dependency validation - **WS /ws**: Audio streaming endpoint with codec parameter (Wyoming protocol, supports pcm and opus codecs) - **GET /api/conversations**: User's conversations with transcripts -- **GET /api/memories/search**: Semantic memory search with relevance scoring +- **GET /api/memories/search**: Agentic vault search (retrieval agent over the Markdown vault) - **POST /auth/jwt/login**: Email-based login (returns JWT token) ### Authentication Flow @@ -458,6 +445,51 @@ The relay will automatically: - Forward ESP32 audio to the backend with proper authentication - Handle token refresh and reconnection +## TTS Services + +Provider-based text-to-speech (`extras/tts/`), built on the same provider pattern as `extras/asr-services/`. Run **one provider at a time**, all serving on port `8770` (configurable via `TTS_PORT`). + +### Providers + +| Provider | Service | Hardware | Highlights | +|----------|---------|----------|-----------| +| **TADA** (HumeAI) | `tada-tts` | GPU | Zero-shot voice cloning, 1:1 token alignment (no hallucinations), MIT. `tada-1b` (English) / `tada-3b-ml` (9 langs). Needs `HF_TOKEN` (Llama 3.2 base is gated). | +| **Fish Speech** (Fish Audio) | `fish-tts` | GPU | Dual-AR, 50+ langs, inline emotion/prosody tags (`[laugh]`, `[whispers]`), streaming. `s2-pro` (default) / `openaudio-s1-mini` / `fish-speech-1.5`. Optional `torch.compile`. | +| **KittenTTS** (KittenML) | `kittentts-tts` | CPU | Ultra-light (~25MB) ONNX, no GPU/API key, preset voices, English only. Uses dedicated `KITTEN_TTS_*` env vars. | +| **Kokoro** (hexgrad) | `kokoro-tts` | GPU/CPU | Lightweight (~82M, **<~1GB VRAM**) StyleTTS2, preset voices, 8 langs, Apache-2.0. Quality-per-VRAM sweet spot. Uses dedicated `KOKORO_TTS_*` env vars. | + +### Setup & Run + +```bash +cd extras/tts + +# Configure (selects provider, model, CUDA version) +uv run --with-requirements ../../setup-requirements.txt python init.py + +# Start ONE provider +docker compose up tada-tts -d --build # or fish-tts / kittentts-tts + +# Test +curl http://localhost:8770/health +curl -X POST http://localhost:8770/synthesize -F "text=Hello world." -o output.wav +``` + +### API Endpoints + +| Endpoint | Method | Description | +|----------|--------|-------------| +| `/health` | GET | Service health (`healthy` / `initializing`) | +| `/info` | GET | Model id, provider, capabilities, supported languages | +| `/synthesize` | POST | Generate speech (multipart form) | + +**POST /synthesize** — `text` (required); optional `reference_audio` (WAV) + `reference_text` for voice cloning; optional generation params (`temperature`, `top_p`, `repetition_penalty`, `seed`, `max_new_tokens`). Returns WAV bytes with `X-Sample-Rate`, `X-Provider`, `X-Model` headers. + +**Notes:** +- Not registered in `services.py` — manage with `docker compose` directly (like the HAVPE relay). +- GPU providers require CUDA 12.6+ (`PYTORCH_CUDA_VERSION=cu126`/`cu128`); `cu121` is unsupported (torch>=2.7). +- Add a provider by creating `extras/tts/providers/{name}/` with `service.py`, `synthesizer.py`, and `Dockerfile` (subclass `BaseTTSService`). +- An optional `edge-agent` sidecar (`--profile edge`) advertises the service on the Tailnet. + ## Distributed Deployment ### Single Machine vs Distributed Setup @@ -515,14 +547,14 @@ tailscale ip -4 **Service Examples:** - GPU machine: LLM inference, ASR, speaker recognition - Backend machine: FastAPI, WebUI, databases -- Database machine: MongoDB, Qdrant (optional separation) +- Database machine: MongoDB (optional separation) ## Development Notes ### Package Management - **Backend**: Uses `uv` for Python dependency management (faster than pip) - **Mobile**: Uses `npm` with React Native and Expo -- **Docker**: Primary deployment method with docker-compose +- **Containers**: The service lifecycle (`services.py`/`status.py`/service-manager) supports **both Docker and Podman**, selected via `container_engine: docker|podman` in `config/config.yml` (or the `CONTAINER_ENGINE`/`COMPOSE_CMD` env vars). The repo's compose files run unmodified under either engine. For Podman it drives `podman-compose` and needs CDI for GPU. See **[@docs/podman.md](docs/podman.md)** for rootless/GPU setup and migration notes. ### Testing Strategy - **Makefile-Based**: All test operations through simple `make` commands (`make test`, `make start`, `make stop`) @@ -562,20 +594,28 @@ The system includes comprehensive health checks: - **Log Preservation**: All cleanup operations save logs to `tests/logs/` automatically - **CI Compatibility**: Same test logic runs locally and in GitHub Actions -### Cursor Rule Integration -Project includes `.cursor/rules/always-plan-first.mdc` requiring understanding before coding. Always explain the task and confirm approach before implementation. - ## Extended Documentation For detailed technical documentation, see: -- **[@Docs/overview.md](Docs/overview.md)**: Architecture overview and technical deep dive -- **[@Docs/init-system.md](Docs/init-system.md)**: Initialization system and service management -- **[@Docs/ssl-certificates.md](Docs/ssl-certificates.md)**: HTTPS/SSL setup details -- **[@Docs/audio-pipeline-architecture.md](Docs/audio-pipeline-architecture.md)**: Audio pipeline design -- **[@backends/advanced/Docs/auth.md](backends/advanced/Docs/auth.md)**: Authentication architecture -- **[backends/advanced/Docs/architecture.md](backends/advanced/Docs/architecture.md)**: Backend architecture details -- **[@backends/advanced/Docs/memories.md](backends/advanced/Docs/memories.md)**: Memory system documentation -- **[@backends/advanced/Docs/plugin-development-guide.md](backends/advanced/Docs/plugin-development-guide.md)**: Plugin development guide +- **[@docs/README.md](docs/README.md)**: Documentation index +- **[@docs/overview.md](docs/overview.md)**: Architecture overview and technical deep dive +- **[@docs/init-system.md](docs/init-system.md)**: Initialization system and service management +- **[@docs/ssl-certificates.md](docs/ssl-certificates.md)**: HTTPS/SSL setup details +- **[@docs/podman.md](docs/podman.md)**: Running with Podman instead of Docker (engine selection, rootless/GPU setup) +- **[@docs/screenpipe.md](docs/screenpipe.md)**: ScreenPipe capture-node architecture, services, desktop controls, and troubleshooting +- **[@docs/audio-pipeline-architecture.md](docs/audio-pipeline-architecture.md)**: Audio pipeline design +- **[@docs/backend/auth.md](docs/backend/auth.md)**: Authentication architecture +- **[@docs/backend/memories.md](docs/backend/memories.md)**: Memory system documentation +- **[@docs/backend/plugin-development-guide.md](docs/backend/plugin-development-guide.md)**: Plugin development guide + +### ScreenPipe Capture Nodes + +Before changing ScreenPipe ingestion, the desktop tray, or capture-node services, read +[@docs/screenpipe.md](docs/screenpipe.md). ScreenPipe owns the high-volume local capture +store; Chronicle's companion sends compact activity metadata and serves bounded +snapshot/OCR requests. The desktop entry point is shared across macOS and Linux, with +platform UI adapters over common state, logging, and vault-sync code. The ScreenPipe UI +is an optional on-demand viewer and must not be required for background capture. ## Robot Framework Testing @@ -583,13 +623,13 @@ For detailed technical documentation, see: Before writing any Robot Framework test: 1. **Read [@tests/TESTING_GUIDELINES.md](tests/TESTING_GUIDELINES.md)** for comprehensive testing patterns and standards -2. **Check [@tests/tags.md](tests/tags.md)** for approved tags - ONLY 11 tags are permitted +2. **Check [@tests/tags.md](tests/tags.md)** for approved tags - only the 11 business tags and 4 execution tags are permitted 3. **SCAN existing resource files** for keywords - NEVER write code that duplicates existing keywords 4. **Follow the Arrange-Act-Assert pattern** with inline verifications (not abstracted to keywords) Key Testing Rules: - **Check Existing Keywords FIRST**: Before writing ANY test code, scan relevant resource files (`websocket_keywords.robot`, `queue_keywords.robot`, `conversation_keywords.robot`, etc.) for existing keywords -- **Tags**: ONLY use the 11 approved tags from tags.md, tab-separated (e.g., `[Tags] infra audio-streaming`) +- **Tags**: ONLY use the 15 approved tags from tags.md, tab-separated (e.g., `[Tags] infra audio-streaming`) - **Verifications**: Write assertions directly in tests, not in resource keywords - **Keywords**: Only create keywords for reusable setup/action operations AFTER confirming no existing keyword exists - **Resources**: Always check existing resource files before creating new keywords or duplicating logic @@ -597,20 +637,34 @@ Key Testing Rules: **DO NOT:** - Write inline code without checking if a keyword already exists for that operation -- Create custom tags (use only the 11 approved tags) +- Create custom tags (use only the 15 approved tags) - Abstract verifications into keywords (keep them inline in tests) - Use space-separated tags (must be tab-separated) - Skip reading the guidelines before writing tests -## Notes for Claude +## Notes for Coding Agents +For frontend UI changes and reviews, read +**[@docs/agents/frontend-ux-review.md](docs/agents/frontend-ux-review.md)**. Apply the +installed `frontend-design-principles` skill for hierarchy and visual judgment, and +the repository `screenshots` skill for rendered-page verification. + Check if the src/ is volume mounted. If not, do compose build so that code changes are reflected. Do not simply run `docker compose restart` as it will not rebuild the image. -Check backends/advanced/Docs for up to date information on advanced backend. +Check `docs/backend/` for up-to-date information on the advanced backend. All docker projects have .dockerignore following the exclude pattern. That means files need to be included for them to be visible to docker. The uv package manager is used for all python projects. Wherever you'd call `python3 main.py` you'd call `uv run python main.py` +For temporary Python-backed tooling that is not part of the repo dependencies, prefer `uv run --with python ...` instead of installing packages into the project. Use `uvx --from ` when invoking a package's own CLI. For browser scripts/screenshots, use `uv run --with playwright python - <<'PY'` and import `playwright.sync_api`; for the Playwright CLI use `uvx --from playwright playwright ...`. Do not add transient Playwright/npm packages to `package.json` just to drive a one-off check. + +**Compute-Intensive Workloads:** +- Chronicle is designed for heavy data and AI processing. Re-encoding or recomputation is acceptable when it improves correctness or output quality. +- Avoid wasteful repeated work: cache reusable artifacts, fingerprint model and configuration inputs, and reuse valid intermediate results. +- Prefer GPU acceleration whenever the workload and deployed service support it. + +**Container Engine (Docker or Podman):** +- The project supports **both Docker and Podman**. The active engine is set by `container_engine` in `config/config.yml` (default `docker`); prefer the lifecycle scripts (`./start.sh`/`./stop.sh`/`./restart.sh`) which route through the selected engine. For one-off manual commands under Podman use `podman-compose` (not `docker compose`). See **[@docs/podman.md](docs/podman.md)**. **Docker Build Guidelines:** -- Use `docker compose build` without `--no-cache` by default for faster builds +- Use `docker compose build` (or `podman-compose build`) without `--no-cache` by default for faster builds - Only use `--no-cache` when explicitly needed (e.g., if cached layers are causing issues or when troubleshooting build problems) -- Docker's build cache is efficient and saves significant time during development +- The build cache is efficient and saves significant time during development -- Remember that whenever there's a python command, you should use uv run python3 instead \ No newline at end of file +- Remember that whenever there's a python command, you should use uv run python3 instead diff --git a/Docs/audio-pipeline-architecture.md b/Docs/audio-pipeline-architecture.md deleted file mode 100644 index afba52db..00000000 --- a/Docs/audio-pipeline-architecture.md +++ /dev/null @@ -1,1241 +0,0 @@ -# Audio Pipeline Architecture - -This document explains how audio flows through the Chronicle system from initial capture to final storage, including all intermediate processing stages, Redis streams, and data storage locations. - -## Table of Contents - -- [Overview](#overview) -- [Architecture Diagram](#architecture-diagram) -- [Data Sources](#data-sources) -- [Redis Streams: The Central Pipeline](#redis-streams-the-central-pipeline) -- [Producer: AudioStreamProducer](#producer-audiostreamproducer) -- [Dual-Consumer Architecture](#dual-consumer-architecture) -- [Transcription Results Aggregator](#transcription-results-aggregator) -- [Job Queue Orchestration (RQ)](#job-queue-orchestration-rq) -- [Data Storage](#data-storage) -- [Complete End-to-End Flow](#complete-end-to-end-flow) -- [Key Design Patterns](#key-design-patterns) -- [Failure Handling](#failure-handling) - -## Overview - -Chronicle's audio pipeline is built on three core technologies: - -- **Redis Streams**: Distributed message queues for audio chunks and transcription results -- **Background Tasks**: Async consumers that process streams independently -- **RQ Job Queue**: Orchestrates session-level and conversation-level workflows - -**Key Insight**: Multiple workers can independently consume the **same audio stream** using Redis Consumer Groups, enabling parallel processing paths (transcription + disk persistence) without duplication. - -## Architecture Diagram - -``` -┌─────────────────────────────────────────────────────────────────┐ -│ AUDIO INPUT │ -│ WebSocket (/ws) │ File Upload (/audio/upload) │ Google Drive │ -└────────────────────────────────┬────────────────────────────────┘ - ↓ - ┌────────────────────────┐ - │ AudioStreamProducer │ - │ - Chunk audio (0.25s) │ - │ - Session metadata │ - └────────────┬───────────┘ - ↓ - ┌────────────────────────────────┐ - │ Redis Stream (Per Client) │ - │ audio:stream:{client_id} │ - └─────┬──────────────────┬───────┘ - ↓ ↓ - ┌───────────────────────┐ ┌──────────────────────┐ - │ Transcription Consumer│ │ Audio Persistence │ - │ Group (streaming/batch)│ │ Consumer Group │ - │ │ │ │ - │ → Deepgram WebSocket │ │ → Writes WAV files │ - │ → Batch buffering │ │ → Monitors rotation │ - │ → Publish results │ │ → Stores file paths │ - └───────────┬───────────┘ └──────────┬───────────┘ - ↓ ↓ - ┌───────────────────────┐ ┌──────────────────────┐ - │ transcription:results │ │ Disk Storage │ - │ :{session_id} │ │ data/chunks/*.wav │ - └───────────┬───────────┘ └──────────────────────┘ - ↓ - ┌───────────────────────┐ - │ TranscriptionResults │ - │ Aggregator │ - │ - Combines chunks │ - │ - Merges timestamps │ - └───────────┬───────────┘ - ↓ - ┌───────────────────────┐ - │ RQ Job Pipeline │ - ├───────────────────────┤ - │ speech_detection_job │ ← Session-level - │ ↓ │ - │ open_conversation_job │ ← Conversation-level - │ ↓ │ - │ Post-Conversation: │ - │ • transcribe_full │ - │ • speaker_recognition │ - │ • memory_extraction │ - │ • title_generation │ - └───────────┬───────────┘ - ↓ - ┌───────────────────────┐ - │ Final Storage │ - ├───────────────────────┤ - │ MongoDB: conversations│ - │ Disk: WAV files │ - │ Qdrant: Memories │ - └───────────────────────┘ -``` - -## Data Sources - -### 1. WebSocket Streaming (`/ws`) - -**Endpoint**: `/ws?codec=pcm|opus&token=xxx&device_name=xxx` - -**Handlers**: -- `handle_pcm_websocket()` - Raw PCM audio -- `handle_omi_websocket()` - Opus-encoded audio (compressed, used by OMI devices) - -**Protocol**: Wyoming Protocol (JSON lines + binary frames) - -**Authentication**: JWT token required - -**Location**: `backends/advanced/src/advanced_omi_backend/routers/websocket_routes.py` - -**Container**: `chronicle-backend` - -### 2. File Upload (`/audio/upload`) - -**Endpoint**: `POST /api/audio/upload` - -**Accepts**: Multiple WAV files (multipart form data) - -**Authentication**: Admin only - -**Device ID**: Auto-generated as `{user_id_suffix}-upload` or custom `device_name` - -**Location**: `backends/advanced/src/advanced_omi_backend/routers/api_router.py` - -**Container**: `chronicle-backend` - -### 3. Google Drive Upload - -**Endpoint**: `POST /api/audio/upload_audio_from_gdrive` - -**Source**: Google Drive folder ID - -**Processing**: Downloads files and enqueues for processing - -**Container**: `chronicle-backend` - -## Redis Streams: The Central Pipeline - -### Stream Naming Convention - -``` -audio:stream:{client_id} -``` - -**Examples**: -- `audio:stream:user01-phone` -- `audio:stream:user01-omi-device` -- `audio:stream:user01-upload` - -**Characteristics**: -- **Client-specific isolation**: Each device has its own stream -- **Fan-out pattern**: Multiple consumer groups read the same stream -- **MAXLEN constraint**: Keeps last 25,000 entries (auto-trimming) -- **No TTL**: Streams persist until manually deleted -- **Container**: `redis` service - -### Session Metadata Storage - -``` -audio:session:{session_id} -``` - -**Type**: Redis Hash - -**Fields**: -- `user_id`: MongoDB ObjectId -- `client_id`: Device identifier -- `connection_id`: WebSocket connection ID -- `stream_name`: `audio:stream:{client_id}` -- `status`: `"active"` → `"finalizing"` → `"complete"` -- `chunks_published`: Integer count -- `speech_detection_job_id`: RQ job ID -- `audio_persistence_job_id`: RQ job ID -- `websocket_connected`: `true|false` -- `transcription_error`: Error message (if any) - -**TTL**: 1 hour - -**Container**: `redis` - -### Transcription Results Stream - -``` -transcription:results:{session_id} -``` - -**Type**: Redis Stream - -**Written by**: Transcription consumers (streaming or batch) - -**Read by**: `TranscriptionResultsAggregator` - -**Message Fields**: -- `text`: Transcribed text for this chunk -- `chunk_id`: Redis message ID from audio stream -- `provider`: `"deepgram"` or `"parakeet"` -- `confidence`: Float (0.0-1.0) -- `words`: JSON array of word-level timestamps -- `segments`: JSON array of speaker segments - -**Lifecycle**: Deleted when conversation completes - -**Container**: `redis` - -### Conversation Tracking - -``` -conversation:current:{session_id} -``` - -**Type**: Redis String - -**Value**: Current `conversation_id` (UUID) - -**Purpose**: Signals audio persistence job to rotate WAV file - -**TTL**: 24 hours - -**Container**: `redis` - -### Audio File Path Mapping - -``` -audio:file:{conversation_id} -``` - -**Type**: Redis String - -**Value**: File path (e.g., `1704067200000_user01-phone_convid.wav`) - -**Purpose**: Links conversation to its audio file on disk - -**TTL**: 24 hours - -**Container**: `redis` - -## Producer: AudioStreamProducer - -**File**: `backends/advanced/src/advanced_omi_backend/services/audio_stream/producer.py` - -**Container**: `chronicle-backend` (in-memory, no persistence) - -### Responsibilities - -#### 1. Session Initialization - -```python -async def init_session( - session_id: str, - user_id: str, - client_id: str, - provider: str, - mode: str -) -> None -``` - -**Actions**: -- Creates `audio:session:{session_id}` hash in Redis -- Initializes in-memory buffer for chunking -- Stores session metadata (user, client, provider) - -#### 2. Audio Chunking - -```python -async def add_audio_chunk( - session_id: str, - audio_data: bytes -) -> list[str] -``` - -**Process**: -1. Buffers incoming audio (arbitrary size from WebSocket) -2. Creates **fixed-size chunks**: 0.25 seconds = 8,000 bytes - - Assumes: 16kHz sample rate, 16-bit mono PCM -3. Prevents cutting audio mid-word (aligned chunks) -4. Publishes each chunk to `audio:stream:{client_id}` via `XADD` -5. Returns Redis message IDs for tracking - -**In-Memory Storage**: Session buffers stored in `AudioStreamProducer._session_buffers` dict - -#### 3. Session End Signal - -```python -async def send_session_end_signal(session_id: str) -> None -``` - -**Actions**: -- Publishes special `{"type": "END"}` message to stream -- Signals all consumers to flush buffers and finalize -- Updates session status to `"finalizing"` - -### Data Location - -**Memory**: `chronicle-backend` container (in-memory buffers) - -**Redis**: Published chunks in `audio:stream:{client_id}` (redis container) - -## Dual-Consumer Architecture - -Chronicle uses **Redis Consumer Groups** to enable multiple independent consumers to read the **same audio stream** without message duplication. - -### Consumer Group 1: Transcription - -Two implementations available: - -#### A. Streaming Transcription Consumer - -**File**: `backends/advanced/src/advanced_omi_backend/services/transcription/streaming_consumer.py` - -**Class**: `StreamingTranscriptionConsumer` - -**Consumer Group**: `streaming-transcription` - -**Provider**: Deepgram (WebSocket-based) - -**Process**: -1. Discovers `audio:stream:*` streams dynamically using `SCAN` -2. Opens persistent WebSocket connection to Deepgram per stream -3. Sends audio chunks **immediately** (no buffering) -4. Publishes **interim results** to `transcription:interim:{session_id}` (Redis Pub/Sub) -5. Publishes **final results** to `transcription:results:{session_id}` (Redis Stream) -6. Triggers plugins on final results only -7. ACKs messages with `XACK` to prevent reprocessing -8. Handles END signal: closes WebSocket, cleans up - -**Container**: `chronicle-backend` (Background Task via `BackgroundTaskManager`) - -**Real-time Updates**: Interim results pushed to WebSocket clients via Pub/Sub - -#### B. Batch Transcription Consumer - -**File**: `backends/advanced/src/advanced_omi_backend/services/audio_stream/consumer.py` - -**Class**: `BaseAudioStreamConsumer` - -**Consumer Group**: `{provider_name}_workers` (e.g., `deepgram_workers`, `parakeet_workers`) - -**Providers**: Deepgram (batch), Parakeet ASR (offline) - -**Process**: -1. Reads from `audio:stream:{client_id}` using `XREADGROUP` -2. Buffers chunks per session (default: 30 chunks = ~7.5 seconds) -3. When buffer full: - - Combines chunks into single audio buffer - - Transcribes using provider API - - Adjusts word/segment timestamps relative to session start - - Publishes result to `transcription:results:{session_id}` -4. Flushes remaining buffer on END signal -5. ACKs all buffered messages with `XACK` -6. Trims stream to keep only last 1,000 entries (`XTRIM MAXLEN`) - -**Container**: `chronicle-backend` (Background Task) - -**Batching Benefits**: Reduces API calls, improves transcription accuracy (more context) - -### Consumer Group 2: Audio Persistence - -**File**: `backends/advanced/src/advanced_omi_backend/workers/audio_jobs.py` - -**Function**: `audio_streaming_persistence_job()` - -**Consumer Group**: `audio_persistence` - -**Consumer Name**: `persistence-worker-{session_id}` - -**Process**: -1. Reads audio chunks from `audio:stream:{client_id}` using `XREADGROUP` -2. Monitors `conversation:current:{session_id}` for rotation signals -3. On conversation rotation: - - Closes current WAV file - - Opens new WAV file with new conversation ID -4. Writes chunks immediately to disk (real-time persistence) -5. Stores file path in `audio:file:{conversation_id}` (Redis) -6. Handles END signal: closes file, returns statistics -7. ACKs messages after writing to disk - -**Container**: `chronicle-backend` (RQ Worker) - -**Output Location**: `backends/advanced/data/chunks/` (volume-mounted) - -**File Format**: `{timestamp_ms}_{client_id}_{conversation_id}.wav` - -### Fan-Out Pattern Visualization - -``` -audio:stream:user01-phone - ↓ - ├─ Consumer Group: "streaming-transcription" - │ └─ Worker: streaming-worker-12345 - │ → Reads: chunks → Deepgram WS → Results stream - │ - ├─ Consumer Group: "deepgram_workers" - │ ├─ Worker: deepgram-worker-67890 - │ ├─ Worker: deepgram-worker-67891 - │ └─ Reads: chunks → Buffer (30) → Batch API → Results stream - │ - └─ Consumer Group: "audio_persistence" - └─ Worker: persistence-worker-sessionXYZ - → Reads: chunks → WAV file (disk) -``` - -**Key Benefits**: -- **Horizontal scaling**: Multiple workers per group -- **Independent processing**: Each group processes all messages -- **No message loss**: Messages ACKed only after processing -- **Decoupled**: Producer doesn't know about consumers - -## Transcription Results Aggregator - -**File**: `backends/advanced/src/advanced_omi_backend/services/audio_stream/aggregator.py` - -**Class**: `TranscriptionResultsAggregator` - -**Container**: `chronicle-backend` (in-memory, stateless) - -### Methods - -#### Get Combined Results - -```python -async def get_combined_results(session_id: str) -> dict -``` - -**Returns**: -```python -{ - "text": "Full transcript...", - "segments": [SpeakerSegment, ...], - "words": [Word, ...], - "provider": "deepgram", - "chunk_count": 42 -} -``` - -**Process**: -- Reads all entries from `transcription:results:{session_id}` -- For **streaming mode**: Uses latest final result only (supersedes interim) -- For **batch mode**: Combines all chunks sequentially -- Adjusts timestamps across chunks (adds audio offset) -- Merges speaker segments, words - -#### Get Session Results (Raw) - -```python -async def get_session_results(session_id: str) -> list[dict] -``` - -**Returns**: Raw list of transcription result messages - -#### Get Real-time Results - -```python -async def get_realtime_results( - session_id: str, - last_id: str = "0-0" -) -> tuple[list[dict], str] -``` - -**Returns**: `(new_results, new_last_id)` - -**Purpose**: Incremental polling for live UI updates - -### Data Location - -**Input**: `transcription:results:{session_id}` stream (redis container) - -**Processing**: In-memory (chronicle-backend container) - -**Output**: Returned to caller (no persistence) - -## Job Queue Orchestration (RQ) - -**Library**: Python RQ (Redis Queue) - -**File**: `backends/advanced/src/advanced_omi_backend/controllers/queue_controller.py` - -**Containers**: -- `chronicle-backend` (enqueues jobs) -- `rq-worker` (executes jobs) - -### Job Pipeline - -``` -Session Starts - ↓ -┌─────────────────────────────────┐ -│ stream_speech_detection_job │ ← Session-level (long-running) -│ - Polls transcription results │ -│ - Analyzes speech content │ -│ - Checks speaker filters │ -└─────────────┬───────────────────┘ - ↓ (when speech detected) -┌─────────────────────────────────┐ -│ open_conversation_job │ ← Conversation-level (long-running) -│ - Creates conversation │ -│ - Signals file rotation │ -│ - Monitors activity │ -│ - Detects end conditions │ -└─────────────┬───────────────────┘ - ↓ (when conversation ends) -┌─────────────────────────────────┐ -│ Post-Conversation Pipeline │ -├─────────────────────────────────┤ -│ • recognize_speakers_job │ -│ • memory_extraction_job │ -│ • generate_title_summary_job │ -│ • dispatch_conversation_complete│ -└─────────────────────────────────┘ -``` - -### Session-Level Jobs - -#### Speech Detection Job - -**File**: `backends/advanced/src/advanced_omi_backend/workers/transcription_jobs.py` - -**Function**: `stream_speech_detection_job()` - -**Scope**: Entire session (can handle multiple conversations) - -**Max Duration**: 24 hours - -**Process**: -1. Polls `TranscriptionResultsAggregator.get_combined_results()` (1-second intervals) -2. Analyzes speech content: - - Word count > 10 - - Duration > 5 seconds - - Confidence > threshold -3. If speaker filter enabled: checks for enrolled speakers -4. When speech detected: - - Creates conversation in MongoDB - - Enqueues `open_conversation_job` - - **Exits** (restarts when conversation completes) -5. Handles transcription errors (marks session with error flag) - -**RQ Queue**: `speech_detection_queue` (dedicated queue) - -**Container**: `rq-worker` - -### Conversation-Level Jobs - -#### Open Conversation Job - -**File**: `backends/advanced/src/advanced_omi_backend/workers/conversation_jobs.py` - -**Function**: `open_conversation_job()` - -**Scope**: Single conversation - -**Max Duration**: 3 hours - -**Process**: -1. Creates conversation document in MongoDB `conversations` collection -2. Sets `conversation:current:{session_id}` = `conversation_id` (Redis) - - **Triggers audio persistence job to rotate WAV file** -3. Polls for transcription updates (1-second intervals) -4. Tracks speech activity (inactivity timeout = 60 seconds default) -5. Detects end conditions: - - WebSocket disconnect - - User manual stop - - Inactivity timeout -6. Waits for audio file path from persistence job -7. Saves `audio_path` to conversation document -8. Triggers conversation-level plugins -9. Enqueues post-conversation jobs -10. Calls `handle_end_of_conversation()` for cleanup + restart - -**RQ Queue**: `default` - -**Container**: `rq-worker` - -#### Audio Persistence Job - -**File**: `backends/advanced/src/advanced_omi_backend/workers/audio_jobs.py` - -**Function**: `audio_streaming_persistence_job()` - -**Scope**: Entire session (parallel with open_conversation_job) - -**Max Duration**: 24 hours - -**Process**: -1. Monitors `conversation:current:{session_id}` for rotation signals -2. For each conversation: - - Opens new WAV file: `{timestamp}_{client_id}_{conversation_id}.wav` - - Writes chunks immediately as they arrive from stream - - Stores file path in `audio:file:{conversation_id}` -3. On rotation signal: - - Closes current file - - Opens new file for next conversation -4. On END signal: - - Closes file - - Returns statistics (chunk count, bytes, duration) - -**Output**: WAV files in `backends/advanced/data/chunks/` - -**Container**: `rq-worker` - -### Post-Conversation Pipeline - -**Streaming conversations**: Use streaming transcript saved during conversation. No batch re-transcription. - -**File uploads**: Batch transcription job runs first, then post-conversation jobs depend on it. - -#### 1. Recognize Speakers Job - -**File**: `backends/advanced/src/advanced_omi_backend/workers/transcription_jobs.py` - -**Function**: `recognize_speakers_job()` - -**Process**: -- Sends audio + segments to speaker recognition service -- Identifies speakers using voice embeddings -- Updates segment speaker labels in MongoDB - -**Optional**: Only runs if `DISABLE_SPEAKER_RECOGNITION=false` - -**Container**: `rq-worker` - -**External Service**: `speaker-recognition` container (if enabled) - -#### 2. Memory Extraction Job - -**File**: `backends/advanced/src/advanced_omi_backend/workers/memory_jobs.py` - -**Function**: `memory_extraction_job()` - -**Prerequisite**: Speaker recognition job - -**Process**: -- Uses LLM (OpenAI/Ollama) to extract semantic facts -- Stores embeddings in vector database: - - **Chronicle provider**: Qdrant - - **OpenMemory MCP provider**: External OpenMemory server - -**Container**: `rq-worker` - -**External Services**: -- `ollama` or OpenAI API (LLM) -- `qdrant` or OpenMemory MCP (vector storage) - -#### 3. Generate Title Summary Job - -**File**: `backends/advanced/src/advanced_omi_backend/workers/conversation_jobs.py` - -**Function**: `generate_title_summary_job()` - -**Prerequisite**: Speaker recognition job - -**Process**: -- Uses LLM to generate title, summary, detailed summary -- Updates conversation document in MongoDB - -**Container**: `rq-worker` - -#### 4. Dispatch Conversation Complete Event - -**File**: `backends/advanced/src/advanced_omi_backend/workers/conversation_jobs.py` - -**Function**: `dispatch_conversation_complete_event_job()` - -**Process**: -- Triggers `conversation.complete` plugin event - -**Container**: `rq-worker` - -#### Batch Transcription Job - -**File**: `backends/advanced/src/advanced_omi_backend/workers/transcription_jobs.py` - -**Function**: `transcribe_full_audio_job()` - -**When used**: -- File uploads via `/api/process-audio-files` -- Manual reprocessing via `/api/conversations/{id}/reprocess-transcript` -- NOT used for streaming conversations - -**Process**: -- Reconstructs audio from MongoDB chunks -- Batch transcribes entire audio -- Stores transcript with word-level timestamps - -**Container**: `rq-worker` - -### Session Restart - -**File**: `backends/advanced/src/advanced_omi_backend/utils/conversation_utils.py` - -**Function**: `handle_end_of_conversation()` - -**Process**: -1. Deletes transcription results stream: `transcription:results:{session_id}` -2. Increments `session:conversation_count:{session_id}` -3. Checks if session still active (WebSocket connected) -4. If active: Re-enqueues `stream_speech_detection_job` for next conversation -5. Cleans up consumer groups and pending messages - -**Purpose**: Allows continuous recording with multiple conversations per session - -## Data Storage - -### MongoDB Collections - -**Database**: `chronicle` - -**Container**: `mongo` - -**Volume**: `mongodb_data` (persistent) - -#### `conversations` Collection - -**Schema**: -```python -{ - "_id": ObjectId, - "conversation_id": "uuid-string", - "audio_uuid": "session_id", - "user_id": ObjectId, - "client_id": "user01-phone", - - # Content - "title": "Meeting notes", - "summary": "Discussion about...", - "detailed_summary": "Longer summary...", - "transcript": "Full transcript text", - "audio_path": "1704067200000_user01-phone_convid.wav", - - # Versioned Transcripts - "active_transcript_version": "v1", - "transcript_versions": { - "v1": { - "text": "Full transcript", - "segments": [SpeakerSegment], - "words": [Word], - "provider": "deepgram", - "processing_time_seconds": 45.2, - "created_at": "2025-01-11T12:00:00Z" - } - }, - "segments": [SpeakerSegment], # From active version - - # Metadata - "created_at": "2025-01-11T12:00:00Z", - "completed_at": "2025-01-11T12:15:00Z", - "end_reason": "user_stopped|inactivity_timeout|websocket_disconnect", - "deleted": false -} -``` - -**Indexes**: -- `user_id` (for user-scoped queries) -- `client_id` (for device filtering) -- `conversation_id` (unique) - -#### `audio_chunks` Collection - -**Purpose**: Stores raw audio session data - -**Schema**: -```python -{ - "_id": ObjectId, - "audio_uuid": "session_id", - "user_id": ObjectId, - "client_id": "user01-phone", - "created_at": "2025-01-11T12:00:00Z", - "metadata": { ... } -} -``` - -**Use Case**: Speech-driven architecture (sessions without conversations) - -#### `users` Collection - -**Purpose**: User accounts, authentication, preferences - -**Schema**: -```python -{ - "_id": ObjectId, - "email": "user@example.com", - "hashed_password": "...", - "is_active": true, - "is_superuser": false, - "created_at": "2025-01-11T12:00:00Z" -} -``` - -### Disk Storage - -**Location**: `backends/advanced/data/chunks/` - -**Container**: `chronicle-backend` (volume-mounted) - -**Volume**: `./backends/advanced/data/chunks:/app/data/chunks` - -**File Format**: WAV files - -**Naming Convention**: `{timestamp_ms}_{client_id}_{conversation_id}.wav` - -**Example**: `1704067200000_user01-phone_550e8400-e29b-41d4-a716-446655440000.wav` - -**Created by**: `audio_streaming_persistence_job()` - -**Read by**: Post-conversation transcription jobs - -**Retention**: Manual cleanup (no automatic deletion) - -### Redis Storage - -**Container**: `redis` - -**Volume**: `redis_data` (persistent) - -| Key Pattern | Type | Purpose | TTL | Created By | -|-------------|------|---------|-----|------------| -| `audio:stream:{client_id}` | Stream | Audio chunks for transcription | None (MAXLEN=25k) | AudioStreamProducer | -| `audio:session:{session_id}` | Hash | Session metadata | 1 hour | AudioStreamProducer | -| `transcription:results:{session_id}` | Stream | Transcription results | Manual delete | Transcription consumers | -| `transcription:interim:{session_id}` | Pub/Sub | Real-time interim results | N/A (ephemeral) | Streaming consumer | -| `conversation:current:{session_id}` | String | Current conversation ID | 24 hours | open_conversation_job | -| `audio:file:{conversation_id}` | String | Audio file path | 24 hours | audio_persistence_job | -| `session:conversation_count:{session_id}` | Counter | Conversation count | 1 hour | handle_end_of_conversation | -| `speech_detection_job:{client_id}` | String | Job ID for cleanup | 1 hour | speech_detection_job | -| `rq:job:{job_id}` | Hash | RQ job metadata | 24 hours (default) | RQ | - -### Vector Storage (Memory) - -#### Option A: Qdrant (Chronicle Native Provider) - -**Container**: `qdrant` - -**Volume**: `qdrant_data` (persistent) - -**Ports**: 6333 (HTTP), 6334 (gRPC) - -**Collections**: User-specific collections for semantic embeddings - -**Written by**: `memory_extraction_job()` - -**Read by**: Memory search API (`/api/memories/search`) - -#### Option B: OpenMemory MCP - -**Container**: `openmemory-mcp` (external service) - -**Port**: 8765 - -**Protocol**: MCP (Model Context Protocol) - -**Collections**: Cross-client memory storage - -**Written by**: `memory_extraction_job()` (via MCP provider) - -**Read by**: Memory search API (via MCP provider) - -## Complete End-to-End Flow - -### Step-by-Step Data Journey - -``` -┌─────────────────────────────────────────────────────────────────┐ -│ 1. AUDIO INPUT │ -└─────────────────────────────────────────────────────────────────┘ - WebSocket (/ws) or File Upload (/audio/upload) - ↓ - Container: chronicle-backend - ↓ - AudioStreamProducer.init_session() - - Creates: audio:session:{session_id} (Redis) - - Initializes: In-memory buffer (chronicle-backend container) - ↓ - AudioStreamProducer.add_audio_chunk() - - Buffers: In-memory (chronicle-backend) - - Chunks: Fixed 0.25s chunks (8,000 bytes) - - Publishes: audio:stream:{client_id} (Redis) - - Returns: Redis message IDs - -┌─────────────────────────────────────────────────────────────────┐ -│ 2. SESSION-LEVEL JOB (RQ) │ -└─────────────────────────────────────────────────────────────────┘ - stream_speech_detection_job - Container: rq-worker - ↓ - Polls: TranscriptionResultsAggregator.get_combined_results() - Reads: transcription:results:{session_id} (Redis) - ↓ - Analyzes: Word count, duration, confidence - ↓ - When speech detected: - - Creates: Conversation document (MongoDB) - - Enqueues: open_conversation_job (RQ) - - Exits (restarts when conversation ends) - -┌─────────────────────────────────────────────────────────────────┐ -│ 3a. TRANSCRIPTION CONSUMER (Background Task) │ -└─────────────────────────────────────────────────────────────────┘ - StreamingTranscriptionConsumer (or BaseAudioStreamConsumer) - Container: chronicle-backend (Background Task) - ↓ - Reads: audio:stream:{client_id} (Redis, via XREADGROUP) - Consumer Group: streaming-transcription (or batch provider) - ↓ - STREAMING PATH: - • Opens: WebSocket to Deepgram - • Sends: Chunks immediately (no buffering) - • Publishes Interim: transcription:interim:{session_id} (Redis Pub/Sub) - • Publishes Final: transcription:results:{session_id} (Redis Stream) - • Triggers: Plugins on final results - - BATCH PATH: - • Buffers: 30 chunks (~7.5s) in memory (chronicle-backend) - • Combines: All buffered chunks - • Transcribes: Via provider API (Deepgram/Parakeet) - • Adjusts: Timestamps relative to session start - • Publishes: transcription:results:{session_id} (Redis Stream) - -┌─────────────────────────────────────────────────────────────────┐ -│ 3b. AUDIO PERSISTENCE CONSUMER (RQ Job) │ -└─────────────────────────────────────────────────────────────────┘ - audio_streaming_persistence_job - Container: rq-worker - ↓ - Reads: audio:stream:{client_id} (Redis, via XREADGROUP) - Consumer Group: audio_persistence - ↓ - Monitors: conversation:current:{session_id} (Redis) - ↓ - For each conversation: - • Opens: New WAV file (data/chunks/, chronicle-backend volume) - • Writes: Chunks immediately (real-time) - • Stores: audio:file:{conversation_id} = path (Redis) - ↓ - On rotation signal: - • Closes: Current file - • Opens: New file for next conversation - ↓ - On END signal: - • Closes: File - • Returns: Statistics (chunks, bytes, duration) - -┌─────────────────────────────────────────────────────────────────┐ -│ 4. CONVERSATION-LEVEL JOB (RQ) │ -└─────────────────────────────────────────────────────────────────┘ - open_conversation_job - Container: rq-worker - ↓ - Creates: Conversation document (MongoDB conversations collection) - ↓ - Sets: conversation:current:{session_id} = conversation_id (Redis) - → Triggers audio persistence job to rotate WAV file - ↓ - Polls: TranscriptionResultsAggregator for updates (1s intervals) - Reads: transcription:results:{session_id} (Redis) - ↓ - Tracks: Speech activity (inactivity timeout = 60s) - ↓ - Detects End: - - Inactivity (60s) - - User manual stop - - WebSocket disconnect - ↓ - Waits: For audio file path from persistence job - Reads: audio:file:{conversation_id} (Redis) - ↓ - Saves: audio_path to conversation document (MongoDB) - ↓ - Enqueues: POST-CONVERSATION PIPELINE (RQ) - -┌─────────────────────────────────────────────────────────────────┐ -│ 5. POST-CONVERSATION PIPELINE (RQ - Parallel Jobs) │ -└─────────────────────────────────────────────────────────────────┘ - All jobs run in parallel - Container: rq-worker - ↓ - Reads: Audio file from disk (data/chunks/*.wav) - - ┌─ transcribe_full_audio_job - │ - Batch transcribes: Complete audio file - │ - Validates: Meaningful speech - │ - Marks deleted: If no speech - │ - Stores: MongoDB (transcript, segments, words) - │ - │ └─ recognize_speakers_job (if enabled) - │ - Sends: Audio + segments to speaker-recognition service - │ - Identifies: Speakers via voice embeddings - │ - Updates: MongoDB (segment speaker labels) - │ - │ └─ memory_extraction_job - │ - Uses: LLM (OpenAI/Ollama) to extract facts - │ - Stores: Qdrant (Chronicle) or OpenMemory MCP (vector DB) - │ - └─ generate_title_summary_job - - Uses: LLM (OpenAI/Ollama) - - Generates: Title, summary, detailed_summary - - Stores: MongoDB (conversation document) - - └─ dispatch_conversation_complete_event_job - - Triggers: conversation.complete plugins - - Only for: File uploads (not streaming) - - All results stored: MongoDB conversations collection - -┌─────────────────────────────────────────────────────────────────┐ -│ 6. SESSION RESTART │ -└─────────────────────────────────────────────────────────────────┘ - handle_end_of_conversation() - Container: chronicle-backend - ↓ - Deletes: transcription:results:{session_id} (Redis) - ↓ - Increments: session:conversation_count:{session_id} (Redis) - ↓ - Checks: Session still active? (WebSocket connected) - ↓ - If active: - - Re-enqueues: stream_speech_detection_job (RQ) - - Session remains: "active" for next conversation -``` - -### Data Locations Summary - -| Stage | Data Type | Location | Container | -|-------|-----------|----------|-----------| -| Input | Audio bytes | In-memory buffers | chronicle-backend | -| Producer | Fixed chunks | `audio:stream:{client_id}` | redis | -| Session metadata | Hash | `audio:session:{session_id}` | redis | -| Transcription consumer | Interim results | `transcription:interim:{session_id}` (Pub/Sub) | redis | -| Transcription consumer | Final results | `transcription:results:{session_id}` (Stream) | redis | -| Audio persistence | WAV files | `data/chunks/*.wav` (disk volume) | chronicle-backend (volume) | -| Audio persistence | File paths | `audio:file:{conversation_id}` | redis | -| Conversation job | Conversation doc | MongoDB `conversations` | mongo | -| Post-processing | Transcript | MongoDB `conversations` | mongo | -| Post-processing | Memories | Qdrant or OpenMemory MCP | qdrant / openmemory-mcp | -| Post-processing | Title/summary | MongoDB `conversations` | mongo | - -## Key Design Patterns - -### 1. Speech-Driven Architecture - -**Principle**: Conversations only created when speech is detected - -**Benefits**: -- Clean user experience (no noise-only sessions in UI) -- Reduced memory processing load -- Automatic quality filtering - -**Implementation**: -- `audio_chunks` collection: Always stores sessions -- `conversations` collection: Only created with speech -- Speech detection: Analyzes word count, duration, confidence - -### 2. Versioned Processing - -**Principle**: Store multiple versions of transcripts/memories - -**Benefits**: -- Reprocess without losing originals -- A/B testing different providers -- Rollback to previous versions - -**Implementation**: -- `transcript_versions` dict with version IDs (v1, v2, ...) -- `active_transcript_version` pointer -- `segments` field mirrors active version (quick access) - -### 3. Session-Level vs Conversation-Level - -**Session**: WebSocket connection lifetime (multiple conversations) -- Duration: Up to 24 hours -- Job: `stream_speech_detection_job` -- Purpose: Continuous monitoring for speech - -**Conversation**: Speech burst between silence periods -- Duration: Typically minutes -- Job: `open_conversation_job` -- Purpose: Process single meaningful exchange - -**Benefits**: -- Continuous recording without manual start/stop -- Automatic conversation segmentation -- Efficient resource usage (one session, many conversations) - -### 4. Job Metadata Cascading - -**Pattern**: Parent jobs link to child jobs - -**Example**: -``` -speech_detection_job - ↓ job_id stored in -audio:session:{session_id} - ↓ creates -open_conversation_job - ↓ job_id stored in -conversation document - ↓ creates -post-conversation jobs (parallel) -``` - -**Benefits**: -- Job grouping and cleanup -- Dependency tracking -- Debugging (trace job lineage) - -### 5. Real-Time + Batch Hybrid - -**Real-Time Path** (Streaming Consumer): -- Low latency (interim results in <1 second) -- WebSocket to Deepgram -- Publishes to Pub/Sub for live UI updates - -**Batch Path** (Batch Consumer): -- High accuracy (more context) -- Buffers 7.5 seconds -- API-based transcription - -**Both paths** write to same `transcription:results:{session_id}` stream - -**Benefits**: -- Live UI updates (interim results) -- Accurate final results (batch processing) -- Provider flexibility (switch between streaming/batch) - -### 6. Fan-Out via Redis Consumer Groups - -**Pattern**: Multiple consumer groups read same stream - -**Example**: `audio:stream:{client_id}` consumed by: -- Transcription consumer group -- Audio persistence consumer group - -**Benefits**: -- Parallel processing paths -- Horizontal scaling (multiple workers per group) -- No message duplication (each group processes independently) - -### 7. File Rotation via Redis Signals - -**Pattern**: Conversation job signals persistence job via Redis key - -**Implementation**: -```python -# Conversation job -redis.set(f"conversation:current:{session_id}", conversation_id) - -# Persistence job (monitors key) -current_conv = redis.get(f"conversation:current:{session_id}") -if current_conv != last_conv: - close_current_file() - open_new_file(current_conv) -``` - -**Benefits**: -- Decoupled jobs (no direct communication) -- Real-time file rotation -- Multiple files per session (one per conversation) - -## Failure Handling - -### Transcription Errors - -**Detection**: `stream_speech_detection_job` polls results - -**Action**: -- Sets `transcription_error` field in `audio:session:{session_id}` -- Logs error for debugging -- Session remains active (can recover) - -### No Meaningful Speech - -**Detection**: `transcribe_full_audio_job` validates transcript - -**Criteria**: -- Word count < 10 -- Duration < 5 seconds -- All words low confidence - -**Action**: -- Marks conversation `deleted=True` -- Sets `end_reason="no_meaningful_speech"` -- Conversation hidden from UI - -### Audio File Not Ready - -**Detection**: `open_conversation_job` waits for file path - -**Timeout**: 30 seconds (configurable) - -**Action**: -- Marks conversation `deleted=True` -- Sets `end_reason="audio_file_not_ready"` -- Logs error for debugging - -### Job Zombies (Stuck Jobs) - -**Detection**: `check_job_alive()` utility - -**Method**: Checks Redis for job existence - -**Action**: -- Returns `False` if job missing -- Caller can retry or fail gracefully - -### Dead Consumers - -**Detection**: Consumer group lag monitoring - -**Cleanup**: -- Removes idle consumers (>30 seconds) -- Claims pending messages from dead consumers -- Redistributes to active workers - -### Stream Trimming - -**Prevention**: Streams don't grow unbounded - -**Implementation**: -- `XTRIM MAXLEN 25000` on `audio:stream:{client_id}` -- Keeps last 25k messages (~104 minutes @ 0.25s chunks) -- Deletes `transcription:results:{session_id}` after conversation ends - -### Session Timeout - -**Max Duration**: 24 hours - -**Action**: -- Jobs exit gracefully -- Session marked `"complete"` -- Resources cleaned up (streams deleted, consumer groups removed) - ---- - -## Conclusion - -Chronicle's audio pipeline is designed for: -- **Real-time processing**: Low-latency transcription and live UI updates -- **Horizontal scalability**: Redis Consumer Groups enable multiple workers -- **Fault tolerance**: Decoupled components, job retries, graceful error handling -- **Resource efficiency**: Speech-driven architecture filters noise automatically -- **Flexibility**: Pluggable providers (Deepgram/Parakeet, OpenAI/Ollama, Qdrant/OpenMemory) - -All coordinated through **Redis Streams** for data flow and **RQ** for orchestration, with **MongoDB** for final storage and **disk** for audio archives. diff --git a/Docs/ssl-certificates.md b/Docs/ssl-certificates.md deleted file mode 100644 index 1980c833..00000000 --- a/Docs/ssl-certificates.md +++ /dev/null @@ -1,73 +0,0 @@ -# SSL Certificates & HTTPS - -Chronicle uses automatic HTTPS setup for secure microphone access and remote connections. - -## Why HTTPS is Needed - -Modern browsers require HTTPS for: -- **Microphone access** over network (not localhost) -- **Secure WebSocket connections** (WSS) -- **Remote access** via Tailscale/VPN -- **Production deployments** - -## SSL Implementation - -### Advanced Backend → Caddy - -The main backend uses **Caddy** for automatic HTTPS: - -**Configuration**: `backends/advanced/Caddyfile` -**Activation**: Caddy starts when using `--profile https` or when wizard enables HTTPS -**Certificate**: Self-signed for local/Tailscale IPs, automatic Let's Encrypt for domains - -**Ports**: -- `443` - HTTPS (main access) -- `80` - HTTP (redirects to HTTPS) - -**Access**: `https://localhost` or `https://your-tailscale-ip` - -### Speaker Recognition → nginx - -The speaker recognition service uses **nginx** for HTTPS: - -**Configuration**: `extras/speaker-recognition/nginx.conf` -**Certificate**: Self-signed via `ssl/generate-ssl.sh` - -**Ports**: -- `8444` - HTTPS -- `8081` - HTTP (redirects to HTTPS) - -**Access**: `https://localhost:8444` - -## Setup via Wizard - -When you run `./wizard.sh`, the setup wizard: -1. Asks if you want to enable HTTPS -2. Prompts for your Tailscale IP or domain -3. Generates SSL certificates automatically -4. Configures Caddy/nginx as needed -5. Updates CORS settings for HTTPS origins - -**No manual setup required** - the wizard handles everything. - -## Browser Certificate Warnings - -Since we use self-signed certificates for local/Tailscale IPs, browsers will show security warnings: - -1. Click "Advanced" -2. Click "Proceed to localhost (unsafe)" or similar -3. Microphone access will now work - -For production with real domains, Caddy automatically obtains valid Let's Encrypt certificates. - -## Troubleshooting - -**HTTPS not working**: -- Check Caddy/nginx containers are running: `docker compose ps` -- Verify certificates exist: `ls backends/advanced/ssl/` or `ls extras/speaker-recognition/ssl/` -- Check you're using `https://` not `http://` - -**Microphone not accessible**: -- Ensure you're accessing via HTTPS (not HTTP) -- Accept browser certificate warning -- Verify you're not using `localhost` from remote device (use Tailscale IP instead) diff --git a/LICENSE b/LICENSE index 6e9268e9..4130f88b 100644 --- a/LICENSE +++ b/LICENSE @@ -18,4 +18,4 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. \ No newline at end of file +SOFTWARE. diff --git a/Makefile b/Makefile index d821819e..79617249 100644 --- a/Makefile +++ b/Makefile @@ -74,11 +74,11 @@ help: ## Show detailed help for all targets @echo "🏗️ KUBERNETES SETUP:" @echo " setup-k8s Complete initial Kubernetes setup" @echo " - Configures insecure registry access" - @echo " - Sets up infrastructure services (MongoDB, Qdrant)" + @echo " - Sets up infrastructure services (MongoDB, FalkorDB)" @echo " - Creates shared models PVC" @echo " - Sets up cross-namespace RBAC" @echo " - Generates and applies configuration" - @echo " setup-infrastructure Deploy infrastructure services (MongoDB, Qdrant)" + @echo " setup-infrastructure Deploy infrastructure services (MongoDB, FalkorDB)" @echo " setup-rbac Set up cross-namespace RBAC" @echo " setup-storage-pvc Create shared models PVC" @echo @@ -122,8 +122,9 @@ help: ## Show detailed help for all targets setup-dev: ## Setup development environment (git hooks, pre-commit) @echo "🛠️ Setting up development environment..." @echo "" + @bash scripts/check_uv.sh @echo "📦 Installing pre-commit..." - @pip install pre-commit 2>/dev/null || pip3 install pre-commit + @uv tool install pre-commit @echo "" @echo "🔧 Installing git hooks..." @pre-commit install --hook-type pre-push @@ -148,7 +149,7 @@ setup-k8s: ## Initial Kubernetes setup (registry + infrastructure) @echo @echo "📋 Setup includes:" @echo " • Insecure registry configuration" - @echo " • Infrastructure services (MongoDB, Qdrant)" + @echo " • Infrastructure services (MongoDB, FalkorDB)" @echo " • Shared models PVC for speaker recognition" @echo " • Cross-namespace RBAC" @echo " • Configuration generation and application" @@ -177,13 +178,12 @@ setup-k8s: ## Initial Kubernetes setup (registry + infrastructure) @echo " • Run 'make k8s-status' to check cluster status" @echo " • Run 'make help' for more options" -setup-infrastructure: ## Set up infrastructure services (MongoDB, Qdrant) +setup-infrastructure: ## Set up infrastructure services (MongoDB, FalkorDB) @echo "🏗️ Setting up infrastructure services..." - @echo "Deploying MongoDB and Qdrant to $(INFRASTRUCTURE_NAMESPACE) namespace..." + @echo "Deploying MongoDB and FalkorDB to $(INFRASTRUCTURE_NAMESPACE) namespace..." @set -a; source skaffold.env; set +a; skaffold run --profile=infrastructure --default-repo=$(CONTAINER_REGISTRY) @echo "⏳ Waiting for infrastructure services to be ready..." @kubectl wait --for=condition=ready pod -l app.kubernetes.io/name=mongodb -n $(INFRASTRUCTURE_NAMESPACE) --timeout=300s || echo "⚠️ MongoDB not ready yet" - @kubectl wait --for=condition=ready pod -l app.kubernetes.io/name=qdrant -n $(INFRASTRUCTURE_NAMESPACE) --timeout=300s || echo "⚠️ Qdrant not ready yet" @echo "✅ Infrastructure services deployed" setup-rbac: ## Set up cross-namespace RBAC diff --git a/README-K8S.md b/README-K8S.md index 8bbe22fa..5a243400 100644 --- a/README-K8S.md +++ b/README-K8S.md @@ -47,7 +47,7 @@ This guide walks you through setting up Chronicle from scratch on a fresh Ubuntu 2. **Install Ubuntu Server** - Boot from USB/DVD - Choose "Install Ubuntu Server" - - Configure network with static IP (recommended: 192.168.1.42) + - Configure network with static IP (recommended: 192.168.1.42) - Set hostname (e.g., `k8s_control_plane`) - Create user account - Install OpenSSH server @@ -56,10 +56,10 @@ This guide walks you through setting up Chronicle from scratch on a fresh Ubuntu ```bash # Update system sudo apt update && sudo apt upgrade -y - + # Install essential packages sudo apt install -y curl wget git vim htop tree - + # Configure firewall sudo ufw allow ssh sudo ufw allow 6443 # Kubernetes API @@ -75,11 +75,11 @@ This guide walks you through setting up Chronicle from scratch on a fresh Ubuntu ```bash # Install MicroK8s sudo snap install microk8s --classic - + # Add user to microk8s group sudo usermod -a -G microk8s $USER sudo chown -f -R $USER ~/.kube - + # Log out and back in, or run: newgrp microk8s ``` @@ -88,10 +88,10 @@ This guide walks you through setting up Chronicle from scratch on a fresh Ubuntu ```bash # Start MicroK8s sudo microk8s start - + # Wait for all services to be ready sudo microk8s status --wait-ready - + # Generate join token for worker nodes sudo microk8s add-node # This will output a command like: @@ -103,7 +103,7 @@ This guide walks you through setting up Chronicle from scratch on a fresh Ubuntu ```bash # Start MicroK8s sudo microk8s start - + # Wait for all services to be ready sudo microk8s status --wait-ready ``` @@ -115,7 +115,7 @@ This guide walks you through setting up Chronicle from scratch on a fresh Ubuntu sudo microk8s enable ingress sudo microk8s enable storage sudo microk8s enable metrics-server - + # Wait for add-ons to be ready sudo microk8s status --wait-ready ``` @@ -125,7 +125,7 @@ This guide walks you through setting up Chronicle from scratch on a fresh Ubuntu # Create kubectl alias echo 'alias kubectl="microk8s kubectl"' >> ~/.bashrc source ~/.bashrc - + # Verify installation kubectl get nodes kubectl get pods -A @@ -147,11 +147,11 @@ This guide walks you through setting up Chronicle from scratch on a fresh Ubuntu ```bash # Install MicroK8s sudo snap install microk8s --classic - + # Add user to microk8s group sudo usermod -a -G microk8s $USER sudo chown -f -R $USER ~/.kube - + # Log out and back in, or run: newgrp microk8s ``` @@ -161,7 +161,7 @@ This guide walks you through setting up Chronicle from scratch on a fresh Ubuntu # Use the join command from the control plane # Replace with your actual join token sudo microk8s join 192.168.1.42:25000/xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx - + # Wait for node to join sudo microk8s status --wait-ready ``` @@ -170,7 +170,7 @@ This guide walks you through setting up Chronicle from scratch on a fresh Ubuntu ```bash # On the control plane, verify the worker node joined kubectl get nodes - + # The worker node should show as Ready # Example output: # NAME STATUS ROLES AGE VERSION @@ -182,7 +182,7 @@ This guide walks you through setting up Chronicle from scratch on a fresh Ubuntu ```bash # From your build machine, configure the worker node ./configure-insecure-registry-remote.sh 192.168.1.43 - + # Repeat for each worker node with their respective IPs ``` @@ -194,10 +194,10 @@ This guide walks you through setting up Chronicle from scratch on a fresh Ubuntu ```bash # Enable the built-in MicroK8s registry (not enabled by default) sudo microk8s enable registry - + # Wait for registry to be ready sudo microk8s status --wait-ready - + # Verify registry is running kubectl get pods -n container-registry ``` @@ -213,11 +213,11 @@ This guide walks you through setting up Chronicle from scratch on a fresh Ubuntu ```bash # From your build machine, configure MicroK8s to trust the insecure registry chmod +x scripts/configure-insecure-registry-remote.sh - + # Run the configuration script with your node IP address # Usage: ./scripts/configure-insecure-registry-remote.sh [ssh_user] ./scripts/configure-insecure-registry-remote.sh 192.168.1.42 - + # Or with custom SSH user: # ./scripts/configure-insecure-registry-remote.sh 192.168.1.42 myuser ``` @@ -234,7 +234,7 @@ This guide walks you through setting up Chronicle from scratch on a fresh Ubuntu ```bash # Apply the hostpath provisioner kubectl apply -f k8s-manifests/hostpath-provisioner-official.yaml - + # Verify storage class kubectl get storageclass ``` @@ -281,19 +281,19 @@ chronicle/ > **Note:** The `--recursive` flag downloads the optional Mycelia submodule (an alternative memory backend with timeline visualization). Most deployments use the default Chronicle memory system and don't need Mycelia. 2. **Install Required Tools** - + **kubectl** (required for Skaffold and Helm): - Visit: https://kubernetes.io/docs/tasks/tools/ - Follow the official installation guide for your platform - + **Skaffold**: - Visit: https://skaffold.dev/docs/install/ - - Follow the official installation guide - + - Follow the official installation guide + **Helm**: - Visit: https://helm.sh/docs/intro/install/ - - Follow the official installation guide - + - Follow the official installation guide + **Verify installations:** ```bash kubectl version --client @@ -311,53 +311,52 @@ chronicle/ ```bash # Copy template (if it exists) # cp backends/advanced/.env.template backends/advanced/.env - + # Note: Most environment variables are automatically set by Skaffold during deployment - # including MONGODB_URI, QDRANT_BASE_URL, and other Kubernetes-specific values + # including MONGODB_URI and other Kubernetes-specific values ``` 2. **Configure Skaffold Environment** ```bash # Copy the template file cp skaffold.env.template skaffold.env - + # Edit skaffold.env with your specific values vim skaffold.env - + # Essential variables to configure: REGISTRY=192.168.1.42:32000 # Use IP address for immediate access # Alternative: REGISTRY=k8s_control_plane:32000 (requires adding 'k8s_control_plane 192.168.1.42' to /etc/hosts) BACKEND_IP=192.168.1.42 BACKEND_NODEPORT=30270 WEBUI_NODEPORT=31011 - + # Optional: Configure speaker recognition service HF_TOKEN=hf_your_huggingface_token_here DEEPGRAM_API_KEY=your_deepgram_api_key_here - - # Note: MONGODB_URI and QDRANT_BASE_URL are automatically generated - # by Skaffold based on your infrastructure namespace and service names + + # Note: MONGODB_URI is automatically generated by Skaffold based on + # your infrastructure namespace and service names ``` 3. **Configuration Variables Reference** - + **Required Variables:** - `REGISTRY`: Docker registry for image storage - `BACKEND_IP`: IP address of your Kubernetes control plane - `BACKEND_NODEPORT`: Port for backend service (30000-32767) - `WEBUI_NODEPORT`: Port for WebUI service (30000-32767) - - `INFRASTRUCTURE_NAMESPACE`: Namespace for MongoDB and Qdrant + - `INFRASTRUCTURE_NAMESPACE`: Namespace for MongoDB and FalkorDB - `APPLICATION_NAMESPACE`: Namespace for your application - + **Optional Variables (for Speaker Recognition):** - `HF_TOKEN`: Hugging Face token for Pyannote models - `DEEPGRAM_API_KEY`: Deepgram API key for speech-to-text - `COMPUTE_MODE`: GPU or CPU mode for ML services - `SIMILARITY_THRESHOLD`: Speaker identification threshold - + **Automatically Generated:** - `MONGODB_URI`: Generated from infrastructure namespace - - `QDRANT_BASE_URL`: Generated from infrastructure namespace - `IMAGE_REPO_*`: Generated from Skaffold build process - `IMAGE_TAG_*`: Generated from Skaffold build process @@ -365,11 +364,11 @@ chronicle/ ```bash # Note: Most environment variables are handled by Skaffold automatically # If you need custom environment variables, you can: - + # Option 1: Use the script (if it exists) # chmod +x scripts/generate-helm-configmap.sh # ./scripts/generate-helm-configmap.sh - + # Option 2: Add them directly to the Helm chart values # Edit backends/charts/advanced-backend/values.yaml ``` @@ -460,9 +459,9 @@ This directory contains standalone Kubernetes manifests that are not managed by ```bash # Deploy everything in the correct order ./scripts/deploy-all-services.sh - + # This will automatically: - # - Deploy infrastructure (MongoDB, Qdrant) + # - Deploy infrastructure (MongoDB, FalkorDB) # - Deploy main application (Backend, WebUI) # - Deploy additional services (if configured) # - Wait for each service to be ready @@ -473,13 +472,13 @@ This directory contains standalone Kubernetes manifests that are not managed by ```bash # Deploy infrastructure first skaffold run --profile=infrastructure - + # Wait for infrastructure to be ready kubectl get pods -n root - + # Deploy main application skaffold run --profile=advanced-backend --default-repo=192.168.1.42:32000 - + # Monitor deployment skaffold run --profile=advanced-backend --default-repo=192.168.1.42:32000 --tail ``` @@ -489,10 +488,10 @@ This directory contains standalone Kubernetes manifests that are not managed by # Check all resources kubectl get all -n chronicle kubectl get all -n root - + # Check Ingress kubectl get ingress -n chronicle - + # Check services kubectl get svc -n chronicle ``` @@ -636,7 +635,7 @@ spec: ```bash # Check backend health curl -k https://chronicle.192-168-1-42.nip.io:32623/health - + # Check WebUI curl -k https://chronicle.192-168-1-42.nip.io:32623/ ``` @@ -660,7 +659,7 @@ spec: ```bash # Test registry connectivity (run on Kubernetes node) curl http://k8s_control_plane:32000/v2/ - + # Check MicroK8s containerd config (run on Kubernetes node) sudo cat /var/snap/microk8s/current/args/certs.d/k8s_control_plane:32000/hosts.toml ``` @@ -669,7 +668,7 @@ spec: ```bash # Check storage class (run on build machine) kubectl get storageclass - + # Check persistent volumes (run on build machine) kubectl get pv kubectl get pvc -A @@ -679,7 +678,7 @@ spec: ```bash # Check Ingress controller (run on build machine) kubectl get pods -n ingress-nginx - + # Check Ingress configuration (run on build machine) kubectl describe ingress -n chronicle ``` @@ -696,13 +695,13 @@ spec: # Check GPU operator status (run on build machine) kubectl get pods -n gpu-operator kubectl describe pod -n gpu-operator - + # Check GPU detection on nodes kubectl get nodes -o json | jq '.items[] | {name: .metadata.name, gpu: .status.allocatable."nvidia.com/gpu"}' - + # Check GPU operator logs kubectl logs -n gpu-operator deployment/gpu-operator - + # Verify NVIDIA drivers on host (run on Kubernetes node) nvidia-smi ``` @@ -712,18 +711,18 @@ spec: # Check node connectivity (run on build machine) kubectl get nodes kubectl describe node - + # Check node status and conditions kubectl get nodes -o json | jq '.items[] | {name: .metadata.name, status: .status.conditions[] | select(.type=="Ready") | .status, message: .message}' - + # Check if pods can be scheduled kubectl get pods -A -o wide kubectl describe pod -n - + # Check node resources and capacity kubectl top nodes kubectl describe node | grep -A 10 "Allocated resources" - + # Verify network connectivity between nodes # Run on each node: ping @@ -766,7 +765,7 @@ kubectl rollout restart deployment/webui -n chronicle ```bash # Update system packages sudo apt update && sudo apt upgrade -y - + # Update MicroK8s sudo snap refresh microk8s ``` @@ -776,7 +775,7 @@ kubectl rollout restart deployment/webui -n chronicle # Backup environment files (run on build machine) cp backends/advanced/.env backends/advanced/.env.backup cp skaffold.env skaffold.env.backup - + # Backup Kubernetes manifests (run on build machine) kubectl get all -n chronicle -o yaml > chronicle-backup.yaml kubectl get all -n root -o yaml > infrastructure-backup.yaml @@ -818,8 +817,7 @@ This script handles speaker recognition service deployment with proper environme For additional support: - Check the main [README.md](README.md) -- Review [CLAUDE.md](CLAUDE.md) for development notes -- Check [README-skaffold.md](README-skaffold.md) for Skaffold-specific information +- Review [AGENTS.md](AGENTS.md) for development notes --- diff --git a/README.md b/README.md index 7e342210..a775f83f 100644 --- a/README.md +++ b/README.md @@ -2,9 +2,15 @@ Self-hostable AI system that captures audio/video data from OMI devices and other sources to generate memories, action items, and contextual insights about your conversations and daily interactions. -## Quick Start → [Get Started](quickstart.md) +## Quick Start -Run setup wizard, start services, access at http://localhost:5173 +```bash +curl -fsSL https://raw.githubusercontent.com/SimpleOpenSoftware/chronicle/main/install.sh | sh +``` + +This clones the latest release, installs dependencies, and launches the interactive setup wizard. + +For step-by-step instructions, see the [setup guide](quickstart.md). ## Screenshots @@ -16,9 +22,11 @@ Run setup wizard, start services, access at http://localhost:5173 ![Memory Search](.assets/memory-dashboard.png) -*[Mobile App - Screenshot coming soon]* +### Desktop Menu Bar Client + +![Menu Bar Client](.assets/menu-bar-client.png) -![Mobile App](screenshots/mobile-app.png) +*[Mobile App - Screenshot coming soon]* ## What's Included @@ -30,8 +38,8 @@ Run setup wizard, start services, access at http://localhost:5173 ## Links - **📚 [Setup Guide](quickstart.md)** - Start here -- **🔧 [Full Documentation](CLAUDE.md)** - Comprehensive reference -- **🏗️ [Project Overview](Docs/overview.md)** - Architecture and vision +- **🔧 [Full Documentation](AGENTS.md)** - Comprehensive reference +- **🏗️ [Project Overview](docs/overview.md)** - Architecture and vision - **🐳 [Docker/K8s](README-K8S.md)** - Container deployment ## Project Structure @@ -52,7 +60,7 @@ chronicle/ │ ├── speaker-recognition/ # Voice identification service │ ├── asr-services/ # Offline speech-to-text (Parakeet) │ └── openmemory-mcp/ # External memory server -├── Docs/ # Technical documentation +├── docs/ # Technical documentation ├── config/ # Central configuration files ├── tests/ # Integration & unit tests ├── wizard.py # Root setup orchestrator @@ -77,9 +85,9 @@ chronicle/ │ ┌────────────────────┴────────────────┐ │ │ │ │ │ │ ┌────▼─────┐ ┌───────────┐ ┌──────────▼──┐ │ -│ │ Deepgram │ │ OpenAI │ │ Qdrant │ │ -│ │ STT │ │ LLM │ │ (Vector │ │ -│ │ │ │ │ │ Store) │ │ +│ │ Deepgram │ │ OpenAI │ │ FalkorDB │ │ +│ │ STT │ │ LLM │ │ (Graph + │ │ +│ │ │ │ │ │ Vector) │ │ │ └──────────┘ └───────────┘ └─────────────┘ │ │ │ │ Optional Services: │ @@ -162,7 +170,7 @@ Usecases are numerous - OMI Mentor is one of them. Friend/Omi/pendants are a sma Regardless - this repo will try to do the minimal of this - multiple OMI-like audio devices feeding audio data - and from it: - Memories -- Action items +- Action items - Home automation ## Golden Goals (Not Yet Achieved) @@ -171,4 +179,3 @@ Regardless - this repo will try to do the minimal of this - multiple OMI-like au - **Home automation integration** (planned) - **Multi-device coordination** (planned) - **Visual context capture** (smart glasses integration planned) - diff --git a/app/.easignore b/app/.easignore new file mode 100644 index 00000000..54c84429 --- /dev/null +++ b/app/.easignore @@ -0,0 +1,46 @@ +# NOTE: .easignore fully replaces .gitignore for EAS uploads. +# Must exclude everything we don't want in the build tarball. + +# Dependencies — reinstalled on EAS build server +node_modules/ + +# Expo +.expo/ +dist/ +web-build/ + +# Native build outputs — regenerated on EAS build servers +android/.gradle/ +android/build/ +android/app/build/ +android/app/.cxx/ +ios/build/ +ios/Pods/ +ios/DerivedData/ +ios/*.xcworkspace/xcuserdata/ +ios/*.xcodeproj/xcuserdata/ +ios/*.xcodeproj/project.xcworkspace/xcuserdata/ + +# Local artifacts +*.ipa +*.apk +*.aab +build-*.ipa + +# Credentials (EAS uses server-side credentials) +*.jks +*.p8 +*.p12 +*.key +*.mobileprovision +*.pem + +# Metro / debug +.metro-health-check* +npm-debug.* +yarn-debug.* +yarn-error.* + +# OS / editor +.DS_Store +*.log diff --git a/app/.gitignore b/app/.gitignore index 6bf33056..f6e07bb9 100644 --- a/app/.gitignore +++ b/app/.gitignore @@ -34,4 +34,4 @@ yarn-error.* # typescript *.tsbuildinfo -android/* \ No newline at end of file +android/* diff --git a/app/App.tsx b/app/App.tsx index 60e44938..0397f908 100644 --- a/app/App.tsx +++ b/app/App.tsx @@ -1,3 +1,3 @@ // App.tsx import App from './app/index'; // your actual entry file -export default App; \ No newline at end of file +export default App; diff --git a/app/README.md b/app/README.md index e85e83e5..7041c19b 100644 --- a/app/README.md +++ b/app/README.md @@ -174,7 +174,7 @@ Stream audio directly from your phone's microphone to Chronicle backend, bypassi #### Requirements - **iOS**: iOS 13+ with microphone permissions -- **Android**: Android API 21+ with microphone permissions +- **Android**: Android API 21+ with microphone permissions - **Network**: Stable connection to Chronicle backend - **Backend**: Advanced backend running with `/ws?codec=pcm` endpoint @@ -191,7 +191,7 @@ Stream audio directly from your phone's microphone to Chronicle backend, bypassi - **Network Connection**: Test backend connectivity - **Authentication**: Verify JWT token is valid -#### Poor Audio Quality +#### Poor Audio Quality - **Check Signal Strength**: Ensure stable network connection - **Reduce Background Noise**: Use in quiet environment - **Restart Recording**: Stop and restart phone audio streaming @@ -365,4 +365,4 @@ BluetoothService.onAudioData = (audioBuffer) => { - **[Backend Setup](../backends/)**: Choose and configure backend services - **[Quick Start Guide](../quickstart.md)**: Complete system setup - **[Advanced Backend](../backends/advanced/)**: Full-featured backend option -- **[Simple Backend](../backends/simple/)**: Basic backend for testing \ No newline at end of file +- **[Simple Backend](../backends/simple/)**: Basic backend for testing diff --git a/app/app.json b/app/app.json index 66fbb8c2..3772aa2f 100644 --- a/app/app.json +++ b/app/app.json @@ -1,12 +1,12 @@ { "expo": { - "name": "friend-lite-app", + "name": "chronicle", "slug": "friend-lite-app", - "version": "1.0.0", + "version": "1.0.8", + "scheme": "chronicle", "orientation": "portrait", "icon": "./assets/icon.png", - "entryPoint": "./app/index.tsx", - "userInterfaceStyle": "light", + "userInterfaceStyle": "automatic", "splash": { "image": "./assets/splash.png", "resizeMode": "contain", @@ -17,9 +17,15 @@ ], "ios": { "supportsTablet": true, - "bundleIdentifier": "com.cupbearer5517.friendlite", + "bundleIdentifier": "com.cupbearer5517.chronicle", "infoPlist": { - "NSMicrophoneUsageDescription": "Friend Lite needs access to your microphone to stream audio to the backend for processing." + "NSCameraUsageDescription": "Chronicle uses the camera to scan QR codes for backend connection setup.", + "NSMicrophoneUsageDescription": "Chronicle needs access to your microphone to stream audio to the backend for processing.", + "NSAppTransportSecurity": { + "NSAllowsArbitraryLoads": true, + "NSAllowsLocalNetworking": true + }, + "ITSAppUsesNonExemptEncryption": false } }, "android": { @@ -27,7 +33,7 @@ "foregroundImage": "./assets/adaptive-icon.png", "backgroundColor": "#ffffff" }, - "package": "com.cupbearer5517.friendlite", + "package": "com.cupbearer5517.chronicle", "permissions": [ "android.permission.BLUETOOTH", "android.permission.BLUETOOTH_ADMIN", @@ -36,11 +42,19 @@ "android.permission.FOREGROUND_SERVICE", "android.permission.FOREGROUND_SERVICE_DATA_SYNC", "android.permission.POST_NOTIFICATIONS", - "android.permission.RECORD_AUDIO" - ], - "usesCleartextTraffic": true + "android.permission.RECORD_AUDIO", + "android.permission.CAMERA", + "android.permission.BLUETOOTH", + "android.permission.BLUETOOTH_ADMIN", + "android.permission.BLUETOOTH_CONNECT", + "android.permission.ACCESS_NETWORK_STATE", + "android.permission.FOREGROUND_SERVICE", + "android.permission.FOREGROUND_SERVICE_DATA_SYNC", + "android.permission.POST_NOTIFICATIONS", + "android.permission.RECORD_AUDIO", + "android.permission.CAMERA" + ] }, - "newArchEnabled": true, "plugins": [ [ "@siteed/expo-audio-studio", @@ -49,7 +63,10 @@ "enableNotifications": true, "enableBackgroundAudio": true, "enableDeviceDetection": true, - "iosBackgroundModes": { "useProcessing": true }, + "iosBackgroundModes": { + "useAudio": true, + "useProcessing": true + }, "iosConfig": { "microphoneUsageDescription": "We use the mic for live audio streaming" } @@ -79,6 +96,7 @@ } } ], + "expo-audio", [ "expo-build-properties", { @@ -91,12 +109,27 @@ ] } } - ] + ], + [ + "expo-camera", + { + "cameraPermission": "Chronicle uses the camera to scan QR codes for backend connection setup." + } + ], + "expo-image-picker", + "./plugins/with-ats", + "expo-asset", + "expo-secure-store" ], + "owner": "cupbearer5517", "extra": { "eas": { "projectId": "05d8598e-6fe7-4373-81e4-1654f3d8e181" } + }, + "runtimeVersion": "1.0.0", + "updates": { + "enabled": false } } -} \ No newline at end of file +} diff --git a/app/app/_layout.tsx b/app/app/_layout.tsx index d2a8b0bc..49bc72d5 100644 --- a/app/app/_layout.tsx +++ b/app/app/_layout.tsx @@ -1,5 +1,36 @@ +import { useEffect } from "react"; import { Stack } from "expo-router"; +import { useTheme } from "@/theme"; +import { ConnectionLogProvider } from "@/contexts/ConnectionLogContext"; +import { AppSettingsProvider } from "@/contexts/AppSettingsContext"; +import ErrorBoundary from "@/components/ErrorBoundary"; +import { initLogger, logInfo } from "@/utils/logger"; export default function RootLayout() { - return ; + const { colors, isDark } = useTheme(); + + useEffect(() => { + initLogger().then(() => logInfo('RootLayout', 'app mounted')); + }, []); + + return ( + + + + + + + + + + + + ); } diff --git a/app/app/components/BackendStatus.tsx b/app/app/components/BackendStatus.tsx deleted file mode 100644 index 4f55d37f..00000000 --- a/app/app/components/BackendStatus.tsx +++ /dev/null @@ -1,319 +0,0 @@ -import React, { useState, useEffect } from 'react'; -import { View, Text, TextInput, TouchableOpacity, StyleSheet, Alert, ActivityIndicator } from 'react-native'; - -interface BackendStatusProps { - backendUrl: string; - onBackendUrlChange: (url: string) => void; - jwtToken: string | null; -} - -interface HealthStatus { - status: 'unknown' | 'checking' | 'healthy' | 'unhealthy' | 'auth_required'; - message: string; - lastChecked?: Date; -} - -export const BackendStatus: React.FC = ({ - backendUrl, - onBackendUrlChange, - jwtToken, -}) => { - const [healthStatus, setHealthStatus] = useState({ - status: 'unknown', - message: 'Not checked', - }); - - const checkBackendHealth = async (showAlert: boolean = false) => { - if (!backendUrl.trim()) { - setHealthStatus({ - status: 'unhealthy', - message: 'Backend URL not set', - }); - return; - } - - setHealthStatus({ - status: 'checking', - message: 'Checking connection...', - }); - - try { - // Convert WebSocket URL to HTTP URL for health check - let baseUrl = backendUrl.trim(); - - // Handle different URL formats - if (baseUrl.startsWith('ws://')) { - baseUrl = baseUrl.replace('ws://', 'http://'); - } else if (baseUrl.startsWith('wss://')) { - baseUrl = baseUrl.replace('wss://', 'https://'); - } - - // Remove any WebSocket path if present - baseUrl = baseUrl.split('/ws')[0]; - - // Try health endpoint first - const healthUrl = `${baseUrl}/health`; - console.log('[BackendStatus] Checking health at:', healthUrl); - - const response = await fetch(healthUrl, { - method: 'GET', - headers: { - 'Accept': 'application/json', - 'Content-Type': 'application/json', - ...(jwtToken ? { 'Authorization': `Bearer ${jwtToken}` } : {}), - }, - }); - - console.log('[BackendStatus] Health check response status:', response.status); - - if (response.ok) { - const healthData = await response.json(); - setHealthStatus({ - status: 'healthy', - message: `Connected (${healthData.status || 'OK'})`, - lastChecked: new Date(), - }); - - if (showAlert) { - Alert.alert('Connection Success', 'Successfully connected to backend!'); - } - } else if (response.status === 401 || response.status === 403) { - setHealthStatus({ - status: 'auth_required', - message: 'Authentication required', - lastChecked: new Date(), - }); - - if (showAlert) { - Alert.alert('Authentication Required', 'Please login to access the backend.'); - } - } else { - throw new Error(`HTTP ${response.status}: ${response.statusText}`); - } - } catch (error) { - console.error('[BackendStatus] Health check error:', error); - - let errorMessage = 'Connection failed'; - if (error instanceof Error) { - if (error.message.includes('Network request failed')) { - errorMessage = 'Network request failed - check URL and network connection'; - } else if (error.name === 'AbortError') { - errorMessage = 'Request timeout'; - } else { - errorMessage = error.message; - } - } - - setHealthStatus({ - status: 'unhealthy', - message: errorMessage, - lastChecked: new Date(), - }); - - if (showAlert) { - Alert.alert( - 'Connection Failed', - `Could not connect to backend: ${errorMessage}\n\nMake sure the backend is running and accessible.` - ); - } - } - }; - - // Auto-check health when backend URL or JWT token changes - useEffect(() => { - if (backendUrl.trim()) { - const timer = setTimeout(() => { - checkBackendHealth(false); - }, 500); // Debounce - - return () => clearTimeout(timer); - } - }, [backendUrl, jwtToken]); - - const getStatusColor = (status: HealthStatus['status']): string => { - switch (status) { - case 'healthy': - return '#4CD964'; - case 'checking': - return '#FF9500'; - case 'unhealthy': - return '#FF3B30'; - case 'auth_required': - return '#FF9500'; - default: - return '#8E8E93'; - } - }; - - const getStatusIcon = (status: HealthStatus['status']): string => { - switch (status) { - case 'healthy': - return '✅'; - case 'checking': - return '🔄'; - case 'unhealthy': - return '❌'; - case 'auth_required': - return '🔐'; - default: - return '❓'; - } - }; - - return ( - - Backend Connection - - Backend URL: - - - - - Status: - - {getStatusIcon(healthStatus.status)} - - {healthStatus.message} - - {healthStatus.status === 'checking' && ( - - )} - - - - {healthStatus.lastChecked && ( - - Last checked: {healthStatus.lastChecked.toLocaleTimeString()} - - )} - - - checkBackendHealth(true)} - disabled={healthStatus.status === 'checking'} - > - - {healthStatus.status === 'checking' ? 'Checking...' : 'Test Connection'} - - - - - Enter the WebSocket URL of your backend server. Simple backend: http://localhost:8000/ (no auth). - Advanced backend: http://localhost:8080/ (requires login). Status is automatically checked. - The websocket URL can be different or the same as the HTTP URL, with /ws endpoint and codec parameter (e.g., /ws?codec=pcm) - - - ); -}; - -const styles = StyleSheet.create({ - section: { - marginBottom: 25, - padding: 15, - backgroundColor: 'white', - borderRadius: 10, - shadowColor: '#000', - shadowOffset: { width: 0, height: 1 }, - shadowOpacity: 0.1, - shadowRadius: 3, - elevation: 2, - }, - sectionTitle: { - fontSize: 18, - fontWeight: '600', - marginBottom: 15, - color: '#333', - }, - inputLabel: { - fontSize: 14, - color: '#333', - marginBottom: 5, - fontWeight: '500', - }, - textInput: { - backgroundColor: '#f0f0f0', - borderWidth: 1, - borderColor: '#ddd', - borderRadius: 6, - padding: 10, - fontSize: 14, - width: '100%', - marginBottom: 15, - color: '#333', - }, - statusContainer: { - marginBottom: 15, - padding: 10, - backgroundColor: '#f8f9fa', - borderRadius: 6, - borderWidth: 1, - borderColor: '#e9ecef', - }, - statusRow: { - flexDirection: 'row', - alignItems: 'center', - justifyContent: 'space-between', - }, - statusLabel: { - fontSize: 14, - fontWeight: '500', - color: '#333', - }, - statusValue: { - flexDirection: 'row', - alignItems: 'center', - flex: 1, - justifyContent: 'flex-end', - }, - statusIcon: { - fontSize: 16, - marginRight: 6, - }, - statusText: { - fontSize: 14, - fontWeight: '500', - }, - lastCheckedText: { - fontSize: 12, - color: '#666', - marginTop: 5, - textAlign: 'center', - fontStyle: 'italic', - }, - button: { - backgroundColor: '#007AFF', - paddingVertical: 12, - paddingHorizontal: 20, - borderRadius: 8, - alignItems: 'center', - marginBottom: 10, - elevation: 2, - }, - buttonDisabled: { - backgroundColor: '#A0A0A0', - opacity: 0.7, - }, - buttonText: { - color: 'white', - fontSize: 16, - fontWeight: '600', - }, - helpText: { - fontSize: 12, - color: '#666', - textAlign: 'center', - fontStyle: 'italic', - }, -}); - -export default BackendStatus; \ No newline at end of file diff --git a/app/app/components/DeviceDetails.tsx b/app/app/components/DeviceDetails.tsx deleted file mode 100644 index ebf204c3..00000000 --- a/app/app/components/DeviceDetails.tsx +++ /dev/null @@ -1,343 +0,0 @@ -import React from 'react'; -import { View, Text, TouchableOpacity, StyleSheet, TextInput } from 'react-native'; -import { BleAudioCodec } from 'friend-lite-react-native'; - -interface DeviceDetailsProps { - // Device Info - connectedDeviceId: string | null; - onGetAudioCodec: () => void; - currentCodec: BleAudioCodec | null; - onGetBatteryLevel: () => void; - batteryLevel: number; - - // Audio Listener - isListeningAudio: boolean; - onStartAudioListener: () => void; - onStopAudioListener: () => void; - audioPacketsReceived: number; - - // WebSocket URL for custom backend - webSocketUrl: string; - onSetWebSocketUrl: (url: string) => void; - - // Custom Audio Streamer Status - isAudioStreaming: boolean; - isConnectingAudioStreamer: boolean; - audioStreamerError: string | null; - - // User ID Management - userId: string; - onSetUserId: (userId: string) => void; - - // Audio Listener Retry State - isAudioListenerRetrying?: boolean; - audioListenerRetryAttempts?: number; -} - -export const DeviceDetails: React.FC = ({ - connectedDeviceId, - onGetAudioCodec, - currentCodec, - onGetBatteryLevel, - batteryLevel, - isListeningAudio, - onStartAudioListener, - onStopAudioListener, - audioPacketsReceived, - webSocketUrl, - onSetWebSocketUrl, - isAudioStreaming, - isConnectingAudioStreamer, - audioStreamerError, - userId, - onSetUserId, - isAudioListenerRetrying, - audioListenerRetryAttempts -}) => { - if (!connectedDeviceId) return null; - - - return ( - - Device Functions - - {/* Audio Codec */} - - Get Audio Codec - - {currentCodec && ( - - Current Audio Codec: - {currentCodec} - - )} - - {/* Battery Level */} - - Get Battery Level - - {batteryLevel >= 0 && ( - - Battery Level: - - - {batteryLevel}% - - - )} - - {/* User ID Management */} - - User ID (optional) - Enter User ID (for device identification): - - - - {userId && ( - - Current User ID: - {userId} - - )} - - - {/* Audio Controls */} - - - - {isListeningAudio ? "Stop Audio Listener" : - isAudioListenerRetrying ? "Stop Retry" : "Start Audio Listener"} - - - - {isAudioListenerRetrying && ( - - - 🔄 Retrying audio listener... (Attempt {audioListenerRetryAttempts || 0}/10) - - - )} - - {isListeningAudio && ( - - Audio Packets Received: - {audioPacketsReceived} - - )} - - - {/* Transcription Controls - Entire section REMOVED and replaced by WebSocket URL input */} - - Custom Audio Streaming - Backend WebSocket URL: - - - {/* Display Streamer Status */} - {isConnectingAudioStreamer && ( - Connecting to WebSocket... - )} - {isAudioStreaming && ( - Streaming audio to WebSocket... - )} - {audioStreamerError && ( - Error: {audioStreamerError} - )} - - - - ); -}; - -const styles = StyleSheet.create({ - section: { - marginBottom: 25, - padding: 15, - backgroundColor: 'white', - borderRadius: 10, - shadowColor: '#000', - shadowOffset: { width: 0, height: 1 }, - shadowOpacity: 0.1, - shadowRadius: 3, - elevation: 2, - }, - sectionTitle: { - fontSize: 18, - fontWeight: '600', - marginBottom: 15, - color: '#333', - }, - subSection: { - marginTop: 20, - }, - subSectionTitle: { - fontSize: 16, - fontWeight: '600', - marginBottom: 12, - color: '#444', - }, - button: { - backgroundColor: '#007AFF', - paddingVertical: 12, - paddingHorizontal: 20, - borderRadius: 8, - alignItems: 'center', - elevation: 2, - }, - buttonWarning: { - backgroundColor: '#FF9500', - }, - buttonDisabled: { - backgroundColor: '#A0A0A0', - opacity: 0.7, - }, - buttonSecondary: { - backgroundColor: '#8E8E93', - }, - buttonSecondaryText: { - color: 'white', - }, - buttonText: { - color: 'white', - fontSize: 16, - fontWeight: '600', - }, - infoContainerSM: { - marginTop: 10, - padding: 10, - backgroundColor: '#f0f0f0', - borderRadius: 8, - alignItems: 'center', - }, - infoTitle: { - fontSize: 14, - fontWeight: '500', - color: '#555', - }, - infoValue: { - fontSize: 16, - fontWeight: 'bold', - color: '#007AFF', - marginTop: 5, - }, - infoValueLg: { - fontSize: 18, - fontWeight: 'bold', - color: '#FF9500', - marginTop: 5, - }, - batteryContainer: { - marginTop: 10, - padding: 12, - backgroundColor: '#f0f0f0', - borderRadius: 8, - alignItems: 'center', - borderLeftWidth: 4, - borderLeftColor: '#4CD964', - }, - batteryLevelDisplayContainer: { - width: '100%', - height: 24, - backgroundColor: '#e0e0e0', - borderRadius: 12, - marginTop: 8, - overflow: 'hidden', - position: 'relative', - }, - batteryLevelBar: { - height: '100%', - backgroundColor: '#4CD964', - borderRadius: 12, - position: 'absolute', - left: 0, - top: 0, - }, - batteryLevelText: { - position: 'absolute', - width: '100%', - textAlign: 'center', - lineHeight: 24, - fontSize: 12, - fontWeight: 'bold', - color: '#333', - }, - // Transcription Specific Styles - Some can be repurposed or removed - customStreamerSection: { - marginTop: 20, - paddingTop: 15, - borderTopWidth: 1, - borderTopColor: '#e0e0e0', - // alignItems: 'center', // No longer centering checkbox etc. - }, - inputLabel: { - fontSize: 14, - color: '#333', - marginBottom: 5, - fontWeight: '500', - }, - textInput: { - backgroundColor: '#f0f0f0', - borderWidth: 1, - borderColor: '#ddd', - borderRadius: 6, - padding: 10, - fontSize: 14, - width: '100%', // Ensure input takes full width of its container - marginBottom: 10, - color: '#333', - }, - statusText: { // New style for status messages - marginTop: 8, - fontSize: 13, - color: '#555', - textAlign: 'left', - }, - statusStreaming: { - color: 'green', - }, - statusError: { - color: 'red', - fontWeight: 'bold', - }, - retryContainer: { - marginTop: 10, - padding: 12, - backgroundColor: '#FFF3CD', - borderRadius: 8, - borderLeftWidth: 4, - borderLeftColor: '#FF9500', - }, - retryText: { - fontSize: 14, - color: '#856404', - fontWeight: '500', - textAlign: 'center', - }, -}); - -export default DeviceDetails; \ No newline at end of file diff --git a/app/app/components/DeviceListItem.tsx b/app/app/components/DeviceListItem.tsx deleted file mode 100644 index a8083035..00000000 --- a/app/app/components/DeviceListItem.tsx +++ /dev/null @@ -1,107 +0,0 @@ -import React from 'react'; -import { View, Text, TouchableOpacity, StyleSheet } from 'react-native'; -import { OmiDevice } from 'friend-lite-react-native'; - -interface DeviceListItemProps { - device: OmiDevice; - onConnect: (deviceId: string) => void; - onDisconnect: () => void; - isConnecting: boolean; - connectedDeviceId: string | null; -} - -export const DeviceListItem: React.FC = ({ - device, - onConnect, - onDisconnect, - isConnecting, - connectedDeviceId -}) => { - const isThisDeviceConnected = connectedDeviceId === device.id; - const isAnotherDeviceConnected = connectedDeviceId !== null && connectedDeviceId !== device.id; - - return ( - - - {device.name || 'Unknown Device'} - ID: {device.id} - {device.rssi != null && RSSI: {device.rssi} dBm} - - { - isThisDeviceConnected ? ( - - {isConnecting ? 'Disconnecting...' : 'Disconnect'} - - ) : ( - onConnect(device.id)} - disabled={isConnecting || isAnotherDeviceConnected} // Disable if connecting to this/another device or another device is connected - > - {isConnecting && connectedDeviceId === device.id ? 'Connecting...' : 'Connect'} - - ) - } - - ); -}; - -const styles = StyleSheet.create({ - deviceItem: { - flexDirection: 'row', - justifyContent: 'space-between', - alignItems: 'center', - paddingVertical: 12, - paddingHorizontal: 5, // Added some horizontal padding - borderBottomWidth: 1, - borderBottomColor: '#eee', - }, - deviceInfoContainer: { - flex: 1, // Allow text to take available space and wrap if needed - marginRight: 10, // Space between text and button - }, - deviceName: { - fontSize: 16, - fontWeight: '500', - color: '#333', - }, - deviceInfo: { - fontSize: 12, - color: '#666', - marginTop: 2, - }, - button: { - backgroundColor: '#007AFF', - paddingVertical: 12, - paddingHorizontal: 20, - borderRadius: 8, - alignItems: 'center', - elevation: 1, - }, - smallButton: { - paddingVertical: 8, - paddingHorizontal: 12, - }, - buttonDanger: { - backgroundColor: '#FF3B30', - }, - buttonDisabled: { - backgroundColor: '#A0A0A0', - opacity: 0.7, - }, - buttonText: { - color: 'white', - fontSize: 14, // Slightly smaller for small buttons - fontWeight: '600', - }, -}); - -export default DeviceListItem; \ No newline at end of file diff --git a/app/app/components/ObsidianIngest.tsx b/app/app/components/ObsidianIngest.tsx deleted file mode 100644 index d14ca367..00000000 --- a/app/app/components/ObsidianIngest.tsx +++ /dev/null @@ -1,154 +0,0 @@ - -import React, { useState } from 'react'; -import { View, Text, TextInput, TouchableOpacity, StyleSheet, Alert, ActivityIndicator } from 'react-native'; - -interface ObsidianIngestProps { - backendUrl: string; - jwtToken: string | null; -} - -export const ObsidianIngest: React.FC = ({ - backendUrl, - jwtToken, -}) => { - const [vaultPath, setVaultPath] = useState('/app/data/obsidian_vault'); - const [loading, setLoading] = useState(false); - - const handleIngest = async () => { - if (!backendUrl) { - Alert.alert("Error", "Backend URL not set"); - return; - } - - if (!jwtToken) { - Alert.alert("Authentication Required", "Please login to ingest Obsidian vault."); - return; - } - - setLoading(true); - try { - let baseUrl = backendUrl.trim(); - // Handle different URL formats - if (baseUrl.startsWith('ws://')) { - baseUrl = baseUrl.replace('ws://', 'http://'); - } else if (baseUrl.startsWith('wss://')) { - baseUrl = baseUrl.replace('wss://', 'https://'); - } - baseUrl = baseUrl.split('/ws')[0]; - - const response = await fetch(`${baseUrl}/api/obsidian/ingest`, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'Authorization': `Bearer ${jwtToken}` - }, - body: JSON.stringify({ vault_path: vaultPath }) - }); - - if (response.ok) { - Alert.alert("Success", "Ingestion started in background."); - } else { - const errorText = await response.text(); - Alert.alert("Error", `Ingestion failed: ${response.status} - ${errorText}`); - } - } catch (e) { - Alert.alert("Error", `Network request failed: ${e}`); - } finally { - setLoading(false); - } - }; - - return ( - - Obsidian Ingestion - - Vault Path (Backend Container): - - - - - {loading ? 'Starting Ingestion...' : 'Ingest to Neo4j'} - - - - - Enter the absolute path to the Obsidian vault INSIDE the backend container. - Ensure the folder is mounted to the container. - - - ); -}; - -const styles = StyleSheet.create({ - section: { - marginBottom: 25, - padding: 15, - backgroundColor: 'white', - borderRadius: 10, - shadowColor: '#000', - shadowOffset: { width: 0, height: 1 }, - shadowOpacity: 0.1, - shadowRadius: 3, - elevation: 2, - }, - sectionTitle: { - fontSize: 18, - fontWeight: '600', - marginBottom: 15, - color: '#333', - }, - inputLabel: { - fontSize: 14, - color: '#333', - marginBottom: 5, - fontWeight: '500', - }, - textInput: { - backgroundColor: '#f0f0f0', - borderWidth: 1, - borderColor: '#ddd', - borderRadius: 6, - padding: 10, - fontSize: 14, - width: '100%', - marginBottom: 15, - color: '#333', - }, - button: { - backgroundColor: '#9b59b6', // Purple for Obsidian - paddingVertical: 12, - paddingHorizontal: 20, - borderRadius: 8, - alignItems: 'center', - marginBottom: 10, - elevation: 2, - }, - buttonDisabled: { - backgroundColor: '#A0A0A0', - opacity: 0.7, - }, - buttonText: { - color: 'white', - fontSize: 16, - fontWeight: '600', - }, - helpText: { - fontSize: 12, - color: '#666', - textAlign: 'center', - fontStyle: 'italic', - }, -}); - -export default ObsidianIngest; diff --git a/app/app/components/PhoneAudioButton.tsx b/app/app/components/PhoneAudioButton.tsx deleted file mode 100644 index 1f486e55..00000000 --- a/app/app/components/PhoneAudioButton.tsx +++ /dev/null @@ -1,201 +0,0 @@ -// PhoneAudioButton.tsx -import React from 'react'; -import { - TouchableOpacity, - Text, - View, - StyleSheet, - ActivityIndicator, -} from 'react-native'; - -interface PhoneAudioButtonProps { - isRecording: boolean; - isInitializing: boolean; - isDisabled: boolean; - audioLevel: number; - error: string | null; - onPress: () => void; -} - -const PhoneAudioButton: React.FC = ({ - isRecording, - isInitializing, - isDisabled, - audioLevel, - error, - onPress, -}) => { - - const getButtonStyle = () => { - if (isDisabled && !isRecording) { - return [styles.button, styles.buttonDisabled]; - } - if (isRecording) { - return [styles.button, styles.buttonRecording]; - } - if (error) { - return [styles.button, styles.buttonError]; - } - return [styles.button, styles.buttonIdle]; - }; - - const getButtonText = () => { - if (isInitializing) { - return 'Initializing...'; - } - if (isRecording) { - return 'Stop Phone Audio'; - } - return 'Stream Phone Audio'; - }; - - const getMicrophoneIcon = () => { - if (isRecording) { - return '🎤'; // Recording microphone - } - return '🎙️'; // Idle microphone - }; - - return ( - - - - {isInitializing ? ( - - ) : ( - - {getMicrophoneIcon()} - {getButtonText()} - - )} - - - - {/* Audio Level Indicator */} - {isRecording && ( - - - - - Audio Level - - )} - - {/* Status Message */} - {isRecording && ( - - Streaming audio to backend... - - )} - - {/* Error Message */} - {error && !isRecording && ( - {error} - )} - - {/* Disabled Message */} - {isDisabled && !isRecording && ( - - Disconnect Bluetooth device to use phone audio - - )} - - ); -}; - -const styles = StyleSheet.create({ - container: { - marginVertical: 10, - paddingHorizontal: 20, - }, - buttonWrapper: { - alignSelf: 'stretch', - }, - button: { - flexDirection: 'row', - alignItems: 'center', - justifyContent: 'center', - paddingVertical: 12, - paddingHorizontal: 20, - borderRadius: 8, - minHeight: 48, - }, - buttonContent: { - flexDirection: 'row', - alignItems: 'center', - justifyContent: 'center', - }, - buttonIdle: { - backgroundColor: '#007AFF', - }, - buttonRecording: { - backgroundColor: '#FF3B30', - }, - buttonDisabled: { - backgroundColor: '#C7C7CC', - }, - buttonError: { - backgroundColor: '#FF9500', - }, - buttonText: { - color: '#FFFFFF', - fontSize: 16, - fontWeight: '600', - marginLeft: 8, - }, - icon: { - fontSize: 20, - }, - statusText: { - textAlign: 'center', - marginTop: 8, - fontSize: 12, - color: '#8E8E93', - }, - errorText: { - textAlign: 'center', - marginTop: 8, - fontSize: 12, - color: '#FF3B30', - }, - disabledText: { - textAlign: 'center', - marginTop: 8, - fontSize: 12, - color: '#8E8E93', - fontStyle: 'italic', - }, - audioLevelContainer: { - marginTop: 12, - alignItems: 'center', - }, - audioLevelBackground: { - width: '100%', - height: 4, - backgroundColor: '#E5E5EA', - borderRadius: 2, - overflow: 'hidden', - }, - audioLevelBar: { - height: '100%', - backgroundColor: '#34C759', - borderRadius: 2, - }, - audioLevelText: { - marginTop: 4, - fontSize: 10, - color: '#8E8E93', - }, -}); - -export default PhoneAudioButton; \ No newline at end of file diff --git a/app/app/diagnostics.tsx b/app/app/diagnostics.tsx new file mode 100644 index 00000000..d780a528 --- /dev/null +++ b/app/app/diagnostics.tsx @@ -0,0 +1,221 @@ +import React from 'react'; +import { View, Text, FlatList, TouchableOpacity, StyleSheet, SafeAreaView, Share, Platform, Alert } from 'react-native'; +import { useTheme, ThemeColors } from '@/theme'; +import { useConnectionLog, ConnectionEvent, ConnectionEventType } from '@/contexts/ConnectionLogContext'; +import { getLogPath, readLog, clearLog } from '@/utils/logger'; + +const EVENT_BADGE_COLORS: Record = { + scan_start: '#007AFF', + scan_stop: '#8E8E93', + scan_result: '#5856D6', + connect_start: '#FF9500', + connect_success: '#34C759', + connect_fail: '#FF3B30', + disconnect: '#FF3B30', + battery_read: '#34C759', + audio_start: '#007AFF', + audio_stop: '#8E8E93', + error: '#FF3B30', + health_ping: '#34C759', + reconnect_attempt: '#FF9500', + reconnect_backoff: '#FF9500', + bt_state_change: '#5856D6', + ws_connecting: '#FF9500', + ws_open: '#34C759', + ws_close: '#FF3B30', + ws_error: '#FF3B30', + ws_reconnect: '#FF9500', + ws_reauth: '#AF52DE', + net_change: '#5856D6', +}; + +function formatTime(date: Date): string { + return date.toLocaleTimeString('en-US', { hour12: false, hour: '2-digit', minute: '2-digit', second: '2-digit' }); +} + +function EventItem({ event, colors }: { event: ConnectionEvent; colors: ThemeColors }) { + const badgeColor = EVENT_BADGE_COLORS[event.type] || colors.textTertiary; + + return ( + + {formatTime(event.timestamp)} + + {event.type.replace(/_/g, ' ')} + + + {event.deviceName && {event.deviceName}} + {event.details && {event.details}} + {event.rssi != null && RSSI: {event.rssi} dBm} + + + ); +} + +const itemStyles = StyleSheet.create({ + row: { + flexDirection: 'row', + alignItems: 'flex-start', + paddingVertical: 8, + paddingHorizontal: 12, + borderBottomWidth: 1, + }, + time: { + fontSize: 11, + fontFamily: 'monospace', + width: 65, + marginTop: 3, + }, + badge: { + paddingHorizontal: 6, + paddingVertical: 2, + borderRadius: 4, + marginRight: 8, + marginTop: 2, + }, + badgeText: { + color: 'white', + fontSize: 10, + fontWeight: '600', + textTransform: 'uppercase', + }, + details: { + flex: 1, + }, + device: { + fontSize: 13, + fontWeight: '500', + }, + detail: { + fontSize: 12, + marginTop: 1, + }, +}); + +export default function DiagnosticsScreen() { + const { colors } = useTheme(); + const { events, clearEvents } = useConnectionLog(); + + const shareLogFile = async () => { + try { + const contents = await readLog(); + if (!contents) { + Alert.alert('No log yet', 'The crash log file is empty.'); + return; + } + if (Platform.OS === 'ios') { + await Share.share({ url: `file://${getLogPath()}`, message: contents.slice(-4000) }); + } else { + await Share.share({ message: contents.slice(-4000) }); + } + } catch (err) { + Alert.alert('Share failed', String(err)); + } + }; + + const wipeLogFile = async () => { + Alert.alert('Clear crash log?', 'Removes the on-device crash log file.', [ + { text: 'Cancel', style: 'cancel' }, + { text: 'Clear', style: 'destructive', onPress: async () => { await clearLog(); } }, + ]); + }; + + return ( + + + Crash Log + {getLogPath()} + + + Share Log File + + + Clear File + + + + + Connection Log ({events.length}) + + Clear + + + + {events.length === 0 ? ( + + No events recorded yet. Scan or connect a device to see events here. + + ) : ( + } + keyExtractor={(item) => item.id} + style={{ backgroundColor: colors.card }} + /> + )} + + ); +} + +const screenStyles = StyleSheet.create({ + header: { + flexDirection: 'row', + justifyContent: 'space-between', + alignItems: 'center', + paddingHorizontal: 16, + paddingVertical: 12, + borderBottomWidth: 1, + }, + title: { + fontSize: 17, + fontWeight: '600', + }, + clearButton: { + paddingHorizontal: 12, + paddingVertical: 6, + borderRadius: 6, + }, + clearText: { + fontSize: 14, + fontWeight: '500', + }, + empty: { + flex: 1, + justifyContent: 'center', + alignItems: 'center', + padding: 40, + }, + emptyText: { + fontSize: 15, + textAlign: 'center', + }, + logBar: { + paddingHorizontal: 16, + paddingVertical: 10, + borderBottomWidth: 1, + }, + logBarTitle: { + fontSize: 15, + fontWeight: '600', + }, + logBarPath: { + fontSize: 10, + fontFamily: 'monospace', + marginTop: 2, + marginBottom: 8, + }, + logBarRow: { + flexDirection: 'row', + gap: 8, + }, + logBtn: { + flex: 1, + paddingHorizontal: 12, + paddingVertical: 8, + borderRadius: 6, + alignItems: 'center', + }, + logBtnText: { + fontSize: 13, + fontWeight: '500', + }, +}); diff --git a/app/app/hooks/.gitkeep b/app/app/hooks/.gitkeep deleted file mode 100644 index 0519ecba..00000000 --- a/app/app/hooks/.gitkeep +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/app/app/index.tsx b/app/app/index.tsx index 649a2e2b..14180904 100644 --- a/app/app/index.tsx +++ b/app/app/index.tsx @@ -1,600 +1,388 @@ import React, { useRef, useCallback, useEffect, useState } from 'react'; -import { StyleSheet, Text, View, SafeAreaView, ScrollView, Platform, FlatList, ActivityIndicator, Alert, Switch, Button, TouchableOpacity, KeyboardAvoidingView } from 'react-native'; -import { OmiConnection } from 'friend-lite-react-native'; // OmiDevice also comes from here -import { State as BluetoothState } from 'react-native-ble-plx'; // Import State from ble-plx +import { Text, View, SafeAreaView, ScrollView, Platform, FlatList, ActivityIndicator, Alert, Switch, TouchableOpacity, KeyboardAvoidingView, StyleSheet, RefreshControl } from 'react-native'; +import { OmiConnection } from 'friend-lite-react-native'; +import { State as BluetoothState } from 'react-native-ble-plx'; +import { Link } from 'expo-router'; +import Constants from 'expo-constants'; +import { useTheme, ThemeColors } from '@/theme'; // Hooks -import { useBluetoothManager } from './hooks/useBluetoothManager'; -import { useDeviceScanning } from './hooks/useDeviceScanning'; -import { useDeviceConnection } from './hooks/useDeviceConnection'; -import { - saveLastConnectedDeviceId, - getLastConnectedDeviceId, - saveWebSocketUrl, - getWebSocketUrl, - saveUserId, - getUserId, - getAuthEmail, - getJwtToken, -} from './utils/storage'; -import { useAudioListener } from './hooks/useAudioListener'; -import { useAudioStreamer } from './hooks/useAudioStreamer'; -import { usePhoneAudioRecorder } from './hooks/usePhoneAudioRecorder'; +import { useBluetoothManager } from '@/hooks/useBluetoothManager'; +import { useDeviceScanning } from '@/hooks/useDeviceScanning'; +import { useDeviceConnection } from '@/hooks/useDeviceConnection'; +import { useSharedAppSettings } from '@/contexts/AppSettingsContext'; +import { useAutoReconnect } from '@/hooks/useAutoReconnect'; +import { useAudioStreamingOrchestrator } from '@/hooks/useAudioStreamingOrchestrator'; +import { useAudioListener } from '@/hooks/useAudioListener'; +import { useAudioStreamer } from '@/hooks/useAudioStreamer'; +import { usePhoneAudioRecorder } from '@/hooks/usePhoneAudioRecorder'; +import { usePhoneAudioDevices } from '@/hooks/usePhoneAudioDevices'; +import { useBatteryMonitor } from '@/hooks/useBatteryMonitor'; +import { useBackendHealth, isNotConfigured } from '@/hooks/useBackendHealth'; +import { saveLastConnectedDeviceId } from '@/utils/storage'; // Components -import BluetoothStatusBanner from './components/BluetoothStatusBanner'; -import ScanControls from './components/ScanControls'; -import DeviceListItem from './components/DeviceListItem'; -import DeviceDetails from './components/DeviceDetails'; -import AuthSection from './components/AuthSection'; -import BackendStatus from './components/BackendStatus'; -import ObsidianIngest from './components/ObsidianIngest'; -import PhoneAudioButton from './components/PhoneAudioButton'; +import BluetoothStatusBanner from '@/components/BluetoothStatusBanner'; +import ScanControls from '@/components/ScanControls'; +import DeviceListItem from '@/components/DeviceListItem'; +import DeviceDetails from '@/components/DeviceDetails'; +import PhoneAudioButton from '@/components/PhoneAudioButton'; +import PhoneAudioMicPicker from '@/components/PhoneAudioMicPicker'; export default function App() { - // Initialize OmiConnection + const { colors } = useTheme(); + const s = createStyles(colors); const omiConnection = useRef(new OmiConnection()).current; - - // Filter state const [showOnlyOmi, setShowOnlyOmi] = useState(false); + const [activeTab, setActiveTab] = useState<'backend' | 'connection'>('backend'); - // State for remembering the last connected device - const [lastKnownDeviceId, setLastKnownDeviceId] = useState(null); - const [isAttemptingAutoReconnect, setIsAttemptingAutoReconnect] = useState(false); - const [triedAutoReconnectForCurrentId, setTriedAutoReconnectForCurrentId] = useState(false); - - // State for WebSocket URL for custom audio streaming - const [webSocketUrl, setWebSocketUrl] = useState(''); - - // State for User ID - const [userId, setUserId] = useState(''); - - // Authentication state - const [isAuthenticated, setIsAuthenticated] = useState(false); - const [currentUserEmail, setCurrentUserEmail] = useState(null); - const [jwtToken, setJwtToken] = useState(null); - - // Bluetooth Management Hook - const { - bleManager, - bluetoothState, - permissionGranted, - requestBluetoothPermission, - isPermissionsLoading, - } = useBluetoothManager(); - - // Custom Audio Streamer Hook - const audioStreamer = useAudioStreamer(); - - // Phone Audio Recorder Hook - const phoneAudioRecorder = usePhoneAudioRecorder(); - const [isPhoneAudioMode, setIsPhoneAudioMode] = useState(false); + // Bluetooth + const { bleManager, bluetoothState, permissionGranted, requestBluetoothPermission, isPermissionsLoading } = useBluetoothManager(); + // Settings (must be before audioStreamer so the token refresh callback can reference it) + const settings = useSharedAppSettings(); - const { - isListeningAudio: isOmiAudioListenerActive, - audioPacketsReceived, - startAudioListener: originalStartAudioListener, - stopAudioListener: originalStopAudioListener, - isRetrying: isAudioListenerRetrying, - retryAttempts: audioListenerRetryAttempts, - } = useAudioListener( - omiConnection, - () => !!deviceConnection.connectedDeviceId - ); + // Live backend reachability (Connection Doctor), re-probed on pull-to-refresh. + const { healthStatus, checkBackendHealth } = useBackendHealth(settings.webSocketUrl, settings.jwtToken); + const [refreshing, setRefreshing] = useState(false); - // Refs to hold the current state for onDeviceDisconnect without causing re-memoization - const isOmiAudioListenerActiveRef = useRef(isOmiAudioListenerActive); - const isAudioStreamingRef = useRef(audioStreamer.isStreaming); + // Audio + const audioStreamer = useAudioStreamer({ + autoReconnectEnabled: settings.autoReconnectEnabled, + onTokenRefreshed: (newToken) => { + // Update app-level auth state when auto-re-login refreshes the token + if (settings.currentUserEmail) { + settings.handleAuthStatusChange(true, settings.currentUserEmail, newToken); + } + }, + }); + const phoneAudioRecorder = usePhoneAudioRecorder(); + const phoneAudioDevices = usePhoneAudioDevices(); - useEffect(() => { - isOmiAudioListenerActiveRef.current = isOmiAudioListenerActive; - }, [isOmiAudioListenerActive]); + const { isListeningAudio: isOmiAudioListenerActive, audioPacketsReceived, startAudioListener: originalStartAudioListener, stopAudioListener: originalStopAudioListener, isRetrying: isAudioListenerRetrying, retryAttempts: audioListenerRetryAttempts } = useAudioListener(omiConnection, () => !!deviceConnection.connectedDeviceId); - useEffect(() => { - isAudioStreamingRef.current = audioStreamer.isStreaming; - }, [audioStreamer.isStreaming]); - - // Now define the stable onDeviceConnect and onDeviceDisconnect callbacks + // Refs for disconnect cleanup + const isOmiAudioListenerActiveRef = useRef(isOmiAudioListenerActive); + const isAudioStreamingRef = useRef(audioStreamer.isStreaming); + // Track if audio pipeline was active before BLE disconnect (for auto-restart on reconnect) + const wasStreamingBeforeDisconnectRef = useRef(false); + useEffect(() => { isOmiAudioListenerActiveRef.current = isOmiAudioListenerActive; }, [isOmiAudioListenerActive]); + useEffect(() => { isAudioStreamingRef.current = audioStreamer.isStreaming; }, [audioStreamer.isStreaming]); + + // Refs to break the declaration-order cycle: + // onDeviceConnect/onDeviceDisconnect need orchestrator + autoReconnect, + // but deviceConnection (which needs those callbacks) must be declared + // before orchestrator and autoReconnect. + type OrchestratorHandle = ReturnType; + type AutoReconnectHandle = ReturnType; + const orchestratorRef = useRef(null); + const autoReconnectRef = useRef(null); + + // Device callbacks const onDeviceConnect = useCallback(async () => { - console.log('[App.tsx] Device connected callback.'); - const deviceIdToSave = omiConnection.connectedDeviceId; // Corrected: Use property from OmiConnection instance - + const deviceIdToSave = omiConnection.connectedDeviceId; if (deviceIdToSave) { - console.log('[App.tsx] Saving connected device ID to storage:', deviceIdToSave); await saveLastConnectedDeviceId(deviceIdToSave); - setLastKnownDeviceId(deviceIdToSave); // Update state for consistency - setTriedAutoReconnectForCurrentId(false); // Reset if a new device connects successfully - } else { - console.warn('[App.tsx] onDeviceConnect: Could not determine connected device ID to save. omiConnection.connectedDeviceId was null/undefined.'); + autoReconnectRef.current?.setLastKnownDeviceId(deviceIdToSave); + autoReconnectRef.current?.setTriedAutoReconnectForCurrentId(false); } - // Actions on connect (e.g., auto-fetch codec/battery) - }, [omiConnection]); // saveLastConnectedDeviceId is stable, omiConnection is stable ref - const onDeviceDisconnect = useCallback(async () => { - console.log('[App.tsx] Device disconnected callback.'); - if (isOmiAudioListenerActiveRef.current) { - console.log('[App.tsx] Disconnect: Stopping audio listener.'); - await originalStopAudioListener(); + // Auto-restart audio pipeline if it was active before BLE disconnect + if (wasStreamingBeforeDisconnectRef.current) { + wasStreamingBeforeDisconnectRef.current = false; + console.log('[App] BLE reconnected — auto-restarting audio pipeline'); + // Short delay to let BLE connection stabilize + setTimeout(() => { + orchestratorRef.current?.handleStartAudioListeningAndStreaming().catch(err => { + console.error('[App] Failed to auto-restart audio pipeline:', err); + }); + }, 1000); } - if (isAudioStreamingRef.current) { - console.log('[App.tsx] Disconnect: Stopping custom audio streaming.'); - audioStreamer.stopStreaming(); + }, [omiConnection]); + + const onDeviceDisconnect = useCallback(async () => { + // Remember if audio was active so we can auto-restart on reconnect + if (isOmiAudioListenerActiveRef.current || isAudioStreamingRef.current) { + wasStreamingBeforeDisconnectRef.current = true; } - // Also stop phone audio if it's running + + // Stop audio listener (BLE is gone, can't read audio) + if (isOmiAudioListenerActiveRef.current) await originalStopAudioListener(); + + // Keep WebSocket alive — it will reconnect or idle until BLE comes back. + // Only stop WebSocket for phone audio mode (no BLE needed there). if (phoneAudioRecorder.isRecording) { - console.log('[App.tsx] Disconnect: Stopping phone audio recording.'); + audioStreamer.stopStreaming(); await phoneAudioRecorder.stopRecording(); - setIsPhoneAudioMode(false); + orchestratorRef.current?.setIsPhoneAudioMode(false); } - }, [originalStopAudioListener, audioStreamer.stopStreaming, phoneAudioRecorder.stopRecording, phoneAudioRecorder.isRecording, setIsPhoneAudioMode]); + }, [originalStopAudioListener, audioStreamer.stopStreaming, phoneAudioRecorder.stopRecording, phoneAudioRecorder.isRecording]); - // Initialize Device Connection hook, passing the memoized callbacks - const deviceConnection = useDeviceConnection( - omiConnection, - onDeviceDisconnect, - onDeviceConnect - ); - - // Effect to load settings on app startup - useEffect(() => { - const loadSettings = async () => { - const deviceId = await getLastConnectedDeviceId(); - if (deviceId) { - console.log('[App.tsx] Loaded last known device ID from storage:', deviceId); - setLastKnownDeviceId(deviceId); - setTriedAutoReconnectForCurrentId(false); - } else { - console.log('[App.tsx] No last known device ID found in storage. Auto-reconnect will not be attempted.'); - setLastKnownDeviceId(null); // Explicitly ensure it's null - setTriedAutoReconnectForCurrentId(true); // Mark that we shouldn't try (as no ID is known) - } - - const storedWsUrl = await getWebSocketUrl(); - if (storedWsUrl) { - console.log('[App.tsx] Loaded WebSocket URL from storage:', storedWsUrl); - setWebSocketUrl(storedWsUrl); - } else { - // Set default to simple backend - const defaultUrl = 'ws://localhost:8000/ws'; - console.log('[App.tsx] No stored WebSocket URL, setting default for simple backend:', defaultUrl); - setWebSocketUrl(defaultUrl); - await saveWebSocketUrl(defaultUrl); - } - - const storedUserId = await getUserId(); - if (storedUserId) { - console.log('[App.tsx] Loaded User ID from storage:', storedUserId); - setUserId(storedUserId); - } - - // Load authentication data - const storedEmail = await getAuthEmail(); - const storedToken = await getJwtToken(); - if (storedEmail && storedToken) { - console.log('[App.tsx] Loaded auth data from storage for:', storedEmail); - setCurrentUserEmail(storedEmail); - setJwtToken(storedToken); - setIsAuthenticated(true); - } - }; - loadSettings(); - }, []); + const deviceConnection = useDeviceConnection(omiConnection, onDeviceDisconnect, onDeviceConnect); + // Battery monitor + const batteryMonitor = useBatteryMonitor({ + connectedDeviceId: deviceConnection.connectedDeviceId, + getBatteryLevel: deviceConnection.getRawBatteryLevel, + onConnectionLost: deviceConnection.disconnectFromDevice, + }); - // Device Scanning Hook - const { - devices: scannedDevices, - scanning, - startScan, - stopScan: stopDeviceScanAction, - } = useDeviceScanning( - bleManager, // From useBluetoothManager - omiConnection, - permissionGranted, // From useBluetoothManager - bluetoothState === BluetoothState.PoweredOn, // Derived from useBluetoothManager - requestBluetoothPermission // From useBluetoothManager, should be stable - ); - - // Effect for attempting auto-reconnection - useEffect(() => { - if ( - bluetoothState === BluetoothState.PoweredOn && - permissionGranted && - lastKnownDeviceId && - !deviceConnection.connectedDeviceId && // Only if not already connected - !deviceConnection.isConnecting && // Only if not currently trying to connect by other means - !scanning && // Only if not currently scanning - !isAttemptingAutoReconnect && // Only if not already attempting auto-reconnect - !triedAutoReconnectForCurrentId // Only try once per loaded/set lastKnownDeviceId - ) { - const attemptAutoConnect = async () => { - console.log(`[App.tsx] Attempting to auto-reconnect to device: ${lastKnownDeviceId}`); - setIsAttemptingAutoReconnect(true); - setTriedAutoReconnectForCurrentId(true); // Mark that we've initiated an attempt for this ID - try { - // useDeviceConnection.connectToDevice can take a device ID string directly - await deviceConnection.connectToDevice(lastKnownDeviceId); - // If connectToDevice throws, catch block handles it. - // If it resolves, the connection attempt was made. - // The onDeviceConnect callback will be triggered if successful. - console.log(`[App.tsx] Auto-reconnect attempt initiated for ${lastKnownDeviceId}. Waiting for connection event.`); - // Removed the if(success) block as connectToDevice is void - } catch (error) { - console.error(`[App.tsx] Error auto-reconnecting to ${lastKnownDeviceId}:`, error); - // Clear the problematic device ID from storage and state - if (lastKnownDeviceId) { // Ensure we have an ID to clear - console.log(`[App.tsx] Clearing problematic device ID ${lastKnownDeviceId} from storage due to auto-reconnect failure.`); - await saveLastConnectedDeviceId(null); // Clears from AsyncStorage - setLastKnownDeviceId(null); // Clears from current app state - } - } finally { - setIsAttemptingAutoReconnect(false); - } - }; - attemptAutoConnect(); - } - }, [ + // Auto-reconnect + const autoReconnect = useAutoReconnect({ bluetoothState, permissionGranted, - lastKnownDeviceId, - deviceConnection.connectedDeviceId, - deviceConnection.isConnecting, - scanning, - deviceConnection.connectToDevice, // Stable function from the hook - triedAutoReconnectForCurrentId, - isAttemptingAutoReconnect, // Added to prevent re-triggering while one is in progress - // Added saveLastConnectedDeviceId and setLastKnownDeviceId to dependency array if they were not already implicitly covered - // saveLastConnectedDeviceId is an import, setLastKnownDeviceId is a state setter - typically stable - ]); - - const handleStartAudioListeningAndStreaming = useCallback(async () => { - if (!webSocketUrl || webSocketUrl.trim() === '') { - Alert.alert('WebSocket URL Required', 'Please enter the WebSocket URL for streaming.'); - return; - } - if (!omiConnection.isConnected() || !deviceConnection.connectedDeviceId) { - Alert.alert('Device Not Connected', 'Please connect to an OMI device first.'); - return; - } - - try { - let finalWebSocketUrl = webSocketUrl.trim(); - - // Check if this is the advanced backend (requires authentication) or simple backend - const isAdvancedBackend = jwtToken && isAuthenticated; - - if (isAdvancedBackend) { - // Advanced backend: include JWT token and device parameters - const params = new URLSearchParams(); - params.append('token', jwtToken); - - if (userId && userId.trim() !== '') { - params.append('device_name', userId.trim()); - console.log('[App.tsx] Using advanced backend with token and device_name:', userId.trim()); - } else { - params.append('device_name', 'phone'); // Default device name - console.log('[App.tsx] Using advanced backend with token and default device_name'); - } - - const separator = webSocketUrl.includes('?') ? '&' : '?'; - finalWebSocketUrl = `${webSocketUrl}${separator}${params.toString()}`; - console.log('[App.tsx] Advanced backend WebSocket URL constructed (token hidden for security)'); - } else { - // Simple backend: use URL as-is without authentication - console.log('[App.tsx] Using simple backend without authentication:', finalWebSocketUrl); - } - - // Start custom WebSocket streaming first - await audioStreamer.startStreaming(finalWebSocketUrl); - - // Then start OMI audio listener - await originalStartAudioListener(async (audioBytes) => { - const wsReadyState = audioStreamer.getWebSocketReadyState(); - if (wsReadyState === WebSocket.OPEN && audioBytes.length > 0) { - await audioStreamer.sendAudio(audioBytes); - } - }); - } catch (error) { - console.error('[App.tsx] Error starting audio listening/streaming:', error); - Alert.alert('Error', 'Could not start audio listening or streaming.'); - // Ensure cleanup if one part started but the other failed - if (audioStreamer.isStreaming) audioStreamer.stopStreaming(); - } - }, [originalStartAudioListener, audioStreamer, webSocketUrl, userId, omiConnection, deviceConnection.connectedDeviceId, jwtToken, isAuthenticated]); - - const handleStopAudioListeningAndStreaming = useCallback(async () => { - console.log('[App.tsx] Stopping audio listening and streaming.'); - await originalStopAudioListener(); - audioStreamer.stopStreaming(); - }, [originalStopAudioListener, audioStreamer]); - - // Phone Audio Streaming Functions - const handleStartPhoneAudioStreaming = useCallback(async () => { - if (!webSocketUrl || webSocketUrl.trim() === '') { - Alert.alert('WebSocket URL Required', 'Please enter the WebSocket URL for streaming.'); - return; - } - - try { - let finalWebSocketUrl = webSocketUrl.trim(); - - // Convert HTTP/HTTPS to WS/WSS protocol - finalWebSocketUrl = finalWebSocketUrl.replace(/^http:/, 'ws:').replace(/^https:/, 'wss:'); - - // Ensure /ws endpoint is included - if (!finalWebSocketUrl.includes('/ws')) { - // Remove trailing slash if present, then add /ws - finalWebSocketUrl = finalWebSocketUrl.replace(/\/$/, '') + '/ws'; - } - - // Add codec parameter if not present - if (!finalWebSocketUrl.includes('codec=')) { - const separator = finalWebSocketUrl.includes('?') ? '&' : '?'; - finalWebSocketUrl = finalWebSocketUrl + separator + 'codec=pcm'; - } - - // Check if this is the advanced backend (requires authentication) or simple backend - const isAdvancedBackend = jwtToken && isAuthenticated; - - if (isAdvancedBackend) { - // Advanced backend: include JWT token and device parameters - const params = new URLSearchParams(); - params.append('token', jwtToken); - - const deviceName = userId && userId.trim() !== '' ? userId.trim() : 'phone-mic'; - params.append('device_name', deviceName); - console.log('[App.tsx] Using advanced backend with token and device_name:', deviceName); - - const separator = finalWebSocketUrl.includes('?') ? '&' : '?'; - finalWebSocketUrl = `${finalWebSocketUrl}${separator}${params.toString()}`; - console.log('[App.tsx] Advanced backend WebSocket URL constructed for phone audio'); - } else { - // Simple backend: use URL as-is without authentication - console.log('[App.tsx] Using simple backend without authentication for phone audio'); - } + deviceConnection, + scanning: false, + autoReconnectEnabled: settings.autoReconnectEnabled, + }); - // Start WebSocket streaming first - await audioStreamer.startStreaming(finalWebSocketUrl); - - // Start phone audio recording - await phoneAudioRecorder.startRecording(async (pcmBuffer) => { - const wsReadyState = audioStreamer.getWebSocketReadyState(); - if (wsReadyState === WebSocket.OPEN && pcmBuffer.length > 0) { - await audioStreamer.sendAudio(pcmBuffer); - } - }); - - setIsPhoneAudioMode(true); - console.log('[App.tsx] Phone audio streaming started successfully'); - } catch (error) { - console.error('[App.tsx] Error starting phone audio streaming:', error); - Alert.alert('Error', 'Could not start phone audio streaming.'); - // Ensure cleanup if one part started but the other failed - if (audioStreamer.isStreaming) audioStreamer.stopStreaming(); - if (phoneAudioRecorder.isRecording) await phoneAudioRecorder.stopRecording(); - setIsPhoneAudioMode(false); - } - }, [audioStreamer, phoneAudioRecorder, webSocketUrl, userId, jwtToken, isAuthenticated]); - - const handleStopPhoneAudioStreaming = useCallback(async () => { - console.log('[App.tsx] Stopping phone audio streaming.'); - await phoneAudioRecorder.stopRecording(); - audioStreamer.stopStreaming(); - setIsPhoneAudioMode(false); - }, [phoneAudioRecorder, audioStreamer]); - - const handleTogglePhoneAudio = useCallback(async () => { - if (isPhoneAudioMode || phoneAudioRecorder.isRecording) { - await handleStopPhoneAudioStreaming(); - } else { - await handleStartPhoneAudioStreaming(); - } - }, [isPhoneAudioMode, phoneAudioRecorder.isRecording, handleStartPhoneAudioStreaming, handleStopPhoneAudioStreaming]); + // Scanning + const { devices: scannedDevices, scanning, startScan, stopScan: stopDeviceScanAction } = useDeviceScanning(bleManager, omiConnection, permissionGranted, bluetoothState === BluetoothState.PoweredOn, requestBluetoothPermission); - // Store stable references for cleanup - const cleanupRefs = useRef({ + // Audio orchestrator + const orchestrator = useAudioStreamingOrchestrator({ omiConnection, - bleManager, - disconnectFromDevice: deviceConnection.disconnectFromDevice, - stopAudioStreaming: audioStreamer.stopStreaming, - stopPhoneAudio: phoneAudioRecorder.stopRecording, + deviceConnection, + audioStreamer, + phoneAudioRecorder, + originalStartAudioListener, + originalStopAudioListener, + resolvePhoneInputDeviceId: phoneAudioDevices.resolveEffectiveDeviceId, + settings, }); - // Update refs when functions change - useEffect(() => { - cleanupRefs.current = { - omiConnection, - bleManager, - disconnectFromDevice: deviceConnection.disconnectFromDevice, - stopAudioStreaming: audioStreamer.stopStreaming, - stopPhoneAudio: phoneAudioRecorder.stopRecording, - }; - }); + // Keep forward-declared refs in sync so device callbacks can call through. + orchestratorRef.current = orchestrator; + autoReconnectRef.current = autoReconnect; - // Cleanup only on actual unmount (no dependencies to avoid re-runs) + // Cleanup + const cleanupRefs = useRef({ omiConnection, bleManager, disconnectFromDevice: deviceConnection.disconnectFromDevice, stopAudioStreaming: audioStreamer.stopStreaming, stopPhoneAudio: phoneAudioRecorder.stopRecording }); + useEffect(() => { cleanupRefs.current = { omiConnection, bleManager, disconnectFromDevice: deviceConnection.disconnectFromDevice, stopAudioStreaming: audioStreamer.stopStreaming, stopPhoneAudio: phoneAudioRecorder.stopRecording }; }); useEffect(() => { return () => { - console.log('App unmounting - cleaning up OmiConnection, BleManager, AudioStreamer, and PhoneAudioRecorder'); const refs = cleanupRefs.current; - - if (refs.omiConnection.isConnected()) { - refs.disconnectFromDevice().catch(err => console.error("Error disconnecting in cleanup:", err)); - } - if (refs.bleManager) { - refs.bleManager.destroy(); - } + if (refs.omiConnection.isConnected()) refs.disconnectFromDevice().catch(() => {}); + if (refs.bleManager) refs.bleManager.destroy(); refs.stopAudioStreaming(); - // Phone audio stopRecording now handles inactive state gracefully - refs.stopPhoneAudio().catch(err => console.error("Error stopping phone audio in cleanup:", err)); + refs.stopPhoneAudio().catch(() => {}); }; - }, []); // Empty dependency array - only run on mount/unmount + }, []); const canScan = React.useMemo(() => ( - permissionGranted && - bluetoothState === BluetoothState.PoweredOn && - !isAttemptingAutoReconnect && + permissionGranted && bluetoothState === BluetoothState.PoweredOn && + !autoReconnect.isAttemptingAutoReconnect && !autoReconnect.isRetryingConnection && !deviceConnection.isConnecting && !deviceConnection.connectedDeviceId && - (triedAutoReconnectForCurrentId || !lastKnownDeviceId) - // Removed authentication requirement for scanning - ), [ - permissionGranted, - bluetoothState, - isAttemptingAutoReconnect, - deviceConnection.isConnecting, - deviceConnection.connectedDeviceId, - triedAutoReconnectForCurrentId, - lastKnownDeviceId, - ]); + (autoReconnect.triedAutoReconnectForCurrentId || !autoReconnect.lastKnownDeviceId) + ), [permissionGranted, bluetoothState, autoReconnect.isAttemptingAutoReconnect, autoReconnect.isRetryingConnection, deviceConnection.isConnecting, deviceConnection.connectedDeviceId, autoReconnect.triedAutoReconnectForCurrentId, autoReconnect.lastKnownDeviceId]); const filteredDevices = React.useMemo(() => { - if (!showOnlyOmi) { - return scannedDevices; - } - return scannedDevices.filter(device => { - const name = device.name?.toLowerCase() || ''; - return name.includes('omi') || name.includes('friend'); + if (!showOnlyOmi) return scannedDevices; + return scannedDevices.filter(d => { + const name = d.name?.toLowerCase() || ''; + return name.includes('omi') || name.includes('friend') || name.includes('neo') || name.includes('elato'); }); }, [scannedDevices, showOnlyOmi]); - const handleSetAndSaveWebSocketUrl = useCallback(async (url: string) => { - setWebSocketUrl(url); - await saveWebSocketUrl(url); - }, []); - - const handleSetAndSaveUserId = useCallback(async (id: string) => { - setUserId(id); - await saveUserId(id || null); - }, []); - - // Authentication status change handler - const handleAuthStatusChange = useCallback((authenticated: boolean, email: string | null, token: string | null) => { - setIsAuthenticated(authenticated); - setCurrentUserEmail(email); - setJwtToken(token); - console.log('[App.tsx] Auth status changed:', { authenticated, email: email ? 'logged in' : 'logged out' }); - }, []); - - const handleCancelAutoReconnect = useCallback(async () => { - console.log('[App.tsx] Cancelling auto-reconnection attempt.'); - if (lastKnownDeviceId) { - // Clear the last known device ID to prevent further auto-reconnect attempts in this session - await saveLastConnectedDeviceId(null); - setLastKnownDeviceId(null); - setTriedAutoReconnectForCurrentId(true); // Mark as tried to prevent immediate re-trigger if conditions meet again + // Pull-to-refresh: re-probe backend reachability and refresh live device state. + const onRefresh = useCallback(async () => { + setRefreshing(true); + try { + await Promise.all([ + checkBackendHealth(false), + deviceConnection.connectedDeviceId ? batteryMonitor.refreshBattery() : Promise.resolve(), + phoneAudioDevices.refresh().then(() => {}, () => {}), + ]); + } finally { + setRefreshing(false); } - // Attempt to stop any ongoing connection process - // disconnectFromDevice also sets isConnecting to false internally. - await deviceConnection.disconnectFromDevice(); - setIsAttemptingAutoReconnect(false); // Explicitly set to false to hide the auto-reconnect screen - }, [deviceConnection, lastKnownDeviceId, saveLastConnectedDeviceId, setLastKnownDeviceId, setTriedAutoReconnectForCurrentId, setIsAttemptingAutoReconnect]); - + }, [checkBackendHealth, deviceConnection.connectedDeviceId, batteryMonitor.refreshBattery, phoneAudioDevices.refresh]); + + const bluetoothReady = bluetoothState === BluetoothState.PoweredOn && permissionGranted; + // A fresh install points at localhost (the phone itself), which can never be a + // real backend — treat that (and empty) as "not paired yet" so the setup card + // and health pill reflect reality. + const backendConfigured = !isNotConfigured(settings.webSocketUrl); + // The pill reflects the live probe, not just config: a confirmed-bad probe + // (offline / unreachable / down / unhealthy) turns it red; pending or healthy + // probes fall back to the config+bluetooth view. + const backendDown = ['offline', 'backend_down', 'unreachable', 'unhealthy'].includes(healthStatus.status); + const isOperational = bluetoothReady && backendConfigured && !backendDown; + const healthLabel = backendDown + ? (healthStatus.status === 'offline' ? "You're Offline" : 'Backend Unreachable') + : isOperational ? 'System Operational' : 'Action Needed'; + const healthTone = backendDown ? colors.danger : isOperational ? colors.success : colors.warning; + const batteryDisplay = deviceConnection.connectedDeviceId + ? batteryMonitor.batteryLevel >= 0 ? `${batteryMonitor.batteryLevel}%` : '...' + : '--'; + const streamDisplay = audioStreamer.isStreaming + ? 'Streaming' + : (phoneAudioRecorder.isRecording || orchestrator.isPhoneAudioMode) + ? 'Phone Mic' + : 'Idle'; + + // Loading / auto-reconnect screens if (isPermissionsLoading && bluetoothState === BluetoothState.Unknown) { return ( - - - - {isAttemptingAutoReconnect - ? `Attempting to reconnect to the last device (${lastKnownDeviceId ? lastKnownDeviceId.substring(0, 10) + '...' : ''})...` + + + + {autoReconnect.isAttemptingAutoReconnect + ? `Reconnecting to ${autoReconnect.lastKnownDeviceId?.substring(0, 10)}...` : 'Initializing Bluetooth...'} ); } - if (isAttemptingAutoReconnect) { - return ( - - - - - Attempting to reconnect to the last device ({lastKnownDeviceId ? lastKnownDeviceId.substring(0, 10) + '...' : ''})... - -