diff --git a/.github/workflows/browser-benchmarks.yml b/.github/workflows/browser-benchmarks.yml index e6518270..a54e496b 100644 --- a/.github/workflows/browser-benchmarks.yml +++ b/.github/workflows/browser-benchmarks.yml @@ -67,9 +67,13 @@ jobs: run: | . benchmarks/scripts/load-vault-secrets.sh '^(BROWSERBASE_API_KEY|BROWSERBASE_PROJECT_ID|BROWSER_USE_API_KEY|HYPERBROWSER_API_KEY|KERNEL_API_KEY|NOTTE_API_KEY|STEEL_API_KEY|TILION_API_KEY|TILION_BASE_URL|COMPUTESDK_ADMIN_API_KEY|BENCHMARKS_PLATFORM_API_KEY)' + # All matrix jobs for this workflow run share a single platform run. The + # attempt number is included so re-runs do not rejoin completed runs. + RUN_KEY="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" npx tsx packages/benchsdk-runner/dist/bin.js run benchmarks/browser/browser.bench.ts \ --provider ${{ matrix.provider }} \ - --iterations ${{ github.event_name == 'push' && '10' || github.event.inputs.iterations || '100' }} + --iterations ${{ github.event_name == 'push' && '10' || github.event.inputs.iterations || '100' }} \ + --run-key "$RUN_KEY" - name: Upload results if: always() uses: actions/upload-artifact@v4 diff --git a/.github/workflows/browser-throughput-benchmarks.yml b/.github/workflows/browser-throughput-benchmarks.yml index cd96446a..2a2a3a69 100644 --- a/.github/workflows/browser-throughput-benchmarks.yml +++ b/.github/workflows/browser-throughput-benchmarks.yml @@ -120,9 +120,13 @@ jobs: run: | . benchmarks/scripts/load-vault-secrets.sh '^(BROWSERBASE_API_KEY|BROWSERBASE_PROJECT_ID|BROWSER_USE_API_KEY|HYPERBROWSER_API_KEY|KERNEL_API_KEY|NOTTE_API_KEY|STEEL_API_KEY|TILION_API_KEY|TILION_BASE_URL|COMPUTESDK_ADMIN_API_KEY|BENCHMARKS_PLATFORM_API_KEY)' + # All matrix jobs for this workflow run share a single platform run. The + # attempt number is included so re-runs do not rejoin completed runs. + RUN_KEY="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" npx tsx packages/benchsdk-runner/dist/bin.js run benchmarks/browser/browser-throughput.bench.ts \ --provider ${{ matrix.provider }} \ - --iterations ${{ github.event_name == 'push' && '3' || github.event.inputs.iterations || '100' }} + --iterations ${{ github.event_name == 'push' && '3' || github.event.inputs.iterations || '100' }} \ + --run-key "$RUN_KEY" - name: Upload results if: always() uses: actions/upload-artifact@v4 diff --git a/.github/workflows/sandbox-dax-benchmarks.yml b/.github/workflows/sandbox-dax-benchmarks.yml index c8a46001..bb5fbc92 100644 --- a/.github/workflows/sandbox-dax-benchmarks.yml +++ b/.github/workflows/sandbox-dax-benchmarks.yml @@ -111,9 +111,13 @@ jobs: run: | . benchmarks/scripts/load-vault-secrets.sh '^(ARCHIL_API_KEY|ARCHIL_REGION|ARCHIL_DISK_ID|ARKER_API_KEY|BEAM_TOKEN|BEAM_WORKSPACE_ID|BL_API_KEY|BL_WORKSPACE|CLOUD_RUN_SANDBOX_URL|CLOUD_RUN_SANDBOX_SECRET|CLOUDFLARE_SANDBOX_URL|CLOUDFLARE_SANDBOX_SECRET|CSB_API_KEY|CREATEOS_SANDBOX_API_KEY|DAYTONA_API_KEY|DECLAW_API_KEY|E2B_API_KEY|HOPX_API_KEY|ISORUN_API_KEY|LIGHTNING_API_KEY|MODAL_TOKEN_ID|MODAL_TOKEN_SECRET|NSC_TOKEN|NORTHFLANK_TOKEN|NORTHFLANK_PROJECT_ID|OPENCOMPUTER_API_KEY|OPENCOMPUTER_API_URL|RUNLOOP_API_KEY|SAIL_API_KEY|SANDBOX0_TOKEN|SPRITES_TOKEN|SUPERSERVE_API_KEY|TENKI_API_KEY|TENSORLAKE_API_KEY|UPSTASH_BOX_API_KEY|VERCEL_TOKEN|VERCEL_TEAM_ID|VERCEL_PROJECT_ID|COMPUTESDK_ADMIN_API_KEY|BENCHMARKS_PLATFORM_API_KEY)' + # All matrix jobs for this workflow run share a single platform run. The + # attempt number is included so re-runs do not rejoin completed runs. + RUN_KEY="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" npx tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/dax.bench.ts \ --provider ${{ matrix.provider }} \ - --iterations ${{ github.event.inputs.iterations || '1' }} + --iterations ${{ github.event.inputs.iterations || '1' }} \ + --run-key "$RUN_KEY" - name: Upload results if: always() uses: actions/upload-artifact@v4 diff --git a/.github/workflows/snapshot-fork-benchmarks.yml b/.github/workflows/snapshot-fork-benchmarks.yml index 0bdebff6..57f5375d 100644 --- a/.github/workflows/snapshot-fork-benchmarks.yml +++ b/.github/workflows/snapshot-fork-benchmarks.yml @@ -84,10 +84,14 @@ jobs: run: | . benchmarks/scripts/load-vault-secrets.sh '^(AWS_ACCESS_KEY_ID|AWS_SECRET_ACCESS_KEY|AWS_REGION|S3_BUCKET|S3_SNAPSHOT_ACCESS_KEY_ID|S3_SNAPSHOT_SECRET_ACCESS_KEY|S3_SNAPSHOT_BUCKET|R2_ACCESS_KEY_ID|R2_SECRET_ACCESS_KEY|R2_BUCKET|R2_ACCOUNT_ID|R2_SNAPSHOT_ACCESS_KEY_ID|R2_SNAPSHOT_SECRET_ACCESS_KEY|R2_SNAPSHOT_BUCKET|TIGRIS_STORAGE_ACCESS_KEY_ID|TIGRIS_STORAGE_SECRET_ACCESS_KEY|TIGRIS_STORAGE_BUCKET|TIGRIS_SNAPSHOT_ACCESS_KEY|TIGRIS_SNAPSHOT_SECRET_KEY|TIGRIS_SNAPSHOT_STORAGE_BUCKET|BLOB_READ_WRITE_TOKEN|VERCEL_BLOB_BUCKET|GCS_PROJECT_ID|GCS_BUCKET|GCS_CLIENT_EMAIL|GCS_PRIVATE_KEY|AZURE_ACCOUNT_NAME|AZURE_ACCOUNT_KEY|AZURE_CONTAINER|AZURE_SNAPSHOT_ACCOUNT_NAME|AZURE_SNAPSHOT_ACCOUNT_KEY|AZURE_SNAPSHOT_CONTAINER|TENSORLAKE_API_KEY|TENSORLAKE_API_URL|TENSORLAKE_ORGANIZATION_ID|TENSORLAKE_PROJECT_ID|TENSORLAKE_FILESYSTEM|ARCHIL_S3_ACCESS_KEY_ID|ARCHIL_S3_SECRET_ACCESS_KEY|ARCHIL_BUCKET|ARCHIL_REGION|ARCHIL_BRANCH|COMPUTESDK_ADMIN_API_KEY|BENCHMARKS_PLATFORM_API_KEY)' + # All matrix jobs for this workflow run share a single platform run. The + # attempt number is included so re-runs do not rejoin completed runs. + RUN_KEY="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" npx tsx packages/benchsdk-runner/dist/bin.js run benchmarks/storage/snapshot-fork.bench.ts \ --provider ${{ matrix.provider }} \ --dataset ${{ github.event.inputs.dataset || 'small' }} \ - --iterations ${{ (github.event_name == 'schedule' && '100') || github.event.inputs.iterations || (github.event_name == 'push' && '1') || '3' }} + --iterations ${{ (github.event_name == 'schedule' && '100') || github.event.inputs.iterations || (github.event_name == 'push' && '1') || '3' }} \ + --run-key "$RUN_KEY" - name: Upload results if: always() uses: actions/upload-artifact@v4 diff --git a/.github/workflows/storage-benchmarks.yml b/.github/workflows/storage-benchmarks.yml index 9e4168b1..73aa67b2 100644 --- a/.github/workflows/storage-benchmarks.yml +++ b/.github/workflows/storage-benchmarks.yml @@ -84,11 +84,16 @@ jobs: . benchmarks/scripts/load-vault-secrets.sh '^(AWS_ACCESS_KEY_ID|AWS_SECRET_ACCESS_KEY|AWS_REGION|S3_BUCKET|R2_ACCESS_KEY_ID|R2_SECRET_ACCESS_KEY|R2_BUCKET|R2_ACCOUNT_ID|TIGRIS_STORAGE_ACCESS_KEY_ID|TIGRIS_STORAGE_SECRET_ACCESS_KEY|TIGRIS_STORAGE_BUCKET|BLOB_READ_WRITE_TOKEN|VERCEL_BLOB_BUCKET|GCS_PROJECT_ID|GCS_BUCKET|GCS_CLIENT_EMAIL|GCS_PRIVATE_KEY|AZURE_ACCOUNT_NAME|AZURE_ACCOUNT_KEY|AZURE_CONTAINER|TENSORLAKE_API_KEY|TENSORLAKE_API_URL|TENSORLAKE_ORGANIZATION_ID|TENSORLAKE_PROJECT_ID|TENSORLAKE_FILESYSTEM|ARCHIL_S3_ACCESS_KEY_ID|ARCHIL_S3_SECRET_ACCESS_KEY|ARCHIL_BUCKET|ARCHIL_REGION|ARCHIL_BRANCH|COMPUTESDK_ADMIN_API_KEY|BENCHMARKS_PLATFORM_API_KEY)' FILE_SIZE="${{ matrix.file_size }}" + # Matrix jobs for the same file size share one platform run (the benchmark + # slug is size-independent, so the size is part of the key). The attempt + # number is included so re-runs do not rejoin completed runs. + RUN_KEY="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${FILE_SIZE}" npx tsx packages/benchsdk-runner/dist/bin.js run benchmarks/storage/storage.bench.ts \ --provider ${{ matrix.provider }} \ --file-size $FILE_SIZE \ --concurrency ${{ (github.event_name == 'schedule' && '16') || (github.event_name == 'push' && '8') || github.event.inputs.storage_concurrency || '1' }} \ - --iterations ${{ (github.event_name == 'schedule' && '110') || (github.event_name == 'push' && '10') || github.event.inputs.iterations || '100' }} + --iterations ${{ (github.event_name == 'schedule' && '110') || (github.event_name == 'push' && '10') || github.event.inputs.iterations || '100' }} \ + --run-key "$RUN_KEY" - name: Upload results if: always() uses: actions/upload-artifact@v4