diff --git a/.github/workflows/sandbox-dax-benchmarks.yml b/.github/workflows/sandbox-dax-benchmarks.yml index 6aad5546..01d1c8de 100644 --- a/.github/workflows/sandbox-dax-benchmarks.yml +++ b/.github/workflows/sandbox-dax-benchmarks.yml @@ -110,7 +110,7 @@ jobs: LIGHTNING_INSTANCE_TYPE: cpu-8 # 8 vCPU / 16 GiB, matching the standardized dax profile ARKER_PLATFORMS: graviton4 # dax only; elsewhere Arker picks the platform automatically run: | - . benchmarks/scripts/load-vault-secrets.sh '^(ARCHIL_API_KEY|ARCHIL_REGION|ARCHIL_DISK_ID|ARKER_API_KEY|BEAM_TOKEN|BEAM_WORKSPACE_ID|BL_API_KEY|BL_WORKSPACE|CLOUD_RUN_SANDBOX_URL|CLOUD_RUN_SANDBOX_SECRET|CLOUDFLARE_SANDBOX_URL|CLOUDFLARE_SANDBOX_SECRET|CSB_API_KEY|CREATEOS_SANDBOX_API_KEY|DAYTONA_API_KEY|DECLAW_API_KEY|E2B_API_KEY|HOPX_API_KEY|ISORUN_API_KEY|LIGHTNING_API_KEY|MODAL_TOKEN_ID|MODAL_TOKEN_SECRET|MOSAIC_API_URL|MOSAIC_API_TOKEN|NSC_TOKEN|NORTHFLANK_TOKEN|NORTHFLANK_PROJECT_ID|OPENCOMPUTER_API_KEY|OPENCOMPUTER_API_URL|RUNLOOP_API_KEY|SAIL_API_KEY|SANDBOX0_TOKEN|SPRITES_TOKEN|SUPERSERVE_API_KEY|TENKI_API_KEY|TENSORLAKE_API_KEY|UPSTASH_BOX_API_KEY|VERCEL_TOKEN|VERCEL_TEAM_ID|VERCEL_PROJECT_ID|COMPUTESDK_ADMIN_API_KEY|BENCHMARKS_PLATFORM_API_KEY)' + . benchmarks/scripts/load-vault-secrets.sh '^(ARCHIL_API_KEY|ARCHIL_REGION|ARCHIL_DISK_ID|ARKER_API_KEY|BEAM_TOKEN|BEAM_WORKSPACE_ID|BL_DAX_API_KEY|BL_DAX_WORKSPACE|CLOUD_RUN_SANDBOX_URL|CLOUD_RUN_SANDBOX_SECRET|CLOUDFLARE_SANDBOX_URL|CLOUDFLARE_SANDBOX_SECRET|CSB_API_KEY|CREATEOS_SANDBOX_API_KEY|DAYTONA_API_KEY|DECLAW_API_KEY|E2B_API_KEY|HOPX_API_KEY|ISORUN_API_KEY|LIGHTNING_API_KEY|MODAL_TOKEN_ID|MODAL_TOKEN_SECRET|MOSAIC_API_URL|MOSAIC_API_TOKEN|NSC_TOKEN|NORTHFLANK_TOKEN|NORTHFLANK_PROJECT_ID|OPENCOMPUTER_API_KEY|OPENCOMPUTER_API_URL|RUNLOOP_API_KEY|SAIL_API_KEY|SANDBOX0_TOKEN|SPRITES_TOKEN|SUPERSERVE_API_KEY|TENKI_API_KEY|TENSORLAKE_API_KEY|UPSTASH_BOX_API_KEY|VERCEL_TOKEN|VERCEL_TEAM_ID|VERCEL_PROJECT_ID|COMPUTESDK_ADMIN_API_KEY|BENCHMARKS_PLATFORM_API_KEY)' # All matrix jobs for this workflow run share a single platform run. The # attempt number is included so re-runs do not rejoin completed runs. diff --git a/benchmarks/sandbox/dax.bench.ts b/benchmarks/sandbox/dax.bench.ts index e2b62888..f32a111c 100644 --- a/benchmarks/sandbox/dax.bench.ts +++ b/benchmarks/sandbox/dax.bench.ts @@ -21,6 +21,7 @@ import path from 'node:path'; import { fileURLToPath } from 'node:url'; import { defineBenchmarkConfig, defineTask, TaskError } from '@benchsdk/runner'; import type { JsonObject, TaskStepRecord } from '@benchsdk/client'; +import { blaxel } from '@computesdk/blaxel'; import { VMTier } from '@codesandbox/sdk'; import { withTimeout } from '../src/util/timeout.js'; import { formatError } from '../src/util/error.js'; @@ -35,6 +36,29 @@ const __dirname = path.dirname(fileURLToPath(import.meta.url)); const timeout = 600_000; const destroyTimeoutMs = 15_000; +// DAX uses a dedicated workspace, isolated from the pooled TTI credentials. +function requireDaxEnv(name: 'BL_DAX_API_KEY' | 'BL_DAX_WORKSPACE'): string { + const value = process.env[name]; + if (!value) throw new Error(`${name} is required for the Blaxel DAX benchmark`); + return value; +} + +const daxProviders: ProviderConfig[] = providers.map((provider) => + provider.name === 'blaxel' + ? { + ...provider, + requiredEnvVars: ['BL_DAX_API_KEY', 'BL_DAX_WORKSPACE'], + createCompute: () => + blaxel({ + // Pass explicit values so the SDK cannot fall back to BL_*. + apiKey: requireDaxEnv('BL_DAX_API_KEY'), + workspace: requireDaxEnv('BL_DAX_WORKSPACE'), + region: 'us-was-1', + }), + } + : provider, +); + export const config = defineBenchmarkConfig({ benchmarkSlug: 'sandbox-dax-local', benchmarkName: 'Dax sandbox benchmark (local)', @@ -43,7 +67,7 @@ export const config = defineBenchmarkConfig({ concurrency: 1, groupBy: 'round', defaultProviders: ['e2b', 'modal', 'tensorlake'], - participants: providers, + participants: daxProviders, onComplete: (outcome) => writeDaxLegacyResults(outcome.participants, { resultsDir: path.resolve(__dirname, '../../results/sandbox-dax'), @@ -66,7 +90,18 @@ const DAX_RESOURCE_OPTIONS: Record> = { runloop: { launch_parameters: { resource_size_request: 'CUSTOM_SIZE', custom_cpu_cores: 8, custom_gb_memory: 16 } }, upstash: { size: 'large' }, // large = 8 cores, 16 GB vercel: { resources: { vcpus: 8 } }, // no memory control - blaxel: { memory: 16384 }, // CPU derived: cores = memory_MB / 2048 = 8 + blaxel: { + memory: 16384, // CPU derived: cores = memory_MB / 2048 = 8 + timeout: 900_000, // Server-side TTL backstop if client cleanup cannot run + volumes: [ + { + name: 'dax-root', + type: 'ephemeral', + sizeMb: 8192, + mountPath: '/', // Keep the writable root off RAM + }, + ], + }, beam: { cpu: 8, memory: 16384 }, // cpu = cores, memory = MiB codesandbox: { vmTier: VMTier.Small }, // Small = 8 CPU, 16 GiB daytona: { resources: { cpu: 8, memory: 16 } }, // memory in GiB; requires image-based creation (see providers.ts)