diff --git a/.github/workflows/sandbox-dax-benchmarks.yml b/.github/workflows/sandbox-dax-benchmarks.yml index b829f1c1..6085c6f6 100644 --- a/.github/workflows/sandbox-dax-benchmarks.yml +++ b/.github/workflows/sandbox-dax-benchmarks.yml @@ -6,6 +6,7 @@ on: paths: - 'benchmarks/sandbox/dax.ts' - 'benchmarks/sandbox/providers.ts' + - 'benchmarks/sandbox/run-cloud.ts' - 'benchmarks/src/run.ts' - 'benchmarks/src/merge-results.ts' - 'benchmarks/scripts/dax-benchmark.sh' @@ -63,6 +64,7 @@ jobs: - namespace - northflank - opencomputer + - run-cloud - runloop - superserve - tenki @@ -130,6 +132,8 @@ jobs: NORTHFLANK_PROJECT_ID: ${{ secrets.NORTHFLANK_PROJECT_ID }} OPENCOMPUTER_API_KEY: ${{ secrets.OPENCOMPUTER_API_KEY }} OPENCOMPUTER_API_URL: ${{ secrets.OPENCOMPUTER_API_URL }} + RUN_CLOUD_API_KEY: ${{ secrets.RUN_CLOUD_API_KEY }} + RUN_CLOUD_API_URL: https://api.run.cloud RUNLOOP_API_KEY: ${{ secrets.RUNLOOP_API_KEY }} SUPERSERVE_API_KEY: ${{ secrets.SUPERSERVE_API_KEY }} TENKI_API_KEY: ${{ secrets.TENKI_API_KEY }} diff --git a/benchmarks/.env.example b/benchmarks/.env.example index 38fe7cca..614b7389 100644 --- a/benchmarks/.env.example +++ b/benchmarks/.env.example @@ -114,6 +114,11 @@ LIGHTNING_INSTANCE_TYPE=cpu-1 OPENCOMPUTER_API_KEY=your_opencomputer_api_key OPENCOMPUTER_API_URL=https://app.opencomputer.dev +######### RUN CLOUD ######## +RUN_CLOUD_API_KEY=your_run_cloud_api_key +# Defaults to https://api.run.cloud +RUN_CLOUD_API_URL=https://api.run.cloud + ######### TILION ######## TILION_API_KEY=your_tilion_api_key TILION_BASE_URL=https://api.tilion.dev diff --git a/benchmarks/sandbox/dax.ts b/benchmarks/sandbox/dax.ts index 30f75958..a6d508c5 100644 --- a/benchmarks/sandbox/dax.ts +++ b/benchmarks/sandbox/dax.ts @@ -35,6 +35,7 @@ const DAX_RESOURCE_OPTIONS: Record> = { superserve: { templateId: 'node22-8cpu-16gb' }, // 8 vCPU / 16 GiB template built in the pre-step createos: { shape: 's-8vcpu-16gb', ephemeralDiskMb: 61440 }, // 8 vCPU, 16 GiB RAM, 60 GiB disk opencomputer: { cpuCount: 4, memoryMB: 16384, timeout: 600_000 }, + 'run-cloud': { cpu: 8, memory: 16384, disk: 40 }, // cpu = cores, memory = MiB, disk = GiB }; function getSandboxOptionsWithResources(providerName: string, baseOptions?: Record): Record { diff --git a/benchmarks/sandbox/providers.ts b/benchmarks/sandbox/providers.ts index 2048c655..4c85cc91 100644 --- a/benchmarks/sandbox/providers.ts +++ b/benchmarks/sandbox/providers.ts @@ -26,6 +26,7 @@ import { tenki } from '@computesdk/tenki'; import { tensorlake } from '@computesdk/tensorlake' import { upstash } from '@computesdk/upstash'; import { vercel } from '@computesdk/vercel'; +import { runCloud } from './run-cloud.js'; import type { ProviderConfig } from './types.js'; /** @@ -175,6 +176,15 @@ export const providers: ProviderConfig[] = [ requiredEnvVars: ['RUNLOOP_API_KEY'], createCompute: () => runloop({ apiKey: process.env.RUNLOOP_API_KEY! }), }, + { + name: 'run-cloud', + requiredEnvVars: ['RUN_CLOUD_API_KEY'], + createCompute: () => runCloud({ + apiKey: process.env.RUN_CLOUD_API_KEY!, + apiUrl: process.env.RUN_CLOUD_API_URL, + }), + sandboxOptions: { disk: 40 }, + }, { name: 'sprites', requiredEnvVars: ['SPRITES_TOKEN'], diff --git a/benchmarks/sandbox/run-cloud.ts b/benchmarks/sandbox/run-cloud.ts new file mode 100644 index 00000000..ed10accc --- /dev/null +++ b/benchmarks/sandbox/run-cloud.ts @@ -0,0 +1,171 @@ +import { + Client, + type CreateSandboxOptions, + type ExecResult, +} from '@run-cloud/sdk'; +import { randomUUID } from 'node:crypto'; + +interface RunCommandOptions { + timeout?: number; +} + +interface RunCloudComputeOptions { + apiKey: string; + apiUrl?: string; +} + +export function runCloud(options: RunCloudComputeOptions) { + const client = new Client(options); + + return { + sandbox: { + async create(createOptions: CreateSandboxOptions = {}) { + const sandbox = await client.sandboxes.create(createOptions); + + return { + id: sandbox.id, + runCommand( + command: string, + commandOptions: RunCommandOptions = {}, + ): Promise { + // The public API proxy terminates long synchronous requests. Detach + // benchmark workloads and poll them through short exec requests. + if ((commandOptions.timeout ?? 0) > 45_000) { + return runLongCommand( + client, + sandbox.id, + command, + commandOptions.timeout!, + ); + } + + return client.sandboxes.exec(sandbox.id, command, { + timeoutSeconds: toTimeoutSeconds(commandOptions.timeout), + }); + }, + destroy(): Promise { + return client.sandboxes.destroy(sandbox.id); + }, + }; + }, + }, + }; +} + +async function runLongCommand( + client: Client, + sandboxId: string, + command: string, + timeoutMs: number, +): Promise { + const runId = randomUUID().replaceAll('-', ''); + const prefix = `/tmp/run-cloud-benchmark-${runId}`; + const scriptPath = `${prefix}.sh`; + const stdoutPath = `${prefix}.stdout`; + const stderrPath = `${prefix}.stderr`; + const statusPath = `${prefix}.status`; + const unitName = `run-cloud-benchmark-${runId}`; + const encodedCommand = Buffer.from(command).toString('base64'); + + const launch = [ + `printf '%s' '${encodedCommand}' | base64 -d > '${scriptPath}'`, + `chmod 700 '${scriptPath}'`, + `systemd-run --unit='${unitName}' --collect --quiet --property=OOMPolicy=continue /bin/sh -c 'bash "$1" >"$2" 2>"$3"; printf "%s" "$?" >"$4"' _ '${scriptPath}' '${stdoutPath}' '${stderrPath}' '${statusPath}'`, + ].join('\n'); + + const launched = await client.sandboxes.exec(sandboxId, launch, { + timeoutSeconds: 30, + }); + if (launched.exitCode !== 0) return launched; + + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + let status: ExecResult; + try { + status = await client.sandboxes.exec( + sandboxId, + `if [ -f '${statusPath}' ]; then cat '${statusPath}'; else printf pending; fi`, + { timeoutSeconds: 30 }, + ); + } catch (error) { + if (!isTransientApiError(error)) throw error; + await new Promise((resolve) => setTimeout(resolve, 500)); + continue; + } + + if (status.stdout.trim() !== 'pending') { + const exitCode = Number.parseInt(status.stdout.trim(), 10); + const stdout = await execWithTransientRetry( + client, + sandboxId, + ['cat', stdoutPath], + deadline, + ); + const stderr = await execWithTransientRetry( + client, + sandboxId, + ['cat', stderrPath], + deadline, + ); + await cleanupCommandFiles(client, sandboxId, prefix); + + return { + exit_code: Number.isFinite(exitCode) ? exitCode : 1, + exitCode: Number.isFinite(exitCode) ? exitCode : 1, + stdout: stdout.stdout, + stderr: stderr.stdout, + }; + } + + await new Promise((resolve) => setTimeout(resolve, 500)); + } + + await client.sandboxes.exec( + sandboxId, + `systemctl stop '${unitName}' 2>/dev/null || true`, + { timeoutSeconds: 30 }, + ).catch(() => {}); + await cleanupCommandFiles(client, sandboxId, prefix); + throw new Error(`Run Cloud command timed out after ${timeoutMs}ms`); +} + +async function execWithTransientRetry( + client: Client, + sandboxId: string, + command: string[], + deadline: number, +): Promise { + while (Date.now() < deadline) { + try { + return await client.sandboxes.exec(sandboxId, command, { + timeoutSeconds: 30, + }); + } catch (error) { + if (!isTransientApiError(error)) throw error; + await new Promise((resolve) => setTimeout(resolve, 500)); + } + } + + throw new Error('Run Cloud API remained unavailable while collecting benchmark output'); +} + +function isTransientApiError(error: unknown): boolean { + return error instanceof Error + && /run\.cloud API (?:429|5\d\d)\b/.test(error.message); +} + +async function cleanupCommandFiles( + client: Client, + sandboxId: string, + prefix: string, +): Promise { + await client.sandboxes.exec( + sandboxId, + `rm -f '${prefix}.sh' '${prefix}.stdout' '${prefix}.stderr' '${prefix}.status'`, + { timeoutSeconds: 30 }, + ).catch(() => {}); +} + +function toTimeoutSeconds(timeoutMs?: number): number | undefined { + return timeoutMs === undefined ? undefined : Math.ceil(timeoutMs / 1_000); +} diff --git a/package.json b/package.json index 3e81d6fd..82193313 100644 --- a/package.json +++ b/package.json @@ -21,6 +21,7 @@ "bench:northflank": "tsx benchmarks/src/run.ts --provider northflank", "bench:railway": "tsx benchmarks/src/run.ts --provider railway", "bench:render": "tsx benchmarks/src/run.ts --provider render", + "bench:run-cloud": "tsx benchmarks/src/run.ts --provider run-cloud", "bench:runloop": "tsx benchmarks/src/run.ts --provider runloop", "bench:vercel": "tsx benchmarks/src/run.ts --provider vercel", "bench:just-bash": "tsx benchmarks/src/run.ts --provider just-bash", @@ -132,6 +133,7 @@ "@computesdk/vercel": "^1.7.31", "@superserve/sdk": "^0.8.1", "@google-cloud/storage": "^7.21.0", + "@run-cloud/sdk": "^0.5.1", "@storagesdk/adapters": "^0.7.1", "@storagesdk/core": "^0.4.2", "@tigrisdata/storage": "^3.16.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 1cbc647c..81fac06e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -140,6 +140,9 @@ importers: '@google-cloud/storage': specifier: ^7.21.0 version: 7.21.0 + '@run-cloud/sdk': + specifier: ^0.5.1 + version: 0.5.1 '@storagesdk/adapters': specifier: ^0.7.1 version: 0.7.1(@aws-sdk/client-s3@3.1092.0)(@aws-sdk/lib-storage@3.1092.0(@aws-sdk/client-s3@3.1092.0))(@aws-sdk/s3-presigned-post@3.1092.0)(@aws-sdk/s3-request-presigner@3.1092.0)(@azure/storage-blob@12.33.0)(@google-cloud/storage@7.21.0)(@tigrisdata/storage@3.17.2)(@vercel/blob@2.6.1) @@ -1892,6 +1895,10 @@ packages: cpu: [x64] os: [win32] + '@run-cloud/sdk@0.5.1': + resolution: {integrity: sha512-eqvlA4nzCsr8A6zQ6a8kQ+C3u+oUZqx+N47kjTqhIYrbdRNsY2chfvfTw5wOJja6AoaF5dqg25Ium/2u0KH/5A==} + engines: {node: '>=20'} + '@runloop/api-client@1.25.0': resolution: {integrity: sha512-I1LMbxrB4hqnkERLrnCoy+J0CeXv+3SbiaHDQZpw7UBRyvd817fs1JblP3i4Crr77xLToMsYqL3htCwxWZzlEA==} @@ -7144,6 +7151,8 @@ snapshots: '@rollup/rollup-win32-x64-msvc@4.62.2': optional: true + '@run-cloud/sdk@0.5.1': {} + '@runloop/api-client@1.25.0': dependencies: '@types/node': 18.19.130