|
| 1 | +name: Sticky Disk GC |
| 2 | + |
| 3 | +# node_modules sticky disks are keyed on hashFiles('bun.lock') by design — see the |
| 4 | +# "Mount node_modules" comment in test-build.yml. A sticky disk is a mutable volume |
| 5 | +# and `bun install --frozen-lockfile` adds what the lockfile needs without pruning |
| 6 | +# what it dropped, so branches on different lockfiles must not share one. That |
| 7 | +# design is correct and is preserved here. |
| 8 | +# |
| 9 | +# Its cost is a fresh ~13.7 GB disk per lockfile hash per event class, created at |
| 10 | +# ~6.3/day. Blacksmith evicts a sticky disk after 7 days of inactivity, so this is |
| 11 | +# not a leak — it is a retention window far too generous for a key that churns |
| 12 | +# this fast. Steady state is (new disks/day x GB/disk x retention days), which at |
| 13 | +# 7 days predicts 606 GB against 601 GB actually observed. |
| 14 | +# |
| 15 | +# Age-based on purpose. The key contains NO PR identifier, so every open PR whose |
| 16 | +# checkout has the same bun.lock mounts the SAME disk — there are far more open |
| 17 | +# PRs than distinct lockfile hashes, so sharing is the common case, not an edge |
| 18 | +# case. Deleting on PR close would destroy a disk many other open PRs are using. |
| 19 | +# Never add a pull_request or pull_request_target trigger here. |
| 20 | + |
| 21 | +on: |
| 22 | + schedule: |
| 23 | + - cron: '17 9 * * *' |
| 24 | + workflow_dispatch: |
| 25 | + inputs: |
| 26 | + retention_days: |
| 27 | + description: Delete node_modules disks unused for more than this many days. |
| 28 | + required: false |
| 29 | + default: '3' |
| 30 | + dry_run: |
| 31 | + description: List what would be deleted without deleting it. |
| 32 | + type: boolean |
| 33 | + required: false |
| 34 | + default: false |
| 35 | + |
| 36 | +# Nothing in this job reads the repository. |
| 37 | +permissions: {} |
| 38 | + |
| 39 | +concurrency: |
| 40 | + group: stickydisk-gc |
| 41 | + cancel-in-progress: false |
| 42 | + |
| 43 | +jobs: |
| 44 | + gc: |
| 45 | + name: Reclaim idle node_modules sticky disks |
| 46 | + # GitHub-hosted on purpose, not a Blacksmith runner: the CLI is a pure API |
| 47 | + # client, and collection has to keep working during a Blacksmith outage or a |
| 48 | + # CI_PROVIDER break-glass switch — exactly when disks sit idle and still bill. |
| 49 | + runs-on: ubuntu-latest |
| 50 | + timeout-minutes: 10 |
| 51 | + env: |
| 52 | + # Pinned binary + checksum rather than `curl https://get.blacksmith.sh | sh`: |
| 53 | + # this job holds an org-wide token, so it must not execute unpinned remote |
| 54 | + # shell. The vendor publishes a .sha256 sidecar next to each binary; check a |
| 55 | + # new version against it and bump both values together. |
| 56 | + BLACKSMITH_CLI_VERSION: v0.4.58 |
| 57 | + BLACKSMITH_CLI_SHA256: 0b54a4398e9b35344d8fb32891703d8a393343f5001914d7482f93d068c76822 |
| 58 | + # The CLI self-updates in the background on every invocation, which would |
| 59 | + # silently defeat the pin above. |
| 60 | + BLACKSMITH_DISABLE_AUTO_UPDATE: '1' |
| 61 | + BLACKSMITH_ORG: simstudioai |
| 62 | + TARGET_REPO: ${{ github.repository }} |
| 63 | + RETENTION_DAYS: ${{ inputs.retention_days || '3' }} |
| 64 | + DRY_RUN: ${{ inputs.dry_run || 'false' }} |
| 65 | + |
| 66 | + steps: |
| 67 | + - name: Install Blacksmith CLI |
| 68 | + run: | |
| 69 | + set -euo pipefail |
| 70 | + url="https://clireleases.blacksmith.sh/cli/${BLACKSMITH_CLI_VERSION}/linux/amd64/blacksmith" |
| 71 | + curl -fsSL "$url" -o /usr/local/bin/blacksmith |
| 72 | + echo "${BLACKSMITH_CLI_SHA256} /usr/local/bin/blacksmith" | sha256sum -c - |
| 73 | + chmod +x /usr/local/bin/blacksmith |
| 74 | +
|
| 75 | + - name: Authenticate |
| 76 | + env: |
| 77 | + BLACKSMITH_CLI_TOKEN: ${{ secrets.BLACKSMITH_CLI_TOKEN }} |
| 78 | + run: | |
| 79 | + set -euo pipefail |
| 80 | + if [ -z "${BLACKSMITH_CLI_TOKEN:-}" ]; then |
| 81 | + echo "::error::BLACKSMITH_CLI_TOKEN is not set. Mint one with 'blacksmith org-token create' and add it as a repository secret." |
| 82 | + exit 1 |
| 83 | + fi |
| 84 | + printf '%s' "$BLACKSMITH_CLI_TOKEN" \ |
| 85 | + | blacksmith auth login --api-token - --non-interactive --organization "$BLACKSMITH_ORG" |
| 86 | +
|
| 87 | + - name: Delete node_modules disks idle beyond the retention window |
| 88 | + run: | |
| 89 | + set -euo pipefail |
| 90 | +
|
| 91 | + case "$RETENTION_DAYS" in |
| 92 | + ''|*[!0-9]*) |
| 93 | + echo "::error::retention_days must be a whole number of days, got '${RETENTION_DAYS}'" |
| 94 | + exit 1 |
| 95 | + ;; |
| 96 | + esac |
| 97 | + if [ "$RETENTION_DAYS" -lt 1 ]; then |
| 98 | + echo "::error::retention_days must be at least 1; 0 would delete disks a running job is using" |
| 99 | + exit 1 |
| 100 | + fi |
| 101 | +
|
| 102 | + blacksmith stickydisk list \ |
| 103 | + --repo "$TARGET_REPO" \ |
| 104 | + --search '-node-modules-' \ |
| 105 | + --per-page 100 \ |
| 106 | + --format json > disks.json |
| 107 | +
|
| 108 | + # Reduce the listing to the deletable set ONCE, so the staleness test and |
| 109 | + # the bulk-delete guard below both count the same things. Deriving the |
| 110 | + # guard's denominator from the raw entries instead would double-count |
| 111 | + # architecture variants (which are deliberately grouped into one key) and |
| 112 | + # would also count entries the regex rejected, so the guard could never |
| 113 | + # fire. |
| 114 | + # |
| 115 | + # Two independent filters, because the blast radius of a wrong key is a |
| 116 | + # cache every CI job depends on: |
| 117 | + # 1. --search narrows server-side to the node_modules family. |
| 118 | + # 2. The regex re-proves each key's full shape locally. |
| 119 | + # The event segment is [a-z_]+ rather than an enumerated push|pull_request |
| 120 | + # because the key interpolates ${{ github.event_name }} and a |
| 121 | + # workflow_dispatch disk already exists that an enumerated list would have |
| 122 | + # skipped forever. `-fork` is a separate optional segment rather than part |
| 123 | + # of that class: test-build.yml appends it after the event name, so a fork |
| 124 | + # key reads `pull_request-fork` and a character class cannot span the |
| 125 | + # hyphen. Without it, fork disks would never be collected. |
| 126 | + # |
| 127 | + # Grouped by key because `delete` without --arch removes every |
| 128 | + # architecture variant, so a key may only go when its NEWEST variant is |
| 129 | + # stale. |
| 130 | + jq --arg repo "$TARGET_REPO" ' |
| 131 | + .entries |
| 132 | + | map(select(.type == "stickydisk")) |
| 133 | + | map(select(.key | test("^" + ($repo | gsub("/"; "\\/")) + "-node-modules-[a-z_]+(-fork)?-[0-9a-f]{64}$"))) |
| 134 | + | group_by(.key) |
| 135 | + | map({ |
| 136 | + key: .[0].key, |
| 137 | + gb: (map(.size_bytes) | add / 1000000000 * 100 | round / 100), |
| 138 | + last_used: (map(.last_used_at | sub("\\.[0-9]+Z$"; "Z") | fromdateiso8601) | max) |
| 139 | + }) |
| 140 | + ' disks.json > eligible.json |
| 141 | +
|
| 142 | + jq -r --argjson days "$RETENTION_DAYS" ' |
| 143 | + (now - ($days * 86400)) as $cutoff |
| 144 | + | map(select(.last_used < $cutoff)) |
| 145 | + | .[] | "\(.key)\t\(.gb)" |
| 146 | + ' eligible.json > stale.tsv |
| 147 | +
|
| 148 | + total=$(jq 'length' eligible.json) |
| 149 | + count=$(wc -l < stale.tsv | tr -d ' ') |
| 150 | + reclaimed=$(awk -F'\t' '{s+=$2} END {printf "%.1f", s+0}' stale.tsv) |
| 151 | + { |
| 152 | + echo "### Sticky disk GC" |
| 153 | + echo "" |
| 154 | + echo "Retention **${RETENTION_DAYS}d** · dry run **${DRY_RUN}** · **${count}** of ${total} node_modules disks idle (**${reclaimed} GB**)" |
| 155 | + echo "" |
| 156 | + } >> "$GITHUB_STEP_SUMMARY" |
| 157 | +
|
| 158 | + # A run that would delete everything means the listing or the clock is |
| 159 | + # wrong, not that every disk went idle at once. Refuse rather than wipe |
| 160 | + # the caches every CI job depends on. |
| 161 | + if [ "$count" -gt 0 ] && [ "$count" -eq "$total" ]; then |
| 162 | + echo "::error::Refusing to delete all ${total} node_modules disks — that indicates a listing or clock fault, not genuine idleness." |
| 163 | + exit 1 |
| 164 | + fi |
| 165 | +
|
| 166 | + failed=0 |
| 167 | + while IFS=$'\t' read -r key gb; do |
| 168 | + [ -n "$key" ] || continue |
| 169 | + if [ "$DRY_RUN" = "true" ]; then |
| 170 | + echo "- would delete \`${key}\` (${gb} GB)" >> "$GITHUB_STEP_SUMMARY" |
| 171 | + continue |
| 172 | + fi |
| 173 | + if blacksmith stickydisk delete --repo "$TARGET_REPO" --key "$key" --yes; then |
| 174 | + echo "- deleted \`${key}\` (${gb} GB)" >> "$GITHUB_STEP_SUMMARY" |
| 175 | + else |
| 176 | + echo "- FAILED \`${key}\`" >> "$GITHUB_STEP_SUMMARY" |
| 177 | + failed=1 |
| 178 | + fi |
| 179 | + done < stale.tsv |
| 180 | +
|
| 181 | + # Fail loudly rather than continue-on-error: a revoked token or a changed |
| 182 | + # CLI JSON shape would otherwise silently revert us to 7-day billing. |
| 183 | + exit "$failed" |
0 commit comments