diff --git a/deploy/helm/container-cache/README.md b/deploy/helm/container-cache/README.md index 5e345ef698..edfe95ecc0 100644 --- a/deploy/helm/container-cache/README.md +++ b/deploy/helm/container-cache/README.md @@ -289,7 +289,7 @@ Key behaviors: | `persistentVolumeClaim.storageClassName` | `emptydir` | Storage class (`emptydir` for ephemeral) | | `persistentVolumeClaim.sizeGB` | `100` | Container cache volume (Gi) | | `persistentVolumeClaim.sizeProxyGB` | `200` | Proxy cache volume (Gi) | -| `persistentVolumeClaim.freeProxyPct` | `10` | Min free space percentage | +| `persistentVolumeClaim.freeProxyPct` | `15` | Evict at (100 - pct) percent usage: `min_free` on a PersistentVolume, plus per-zone `max_size` on `emptydir` | ### Service diff --git a/deploy/helm/container-cache/deploy/files/nginx.conf b/deploy/helm/container-cache/deploy/files/nginx.conf index 1fb5a5b0b5..c38eb92341 100644 --- a/deploy/helm/container-cache/deploy/files/nginx.conf +++ b/deploy/helm/container-cache/deploy/files/nginx.conf @@ -161,15 +161,34 @@ gzip off; # Configure cache size and Path {{- $proxyMaxSize := default 100 $.Values.persistentVolumeClaim.sizeProxyGB -}} + {{- $containerMaxSize := default 100 $.Values.persistentVolumeClaim.sizeGB -}} {{- $proxyMinFreePct := default 7 $.Values.persistentVolumeClaim.freeProxyPct -}} {{- $proxyMinFreeGB := max (mulf (divf $proxyMaxSize 100) $proxyMinFreePct) 1 }} - {{- /* No per-zone max_size: a deployment serves either s3 or ngc/hf - traffic, so a per-zone split would strand the idle zone's share of the - volume. Eviction is driven by the min_free watermark alone, sized by - persistentVolumeClaim.freeProxyPct. */}} - proxy_cache_path /container_cache levels=1:2 min_free={{ printf "%dg" $proxyMinFreeGB }} inactive={{ $.Values.cache.inactive | default "30d" }} keys_zone=container:{{ $.Values.cache.keyStorageSize | default "10m" }} use_temp_path=off; - proxy_cache_path /proxy_cache/s3 levels=1:2 min_free={{ printf "%dg" $proxyMinFreeGB }} inactive={{ $.Values.cache.inactive | default "30d" }} keys_zone=proxy_s3:{{ $.Values.cache.keyStorageSize | default "10m" }} use_temp_path=off; - proxy_cache_path /proxy_cache/ngc levels=1:2 min_free={{ printf "%dg" $proxyMinFreeGB }} inactive={{ $.Values.cache.inactive | default "30d" }} keys_zone=proxy_ngc:{{ $.Values.cache.keyStorageSize | default "10m" }} use_temp_path=off; + {{- /* On a PersistentVolume eviction is driven by the min_free watermark + alone, sized by persistentVolumeClaim.freeProxyPct. There is no per-zone + max_size: a deployment serves either s3 or ngc/hf traffic, so a per-zone + split would strand the idle zone's share of the volume. + + An emptyDir has no filesystem of its own: min_free then reads free space + on the node's ephemeral-storage volume, which is far larger than the + cache, so the watermark never fires and kubelet evicts the whole pod + when the directory reaches the emptyDir sizeLimit (sizeProxyGB, sizeGB). + For that mode each zone also gets a max_size of (100 - freeProxyPct) + percent of its own configured size. This keeps the meaning of + freeProxyPct ("start evicting at 100 - pct percent usage") and leaves + the remaining pct as headroom for in-flight temp files and cache-manager + lag before kubelet acts. The s3 and ngc zones each get the full share + because only one of them carries traffic in a given deployment. */}} + {{- $proxyCap := "" -}} + {{- $containerCap := "" -}} + {{- if eq (toString $.Values.persistentVolumeClaim.storageClassName) "emptydir" }} + {{- $usablePct := sub 100 (int $proxyMinFreePct) }} + {{- $proxyCap = printf " max_size=%dg" (max (int (mulf (divf $proxyMaxSize 100) $usablePct)) 1) }} + {{- $containerCap = printf " max_size=%dg" (max (int (mulf (divf $containerMaxSize 100) $usablePct)) 1) }} + {{- end }} + proxy_cache_path /container_cache levels=1:2{{ $containerCap }} min_free={{ printf "%dg" $proxyMinFreeGB }} inactive={{ $.Values.cache.inactive | default "30d" }} keys_zone=container:{{ $.Values.cache.keyStorageSize | default "10m" }} use_temp_path=off; + proxy_cache_path /proxy_cache/s3 levels=1:2{{ $proxyCap }} min_free={{ printf "%dg" $proxyMinFreeGB }} inactive={{ $.Values.cache.inactive | default "30d" }} keys_zone=proxy_s3:{{ $.Values.cache.keyStorageSize | default "10m" }} use_temp_path=off; + proxy_cache_path /proxy_cache/ngc levels=1:2{{ $proxyCap }} min_free={{ printf "%dg" $proxyMinFreeGB }} inactive={{ $.Values.cache.inactive | default "30d" }} keys_zone=proxy_ngc:{{ $.Values.cache.keyStorageSize | default "10m" }} use_temp_path=off; {{- if (.Values.consistentHashRouting).enabled }} # Consistent-hash peer routing: one upstream per pod ordinal. diff --git a/deploy/helm/container-cache/deploy/values.yaml b/deploy/helm/container-cache/deploy/values.yaml index 727007d5c1..2a4d0f74aa 100644 --- a/deploy/helm/container-cache/deploy/values.yaml +++ b/deploy/helm/container-cache/deploy/values.yaml @@ -140,12 +140,17 @@ persistentVolumeClaim: sizeProxyGB: 200 # Container cache volume size (Gi). sizeGB: 100 - # Cleanup watermark percentage (keep this much free on the proxy cache - # volume): eviction starts when the volume passes (100 - freeProxyPct) - # percent usage, i.e. 15 means evict at 85 percent full. This is the only - # space-based eviction trigger; there is no per-zone max_size because a + # Cleanup watermark percentage: eviction starts when usage passes + # (100 - freeProxyPct) percent, i.e. 15 means evict at 85 percent full. + # On a PersistentVolume this is an nginx min_free watermark on the volume + # and the only space-based trigger; there is no per-zone max_size because a # deployment serves either s3 or ngc/hf traffic, and a per-zone split would - # strand the idle zone's share of the volume. + # strand the idle zone's share of the volume. With storageClassName + # "emptydir" the directory shares the node's ephemeral-storage filesystem, + # so min_free cannot see the cache's own usage; each zone then also gets a + # max_size of (100 - freeProxyPct) percent of sizeProxyGB (sizeGB for the + # container zone), so nginx evicts before kubelet reaches the emptyDir + # sizeLimit and evicts the pod. freeProxyPct: 15 # Service configuration. diff --git a/deploy/helm/container-cache/tests/render-emptydir-max-size-test.sh b/deploy/helm/container-cache/tests/render-emptydir-max-size-test.sh new file mode 100755 index 0000000000..0258d9a08a --- /dev/null +++ b/deploy/helm/container-cache/tests/render-emptydir-max-size-test.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Rendered-output regression tests for cache eviction sizing. An emptyDir +# shares the node filesystem, so min_free alone never fires there; the chart +# must add a per-zone max_size in that mode and only in that mode. Run from +# the chart subtree: +# bash tests/render-emptydir-max-size-test.sh +set -euo pipefail +CHART_DIR="$(cd "$(dirname "$0")/.." && pwd)/deploy" +TMP="$(mktemp -d)" +trap 'rm -rf "$TMP"' EXIT +fail() { echo "FAIL: $*" >&2; exit 1; } +count() { grep -Ec "$1" "$2" || true; } +paths() { grep -E '^\s*proxy_cache_path ' "$1"; } + +helm template t "$CHART_DIR" > "$TMP/default.yaml" 2>/dev/null +helm template t "$CHART_DIR" --set persistentVolumeClaim.storageClassName=nvcf-cc-sc > "$TMP/pvc.yaml" 2>/dev/null +helm template t "$CHART_DIR" --set persistentVolumeClaim.sizeProxyGB=5000 --set persistentVolumeClaim.sizeGB=1000 --set persistentVolumeClaim.freeProxyPct=15 > "$TMP/big.yaml" 2>/dev/null +helm template t "$CHART_DIR" --set persistentVolumeClaim.freeProxyPct=100 > "$TMP/edge.yaml" 2>/dev/null + +echo "1. every render declares exactly three cache paths" +for f in default pvc big edge; do + [ "$(count '^\s*proxy_cache_path ' "$TMP/$f.yaml")" = 3 ] || fail "$f: expected 3 proxy_cache_path directives" +done + +echo "2. emptydir (chart default 200/100/15): max_size is 85 percent of each zone's own size" +paths "$TMP/default.yaml" | grep -q '/container_cache levels=1:2 max_size=85g min_free=30g' || fail "container zone: expected max_size=85g min_free=30g" +paths "$TMP/default.yaml" | grep -q '/proxy_cache/s3 levels=1:2 max_size=170g min_free=30g' || fail "s3 zone: expected max_size=170g min_free=30g" +paths "$TMP/default.yaml" | grep -q '/proxy_cache/ngc levels=1:2 max_size=170g min_free=30g' || fail "ngc zone: expected max_size=170g min_free=30g" + +echo "3. emptydir: max_size stays below the emptyDir sizeLimit kubelet enforces" +grep -q 'sizeLimit: "200Gi"' "$TMP/default.yaml" || fail "proxy-cache emptyDir sizeLimit must be 200Gi" +grep -q 'sizeLimit: "100Gi"' "$TMP/default.yaml" || fail "cache emptyDir sizeLimit must be 100Gi" + +echo "4. PersistentVolume: no max_size renders and min_free is unchanged" +[ "$(count 'max_size=' "$TMP/pvc.yaml")" = 0 ] || fail "max_size leaked into the PVC render" +[ "$(count 'min_free=30g' "$TMP/pvc.yaml")" = 3 ] || fail "PVC render must keep min_free=30g on all three zones" +grep -q 'storageClassName: "nvcf-cc-sc"' "$TMP/pvc.yaml" || fail "PVC render must carry the storage class" + +echo "5. emptydir sized for a large node (5000/1000/15): caps scale with the configured sizes" +paths "$TMP/big.yaml" | grep -q '/proxy_cache/ngc levels=1:2 max_size=4250g min_free=750g' || fail "ngc zone: expected max_size=4250g min_free=750g" +paths "$TMP/big.yaml" | grep -q '/container_cache levels=1:2 max_size=850g min_free=750g' || fail "container zone: expected max_size=850g" +grep -q 'sizeLimit: "5000Gi"' "$TMP/big.yaml" || fail "proxy-cache emptyDir sizeLimit must follow sizeProxyGB" + +echo "6. freeProxyPct=100 clamps max_size to 1g instead of rendering 0g" +[ "$(count 'max_size=1g' "$TMP/edge.yaml")" = 3 ] || fail "expected max_size=1g on all zones at freeProxyPct=100" +[ "$(count 'max_size=0g' "$TMP/edge.yaml")" = 0 ] || fail "max_size=0g must never render" + +echo "PASS: emptydir max_size render tests"