diff --git a/tests/bdd/AGENTS.md b/tests/bdd/AGENTS.md index 7c39d64d75..3bce9a05ae 100644 --- a/tests/bdd/AGENTS.md +++ b/tests/bdd/AGENTS.md @@ -62,8 +62,9 @@ logic into `dsl/`. namespace, and intended Gateway parent plus the shared context and timeout. The step requires `Accepted=True` and `ResolvedRefs=True` for that parent but does not allowlist route kinds or duplicate Gateway API validation. -- File-mutating steps (`I copy the file`, `I update yaml file`, - `I prepare self-managed secrets file`, `I substitute a block`) +- File-mutating steps (`I copy the file`, `I write yaml file`, + `I update yaml file`, `I prepare self-managed secrets file`, + `I substitute a block`) snapshot the destination through `Suite.Ledger` before the first write. Suite teardown restores every snapshotted path. - `Given command has succeeded:` keys on the fully resolved command diff --git a/tests/bdd/PLAN.md b/tests/bdd/PLAN.md index cdb9b577f9..1ced7c9630 100644 --- a/tests/bdd/PLAN.md +++ b/tests/bdd/PLAN.md @@ -106,6 +106,7 @@ refactor in every consumer; that is a feature. | Step | Notes | |------|-------| | `And I copy the file {string} to {string}` | Both paths are repo-relative. | +| `And I write yaml file {string} with values:` (two-column table of dotted-path and value) | Creates a new YAML file from the visible table. The destination must not already exist; the step fails instead of overwriting so an authored file is never silently replaced. Parent directories are created. Path syntax and `${VAR}` expansion match `I update yaml file`. Boolean literals and collection literals such as `[]` are written as native YAML types, not quoted strings, because Helm treats the string `"false"` as truthy. Numbers stay as strings. The destination is ledger-backed and removed at teardown. | | `And I update yaml file {string} with keys:` (two-column table of dotted-path and value) | Path supports dotted notation and `[n]` indices (e.g. `global.imagePullSecrets[0].name`). Missing intermediate maps and missing list indices are upserted: writing `global.imagePullSecrets[0].name` against a file that has neither `global.imagePullSecrets` nor any list entry creates both. Existing scalars at intermediate positions cause the step to fail rather than silently overwrite a non-map. Value cells expand `${VAR}` from `os.Environ`. | | `And I prepare Helmfile environment {string} for stack {string} from fixture {string} with values:` (two-column table of dotted-path and value) | Validates the stack and environment names, derives `deploy/stacks//environments/.yaml` from the absolute repository root, copies the explicit fixture, and applies the visible values table with the same YAML update and `${VAR}` interpolation behavior. Supported stacks are `self-managed`, `observability`, and `nvcf-compute-plane`. The destination is ledger-backed. | | `And I prepare self-managed secrets file {string} from template {string} using the current NGC registry credential` | The destination and template are explicit repo-relative paths with `${VAR}` interpolation. Replaces the template's registry credential placeholder with base64 of the current `$oauthtoken:` credential and writes the destination with mode `0600`. The destination is ledger-backed, and secret material never enters Gherkin, command logs, or failure messages. | @@ -273,7 +274,7 @@ contract verified in `src/clis/nvcf-cli/cmd/`): ## File restoration Every step that writes into a path under the repo working tree -(`I copy the file ... to ...`, `I update yaml file ...`, +(`I copy the file ... to ...`, `I write yaml file ...`, `I update yaml file ...`, `I prepare self-managed secrets file ...`, `I substitute a block ...`) registers that path with the runner's restoration ledger: diff --git a/tests/bdd/dsl/yamledit.go b/tests/bdd/dsl/yamledit.go index 82d02cd3d9..021d4f2ef0 100644 --- a/tests/bdd/dsl/yamledit.go +++ b/tests/bdd/dsl/yamledit.go @@ -45,6 +45,32 @@ const ( MatchSubset ) +// RenderYAMLFromKeys builds a YAML document from the supplied +// dotted-path/value pairs and returns its serialized bytes. It is the +// pure counterpart of UpdateYAMLKeys for a file that does not exist +// yet: the caller owns the destination path, the existence check, and +// the write. Path syntax matches UpdateYAMLKeys. Value cells run +// through Interpolate and then decodeTypedValue so booleans and +// collection literals reach Helm as native YAML types rather than +// quoted strings. +func RenderYAMLFromKeys(keys [][2]string) ([]byte, error) { + rootMap := map[string]any{} + for _, kv := range keys { + segments, err := parsePath(kv[0]) + if err != nil { + return nil, fmt.Errorf("render yaml: %w", err) + } + if err := setNested(rootMap, segments, decodeTypedValue(Interpolate(kv[1]))); err != nil { + return nil, fmt.Errorf("render yaml: %w", err) + } + } + body, err := yaml.Marshal(rootMap) + if err != nil { + return nil, fmt.Errorf("render yaml: marshal: %w", err) + } + return body, nil +} + // UpdateYAMLKeys reads the YAML file at path, applies each (dotted-path, // value) pair as an upsert, and writes the file back. Path syntax uses // "." between segments and "[n]" for list indices; missing intermediate @@ -210,6 +236,30 @@ func SubstituteFileBlock(path, spec string) error { return SubstituteFile(path, oldBlock, newBlock) } +// decodeTypedValue converts the YAML-significant literals that Helm +// evaluates differently when quoted. Booleans are decoded because +// Go templates treat the string "false" as truthy. Collection +// literals like "[]" are decoded so Helm sees an empty list instead +// of a non-empty string. Numbers are left as strings: Helm coerces +// them in template expressions, and eagerly parsing "1.0" as a float +// would lose the trailing zero on round-trip. +func decodeTypedValue(s string) any { + switch s { + case "true": + return true + case "false": + return false + } + var decoded any + if err := yaml.Unmarshal([]byte(s), &decoded); err == nil { + switch decoded.(type) { + case []any, map[string]any: + return decoded + } + } + return s +} + // readYAMLAny reads path and unmarshals into a generic any value. // An empty document parses to nil. func readYAMLAny(path string) (any, error) { diff --git a/tests/bdd/dsl/yamledit_test.go b/tests/bdd/dsl/yamledit_test.go index 1b14576580..fb13376cf9 100644 --- a/tests/bdd/dsl/yamledit_test.go +++ b/tests/bdd/dsl/yamledit_test.go @@ -313,6 +313,107 @@ func TestSubstituteFileBlockRejectsMissingOldBlock(t *testing.T) { } } +// TestRenderYAMLFromKeysBuildsNestedDocument verifies that +// RenderYAMLFromKeys produces a nested YAML structure from dotted-path +// key/value pairs without touching the filesystem. +func TestRenderYAMLFromKeysBuildsNestedDocument(t *testing.T) { + keys := [][2]string{ + {"llmRequestRouter.fullnameOverride", "llm-request-router-region-b"}, + {"llmRequestRouter.replicaCount", "2"}, + {"llmRequestRouter.workload.kind", "StatefulSet"}, + } + body, err := RenderYAMLFromKeys(keys) + if err != nil { + t.Fatalf("render: %v", err) + } + + out := string(body) + for _, want := range []string{ + "fullnameOverride: llm-request-router-region-b", + "replicaCount: \"2\"", + "kind: StatefulSet", + } { + if !strings.Contains(out, want) { + t.Fatalf("output missing %q:\n%s", want, out) + } + } +} + +// TestRenderYAMLFromKeysPreservesBoolsAndCollections verifies that +// booleans and collection literals are decoded to native YAML types +// while numbers remain as quoted strings. +func TestRenderYAMLFromKeysPreservesBoolsAndCollections(t *testing.T) { + keys := [][2]string{ + {"router.enabled", "true"}, + {"router.pki.enabled", "false"}, + {"router.replicaCount", "2"}, + {"router.discovery.remoteWatchUrls", "[]"}, + {"router.name", "region-b"}, + } + body, err := RenderYAMLFromKeys(keys) + if err != nil { + t.Fatalf("render: %v", err) + } + out := string(body) + + for _, want := range []string{ + "enabled: true", + "enabled: false", + "remoteWatchUrls: []", + "name: region-b", + } { + if !strings.Contains(out, want) { + t.Fatalf("output missing %q:\n%s", want, out) + } + } + for _, unwanted := range []string{ + `enabled: "true"`, + `enabled: "false"`, + `remoteWatchUrls: "[]"`, + } { + if strings.Contains(out, unwanted) { + t.Fatalf("value emitted as quoted string %q:\n%s", unwanted, out) + } + } + // Numbers stay as quoted strings; Helm coerces them in templates. + if !strings.Contains(out, `replicaCount: "2"`) { + t.Fatalf("replicaCount should remain a quoted string:\n%s", out) + } +} + +// TestRenderYAMLFromKeysRejectsInvalidPath confirms that a malformed +// dotted path surfaces as an error instead of a partial document. +func TestRenderYAMLFromKeysRejectsInvalidPath(t *testing.T) { + keys := [][2]string{ + {"router.name", "region-b"}, + {"router..enabled", "true"}, + } + body, err := RenderYAMLFromKeys(keys) + if err == nil || !strings.Contains(err.Error(), "empty segment") { + t.Fatalf("err = %v, want invalid-path error", err) + } + if body != nil { + t.Fatalf("body should be nil on error, got:\n%s", body) + } +} + +// TestRenderYAMLFromKeysInterpolatesValues confirms that ${VAR} +// references in value cells are expanded before serialization. +func TestRenderYAMLFromKeysInterpolatesValues(t *testing.T) { + t.Setenv("BDD_TEST_HOST", "region-b.example.invalid") + + keys := [][2]string{ + {"service.host", "${BDD_TEST_HOST}"}, + } + body, err := RenderYAMLFromKeys(keys) + if err != nil { + t.Fatalf("render: %v", err) + } + if !strings.Contains(string(body), "host: region-b.example.invalid") { + t.Fatalf("interpolation failed:\n%s", body) + } +} + func TestParsePathInvalidShapes(t *testing.T) { bads := []string{ "a..b", diff --git a/tests/bdd/features/multi-cluster-helmfile-llm-registration-multiregion.feature b/tests/bdd/features/multi-cluster-helmfile-llm-registration-multiregion.feature index 6e39d43af0..050ced6729 100644 --- a/tests/bdd/features/multi-cluster-helmfile-llm-registration-multiregion.feature +++ b/tests/bdd/features/multi-cluster-helmfile-llm-registration-multiregion.feature @@ -70,12 +70,180 @@ Feature: Register an LLM worker securely with routers in two local regions When I run command "kubectl --context k3d-ncp-local-cp rollout status deployment/llm-request-router -n nvcf --timeout=10m" Then the command exit code should be 0 - # This script still hides the Region B release values, routes, alias - # endpoints, and rollout waits. Replacing it with visible DSL steps is - # tracked in https://github.com/NVIDIA/nvcf/issues/1391. - When I run command "tests/bdd/scripts/install-llm-region-b.sh" + # Region B: write visible override values for a second LLM request + # router with a distinct StatefulSet identity. + Given I write yaml file "tests/bdd/out/region-b-values.yaml" with values: + | llmRequestRouter.fullnameOverride | llm-request-router-region-b | + | llmRequestRouter.replicaCount | 2 | + | llmRequestRouter.workload.kind | StatefulSet | + | llmRequestRouter.service.headlessName | llm-request-router-region-b-headless | + | llmRequestRouter.kubernetes.advertisedHostnameTemplate | {pod_name}.llm-request-router-region-b-headless.nvcf.svc.cluster.local | + | llmRequestRouter.discovery.remoteWatchUrls | [] | + | llmRequestRouter.backendRouter.enabled | true | + | llmRequestRouter.backendRouter.pylonGrpcDialAddress | https://region-b-watch.nvcf.svc.cluster.local:50071 | + | llmRequestRouter.backendRouter.pylonReverseTunnelDialAddress | region-b-watch.nvcf.svc.cluster.local:50072 | + | llmRequestRouter.backendRouter.image.pullPolicy | IfNotPresent | + | llmRequestRouter.serviceAccount.create | false | + | llmRequestRouter.serviceAccount.name | llm-request-router | + | llmRequestRouter.pki.enabled | false | + | llmRequestRouter.certificate.enabled | false | + | llmRequestRouter.tls.mode | existingSecret | + | llmRequestRouter.tls.secretName | stargate-quic-tls | + | llmRequestRouter.image.pullPolicy | IfNotPresent | + + # Export Region A base values so Region B inherits image tags and + # shared config; then install Region B with the visible overrides. + When I successfully run command: + """ + /bin/bash -c 'helm --kube-context k3d-ncp-local-cp get values llm-request-router --namespace nvcf --output json | jq "{llmRequestRouter: .llmRequestRouter}" > ${REPO_ROOT}/tests/bdd/out/region-a-base-values.json' + """ + When I run command: + """ + helm --kube-context k3d-ncp-local-cp upgrade --install llm-request-router-region-b ${REPO_ROOT}/deploy/helm/llm-request-router/llm-request-router --namespace nvcf --values ${REPO_ROOT}/tests/bdd/out/region-a-base-values.json --values ${REPO_ROOT}/tests/bdd/out/region-b-values.yaml --wait --timeout 10m + """ Then the command exit code should be 0 + # Region B gateway resources: GRPCRoute, BackendTrafficPolicy, + # and cross-namespace ReferenceGrant. + When I successfully run command: + """ + kubectl --context k3d-ncp-local-cp apply -f - <<'YAML' + apiVersion: gateway.networking.k8s.io/v1 + kind: GRPCRoute + metadata: + name: llm-worker-region-b-grpc + namespace: envoy-gateway-system + spec: + parentRefs: + - name: grpc-gw + namespace: envoy-gateway-system + sectionName: llm-grpc + hostnames: + - "region-b-watch.nvcf.svc.cluster.local" + - "*.llm-request-router-region-b-headless.nvcf.svc.cluster.local" + rules: + - backendRefs: + - name: llm-request-router-region-b-backend-router + namespace: nvcf + port: 50071 + --- + apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: BackendTrafficPolicy + metadata: + name: llm-worker-region-b-grpc-streams + namespace: envoy-gateway-system + spec: + targetRefs: + - group: gateway.networking.k8s.io + kind: GRPCRoute + name: llm-worker-region-b-grpc + timeout: + http: + requestTimeout: 0s + --- + apiVersion: gateway.networking.k8s.io/v1beta1 + kind: ReferenceGrant + metadata: + name: allow-llm-worker-region-b-route + namespace: nvcf + spec: + from: + - group: gateway.networking.k8s.io + kind: GRPCRoute + namespace: envoy-gateway-system + to: + - group: "" + kind: Service + name: llm-request-router-region-b-backend-router + YAML + """ + + # Discover the control-plane endpoint IP for the region-b-watch alias. + When I run command "kubectl --context k3d-ncp-local-compute-1 get endpoints llm-request-router --namespace nvcf --output jsonpath={.subsets[0].addresses[0].ip}" + Then the command exit code should be 0 + And I export command output to environment variable "CONTROL_PLANE_IP" + + # Apply the region-b-watch Service and Endpoints alias in both + # clusters so each cluster can reach the Region B backend-router + # by name. The discovered control-plane IP is interpolated by the + # DSL so no external templating tool is needed. + When I successfully run command: + """ + kubectl --context k3d-ncp-local-cp apply -f - <<'YAML' + apiVersion: v1 + kind: Service + metadata: + name: region-b-watch + namespace: nvcf + spec: + ports: + - name: llm-grpc + port: 50071 + targetPort: llm-grpc + protocol: TCP + - name: llm-quic + port: 50072 + targetPort: llm-quic + protocol: UDP + --- + apiVersion: v1 + kind: Endpoints + metadata: + name: region-b-watch + namespace: nvcf + subsets: + - addresses: + - ip: ${CONTROL_PLANE_IP} + ports: + - name: llm-grpc + port: 50071 + protocol: TCP + - name: llm-quic + port: 50072 + protocol: UDP + YAML + """ + When I successfully run command: + """ + kubectl --context k3d-ncp-local-compute-1 apply -f - <<'YAML' + apiVersion: v1 + kind: Service + metadata: + name: region-b-watch + namespace: nvcf + spec: + ports: + - name: llm-grpc + port: 50071 + targetPort: llm-grpc + protocol: TCP + - name: llm-quic + port: 50072 + targetPort: llm-quic + protocol: UDP + --- + apiVersion: v1 + kind: Endpoints + metadata: + name: region-b-watch + namespace: nvcf + subsets: + - addresses: + - ip: ${CONTROL_PLANE_IP} + ports: + - name: llm-grpc + port: 50071 + protocol: TCP + - name: llm-quic + port: 50072 + protocol: UDP + YAML + """ + + # Wait for Region B workloads to be ready. + When I successfully run command "kubectl --context k3d-ncp-local-cp rollout status statefulset/llm-request-router-region-b --namespace nvcf --timeout=10m" + And deployment "llm-request-router-region-b-backend-router" in namespace "nvcf" using context "k3d-ncp-local-cp" should complete rollout within "10m" + # The initial region advertises an explicit HTTPS recursive seed while # retaining every concrete Deployment pod identity. Three distinct # - identities prove per-pod discovery diff --git a/tests/bdd/godog_test.go b/tests/bdd/godog_test.go index 60002ef6cf..6856ad0658 100644 --- a/tests/bdd/godog_test.go +++ b/tests/bdd/godog_test.go @@ -1312,6 +1312,8 @@ func TestMultiClusterHelmfileLLMRegistrationMultiregionFeatureFileWiresToSteps(t " pylon_reverse_tunnel_connected 'at least' 3" grpcCertificateCommand = "kubectl --context k3d-ncp-local-cp get certificate llm-request-router-grpc-tls" + " -n envoy-gateway-system -o jsonpath={.spec.dnsNames}" + endpointDiscoveryCommand = "kubectl --context k3d-ncp-local-compute-1 get endpoints llm-request-router" + + " --namespace nvcf --output jsonpath={.subsets[0].addresses[0].ip}" invokeCommand = "/usr/bin/nvcf-cli --config /repo-root-placeholder/tests/bdd/fixtures/nvcf-cli-local.yaml function invoke" + " --inference-url /v1/chat/completions --model-name openai-compatible-sample" + " --request-body '{\"messages\":[{\"role\":\"user\",\"content\":\"bdd-registration-multiregion\"}]}' --timeout 120" @@ -1337,6 +1339,10 @@ func TestMultiClusterHelmfileLLMRegistrationMultiregionFeatureFileWiresToSteps(t ExitCode: 0, Stdout: "[llm-request-router.nvcf.svc.cluster.local region-b-watch.nvcf.svc.cluster.local]", }, + endpointDiscoveryCommand: { + ExitCode: 0, + Stdout: "192.0.2.10", + }, pylonMetricsCommand: { ExitCode: 0, Stdout: "pylon_registration_stream_connected=5\n" + @@ -1383,6 +1389,7 @@ func TestMultiClusterHelmfileLLMRegistrationMultiregionFeatureFileWiresToSteps(t } for _, command := range []string{ grpcCertificateCommand, + endpointDiscoveryCommand, regionAWatchCommand, regionBWatchCommand, pylonMetricsCommand, @@ -1391,8 +1398,99 @@ func TestMultiClusterHelmfileLLMRegistrationMultiregionFeatureFileWiresToSteps(t t.Fatalf("exact multi-region observation command was not invoked: %s", command) } } - if !commandRanThatContainsAll( - suite.Runner.(*fakeRunner).runs, + runs := suite.Runner.(*fakeRunner).runs + + // Region B base values export (the jq pipeline that captures Region A config). + if !commandRanThatContainsAll(runs, + "helm --kube-context k3d-ncp-local-cp get values llm-request-router", + "region-a-base-values.json", + ) { + t.Fatal("Region A base values export was not invoked") + } + + // Region B helm install references both the base values and the + // DSL-generated override file. + if !commandRanThatContainsAll(runs, + "upgrade --install llm-request-router-region-b", + "region-a-base-values.json", + "region-b-values.yaml", + ) { + t.Fatal("Region B helm install was not invoked with both base and override values") + } + + // The override file was written by the I write yaml file step and + // should contain the visible table values. + regionBValuesPath := filepath.Join(suite.Config.RepoRoot, "tests", "bdd", "out", "region-b-values.yaml") + regionBValues, err := os.ReadFile(regionBValuesPath) + if err != nil { + t.Fatalf("read region-b-values.yaml: %v", err) + } + for _, want := range []string{ + "fullnameOverride: llm-request-router-region-b", + "kind: StatefulSet", + "headlessName: llm-request-router-region-b-headless", + "pylonGrpcDialAddress: https://region-b-watch.nvcf.svc.cluster.local:50071", + "mode: existingSecret", + "secretName: stargate-quic-tls", + } { + if !strings.Contains(string(regionBValues), want) { + t.Fatalf("region-b-values.yaml missing %q:\n%s", want, regionBValues) + } + } + // Boolean and collection values must be emitted as native YAML + // types so Helm evaluates them correctly (string "false" is truthy + // in Go templates). + for _, unwanted := range []string{ + `enabled: "true"`, + `enabled: "false"`, + `create: "false"`, + `remoteWatchUrls: "[]"`, + } { + if strings.Contains(string(regionBValues), unwanted) { + t.Fatalf("region-b-values.yaml has quoted %s:\n%s", unwanted, regionBValues) + } + } + + // Gateway resources applied inline (GRPCRoute, BackendTrafficPolicy, + // ReferenceGrant visible in the feature file docstring). + if !commandRanThatContainsAll(runs, + "kubectl --context k3d-ncp-local-cp apply -f -", + "kind: GRPCRoute", + "kind: BackendTrafficPolicy", + "kind: ReferenceGrant", + ) { + t.Fatal("Region B gateway resources were not applied inline") + } + + // Watch alias applied inline to both clusters with DSL-interpolated + // control-plane IP (no envsubst dependency). Each apply must carry + // both the Service and the Endpoints resource; a name and IP alone + // would also match an unrelated resource. + for _, cluster := range []struct { + context string + label string + }{ + {context: "k3d-ncp-local-cp", label: "control-plane"}, + {context: "k3d-ncp-local-compute-1", label: "compute"}, + } { + if !commandRanThatContainsAll(runs, + "kubectl --context "+cluster.context+" apply -f -", + "kind: Service", + "kind: Endpoints", + "name: region-b-watch", + "ip: 192.0.2.10", + ) { + t.Fatalf("Region B watch alias Service and Endpoints were not applied to the %s cluster", cluster.label) + } + } + + if !commandRanThatContains(runs, "rollout status statefulset/llm-request-router-region-b") { + t.Fatal("Region B StatefulSet rollout wait was not invoked") + } + if !commandRanThatContains(runs, "rollout status deployment/llm-request-router-region-b-backend-router") { + t.Fatal("Region B backend-router Deployment rollout wait was not invoked") + } + if !commandRanThatContainsAll(runs, "function create --name bdd-registration-multiregion", "--function-type LLM", "--llm-model", diff --git a/tests/bdd/region_b_script_test.go b/tests/bdd/region_b_script_test.go deleted file mode 100644 index 23b6e30440..0000000000 --- a/tests/bdd/region_b_script_test.go +++ /dev/null @@ -1,101 +0,0 @@ -/* -SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. -SPDX-License-Identifier: Apache-2.0 - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package bdd_tmp - -import ( - "os" - "os/exec" - "path/filepath" - "strings" - "testing" -) - -func TestInstallLLMRegionBCreatesWatchAliasInBothClusters(t *testing.T) { - binDir := t.TempDir() - applyDir := t.TempDir() - - helmScript := `#!/usr/bin/env bash -set -euo pipefail -case " $* " in - *" get values "*) printf '{"llmRequestRouter":{}}\n' ;; - *) cat >/dev/null ;; -esac -` - kubectlScript := `#!/usr/bin/env bash -set -euo pipefail -context="" -previous="" -for argument in "$@"; do - if [[ "${previous}" == "--context" ]]; then - context="${argument}" - fi - previous="${argument}" -done -case " $* " in - *" get endpoints llm-request-router "*) printf '192.0.2.10' ;; - *" apply -f - "*) - cat >>"${FAKE_APPLY_DIR}/${context}.yaml" - printf '\n---\n' >>"${FAKE_APPLY_DIR}/${context}.yaml" - ;; -esac -` - jqScript := `#!/usr/bin/env bash -set -euo pipefail -cat -` - for name, body := range map[string]string{ - "helm": helmScript, - "jq": jqScript, - "kubectl": kubectlScript, - } { - if err := os.WriteFile(filepath.Join(binDir, name), []byte(body), 0o755); err != nil { - t.Fatalf("write fake %s: %v", name, err) - } - } - - cmd := exec.Command("bash", "scripts/install-llm-region-b.sh") - cmd.Env = append(os.Environ(), - "CONTROL_CONTEXT=bdd-control", - "COMPUTE_CONTEXT=bdd-compute", - "FAKE_APPLY_DIR="+applyDir, - "PATH="+binDir+":"+os.Getenv("PATH"), - "REPO_ROOT="+t.TempDir(), - ) - if output, err := cmd.CombinedOutput(); err != nil { - t.Fatalf("install region B: %v\n%s", err, output) - } - - for _, context := range []string{"bdd-control", "bdd-compute"} { - manifestPath := filepath.Join(applyDir, context+".yaml") - manifest, err := os.ReadFile(manifestPath) - if err != nil { - t.Fatalf("read %s aliases: %v", context, err) - } - for _, want := range []string{ - "kind: Service\nmetadata:\n name: region-b-watch", - "kind: Endpoints\nmetadata:\n name: region-b-watch", - "- ip: 192.0.2.10", - "name: llm-grpc", - "name: llm-quic", - } { - if !strings.Contains(string(manifest), want) { - t.Fatalf("%s aliases missing %q:\n%s", context, want, manifest) - } - } - } -} diff --git a/tests/bdd/scripts/install-llm-region-b.sh b/tests/bdd/scripts/install-llm-region-b.sh deleted file mode 100755 index 26a7e03f0e..0000000000 --- a/tests/bdd/scripts/install-llm-region-b.sh +++ /dev/null @@ -1,140 +0,0 @@ -#!/usr/bin/env bash -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -set -euo pipefail - -control_context="${CONTROL_CONTEXT:-k3d-ncp-local-cp}" -compute_context="${COMPUTE_CONTEXT:-k3d-ncp-local-compute-1}" -namespace="nvcf" -region_b_release="llm-request-router-region-b" -region_b_watch_host="region-b-watch.nvcf.svc.cluster.local" -region_b_headless_host="*.llm-request-router-region-b-headless.nvcf.svc.cluster.local" -chart="${REPO_ROOT:?REPO_ROOT is required}/deploy/helm/llm-request-router/llm-request-router" - -values_json="$(helm --kube-context "${control_context}" get values llm-request-router \ - --namespace "${namespace}" --output json)" - -printf '%s' "${values_json}" | jq --arg watch_host "${region_b_watch_host}" ' - { - llmRequestRouter: ( - .llmRequestRouter - | .fullnameOverride = "llm-request-router-region-b" - | .replicaCount = 2 - | .workload.kind = "StatefulSet" - | .service.headlessName = "llm-request-router-region-b-headless" - | .kubernetes.advertisedHostnameTemplate = "{pod_name}.llm-request-router-region-b-headless.nvcf.svc.cluster.local" - | .discovery.remoteWatchUrls = [] - | .backendRouter.enabled = true - | .backendRouter.pylonGrpcDialAddress = ("https://" + $watch_host + ":50071") - | .backendRouter.pylonReverseTunnelDialAddress = ($watch_host + ":50072") - | .serviceAccount.create = false - | .serviceAccount.name = "llm-request-router" - | .pki.enabled = false - | .certificate.enabled = false - | .tls.mode = "existingSecret" - | .tls.secretName = "stargate-quic-tls" - | .image.pullPolicy = "IfNotPresent" - | .backendRouter.image.pullPolicy = "IfNotPresent" - ) - } -' | helm --kube-context "${control_context}" upgrade --install "${region_b_release}" "${chart}" \ - --namespace "${namespace}" --values - --wait --timeout 10m - -kubectl --context "${control_context}" apply -f - <&2 - exit 1 -fi - -for alias_context in "${control_context}" "${compute_context}"; do - kubectl --context "${alias_context}" apply -f - <>> %s\n", st.Text) + if arg := st.Argument; arg != nil { + if dt := arg.DataTable; dt != nil { + for _, row := range dt.Rows { + cells := make([]string, len(row.Cells)) + for i, c := range row.Cells { + cells[i] = c.Value + } + fmt.Fprintf(os.Stderr, ">>> | %s |\n", strings.Join(cells, " | ")) + } + } + if ds := arg.DocString; ds != nil { + for _, line := range strings.Split(ds.Content, "\n") { + fmt.Fprintf(os.Stderr, ">>> %s\n", line) + } + } + } return c, nil }) registerFileSteps(ctx, sc) diff --git a/tests/bdd/steps/file_steps.go b/tests/bdd/steps/file_steps.go index 9903b43dfd..e9c64f59c3 100644 --- a/tests/bdd/steps/file_steps.go +++ b/tests/bdd/steps/file_steps.go @@ -33,6 +33,7 @@ import ( // to the Ledger before its first write so suite teardown can restore. func registerFileSteps(ctx *godog.ScenarioContext, sc *ScenarioContext) { ctx.Step(`^I copy the file "([^"]*)" to "([^"]*)"$`, sc.iCopyFile) + ctx.Step(`^I write yaml file "([^"]*)" with values:$`, sc.iWriteYAMLFile) ctx.Step(`^I update yaml file "([^"]*)" with keys:$`, sc.iUpdateYAMLFile) ctx.Step(`^I prepare Helmfile environment "([^"]*)" for stack "([^"]*)" from fixture "([^"]*)" with values:$`, sc.iPrepareHelmfileEnvironment) ctx.Step(`^I prepare self-managed secrets file "([^"]*)" from template "([^"]*)" using the current NGC registry credential$`, sc.iPrepareSelfManagedSecretsFile) @@ -81,6 +82,31 @@ func (sc *ScenarioContext) iCopyFile(src, dest string) error { return copyFile(resolvedSrc, resolvedDest) } +// iWriteYAMLFile creates a new YAML file from the supplied table of +// dotted-path/value rows. The step refuses to overwrite an existing +// file; use I update yaml file for that. The destination is recorded +// with the Ledger before the write so suite teardown removes it. YAML +// construction stays in dsl.RenderYAMLFromKeys; this handler owns only +// path resolution, the existence check, and the write. +func (sc *ScenarioContext) iWriteYAMLFile(path string, table *godog.Table) error { + resolved := sc.resolvePath(dsl.Interpolate(path)) + if _, err := os.Stat(resolved); err == nil { + return fmt.Errorf("write yaml %s: file already exists (use I update yaml file to modify)", resolved) + } + keys, err := tableToKeyValuePairs(table) + if err != nil { + return err + } + body, err := dsl.RenderYAMLFromKeys(keys) + if err != nil { + return fmt.Errorf("write yaml %s: %w", resolved, err) + } + if err := sc.Suite.Ledger.Snapshot(resolved); err != nil { + return err + } + return writeNewFile(resolved, body) +} + // iUpdateYAMLFile applies the supplied table of dotted-path/value rows // to path. Path keys do not interpolate; value cells do. func (sc *ScenarioContext) iUpdateYAMLFile(path string, table *godog.Table) error { @@ -212,6 +238,28 @@ func copyFile(src, dest string) error { return nil } +// writeNewFile creates dest with mode 0644, creating parent directories +// as needed. O_EXCL guarantees the write never clobbers a file that +// appeared between the caller's existence check and this call, and the +// Close error is checked so a flush failure surfaces. +func writeNewFile(dest string, body []byte) error { + if err := os.MkdirAll(filepath.Dir(dest), 0o755); err != nil { + return fmt.Errorf("mkdir %s: %w", filepath.Dir(dest), err) + } + out, err := os.OpenFile(dest, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644) + if err != nil { + return fmt.Errorf("create %s: %w", dest, err) + } + if _, err := out.Write(body); err != nil { + _ = out.Close() + return fmt.Errorf("write %s: %w", dest, err) + } + if err := out.Close(); err != nil { + return fmt.Errorf("close %s: %w", dest, err) + } + return nil +} + // tableToKeyValuePairs converts a two-column Godog table into the // (path, value) slice that dsl.UpdateYAMLKeys consumes. func tableToKeyValuePairs(table *godog.Table) ([][2]string, error) { diff --git a/tests/bdd/steps/steps_test.go b/tests/bdd/steps/steps_test.go index 6a34e0c5d7..78c928ce19 100644 --- a/tests/bdd/steps/steps_test.go +++ b/tests/bdd/steps/steps_test.go @@ -365,6 +365,53 @@ func TestIUpdateYAMLFileWritesKeys(t *testing.T) { } } +// TestIWriteYAMLFileCreatesAndRestores verifies that the write-yaml +// step creates the file and that Suite.Teardown removes it. +func TestIWriteYAMLFileCreatesAndRestores(t *testing.T) { + sc, _ := newScenarioContext(t) + rel := "out/region-b-values.yaml" + abs := filepath.Join(sc.Suite.Config.RepoRoot, rel) + + table := docTable(t, [][]string{ + {"llmRequestRouter.fullnameOverride", "llm-request-router-region-b"}, + {"llmRequestRouter.replicaCount", "2"}, + }) + if err := sc.iWriteYAMLFile(rel, table); err != nil { + t.Fatalf("write: %v", err) + } + got, err := os.ReadFile(abs) + if err != nil { + t.Fatalf("read: %v", err) + } + if !strings.Contains(string(got), "fullnameOverride: llm-request-router-region-b") { + t.Fatalf("missing key:\n%s", got) + } + + if err := sc.Suite.Teardown(); err != nil { + t.Fatalf("teardown: %v", err) + } + if _, err := os.Stat(abs); err == nil { + t.Fatal("file should be removed after restore") + } +} + +// TestIWriteYAMLFileRejectsExistingFile confirms that the write-yaml +// step refuses to overwrite an existing file. +func TestIWriteYAMLFileRejectsExistingFile(t *testing.T) { + sc, _ := newScenarioContext(t) + rel := "existing.yaml" + abs := filepath.Join(sc.Suite.Config.RepoRoot, rel) + if err := os.WriteFile(abs, []byte("key: value\n"), 0o644); err != nil { + t.Fatalf("seed: %v", err) + } + + table := docTable(t, [][]string{{"key", "new"}}) + err := sc.iWriteYAMLFile(rel, table) + if err == nil || !strings.Contains(err.Error(), "already exists") { + t.Fatalf("err = %v, want already-exists error", err) + } +} + func TestISubstituteBlockReplacesAndRestoresFile(t *testing.T) { sc, _ := newScenarioContext(t) rel := "global.yaml.gotmpl"