From 34d36627cef33f95bcb83518d1438ae265a7d53b Mon Sep 17 00:00:00 2001 From: kbhos Date: Fri, 12 Jun 2026 09:37:56 +0530 Subject: [PATCH 01/80] feat(AIP-3938): AI tier openshift support --- config/configs/applications.yaml | 125 +- config/configs/features/saia.yaml | 7 +- tools/cluster_setup/artifacts.yaml | 67 +- .../openshift-cluster-config.yaml | 105 ++ tools/cluster_setup/openshift_with_stack.sh | 1417 +++++++++++++++++ 5 files changed, 1571 insertions(+), 150 deletions(-) create mode 100644 tools/cluster_setup/openshift-cluster-config.yaml create mode 100755 tools/cluster_setup/openshift_with_stack.sh diff --git a/config/configs/applications.yaml b/config/configs/applications.yaml index a9a0869f..dbb0c5c6 100644 --- a/config/configs/applications.yaml +++ b/config/configs/applications.yaml @@ -34,123 +34,6 @@ applications: SERVICE_NAME: "ai_platform_models" SKIP_VERIFICATION: "true" USE_SYSTEM_PERMISSIONS: "true" - - args: - application_name: Gemma431bIt - deployment_configs: - LLMDeployment: - gpu_type_options_override: - H100: - autoscaling_config: - max_replicas: {{.Replicas.Gemma431bIt}} - min_replicas: {{.Replicas.Gemma431bIt}} - target_ongoing_requests: 6 - max_ongoing_requests: 8 - ray_actor_options: - num_gpus: 1 - L40S: - autoscaling_config: - max_replicas: {{.Replicas.Gemma431bIt}} - min_replicas: {{.Replicas.Gemma431bIt}} - target_ongoing_requests: 4 - max_ongoing_requests: 6 - ray_actor_options: - num_gpus: 2 - options: - autoscaling_config: - max_replicas: {{.Replicas.Gemma431bIt}} - min_replicas: {{.Replicas.Gemma431bIt}} - deployment_type: text_gen_model_deployment - gpu_types: '["{{.AcceleratorType}}"]' - model_definition: - gpu_type_model_config_override: - H100: - engine_args: - dtype: bfloat16 - gpu_memory_utilization: 0.9 - max_model_len: 32768 - max_num_batched_tokens: 4096 - tensor_parallel_size: 1 - L40S: - engine_args: - dtype: bfloat16 - gpu_memory_utilization: 0.85 - max_model_len: 120000 - max_num_batched_tokens: 4096 - max_num_seqs: 2 - tensor_parallel_size: 2 - model_config: - openai_serving_config: - chat: - enable_auto_tools: true - reasoning_parser: gemma4 - tool_parser: gemma4 - responses: - enable_auto_tools: true - reasoning_parser: gemma4 - tool_parser: gemma4 - model_id: gemma4_31b_it - model_loader: - blob_storage: - blob_prefix: model_artifacts/gemma-4-31b-it - tokenizer_definition: - model_id: gemma4_31b_it - model_loader: - blob_storage: - artifacts_list: - - chat_template.jinja - - config.json - - processor_config.json - - tokenizer_config.json - - tokenizer.json - blob_prefix: model_artifacts/gemma-4-31b-it - name: Gemma431bIt - import_path: main:create_serve_app - route_prefix: /gemma4_31b_it - runtime_env: - working_dir: "file:///home/ray/ray/applications/generic_application.zip" - env_vars: - API_VERSION: "v1" - APPLICATION_NAME: gemma4_31b_it - VLLM_ATTENTION_BACKEND: TRITON_ATTN - ARTIFACTS_S3_BUCKET: "{{.ArtifactBucketName}}" - S3_BUCKET: "{{.ArtifactBucketName}}" - ARTIFACTS_PROVIDER: "{{.ArtifactsProvider}}" - CLOUD_PROVIDER: "{{.CloudProvider}}" - S3COMPAT_OBJECT_STORE_ENDPOINT_URL: "{{.S3CompatObjectStoreEndpointUrl}}" - S3COMPAT_OBJECT_STORE_ACCESS_KEY: "{{.S3CompatObjectStoreAccessKey}}" - S3COMPAT_OBJECT_STORE_SECRET_KEY: "{{.S3CompatObjectStoreSecretKey}}" - # AWS / boto3 standard credential names — populated whenever the - # operator can load credentials from spec.objectStorage.secretRef. For - # CLOUD_PROVIDER=aws these are the values boto3 reads (the S3COMPAT_* - # names above are only consumed by the s3compat shim). Both code paths - # share the same source-of-truth Secret keys (s3_access_key / - # s3_secret_key) so emitting both pairs is safe — each provider only - # reads its own. AWS_REGION lets boto3 resolve the default regional S3 - # endpoint when no AWS_ENDPOINT_URL is set; required for any AWS S3 - # bucket outside us-east-1 to avoid PermanentRedirect on the first call. - AWS_ACCESS_KEY_ID: "{{.S3CompatObjectStoreAccessKey}}" - AWS_SECRET_ACCESS_KEY: "{{.S3CompatObjectStoreSecretKey}}" - AWS_REGION: "{{.Region}}" - AWS_DEFAULT_REGION: "{{.Region}}" - ENABLE_AUTHN: "false" - ENABLE_AUTHZ: "false" - SERVICE_EXTERNAL_NAME: "ai-platform-models" - SERVICE_INTERNAL_NAME: "ai_platform_models" - SERVICE_NAME: "ai_platform_models" - SKIP_VERIFICATION: "true" - USE_SYSTEM_PERMISSIONS: "true" - VLLM_WORKER_MULTIPROC_METHOD: spawn - # Disable the Redis-backed Responses API store (see ai-platform-models - # commit c1f9aef3: "feat: add a no-op store"). When True, the vLLM - # TextGen deployment constructs NoOpOpenAIServingResponses instead of - # RedisOpenAIServingResponses, so /v1/responses works without a Redis - # infra. Without this flag the deployment raises - # RuntimeError: Responses Redis URL not set - # on every request, which surfaces as an empty SSE stream and the SAIA - # v2 /query path fails with "An error occurred processing your request". - # Airgap k0s has no Redis; cloud sets this to "False" and wires - # RESPONSES_REDIS_ADDRESS to its in-namespace Redis StatefulSet. - DISABLE_RESPONSES_API_REDIS: "True" - args: application_name: GptOss20b deployment_configs: @@ -162,6 +45,14 @@ applications: L40S: ray_actor_options: num_gpus: 1 + RTX_PRO_6000_BLACKWELL: + autoscaling_config: + max_replicas: {{.Replicas.GptOss20b}} + min_replicas: {{.Replicas.GptOss20b}} + target_ongoing_requests: 4 + max_ongoing_requests: 8 + ray_actor_options: + num_gpus: 1 options: autoscaling_config: max_replicas: {{.Replicas.GptOss20b}} diff --git a/config/configs/features/saia.yaml b/config/configs/features/saia.yaml index bfe5d96d..4e94b27d 100644 --- a/config/configs/features/saia.yaml +++ b/config/configs/features/saia.yaml @@ -4,7 +4,6 @@ applicationScale: CrossEncoder: 1 E5LanguageClassifier: 1 Entrypoint: 1 - Gemma431bIt: 1 GptOss20b: 1 MbartTranslator: 1 PromptInjectionClassifier: 1 @@ -23,4 +22,8 @@ instanceScale: h100-1-gpu: 2 H100_NVL: h100-nvl-0-gpu: 1 - h100-nvl-1-gpu: 2 \ No newline at end of file + h100-nvl-1-gpu: 2 + RTX_PRO_6000_BLACKWELL: + rtx-pro-6000-blackwell-0-gpu: 1 + rtx-pro-6000-blackwell-1-gpu: 1 + rtx-pro-6000-blackwell-2-gpu: 0 \ No newline at end of file diff --git a/tools/cluster_setup/artifacts.yaml b/tools/cluster_setup/artifacts.yaml index c6953e7b..f2347653 100644 --- a/tools/cluster_setup/artifacts.yaml +++ b/tools/cluster_setup/artifacts.yaml @@ -1061,11 +1061,18 @@ spec: items: description: FeatureSpec defines the features to enable in the AIPlatform properties: + env: + additionalProperties: + type: string + description: Env specifies environment variables to propagate + to the child AIService. + type: object name: description: Name of the feature, e.g. "saia" or "seca" enum: - saia - seca + - weaviate-service type: string scaleFactor: description: ScaleFactor is the desired fixed number of replicas @@ -2085,6 +2092,11 @@ spec: type: object x-kubernetes-map-type: atomic type: array + otelImage: + default: otel/opentelemetry-collector-contrib:0.122.1 + description: OTelImage is the OpenTelemetry Collector sidecar + image + type: string rayHeadGroupImage: description: Ray head group image, e.g. "rayproject/ray-head:latest" type: string @@ -2225,7 +2237,8 @@ spec: type: object objectStorage: description: |- - ObjectStorage defines the object storage configuration for AI artifacts, tasks, and models + ObjectStorage defines the object storage configuration for AI artifacts, tasks, and models. + It is optional for platforms that only enable features that do not require object storage. Supported providers: S3, GCS, Azure Blob Storage, MinIO properties: endpoint: @@ -2237,8 +2250,8 @@ spec: path: description: |- Remote volume URI in the format s3://bucketname/, gs://bucketname/, - azure://containername/, minio://bucketname/, seaweedfs://bucketname/, or s3compat://bucketname/ - pattern: ^(s3|gs|azure|minio|seaweedfs|s3compat)://[a-zA-Z0-9.\-_]+(/.*)?$ + azure://containername/, or minio://bucketname/ + pattern: ^(s3|gs|azure|minio)://[a-zA-Z0-9.\-_]+(/.*)?$ type: string region: description: Region of the remote storage volume. Required for @@ -2908,8 +2921,6 @@ spec: pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ type: string type: object - required: - - objectStorage type: object status: description: AIPlatformStatus defines observed state @@ -4084,11 +4095,18 @@ spec: features: description: Feature defines the features to be enabled for the AIService properties: + env: + additionalProperties: + type: string + description: Env specifies environment variables to propagate + to the child AIService. + type: object name: description: Name of the feature, e.g. "saia" or "seca" enum: - saia - seca + - weaviate-service type: string scaleFactor: description: ScaleFactor is the desired fixed number of replicas @@ -4866,27 +4884,15 @@ spec: properties: endpoint: description: |- - Optional override endpoint (only needed for S3-compatible services like MinIO, SeaweedFS) - Must be a valid HTTP/HTTPS URL. When set with s3:// path, backend is treated as S3-compatible (MinIO, SeaweedFS, etc.) + Optional override endpoint (only needed for S3-compatible services like MinIO) + Must be a valid HTTP/HTTPS URL pattern: ^https?://.*$ type: string path: description: |- Remote volume URI in the format s3://bucketname/, gs://bucketname/, - azure://containername/, s3compat://bucketname/ (generic S3-compatible), minio://, or seaweedfs:// - pattern: ^(s3|gs|azure|minio|seaweedfs|s3compat)://[a-zA-Z0-9.\-_]+(/.*)?$ - type: string - provider: - description: |- - Provider is an optional hint for documentation and tooling. Operator derives behavior from path scheme and endpoint. - Values: aws, minio, seaweedfs, s3compat, gcs, azure - enum: - - aws - - minio - - seaweedfs - - s3compat - - gcs - - azure + azure://containername/, or minio://bucketname/ + pattern: ^(s3|gs|azure|minio)://[a-zA-Z0-9.\-_]+(/.*)?$ type: string region: description: Region of the remote storage volume. Required for @@ -4894,8 +4900,7 @@ spec: minLength: 1 type: string secretRef: - description: Secret name containing storage credentials (e.g. - s3_access_key, s3_secret_key for S3-compatible backends) + description: Secret name containing storage credentials maxLength: 253 minLength: 1 type: string @@ -5682,19 +5687,19 @@ spec: fieldRef: fieldPath: metadata.name - name: RELATED_IMAGE_RAY_HEAD - value: 658391232643.dkr.ecr.us-east-2.amazonaws.com/ml-platform/ray/ray-head:build-v2-010 + value: 658391232643.dkr.ecr.us-east-2.amazonaws.com/ml-platform/ray/ray-head:build-953 - name: RELATED_IMAGE_RAY_WORKER - value: 658391232643.dkr.ecr.us-east-2.amazonaws.com/ml-platform/ray/ray-worker-gpu:build-v2-010 + value: 658391232643.dkr.ecr.us-east-2.amazonaws.com/ml-platform/ray/ray-worker-gpu:build-953 - name: RELATED_IMAGE_WEAVIATE value: docker.io/semitechnologies/weaviate:stable-v1.28-007846a + - name: RELATED_IMAGE_WEAVIATE_SERVICE + value: docker.io/semitechnologies/weaviate:stable-v1.28-007846a - name: RELATED_IMAGE_SAIA_API - value: 658391232643.dkr.ecr.us-east-2.amazonaws.com/ml-platform/saia/saia-api:build-v2-012 + value: 658391232643.dkr.ecr.us-east-2.amazonaws.com/ml-platform/saia/saia-api:build-v2-main-c3b489d - name: RELATED_IMAGE_SAIA_API_V2 - value: 658391232643.dkr.ecr.us-east-2.amazonaws.com/ml-platform/saia/saia-api-v2:build-v2-012 + value: 658391232643.dkr.ecr.us-east-2.amazonaws.com/ml-platform/saia/saia-api-v2:build-v2-main-c3b489d - name: RELATED_IMAGE_POST_INSTALL_HOOK - value: 658391232643.dkr.ecr.us-east-2.amazonaws.com/ml-platform/saia/saia-data-loader:build-v2-012 - - name: SPLUNK_METRICS_INDEX_NAME - value: _metrics + value: 658391232643.dkr.ecr.us-east-2.amazonaws.com/ml-platform/saia/saia-data-loader:build-v2-main-c3b489d - name: RELATED_IMAGE_FLUENT_BIT value: docker.io/fluent/fluent-bit:1.9.6 - name: RELATED_IMAGE_OTEL_COLLECTOR @@ -5705,7 +5710,7 @@ spec: value: v0.3.14-36-g1549f5a - name: RAY_VERSION value: 2.53.0 - image: 658391232643.dkr.ecr.us-east-2.amazonaws.com/arif/splunk/splunk-ai-operator:v0.1.29 + image: 658391232643.dkr.ecr.us-east-2.amazonaws.com/kiran/splunk/splunk-ai-operator:openshift-0.1 livenessProbe: httpGet: path: /healthz diff --git a/tools/cluster_setup/openshift-cluster-config.yaml b/tools/cluster_setup/openshift-cluster-config.yaml new file mode 100644 index 00000000..2eb1c105 --- /dev/null +++ b/tools/cluster_setup/openshift-cluster-config.yaml @@ -0,0 +1,105 @@ +# ============================================================================= +# OpenShift Cluster Config for Splunk AI Platform +# ============================================================================= +# Used by openshift_with_stack.sh +# ============================================================================= + +kubernetes: + namespace: ai-platform + +# OpenShift-specific settings +openshift: + # Grant privileged SCC to Ray worker and operator service accounts. + # Required when running GPU workloads (nvidia.com/gpu requests). + # Set to "false" only if your cluster policy already covers this. + grantPrivilegedSCC: "true" + + # Node labeling for splunk.ai/* workload selectors. + # The operator schedules weaviate/ray-head on cpu nodes and Ray workers on gpu nodes. + # Use "auto" to detect by nvidia.com/gpu.present label (works when GPU Operator is installed). + # Use "manual" to specify node names explicitly below. + nodeLabelStrategy: "manual" + + # L40S nodes handle CPU workloads (weaviate, ray-head, saia-api). + # RTX 6000 Blackwell node is dedicated to GPU model pods (ray-worker). + nodes: + cpu: + - 00-25-b5-b5-00-31 + - 00-25-b5-b5-00-33 + gpu: + - cc-40-f3-9f-e2-3c + +images: + # Registry prefix applied to images that are not fully qualified + registry: "658391232643.dkr.ecr.us-east-2.amazonaws.com" + + operator: + image: "658391232643.dkr.ecr.us-east-2.amazonaws.com/kiran/splunk/splunk-ai-operator:openshift-0.1" + + ray: + headImage: "ml-platform/ray/ray-head:build-953" + workerImage: "ml-platform/ray/ray-worker-gpu:build-953" + + weaviate: + image: "docker.io/semitechnologies/weaviate:stable-v1.28-007846a" + + saia: + apiImage: "ml-platform/saia/saia-api:build-v2-main-c3b489d" + apiV2Image: "ml-platform/saia/saia-api-v2:build-v2-main-c3b489d" + dataLoaderImage: "ml-platform/saia/saia-data-loader:build-v2-main-c3b489d" + + splunk: + image: "658391232643.dkr.ecr.us-east-2.amazonaws.com/splunk/splunk:10-2-ai-custom" + operatorImage: "docker.io/splunk/splunk-operator:3.0.0" + + fluentBit: + image: "docker.io/fluent/fluent-bit:1.9.6" + + otelCollector: + image: "docker.io/otel/opentelemetry-collector-contrib:0.122.1" + + nginx: + image: "docker.io/library/nginx:1.27-alpine" + +storage: + storageClass: "local-path" + vectorDbSize: "50Gi" + objectStore: + type: "minio" # aws | s3compat | minio | seaweedfs + bucket: "ai-platform-bucket" + endpoint: "http://18.116.39.79:8333" + auth: + rootUser: "minioadmin" + rootPassword: "minioadmin" + +splunk: + standaloneName: splunk-standalone + +aiPlatform: + name: "openshift-ai-platform" + defaultAcceleratorType: "RTX_PRO_6000_BLACKWELL" + workerGroupConfig: + imageRegistry: "" + serviceTemplate: + type: NodePort + nodePort: 30080 + features: + - name: "saia" + version: "1.1.0" + +operators: + ray: + modelVersion: "v0.3.14-36-g1549f5a" + rayVersion: "2.53.0" + +files: + aiPlatform: "./artifacts.yaml" + splunkOperator: "./splunk-operator-cluster.yaml" + +# ECR pull secret — created automatically in all relevant namespaces during install. +# Requires AWS credentials in the environment (e.g. AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY +# or an instance profile). Set enabled: false for non-ECR registries. +ecr: + enabled: true + account: "658391232643" + region: "us-east-2" diff --git a/tools/cluster_setup/openshift_with_stack.sh b/tools/cluster_setup/openshift_with_stack.sh new file mode 100755 index 00000000..ced27236 --- /dev/null +++ b/tools/cluster_setup/openshift_with_stack.sh @@ -0,0 +1,1417 @@ +#!/bin/bash +set -euo pipefail + +# ============================================================================= +# OpenShift Cluster Setup Script for Splunk AI Platform +# ============================================================================= +# Installs/removes the Splunk AI Operator stack onto an existing OpenShift +# cluster. Assumes you are already logged in via `oc login` or have a valid +# KUBECONFIG pointing at the cluster. +# +# Usage: +# ./openshift_with_stack.sh [install|delete] +# +# The script reads openshift-cluster-config.yaml in the same directory. +# Override with: CONFIG_FILE=/path/to/config.yaml ./openshift_with_stack.sh +# ============================================================================= + +export PAGER=cat +export LANG=C LC_ALL=C + +# ====== CONFIG FILE LOCATION ====== +CONFIG_FILE="${CONFIG_FILE:-$(dirname "$0")/openshift-cluster-config.yaml}" + +# ====== SESSION LOG ====== +LOG_DIR="${LOG_DIR:-$(dirname "$0")/logs}" +mkdir -p "${LOG_DIR}" +LOG_FILE="${LOG_DIR}/openshift-install-$(date '+%Y-%m-%d_%H-%M-%S').log" +exec > >(tee -a "${LOG_FILE}") 2>&1 +echo "[LOG] Session log: ${LOG_FILE}" + +# ====== COLORS & LOGGING ====== +log() { echo -e "\033[1;36m[INFO]\033[0m $*" >&2; } +warn() { echo -e "\033[1;33m[WARN]\033[0m $*" >&2; } +err() { echo -e "\033[1;31m[ERROR]\033[0m $*" >&2; exit 1; } +need() { command -v "$1" >/dev/null 2>&1 || err "Missing $1 in PATH"; } + +# ====== LOAD CONFIGURATION ====== +load_config() { + log "Loading configuration from: ${CONFIG_FILE}" + [[ -f "${CONFIG_FILE}" ]] || err "Config file not found: ${CONFIG_FILE}" + + if command -v yq >/dev/null 2>&1; then + local yq_err + if ! yq_err=$(yq eval '.' "${CONFIG_FILE}" 2>&1 >/dev/null); then + err "Config file ${CONFIG_FILE} has YAML syntax errors: +${yq_err}" + fi + fi + + AI_NS=$(yq eval '.kubernetes.namespace // "ai-platform"' "${CONFIG_FILE}" 2>/dev/null || echo "ai-platform") + IMAGE_REGISTRY=$(yq eval '.images.registry // ""' "${CONFIG_FILE}" 2>/dev/null || echo "") + OPERATOR_IMAGE=$(yq eval '.images.operator.image // ""' "${CONFIG_FILE}" 2>/dev/null || echo "") + RAY_HEAD_IMAGE=$(yq eval '.images.ray.headImage // ""' "${CONFIG_FILE}" 2>/dev/null || echo "") + RAY_WORKER_IMAGE=$(yq eval '.images.ray.workerImage // ""' "${CONFIG_FILE}" 2>/dev/null || echo "") + WEAVIATE_IMAGE=$(yq eval '.images.weaviate.image // ""' "${CONFIG_FILE}" 2>/dev/null || echo "") + SAIA_API_IMAGE=$(yq eval '.images.saia.apiImage // ""' "${CONFIG_FILE}" 2>/dev/null || echo "") + SAIA_API_V2_IMAGE=$(yq eval '.images.saia.apiV2Image // ""' "${CONFIG_FILE}" 2>/dev/null || echo "") + SAIA_DATALOADER_IMAGE=$(yq eval '.images.saia.dataLoaderImage // ""' "${CONFIG_FILE}" 2>/dev/null || echo "") + SPLUNK_IMAGE=$(yq eval '.images.splunk.image // ""' "${CONFIG_FILE}" 2>/dev/null || echo "") + SPLUNK_OPERATOR_IMAGE=$(yq eval '.images.splunk.operatorImage // ""' "${CONFIG_FILE}" 2>/dev/null || echo "") + FLUENT_BIT_IMAGE=$(yq eval '.images.fluentBit.image // "fluent/fluent-bit:1.9.6"' "${CONFIG_FILE}" 2>/dev/null || echo "fluent/fluent-bit:1.9.6") + OTEL_COLLECTOR_IMAGE=$(yq eval '.images.otelCollector.image // "otel/opentelemetry-collector-contrib:0.122.1"' "${CONFIG_FILE}" 2>/dev/null || echo "otel/opentelemetry-collector-contrib:0.122.1") + NGINX_IMAGE=$(yq eval '.images.nginx.image // "docker.io/library/nginx:1.27-alpine"' "${CONFIG_FILE}" 2>/dev/null || echo "docker.io/library/nginx:1.27-alpine") + MODEL_VERSION=$(yq eval '.operators.ray.modelVersion // ""' "${CONFIG_FILE}" 2>/dev/null || echo "") + RAY_RUNTIME_VERSION=$(yq eval '.operators.ray.rayVersion // "2.44.0"' "${CONFIG_FILE}" 2>/dev/null || echo "2.44.0") + SPLUNK_AI_FILE=$(yq eval '.files.aiPlatform // "./artifacts.yaml"' "${CONFIG_FILE}" 2>/dev/null || echo "./artifacts.yaml") + SPLUNK_OPERATOR_FILE=$(yq eval '.files.splunkOperator // "./splunk-operator-cluster.yaml"' "${CONFIG_FILE}" 2>/dev/null || echo "./splunk-operator-cluster.yaml") + + # OpenShift-specific + # Whether to grant the operator service account privileged SCC. + # Required for Ray worker pods that request nvidia.com/gpu resources. + GRANT_PRIVILEGED_SCC=$(yq eval '.openshift.grantPrivilegedSCC // "true"' "${CONFIG_FILE}" 2>/dev/null || echo "true") + + NODE_LABEL_STRATEGY=$(yq eval '.openshift.nodeLabelStrategy // "auto"' "${CONFIG_FILE}" 2>/dev/null || echo "auto") + + ECR_ENABLED=$(yq eval '.ecr.enabled // "false"' "${CONFIG_FILE}" 2>/dev/null || echo "false") + ECR_ACCOUNT=$(yq eval '.ecr.account // ""' "${CONFIG_FILE}" 2>/dev/null || echo "") + ECR_REGION=$(yq eval '.ecr.region // "us-east-2"' "${CONFIG_FILE}" 2>/dev/null || echo "us-east-2") + + AI_PLATFORM_NAME=$(yq eval '.aiPlatform.name // "openshift-ai-platform"' "${CONFIG_FILE}" 2>/dev/null || echo "openshift-ai-platform") + DEFAULT_ACCELERATOR=$(yq eval '.aiPlatform.defaultAcceleratorType // ""' "${CONFIG_FILE}" 2>/dev/null || echo "") + WORKER_IMAGE_REGISTRY=$(yq eval '.aiPlatform.workerGroupConfig.imageRegistry // ""' "${CONFIG_FILE}" 2>/dev/null || echo "") + STORAGE_CLASS=$(yq eval '.storage.storageClass // ""' "${CONFIG_FILE}" 2>/dev/null || echo "") + VECTORDB_SIZE=$(yq eval '.storage.vectorDbSize // "50Gi"' "${CONFIG_FILE}" 2>/dev/null || echo "50Gi") + OBJ_STORE_TYPE=$(yq eval '.storage.objectStore.type // "minio"' "${CONFIG_FILE}" 2>/dev/null || echo "minio") + OBJ_STORE_BUCKET=$(yq eval '.storage.objectStore.bucket // ""' "${CONFIG_FILE}" 2>/dev/null || echo "") + OBJ_STORE_ENDPOINT=$(yq eval '.storage.objectStore.endpoint // ""' "${CONFIG_FILE}" 2>/dev/null || echo "") + MINIO_ROOT_USER=$(yq eval '.storage.objectStore.auth.rootUser // ""' "${CONFIG_FILE}" 2>/dev/null || echo "") + MINIO_ROOT_PASSWORD=$(yq eval '.storage.objectStore.auth.rootPassword // ""' "${CONFIG_FILE}" 2>/dev/null || echo "") + AI_STANDALONE_NAME=$(yq eval '.splunk.standaloneName // "splunk-standalone"' "${CONFIG_FILE}" 2>/dev/null || echo "splunk-standalone") + + log "Configuration loaded: namespace=${AI_NS}, accelerator=${DEFAULT_ACCELERATOR}" +} + +# ====== IMAGE HELPERS ====== +build_image_url() { + local registry="$1" + local image_path="$2" + # If the image is already fully qualified (contains a registry host) return as-is + if [[ "$image_path" =~ ^([a-zA-Z0-9.-]+\.[a-zA-Z]{2,}|[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+(:[0-9]+)?)/.*:.+ ]]; then + echo "$image_path" + return 0 + fi + if [[ -n "$registry" && "$registry" != "null" ]]; then + echo "${registry}/${image_path}" + else + echo "$image_path" + fi +} + +validate_image_config() { + log "Validating image configuration..." + [[ -z "$OPERATOR_IMAGE" || "$OPERATOR_IMAGE" == "null" ]] && err "REQUIRED: images.operator.image must be set in config" + [[ -z "$RAY_HEAD_IMAGE" || "$RAY_HEAD_IMAGE" == "null" ]] && err "REQUIRED: images.ray.headImage must be set in config" + [[ -z "$RAY_WORKER_IMAGE" || "$RAY_WORKER_IMAGE" == "null" ]] && err "REQUIRED: images.ray.workerImage must be set in config" + [[ -z "$WEAVIATE_IMAGE" || "$WEAVIATE_IMAGE" == "null" ]] && err "REQUIRED: images.weaviate.image must be set in config" + [[ -z "$SAIA_API_IMAGE" || "$SAIA_API_IMAGE" == "null" ]] && err "REQUIRED: images.saia.apiImage must be set in config" + [[ -z "$SAIA_API_V2_IMAGE" || "$SAIA_API_V2_IMAGE" == "null" ]] && err "REQUIRED: images.saia.apiV2Image must be set in config" + [[ -z "$SAIA_DATALOADER_IMAGE" || "$SAIA_DATALOADER_IMAGE" == "null" ]] && err "REQUIRED: images.saia.dataLoaderImage must be set in config" + [[ -z "$SPLUNK_IMAGE" || "$SPLUNK_IMAGE" == "null" ]] && err "REQUIRED: images.splunk.image must be set in config" + [[ -z "$MODEL_VERSION" || "$MODEL_VERSION" == "null" ]] && { MODEL_VERSION="v0.3.14-36-g1549f5a"; log "Using default MODEL_VERSION: $MODEL_VERSION"; } + log "✓ Image configuration validated" +} + +configure_images() { + log "Patching image references in manifest files..." + + [[ -f "${SPLUNK_AI_FILE}" ]] || err "Manifest not found: ${SPLUNK_AI_FILE}" + + if [[ ! -f "${SPLUNK_AI_FILE}.original" ]]; then + cp "$SPLUNK_AI_FILE" "${SPLUNK_AI_FILE}.original" + fi + cp "${SPLUNK_AI_FILE}.original" "$SPLUNK_AI_FILE" + + local operator_full ray_head_full ray_worker_full weaviate_full + local saia_api_full saia_api_v2_full saia_dataloader_full + local fluent_bit_full otel_collector_full nginx_full + + operator_full=$(build_image_url "$IMAGE_REGISTRY" "$OPERATOR_IMAGE") + ray_head_full=$(build_image_url "$IMAGE_REGISTRY" "$RAY_HEAD_IMAGE") + ray_worker_full=$(build_image_url "$IMAGE_REGISTRY" "$RAY_WORKER_IMAGE") + weaviate_full=$(build_image_url "$IMAGE_REGISTRY" "$WEAVIATE_IMAGE") + saia_api_full=$(build_image_url "$IMAGE_REGISTRY" "$SAIA_API_IMAGE") + saia_api_v2_full=$(build_image_url "$IMAGE_REGISTRY" "$SAIA_API_V2_IMAGE") + saia_dataloader_full=$(build_image_url "$IMAGE_REGISTRY" "$SAIA_DATALOADER_IMAGE") + fluent_bit_full=$(build_image_url "$IMAGE_REGISTRY" "$FLUENT_BIT_IMAGE") + otel_collector_full=$(build_image_url "$IMAGE_REGISTRY" "$OTEL_COLLECTOR_IMAGE") + nginx_full=$(build_image_url "$IMAGE_REGISTRY" "$NGINX_IMAGE") + + # BSD (macOS) sed requires an explicit backup-suffix arg after -i. + local SED_INPLACE + if [[ "$OSTYPE" == "darwin"* ]]; then + SED_INPLACE=(sed -i "") + else + SED_INPLACE=(sed -i) + fi + + local ray_head_esc ray_worker_esc weaviate_esc saia_api_esc saia_api_v2_esc + local saia_dl_esc fluent_esc otel_esc nginx_esc operator_esc + + ray_head_esc=$(echo "$ray_head_full" | sed 's/[\/&]/\\&/g') + ray_worker_esc=$(echo "$ray_worker_full" | sed 's/[\/&]/\\&/g') + weaviate_esc=$(echo "$weaviate_full" | sed 's/[\/&]/\\&/g') + saia_api_esc=$(echo "$saia_api_full" | sed 's/[\/&]/\\&/g') + saia_api_v2_esc=$(echo "$saia_api_v2_full" | sed 's/[\/&]/\\&/g') + saia_dl_esc=$(echo "$saia_dataloader_full" | sed 's/[\/&]/\\&/g') + fluent_esc=$(echo "$fluent_bit_full" | sed 's/[\/&]/\\&/g') + otel_esc=$(echo "$otel_collector_full" | sed 's/[\/&]/\\&/g') + nginx_esc=$(echo "$nginx_full" | sed 's/[\/&]/\\&/g') + operator_esc=$(echo "$operator_full" | sed 's/[\/&]/\\&/g') + + "${SED_INPLACE[@]}" "/name: RELATED_IMAGE_RAY_HEAD/,/value:/ s|value:.*|value: ${ray_head_esc}|" "$SPLUNK_AI_FILE" + "${SED_INPLACE[@]}" "/name: RELATED_IMAGE_RAY_WORKER/,/value:/ s|value:.*|value: ${ray_worker_esc}|" "$SPLUNK_AI_FILE" + "${SED_INPLACE[@]}" "/name: RELATED_IMAGE_WEAVIATE/,/value:/ s|value:.*|value: ${weaviate_esc}|" "$SPLUNK_AI_FILE" + "${SED_INPLACE[@]}" "/name: RELATED_IMAGE_SAIA_API$/,/value:/ s|value:.*|value: ${saia_api_esc}|" "$SPLUNK_AI_FILE" + "${SED_INPLACE[@]}" "/name: RELATED_IMAGE_SAIA_API_V2/,/value:/ s|value:.*|value: ${saia_api_v2_esc}|" "$SPLUNK_AI_FILE" + "${SED_INPLACE[@]}" "/name: RELATED_IMAGE_POST_INSTALL_HOOK/,/value:/ s|value:.*|value: ${saia_dl_esc}|" "$SPLUNK_AI_FILE" + "${SED_INPLACE[@]}" "/name: RELATED_IMAGE_FLUENT_BIT/,/value:/ s|value:.*|value: ${fluent_esc}|" "$SPLUNK_AI_FILE" + "${SED_INPLACE[@]}" "/name: RELATED_IMAGE_OTEL_COLLECTOR/,/value:/ s|value:.*|value: ${otel_esc}|" "$SPLUNK_AI_FILE" + "${SED_INPLACE[@]}" "/name: RELATED_IMAGE_NGINX/,/value:/ s|value:.*|value: ${nginx_esc}|" "$SPLUNK_AI_FILE" + "${SED_INPLACE[@]}" "/name: MODEL_VERSION/,/value:/ s|value:.*|value: ${MODEL_VERSION}|" "$SPLUNK_AI_FILE" + "${SED_INPLACE[@]}" "/name: RAY_VERSION/,/value:/ s|value:.*|value: ${RAY_RUNTIME_VERSION}|" "$SPLUNK_AI_FILE" + "${SED_INPLACE[@]}" "s|image: .*splunk.*ai.*operator.*|image: ${operator_esc}|I" "$SPLUNK_AI_FILE" + + log " ✓ RELATED_IMAGE_RAY_HEAD: $ray_head_full" + log " ✓ RELATED_IMAGE_RAY_WORKER: $ray_worker_full" + log " ✓ RELATED_IMAGE_WEAVIATE: $weaviate_full" + log " ✓ RELATED_IMAGE_SAIA_API: $saia_api_full" + log " ✓ RELATED_IMAGE_SAIA_API_V2: $saia_api_v2_full" + log " ✓ RELATED_IMAGE_POST_INSTALL_HOOK: $saia_dataloader_full" + log " ✓ RELATED_IMAGE_FLUENT_BIT: $fluent_bit_full" + log " ✓ RELATED_IMAGE_OTEL_COLLECTOR: $otel_collector_full" + log " ✓ RELATED_IMAGE_NGINX: $nginx_full" + log " ✓ Operator image: $operator_full" + log " ✓ MODEL_VERSION: $MODEL_VERSION" + log " ✓ RAY_VERSION: $RAY_RUNTIME_VERSION" +} + +# ====== PREFLIGHT CHECKS ====== +preflight_checks() { + log "Running preflight checks..." + + for tool in oc yq; do + command -v "$tool" >/dev/null 2>&1 && log " ✓ $tool found" || err "Missing $tool in PATH" + done + + # Verify we are connected to the cluster + if ! oc whoami &>/dev/null; then + err "Not logged in to OpenShift. Run: oc login " + fi + log " ✓ Logged in as: $(oc whoami)" + + # Verify cluster admin access (needed to install CRDs and grant SCCs) + if ! oc auth can-i create clusterrolebinding --all-namespaces &>/dev/null; then + warn " May not have cluster-admin; CRD and SCC operations might fail" + else + log " ✓ Cluster-admin access confirmed" + fi + + [[ -f "${SPLUNK_AI_FILE}" ]] && log " ✓ Manifest: ${SPLUNK_AI_FILE}" || err "Manifest not found: ${SPLUNK_AI_FILE}" + + log "Preflight checks passed" +} + +# ====== WAIT FOR CRD ====== +wait_for_crd() { + local crd_name="$1" + local timeout="${2:-300}" + log "Waiting for CRD ${crd_name} (timeout: ${timeout}s)..." + local elapsed=0 + while ! oc get crd "${crd_name}" >/dev/null 2>&1; do + sleep 5 + elapsed=$((elapsed + 5)) + if [[ ${elapsed} -ge ${timeout} ]]; then + err "Timeout waiting for CRD ${crd_name}" + fi + done + log " ✓ CRD ${crd_name} ready" +} + +# ====== ENSURE NAMESPACE ====== +ensure_namespace() { + local ns="$1" + if ! oc get namespace "${ns}" &>/dev/null; then + log "Creating namespace ${ns}..." + oc create namespace "${ns}" + fi +} + +# ====== OPENSHIFT: GRANT PRIVILEGED SCC ====== +# Ray worker pods request nvidia.com/gpu resources and run as non-root. +# On OpenShift the default restricted SCC blocks this — privileged SCC is needed. +grant_privileged_scc() { + if [[ "${GRANT_PRIVILEGED_SCC}" != "true" ]]; then + log "Skipping privileged SCC grant (openshift.grantPrivilegedSCC=false)" + return 0 + fi + + local ai_operator_ns="splunk-ai-operator-system" + log "Granting SCC policies to service account groups in ${ai_operator_ns} and ${AI_NS}..." + + # Use `oc adm policy add-scc-to-group` which modifies the SCC's groups list directly + # and is honored by OCP SCC admission (unlike ClusterRoleBinding which can be ignored). + # + # - privileged: operator namespace (webhook + leader election need elevated perms) + # - anyuid: AI platform namespace so operator-created SAs (saia-sa, weaviate, + # raycluster-*) run as the UID defined in their images, not OCP's random UID range. + # - privileged: also on AI platform so Splunk Standalone can write to hostPath PVCs. + oc adm policy add-scc-to-group privileged \ + "system:serviceaccounts:${ai_operator_ns}" 2>/dev/null || true + oc adm policy add-scc-to-group anyuid \ + "system:serviceaccounts:${AI_NS}" 2>/dev/null || true + oc adm policy add-scc-to-group privileged \ + "system:serviceaccounts:${AI_NS}" 2>/dev/null || true + # Splunk Operator pod adds NET_BIND_SERVICE capability which anyuid blocks; needs privileged. + oc adm policy add-scc-to-group privileged \ + "system:serviceaccounts:splunk-operator" 2>/dev/null || true + + log " ✓ anyuid + privileged SCC granted to all SAs in ${AI_NS} and splunk-operator" +} + +# ====== INSTALL NFD (Node Feature Discovery) via OLM ====== +# NFD labels nodes with hardware capabilities including nvidia.com/gpu.present=true. +# The GPU Operator depends on NFD labels to know which nodes to target. +install_nfd() { + log "Installing Node Feature Discovery Operator (NFD)..." + + if oc get subscription nfd -n openshift-nfd &>/dev/null; then + log " ✓ NFD subscription already exists, skipping" + return 0 + fi + + oc apply -f - <<'EOF' +apiVersion: v1 +kind: Namespace +metadata: + name: openshift-nfd +--- +apiVersion: operators.coreos.com/v1 +kind: OperatorGroup +metadata: + name: openshift-nfd + namespace: openshift-nfd +spec: + targetNamespaces: + - openshift-nfd +--- +apiVersion: operators.coreos.com/v1alpha1 +kind: Subscription +metadata: + name: nfd + namespace: openshift-nfd +spec: + channel: stable + name: nfd + source: redhat-operators + sourceNamespace: openshift-marketplace + installPlanApproval: Automatic +EOF + + log "Waiting for NFD CSV to succeed..." + local retries=0 + while (( retries < 36 )); do + local phase + phase=$(oc get csv -n openshift-nfd -o jsonpath='{.items[0].status.phase}' 2>/dev/null || echo "") + if [[ "${phase}" == "Succeeded" ]]; then + log " ✓ NFD operator ready" + break + fi + sleep 10 + retries=$(( retries + 1 )) + log " Waiting for NFD CSV... (${retries}/36, phase=${phase:-pending})" + done + + # Create the NodeFeatureDiscovery CR to start labeling nodes + if ! oc get nodefeaturediscovery nfd-instance -n openshift-nfd &>/dev/null; then + log "Creating NodeFeatureDiscovery CR..." + oc apply -f - <<'EOF' +apiVersion: nfd.openshift.io/v1 +kind: NodeFeatureDiscovery +metadata: + name: nfd-instance + namespace: openshift-nfd +spec: + operand: + image: registry.redhat.io/openshift4/ose-node-feature-discovery-rhel9:v4.21 + imagePullPolicy: Always + workerConfig: + configData: | + core: + sleepInterval: 60s + sources: + pci: + deviceClassWhitelist: + - "03" + deviceLabelFields: + - "vendor" +EOF + fi + + log " ✓ NFD installed" +} + +# ====== INSTALL NVIDIA GPU OPERATOR via OLM ====== +# Installs driver, container toolkit, device plugin, and DCGM on GPU nodes. +# Uses OCP Driver Toolkit (use_ocp_driver_toolkit: true) so no SSH to nodes needed. +install_nvidia_gpu_operator() { + log "Installing NVIDIA GPU Operator..." + + if oc get subscription gpu-operator-certified -n nvidia-gpu-operator &>/dev/null; then + log " ✓ GPU Operator subscription already exists, skipping" + return 0 + fi + + oc apply -f - <<'EOF' +apiVersion: v1 +kind: Namespace +metadata: + name: nvidia-gpu-operator +--- +apiVersion: operators.coreos.com/v1 +kind: OperatorGroup +metadata: + name: nvidia-gpu-operator + namespace: nvidia-gpu-operator +spec: + targetNamespaces: + - nvidia-gpu-operator +--- +apiVersion: operators.coreos.com/v1alpha1 +kind: Subscription +metadata: + name: gpu-operator-certified + namespace: nvidia-gpu-operator +spec: + channel: v26.3 + name: gpu-operator-certified + source: certified-operators + sourceNamespace: openshift-marketplace + installPlanApproval: Automatic +EOF + + log "Waiting for GPU Operator CSV to succeed..." + local retries=0 + while (( retries < 36 )); do + local phase + phase=$(oc get csv -n nvidia-gpu-operator -o jsonpath='{.items[0].status.phase}' 2>/dev/null || echo "") + if [[ "${phase}" == "Succeeded" ]]; then + log " ✓ GPU Operator CSV ready" + break + fi + sleep 10 + retries=$(( retries + 1 )) + log " Waiting for GPU Operator CSV... (${retries}/36, phase=${phase:-pending})" + done + + # Create ClusterPolicy to trigger driver + toolkit + device-plugin rollout + if ! oc get clusterpolicy gpu-cluster-policy &>/dev/null; then + log "Creating ClusterPolicy CR..." + oc apply -f - <<'EOF' +apiVersion: nvidia.com/v1 +kind: ClusterPolicy +metadata: + name: gpu-cluster-policy +spec: + operator: {} + daemonsets: {} + driver: + enabled: true + use_ocp_driver_toolkit: true + toolkit: + enabled: true + devicePlugin: + enabled: true + dcgm: + enabled: true + dcgmExporter: + enabled: true + gfd: + enabled: true + nodeStatusExporter: + enabled: true + validator: + enabled: true +EOF + fi + + # Wait for nvidia.com/gpu.present=true to appear on at least one worker node. + # This confirms NFD + GFD have finished their discovery pass. + log "Waiting for GPU nodes to be labeled by GPU Operator / GFD..." + local retries=0 + while (( retries < 60 )); do + local count + count=$(oc get nodes -l nvidia.com/gpu.present=true --no-headers 2>/dev/null | wc -l | tr -d ' ') + if (( count > 0 )); then + log " ✓ ${count} GPU node(s) labeled with nvidia.com/gpu.present=true" + break + fi + sleep 15 + retries=$(( retries + 1 )) + log " Waiting for GPU node labels... (${retries}/60)" + done + + if (( retries >= 60 )); then + warn "GPU nodes not labeled after 15m — label_nodes will fall back to 0 GPU workers. + Check: oc get pods -n nvidia-gpu-operator + oc get clusterpolicy gpu-cluster-policy -o yaml" + fi + + log " ✓ NVIDIA GPU Operator installed" +} + +# ====== NODE LABELING ====== +# Applies splunk.ai/* labels that the operator uses to schedule workloads. +# Without these labels all operator-managed pods (weaviate, ray-head, ray-worker) +# will stay Pending forever because their nodeSelectors won't match any node. +# Runs after install_nvidia_gpu_operator so nvidia.com/gpu.present=true is already set. +label_nodes() { + log "Applying splunk.ai/* node labels (strategy: ${NODE_LABEL_STRATEGY})..." + + local cpu_nodes=() gpu_nodes=() control_nodes=() + + # Always label master/control-plane nodes + while IFS= read -r node; do + [[ -n "$node" ]] && control_nodes+=("$node") + done < <(oc get nodes -l node-role.kubernetes.io/master -o name 2>/dev/null | sed 's|node/||') + + case "${NODE_LABEL_STRATEGY}" in + auto) + # GPU nodes: detected by nvidia.com/gpu.present=true (set by NVIDIA GPU Operator / NFD) + while IFS= read -r node; do + [[ -n "$node" ]] && gpu_nodes+=("$node") + done < <(oc get nodes -l nvidia.com/gpu.present=true,node-role.kubernetes.io/worker -o name 2>/dev/null | sed 's|node/||') + + # CPU nodes: worker nodes without GPU label + while IFS= read -r node; do + [[ -n "$node" ]] && cpu_nodes+=("$node") + done < <(oc get nodes -l '!nvidia.com/gpu.present,node-role.kubernetes.io/worker' -o name 2>/dev/null | sed 's|node/||') + ;; + + manual) + local cpu_count gpu_count + cpu_count=$(yq eval '.openshift.nodes.cpu | length' "${CONFIG_FILE}" 2>/dev/null || echo "0") + gpu_count=$(yq eval '.openshift.nodes.gpu | length' "${CONFIG_FILE}" 2>/dev/null || echo "0") + local i=0 + while [[ $i -lt $cpu_count ]]; do + local n; n=$(yq eval ".openshift.nodes.cpu[$i]" "${CONFIG_FILE}" 2>/dev/null || echo "") + [[ -n "$n" && "$n" != "null" ]] && cpu_nodes+=("$n") + i=$((i+1)) + done + i=0 + while [[ $i -lt $gpu_count ]]; do + local n; n=$(yq eval ".openshift.nodes.gpu[$i]" "${CONFIG_FILE}" 2>/dev/null || echo "") + [[ -n "$n" && "$n" != "null" ]] && gpu_nodes+=("$n") + i=$((i+1)) + done + ;; + + *) + err "Unknown nodeLabelStrategy: ${NODE_LABEL_STRATEGY}. Use 'auto' or 'manual'." + ;; + esac + + # Label control-plane nodes + for node in "${control_nodes[@]}"; do + log " Labeling control-plane node: ${node}" + oc label node "${node}" \ + splunk.ai/node-role=controller \ + splunk.ai/workload-type=control-plane \ + --overwrite + done + + # Label CPU worker nodes + for node in "${cpu_nodes[@]}"; do + log " Labeling CPU worker node: ${node}" + oc label node "${node}" \ + splunk.ai/node-role=worker \ + splunk.ai/workload-type=cpu \ + splunk.ai/instance-type=cpu-worker \ + --overwrite + done + + # Label GPU worker nodes + for node in "${gpu_nodes[@]}"; do + log " Labeling GPU worker node: ${node}" + oc label node "${node}" \ + splunk.ai/node-role=worker \ + splunk.ai/workload-type=gpu \ + splunk.ai/instance-type=gpu-worker \ + --overwrite + # Taint GPU nodes so non-GPU workloads don't land on them + oc adm taint node "${node}" nvidia.com/gpu=true:NoSchedule --overwrite 2>/dev/null || true + done + + # Verify no worker node is left unlabeled — unlabeled workers cause silent Pending forever + local unlabeled + unlabeled=$(oc get nodes -l node-role.kubernetes.io/worker -o json 2>/dev/null \ + | python3 -c " +import json,sys +data=json.load(sys.stdin) +for n in data['items']: + if 'splunk.ai/workload-type' not in n['metadata']['labels']: + print(n['metadata']['name']) +" 2>/dev/null || echo "") + + if [[ -n "${unlabeled}" ]]; then + err "Worker node(s) still missing splunk.ai/workload-type after labeling: +$(echo "${unlabeled}" | sed 's/^/ /') + +If using nodeLabelStrategy: auto, check that the NVIDIA GPU Operator is installed +and nodes have nvidia.com/gpu.present=true, or switch to nodeLabelStrategy: manual +and list nodes explicitly under openshift.nodes.cpu / openshift.nodes.gpu in the config." + fi + + log " ✓ Control-plane nodes: ${#control_nodes[@]}" + log " ✓ CPU worker nodes: ${#cpu_nodes[@]}" + log " ✓ GPU worker nodes: ${#gpu_nodes[@]}" + log "Node labeling complete" +} + +# ====== INSTALL CERT-MANAGER ====== +install_cert_manager() { + log "Installing cert-manager..." + + if oc get namespace cert-manager &>/dev/null; then + log " cert-manager namespace already exists, checking if running..." + if oc get deployment cert-manager -n cert-manager &>/dev/null; then + log " ✓ cert-manager already installed, skipping" + return 0 + fi + fi + + oc apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.13.0/cert-manager.yaml + + log "Waiting for cert-manager to be ready..." + oc wait --for=condition=ready pod \ + -l app.kubernetes.io/instance=cert-manager \ + -n cert-manager --timeout=300s + + # On OpenShift, cert-manager pods may need anyuid SCC + oc adm policy add-scc-to-user anyuid \ + -z cert-manager -n cert-manager 2>/dev/null || true + oc adm policy add-scc-to-user anyuid \ + -z cert-manager-cainjector -n cert-manager 2>/dev/null || true + oc adm policy add-scc-to-user anyuid \ + -z cert-manager-webhook -n cert-manager 2>/dev/null || true + + log "Waiting for cert-manager webhook to be reachable with a valid TLS certificate..." + # The webhook endpoint being ready is not enough — the TLS cert has a notBefore + # timestamp ~30s in the future right after issuance. Probe by applying a test + # Issuer and retrying until the x509 clock-skew error clears. + # NOTE: heredoc inside $(...) is unreliable under set -euo pipefail; use a temp file. + local probe_file + probe_file=$(mktemp /tmp/cert-manager-probe-XXXXXX.yaml) + cat > "${probe_file}" <<'EOF' +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: cert-manager-webhook-probe + namespace: cert-manager +spec: + selfSigned: {} +EOF + local retries=0 + while (( retries < 60 )); do + local out + out=$(oc apply -f "${probe_file}" 2>&1) || true + if echo "${out}" | grep -q "x509: certificate\|failed to call webhook\|i/o timeout"; then + sleep 5 + retries=$((retries + 1)) + (( retries % 6 == 0 )) && log " Still waiting for cert-manager webhook TLS... (${retries}/60)" + continue + fi + oc delete issuer cert-manager-webhook-probe -n cert-manager --ignore-not-found=true 2>/dev/null || true + rm -f "${probe_file}" + break + done + rm -f "${probe_file}" 2>/dev/null || true + log " ✓ cert-manager installed" +} + +# ====== INSTALL LOCAL-PATH PROVISIONER ====== +# k0s installs this as part of cluster setup. OpenShift has no default storage +# class on bare-metal, so we install local-path-provisioner the same way. +install_local_path_provisioner() { + if oc get storageclass 2>/dev/null | grep -q "(default)"; then + log " ✓ Default storage class already exists, skipping local-path install" + oc get storageclass + return 0 + fi + + log "Installing local-path-provisioner (no default storage class found)..." + oc apply -f https://raw.githubusercontent.com/rancher/local-path-provisioner/v0.0.26/deploy/local-path-storage.yaml + + log "Waiting for local-path-provisioner to be ready..." + oc rollout status deployment local-path-provisioner -n local-path-storage --timeout=120s || true + + log "Setting local-path as default storage class..." + oc patch storageclass local-path \ + -p '{"metadata": {"annotations":{"storageclass.kubernetes.io/is-default-class":"true"}}}' + + # The main provisioner pod and the helper pod it spawns both need privileged SCC. + # The main provisioner runs as local-path-provisioner-service-account. + # The helper pod runs as the namespace's default SA (no serviceAccountName set). + oc create clusterrolebinding local-path-provisioner-privileged \ + --clusterrole=system:openshift:scc:privileged \ + --serviceaccount=local-path-storage:local-path-provisioner-service-account \ + 2>/dev/null || true + oc create clusterrolebinding local-path-helper-privileged \ + --clusterrole=system:openshift:scc:privileged \ + --serviceaccount=local-path-storage:default \ + 2>/dev/null || true + + # Patch the helper pod template to run privileged and relabel the created directory + # with svirt_sandbox_file_t so containers can read/write it (SELinux on OpenShift). + # Without the chcon, directories get var_t which containers cannot access. + oc patch configmap local-path-config -n local-path-storage --type=merge -p "$(cat <<'PATCH' +{ + "data": { + "helperPod.yaml": "apiVersion: v1\nkind: Pod\nmetadata:\n name: helper-pod\nspec:\n priorityClassName: system-node-critical\n tolerations:\n - key: node.kubernetes.io/disk-pressure\n operator: Exists\n effect: NoSchedule\n containers:\n - name: helper-pod\n image: busybox\n imagePullPolicy: IfNotPresent\n securityContext:\n privileged: true\n", + "setup": "#!/bin/sh\nset -eu\nmkdir -m 0777 -p \"$VOL_DIR\"\nchcon -Rt container_file_t -l s0 \"$VOL_DIR\" 2>/dev/null || true\n" + } +} +PATCH + )" + + # Restart the provisioner so it picks up the new helper pod template + oc rollout restart deployment local-path-provisioner -n local-path-storage + oc rollout status deployment local-path-provisioner -n local-path-storage --timeout=60s || true + + log " ✓ local-path-provisioner installed and set as default storage class" +} + +# ====== RELABEL WORKER NODE HOST PATHS FOR SELINUX ====== +# On OpenShift with SELinux enforcing, hostPath directories created by root get +# var_t label which containers cannot access. Relabel to container_file_t:s0 +# (no MCS categories) so any container can read/write the volume. +relabel_worker_nodes_for_selinux() { + log "Relabeling /opt/local-path-provisioner on worker nodes for SELinux..." + local workers + workers=$(oc get nodes -l '!node-role.kubernetes.io/master,!node-role.kubernetes.io/control-plane' \ + -o jsonpath='{.items[*].metadata.name}' 2>/dev/null) + for node in ${workers}; do + log " Relabeling node ${node}..." + oc debug "node/${node}" --image=registry.access.redhat.com/ubi8/ubi-minimal -- \ + sh -c "mkdir -p /host/opt/local-path-provisioner && \ + chcon -Rt container_file_t -l s0 /host/opt/local-path-provisioner/ 2>/dev/null || true; \ + echo relabeled" 2>/dev/null || \ + oc debug "node/${node}" -- \ + chroot /host sh -c "mkdir -p /opt/local-path-provisioner && \ + chcon -Rt container_file_t -l s0 /opt/local-path-provisioner/ 2>/dev/null || true" 2>/dev/null || true + done + log " ✓ SELinux labels set on worker nodes" +} + +# ====== INSTALL OPENTELEMETRY OPERATOR ====== +install_otel_operator() { + log "Installing OpenTelemetry Operator..." + + if oc get deployment opentelemetry-operator-controller-manager \ + -n opentelemetry-operator-system &>/dev/null; then + log " ✓ OpenTelemetry Operator already installed, skipping" + return 0 + fi + + helm repo add open-telemetry https://open-telemetry.github.io/opentelemetry-helm-charts 2>/dev/null || true + helm repo update open-telemetry 2>/dev/null || true + + local otel_retries=0 + while (( otel_retries < 6 )); do + local otel_out + otel_out=$(helm upgrade --install opentelemetry-operator open-telemetry/opentelemetry-operator \ + --namespace opentelemetry-operator-system --create-namespace \ + --set manager.collectorImage.repository=otel/opentelemetry-collector-contrib \ + --set admissionWebhooks.certManager.enabled=true \ + --wait=false --timeout=10m 2>&1) + if echo "${otel_out}" | grep -q "x509: certificate\|failed to call webhook\|i/o timeout"; then + warn "cert-manager webhook not ready yet, waiting 10s (${otel_retries}/6)..." + sleep 10 + otel_retries=$((otel_retries + 1)) + continue + fi + echo "${otel_out}" + break + done + + # Grant privileged SCC before pods start (runs as UID 65532 which is outside OCP's range) + oc create clusterrolebinding otel-operator-privileged \ + --clusterrole=system:openshift:scc:privileged \ + --serviceaccount=opentelemetry-operator-system:opentelemetry-operator \ + 2>/dev/null || true + + oc rollout status deployment opentelemetry-operator \ + -n opentelemetry-operator-system --timeout=5m || \ + oc rollout restart deployment opentelemetry-operator \ + -n opentelemetry-operator-system + + wait_for_crd opentelemetrycollectors.opentelemetry.io 300 + log " ✓ OpenTelemetry Operator installed" +} + +# ====== INSTALL KUBERAY OPERATOR ====== +install_ray_operator() { + log "Installing KubeRay Operator..." + + if oc get deployment kuberay-operator -n ray-system &>/dev/null; then + log " ✓ KubeRay Operator already installed, skipping" + return 0 + fi + + helm repo add kuberay https://ray-project.github.io/kuberay-helm/ 2>/dev/null || true + helm repo update kuberay + + helm upgrade --install kuberay-operator kuberay/kuberay-operator \ + --namespace ray-system --create-namespace \ + --version 1.2.2 \ + --set image.repository=quay.io/kuberay/operator \ + --set image.tag=v1.2.2 \ + --wait --timeout=10m + + wait_for_crd rayservices.ray.io 300 + wait_for_crd rayclusters.ray.io 300 + + log " ✓ KubeRay Operator installed" +} + +# ====== ECR PULL SECRET ====== +# Creates ecr-registry-secret in every namespace that pulls ECR images. +# Uses --dry-run=client | apply so it is idempotent (safe to re-run). +ensure_ecr_pull_secret() { + if [[ "${ECR_ENABLED}" != "true" ]]; then + log "ECR pull secret disabled (ecr.enabled=false), skipping" + return 0 + fi + + log "Creating ECR pull secret (account=${ECR_ACCOUNT}, region=${ECR_REGION})..." + + if ! aws sts get-caller-identity &>/dev/null; then + warn "AWS credentials not available — skipping ECR secret creation." + warn "Pods pulling from ECR will fail. Export AWS credentials and re-run install." + return 0 + fi + + local ecr_password + if ! ecr_password=$(aws ecr get-login-password --region "${ECR_REGION}" 2>/dev/null); then + warn "Failed to get ECR token — skipping secret creation" + return 0 + fi + + local server="${ECR_ACCOUNT}.dkr.ecr.${ECR_REGION}.amazonaws.com" + for ns in splunk-ai-operator-system "${AI_NS}"; do + ensure_namespace "${ns}" + oc create secret docker-registry ecr-registry-secret \ + --docker-server="${server}" \ + --docker-username=AWS \ + --docker-password="${ecr_password}" \ + --namespace="${ns}" \ + --dry-run=client -o yaml | oc apply -f - + + # Patch the default SA so pods without explicit imagePullSecrets also pull correctly + oc patch serviceaccount default -n "${ns}" \ + -p '{"imagePullSecrets": [{"name": "ecr-registry-secret"}]}' 2>/dev/null || true + + log " ✓ ecr-registry-secret created in ${ns}" + done + + # Also patch the operator SA specifically + oc patch serviceaccount splunk-ai-operator-controller-manager \ + -n splunk-ai-operator-system \ + -p '{"imagePullSecrets": [{"name": "ecr-registry-secret"}]}' 2>/dev/null || true +} + +# ====== INSTALL SPLUNK AI OPERATOR ====== +install_splunk_ai_operator() { + log "Installing Splunk AI Operator from ${SPLUNK_AI_FILE}..." + + [[ -f "${SPLUNK_AI_FILE}" ]] || { warn "Manifest not found: ${SPLUNK_AI_FILE}"; return 0; } + + local ai_operator_ns="splunk-ai-operator-system" + ensure_namespace "${ai_operator_ns}" + + # Grant SCCs before applying manifests so pods start on first attempt + grant_privileged_scc + + log "Applying Splunk AI Operator manifests (server-side apply)..." + local apply_output + apply_output=$(oc apply --server-side --force-conflicts -f "${SPLUNK_AI_FILE}" 2>&1) || true + echo "${apply_output}" + + # Retry if cert-manager webhook not ready OR if cert-manager CRD mapping was missing. + # Certificate/Issuer resources silently fail with "resource mapping not found" when + # cert-manager pods are up but CRDs haven't been registered in the API server yet. + if echo "${apply_output}" | grep -qi "webhook.*cert-manager\|failed calling webhook.*cert-manager\|i/o timeout\|resource mapping not found\|no matches for kind.*cert-manager"; then + warn "cert-manager CRDs not ready, waiting 20s and retrying full apply..." + sleep 20 + oc apply --server-side --force-conflicts -f "${SPLUNK_AI_FILE}" 2>&1 || true + fi + + # Inject the local instance.yaml so the operator knows about RTX_PRO_6000_BLACKWELL + # and other accelerators that may not be baked into the operator image. + local instance_src + instance_src="$(dirname "${SPLUNK_AI_FILE}")/../../config/configs/instance.yaml" + if [[ ! -f "${instance_src}" ]]; then + instance_src="$(cd "$(dirname "$0")/../.." && pwd)/config/configs/instance.yaml" + fi + if [[ -f "${instance_src}" ]]; then + oc create configmap splunk-ai-operator-instance-yaml \ + -n "${ai_operator_ns}" \ + --from-file=instance.yaml="${instance_src}" \ + --dry-run=client -o yaml | oc -n "${ai_operator_ns}" apply -f - + # Mount the ConfigMap and set INSTANCE_FILE so the operator uses it + oc patch deployment splunk-ai-operator-controller-manager \ + -n "${ai_operator_ns}" --type=json -p='[ + {"op":"add","path":"/spec/template/spec/volumes/-","value":{"name":"instance-yaml","configMap":{"name":"splunk-ai-operator-instance-yaml"}}}, + {"op":"add","path":"/spec/template/spec/containers/0/volumeMounts/-","value":{"name":"instance-yaml","mountPath":"/etc/instance","readOnly":true}}, + {"op":"add","path":"/spec/template/spec/containers/0/env/-","value":{"name":"INSTANCE_FILE","value":"/etc/instance/instance.yaml"}} + ]' 2>/dev/null || true + log " ✓ instance.yaml ConfigMap injected into operator" + else + warn "instance.yaml not found at ${instance_src} — defaultAcceleratorType may not resolve" + fi + + # Patch the operator SA and deployment with ECR pull secret AFTER the manifest apply + # (the SA is created by the manifest; patching before apply silently does nothing). + if [[ "${ECR_ENABLED}" == "true" ]]; then + oc patch serviceaccount splunk-ai-operator-controller-manager \ + -n "${ai_operator_ns}" \ + -p '{"imagePullSecrets": [{"name": "ecr-registry-secret"}]}' 2>/dev/null || true + oc patch deployment splunk-ai-operator-controller-manager \ + -n "${ai_operator_ns}" --type=json \ + -p='[{"op":"add","path":"/spec/template/spec/imagePullSecrets","value":[{"name":"ecr-registry-secret"}]}]' \ + 2>/dev/null || true + log " ✓ ECR pull secret patched into operator SA and deployment" + fi + + # Rollout restart so the deployment picks up pull secrets and instance.yaml. + oc rollout restart deployment splunk-ai-operator-controller-manager \ + -n "${ai_operator_ns}" 2>/dev/null || true + + # Wait for operator deployment to be ready — use the deployment name directly, + # not a label selector, to avoid matching stale ReplicaSets. + # A generous timeout per attempt; the outer loop gives up to 10 minutes total. + log "Waiting for Splunk AI Operator deployment to be ready..." + local retries=0 + while (( retries < 40 )); do + if oc rollout status deployment/splunk-ai-operator-controller-manager \ + -n "${ai_operator_ns}" --timeout=30s 2>/dev/null; then + break + fi + # If the pod is stuck terminating, force-delete it to unblock the rollout + local terminating + terminating=$(oc get pods -n "${ai_operator_ns}" \ + --field-selector=status.phase=Running \ + -l control-plane=controller-manager \ + -o jsonpath='{.items[?(@.metadata.deletionTimestamp)].metadata.name}' 2>/dev/null || true) + if [[ -n "${terminating}" ]]; then + log " Force-deleting stuck terminating pod: ${terminating}" + oc delete pod "${terminating}" -n "${ai_operator_ns}" --grace-period=0 --force 2>/dev/null || true + fi + sleep 10 + retries=$((retries + 1)) + (( retries % 3 == 0 )) && log " Waiting for operator... (${retries}/40)" + done + + # Wait for the webhook service to have endpoints — the pod being Running is not + # enough; the API server needs to register the endpoint before we apply CRs. + log "Waiting for Splunk AI Operator webhook endpoint to be ready..." + local wh_retries=0 + while (( wh_retries < 60 )); do + local ep_count + ep_count=$(oc get endpoints splunk-ai-operator-webhook-service \ + -n "${ai_operator_ns}" -o jsonpath='{.subsets[*].addresses}' 2>/dev/null | wc -w | tr -d ' ') + if [[ "${ep_count}" -gt 0 ]]; then + log " ✓ Webhook endpoint ready" + break + fi + sleep 5 + wh_retries=$((wh_retries + 1)) + (( wh_retries % 6 == 0 )) && log " Still waiting for webhook endpoint... (${wh_retries}/60)" + done + + log " ✓ Splunk AI Operator installed" +} + +# ====== INSTALL SPLUNK OPERATOR ====== +install_splunk_operator() { + log "Installing Splunk Operator..." + + [[ -f "${SPLUNK_OPERATOR_FILE}" ]] || { warn "Splunk operator file not found: ${SPLUNK_OPERATOR_FILE}, skipping"; return 0; } + + local splunk_operator_ns="splunk-operator" + ensure_namespace "${splunk_operator_ns}" + + # Create ECR pull secret in splunk-operator namespace + if [[ "${ECR_ENABLED}" == "true" ]]; then + local ecr_password + if ecr_password=$(aws ecr get-login-password --region "${ECR_REGION}" 2>/dev/null); then + oc create secret docker-registry ecr-registry-secret \ + --docker-server="${ECR_ACCOUNT}.dkr.ecr.${ECR_REGION}.amazonaws.com" \ + --docker-username=AWS \ + --docker-password="${ecr_password}" \ + --namespace="${splunk_operator_ns}" \ + --dry-run=client -o yaml | oc apply -f - + fi + fi + + if oc create -f "${SPLUNK_OPERATOR_FILE}" 2>/dev/null; then + log " Splunk Operator resources created" + else + log " Resources already exist, updating..." + oc replace --force -f "${SPLUNK_OPERATOR_FILE}" 2>&1 | grep -v "Warning: --force is deprecated" || true + fi + + # Grant privileged SCC to the whole namespace group — this is the pattern OCP SCC admission + # actually honours. The operator pod adds NET_BIND_SERVICE which anyuid blocks; privileged + # covers both. group-based grant survives replace --force (which recreates the namespace). + oc adm policy add-scc-to-group privileged \ + "system:serviceaccounts:${splunk_operator_ns}" 2>/dev/null || true + # Force pod recreation so it picks up the new SCC grant + oc delete replicaset -n "${splunk_operator_ns}" --all 2>/dev/null || true + + # Patch deployment with pull secret if present + local dep_name + dep_name=$(oc -n "${splunk_operator_ns}" get deploy -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || echo "") + if [[ -n "${dep_name}" ]] && oc get secret ecr-registry-secret -n "${splunk_operator_ns}" &>/dev/null; then + oc -n "${splunk_operator_ns}" patch deployment "${dep_name}" \ + --type='json' \ + -p='[{"op":"add","path":"/spec/template/spec/imagePullSecrets","value":[{"name":"ecr-registry-secret"}]}]' \ + 2>/dev/null || true + oc rollout restart deployment "${dep_name}" -n "${splunk_operator_ns}" 2>/dev/null || true + fi + + wait_for_crd standalones.enterprise.splunk.com 300 + log " ✓ Splunk Operator installed" +} + +# ====== INSTALL SPLUNK STANDALONE ====== +install_splunk_standalone() { + log "Installing Splunk Standalone: ${AI_STANDALONE_NAME} in ${AI_NS}..." + + ensure_namespace "${AI_NS}" + wait_for_crd standalones.enterprise.splunk.com 600 + + # Object storage credentials secret + oc -n "${AI_NS}" create secret generic minio-credentials \ + --from-literal=AWS_ACCESS_KEY_ID="${MINIO_ROOT_USER}" \ + --from-literal=AWS_SECRET_ACCESS_KEY="${MINIO_ROOT_PASSWORD}" \ + --from-literal=s3_access_key="${MINIO_ROOT_USER}" \ + --from-literal=s3_secret_key="${MINIO_ROOT_PASSWORD}" \ + --from-literal=MINIO_ACCESS_KEY="${MINIO_ROOT_USER}" \ + --from-literal=MINIO_SECRET_KEY="${MINIO_ROOT_PASSWORD}" \ + --dry-run=client -o yaml | oc -n "${AI_NS}" apply -f - + + # Derive S3 endpoint for Splunk appRepo (endpoint is required by the Splunk Operator) + local minio_endpoint="${OBJ_STORE_ENDPOINT}" + if [[ -z "${minio_endpoint}" && "${OBJ_STORE_TYPE}" == "aws" ]]; then + minio_endpoint="https://s3.${ECR_REGION}.amazonaws.com" + log " type=aws: using S3 endpoint ${minio_endpoint}" + fi + [[ -z "${minio_endpoint}" ]] && err "storage.objectStore.endpoint must be set for type=${OBJ_STORE_TYPE}" + + oc apply --server-side --force-conflicts -f - </dev/null || true + oc delete pods -n "${AI_NS}" --field-selector status.phase=Failed --wait=false 2>/dev/null || true + + # Build imagePullSecrets block + local secrets_yaml="" + for secret_name in ecr-registry-secret; do + oc get secret "${secret_name}" -n "${AI_NS}" &>/dev/null && \ + secrets_yaml+=" - name: ${secret_name}"$'\n' + done + local image_pull_secrets="" + [[ -n "${secrets_yaml}" ]] && image_pull_secrets=" imagePullSecrets:"$'\n'"${secrets_yaml}" + + # Object storage path and endpoint + local obj_path obj_endpoint + case "${OBJ_STORE_TYPE}" in + aws) obj_path="s3://${OBJ_STORE_BUCKET}"; obj_endpoint="" ;; + s3compat) obj_path="s3compat://${OBJ_STORE_BUCKET}"; obj_endpoint="${OBJ_STORE_ENDPOINT}" ;; + minio) obj_path="minio://${OBJ_STORE_BUCKET}"; obj_endpoint="${OBJ_STORE_ENDPOINT}" ;; + seaweedfs) obj_path="seaweedfs://${OBJ_STORE_BUCKET}";obj_endpoint="${OBJ_STORE_ENDPOINT}" ;; + *) err "Unsupported objectStore.type: ${OBJ_STORE_TYPE}" ;; + esac + + # Features + local features_yaml="" + local feature_count + feature_count=$(yq eval '.aiPlatform.features | length' "${CONFIG_FILE}" 2>/dev/null || echo "0") + if [[ "${feature_count}" -gt 0 ]]; then + local i=0 + while [[ $i -lt $feature_count ]]; do + local fname fver + fname=$(yq eval ".aiPlatform.features[$i].name" "${CONFIG_FILE}") + fver=$(yq eval ".aiPlatform.features[$i].version // \"1.0.0\"" "${CONFIG_FILE}") + [[ -n "$fname" && "$fname" != "null" ]] && \ + features_yaml+=" - name: ${fname}"$'\n'" version: \"${fver}\""$'\n' + i=$((i + 1)) + done + else + features_yaml=" - name: saia"$'\n'" version: \"1.1.0\""$'\n' + fi + + # Service template + local svc_template_yaml="" + local svc_type + svc_type=$(yq eval '.aiPlatform.serviceTemplate.type // ""' "${CONFIG_FILE}" 2>/dev/null || echo "") + if [[ -n "${svc_type}" && "${svc_type}" != "null" && "${svc_type}" != "ClusterIP" ]]; then + local svc_node_port + svc_node_port=$(yq eval '.aiPlatform.serviceTemplate.nodePort // ""' "${CONFIG_FILE}" 2>/dev/null || echo "") + svc_template_yaml=" serviceTemplate:"$'\n'" spec:"$'\n'" type: ${svc_type}"$'\n' + if [[ -n "${svc_node_port}" && "${svc_type}" == "NodePort" ]]; then + svc_template_yaml+=" ports:"$'\n'" - name: http"$'\n'" port: 8080"$'\n'" targetPort: 8080"$'\n'" nodePort: ${svc_node_port}"$'\n' + fi + fi + + # The operator looks up splunk--secret for the HEC token. + # Extract it from the Splunk standalone secret created by the Splunk Operator. + local splunk_ns_secret="splunk-${AI_NS}-secret" + local standalone_secret="splunk-${AI_STANDALONE_NAME}-standalone-secret-v1" + log " Waiting for Splunk standalone secret ${standalone_secret}..." + local retries=0 + while (( retries < 60 )); do + if oc get secret "${standalone_secret}" -n "${AI_NS}" &>/dev/null; then + local hec_token + hec_token=$(oc get secret "${standalone_secret}" -n "${AI_NS}" \ + -o jsonpath='{.data.hec_token}' 2>/dev/null || echo "") + if [[ -n "${hec_token}" ]]; then + oc -n "${AI_NS}" create secret generic "${splunk_ns_secret}" \ + --from-literal=hec_token="$(echo "${hec_token}" | base64 -d)" \ + --dry-run=client -o yaml | oc apply -f - + log " ✓ ${splunk_ns_secret} created" + break + fi + fi + sleep 10 + retries=$(( retries + 1 )) + log " Waiting for Splunk secret... (${retries}/60)" + done + if (( retries >= 60 )); then + warn "Splunk secret not ready after 10m — AIPlatform reconcile will retry automatically" + fi + + local storage_yaml="" + if [[ -n "${STORAGE_CLASS}" && "${STORAGE_CLASS}" != "null" ]]; then + storage_yaml=" storage:"$'\n'" vectorDB:"$'\n'" size: ${VECTORDB_SIZE}"$'\n'" storageClassName: ${STORAGE_CLASS}"$'\n' + fi + + # Probe the AIPlatform webhook TLS cert immediately before applying. + # cert-manager issues certs with notBefore ~30-60s in the future (clock skew); + # retry until the x509 error clears. Using --dry-run=server hits the exact + # same webhook (maiplatform-v1.kb.io) without creating anything. + local ai_operator_ns="splunk-ai-operator-system" + local tls_probe_file + tls_probe_file=$(mktemp /tmp/aiplatform-tls-probe-XXXXXX.yaml) + cat > "${tls_probe_file}" <<'PROBE_EOF' +apiVersion: ai.splunk.com/v1 +kind: AIPlatform +metadata: + name: webhook-tls-probe + namespace: splunk-ai-operator-system +spec: + defaultAcceleratorType: L40S + objectStorage: + path: s3://probe/probe +PROBE_EOF + local tls_retries=0 + while (( tls_retries < 60 )); do + local tls_out + tls_out=$(oc apply --dry-run=server -f "${tls_probe_file}" 2>&1) || true + if echo "${tls_out}" | grep -q "x509:\|not yet valid\|certificate has expired\|failed to verify certificate\|failed to call webhook"; then + sleep 5 + tls_retries=$((tls_retries + 1)) + (( tls_retries % 6 == 0 )) && log " Still waiting for operator webhook TLS cert... (${tls_retries}/60)" + continue + fi + log " ✓ Operator webhook TLS certificate valid" + break + done + rm -f "${tls_probe_file}" 2>/dev/null || true + + oc -n "${AI_NS}" apply --server-side --force-conflicts -f - </dev/null 2>&1; do + sleep 5; elapsed=$((elapsed + 5)) + [[ ${elapsed} -ge ${timeout} ]] && { warn "Timeout waiting for AIPlatform CR"; break; } + done + + oc get aiplatform "${AI_PLATFORM_NAME}" -n "${AI_NS}" -o wide || true + log " ✓ AIPlatform CR installed" +} + +# ====== MAIN INSTALL ====== +main_install() { + log "============================================" + log " Splunk AI Platform — OpenShift Install" + log "============================================" + + load_config + preflight_checks + validate_image_config + configure_images + install_nfd + install_nvidia_gpu_operator + label_nodes + install_local_path_provisioner + relabel_worker_nodes_for_selinux + install_cert_manager + install_otel_operator + install_ray_operator + ensure_ecr_pull_secret + install_splunk_ai_operator + install_splunk_operator + install_splunk_standalone + install_ai_platform_cr + + log "============================================" + log " Install complete" + log "============================================" + log "" + log "Next steps:" + log " 1. Create an AIPlatform CR in namespace '${AI_NS}'" + log " 2. Check operator logs:" + log " oc logs -n splunk-ai-operator-system -l control-plane=controller-manager -f" + log " 3. Watch resources:" + log " oc get aiplatform,raycluster,rayservice -n ${AI_NS}" + log "" + log "Log file: ${LOG_FILE}" +} + +# ====== MAIN DELETE ====== +main_delete() { + log "============================================" + log " Splunk AI Platform — OpenShift Delete" + log "============================================" + + load_config + + if ! oc whoami &>/dev/null; then + err "Not logged in to OpenShift. Run: oc login " + fi + + local ai_operator_ns="splunk-ai-operator-system" + local splunk_operator_ns="splunk-operator" + + # ── 1. AI Platform CRs (trigger operator finalizers before namespace delete) ── + log "Removing AIPlatform CR and waiting for finalizers..." + oc delete aiplatform --all -n "${AI_NS}" --timeout=120s 2>/dev/null || true + oc delete standalone --all -n "${AI_NS}" --timeout=60s 2>/dev/null || true + + # ── 2. AI Platform namespace (cascades all pods, PVCs, services, etc.) ── + log "Deleting namespace ${AI_NS}..." + oc delete namespace "${AI_NS}" --timeout=180s 2>/dev/null || true + + # ── 3. Splunk AI Operator ── + log "Removing Splunk AI Operator..." + oc delete namespace "${ai_operator_ns}" --timeout=60s 2>/dev/null || true + # Remove cluster-scoped resources (CRDs, ClusterRoles, webhooks) from manifests + [[ -f "${SPLUNK_AI_FILE}" ]] && \ + oc delete -f "${SPLUNK_AI_FILE}" --ignore-not-found=true 2>/dev/null || true + + # ── 4. Splunk Operator ── + log "Removing Splunk Operator..." + oc delete namespace "${splunk_operator_ns}" --timeout=60s 2>/dev/null || true + [[ -f "${SPLUNK_OPERATOR_FILE}" ]] && \ + oc delete -f "${SPLUNK_OPERATOR_FILE}" --ignore-not-found=true 2>/dev/null || true + + # ── 5. KubeRay Operator (helm) ── + log "Removing KubeRay Operator..." + helm uninstall kuberay-operator -n ray-system 2>/dev/null || true + oc delete namespace ray-system --timeout=60s 2>/dev/null || true + + # ── 6. OpenTelemetry Operator (helm) ── + log "Removing OpenTelemetry Operator..." + helm uninstall opentelemetry-operator -n opentelemetry-operator-system 2>/dev/null || true + oc delete namespace opentelemetry-operator-system --timeout=60s 2>/dev/null || true + + # ── 7. cert-manager (helm) ── + log "Removing cert-manager..." + helm uninstall cert-manager -n cert-manager 2>/dev/null || true + oc delete namespace cert-manager --timeout=60s 2>/dev/null || true + # Remove CRDs left by cert-manager (helm uninstall doesn't remove CRDs by default) + oc get crd -o name 2>/dev/null | grep cert-manager | xargs -r oc delete --ignore-not-found=true 2>/dev/null || true + + # ── 8. local-path-provisioner ── + log "Removing local-path-provisioner..." + oc delete -f https://raw.githubusercontent.com/rancher/local-path-provisioner/v0.0.26/deploy/local-path-storage.yaml \ + --ignore-not-found=true 2>/dev/null || true + oc delete namespace local-path-storage --timeout=60s 2>/dev/null || true + oc delete storageclass local-path --ignore-not-found=true 2>/dev/null || true + + # ── 9. NVIDIA GPU Operator ── + log "Removing NVIDIA GPU Operator..." + oc delete clusterpolicy gpu-cluster-policy --ignore-not-found=true 2>/dev/null || true + oc delete subscription gpu-operator-certified -n nvidia-gpu-operator --ignore-not-found=true 2>/dev/null || true + oc delete csv -n nvidia-gpu-operator --all --ignore-not-found=true 2>/dev/null || true + oc delete namespace nvidia-gpu-operator --timeout=60s 2>/dev/null || true + + # ── 10. NFD ── + log "Removing Node Feature Discovery..." + oc delete nodefeaturediscovery nfd-instance -n openshift-nfd --ignore-not-found=true 2>/dev/null || true + oc delete subscription nfd -n openshift-nfd --ignore-not-found=true 2>/dev/null || true + oc delete csv -n openshift-nfd --all --ignore-not-found=true 2>/dev/null || true + oc delete namespace openshift-nfd --timeout=60s 2>/dev/null || true + + # ── 11. Node labels and taints added by label_nodes() ── + log "Removing splunk.ai/* node labels and GPU taint..." + for node in $(oc get nodes -l 'splunk.ai/workload-type' -o name 2>/dev/null); do + oc label "${node}" splunk.ai/workload-type- 2>/dev/null || true + oc taint "${node}" nvidia.com/gpu=true:NoSchedule- 2>/dev/null || true + done + + # ── 12. SCC grants added during install ── + if [[ "${GRANT_PRIVILEGED_SCC}" == "true" ]]; then + log "Removing SCC grants..." + oc adm policy remove-scc-from-group privileged \ + "system:serviceaccounts:${ai_operator_ns}" 2>/dev/null || true + oc adm policy remove-scc-from-group anyuid \ + "system:serviceaccounts:${AI_NS}" 2>/dev/null || true + oc adm policy remove-scc-from-group privileged \ + "system:serviceaccounts:${AI_NS}" 2>/dev/null || true + oc adm policy remove-scc-from-group privileged \ + "system:serviceaccounts:local-path-storage" 2>/dev/null || true + oc adm policy remove-scc-from-group privileged \ + "system:serviceaccounts:splunk-operator" 2>/dev/null || true + fi + + # Remove individual ClusterRoleBindings created during install + for crb in \ + local-path-provisioner-privileged \ + local-path-helper-privileged \ + splunk-standalone-privileged \ + splunk-operator-privileged \ + splunk-operator-anyuid \ + otel-operator-privileged \ + otel-operator-anyuid \ + scc-privileged-ai-platform-all \ + scc-privileged-splunk-ai-operator-system-default \ + scc-privileged-splunk-ai-operator-system-splunk-ai-operator-controller-manager; do + oc delete clusterrolebinding "${crb}" --ignore-not-found=true 2>/dev/null || true + done + + # ── 13. ECR pull secret ClusterRoleBindings ── + oc delete clusterrolebinding ecr-registry-secret-updater 2>/dev/null || true + + log "============================================" + log " Delete complete" + log "============================================" + log "" + log "Cluster itself is untouched — only the AI Platform stack was removed." + log "Log file: ${LOG_FILE}" +} + +# ====== USAGE ====== +usage() { + cat < + - oc, yq in PATH + - artifacts.yaml (operator manifests) in the same directory, or set files.aiPlatform in config +EOF +} + +# ====== MAIN ====== +case "${1:-install}" in + install) + main_install + ;; + delete) + main_delete + ;; + *) + usage + exit 1 + ;; +esac From 64420d827cebc45b36b860a9142b8c4c1d9bd47f Mon Sep 17 00:00:00 2001 From: kbhos Date: Mon, 15 Jun 2026 11:54:19 +0530 Subject: [PATCH 02/80] UPdate test to reflect Gemma431bIt removal --- pkg/ai/raybuilder/configmap_apps_test.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/ai/raybuilder/configmap_apps_test.go b/pkg/ai/raybuilder/configmap_apps_test.go index 4beb1b4e..fbdad4ef 100644 --- a/pkg/ai/raybuilder/configmap_apps_test.go +++ b/pkg/ai/raybuilder/configmap_apps_test.go @@ -86,9 +86,11 @@ func Test_ApplicationsYAML_DisableResponsesRedis(t *testing.T) { } } - expectedTextGenApps := []string{"Gemma431bIt", "GptOss20b"} + expectedTextGenApps := []string{"GptOss20b"} - // We expect exactly two text-gen apps today (Gemma431bIt, GptOss20b). + // We expect exactly one text-gen app today (GptOss20b). Gemma431bIt was + // removed — it requires 2× RTX PRO 6000 GPUs and cannot run alongside + // GptOss20b on a single 2-GPU node. // If this count changes, someone added a new text-gen model; they MUST // also add DISABLE_RESPONSES_API_REDIS to the new app. require.Len(t, textGenApps, len(expectedTextGenApps), From 377ff1dfbf9a2d9aa4741e6a98d060bc17f3c1d5 Mon Sep 17 00:00:00 2001 From: kbhos Date: Mon, 15 Jun 2026 12:34:57 +0530 Subject: [PATCH 03/80] =?UTF-8?q?fix(AIP-3938):=20Revert=20artifacts.yaml?= =?UTF-8?q?=20to=20main=20=E2=80=94=20upstream=20changes=20leaked=20via=20?= =?UTF-8?q?stash?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tools/cluster_setup/artifacts.yaml | 67 ++++++++++++++---------------- 1 file changed, 31 insertions(+), 36 deletions(-) diff --git a/tools/cluster_setup/artifacts.yaml b/tools/cluster_setup/artifacts.yaml index f2347653..c6953e7b 100644 --- a/tools/cluster_setup/artifacts.yaml +++ b/tools/cluster_setup/artifacts.yaml @@ -1061,18 +1061,11 @@ spec: items: description: FeatureSpec defines the features to enable in the AIPlatform properties: - env: - additionalProperties: - type: string - description: Env specifies environment variables to propagate - to the child AIService. - type: object name: description: Name of the feature, e.g. "saia" or "seca" enum: - saia - seca - - weaviate-service type: string scaleFactor: description: ScaleFactor is the desired fixed number of replicas @@ -2092,11 +2085,6 @@ spec: type: object x-kubernetes-map-type: atomic type: array - otelImage: - default: otel/opentelemetry-collector-contrib:0.122.1 - description: OTelImage is the OpenTelemetry Collector sidecar - image - type: string rayHeadGroupImage: description: Ray head group image, e.g. "rayproject/ray-head:latest" type: string @@ -2237,8 +2225,7 @@ spec: type: object objectStorage: description: |- - ObjectStorage defines the object storage configuration for AI artifacts, tasks, and models. - It is optional for platforms that only enable features that do not require object storage. + ObjectStorage defines the object storage configuration for AI artifacts, tasks, and models Supported providers: S3, GCS, Azure Blob Storage, MinIO properties: endpoint: @@ -2250,8 +2237,8 @@ spec: path: description: |- Remote volume URI in the format s3://bucketname/, gs://bucketname/, - azure://containername/, or minio://bucketname/ - pattern: ^(s3|gs|azure|minio)://[a-zA-Z0-9.\-_]+(/.*)?$ + azure://containername/, minio://bucketname/, seaweedfs://bucketname/, or s3compat://bucketname/ + pattern: ^(s3|gs|azure|minio|seaweedfs|s3compat)://[a-zA-Z0-9.\-_]+(/.*)?$ type: string region: description: Region of the remote storage volume. Required for @@ -2921,6 +2908,8 @@ spec: pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ type: string type: object + required: + - objectStorage type: object status: description: AIPlatformStatus defines observed state @@ -4095,18 +4084,11 @@ spec: features: description: Feature defines the features to be enabled for the AIService properties: - env: - additionalProperties: - type: string - description: Env specifies environment variables to propagate - to the child AIService. - type: object name: description: Name of the feature, e.g. "saia" or "seca" enum: - saia - seca - - weaviate-service type: string scaleFactor: description: ScaleFactor is the desired fixed number of replicas @@ -4884,15 +4866,27 @@ spec: properties: endpoint: description: |- - Optional override endpoint (only needed for S3-compatible services like MinIO) - Must be a valid HTTP/HTTPS URL + Optional override endpoint (only needed for S3-compatible services like MinIO, SeaweedFS) + Must be a valid HTTP/HTTPS URL. When set with s3:// path, backend is treated as S3-compatible (MinIO, SeaweedFS, etc.) pattern: ^https?://.*$ type: string path: description: |- Remote volume URI in the format s3://bucketname/, gs://bucketname/, - azure://containername/, or minio://bucketname/ - pattern: ^(s3|gs|azure|minio)://[a-zA-Z0-9.\-_]+(/.*)?$ + azure://containername/, s3compat://bucketname/ (generic S3-compatible), minio://, or seaweedfs:// + pattern: ^(s3|gs|azure|minio|seaweedfs|s3compat)://[a-zA-Z0-9.\-_]+(/.*)?$ + type: string + provider: + description: |- + Provider is an optional hint for documentation and tooling. Operator derives behavior from path scheme and endpoint. + Values: aws, minio, seaweedfs, s3compat, gcs, azure + enum: + - aws + - minio + - seaweedfs + - s3compat + - gcs + - azure type: string region: description: Region of the remote storage volume. Required for @@ -4900,7 +4894,8 @@ spec: minLength: 1 type: string secretRef: - description: Secret name containing storage credentials + description: Secret name containing storage credentials (e.g. + s3_access_key, s3_secret_key for S3-compatible backends) maxLength: 253 minLength: 1 type: string @@ -5687,19 +5682,19 @@ spec: fieldRef: fieldPath: metadata.name - name: RELATED_IMAGE_RAY_HEAD - value: 658391232643.dkr.ecr.us-east-2.amazonaws.com/ml-platform/ray/ray-head:build-953 + value: 658391232643.dkr.ecr.us-east-2.amazonaws.com/ml-platform/ray/ray-head:build-v2-010 - name: RELATED_IMAGE_RAY_WORKER - value: 658391232643.dkr.ecr.us-east-2.amazonaws.com/ml-platform/ray/ray-worker-gpu:build-953 + value: 658391232643.dkr.ecr.us-east-2.amazonaws.com/ml-platform/ray/ray-worker-gpu:build-v2-010 - name: RELATED_IMAGE_WEAVIATE value: docker.io/semitechnologies/weaviate:stable-v1.28-007846a - - name: RELATED_IMAGE_WEAVIATE_SERVICE - value: docker.io/semitechnologies/weaviate:stable-v1.28-007846a - name: RELATED_IMAGE_SAIA_API - value: 658391232643.dkr.ecr.us-east-2.amazonaws.com/ml-platform/saia/saia-api:build-v2-main-c3b489d + value: 658391232643.dkr.ecr.us-east-2.amazonaws.com/ml-platform/saia/saia-api:build-v2-012 - name: RELATED_IMAGE_SAIA_API_V2 - value: 658391232643.dkr.ecr.us-east-2.amazonaws.com/ml-platform/saia/saia-api-v2:build-v2-main-c3b489d + value: 658391232643.dkr.ecr.us-east-2.amazonaws.com/ml-platform/saia/saia-api-v2:build-v2-012 - name: RELATED_IMAGE_POST_INSTALL_HOOK - value: 658391232643.dkr.ecr.us-east-2.amazonaws.com/ml-platform/saia/saia-data-loader:build-v2-main-c3b489d + value: 658391232643.dkr.ecr.us-east-2.amazonaws.com/ml-platform/saia/saia-data-loader:build-v2-012 + - name: SPLUNK_METRICS_INDEX_NAME + value: _metrics - name: RELATED_IMAGE_FLUENT_BIT value: docker.io/fluent/fluent-bit:1.9.6 - name: RELATED_IMAGE_OTEL_COLLECTOR @@ -5710,7 +5705,7 @@ spec: value: v0.3.14-36-g1549f5a - name: RAY_VERSION value: 2.53.0 - image: 658391232643.dkr.ecr.us-east-2.amazonaws.com/kiran/splunk/splunk-ai-operator:openshift-0.1 + image: 658391232643.dkr.ecr.us-east-2.amazonaws.com/arif/splunk/splunk-ai-operator:v0.1.29 livenessProbe: httpGet: path: /healthz From 87982bb5332e2f6756254d716f2b84b630bd53f5 Mon Sep 17 00:00:00 2001 From: kbhos Date: Mon, 15 Jun 2026 16:13:02 +0530 Subject: [PATCH 04/80] successfully deploy gpt-oss20b model --- config/configs/features/saia.yaml | 2 +- config/configs/instance.yaml | 33 ++++++++- tools/cluster_setup/artifacts.yaml | 67 ++++++++++--------- .../openshift-cluster-config.yaml | 2 +- 4 files changed, 70 insertions(+), 34 deletions(-) diff --git a/config/configs/features/saia.yaml b/config/configs/features/saia.yaml index 4e94b27d..aea4a115 100644 --- a/config/configs/features/saia.yaml +++ b/config/configs/features/saia.yaml @@ -25,5 +25,5 @@ instanceScale: h100-nvl-1-gpu: 2 RTX_PRO_6000_BLACKWELL: rtx-pro-6000-blackwell-0-gpu: 1 - rtx-pro-6000-blackwell-1-gpu: 1 + rtx-pro-6000-blackwell-1-gpu: 2 rtx-pro-6000-blackwell-2-gpu: 0 \ No newline at end of file diff --git a/config/configs/instance.yaml b/config/configs/instance.yaml index 71ea8e78..3183dbf9 100644 --- a/config/configs/instance.yaml +++ b/config/configs/instance.yaml @@ -85,4 +85,35 @@ H100_NVL: cpu: "16" memory: "48Gi" ephemeral-storage: "100Gi" - nvidia.com/gpu: "1" \ No newline at end of file + nvidia.com/gpu: "1" +RTX_PRO_6000_BLACKWELL: + - tier: rtx-pro-6000-blackwell-0-gpu + gpusPerPod: 0 + resources: + limits: + cpu: "16" + memory: "24Gi" + ephemeral-storage: "50Gi" + nvidia.com/gpu: "0" + requests: + cpu: "4" + - tier: rtx-pro-6000-blackwell-1-gpu + gpusPerPod: 1 + resources: + requests: + cpu: "4" + limits: + cpu: "16" + memory: "48Gi" + ephemeral-storage: "200Gi" + nvidia.com/gpu: "1" + - tier: rtx-pro-6000-blackwell-2-gpu + gpusPerPod: 2 + resources: + requests: + cpu: "4" + limits: + cpu: "8" + memory: "96Gi" + ephemeral-storage: "400Gi" + nvidia.com/gpu: "2" \ No newline at end of file diff --git a/tools/cluster_setup/artifacts.yaml b/tools/cluster_setup/artifacts.yaml index c6953e7b..cb480386 100644 --- a/tools/cluster_setup/artifacts.yaml +++ b/tools/cluster_setup/artifacts.yaml @@ -1061,11 +1061,18 @@ spec: items: description: FeatureSpec defines the features to enable in the AIPlatform properties: + env: + additionalProperties: + type: string + description: Env specifies environment variables to propagate + to the child AIService. + type: object name: description: Name of the feature, e.g. "saia" or "seca" enum: - saia - seca + - weaviate-service type: string scaleFactor: description: ScaleFactor is the desired fixed number of replicas @@ -2085,6 +2092,11 @@ spec: type: object x-kubernetes-map-type: atomic type: array + otelImage: + default: otel/opentelemetry-collector-contrib:0.122.1 + description: OTelImage is the OpenTelemetry Collector sidecar + image + type: string rayHeadGroupImage: description: Ray head group image, e.g. "rayproject/ray-head:latest" type: string @@ -2225,7 +2237,8 @@ spec: type: object objectStorage: description: |- - ObjectStorage defines the object storage configuration for AI artifacts, tasks, and models + ObjectStorage defines the object storage configuration for AI artifacts, tasks, and models. + It is optional for platforms that only enable features that do not require object storage. Supported providers: S3, GCS, Azure Blob Storage, MinIO properties: endpoint: @@ -2237,8 +2250,8 @@ spec: path: description: |- Remote volume URI in the format s3://bucketname/, gs://bucketname/, - azure://containername/, minio://bucketname/, seaweedfs://bucketname/, or s3compat://bucketname/ - pattern: ^(s3|gs|azure|minio|seaweedfs|s3compat)://[a-zA-Z0-9.\-_]+(/.*)?$ + azure://containername/, or minio://bucketname/ + pattern: ^(s3|gs|azure|minio)://[a-zA-Z0-9.\-_]+(/.*)?$ type: string region: description: Region of the remote storage volume. Required for @@ -2908,8 +2921,6 @@ spec: pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ type: string type: object - required: - - objectStorage type: object status: description: AIPlatformStatus defines observed state @@ -4084,11 +4095,18 @@ spec: features: description: Feature defines the features to be enabled for the AIService properties: + env: + additionalProperties: + type: string + description: Env specifies environment variables to propagate + to the child AIService. + type: object name: description: Name of the feature, e.g. "saia" or "seca" enum: - saia - seca + - weaviate-service type: string scaleFactor: description: ScaleFactor is the desired fixed number of replicas @@ -4866,27 +4884,15 @@ spec: properties: endpoint: description: |- - Optional override endpoint (only needed for S3-compatible services like MinIO, SeaweedFS) - Must be a valid HTTP/HTTPS URL. When set with s3:// path, backend is treated as S3-compatible (MinIO, SeaweedFS, etc.) + Optional override endpoint (only needed for S3-compatible services like MinIO) + Must be a valid HTTP/HTTPS URL pattern: ^https?://.*$ type: string path: description: |- Remote volume URI in the format s3://bucketname/, gs://bucketname/, - azure://containername/, s3compat://bucketname/ (generic S3-compatible), minio://, or seaweedfs:// - pattern: ^(s3|gs|azure|minio|seaweedfs|s3compat)://[a-zA-Z0-9.\-_]+(/.*)?$ - type: string - provider: - description: |- - Provider is an optional hint for documentation and tooling. Operator derives behavior from path scheme and endpoint. - Values: aws, minio, seaweedfs, s3compat, gcs, azure - enum: - - aws - - minio - - seaweedfs - - s3compat - - gcs - - azure + azure://containername/, or minio://bucketname/ + pattern: ^(s3|gs|azure|minio)://[a-zA-Z0-9.\-_]+(/.*)?$ type: string region: description: Region of the remote storage volume. Required for @@ -4894,8 +4900,7 @@ spec: minLength: 1 type: string secretRef: - description: Secret name containing storage credentials (e.g. - s3_access_key, s3_secret_key for S3-compatible backends) + description: Secret name containing storage credentials maxLength: 253 minLength: 1 type: string @@ -5682,19 +5687,19 @@ spec: fieldRef: fieldPath: metadata.name - name: RELATED_IMAGE_RAY_HEAD - value: 658391232643.dkr.ecr.us-east-2.amazonaws.com/ml-platform/ray/ray-head:build-v2-010 + value: 658391232643.dkr.ecr.us-east-2.amazonaws.com/ml-platform/ray/ray-head:build-953 - name: RELATED_IMAGE_RAY_WORKER - value: 658391232643.dkr.ecr.us-east-2.amazonaws.com/ml-platform/ray/ray-worker-gpu:build-v2-010 + value: 658391232643.dkr.ecr.us-east-2.amazonaws.com/ml-platform/ray/ray-worker-gpu:build-953 - name: RELATED_IMAGE_WEAVIATE value: docker.io/semitechnologies/weaviate:stable-v1.28-007846a + - name: RELATED_IMAGE_WEAVIATE_SERVICE + value: docker.io/semitechnologies/weaviate:stable-v1.28-007846a - name: RELATED_IMAGE_SAIA_API - value: 658391232643.dkr.ecr.us-east-2.amazonaws.com/ml-platform/saia/saia-api:build-v2-012 + value: 658391232643.dkr.ecr.us-east-2.amazonaws.com/ml-platform/saia/saia-api:build-v2-main-c3b489d - name: RELATED_IMAGE_SAIA_API_V2 - value: 658391232643.dkr.ecr.us-east-2.amazonaws.com/ml-platform/saia/saia-api-v2:build-v2-012 + value: 658391232643.dkr.ecr.us-east-2.amazonaws.com/ml-platform/saia/saia-api-v2:build-v2-main-c3b489d - name: RELATED_IMAGE_POST_INSTALL_HOOK - value: 658391232643.dkr.ecr.us-east-2.amazonaws.com/ml-platform/saia/saia-data-loader:build-v2-012 - - name: SPLUNK_METRICS_INDEX_NAME - value: _metrics + value: 658391232643.dkr.ecr.us-east-2.amazonaws.com/ml-platform/saia/saia-data-loader:build-v2-main-c3b489d - name: RELATED_IMAGE_FLUENT_BIT value: docker.io/fluent/fluent-bit:1.9.6 - name: RELATED_IMAGE_OTEL_COLLECTOR @@ -5705,7 +5710,7 @@ spec: value: v0.3.14-36-g1549f5a - name: RAY_VERSION value: 2.53.0 - image: 658391232643.dkr.ecr.us-east-2.amazonaws.com/arif/splunk/splunk-ai-operator:v0.1.29 + image: 658391232643.dkr.ecr.us-east-2.amazonaws.com/kiran/splunk/splunk-ai-operator:openshift-0.2 livenessProbe: httpGet: path: /healthz diff --git a/tools/cluster_setup/openshift-cluster-config.yaml b/tools/cluster_setup/openshift-cluster-config.yaml index 2eb1c105..6eaab469 100644 --- a/tools/cluster_setup/openshift-cluster-config.yaml +++ b/tools/cluster_setup/openshift-cluster-config.yaml @@ -34,7 +34,7 @@ images: registry: "658391232643.dkr.ecr.us-east-2.amazonaws.com" operator: - image: "658391232643.dkr.ecr.us-east-2.amazonaws.com/kiran/splunk/splunk-ai-operator:openshift-0.1" + image: "658391232643.dkr.ecr.us-east-2.amazonaws.com/kiran/splunk/splunk-ai-operator:openshift-0.2" ray: headImage: "ml-platform/ray/ray-head:build-953" From b8ac604b8b5cc5416d5fe1ac493c4062f1b376bc Mon Sep 17 00:00:00 2001 From: kbhos Date: Tue, 16 Jun 2026 09:59:31 +0530 Subject: [PATCH 05/80] misc --- config/configs/applications.yaml | 66 ++++ config/configs/features/saia.yaml | 4 +- tools/cluster_setup/artifacts.yaml | 2 +- .../openshift-cluster-config.yaml | 4 +- tools/cluster_setup/openshift_with_stack.sh | 355 +++++++++++++++++- 5 files changed, 416 insertions(+), 15 deletions(-) diff --git a/config/configs/applications.yaml b/config/configs/applications.yaml index dbb0c5c6..eb29355d 100644 --- a/config/configs/applications.yaml +++ b/config/configs/applications.yaml @@ -53,6 +53,9 @@ applications: max_ongoing_requests: 8 ray_actor_options: num_gpus: 1 + resources: + "gpu_count:2": 0.001 + "accelerator_type:RTX_PRO_6000_BLACKWELL": 0.001 options: autoscaling_config: max_replicas: {{.Replicas.GptOss20b}} @@ -142,6 +145,12 @@ applications: L40S: ray_actor_options: num_gpus: 0.075 + RTX_PRO_6000_BLACKWELL: + ray_actor_options: + num_gpus: 0.031 + resources: + "gpu_count:2": 0.001 + "accelerator_type:RTX_PRO_6000_BLACKWELL": 0.001 options: autoscaling_config: max_replicas: {{.Replicas.UaeLarge}} @@ -157,6 +166,9 @@ applications: L40S: engine_args: gpu_memory_utilization: 0.075 + RTX_PRO_6000_BLACKWELL: + engine_args: + gpu_memory_utilization: 0.031 model_config: engine_args: gpu_memory_utilization: 0.15 @@ -208,6 +220,12 @@ applications: H100: ray_actor_options: num_gpus: 0.005 + RTX_PRO_6000_BLACKWELL: + ray_actor_options: + num_gpus: 0.004 + resources: + "gpu_count:2": 0.001 + "accelerator_type:RTX_PRO_6000_BLACKWELL": 0.001 options: autoscaling_config: max_replicas: {{.Replicas.AllMinilmL6V2}} @@ -220,6 +238,9 @@ applications: H100: engine_args: gpu_memory_utilization: 0.005 + RTX_PRO_6000_BLACKWELL: + engine_args: + gpu_memory_utilization: 0.004 model_config: engine_args: gpu_memory_utilization: 0.01 @@ -271,6 +292,12 @@ applications: H100: ray_actor_options: num_gpus: 0.005 + RTX_PRO_6000_BLACKWELL: + ray_actor_options: + num_gpus: 0.004 + resources: + "gpu_count:2": 0.001 + "accelerator_type:RTX_PRO_6000_BLACKWELL": 0.001 options: autoscaling_config: max_replicas: {{.Replicas.BiEncoder}} @@ -338,6 +365,12 @@ applications: L40S: ray_actor_options: num_gpus: 0.1 + RTX_PRO_6000_BLACKWELL: + ray_actor_options: + num_gpus: 0.05 + resources: + "gpu_count:2": 0.001 + "accelerator_type:RTX_PRO_6000_BLACKWELL": 0.001 options: autoscaling_config: max_replicas: {{.Replicas.MbartTranslator}} @@ -391,6 +424,12 @@ applications: L40S: ray_actor_options: num_gpus: 0.05 + RTX_PRO_6000_BLACKWELL: + ray_actor_options: + num_gpus: 0.021 + resources: + "gpu_count:2": 0.001 + "accelerator_type:RTX_PRO_6000_BLACKWELL": 0.001 options: autoscaling_config: max_replicas: {{.Replicas.XlmRobertaLanguageClassifier}} @@ -406,6 +445,9 @@ applications: L40S: engine_args: gpu_memory_utilization: 0.05 + RTX_PRO_6000_BLACKWELL: + engine_args: + gpu_memory_utilization: 0.021 model_config: engine_args: gpu_memory_utilization: 0.1 @@ -496,6 +538,12 @@ applications: H100: ray_actor_options: num_gpus: 0.005 + RTX_PRO_6000_BLACKWELL: + ray_actor_options: + num_gpus: 0.004 + resources: + "gpu_count:2": 0.001 + "accelerator_type:RTX_PRO_6000_BLACKWELL": 0.001 options: autoscaling_config: max_replicas: {{.Replicas.CrossEncoder}} @@ -508,6 +556,9 @@ applications: H100: engine_args: gpu_memory_utilization: 0.005 + RTX_PRO_6000_BLACKWELL: + engine_args: + gpu_memory_utilization: 0.004 model_config: engine_args: gpu_memory_utilization: 0.01 @@ -563,6 +614,12 @@ applications: L40S: ray_actor_options: num_gpus: 0.05 + RTX_PRO_6000_BLACKWELL: + ray_actor_options: + num_gpus: 0.021 + resources: + "gpu_count:2": 0.001 + "accelerator_type:RTX_PRO_6000_BLACKWELL": 0.001 options: autoscaling_config: max_replicas: {{.Replicas.E5LanguageClassifier}} @@ -578,6 +635,9 @@ applications: L40S: engine_args: gpu_memory_utilization: 0.05 + RTX_PRO_6000_BLACKWELL: + engine_args: + gpu_memory_utilization: 0.021 model_config: engine_args: gpu_memory_utilization: 0.1 @@ -632,6 +692,12 @@ applications: L40S: ray_actor_options: num_gpus: 0.025 + RTX_PRO_6000_BLACKWELL: + ray_actor_options: + num_gpus: 0.013 + resources: + "gpu_count:2": 0.001 + "accelerator_type:RTX_PRO_6000_BLACKWELL": 0.001 options: autoscaling_config: max_replicas: {{.Replicas.PromptInjectionCrossEncoder}} diff --git a/config/configs/features/saia.yaml b/config/configs/features/saia.yaml index aea4a115..69528f49 100644 --- a/config/configs/features/saia.yaml +++ b/config/configs/features/saia.yaml @@ -25,5 +25,5 @@ instanceScale: h100-nvl-1-gpu: 2 RTX_PRO_6000_BLACKWELL: rtx-pro-6000-blackwell-0-gpu: 1 - rtx-pro-6000-blackwell-1-gpu: 2 - rtx-pro-6000-blackwell-2-gpu: 0 \ No newline at end of file + rtx-pro-6000-blackwell-1-gpu: 0 + rtx-pro-6000-blackwell-2-gpu: 1 \ No newline at end of file diff --git a/tools/cluster_setup/artifacts.yaml b/tools/cluster_setup/artifacts.yaml index cb480386..48710094 100644 --- a/tools/cluster_setup/artifacts.yaml +++ b/tools/cluster_setup/artifacts.yaml @@ -5710,7 +5710,7 @@ spec: value: v0.3.14-36-g1549f5a - name: RAY_VERSION value: 2.53.0 - image: 658391232643.dkr.ecr.us-east-2.amazonaws.com/kiran/splunk/splunk-ai-operator:openshift-0.2 + image: 658391232643.dkr.ecr.us-east-2.amazonaws.com/kiran/splunk/splunk-ai-operator:openshift-0.5 livenessProbe: httpGet: path: /healthz diff --git a/tools/cluster_setup/openshift-cluster-config.yaml b/tools/cluster_setup/openshift-cluster-config.yaml index 6eaab469..072de5c6 100644 --- a/tools/cluster_setup/openshift-cluster-config.yaml +++ b/tools/cluster_setup/openshift-cluster-config.yaml @@ -34,7 +34,7 @@ images: registry: "658391232643.dkr.ecr.us-east-2.amazonaws.com" operator: - image: "658391232643.dkr.ecr.us-east-2.amazonaws.com/kiran/splunk/splunk-ai-operator:openshift-0.2" + image: "658391232643.dkr.ecr.us-east-2.amazonaws.com/kiran/splunk/splunk-ai-operator:openshift-0.5" ray: headImage: "ml-platform/ray/ray-head:build-953" @@ -65,7 +65,7 @@ storage: storageClass: "local-path" vectorDbSize: "50Gi" objectStore: - type: "minio" # aws | s3compat | minio | seaweedfs + type: "seaweedfs" # aws | s3compat | minio | seaweedfs bucket: "ai-platform-bucket" endpoint: "http://18.116.39.79:8333" auth: diff --git a/tools/cluster_setup/openshift_with_stack.sh b/tools/cluster_setup/openshift_with_stack.sh index ced27236..5533bb0c 100755 --- a/tools/cluster_setup/openshift_with_stack.sh +++ b/tools/cluster_setup/openshift_with_stack.sh @@ -28,11 +28,182 @@ LOG_FILE="${LOG_DIR}/openshift-install-$(date '+%Y-%m-%d_%H-%M-%S').log" exec > >(tee -a "${LOG_FILE}") 2>&1 echo "[LOG] Session log: ${LOG_FILE}" +# ====== LOG ROTATION (keep last 10 logs) ====== +_rotate_logs() { + local keep=10 + local logs=() + while IFS= read -r f; do logs+=("$f"); done < <(ls -1t "${LOG_DIR}"/openshift-install-*.log 2>/dev/null) + local excess=$(( ${#logs[@]} - keep )) + if (( excess > 0 )); then + for (( i=${#logs[@]}-1; i>=${#logs[@]}-excess; i-- )); do + rm -f "${logs[$i]}" + done + fi +} +_rotate_logs + # ====== COLORS & LOGGING ====== -log() { echo -e "\033[1;36m[INFO]\033[0m $*" >&2; } -warn() { echo -e "\033[1;33m[WARN]\033[0m $*" >&2; } -err() { echo -e "\033[1;31m[ERROR]\033[0m $*" >&2; exit 1; } -need() { command -v "$1" >/dev/null 2>&1 || err "Missing $1 in PATH"; } +_ts() { date '+%Y-%m-%d %H:%M:%S'; } +log() { echo -e "\033[1;36m[$(_ts) INFO]\033[0m $*" >&2; } +warn() { echo -e "\033[1;33m[$(_ts) WARN]\033[0m $*" >&2; } +err() { + echo -e "\033[1;31m[$(_ts) ERROR]\033[0m $*" >&2 + echo -e "\033[1;31m[$(_ts) ERROR]\033[0m Log file: ${LOG_FILE}" >&2 + echo -e "\033[1;31m[$(_ts) ERROR]\033[0m Run '$0 diagnose' to collect a full support bundle." >&2 + exit 1 +} + +# ====== TOOL CHECKER ====== +need() { + command -v "$1" >/dev/null 2>&1 && return 0 + local install_hint="" + case "$1" in + oc) install_hint="https://docs.openshift.com/container-platform/latest/cli_reference/openshift_cli/getting-started-cli.html" ;; + helm) install_hint="brew install helm OR https://helm.sh/docs/intro/install/" ;; + yq) install_hint="brew install yq OR wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O /usr/local/bin/yq && chmod +x /usr/local/bin/yq" ;; + jq) install_hint="brew install jq OR apt-get install jq OR dnf install jq" ;; + curl) install_hint="apt-get install curl OR brew install curl" ;; + aws) install_hint="https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html" ;; + git) install_hint="brew install git OR apt-get install git" ;; + *) install_hint="install '$1' via your system package manager" ;; + esac + err "Required tool not found: $1 + Install: ${install_hint}" +} + +# ====== STEP PROGRESS TRACKER ====== +declare -a _STEP_NAMES=() +declare -a _STEP_STATUS=() +_STEP_CURRENT="" + +step_start() { + _STEP_CURRENT="$1" + _STEP_NAMES+=("$1") + _STEP_STATUS+=("running") + local n=${#_STEP_NAMES[@]} + echo -e "\n\033[1;34m[$(_ts) ── STEP ${n}: $1 ──]\033[0m" >&2 +} + +step_ok() { + local last=$(( ${#_STEP_STATUS[@]} - 1 )) + _STEP_STATUS[$last]="ok" +} + +step_fail() { + local last=$(( ${#_STEP_STATUS[@]} - 1 )) + _STEP_STATUS[$last]="fail:${1:-unknown error}" +} + +step_skip() { + local last=$(( ${#_STEP_STATUS[@]} - 1 )) + _STEP_STATUS[$last]="skip:${1:-}" +} + +show_step_summary() { + echo -e "\n\033[1;34m[$(_ts) ════ INSTALL SUMMARY ════]\033[0m" >&2 + local total=${#_STEP_NAMES[@]} ok=0 fail=0 skip=0 + for i in "${!_STEP_NAMES[@]}"; do + local s="${_STEP_STATUS[$i]}" + local icon color label + case "${s%%:*}" in + ok) icon="✔"; color="\033[1;32m"; label="OK"; ok=$((ok+1)) ;; + fail) icon="✖"; color="\033[1;31m"; label="${s#fail:}"; fail=$((fail+1)) ;; + skip) icon="–"; color="\033[1;33m"; label="${s#skip:}"; skip=$((skip+1)) ;; + running) icon="?"; color="\033[1;33m"; label="interrupted"; fail=$((fail+1)) ;; + *) icon="?"; color="\033[0m"; label="${s}" ;; + esac + printf " ${color}${icon}\033[0m %-45s %s\n" "${_STEP_NAMES[$i]}" "${label}" >&2 + done + echo "" >&2 + if (( fail == 0 )); then + echo -e " \033[1;32mAll ${total} steps completed successfully.\033[0m" >&2 + else + echo -e " \033[1;31m${fail} step(s) failed, ${ok} succeeded, ${skip} skipped.\033[0m" >&2 + echo -e " \033[1;31mSee log: ${LOG_FILE}\033[0m" >&2 + fi + echo "" >&2 +} + +# ====== PHASE SECTION MARKERS ====== +phase_start() { echo -e "\n\033[1;35m[$(_ts) ════════ PHASE: $* ════════]\033[0m" >&2; } +phase_end() { echo -e "\033[1;35m[$(_ts) ════════ END: $* ════════]\033[0m\n" >&2; } + +# ====== WAIT FOR DEPENDENCY (interactive pause-and-retry) ====== +wait_for_dependency() { + local description="$1" + local check_cmd="$2" + local max_wait="${3:-600}" + local elapsed=0 interval=30 + + log "Waiting for external dependency: ${description}" + log " Max wait: ${max_wait}s. Press Enter at any time to retry immediately." + + while (( elapsed < max_wait )); do + if eval "${check_cmd}" >/dev/null 2>&1; then + log " ✔ ${description} — ready" + return 0 + fi + local remaining=$(( max_wait - elapsed )) + warn " ${description} not ready yet. Retrying in ${interval}s (${remaining}s remaining)." + warn " Press Enter to retry now, or wait..." + if read -t "${interval}" -r 2>/dev/null; then + log " Retrying immediately..." + fi + elapsed=$(( elapsed + interval )) + done + + err "Timed out after ${max_wait}s waiting for: ${description} + Resolve the issue, then re-run the installer." +} + +# ====== SHOW INSTALL PLAN ====== +show_install_plan() { + echo -e "\n\033[1;34m╔══════════════════════════════════════════════════════════╗\033[0m" >&2 + echo -e "\033[1;34m║ SPLUNK AI PLATFORM — OPENSHIFT INSTALL PLAN ║\033[0m" >&2 + echo -e "\033[1;34m╚══════════════════════════════════════════════════════════╝\033[0m" >&2 + echo "" >&2 + echo -e " \033[1mNamespace :\033[0m ${AI_NS}" >&2 + echo -e " \033[1mConfig file :\033[0m ${CONFIG_FILE}" >&2 + echo -e " \033[1mLog file :\033[0m ${LOG_FILE}" >&2 + echo "" >&2 + echo -e " \033[1mAccelerator type :\033[0m ${DEFAULT_ACCELERATOR:-}" >&2 + echo -e " \033[1mNode label strat :\033[0m ${NODE_LABEL_STRATEGY}" >&2 + echo -e " \033[1mOperator image :\033[0m ${OPERATOR_IMAGE}" >&2 + echo -e " \033[1mImage registry :\033[0m ${IMAGE_REGISTRY:-}" >&2 + echo -e " \033[1mECR enabled :\033[0m ${ECR_ENABLED}" >&2 + echo "" >&2 + echo -e " \033[1mObject store :\033[0m type=${OBJ_STORE_TYPE} bucket=${OBJ_STORE_BUCKET:-}" >&2 + echo -e " \033[1mObject endpoint :\033[0m ${OBJ_STORE_ENDPOINT:-}" >&2 + echo "" >&2 + echo -e " \033[1mSteps that will run:\033[0m" >&2 + echo -e " 1. Preflight checks (oc login, tools, manifest files)" >&2 + echo -e " 2. NFD Operator (OLM)" >&2 + echo -e " 3. NVIDIA GPU Operator (OLM)" >&2 + echo -e " 4. Node labeling (splunk.ai/workload-type)" >&2 + echo -e " 5. local-path-provisioner + SELinux relabeling" >&2 + echo -e " 6. cert-manager (Helm)" >&2 + echo -e " 7. OpenTelemetry Operator (Helm)" >&2 + echo -e " 8. KubeRay Operator (Helm)" >&2 + echo -e " 9. ECR pull secrets" >&2 + echo -e " 10. Splunk AI Operator" >&2 + echo -e " 11. Splunk Operator" >&2 + echo -e " 12. Splunk Standalone CR" >&2 + echo -e " 13. AIPlatform CR" >&2 + echo "" >&2 + + if [[ "${AUTO_APPROVE:-false}" == "true" ]]; then + log "AUTO_APPROVE=true — skipping confirmation." + return 0 + fi + + echo -e " \033[1mReview the plan above. Type 'yes' to proceed, anything else to abort:\033[0m" >&2 + local answer + read -r answer + if [[ "${answer}" != "yes" ]]; then + echo "Aborted by user." >&2 + exit 0 + fi +} # ====== LOAD CONFIGURATION ====== load_config() { @@ -1001,6 +1172,14 @@ install_splunk_standalone() { ensure_namespace "${AI_NS}" wait_for_crd standalones.enterprise.splunk.com 600 + # Wait for object store endpoint to be reachable before creating credentials secret + if [[ -n "${OBJ_STORE_ENDPOINT}" ]]; then + wait_for_dependency \ + "object store (${OBJ_STORE_TYPE}) at ${OBJ_STORE_ENDPOINT}" \ + "curl -sL --connect-timeout 5 --max-time 10 -o /dev/null -w '%{http_code}' '${OBJ_STORE_ENDPOINT}' 2>/dev/null | grep -qE '^[0-9]'" \ + 300 + fi + # Object storage credentials secret oc -n "${AI_NS}" create secret generic minio-credentials \ --from-literal=AWS_ACCESS_KEY_ID="${MINIO_ROOT_USER}" \ @@ -1073,7 +1252,7 @@ install_ai_platform_cr() { aws) obj_path="s3://${OBJ_STORE_BUCKET}"; obj_endpoint="" ;; s3compat) obj_path="s3compat://${OBJ_STORE_BUCKET}"; obj_endpoint="${OBJ_STORE_ENDPOINT}" ;; minio) obj_path="minio://${OBJ_STORE_BUCKET}"; obj_endpoint="${OBJ_STORE_ENDPOINT}" ;; - seaweedfs) obj_path="seaweedfs://${OBJ_STORE_BUCKET}";obj_endpoint="${OBJ_STORE_ENDPOINT}" ;; + seaweedfs) obj_path="minio://${OBJ_STORE_BUCKET}"; obj_endpoint="${OBJ_STORE_ENDPOINT}" ;; *) err "Unsupported objectStore.type: ${OBJ_STORE_TYPE}" ;; esac @@ -1230,22 +1409,73 @@ main_install() { log "============================================" load_config - preflight_checks validate_image_config configure_images + + show_install_plan + + phase_start "Preflight" + step_start "Preflight checks" + preflight_checks + step_ok + phase_end "Preflight" + + phase_start "Infrastructure" + step_start "NFD Operator" install_nfd + step_ok + + step_start "NVIDIA GPU Operator" install_nvidia_gpu_operator + step_ok + + step_start "Node labeling" label_nodes + step_ok + + step_start "local-path-provisioner + SELinux" install_local_path_provisioner relabel_worker_nodes_for_selinux + step_ok + phase_end "Infrastructure" + + phase_start "Operators" + step_start "cert-manager" install_cert_manager + step_ok + + step_start "OpenTelemetry Operator" install_otel_operator + step_ok + + step_start "KubeRay Operator" install_ray_operator + step_ok + + step_start "ECR pull secrets" ensure_ecr_pull_secret + step_ok + + step_start "Splunk AI Operator" install_splunk_ai_operator + step_ok + + step_start "Splunk Operator" install_splunk_operator + step_ok + phase_end "Operators" + + phase_start "AI Platform Stack" + step_start "Splunk Standalone CR" install_splunk_standalone + step_ok + + step_start "AIPlatform CR" install_ai_platform_cr + step_ok + phase_end "AI Platform Stack" + + show_step_summary log "============================================" log " Install complete" @@ -1273,6 +1503,28 @@ main_delete() { err "Not logged in to OpenShift. Run: oc login " fi + log " Namespace : ${AI_NS}" + log " Cluster : $(oc whoami --show-server 2>/dev/null || echo '')" + log "============================================" + log "" + warn "This will DELETE the AI Platform stack from the OpenShift cluster." + warn "The cluster nodes themselves will remain running." + warn "This action CANNOT be undone." + log "" + + if [[ "${AUTO_APPROVE:-false}" != "true" ]]; then + echo -e " \033[1;31mType 'yes' to confirm deletion, or Ctrl-C to abort:\033[0m" >&2 + local confirm_input + read -r confirm_input + if [[ "${confirm_input}" != "yes" ]]; then + echo "Aborted — confirmation not given." >&2 + exit 0 + fi + log "Confirmed. Proceeding with deletion..." + else + log "AUTO_APPROVE=true — skipping confirmation prompt." + fi + local ai_operator_ns="splunk-ai-operator-system" local splunk_operator_ns="splunk-operator" @@ -1384,20 +1636,100 @@ main_delete() { log "Log file: ${LOG_FILE}" } +# ====== DIAGNOSE SUBCOMMAND ====== +diagnose() { + load_config 2>/dev/null || true + + local bundle_dir + bundle_dir="$(mktemp -d)/splunk-ai-diagnose-$(date '+%Y%m%d-%H%M%S')" + mkdir -p "${bundle_dir}" + + log "=== Collecting support bundle into ${bundle_dir} ===" + + # 1. Installer logs + log "Collecting installer logs..." + cp "${LOG_DIR}"/openshift-install-*.log "${bundle_dir}/" 2>/dev/null || true + + # 2. Cluster state (best-effort — cluster may be unreachable) + if timeout 10 oc cluster-info &>/dev/null 2>&1; then + log "Collecting cluster state..." + oc get nodes -o wide > "${bundle_dir}/nodes.txt" 2>&1 || true + oc get pods --all-namespaces -o wide > "${bundle_dir}/pods.txt" 2>&1 || true + oc get events --all-namespaces --sort-by='.lastTimestamp' > "${bundle_dir}/events.txt" 2>&1 || true + oc get pvc --all-namespaces > "${bundle_dir}/pvcs.txt" 2>&1 || true + oc get svc --all-namespaces > "${bundle_dir}/services.txt" 2>&1 || true + oc describe nodes > "${bundle_dir}/node-details.txt" 2>&1 || true + + # Per-namespace pod logs for failing pods + log "Collecting logs from non-Running pods..." + local ns pod + while IFS= read -r line; do + ns=$(echo "${line}" | awk '{print $1}') + pod=$(echo "${line}" | awk '{print $2}') + mkdir -p "${bundle_dir}/pod-logs/${ns}" + oc logs "${pod}" -n "${ns}" --tail=200 \ + > "${bundle_dir}/pod-logs/${ns}/${pod}.log" 2>&1 || true + oc logs "${pod}" -n "${ns}" --previous --tail=100 \ + > "${bundle_dir}/pod-logs/${ns}/${pod}.previous.log" 2>&1 || true + done < <(oc get pods --all-namespaces --no-headers 2>/dev/null \ + | awk '$4 != "Running" && $4 != "Completed" {print $1, $2}') + + # AI Platform specific resources + oc describe aiplatform --all -n "${AI_NS:-ai-platform}" > "${bundle_dir}/aiplatform-cr.txt" 2>&1 || true + oc describe aiservice --all -n "${AI_NS:-ai-platform}" > "${bundle_dir}/aiservice-cr.txt" 2>&1 || true + + # Operator logs + oc logs -n splunk-ai-operator-system -l control-plane=controller-manager --tail=500 \ + > "${bundle_dir}/operator-logs.txt" 2>&1 || true + else + warn "Cluster not reachable — skipping oc diagnostics." + echo "Cluster unreachable at time of diagnose run." > "${bundle_dir}/CLUSTER_UNREACHABLE.txt" + fi + + # 3. Config file (redact credentials) + if [[ -f "${CONFIG_FILE}" ]]; then + log "Including config file (credentials redacted)..." + sed 's/\(rootUser\|rootPassword\|AWS_ACCESS_KEY_ID\|AWS_SECRET_ACCESS_KEY\|accessKey\|secretKey\):.*/\1: /g' \ + "${CONFIG_FILE}" > "${bundle_dir}/cluster-config-redacted.yaml" + fi + + # 4. Tool versions + { + echo "=== Tool versions ===" + oc version 2>/dev/null || true + helm version 2>/dev/null || true + yq --version 2>/dev/null || true + echo "=== OS ===" + uname -a + } > "${bundle_dir}/versions.txt" 2>&1 + + # 5. Pack into tar.gz + local bundle_tar="${bundle_dir}.tar.gz" + tar -czf "${bundle_tar}" -C "$(dirname "${bundle_dir}")" "$(basename "${bundle_dir}")" 2>/dev/null + rm -rf "${bundle_dir}" + + log "=== Support bundle ready: ${bundle_tar} ===" + log "Attach this file to your support ticket or share with the team." +} + # ====== USAGE ====== usage() { cat < - - oc, yq in PATH + - oc, yq, helm in PATH - artifacts.yaml (operator manifests) in the same directory, or set files.aiPlatform in config EOF } @@ -1410,6 +1742,9 @@ case "${1:-install}" in delete) main_delete ;; + diagnose) + diagnose + ;; *) usage exit 1 From 4f6c8207a474de9d4190d061b3b875ac93dd2e9b Mon Sep 17 00:00:00 2001 From: kbhos Date: Tue, 16 Jun 2026 12:41:09 +0530 Subject: [PATCH 06/80] add back gemma model --- config/configs/applications.yaml | 128 ++++++++++++++++++++++- config/configs/features/saia.yaml | 1 + pkg/ai/raybuilder/configmap_apps_test.go | 6 +- 3 files changed, 129 insertions(+), 6 deletions(-) diff --git a/config/configs/applications.yaml b/config/configs/applications.yaml index eb29355d..72745a90 100644 --- a/config/configs/applications.yaml +++ b/config/configs/applications.yaml @@ -34,6 +34,131 @@ applications: SERVICE_NAME: "ai_platform_models" SKIP_VERIFICATION: "true" USE_SYSTEM_PERMISSIONS: "true" + - args: + application_name: Gemma431bIt + deployment_configs: + LLMDeployment: + gpu_type_options_override: + H100: + autoscaling_config: + max_replicas: {{.Replicas.Gemma431bIt}} + min_replicas: {{.Replicas.Gemma431bIt}} + target_ongoing_requests: 6 + max_ongoing_requests: 8 + ray_actor_options: + num_gpus: 1 + L40S: + autoscaling_config: + max_replicas: {{.Replicas.Gemma431bIt}} + min_replicas: {{.Replicas.Gemma431bIt}} + target_ongoing_requests: 4 + max_ongoing_requests: 6 + ray_actor_options: + num_gpus: 2 + RTX_PRO_6000_BLACKWELL: + autoscaling_config: + max_replicas: {{.Replicas.Gemma431bIt}} + min_replicas: {{.Replicas.Gemma431bIt}} + target_ongoing_requests: 4 + max_ongoing_requests: 10 + ray_actor_options: + num_gpus: 2 + resources: + "gpu_count:2": 0.001 + "accelerator_type:RTX_PRO_6000_BLACKWELL": 0.001 + options: + autoscaling_config: + max_replicas: {{.Replicas.Gemma431bIt}} + min_replicas: {{.Replicas.Gemma431bIt}} + deployment_type: text_gen_model_deployment + gpu_types: '["{{.AcceleratorType}}"]' + model_definition: + gpu_type_model_config_override: + H100: + engine_args: + dtype: bfloat16 + gpu_memory_utilization: 0.9 + max_model_len: 32768 + max_num_batched_tokens: 4096 + tensor_parallel_size: 1 + L40S: + engine_args: + dtype: bfloat16 + gpu_memory_utilization: 0.85 + max_model_len: 120000 + max_num_batched_tokens: 4096 + max_num_seqs: 2 + tensor_parallel_size: 2 + RTX_PRO_6000_BLACKWELL: + engine_args: + dtype: bfloat16 + gpu_memory_utilization: 0.85 + max_model_len: 240000 + max_num_batched_tokens: 4096 + max_num_seqs: 1 + tensor_parallel_size: 2 + model_config: + openai_serving_config: + chat: + enable_auto_tools: true + reasoning_parser: gemma4 + tool_parser: gemma4 + responses: + enable_auto_tools: true + reasoning_parser: gemma4 + tool_parser: gemma4 + model_id: gemma4_31b_it + model_loader: + blob_storage: + blob_prefix: model_artifacts/gemma-4-31b-it + tokenizer_definition: + model_id: gemma4_31b_it + model_loader: + blob_storage: + artifacts_list: + - chat_template.jinja + - config.json + - processor_config.json + - tokenizer_config.json + - tokenizer.json + blob_prefix: model_artifacts/gemma-4-31b-it + name: Gemma431bIt + import_path: main:create_serve_app + route_prefix: /gemma4_31b_it + runtime_env: + working_dir: "file:///home/ray/ray/applications/generic_application.zip" + env_vars: + API_VERSION: "v1" + APPLICATION_NAME: gemma4_31b_it + VLLM_ATTENTION_BACKEND: TRITON_ATTN + ARTIFACTS_S3_BUCKET: "{{.ArtifactBucketName}}" + S3_BUCKET: "{{.ArtifactBucketName}}" + ARTIFACTS_PROVIDER: "{{.ArtifactsProvider}}" + CLOUD_PROVIDER: "{{.CloudProvider}}" + S3COMPAT_OBJECT_STORE_ENDPOINT_URL: "{{.S3CompatObjectStoreEndpointUrl}}" + S3COMPAT_OBJECT_STORE_ACCESS_KEY: "{{.S3CompatObjectStoreAccessKey}}" + S3COMPAT_OBJECT_STORE_SECRET_KEY: "{{.S3CompatObjectStoreSecretKey}}" + # AWS / boto3 standard credential names — populated whenever the + # operator can load credentials from spec.objectStorage.secretRef. For + # CLOUD_PROVIDER=aws these are the values boto3 reads (the S3COMPAT_* + # names above are only consumed by the s3compat shim). Both code paths + # share the same source-of-truth Secret keys (s3_access_key / + # s3_secret_key) so emitting both pairs is safe — each provider only + # reads its own. AWS_REGION lets boto3 resolve the default regional S3 + # bucket outside us-east-1 to avoid PermanentRedirect on the first call. + AWS_ACCESS_KEY_ID: "{{.S3CompatObjectStoreAccessKey}}" + AWS_SECRET_ACCESS_KEY: "{{.S3CompatObjectStoreSecretKey}}" + AWS_REGION: "{{.Region}}" + AWS_DEFAULT_REGION: "{{.Region}}" + ENABLE_AUTHN: "false" + ENABLE_AUTHZ: "false" + SERVICE_EXTERNAL_NAME: "ai-platform-models" + SERVICE_INTERNAL_NAME: "ai_platform_models" + SERVICE_NAME: "ai_platform_models" + SKIP_VERIFICATION: "true" + USE_SYSTEM_PERMISSIONS: "true" + VLLM_WORKER_MULTIPROC_METHOD: spawn + DISABLE_RESPONSES_API_REDIS: "True" - args: application_name: GptOss20b deployment_configs: @@ -131,8 +256,7 @@ applications: SKIP_VERIFICATION: "true" USE_SYSTEM_PERMISSIONS: "true" VLLM_WORKER_MULTIPROC_METHOD: spawn - # See Gemma431bIt above for rationale. Must be "True" in airgap (no - # Redis) so vLLM uses NoOpOpenAIServingResponses. + # Must be "True" in airgap (no Redis) so vLLM uses NoOpOpenAIServingResponses. DISABLE_RESPONSES_API_REDIS: "True" - args: application_name: UaeLarge diff --git a/config/configs/features/saia.yaml b/config/configs/features/saia.yaml index 69528f49..73df509d 100644 --- a/config/configs/features/saia.yaml +++ b/config/configs/features/saia.yaml @@ -4,6 +4,7 @@ applicationScale: CrossEncoder: 1 E5LanguageClassifier: 1 Entrypoint: 1 + Gemma431bIt: 1 GptOss20b: 1 MbartTranslator: 1 PromptInjectionClassifier: 1 diff --git a/pkg/ai/raybuilder/configmap_apps_test.go b/pkg/ai/raybuilder/configmap_apps_test.go index fbdad4ef..4beb1b4e 100644 --- a/pkg/ai/raybuilder/configmap_apps_test.go +++ b/pkg/ai/raybuilder/configmap_apps_test.go @@ -86,11 +86,9 @@ func Test_ApplicationsYAML_DisableResponsesRedis(t *testing.T) { } } - expectedTextGenApps := []string{"GptOss20b"} + expectedTextGenApps := []string{"Gemma431bIt", "GptOss20b"} - // We expect exactly one text-gen app today (GptOss20b). Gemma431bIt was - // removed — it requires 2× RTX PRO 6000 GPUs and cannot run alongside - // GptOss20b on a single 2-GPU node. + // We expect exactly two text-gen apps today (Gemma431bIt, GptOss20b). // If this count changes, someone added a new text-gen model; they MUST // also add DISABLE_RESPONSES_API_REDIS to the new app. require.Len(t, textGenApps, len(expectedTextGenApps), From 5b330f95122635c7d019a07c6302b46da62f18e1 Mon Sep 17 00:00:00 2001 From: kbhos Date: Tue, 16 Jun 2026 12:49:18 +0530 Subject: [PATCH 07/80] clean-ups --- config/configs/applications.yaml | 14 ++++++- tools/cluster_setup/artifacts.yaml | 67 ++++++++++++++---------------- 2 files changed, 44 insertions(+), 37 deletions(-) diff --git a/config/configs/applications.yaml b/config/configs/applications.yaml index 72745a90..c8a38601 100644 --- a/config/configs/applications.yaml +++ b/config/configs/applications.yaml @@ -145,6 +145,7 @@ applications: # share the same source-of-truth Secret keys (s3_access_key / # s3_secret_key) so emitting both pairs is safe — each provider only # reads its own. AWS_REGION lets boto3 resolve the default regional S3 + # endpoint when no AWS_ENDPOINT_URL is set; required for any AWS S3 # bucket outside us-east-1 to avoid PermanentRedirect on the first call. AWS_ACCESS_KEY_ID: "{{.S3CompatObjectStoreAccessKey}}" AWS_SECRET_ACCESS_KEY: "{{.S3CompatObjectStoreSecretKey}}" @@ -158,6 +159,16 @@ applications: SKIP_VERIFICATION: "true" USE_SYSTEM_PERMISSIONS: "true" VLLM_WORKER_MULTIPROC_METHOD: spawn + # Disable the Redis-backed Responses API store (see ai-platform-models + # commit c1f9aef3: "feat: add a no-op store"). When True, the vLLM + # TextGen deployment constructs NoOpOpenAIServingResponses instead of + # RedisOpenAIServingResponses, so /v1/responses works without a Redis + # infra. Without this flag the deployment raises + # RuntimeError: Responses Redis URL not set + # on every request, which surfaces as an empty SSE stream and the SAIA + # v2 /query path fails with "An error occurred processing your request". + # Airgap k0s has no Redis; cloud sets this to "False" and wires + # RESPONSES_REDIS_ADDRESS to its in-namespace Redis StatefulSet. DISABLE_RESPONSES_API_REDIS: "True" - args: application_name: GptOss20b @@ -256,7 +267,8 @@ applications: SKIP_VERIFICATION: "true" USE_SYSTEM_PERMISSIONS: "true" VLLM_WORKER_MULTIPROC_METHOD: spawn - # Must be "True" in airgap (no Redis) so vLLM uses NoOpOpenAIServingResponses. + # See Gemma431bIt above for rationale. Must be "True" in airgap (no + # Redis) so vLLM uses NoOpOpenAIServingResponses. DISABLE_RESPONSES_API_REDIS: "True" - args: application_name: UaeLarge diff --git a/tools/cluster_setup/artifacts.yaml b/tools/cluster_setup/artifacts.yaml index 48710094..c6953e7b 100644 --- a/tools/cluster_setup/artifacts.yaml +++ b/tools/cluster_setup/artifacts.yaml @@ -1061,18 +1061,11 @@ spec: items: description: FeatureSpec defines the features to enable in the AIPlatform properties: - env: - additionalProperties: - type: string - description: Env specifies environment variables to propagate - to the child AIService. - type: object name: description: Name of the feature, e.g. "saia" or "seca" enum: - saia - seca - - weaviate-service type: string scaleFactor: description: ScaleFactor is the desired fixed number of replicas @@ -2092,11 +2085,6 @@ spec: type: object x-kubernetes-map-type: atomic type: array - otelImage: - default: otel/opentelemetry-collector-contrib:0.122.1 - description: OTelImage is the OpenTelemetry Collector sidecar - image - type: string rayHeadGroupImage: description: Ray head group image, e.g. "rayproject/ray-head:latest" type: string @@ -2237,8 +2225,7 @@ spec: type: object objectStorage: description: |- - ObjectStorage defines the object storage configuration for AI artifacts, tasks, and models. - It is optional for platforms that only enable features that do not require object storage. + ObjectStorage defines the object storage configuration for AI artifacts, tasks, and models Supported providers: S3, GCS, Azure Blob Storage, MinIO properties: endpoint: @@ -2250,8 +2237,8 @@ spec: path: description: |- Remote volume URI in the format s3://bucketname/, gs://bucketname/, - azure://containername/, or minio://bucketname/ - pattern: ^(s3|gs|azure|minio)://[a-zA-Z0-9.\-_]+(/.*)?$ + azure://containername/, minio://bucketname/, seaweedfs://bucketname/, or s3compat://bucketname/ + pattern: ^(s3|gs|azure|minio|seaweedfs|s3compat)://[a-zA-Z0-9.\-_]+(/.*)?$ type: string region: description: Region of the remote storage volume. Required for @@ -2921,6 +2908,8 @@ spec: pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ type: string type: object + required: + - objectStorage type: object status: description: AIPlatformStatus defines observed state @@ -4095,18 +4084,11 @@ spec: features: description: Feature defines the features to be enabled for the AIService properties: - env: - additionalProperties: - type: string - description: Env specifies environment variables to propagate - to the child AIService. - type: object name: description: Name of the feature, e.g. "saia" or "seca" enum: - saia - seca - - weaviate-service type: string scaleFactor: description: ScaleFactor is the desired fixed number of replicas @@ -4884,15 +4866,27 @@ spec: properties: endpoint: description: |- - Optional override endpoint (only needed for S3-compatible services like MinIO) - Must be a valid HTTP/HTTPS URL + Optional override endpoint (only needed for S3-compatible services like MinIO, SeaweedFS) + Must be a valid HTTP/HTTPS URL. When set with s3:// path, backend is treated as S3-compatible (MinIO, SeaweedFS, etc.) pattern: ^https?://.*$ type: string path: description: |- Remote volume URI in the format s3://bucketname/, gs://bucketname/, - azure://containername/, or minio://bucketname/ - pattern: ^(s3|gs|azure|minio)://[a-zA-Z0-9.\-_]+(/.*)?$ + azure://containername/, s3compat://bucketname/ (generic S3-compatible), minio://, or seaweedfs:// + pattern: ^(s3|gs|azure|minio|seaweedfs|s3compat)://[a-zA-Z0-9.\-_]+(/.*)?$ + type: string + provider: + description: |- + Provider is an optional hint for documentation and tooling. Operator derives behavior from path scheme and endpoint. + Values: aws, minio, seaweedfs, s3compat, gcs, azure + enum: + - aws + - minio + - seaweedfs + - s3compat + - gcs + - azure type: string region: description: Region of the remote storage volume. Required for @@ -4900,7 +4894,8 @@ spec: minLength: 1 type: string secretRef: - description: Secret name containing storage credentials + description: Secret name containing storage credentials (e.g. + s3_access_key, s3_secret_key for S3-compatible backends) maxLength: 253 minLength: 1 type: string @@ -5687,19 +5682,19 @@ spec: fieldRef: fieldPath: metadata.name - name: RELATED_IMAGE_RAY_HEAD - value: 658391232643.dkr.ecr.us-east-2.amazonaws.com/ml-platform/ray/ray-head:build-953 + value: 658391232643.dkr.ecr.us-east-2.amazonaws.com/ml-platform/ray/ray-head:build-v2-010 - name: RELATED_IMAGE_RAY_WORKER - value: 658391232643.dkr.ecr.us-east-2.amazonaws.com/ml-platform/ray/ray-worker-gpu:build-953 + value: 658391232643.dkr.ecr.us-east-2.amazonaws.com/ml-platform/ray/ray-worker-gpu:build-v2-010 - name: RELATED_IMAGE_WEAVIATE value: docker.io/semitechnologies/weaviate:stable-v1.28-007846a - - name: RELATED_IMAGE_WEAVIATE_SERVICE - value: docker.io/semitechnologies/weaviate:stable-v1.28-007846a - name: RELATED_IMAGE_SAIA_API - value: 658391232643.dkr.ecr.us-east-2.amazonaws.com/ml-platform/saia/saia-api:build-v2-main-c3b489d + value: 658391232643.dkr.ecr.us-east-2.amazonaws.com/ml-platform/saia/saia-api:build-v2-012 - name: RELATED_IMAGE_SAIA_API_V2 - value: 658391232643.dkr.ecr.us-east-2.amazonaws.com/ml-platform/saia/saia-api-v2:build-v2-main-c3b489d + value: 658391232643.dkr.ecr.us-east-2.amazonaws.com/ml-platform/saia/saia-api-v2:build-v2-012 - name: RELATED_IMAGE_POST_INSTALL_HOOK - value: 658391232643.dkr.ecr.us-east-2.amazonaws.com/ml-platform/saia/saia-data-loader:build-v2-main-c3b489d + value: 658391232643.dkr.ecr.us-east-2.amazonaws.com/ml-platform/saia/saia-data-loader:build-v2-012 + - name: SPLUNK_METRICS_INDEX_NAME + value: _metrics - name: RELATED_IMAGE_FLUENT_BIT value: docker.io/fluent/fluent-bit:1.9.6 - name: RELATED_IMAGE_OTEL_COLLECTOR @@ -5710,7 +5705,7 @@ spec: value: v0.3.14-36-g1549f5a - name: RAY_VERSION value: 2.53.0 - image: 658391232643.dkr.ecr.us-east-2.amazonaws.com/kiran/splunk/splunk-ai-operator:openshift-0.5 + image: 658391232643.dkr.ecr.us-east-2.amazonaws.com/arif/splunk/splunk-ai-operator:v0.1.29 livenessProbe: httpGet: path: /healthz From 2a78e36bad21a404b8ec8072c51f9b4af33d6a78 Mon Sep 17 00:00:00 2001 From: kbhos Date: Tue, 16 Jun 2026 14:27:06 +0530 Subject: [PATCH 08/80] config map to store the issuer url --- tools/cluster_setup/openshift_with_stack.sh | 29 +++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/tools/cluster_setup/openshift_with_stack.sh b/tools/cluster_setup/openshift_with_stack.sh index 5533bb0c..53f79650 100755 --- a/tools/cluster_setup/openshift_with_stack.sh +++ b/tools/cluster_setup/openshift_with_stack.sh @@ -1198,6 +1198,30 @@ install_splunk_standalone() { fi [[ -z "${minio_endpoint}" ]] && err "storage.objectStore.endpoint must be set for type=${OBJ_STORE_TYPE}" + # Configure Splunk to use the service URL as the token issuer so that JWT + # tokens have iss=https://splunk-splunk-standalone-standalone-service:8089, + # matching SAIA's SPLUNK_ISSUERS. Without this, Splunk uses the pod hostname + # as issuer (e.g. splunk-splunk-standalone-standalone-0) and SAIA rejects + # tokens with "Issuer not allowed". + cat <<'YAML' | oc -n "${AI_NS}" apply -f - +apiVersion: v1 +kind: ConfigMap +metadata: + name: splunk-defaults +data: + default.yml: | + splunk: + conf: + - key: authentication + value: + directory: /opt/splunk/etc/system/local + content: + oauth2_settings: + issuer_uri: https://splunk-splunk-standalone-standalone-service:8089 + certFile: $SPLUNK_HOME/etc/auth/server.pem + sslPassword: password +YAML + oc apply --server-side --force-conflicts -f - < Date: Tue, 16 Jun 2026 21:44:41 +0530 Subject: [PATCH 09/80] resolve copilot comments --- config/configs/instance.yaml | 2 ++ .../openshift-cluster-config.yaml | 6 ++--- tools/cluster_setup/openshift_with_stack.sh | 22 ++++++++++++------- 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/config/configs/instance.yaml b/config/configs/instance.yaml index 3183dbf9..bf012a00 100644 --- a/config/configs/instance.yaml +++ b/config/configs/instance.yaml @@ -89,6 +89,8 @@ H100_NVL: RTX_PRO_6000_BLACKWELL: - tier: rtx-pro-6000-blackwell-0-gpu gpusPerPod: 0 + env: + NVIDIA_VISIBLE_DEVICES: void resources: limits: cpu: "16" diff --git a/tools/cluster_setup/openshift-cluster-config.yaml b/tools/cluster_setup/openshift-cluster-config.yaml index 072de5c6..9af62f2b 100644 --- a/tools/cluster_setup/openshift-cluster-config.yaml +++ b/tools/cluster_setup/openshift-cluster-config.yaml @@ -67,10 +67,10 @@ storage: objectStore: type: "seaweedfs" # aws | s3compat | minio | seaweedfs bucket: "ai-platform-bucket" - endpoint: "http://18.116.39.79:8333" + endpoint: "" auth: - rootUser: "minioadmin" - rootPassword: "minioadmin" + rootUser: "" + rootPassword: "" splunk: standaloneName: splunk-standalone diff --git a/tools/cluster_setup/openshift_with_stack.sh b/tools/cluster_setup/openshift_with_stack.sh index 53f79650..b090f408 100755 --- a/tools/cluster_setup/openshift_with_stack.sh +++ b/tools/cluster_setup/openshift_with_stack.sh @@ -371,8 +371,8 @@ configure_images() { preflight_checks() { log "Running preflight checks..." - for tool in oc yq; do - command -v "$tool" >/dev/null 2>&1 && log " ✓ $tool found" || err "Missing $tool in PATH" + for tool in oc yq helm aws curl jq base64 tar; do + command -v "$tool" >/dev/null 2>&1 && log " ✓ $tool found" || err "Missing required tool: $tool" done # Verify we are connected to the cluster @@ -988,9 +988,14 @@ ensure_ecr_pull_secret() { --namespace="${ns}" \ --dry-run=client -o yaml | oc apply -f - - # Patch the default SA so pods without explicit imagePullSecrets also pull correctly - oc patch serviceaccount default -n "${ns}" \ - -p '{"imagePullSecrets": [{"name": "ecr-registry-secret"}]}' 2>/dev/null || true + # Append ecr-registry-secret to the default SA only if not already present. + # Using JSON patch add rather than a merge patch to avoid overwriting existing pull secrets. + if ! oc get serviceaccount default -n "${ns}" -o jsonpath='{.imagePullSecrets[*].name}' 2>/dev/null | grep -qw ecr-registry-secret; then + oc patch serviceaccount default -n "${ns}" --type=json \ + -p='[{"op":"add","path":"/imagePullSecrets","value":[]}]' 2>/dev/null || true + oc patch serviceaccount default -n "${ns}" --type=json \ + -p='[{"op":"add","path":"/imagePullSecrets/-","value":{"name":"ecr-registry-secret"}}]' 2>/dev/null || true + fi log " ✓ ecr-registry-secret created in ${ns}" done @@ -1511,11 +1516,12 @@ main_install() { log "============================================" log "" log "Next steps:" - log " 1. Create an AIPlatform CR in namespace '${AI_NS}'" + log " 1. Verify resources:" + log " oc get aiplatform,aiservice,raycluster,rayservice -n ${AI_NS}" log " 2. Check operator logs:" log " oc logs -n splunk-ai-operator-system -l control-plane=controller-manager -f" - log " 3. Watch resources:" - log " oc get aiplatform,raycluster,rayservice -n ${AI_NS}" + log " 3. Watch Ray cluster:" + log " oc get raycluster,rayservice -n ${AI_NS} -w" log "" log "Log file: ${LOG_FILE}" } From cc16e49f15984747e983b280460696dbafaa27ff Mon Sep 17 00:00:00 2001 From: kbhos Date: Thu, 18 Jun 2026 15:50:12 +0530 Subject: [PATCH 10/80] clean up --- tools/cluster_setup/openshift_with_stack.sh | 26 +-------------------- 1 file changed, 1 insertion(+), 25 deletions(-) diff --git a/tools/cluster_setup/openshift_with_stack.sh b/tools/cluster_setup/openshift_with_stack.sh index b090f408..f0f49730 100755 --- a/tools/cluster_setup/openshift_with_stack.sh +++ b/tools/cluster_setup/openshift_with_stack.sh @@ -1032,30 +1032,6 @@ install_splunk_ai_operator() { oc apply --server-side --force-conflicts -f "${SPLUNK_AI_FILE}" 2>&1 || true fi - # Inject the local instance.yaml so the operator knows about RTX_PRO_6000_BLACKWELL - # and other accelerators that may not be baked into the operator image. - local instance_src - instance_src="$(dirname "${SPLUNK_AI_FILE}")/../../config/configs/instance.yaml" - if [[ ! -f "${instance_src}" ]]; then - instance_src="$(cd "$(dirname "$0")/../.." && pwd)/config/configs/instance.yaml" - fi - if [[ -f "${instance_src}" ]]; then - oc create configmap splunk-ai-operator-instance-yaml \ - -n "${ai_operator_ns}" \ - --from-file=instance.yaml="${instance_src}" \ - --dry-run=client -o yaml | oc -n "${ai_operator_ns}" apply -f - - # Mount the ConfigMap and set INSTANCE_FILE so the operator uses it - oc patch deployment splunk-ai-operator-controller-manager \ - -n "${ai_operator_ns}" --type=json -p='[ - {"op":"add","path":"/spec/template/spec/volumes/-","value":{"name":"instance-yaml","configMap":{"name":"splunk-ai-operator-instance-yaml"}}}, - {"op":"add","path":"/spec/template/spec/containers/0/volumeMounts/-","value":{"name":"instance-yaml","mountPath":"/etc/instance","readOnly":true}}, - {"op":"add","path":"/spec/template/spec/containers/0/env/-","value":{"name":"INSTANCE_FILE","value":"/etc/instance/instance.yaml"}} - ]' 2>/dev/null || true - log " ✓ instance.yaml ConfigMap injected into operator" - else - warn "instance.yaml not found at ${instance_src} — defaultAcceleratorType may not resolve" - fi - # Patch the operator SA and deployment with ECR pull secret AFTER the manifest apply # (the SA is created by the manifest; patching before apply silently does nothing). if [[ "${ECR_ENABLED}" == "true" ]]; then @@ -1069,7 +1045,7 @@ install_splunk_ai_operator() { log " ✓ ECR pull secret patched into operator SA and deployment" fi - # Rollout restart so the deployment picks up pull secrets and instance.yaml. + # Rollout restart so the deployment picks up the updated pull secrets. oc rollout restart deployment splunk-ai-operator-controller-manager \ -n "${ai_operator_ns}" 2>/dev/null || true From 108a392f2f06fd355fa0a118b83e42dbcdc3f7d3 Mon Sep 17 00:00:00 2001 From: kbhos Date: Tue, 23 Jun 2026 18:14:05 +0530 Subject: [PATCH 11/80] feat(): AI pod script enhancements --- config/configs/applications.yaml | 18 +-- pkg/ai/raybuilder/builder.go | 4 +- .../openshift-cluster-config.yaml | 25 ++-- tools/cluster_setup/openshift_with_stack.sh | 109 ++++++++++++++++-- 4 files changed, 131 insertions(+), 25 deletions(-) diff --git a/config/configs/applications.yaml b/config/configs/applications.yaml index c8a38601..af08bce6 100644 --- a/config/configs/applications.yaml +++ b/config/configs/applications.yaml @@ -62,7 +62,7 @@ applications: target_ongoing_requests: 4 max_ongoing_requests: 10 ray_actor_options: - num_gpus: 2 + num_gpus: 1 resources: "gpu_count:2": 0.001 "accelerator_type:RTX_PRO_6000_BLACKWELL": 0.001 @@ -92,11 +92,11 @@ applications: RTX_PRO_6000_BLACKWELL: engine_args: dtype: bfloat16 - gpu_memory_utilization: 0.85 - max_model_len: 240000 + gpu_memory_utilization: 0.9 + max_model_len: 120000 max_num_batched_tokens: 4096 max_num_seqs: 1 - tensor_parallel_size: 2 + tensor_parallel_size: 1 model_config: openai_serving_config: chat: @@ -110,7 +110,7 @@ applications: model_id: gemma4_31b_it model_loader: blob_storage: - blob_prefix: model_artifacts/gemma-4-31b-it + blob_prefix: model_artifacts/gemma-4-31b-it-qat-w4a16-ct tokenizer_definition: model_id: gemma4_31b_it model_loader: @@ -121,7 +121,7 @@ applications: - processor_config.json - tokenizer_config.json - tokenizer.json - blob_prefix: model_artifacts/gemma-4-31b-it + blob_prefix: model_artifacts/gemma-4-31b-it-qat-w4a16-ct name: Gemma431bIt import_path: main:create_serve_app route_prefix: /gemma4_31b_it @@ -188,7 +188,7 @@ applications: target_ongoing_requests: 4 max_ongoing_requests: 8 ray_actor_options: - num_gpus: 1 + num_gpus: 0.5 resources: "gpu_count:2": 0.001 "accelerator_type:RTX_PRO_6000_BLACKWELL": 0.001 @@ -208,6 +208,10 @@ applications: engine_args: gpu_memory_utilization: 0.95 tensor_parallel_size: 1 + RTX_PRO_6000_BLACKWELL: + engine_args: + gpu_memory_utilization: 0.5 + tensor_parallel_size: 1 model_config: openai_serving_config: chat: diff --git a/pkg/ai/raybuilder/builder.go b/pkg/ai/raybuilder/builder.go index adf05ce3..dd27dd3d 100644 --- a/pkg/ai/raybuilder/builder.go +++ b/pkg/ai/raybuilder/builder.go @@ -962,13 +962,13 @@ func (b *Builder) makeHeadTemplate() corev1.PodTemplateSpec { Resources: corev1.ResourceRequirements{ Requests: corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("1"), - corev1.ResourceMemory: resource.MustParse("2Gi"), + corev1.ResourceMemory: resource.MustParse("4Gi"), corev1.ResourceEphemeralStorage: resource.MustParse("5Gi"), "nvidia.com/gpu": resource.MustParse("0"), }, Limits: corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("4"), - corev1.ResourceMemory: resource.MustParse("8Gi"), + corev1.ResourceMemory: resource.MustParse("16Gi"), corev1.ResourceEphemeralStorage: resource.MustParse("10Gi"), "nvidia.com/gpu": resource.MustParse("0"), }, diff --git a/tools/cluster_setup/openshift-cluster-config.yaml b/tools/cluster_setup/openshift-cluster-config.yaml index 9af62f2b..05fa99cf 100644 --- a/tools/cluster_setup/openshift-cluster-config.yaml +++ b/tools/cluster_setup/openshift-cluster-config.yaml @@ -24,8 +24,8 @@ openshift: # RTX 6000 Blackwell node is dedicated to GPU model pods (ray-worker). nodes: cpu: - - 00-25-b5-b5-00-31 - - 00-25-b5-b5-00-33 + - 00-25-b5-b5-00-35 + - 00-25-b5-b5-00-37 gpu: - cc-40-f3-9f-e2-3c @@ -34,7 +34,7 @@ images: registry: "658391232643.dkr.ecr.us-east-2.amazonaws.com" operator: - image: "658391232643.dkr.ecr.us-east-2.amazonaws.com/kiran/splunk/splunk-ai-operator:openshift-0.5" + image: "658391232643.dkr.ecr.us-east-2.amazonaws.com/kiran/splunk/splunk-ai-operator:openshift-0.10" ray: headImage: "ml-platform/ray/ray-head:build-953" @@ -65,12 +65,18 @@ storage: storageClass: "local-path" vectorDbSize: "50Gi" objectStore: - type: "seaweedfs" # aws | s3compat | minio | seaweedfs + # type: "seaweedfs" # aws | s3compat | minio | seaweedfs (external only for non-aws) + # bucket: "ai-platform-bucket" + # endpoint: "http://18.116.39.79:8333" + # auth: + # rootUser: "minioadmin" + # rootPassword: "minioadmin" + type: "minio" # aws | s3compat | minio | seaweedfs (external only for non-aws) bucket: "ai-platform-bucket" - endpoint: "" + endpoint: "http://54.253.238.99:9000" auth: - rootUser: "" - rootPassword: "" + rootUser: "minioadmin" + rootPassword: "minioadmin" splunk: standaloneName: splunk-standalone @@ -86,6 +92,11 @@ aiPlatform: features: - name: "saia" version: "1.1.0" + cpuScheduling: + tolerations: + - key: "node-role.kubernetes.io/master" + operator: "Exists" + effect: "NoSchedule" operators: ray: diff --git a/tools/cluster_setup/openshift_with_stack.sh b/tools/cluster_setup/openshift_with_stack.sh index f0f49730..687194fb 100755 --- a/tools/cluster_setup/openshift_with_stack.sh +++ b/tools/cluster_setup/openshift_with_stack.sh @@ -260,6 +260,18 @@ ${yq_err}" MINIO_ROOT_PASSWORD=$(yq eval '.storage.objectStore.auth.rootPassword // ""' "${CONFIG_FILE}" 2>/dev/null || echo "") AI_STANDALONE_NAME=$(yq eval '.splunk.standaloneName // "splunk-standalone"' "${CONFIG_FILE}" 2>/dev/null || echo "splunk-standalone") + # Ingress domain: read from config if set, otherwise auto-detect from the cluster. + # Used to create an OpenShift Route for SAIA so both browsers and in-cluster services + # can reach it via a stable hostname. + local cfg_ingress_domain + cfg_ingress_domain=$(yq eval '.openshift.ingressDomain // ""' "${CONFIG_FILE}" 2>/dev/null || echo "") + if [[ -n "$cfg_ingress_domain" ]]; then + INGRESS_DOMAIN="$cfg_ingress_domain" + else + INGRESS_DOMAIN=$(oc get ingresscontroller default -n openshift-ingress-operator \ + -o jsonpath='{.status.domain}' 2>/dev/null || echo "") + fi + log "Configuration loaded: namespace=${AI_NS}, accelerator=${DEFAULT_ACCELERATOR}" } @@ -723,16 +735,25 @@ label_nodes() { oc adm taint node "${node}" nvidia.com/gpu=true:NoSchedule --overwrite 2>/dev/null || true done - # Verify no worker node is left unlabeled — unlabeled workers cause silent Pending forever - local unlabeled - unlabeled=$(oc get nodes -l node-role.kubernetes.io/worker -o json 2>/dev/null \ - | python3 -c " + # Verify labeled nodes have the label — scope check to listed nodes in manual mode + # (auto mode checks all workers; manual mode only checks what the user explicitly listed) + local unlabeled="" + if [[ "${NODE_LABEL_STRATEGY}" == "manual" ]]; then + for node in "${cpu_nodes[@]}" "${gpu_nodes[@]}"; do + local val + val=$(oc get node "${node}" -o jsonpath='{.metadata.labels.splunk\.ai/workload-type}' 2>/dev/null || echo "") + [[ -z "${val}" ]] && unlabeled+="${node}"$'\n' + done + else + unlabeled=$(oc get nodes -l node-role.kubernetes.io/worker -o json 2>/dev/null \ + | python3 -c " import json,sys data=json.load(sys.stdin) for n in data['items']: if 'splunk.ai/workload-type' not in n['metadata']['labels']: print(n['metadata']['name']) " 2>/dev/null || echo "") + fi if [[ -n "${unlabeled}" ]]; then err "Worker node(s) still missing splunk.ai/workload-type after labeling: @@ -843,13 +864,14 @@ install_local_path_provisioner() { 2>/dev/null || true # Patch the helper pod template to run privileged and relabel the created directory - # with svirt_sandbox_file_t so containers can read/write it (SELinux on OpenShift). - # Without the chcon, directories get var_t which containers cannot access. + # busybox lacks chcon; use ubi-minimal which ships selinux-utils so chcon runs. + # container_file_t allows any container to read/write regardless of MCS categories. + # Without this, local-path-provisioner creates dirs with var_t which blocks all containers. oc patch configmap local-path-config -n local-path-storage --type=merge -p "$(cat <<'PATCH' { "data": { - "helperPod.yaml": "apiVersion: v1\nkind: Pod\nmetadata:\n name: helper-pod\nspec:\n priorityClassName: system-node-critical\n tolerations:\n - key: node.kubernetes.io/disk-pressure\n operator: Exists\n effect: NoSchedule\n containers:\n - name: helper-pod\n image: busybox\n imagePullPolicy: IfNotPresent\n securityContext:\n privileged: true\n", - "setup": "#!/bin/sh\nset -eu\nmkdir -m 0777 -p \"$VOL_DIR\"\nchcon -Rt container_file_t -l s0 \"$VOL_DIR\" 2>/dev/null || true\n" + "helperPod.yaml": "apiVersion: v1\nkind: Pod\nmetadata:\n name: helper-pod\nspec:\n priorityClassName: system-node-critical\n tolerations:\n - key: node.kubernetes.io/disk-pressure\n operator: Exists\n effect: NoSchedule\n containers:\n - name: helper-pod\n image: registry.access.redhat.com/ubi8/ubi-minimal\n imagePullPolicy: IfNotPresent\n securityContext:\n privileged: true\n", + "setup": "#!/bin/sh\nset -eu\nmkdir -m 0777 -p \"$VOL_DIR\"\nchcon -Rt container_file_t \"$VOL_DIR\"\n" } } PATCH @@ -1329,6 +1351,13 @@ install_ai_platform_cr() { storage_yaml=" storage:"$'\n'" vectorDB:"$'\n'" size: ${VECTORDB_SIZE}"$'\n'" storageClassName: ${STORAGE_CLASS}"$'\n' fi + # CPU scheduling tolerations — read from config, default to empty list + local cpu_tols cpu_tolerations_inline="[]" + cpu_tols=$(yq eval '.aiPlatform.cpuScheduling.tolerations // []' "${CONFIG_FILE}" 2>/dev/null || echo "[]") + if [[ "${cpu_tols}" != "[]" && "${cpu_tols}" != "null" && -n "${cpu_tols}" ]]; then + cpu_tolerations_inline=$'\n'"$(echo "${cpu_tols}" | sed 's/^/ /')" + fi + # Probe the AIPlatform webhook TLS cert immediately before applying. # cert-manager issues certs with notBefore ~30-60s in the future (clock skew); # retry until the x509 error clears. Using --dry-run=server hits the exact @@ -1384,7 +1413,7 @@ ${svc_template_yaml}${storage_yaml} cpuScheduler: nodeSelector: splunk.ai/workload-type: cpu - tolerations: [] + tolerations: ${cpu_tolerations_inline} gpuScheduler: nodeSelector: splunk.ai/workload-type: gpu @@ -1412,6 +1441,53 @@ YAML log " ✓ AIPlatform CR installed" } +# ====== CREATE SAIA ROUTE ====== +# Creates an OpenShift Route so SAIA is reachable via a stable external hostname. +# The URL must be reachable from both the browser and from within the cluster +# (Splunk's setup page validates connectivity from the server side). +# NodePort alone doesn't work when node IPs are not externally routable. +create_saia_route() { + if [[ -z "$INGRESS_DOMAIN" ]]; then + warn "Could not determine ingress domain — skipping SAIA Route creation" + warn "Create it manually: oc expose svc/openshift-ai-platform-saia-saia-service -n ${AI_NS}" + return 0 + fi + + local route_host="saia.${INGRESS_DOMAIN}" + local svc_name="${AI_PLATFORM_NAME}-saia-saia-service" + + log "Creating SAIA Route: http://${route_host} ..." + + # Wait for the SAIA service to exist (created by the operator after posthook) + local elapsed=0 timeout=300 + while ! oc get svc "${svc_name}" -n "${AI_NS}" >/dev/null 2>&1; do + sleep 10; elapsed=$((elapsed + 10)) + [[ ${elapsed} -ge ${timeout} ]] && { warn "Timed out waiting for SAIA service — Route not created"; return 0; } + log " Waiting for SAIA service... (${elapsed}s)" + done + + oc apply -f - </dev/null || true + log "Removing AIPlatform CR and waiting for finalizers..." oc delete aiplatform --all -n "${AI_NS}" --timeout=120s 2>/dev/null || true oc delete standalone --all -n "${AI_NS}" --timeout=60s 2>/dev/null || true From 8116909b01748f1c67c9a906975ef8c8a0a6e9de Mon Sep 17 00:00:00 2001 From: kbhos-splunk Date: Tue, 23 Jun 2026 18:25:43 +0530 Subject: [PATCH 12/80] Update openshift-cluster-config.yaml --- tools/cluster_setup/openshift-cluster-config.yaml | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/tools/cluster_setup/openshift-cluster-config.yaml b/tools/cluster_setup/openshift-cluster-config.yaml index 05fa99cf..55fe99e5 100644 --- a/tools/cluster_setup/openshift-cluster-config.yaml +++ b/tools/cluster_setup/openshift-cluster-config.yaml @@ -65,18 +65,12 @@ storage: storageClass: "local-path" vectorDbSize: "50Gi" objectStore: - # type: "seaweedfs" # aws | s3compat | minio | seaweedfs (external only for non-aws) - # bucket: "ai-platform-bucket" - # endpoint: "http://18.116.39.79:8333" - # auth: - # rootUser: "minioadmin" - # rootPassword: "minioadmin" - type: "minio" # aws | s3compat | minio | seaweedfs (external only for non-aws) + type: "seaweedfs" # aws | s3compat | minio | seaweedfs bucket: "ai-platform-bucket" - endpoint: "http://54.253.238.99:9000" + endpoint: "" auth: - rootUser: "minioadmin" - rootPassword: "minioadmin" + rootUser: "" + rootPassword: "" splunk: standaloneName: splunk-standalone From c5046267e8831d438902862497f635cfe2c0332f Mon Sep 17 00:00:00 2001 From: kbhos-splunk Date: Thu, 25 Jun 2026 10:51:05 +0530 Subject: [PATCH 13/80] Apply suggestions from code review Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- tools/cluster_setup/openshift-cluster-config.yaml | 2 +- tools/cluster_setup/openshift_with_stack.sh | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/tools/cluster_setup/openshift-cluster-config.yaml b/tools/cluster_setup/openshift-cluster-config.yaml index 55fe99e5..e9fa9b4b 100644 --- a/tools/cluster_setup/openshift-cluster-config.yaml +++ b/tools/cluster_setup/openshift-cluster-config.yaml @@ -70,7 +70,7 @@ storage: endpoint: "" auth: rootUser: "" - rootPassword: "" + rootPassword: "" splunk: standaloneName: splunk-standalone diff --git a/tools/cluster_setup/openshift_with_stack.sh b/tools/cluster_setup/openshift_with_stack.sh index 687194fb..831b1bc7 100755 --- a/tools/cluster_setup/openshift_with_stack.sh +++ b/tools/cluster_setup/openshift_with_stack.sh @@ -870,9 +870,7 @@ install_local_path_provisioner() { oc patch configmap local-path-config -n local-path-storage --type=merge -p "$(cat <<'PATCH' { "data": { - "helperPod.yaml": "apiVersion: v1\nkind: Pod\nmetadata:\n name: helper-pod\nspec:\n priorityClassName: system-node-critical\n tolerations:\n - key: node.kubernetes.io/disk-pressure\n operator: Exists\n effect: NoSchedule\n containers:\n - name: helper-pod\n image: registry.access.redhat.com/ubi8/ubi-minimal\n imagePullPolicy: IfNotPresent\n securityContext:\n privileged: true\n", - "setup": "#!/bin/sh\nset -eu\nmkdir -m 0777 -p \"$VOL_DIR\"\nchcon -Rt container_file_t \"$VOL_DIR\"\n" - } + "setup": "#!/bin/sh\nset -eu\nmkdir -m 0777 -p \"$VOL_DIR\"\nif command -v chcon >/dev/null 2>&1; then\n chcon -Rt container_file_t -l s0 \"$VOL_DIR\" 2>/dev/null || true\nfi\n" } PATCH )" @@ -1449,7 +1447,7 @@ YAML create_saia_route() { if [[ -z "$INGRESS_DOMAIN" ]]; then warn "Could not determine ingress domain — skipping SAIA Route creation" - warn "Create it manually: oc expose svc/openshift-ai-platform-saia-saia-service -n ${AI_NS}" + warn "Create it manually: oc expose svc/${AI_PLATFORM_NAME}-saia-saia-service -n ${AI_NS}" return 0 fi From 98508836da58771eb1ceec73320df03d56f546ee Mon Sep 17 00:00:00 2001 From: kbhos Date: Fri, 3 Jul 2026 14:34:38 +0530 Subject: [PATCH 14/80] feat(): Openshift scripts for air gapped env --- .../install_from_airgap_bundle_openshift.sh | 229 +++++++++++ tools/cluster_setup/openshift_with_stack.sh | 29 +- .../prepare_airgap_bundle_openshift.sh | 371 ++++++++++++++++++ 3 files changed, 622 insertions(+), 7 deletions(-) create mode 100755 tools/cluster_setup/install_from_airgap_bundle_openshift.sh create mode 100755 tools/cluster_setup/prepare_airgap_bundle_openshift.sh diff --git a/tools/cluster_setup/install_from_airgap_bundle_openshift.sh b/tools/cluster_setup/install_from_airgap_bundle_openshift.sh new file mode 100755 index 00000000..47f062e3 --- /dev/null +++ b/tools/cluster_setup/install_from_airgap_bundle_openshift.sh @@ -0,0 +1,229 @@ +#!/usr/bin/env bash +# install_from_airgap_bundle_openshift.sh +# Run on the air-gapped OpenShift install machine (needs oc + helm pre-installed, +# but NO outbound internet). Extracts the bundle produced by +# prepare_airgap_bundle_openshift.sh, sets environment-variable overrides, then +# invokes openshift_with_stack.sh. +# +# Usage: +# ./install_from_airgap_bundle_openshift.sh \ +# --bundle airgap-bundle-openshift-.tar.gz \ +# --config openshift-cluster-config.yaml [--extract-dir /opt/airgap] +# +# The installer (openshift_with_stack.sh) must be in the same directory as +# this script, or pass --installer /path/to/openshift_with_stack.sh. + +set -euo pipefail + +BUNDLE_TARBALL="" +CONFIG_FILE="" +EXTRACT_DIR="${EXTRACT_DIR:-/opt/airgap}" +INSTALLER_SCRIPT="" +SUBCOMMAND="${SUBCOMMAND:-install}" + +# ── Argument parsing ────────────────────────────────────────────────────────── +while [[ $# -gt 0 ]]; do + case "$1" in + --bundle) BUNDLE_TARBALL="$2"; shift 2 ;; + --config) CONFIG_FILE="$2"; shift 2 ;; + --extract-dir) EXTRACT_DIR="$2"; shift 2 ;; + --installer) INSTALLER_SCRIPT="$2"; shift 2 ;; + --subcommand) SUBCOMMAND="$2"; shift 2 ;; + -h|--help) + cat <<'HELP' +install_from_airgap_bundle_openshift.sh — extract an OpenShift air-gap bundle +and run the Splunk AI Platform installer with no outbound internet required. + +USAGE + ./install_from_airgap_bundle_openshift.sh --bundle BUNDLE.tar.gz [OPTIONS] + +REQUIRED + --bundle FILE Path to the tar.gz produced by prepare_airgap_bundle_openshift.sh. + +OPTIONS + --config FILE Path to your openshift-cluster-config.yaml. + If omitted, the installer looks for CONFIG_FILE in the + environment or its default location. + + --extract-dir DIR Directory where the bundle is extracted. + Default: /opt/airgap + Env: EXTRACT_DIR + + --installer SCRIPT Path to openshift_with_stack.sh. + Default: same directory as this script. + + --subcommand CMD Installer subcommand to run: install | delete + Default: install + Env: SUBCOMMAND + + -h, --help Show this help text. + +WHAT THIS SCRIPT DOES + 1. Extracts the bundle into --extract-dir. + 2. Verifies SHA-256 checksums of all bundled files. + 3. Exports the following environment variables so openshift_with_stack.sh + uses local files instead of downloading from the internet: + + CERT_MANAGER_MANIFEST_URL → file:///manifests/cert-manager.yaml + LOCAL_PATH_MANIFEST_URL → file:///manifests/local-path-storage.yaml + OTEL_CHART_PATH → /charts/opentelemetry-operator-*.tgz + KUBERAY_CHART_PATH → /charts/kuberay-operator-*.tgz + AIRGAP_MODE → true + + 4. Invokes openshift_with_stack.sh . + +PREREQUISITES + On this machine (no internet needed): + oc — logged in to the target OpenShift cluster (oc login ... done) + helm — v3+ + tar + + Before running this script: + - Mirror container images to your internal registry and update images.* + in your cluster config. + - For NFD / GPU Operator: apply the oc mirror ImageContentSourcePolicy and + CatalogSource so OLM can pull from your mirrored catalog. + - Stage model weights via tools/artifacts_download_upload_scripts/. + +MANUAL USE (advanced) + If you extracted the bundle yourself and want to run the installer directly: + + export AIRGAP_BUNDLE_DIR=/opt/airgap/airgap-bundle-openshift- + source "${AIRGAP_BUNDLE_DIR}/airgap-env.sh" + CONFIG_FILE=./openshift-cluster-config.yaml ./openshift_with_stack.sh install + +HELP + exit 0 + ;; + *) echo "Unknown option: $1" >&2 + echo "Run with --help for usage." >&2 + exit 1 ;; + esac +done + +# ── Helpers ─────────────────────────────────────────────────────────────────── +log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] INFO $*"; } +warn() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] WARN $*" >&2; } +err() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] ERROR $*" >&2; exit 1; } + +require_cmd() { + command -v "$1" >/dev/null 2>&1 || err "Required tool not found: $1 — install it on this machine before running." +} + +# ── Pre-flight ───────────────────────────────────────────────────────────────── +[[ -n "${BUNDLE_TARBALL}" ]] || err "No bundle specified. Use --bundle airgap-bundle-openshift-.tar.gz" +[[ -f "${BUNDLE_TARBALL}" ]] || err "Bundle file not found: ${BUNDLE_TARBALL}" + +require_cmd tar +require_cmd helm +require_cmd oc + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +if [[ -z "${INSTALLER_SCRIPT}" ]]; then + INSTALLER_SCRIPT="${SCRIPT_DIR}/openshift_with_stack.sh" +fi +[[ -f "${INSTALLER_SCRIPT}" ]] || err "Installer not found: ${INSTALLER_SCRIPT}. Pass --installer /path/to/openshift_with_stack.sh" +[[ -x "${INSTALLER_SCRIPT}" ]] || chmod +x "${INSTALLER_SCRIPT}" + +log "=== Splunk AI Platform — OpenShift Air-Gap Installation ===" +log "Bundle : ${BUNDLE_TARBALL}" +log "Extract dir : ${EXTRACT_DIR}" +log "Installer : ${INSTALLER_SCRIPT}" +log "Subcommand : ${SUBCOMMAND}" +[[ -n "${CONFIG_FILE}" ]] && log "Config : ${CONFIG_FILE}" +log "" + +# ── Extract bundle ───────────────────────────────────────────────────────────── +log "Extracting bundle..." +mkdir -p "${EXTRACT_DIR}" +tar -xzf "${BUNDLE_TARBALL}" -C "${EXTRACT_DIR}" + +# Find the extracted bundle directory (named airgap-bundle-openshift-) +BUNDLE_DIR="$(find "${EXTRACT_DIR}" -maxdepth 1 -mindepth 1 -type d -name 'airgap-bundle-openshift-*' | sort | tail -1)" +[[ -n "${BUNDLE_DIR}" ]] || err "Could not find extracted bundle directory in ${EXTRACT_DIR}" +log "Bundle extracted to: ${BUNDLE_DIR}" + +# ── Verify checksums ─────────────────────────────────────────────────────────── +if command -v sha256sum >/dev/null 2>&1 || command -v shasum >/dev/null 2>&1; then + log "Verifying checksums..." + ( + cd "${BUNDLE_DIR}" + if command -v sha256sum >/dev/null 2>&1; then + sha256sum --check checksums.sha256 --quiet + else + shasum -a 256 --check checksums.sha256 --quiet + fi + ) || err "Checksum verification failed — bundle may be corrupt or tampered with." + log "Checksums verified OK" +else + warn "sha256sum / shasum not found — skipping checksum verification" +fi + +# ── Read version manifest ────────────────────────────────────────────────────── +VERSION_FILE="${BUNDLE_DIR}/bundle-versions.txt" +[[ -f "${VERSION_FILE}" ]] || err "bundle-versions.txt missing from bundle — bundle is incomplete." + +get_version() { + grep "^${1}=" "${VERSION_FILE}" | cut -d= -f2 +} + +CERT_MANAGER_VERSION="$(get_version cert_manager_version)" +KUBERAY_CHART_VERSION="$(get_version kuberay_chart_version)" +OTEL_CHART_VERSION="$(get_version otel_chart_version)" + +log "Bundle component versions:" +log " cert-manager : ${CERT_MANAGER_VERSION}" +log " kuberay chart : ${KUBERAY_CHART_VERSION}" +log " otel chart : ${OTEL_CHART_VERSION}" + +# ── Resolve chart paths ──────────────────────────────────────────────────────── +LOCAL_CHARTS_DIR="${BUNDLE_DIR}/charts" + +_resolve_chart() { + local pattern="$1" label="$2" + # shellcheck disable=SC2206 + local matches=( ${pattern} ) + if [[ ${#matches[@]} -eq 0 || ! -f "${matches[0]}" ]]; then + err "Chart not found for ${label} (expected: ${pattern}). Bundle may be corrupt or version-mismatched." + fi + echo "${matches[0]}" +} + +OTEL_TGZ="$(_resolve_chart "${LOCAL_CHARTS_DIR}/opentelemetry-operator-${OTEL_CHART_VERSION}*.tgz" "opentelemetry-operator")" +KUBERAY_TGZ="$(_resolve_chart "${LOCAL_CHARTS_DIR}/kuberay-operator-${KUBERAY_CHART_VERSION}*.tgz" "kuberay-operator")" + +# ── Export env-var overrides read by openshift_with_stack.sh ────────────────── +export AIRGAP_BUNDLE_DIR="${BUNDLE_DIR}" + +# Static manifests (installer strips file:// to a bare path for oc apply) +export CERT_MANAGER_MANIFEST_URL="file://${BUNDLE_DIR}/manifests/cert-manager.yaml" +export LOCAL_PATH_MANIFEST_URL="file://${BUNDLE_DIR}/manifests/local-path-storage.yaml" + +# Helm chart paths — installer uses these instead of remote repos +export OTEL_CHART_PATH="${OTEL_TGZ}" +export KUBERAY_CHART_PATH="${KUBERAY_TGZ}" + +# Signal air-gapped mode (skips model staging, enforces offline paths) +export AIRGAP_MODE="true" + +log "" +log "Environment overrides set:" +log " CERT_MANAGER_MANIFEST_URL = ${CERT_MANAGER_MANIFEST_URL}" +log " LOCAL_PATH_MANIFEST_URL = ${LOCAL_PATH_MANIFEST_URL}" +log " OTEL_CHART_PATH = ${OTEL_CHART_PATH}" +log " KUBERAY_CHART_PATH = ${KUBERAY_CHART_PATH}" +log " AIRGAP_MODE = ${AIRGAP_MODE}" +log "" +log "Launching installer..." +log "" + +# ── Invoke main installer ────────────────────────────────────────────────────── +INSTALLER_CMD="${INSTALLER_SCRIPT} ${SUBCOMMAND}" +if [[ -n "${CONFIG_FILE}" ]]; then + CONFIG_FILE="$(realpath "${CONFIG_FILE}")" + exec env CONFIG_FILE="${CONFIG_FILE}" ${INSTALLER_CMD} +else + log "No --config supplied. The installer will look for CONFIG_FILE in the environment" + log "or its default location. Set CONFIG_FILE if it is not in the current directory." + exec ${INSTALLER_CMD} +fi diff --git a/tools/cluster_setup/openshift_with_stack.sh b/tools/cluster_setup/openshift_with_stack.sh index 831b1bc7..a9aa8000 100755 --- a/tools/cluster_setup/openshift_with_stack.sh +++ b/tools/cluster_setup/openshift_with_stack.sh @@ -914,13 +914,20 @@ install_otel_operator() { return 0 fi - helm repo add open-telemetry https://open-telemetry.github.io/opentelemetry-helm-charts 2>/dev/null || true - helm repo update open-telemetry 2>/dev/null || true + local otel_chart_ref + if [[ -n "${OTEL_CHART_PATH:-}" && -f "${OTEL_CHART_PATH}" ]]; then + otel_chart_ref="${OTEL_CHART_PATH}" + log " Using local chart: ${otel_chart_ref}" + else + helm repo add open-telemetry https://open-telemetry.github.io/opentelemetry-helm-charts 2>/dev/null || true + helm repo update open-telemetry 2>/dev/null || true + otel_chart_ref="open-telemetry/opentelemetry-operator" + fi local otel_retries=0 while (( otel_retries < 6 )); do local otel_out - otel_out=$(helm upgrade --install opentelemetry-operator open-telemetry/opentelemetry-operator \ + otel_out=$(helm upgrade --install opentelemetry-operator "${otel_chart_ref}" \ --namespace opentelemetry-operator-system --create-namespace \ --set manager.collectorImage.repository=otel/opentelemetry-collector-contrib \ --set admissionWebhooks.certManager.enabled=true \ @@ -959,12 +966,20 @@ install_ray_operator() { return 0 fi - helm repo add kuberay https://ray-project.github.io/kuberay-helm/ 2>/dev/null || true - helm repo update kuberay + local kuberay_chart_ref kuberay_version_flag=() + if [[ -n "${KUBERAY_CHART_PATH:-}" && -f "${KUBERAY_CHART_PATH}" ]]; then + kuberay_chart_ref="${KUBERAY_CHART_PATH}" + log " Using local chart: ${kuberay_chart_ref}" + else + helm repo add kuberay https://ray-project.github.io/kuberay-helm/ 2>/dev/null || true + helm repo update kuberay + kuberay_chart_ref="kuberay/kuberay-operator" + kuberay_version_flag=(--version 1.2.2) + fi - helm upgrade --install kuberay-operator kuberay/kuberay-operator \ + helm upgrade --install kuberay-operator "${kuberay_chart_ref}" \ --namespace ray-system --create-namespace \ - --version 1.2.2 \ + "${kuberay_version_flag[@]+"${kuberay_version_flag[@]}"}" \ --set image.repository=quay.io/kuberay/operator \ --set image.tag=v1.2.2 \ --wait --timeout=10m diff --git a/tools/cluster_setup/prepare_airgap_bundle_openshift.sh b/tools/cluster_setup/prepare_airgap_bundle_openshift.sh new file mode 100755 index 00000000..eca39d86 --- /dev/null +++ b/tools/cluster_setup/prepare_airgap_bundle_openshift.sh @@ -0,0 +1,371 @@ +#!/usr/bin/env bash +# prepare_airgap_bundle_openshift.sh +# Run on an internet-connected machine to download every Helm chart and static +# manifest needed by openshift_with_stack.sh. Produces a single tar.gz that +# can be copied to an air-gapped OpenShift cluster and consumed by +# install_from_airgap_bundle_openshift.sh. +# +# NOTE: This script does NOT bundle container images or OLM catalog content. +# - Container images: mirror them using `oc mirror` + your internal registry. +# See container-images.txt in the output bundle for the full image list. +# - NFD / GPU Operator: install via OLM from a mirrored catalog +# (oc mirror + ImageContentSourcePolicy). See bundle README section. +# - Model weights: stage separately via tools/artifacts_download_upload_scripts/. +# +# Usage: +# ./prepare_airgap_bundle_openshift.sh [--output-dir DIR] +# +# Requirements on this machine: +# curl, helm, tar, gzip, sha256sum (or shasum on macOS) + +set -euo pipefail + +# ── Versions (keep in sync with openshift_with_stack.sh) ──────────────────── +CERT_MANAGER_VERSION="v1.13.0" +LOCAL_PATH_PROVISIONER_VERSION="v0.0.26" +KUBERAY_CHART_VERSION="1.2.2" + +OUTPUT_DIR="${OUTPUT_DIR:-./airgap-bundle-openshift}" + +# ── Argument parsing ───────────────────────────────────────────────────────── +while [[ $# -gt 0 ]]; do + case "$1" in + --output-dir) OUTPUT_DIR="$2"; shift 2 ;; + -h|--help) + cat <<'HELP' +prepare_airgap_bundle_openshift.sh — download all Splunk AI Platform install +artifacts needed by openshift_with_stack.sh into a self-contained tar.gz bundle +for air-gapped OpenShift deployments. + +USAGE + ./prepare_airgap_bundle_openshift.sh [OPTIONS] + +OPTIONS + --output-dir DIR Directory where the bundle is written. + Default: ./airgap-bundle-openshift + Env: OUTPUT_DIR + + -h, --help Show this help text. + +WHAT IS BUNDLED + manifests/ + cert-manager.yaml — cert-manager CRDs + controller + local-path-storage.yaml — Rancher local-path provisioner + + charts/ + opentelemetry-operator-*.tgz — OTel operator (version resolved at bundle time) + kuberay-operator-1.2.2.tgz — KubeRay operator (pinned) + + airgap-env.sh — Source this to set env-var overrides before a manual install + container-images.txt — Full list of images to mirror to your internal registry + bundle-versions.txt — Records all component versions for reproducibility + checksums.sha256 — SHA-256 checksums for every file in the bundle + +WHAT IS NOT BUNDLED (and why) + Container images + OpenShift uses oc mirror + ImageContentSourcePolicy / ImageDigestMirrorSet to + redirect pulls from public registries to your internal mirror. Run: + oc mirror --config=imageset-config.yaml file:///path/to/mirror + then push the mirror to your registry. See container-images.txt for the list. + + NFD / GPU Operator (OLM) + Install via OLM from a mirrored OperatorHub catalog: + oc mirror --config=imageset-config.yaml file:///path/to/mirror + Apply the resulting ImageContentSourcePolicy + CatalogSource, then create + Subscription objects as normal. + + k0s binary / yq — not applicable (OpenShift provides its own cluster) + MetalLB — not applicable (OpenShift uses Routes; MetalLB is k0s-only) + kube-prometheus-stack — not applicable (OpenShift ships its own monitoring stack) + NVIDIA device plugin manifest — not applicable (GPU Operator via OLM handles this) + GPU node OS packages — not applicable (GPU Operator manages driver lifecycle) + Model weights — stage separately via tools/artifacts_download_upload_scripts/ + +ENVIRONMENT VARIABLE OVERRIDES (set before running the installer manually) + These are exported automatically by install_from_airgap_bundle_openshift.sh. + You only need to set them manually if you extract the bundle yourself. + + CERT_MANAGER_MANIFEST_URL URL/path to cert-manager.yaml (file:// or https://) + LOCAL_PATH_MANIFEST_URL URL/path to local-path-storage.yaml + OTEL_CHART_PATH Local path to opentelemetry-operator .tgz + KUBERAY_CHART_PATH Local path to kuberay-operator .tgz + +EXAMPLES + # Basic bundle + ./prepare_airgap_bundle_openshift.sh + + # Custom output directory + ./prepare_airgap_bundle_openshift.sh --output-dir /mnt/transfer + + # Using env vars instead of flags + OUTPUT_DIR=/mnt/transfer ./prepare_airgap_bundle_openshift.sh + +NEXT STEPS AFTER BUNDLING + 1. Mirror container images listed in container-images.txt to your internal registry. + Update images.registry and images.* in your openshift-cluster-config.yaml. + 2. Stage model weights via tools/artifacts_download_upload_scripts/ (separate step). + 3. For NFD / GPU Operator: mirror OLM catalogs using oc mirror and apply + ImageContentSourcePolicy before running the installer. + 4. Copy the .tar.gz to the air-gapped install machine. + 5. Run: ./install_from_airgap_bundle_openshift.sh \ + --bundle airgap-bundle-openshift-.tar.gz \ + --config openshift-cluster-config.yaml + +HELP + exit 0 + ;; + *) echo "Unknown option: $1" >&2; echo "Run with --help for usage." >&2; exit 1 ;; + esac +done + +BUNDLE_TIMESTAMP="$(date +%Y%m%d-%H%M%S)" +BUNDLE_NAME="airgap-bundle-openshift-${BUNDLE_TIMESTAMP}" +STAGE_DIR="${OUTPUT_DIR}/${BUNDLE_NAME}" + +# ── Helpers ─────────────────────────────────────────────────────────────────── +log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] INFO $*"; } +warn() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] WARN $*" >&2; } +err() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] ERROR $*" >&2; exit 1; } + +require_cmd() { + command -v "$1" >/dev/null 2>&1 || err "Required tool not found: $1 — install it before running this script." +} + +sha256() { + if command -v sha256sum >/dev/null 2>&1; then + sha256sum "$1" | awk '{print $1}' + else + shasum -a 256 "$1" | awk '{print $1}' + fi +} + +download() { + local url="$1" dest="$2" + log "Downloading $(basename "$dest") ..." + if ! curl -fsSL --retry 3 --retry-delay 5 -o "$dest" "$url"; then + err "Download failed: $url" + fi +} + +# ── Pre-flight ───────────────────────────────────────────────────────────────── +require_cmd curl +require_cmd helm +require_cmd tar + +log "=== Splunk AI Platform — OpenShift Air-Gap Bundle Preparation ===" +log "Output directory : ${OUTPUT_DIR}" +log "Bundle name : ${BUNDLE_NAME}" +log "" +log "Component versions:" +log " cert-manager : ${CERT_MANAGER_VERSION}" +log " local-path-provisioner: ${LOCAL_PATH_PROVISIONER_VERSION}" +log " kuberay chart : ${KUBERAY_CHART_VERSION}" +log " otel chart : (resolved at bundle time)" +log "" + +mkdir -p \ + "${STAGE_DIR}/manifests" \ + "${STAGE_DIR}/charts" + +# ── 1. Static Kubernetes manifests ──────────────────────────────────────────── +log "--- Downloading static manifests ---" + +download \ + "https://github.com/cert-manager/cert-manager/releases/download/${CERT_MANAGER_VERSION}/cert-manager.yaml" \ + "${STAGE_DIR}/manifests/cert-manager.yaml" + +download \ + "https://raw.githubusercontent.com/rancher/local-path-provisioner/${LOCAL_PATH_PROVISIONER_VERSION}/deploy/local-path-storage.yaml" \ + "${STAGE_DIR}/manifests/local-path-storage.yaml" + +# ── 2. Helm charts ──────────────────────────────────────────────────────────── +log "--- Pulling Helm charts ---" + +helm repo add open-telemetry https://open-telemetry.github.io/opentelemetry-helm-charts 2>/dev/null || true +helm repo add kuberay https://ray-project.github.io/kuberay-helm/ 2>/dev/null || true +helm repo update open-telemetry kuberay + +# opentelemetry-operator — resolve latest version at bundle time +OTEL_CHART_VERSION="$(helm search repo open-telemetry/opentelemetry-operator \ + --output json | grep -o '"version":"[^"]*"' | head -1 | cut -d'"' -f4)" +[[ -z "${OTEL_CHART_VERSION}" ]] && err "Could not resolve opentelemetry-operator chart version." +log "Resolved opentelemetry-operator chart version: ${OTEL_CHART_VERSION}" + +helm pull open-telemetry/opentelemetry-operator \ + --version "${OTEL_CHART_VERSION}" \ + --destination "${STAGE_DIR}/charts" +echo "${OTEL_CHART_VERSION}" > "${STAGE_DIR}/charts/opentelemetry-operator.version" + +# kuberay — pinned +helm pull kuberay/kuberay-operator \ + --version "${KUBERAY_CHART_VERSION}" \ + --destination "${STAGE_DIR}/charts" + +# ── 3. Env-var override manifest ────────────────────────────────────────────── +log "--- Writing env-var override manifest ---" + +cat > "${STAGE_DIR}/airgap-env.sh" <<'ENVEOF' +# Source this file before running openshift_with_stack.sh in an air-gapped +# environment. It points every internet URL to the local bundle. +# +# Usage: +# source /path/to/bundle/airgap-env.sh +# CONFIG_FILE=./openshift-cluster-config.yaml ./openshift_with_stack.sh install + +# Set by install_from_airgap_bundle_openshift.sh to the extraction directory. +# Override here only if you extracted the bundle manually. +: "${AIRGAP_BUNDLE_DIR:?AIRGAP_BUNDLE_DIR must be set to the bundle extraction path}" + +# Static manifests — oc apply -f does not understand file://, so these are bare +# paths. The installer strips the file:// prefix automatically. +export CERT_MANAGER_MANIFEST_URL="file://${AIRGAP_BUNDLE_DIR}/manifests/cert-manager.yaml" +export LOCAL_PATH_MANIFEST_URL="file://${AIRGAP_BUNDLE_DIR}/manifests/local-path-storage.yaml" + +# Helm chart paths — installer uses these instead of remote repos when set. +export OTEL_CHART_PATH="${AIRGAP_BUNDLE_DIR}/charts/opentelemetry-operator-$(cat "${AIRGAP_BUNDLE_DIR}/charts/opentelemetry-operator.version").tgz" +export KUBERAY_CHART_PATH="${AIRGAP_BUNDLE_DIR}/charts/kuberay-operator-${KUBERAY_CHART_VERSION:-1.2.2}.tgz" + +# Signal to the installer that this is an air-gapped run. +export AIRGAP_MODE="true" +ENVEOF + +# ── 4. Container image list ─────────────────────────────────────────────────── +log "--- Generating container image list ---" + +cat > "${STAGE_DIR}/container-images.txt" <<'IMGEOF' +# Container images required by the Splunk AI Platform stack on OpenShift. +# Mirror ALL of these into your internal registry before running the installer. +# +# Recommended mirroring tool: oc mirror (OpenShift mirror registry) +# oc mirror --config=imageset-config.yaml file:///path/to/local-mirror +# +# Alternative using crane: +# while IFS= read -r img; do +# [[ "$img" =~ ^# ]] && continue +# [[ -z "$img" ]] && continue +# dest="your-internal-registry.example.com/${img##*/}" +# crane copy "$img" "$dest" +# done < container-images.txt +# +# After mirroring, update images.registry in your openshift-cluster-config.yaml +# to your internal registry prefix, and set each images.* field to the mirrored path. + +# ── Splunk ─────────────────────────────────────────────────────────────────── +# Set images.splunk.image / images.splunk.operatorImage in cluster config +splunk/splunk:10.2.0 +docker.io/splunk/splunk-operator:3.0.0 + +# ── Ray ────────────────────────────────────────────────────────────────────── +# Built internally — not on a public registry. +# Set images.ray.headImage and images.ray.workerImage in cluster config. +# Example (replace with your actual build tags): +# /ml-platform/ray/ray-head:build-953 +# /ml-platform/ray/ray-worker-gpu:build-953 + +# ── SAIA ───────────────────────────────────────────────────────────────────── +# Built internally — set images.saia.* in cluster config. +# Example: +# /ml-platform/saia/saia-api:build-v2-main-c3b489d +# /ml-platform/saia/saia-api-v2:build-v2-main-c3b489d +# /ml-platform/saia/saia-data-loader:build-v2-main-c3b489d + +# ── Weaviate ────────────────────────────────────────────────────────────────── +docker.io/semitechnologies/weaviate:stable-v1.28-007846a + +# ── KubeRay Operator ───────────────────────────────────────────────────────── +quay.io/kuberay/operator:v1.2.2 + +# ── OpenTelemetry ───────────────────────────────────────────────────────────── +docker.io/otel/opentelemetry-collector-contrib:0.122.1 + +# ── Fluent Bit ──────────────────────────────────────────────────────────────── +docker.io/fluent/fluent-bit:1.9.6 + +# ── Nginx ───────────────────────────────────────────────────────────────────── +docker.io/library/nginx:1.27-alpine + +# ── cert-manager (installed from manifest) ──────────────────────────────────── +# Exact image tags are embedded in the manifest. Extract them with: +# grep 'image:' manifests/cert-manager.yaml + +# ── local-path-provisioner ──────────────────────────────────────────────────── +# Patched in openshift_with_stack.sh to use ubi9-minimal as the helper pod. +# The provisioner itself uses the image baked into local-path-storage.yaml. +# Extract with: grep 'image:' manifests/local-path-storage.yaml +registry.access.redhat.com/ubi9/ubi-minimal:latest + +# ── OLM Operators (NFD + GPU Operator) ─────────────────────────────────────── +# These are NOT direct image references — they are OLM Subscriptions backed by +# OperatorHub catalog content. Mirror them with oc mirror: +# +# imageset-config.yaml example: +# kind: ImageSetConfiguration +# apiVersion: mirror.openshift.io/v1alpha2 +# mirror: +# operators: +# - catalog: registry.redhat.io/redhat/redhat-operator-index:v4.14 +# packages: +# - name: gpu-operator-certified +# - name: nfd +# +# Then apply the generated ImageContentSourcePolicy and CatalogSource. +# See: https://docs.openshift.com/container-platform/4.14/installing/disconnected_install/ + +# ── NOTE: Model weights (HuggingFace) ───────────────────────────────────────── +# Model weights (~60 GB total) are NOT container images. +# Use tools/artifacts_download_upload_scripts/ to stage them separately to S3/MinIO. +# Models: gemma-4-31b-it, gpt-oss-20b, all-minilm-l6-v2, bi-encoder, +# cross-encoder, e5-language-classifier, mbart-translator, +# pii-classifier, uae-large, xlm-roberta-language-classifier +IMGEOF + +# ── 5. Write version manifest ───────────────────────────────────────────────── +cat > "${STAGE_DIR}/bundle-versions.txt" < "${STAGE_DIR}/checksums.sha256" +log "Checksums written to ${STAGE_DIR}/checksums.sha256" + +# ── 7. Pack the bundle ──────────────────────────────────────────────────────── +log "--- Creating tar.gz bundle ---" +BUNDLE_TARBALL="${OUTPUT_DIR}/${BUNDLE_NAME}.tar.gz" +tar -czf "${BUNDLE_TARBALL}" -C "${OUTPUT_DIR}" "${BUNDLE_NAME}" + +BUNDLE_SIZE="$(du -sh "${BUNDLE_TARBALL}" | cut -f1)" +BUNDLE_SHA="$(sha256 "${BUNDLE_TARBALL}")" +log "" +log "=== Bundle ready ===" +log " File : ${BUNDLE_TARBALL}" +log " Size : ${BUNDLE_SIZE}" +log " SHA256: ${BUNDLE_SHA}" +log "" +log "Next steps:" +log " 1. Mirror the container images listed in:" +log " ${STAGE_DIR}/container-images.txt" +log " to your internal registry. Update images.* in your cluster config." +log " For NFD/GPU Operator, mirror OLM catalogs with oc mirror." +log "" +log " 2. Stage model weights (if not already staged) using:" +log " tools/artifacts_download_upload_scripts/" +log "" +log " 3. Copy ${BUNDLE_TARBALL} to the air-gapped install machine." +log "" +log " 4. On the install machine, run:" +log " ./install_from_airgap_bundle_openshift.sh \\" +log " --bundle ${BUNDLE_NAME}.tar.gz \\" +log " --config openshift-cluster-config.yaml" +log "" +log "Cleaning up staging directory..." +rm -rf "${STAGE_DIR}" From 9049743750a8391332450e3e8687556aa3c5b666 Mon Sep 17 00:00:00 2001 From: kbhos Date: Fri, 3 Jul 2026 16:10:22 +0530 Subject: [PATCH 15/80] fix codex comments --- tools/cluster_setup/openshift_with_stack.sh | 41 +++++++++++++++---- .../prepare_airgap_bundle_openshift.sh | 18 +++++--- 2 files changed, 46 insertions(+), 13 deletions(-) diff --git a/tools/cluster_setup/openshift_with_stack.sh b/tools/cluster_setup/openshift_with_stack.sh index a9aa8000..7227d973 100755 --- a/tools/cluster_setup/openshift_with_stack.sh +++ b/tools/cluster_setup/openshift_with_stack.sh @@ -260,6 +260,9 @@ ${yq_err}" MINIO_ROOT_PASSWORD=$(yq eval '.storage.objectStore.auth.rootPassword // ""' "${CONFIG_FILE}" 2>/dev/null || echo "") AI_STANDALONE_NAME=$(yq eval '.splunk.standaloneName // "splunk-standalone"' "${CONFIG_FILE}" 2>/dev/null || echo "splunk-standalone") + NFD_CATALOG_SOURCE=$(yq eval '.operators.nfd.catalogSource // "redhat-operators"' "${CONFIG_FILE}" 2>/dev/null || echo "redhat-operators") + GPU_CATALOG_SOURCE=$(yq eval '.operators.gpu.catalogSource // "certified-operators"' "${CONFIG_FILE}" 2>/dev/null || echo "certified-operators") + # Ingress domain: read from config if set, otherwise auto-detect from the cluster. # Used to create an OpenShift Route for SAIA so both browsers and in-cluster services # can reach it via a stable hostname. @@ -315,6 +318,13 @@ configure_images() { fi cp "${SPLUNK_AI_FILE}.original" "$SPLUNK_AI_FILE" + if [[ -f "${SPLUNK_OPERATOR_FILE}" ]]; then + if [[ ! -f "${SPLUNK_OPERATOR_FILE}.original" ]]; then + cp "${SPLUNK_OPERATOR_FILE}" "${SPLUNK_OPERATOR_FILE}.original" + fi + cp "${SPLUNK_OPERATOR_FILE}.original" "${SPLUNK_OPERATOR_FILE}" + fi + local operator_full ray_head_full ray_worker_full weaviate_full local saia_api_full saia_api_v2_full saia_dataloader_full local fluent_bit_full otel_collector_full nginx_full @@ -365,6 +375,19 @@ configure_images() { "${SED_INPLACE[@]}" "/name: RAY_VERSION/,/value:/ s|value:.*|value: ${RAY_RUNTIME_VERSION}|" "$SPLUNK_AI_FILE" "${SED_INPLACE[@]}" "s|image: .*splunk.*ai.*operator.*|image: ${operator_esc}|I" "$SPLUNK_AI_FILE" + if [[ -f "${SPLUNK_OPERATOR_FILE}" && -n "${SPLUNK_OPERATOR_IMAGE:-}" && "${SPLUNK_OPERATOR_IMAGE}" != "null" ]]; then + local splunk_op_full splunk_op_esc + splunk_op_full=$(build_image_url "$IMAGE_REGISTRY" "$SPLUNK_OPERATOR_IMAGE") + splunk_op_esc=$(echo "$splunk_op_full" | sed 's/[\/&]/\\&/g') + local splunk_ent_full splunk_ent_esc + splunk_ent_full=$(build_image_url "$IMAGE_REGISTRY" "$SPLUNK_IMAGE") + splunk_ent_esc=$(echo "$splunk_ent_full" | sed 's/[\/&]/\\&/g') + "${SED_INPLACE[@]}" "s|image: .*splunk.*operator.*|image: ${splunk_op_esc}|I" "${SPLUNK_OPERATOR_FILE}" + "${SED_INPLACE[@]}" "/name: RELATED_IMAGE_SPLUNK_ENTERPRISE/,/value:/ s|value:.*|value: ${splunk_ent_esc}|" "${SPLUNK_OPERATOR_FILE}" + log " ✓ Splunk Operator image: $splunk_op_full" + log " ✓ RELATED_IMAGE_SPLUNK_ENTERPRISE: $splunk_ent_full" + fi + log " ✓ RELATED_IMAGE_RAY_HEAD: $ray_head_full" log " ✓ RELATED_IMAGE_RAY_WORKER: $ray_worker_full" log " ✓ RELATED_IMAGE_WEAVIATE: $weaviate_full" @@ -473,7 +496,7 @@ install_nfd() { return 0 fi - oc apply -f - <<'EOF' + oc apply -f - <&1) + --wait=false --timeout=10m 2>&1) || true if echo "${otel_out}" | grep -q "x509: certificate\|failed to call webhook\|i/o timeout"; then warn "cert-manager webhook not ready yet, waiting 10s (${otel_retries}/6)..." sleep 10 diff --git a/tools/cluster_setup/prepare_airgap_bundle_openshift.sh b/tools/cluster_setup/prepare_airgap_bundle_openshift.sh index eca39d86..909ae1f3 100755 --- a/tools/cluster_setup/prepare_airgap_bundle_openshift.sh +++ b/tools/cluster_setup/prepare_airgap_bundle_openshift.sh @@ -284,15 +284,21 @@ docker.io/fluent/fluent-bit:1.9.6 # ── Nginx ───────────────────────────────────────────────────────────────────── docker.io/library/nginx:1.27-alpine -# ── cert-manager (installed from manifest) ──────────────────────────────────── -# Exact image tags are embedded in the manifest. Extract them with: -# grep 'image:' manifests/cert-manager.yaml +# ── cert-manager (images extracted from bundled manifest) ───────────────────── +IMGEOF + +grep -oP '(?<=image: )[^\s]+' "${STAGE_DIR}/manifests/cert-manager.yaml" 2>/dev/null | sort -u >> "${STAGE_DIR}/container-images.txt" || true + +cat >> "${STAGE_DIR}/container-images.txt" <<'IMGEOF' # ── local-path-provisioner ──────────────────────────────────────────────────── -# Patched in openshift_with_stack.sh to use ubi9-minimal as the helper pod. -# The provisioner itself uses the image baked into local-path-storage.yaml. -# Extract with: grep 'image:' manifests/local-path-storage.yaml +# The helper pod image is overridden to ubi-minimal; mirror both. registry.access.redhat.com/ubi9/ubi-minimal:latest +IMGEOF + +grep -oP '(?<=image: )[^\s]+' "${STAGE_DIR}/manifests/local-path-storage.yaml" 2>/dev/null | sort -u >> "${STAGE_DIR}/container-images.txt" || true + +cat >> "${STAGE_DIR}/container-images.txt" <<'IMGEOF' # ── OLM Operators (NFD + GPU Operator) ─────────────────────────────────────── # These are NOT direct image references — they are OLM Subscriptions backed by From 5d89bb3c89f8822b230b910905b4f49e0c848783 Mon Sep 17 00:00:00 2001 From: kbhos-splunk Date: Fri, 10 Jul 2026 11:56:32 +0530 Subject: [PATCH 16/80] fix(openshift): route GPU and CPU workers via ai-tier-node label, fix script issues (#126) * feat(openshift): route GPU/CPU workers via ai-tier-node label * resolve codex comments --- tools/cluster_setup/openshift_with_stack.sh | 155 ++++++++++---------- 1 file changed, 78 insertions(+), 77 deletions(-) diff --git a/tools/cluster_setup/openshift_with_stack.sh b/tools/cluster_setup/openshift_with_stack.sh index 7227d973..f7da1961 100755 --- a/tools/cluster_setup/openshift_with_stack.sh +++ b/tools/cluster_setup/openshift_with_stack.sh @@ -179,7 +179,7 @@ show_install_plan() { echo -e " 1. Preflight checks (oc login, tools, manifest files)" >&2 echo -e " 2. NFD Operator (OLM)" >&2 echo -e " 3. NVIDIA GPU Operator (OLM)" >&2 - echo -e " 4. Node labeling (splunk.ai/workload-type)" >&2 + echo -e " 4. Node labeling (splunk.ai/ai-tier-node)" >&2 echo -e " 5. local-path-provisioner + SELinux relabeling" >&2 echo -e " 6. cert-manager (Helm)" >&2 echo -e " 7. OpenTelemetry Operator (Helm)" >&2 @@ -684,7 +684,7 @@ EOF label_nodes() { log "Applying splunk.ai/* node labels (strategy: ${NODE_LABEL_STRATEGY})..." - local cpu_nodes=() gpu_nodes=() control_nodes=() + local ai_nodes=() control_nodes=() # Always label master/control-plane nodes while IFS= read -r node; do @@ -693,31 +693,20 @@ label_nodes() { case "${NODE_LABEL_STRATEGY}" in auto) - # GPU nodes: detected by nvidia.com/gpu.present=true (set by NVIDIA GPU Operator / NFD) + # AI-tier nodes: all worker nodes. GPU workers are further constrained to + # GPU-capable nodes by their nvidia.com/gpu resource request, not by a label. while IFS= read -r node; do - [[ -n "$node" ]] && gpu_nodes+=("$node") - done < <(oc get nodes -l nvidia.com/gpu.present=true,node-role.kubernetes.io/worker -o name 2>/dev/null | sed 's|node/||') - - # CPU nodes: worker nodes without GPU label - while IFS= read -r node; do - [[ -n "$node" ]] && cpu_nodes+=("$node") - done < <(oc get nodes -l '!nvidia.com/gpu.present,node-role.kubernetes.io/worker' -o name 2>/dev/null | sed 's|node/||') + [[ -n "$node" ]] && ai_nodes+=("$node") + done < <(oc get nodes -l node-role.kubernetes.io/worker -o name 2>/dev/null | sed 's|node/||') ;; manual) - local cpu_count gpu_count - cpu_count=$(yq eval '.openshift.nodes.cpu | length' "${CONFIG_FILE}" 2>/dev/null || echo "0") - gpu_count=$(yq eval '.openshift.nodes.gpu | length' "${CONFIG_FILE}" 2>/dev/null || echo "0") + local node_count + node_count=$(yq eval '.openshift.nodes | length' "${CONFIG_FILE}" 2>/dev/null || echo "0") local i=0 - while [[ $i -lt $cpu_count ]]; do - local n; n=$(yq eval ".openshift.nodes.cpu[$i]" "${CONFIG_FILE}" 2>/dev/null || echo "") - [[ -n "$n" && "$n" != "null" ]] && cpu_nodes+=("$n") - i=$((i+1)) - done - i=0 - while [[ $i -lt $gpu_count ]]; do - local n; n=$(yq eval ".openshift.nodes.gpu[$i]" "${CONFIG_FILE}" 2>/dev/null || echo "") - [[ -n "$n" && "$n" != "null" ]] && gpu_nodes+=("$n") + while [[ $i -lt $node_count ]]; do + local n; n=$(yq eval ".openshift.nodes[$i]" "${CONFIG_FILE}" 2>/dev/null || echo "") + [[ -n "$n" && "$n" != "null" ]] && ai_nodes+=("$n") i=$((i+1)) done ;; @@ -736,35 +725,29 @@ label_nodes() { --overwrite done - # Label CPU worker nodes - for node in "${cpu_nodes[@]}"; do - log " Labeling CPU worker node: ${node}" - oc label node "${node}" \ - splunk.ai/node-role=worker \ - splunk.ai/workload-type=cpu \ - splunk.ai/instance-type=cpu-worker \ - --overwrite - done - - # Label GPU worker nodes - for node in "${gpu_nodes[@]}"; do - log " Labeling GPU worker node: ${node}" + # Label AI-tier worker nodes — every node gets splunk.ai/ai-tier-node=true. + # Both cpuScheduler and gpuScheduler select on this single label; GPU workers are + # further constrained to GPU-capable nodes by their nvidia.com/gpu resource request, + # so CPU and GPU workloads share the same node pool without a cpu/gpu label split. + for node in "${ai_nodes[@]:+${ai_nodes[@]}}"; do + log " Labeling AI-tier worker node: ${node}" oc label node "${node}" \ splunk.ai/node-role=worker \ - splunk.ai/workload-type=gpu \ - splunk.ai/instance-type=gpu-worker \ + splunk.ai/ai-tier-node=true \ --overwrite - # Taint GPU nodes so non-GPU workloads don't land on them - oc adm taint node "${node}" nvidia.com/gpu=true:NoSchedule --overwrite 2>/dev/null || true + # Remove any lingering nvidia.com/gpu taint — a node that previously ran as a GPU + # worker retains the taint across in-place reinstalls; without this, CPU workloads + # selecting ai-tier-node would stay Pending on it. + oc adm taint node "${node}" nvidia.com/gpu=true:NoSchedule- 2>/dev/null || true done # Verify labeled nodes have the label — scope check to listed nodes in manual mode # (auto mode checks all workers; manual mode only checks what the user explicitly listed) local unlabeled="" if [[ "${NODE_LABEL_STRATEGY}" == "manual" ]]; then - for node in "${cpu_nodes[@]}" "${gpu_nodes[@]}"; do + for node in "${ai_nodes[@]:+${ai_nodes[@]}}"; do local val - val=$(oc get node "${node}" -o jsonpath='{.metadata.labels.splunk\.ai/workload-type}' 2>/dev/null || echo "") + val=$(oc get node "${node}" -o jsonpath='{.metadata.labels.splunk\.ai/ai-tier-node}' 2>/dev/null || echo "") [[ -z "${val}" ]] && unlabeled+="${node}"$'\n' done else @@ -773,23 +756,21 @@ label_nodes() { import json,sys data=json.load(sys.stdin) for n in data['items']: - if 'splunk.ai/workload-type' not in n['metadata']['labels']: + if 'splunk.ai/ai-tier-node' not in n['metadata']['labels']: print(n['metadata']['name']) " 2>/dev/null || echo "") fi if [[ -n "${unlabeled}" ]]; then - err "Worker node(s) still missing splunk.ai/workload-type after labeling: + err "Worker node(s) still missing splunk.ai/ai-tier-node after labeling: $(echo "${unlabeled}" | sed 's/^/ /') -If using nodeLabelStrategy: auto, check that the NVIDIA GPU Operator is installed -and nodes have nvidia.com/gpu.present=true, or switch to nodeLabelStrategy: manual -and list nodes explicitly under openshift.nodes.cpu / openshift.nodes.gpu in the config." +If using nodeLabelStrategy: auto, check that worker nodes exist, or switch to +nodeLabelStrategy: manual and list nodes explicitly under openshift.nodes in the config." fi log " ✓ Control-plane nodes: ${#control_nodes[@]}" - log " ✓ CPU worker nodes: ${#cpu_nodes[@]}" - log " ✓ GPU worker nodes: ${#gpu_nodes[@]}" + log " ✓ AI-tier worker nodes: ${#ai_nodes[@]}" log "Node labeling complete" } @@ -894,13 +875,11 @@ install_local_path_provisioner() { # busybox lacks chcon; use ubi-minimal which ships selinux-utils so chcon runs. # container_file_t allows any container to read/write regardless of MCS categories. # Without this, local-path-provisioner creates dirs with var_t which blocks all containers. - oc patch configmap local-path-config -n local-path-storage --type=merge -p "$(cat <<'PATCH' -{ - "data": { - "setup": "#!/bin/sh\nset -eu\nmkdir -m 0777 -p \"$VOL_DIR\"\nif command -v chcon >/dev/null 2>&1; then\n chcon -Rt container_file_t -l s0 \"$VOL_DIR\" 2>/dev/null || true\nfi\n" -} -PATCH - )" + local _patch_file; _patch_file=$(mktemp /tmp/local-path-patch-XXXXXX.json) + printf '%s' '{"data":{"setup":"#!/bin/sh\nset -eu\nmkdir -m 0777 -p \"$VOL_DIR\"\nif command -v chcon >/dev/null 2>&1; then\n chcon -Rt container_file_t -l s0 \"$VOL_DIR\" 2>/dev/null || true\nfi\n"}}' \ + > "${_patch_file}" + oc patch configmap local-path-config -n local-path-storage --type=merge --patch-file="${_patch_file}" + rm -f "${_patch_file}" # Restart the provisioner so it picks up the new helper pod template oc rollout restart deployment local-path-provisioner -n local-path-storage @@ -1050,22 +1029,39 @@ ensure_ecr_pull_secret() { --namespace="${ns}" \ --dry-run=client -o yaml | oc apply -f - - # Append ecr-registry-secret to the default SA only if not already present. - # Using JSON patch add rather than a merge patch to avoid overwriting existing pull secrets. - if ! oc get serviceaccount default -n "${ns}" -o jsonpath='{.imagePullSecrets[*].name}' 2>/dev/null | grep -qw ecr-registry-secret; then - oc patch serviceaccount default -n "${ns}" --type=json \ - -p='[{"op":"add","path":"/imagePullSecrets","value":[]}]' 2>/dev/null || true - oc patch serviceaccount default -n "${ns}" --type=json \ - -p='[{"op":"add","path":"/imagePullSecrets/-","value":{"name":"ecr-registry-secret"}}]' 2>/dev/null || true - fi + # Append the ECR pull secret to the default SA without dropping any it already has + # (OpenShift auto-injects default-dockercfg-* builder/pull secrets there). + add_ecr_pull_secret_to_sa default "${ns}" log " ✓ ecr-registry-secret created in ${ns}" done # Also patch the operator SA specifically - oc patch serviceaccount splunk-ai-operator-controller-manager \ - -n splunk-ai-operator-system \ - -p '{"imagePullSecrets": [{"name": "ecr-registry-secret"}]}' 2>/dev/null || true + add_ecr_pull_secret_to_sa splunk-ai-operator-controller-manager splunk-ai-operator-system +} + +# Append ecr-registry-secret to a service account's imagePullSecrets, preserving +# any existing entries. No-op if the secret is already listed, so reruns are safe. +add_ecr_pull_secret_to_sa() { + local sa="$1" ns="$2" + + # Already present? nothing to do. + if oc get serviceaccount "${sa}" -n "${ns}" \ + -o jsonpath='{.imagePullSecrets[*].name}' 2>/dev/null \ + | tr ' ' '\n' | grep -qx "ecr-registry-secret"; then + return 0 + fi + + # If the SA has no imagePullSecrets yet, a merge patch is enough. Otherwise use a + # JSON add patch to append to the existing array without replacing it. + if oc get serviceaccount "${sa}" -n "${ns}" \ + -o jsonpath='{.imagePullSecrets}' 2>/dev/null | grep -q '\['; then + oc patch serviceaccount "${sa}" -n "${ns}" --type=json \ + -p '[{"op":"add","path":"/imagePullSecrets/-","value":{"name":"ecr-registry-secret"}}]' 2>/dev/null || true + else + oc patch serviceaccount "${sa}" -n "${ns}" \ + -p '{"imagePullSecrets": [{"name": "ecr-registry-secret"}]}' 2>/dev/null || true + fi } # ====== INSTALL SPLUNK AI OPERATOR ====== @@ -1452,16 +1448,11 @@ ${svc_template_yaml}${storage_yaml} imageRegistry: "${WORKER_IMAGE_REGISTRY}" cpuScheduler: nodeSelector: - splunk.ai/workload-type: cpu + splunk.ai/ai-tier-node: "true" tolerations: ${cpu_tolerations_inline} gpuScheduler: nodeSelector: - splunk.ai/workload-type: gpu - tolerations: - - key: "nvidia.com/gpu" - operator: "Equal" - value: "true" - effect: "NoSchedule" + splunk.ai/ai-tier-node: "true" splunkConfiguration: endpoint: http://${AI_STANDALONE_NAME}-standalone-service.${AI_NS}.svc.cluster.local:8089 secretRef: @@ -1487,7 +1478,12 @@ YAML # (Splunk's setup page validates connectivity from the server side). # NodePort alone doesn't work when node IPs are not externally routable. create_saia_route() { - if [[ -z "$INGRESS_DOMAIN" ]]; then + # Re-attempt ingress domain detection in case it was unavailable during load_config. + if [[ -z "${INGRESS_DOMAIN:-}" ]]; then + INGRESS_DOMAIN=$(oc get ingresscontroller default -n openshift-ingress-operator \ + -o jsonpath='{.status.domain}' 2>/dev/null || echo "") + fi + if [[ -z "${INGRESS_DOMAIN:-}" ]]; then warn "Could not determine ingress domain — skipping SAIA Route creation" warn "Create it manually: oc expose svc/${AI_PLATFORM_NAME}-saia-saia-service -n ${AI_NS}" return 0 @@ -1731,10 +1727,15 @@ main_delete() { oc delete namespace openshift-nfd --timeout=60s 2>/dev/null || true # ── 11. Node labels and taints added by label_nodes() ── + # Match both the current ai-tier-node label and the legacy workload-type label so + # teardown cleans up stacks installed before the single-label refactor. Also strip + # the nvidia.com/gpu taint older installs applied to GPU worker nodes. log "Removing splunk.ai/* node labels and GPU taint..." - for node in $(oc get nodes -l 'splunk.ai/workload-type' -o name 2>/dev/null); do - oc label "${node}" splunk.ai/workload-type- 2>/dev/null || true - oc taint "${node}" nvidia.com/gpu=true:NoSchedule- 2>/dev/null || true + for node in $( { oc get nodes -l 'splunk.ai/ai-tier-node' -o name 2>/dev/null; \ + oc get nodes -l 'splunk.ai/workload-type' -o name 2>/dev/null; \ + } | sort -u ); do + oc label "${node}" splunk.ai/ai-tier-node- splunk.ai/workload-type- 2>/dev/null || true + oc adm taint "${node}" nvidia.com/gpu=true:NoSchedule- 2>/dev/null || true done # ── 12. SCC grants added during install ── From 208aa7f30dfdad424f6f4a4c701d5bdad3751b1e Mon Sep 17 00:00:00 2001 From: kbhos-splunk Date: Fri, 10 Jul 2026 12:01:26 +0530 Subject: [PATCH 17/80] feat: Ai pod model downloads (#118) --- .../model_artifacts_configs.yaml | 4 +- tools/cluster_setup/openshift_with_stack.sh | 1012 ++++++++++++++++- 2 files changed, 963 insertions(+), 53 deletions(-) diff --git a/tools/artifacts_download_upload_scripts/model_artifacts_configs.yaml b/tools/artifacts_download_upload_scripts/model_artifacts_configs.yaml index ac86bc56..621f80e1 100755 --- a/tools/artifacts_download_upload_scripts/model_artifacts_configs.yaml +++ b/tools/artifacts_download_upload_scripts/model_artifacts_configs.yaml @@ -33,8 +33,8 @@ artifact-configs: hf-url: https://huggingface.co/facebook/mbart-large-50-many-to-many-mmt is-a-gated-model: false - - artifact-id: gemma-4-31b-it - hf-url: https://huggingface.co/google/gemma-4-31B-it + - artifact-id: gemma-4-31b-it-qat-w4a16-ct + hf-url: https://huggingface.co/google/gemma-4-31B-it-qat-w4a16-ct is-a-gated-model: false - artifact-id: pii-classifier diff --git a/tools/cluster_setup/openshift_with_stack.sh b/tools/cluster_setup/openshift_with_stack.sh index f7da1961..74b74597 100755 --- a/tools/cluster_setup/openshift_with_stack.sh +++ b/tools/cluster_setup/openshift_with_stack.sh @@ -156,6 +156,92 @@ wait_for_dependency() { Resolve the issue, then re-run the installer." } +# ====== RESOLVE MODEL STAGING ====== +# Prompts the user interactively whether to download & stage models. +# In silent/airgap mode the prompt is skipped and MODEL_STAGING_ENABLED is unchanged. +resolve_model_staging() { + # Airgap: models must be pre-staged; staging via this script is not possible. + if [[ "${AIRGAP_MODE:-false}" == "true" ]]; then + log "AIRGAP_MODE=true — model staging skipped (models must be pre-staged in object store)." + MODEL_STAGING_ENABLED="false" + return 0 + fi + + # Already explicitly set in config or env — honour it and skip the prompt. + if [[ "${MODEL_STAGING_ENABLED}" == "true" || "${MODEL_STAGING_ENABLED}" == "false" ]]; then + log "MODEL_STAGING_ENABLED=${MODEL_STAGING_ENABLED} (from config/env)" + return 0 + fi + + # Silent install: cannot prompt — default to false. + if [[ "${SILENT_INSTALL:-false}" == "true" ]]; then + log "SILENT_INSTALL=true — model staging defaulting to false (set storage.modelStaging.enabled=true in config to enable)." + MODEL_STAGING_ENABLED="false" + return 0 + fi + + echo -e "\n \033[1mModel artifact staging:\033[0m" >&2 + echo -e " Download model weights from HuggingFace and upload them to the object store?" >&2 + echo -e " (Requires HF_TOKEN and object store credentials. Type 'yes' to enable.)" >&2 + local answer + read -r -p " Enable model staging? [yes/no]: " answer + if [[ "${answer}" == "yes" ]]; then + MODEL_STAGING_ENABLED="true" + log "Model staging enabled by user." + else + MODEL_STAGING_ENABLED="false" + log "Model staging skipped by user." + fi +} + +# ====== SUPPORTED ACCELERATOR TYPES ====== +readonly SUPPORTED_ACCELERATORS=("L40S" "H100") + +# ====== RESOLVE ACCELERATOR TYPE ====== +# Normalizes and validates DEFAULT_ACCELERATOR. Prompts interactively if missing +# or invalid (unless SILENT_INSTALL is true, in which case it errors out). +resolve_accelerator_type() { + # Normalize to uppercase for comparison, store normalized value. + local _raw="${DEFAULT_ACCELERATOR:-}" + if [[ -n "${_raw}" && "${_raw}" != "null" ]]; then + DEFAULT_ACCELERATOR="${_raw^^}" + fi + + # Validate against supported list. + local _valid=false + for _t in "${SUPPORTED_ACCELERATORS[@]}"; do + [[ "${DEFAULT_ACCELERATOR}" == "${_t}" ]] && _valid=true && break + done + + if ${_valid}; then + log "Accelerator type: ${DEFAULT_ACCELERATOR}" + return 0 + fi + + # Not valid — prompt interactively or error in silent mode. + if [[ "${SILENT_INSTALL:-false}" == "true" ]]; then + err "aiPlatform.defaultAcceleratorType '${DEFAULT_ACCELERATOR:-}' is not supported. Must be one of: ${SUPPORTED_ACCELERATORS[*]}" + fi + + echo -e "\n \033[1mAccelerator type:\033[0m" >&2 + echo -e " Supported types: ${SUPPORTED_ACCELERATORS[*]}" >&2 + local answer="" + while true; do + read -r -p " Enter accelerator type [${SUPPORTED_ACCELERATORS[*]}]: " answer + answer="${answer^^}" + local _v=false + for _t in "${SUPPORTED_ACCELERATORS[@]}"; do + [[ "${answer}" == "${_t}" ]] && _v=true && break + done + if ${_v}; then + DEFAULT_ACCELERATOR="${answer}" + log "Accelerator type set to: ${DEFAULT_ACCELERATOR}" + return 0 + fi + echo -e " \033[1;31mInvalid choice '${answer}'. Must be one of: ${SUPPORTED_ACCELERATORS[*]}\033[0m" >&2 + done +} + # ====== SHOW INSTALL PLAN ====== show_install_plan() { echo -e "\n\033[1;34m╔══════════════════════════════════════════════════════════╗\033[0m" >&2 @@ -171,28 +257,39 @@ show_install_plan() { echo -e " \033[1mOperator image :\033[0m ${OPERATOR_IMAGE}" >&2 echo -e " \033[1mImage registry :\033[0m ${IMAGE_REGISTRY:-}" >&2 echo -e " \033[1mECR enabled :\033[0m ${ECR_ENABLED}" >&2 + echo -e " \033[1mAir-gap mode :\033[0m ${AIRGAP_MODE:-false}" >&2 echo "" >&2 echo -e " \033[1mObject store :\033[0m type=${OBJ_STORE_TYPE} bucket=${OBJ_STORE_BUCKET:-}" >&2 echo -e " \033[1mObject endpoint :\033[0m ${OBJ_STORE_ENDPOINT:-}" >&2 + echo -e " \033[1mModel staging :\033[0m ${MODEL_STAGING_ENABLED}" >&2 echo "" >&2 echo -e " \033[1mSteps that will run:\033[0m" >&2 + echo -e " 0. Model artifact staging (HuggingFace → object store)" >&2 + if [[ "${MODEL_STAGING_ENABLED}" != "true" ]]; then + echo -e " [SKIPPED — modelStaging.enabled=false]" >&2 + elif [[ "${AIRGAP_MODE:-false}" == "true" ]]; then + echo -e " [SKIPPED — AIRGAP_MODE=true, models must be pre-staged]" >&2 + fi echo -e " 1. Preflight checks (oc login, tools, manifest files)" >&2 echo -e " 2. NFD Operator (OLM)" >&2 echo -e " 3. NVIDIA GPU Operator (OLM)" >&2 echo -e " 4. Node labeling (splunk.ai/ai-tier-node)" >&2 echo -e " 5. local-path-provisioner + SELinux relabeling" >&2 - echo -e " 6. cert-manager (Helm)" >&2 + echo -e " 6. cert-manager" >&2 echo -e " 7. OpenTelemetry Operator (Helm)" >&2 echo -e " 8. KubeRay Operator (Helm)" >&2 - echo -e " 9. ECR pull secrets" >&2 + echo -e " 9. Image pull secrets (ECR / DockerHub / GCR / ACR / custom)" >&2 echo -e " 10. Splunk AI Operator" >&2 echo -e " 11. Splunk Operator" >&2 echo -e " 12. Splunk Standalone CR" >&2 echo -e " 13. AIPlatform CR" >&2 echo "" >&2 - if [[ "${AUTO_APPROVE:-false}" == "true" ]]; then - log "AUTO_APPROVE=true — skipping confirmation." + if [[ "${SILENT_INSTALL:-false}" == "true" ]]; then + echo -e " \033[1;33m⚠ SILENT INSTALL — no interactive prompts.\033[0m" >&2 + echo -e " The config file is assumed to have been reviewed and is correct." >&2 + echo -e " The steps listed above will run automatically. Press Ctrl-C within 5 s to abort." >&2 + sleep 5 return 0 fi @@ -220,6 +317,9 @@ ${yq_err}" AI_NS=$(yq eval '.kubernetes.namespace // "ai-platform"' "${CONFIG_FILE}" 2>/dev/null || echo "ai-platform") IMAGE_REGISTRY=$(yq eval '.images.registry // ""' "${CONFIG_FILE}" 2>/dev/null || echo "") + # Set to "true" only for plain-HTTP (no-TLS) registries such as a local mirror. + # Leave false (default) for ECR, Docker Hub, Harbor, or any HTTPS registry. + IMAGE_REGISTRY_INSECURE="$(yq eval '.images.registryInsecure // "false"' "$CONFIG_FILE" 2>/dev/null || echo "false")" OPERATOR_IMAGE=$(yq eval '.images.operator.image // ""' "${CONFIG_FILE}" 2>/dev/null || echo "") RAY_HEAD_IMAGE=$(yq eval '.images.ray.headImage // ""' "${CONFIG_FILE}" 2>/dev/null || echo "") RAY_WORKER_IMAGE=$(yq eval '.images.ray.workerImage // ""' "${CONFIG_FILE}" 2>/dev/null || echo "") @@ -234,8 +334,15 @@ ${yq_err}" NGINX_IMAGE=$(yq eval '.images.nginx.image // "docker.io/library/nginx:1.27-alpine"' "${CONFIG_FILE}" 2>/dev/null || echo "docker.io/library/nginx:1.27-alpine") MODEL_VERSION=$(yq eval '.operators.ray.modelVersion // ""' "${CONFIG_FILE}" 2>/dev/null || echo "") RAY_RUNTIME_VERSION=$(yq eval '.operators.ray.rayVersion // "2.44.0"' "${CONFIG_FILE}" 2>/dev/null || echo "2.44.0") - SPLUNK_AI_FILE=$(yq eval '.files.aiPlatform // "./artifacts.yaml"' "${CONFIG_FILE}" 2>/dev/null || echo "./artifacts.yaml") - SPLUNK_OPERATOR_FILE=$(yq eval '.files.splunkOperator // "./splunk-operator-cluster.yaml"' "${CONFIG_FILE}" 2>/dev/null || echo "./splunk-operator-cluster.yaml") + local _config_dir + _config_dir="$(cd "$(dirname "${CONFIG_FILE}")" && pwd)" + _resolve_manifest() { + local p="$1" + # Absolute paths are used as-is; relative paths are anchored to the config file's directory. + [[ "${p}" = /* ]] && echo "${p}" || echo "${_config_dir}/${p#./}" + } + SPLUNK_AI_FILE=$(_resolve_manifest "$(yq eval '.files.aiPlatform // "./artifacts.yaml"' "${CONFIG_FILE}" 2>/dev/null || echo "./artifacts.yaml")") + SPLUNK_OPERATOR_FILE=$(_resolve_manifest "$(yq eval '.files.splunkOperator // "./splunk-operator-cluster.yaml"' "${CONFIG_FILE}" 2>/dev/null || echo "./splunk-operator-cluster.yaml")") # OpenShift-specific # Whether to grant the operator service account privileged SCC. @@ -247,6 +354,9 @@ ${yq_err}" ECR_ENABLED=$(yq eval '.ecr.enabled // "false"' "${CONFIG_FILE}" 2>/dev/null || echo "false") ECR_ACCOUNT=$(yq eval '.ecr.account // ""' "${CONFIG_FILE}" 2>/dev/null || echo "") ECR_REGION=$(yq eval '.ecr.region // "us-east-2"' "${CONFIG_FILE}" 2>/dev/null || echo "us-east-2") + # S3 bucket region — may differ from ECR_REGION when ECR and S3 are in different regions. + # Defaults to ecr.region for backwards compatibility when not explicitly set. + OBJ_STORE_REGION=$(yq eval ".storage.objectStore.region // \"${ECR_REGION}\"" "${CONFIG_FILE}" 2>/dev/null || echo "${ECR_REGION}") AI_PLATFORM_NAME=$(yq eval '.aiPlatform.name // "openshift-ai-platform"' "${CONFIG_FILE}" 2>/dev/null || echo "openshift-ai-platform") DEFAULT_ACCELERATOR=$(yq eval '.aiPlatform.defaultAcceleratorType // ""' "${CONFIG_FILE}" 2>/dev/null || echo "") @@ -255,11 +365,35 @@ ${yq_err}" VECTORDB_SIZE=$(yq eval '.storage.vectorDbSize // "50Gi"' "${CONFIG_FILE}" 2>/dev/null || echo "50Gi") OBJ_STORE_TYPE=$(yq eval '.storage.objectStore.type // "minio"' "${CONFIG_FILE}" 2>/dev/null || echo "minio") OBJ_STORE_BUCKET=$(yq eval '.storage.objectStore.bucket // ""' "${CONFIG_FILE}" 2>/dev/null || echo "") + OBJ_STORE_BUCKET="$(printf '%s' "${OBJ_STORE_BUCKET}" | tr '[:upper:]' '[:lower:]')" OBJ_STORE_ENDPOINT=$(yq eval '.storage.objectStore.endpoint // ""' "${CONFIG_FILE}" 2>/dev/null || echo "") MINIO_ROOT_USER=$(yq eval '.storage.objectStore.auth.rootUser // ""' "${CONFIG_FILE}" 2>/dev/null || echo "") MINIO_ROOT_PASSWORD=$(yq eval '.storage.objectStore.auth.rootPassword // ""' "${CONFIG_FILE}" 2>/dev/null || echo "") AI_STANDALONE_NAME=$(yq eval '.splunk.standaloneName // "splunk-standalone"' "${CONFIG_FILE}" 2>/dev/null || echo "splunk-standalone") + # Air-gap mode: read from YAML (cluster.airgap: true) and allow env var override. + local _yaml_airgap + _yaml_airgap=$(yq eval '.cluster.airgap // "false"' "${CONFIG_FILE}" 2>/dev/null || echo "false") + [[ "${_yaml_airgap}" == "null" ]] && _yaml_airgap="false" + if [[ "${AIRGAP_MODE:-}" == "true" ]]; then + : # env var wins + elif [[ "${_yaml_airgap}" == "true" ]]; then + export AIRGAP_MODE="true" + else + export AIRGAP_MODE="false" + fi + + # Model staging + MODEL_STAGING_ENABLED="$(yq eval '.storage.modelStaging.enabled' "${CONFIG_FILE}" 2>/dev/null || echo "null")" + [[ "${MODEL_STAGING_ENABLED}" == "null" || -z "${MODEL_STAGING_ENABLED}" ]] && MODEL_STAGING_ENABLED="false" + + # ImagePullSecrets configuration + IMAGE_PULL_SECRETS_ECR_ENABLED=$(yq eval '.imagePullSecrets.autoCreateECR // "false"' "${CONFIG_FILE}" 2>/dev/null || echo "false") + IMAGE_PULL_SECRETS_DOCKERHUB_ENABLED=$(yq eval '.imagePullSecrets.dockerHub.enabled // "false"' "${CONFIG_FILE}" 2>/dev/null || echo "false") + IMAGE_PULL_SECRETS_GCR_ENABLED=$(yq eval '.imagePullSecrets.gcr.enabled // "false"' "${CONFIG_FILE}" 2>/dev/null || echo "false") + IMAGE_PULL_SECRETS_ACR_ENABLED=$(yq eval '.imagePullSecrets.acr.enabled // "false"' "${CONFIG_FILE}" 2>/dev/null || echo "false") + IMAGE_PULL_SECRETS_CUSTOM_ENABLED=$(yq eval '.imagePullSecrets.custom.enabled // "false"' "${CONFIG_FILE}" 2>/dev/null || echo "false") + NFD_CATALOG_SOURCE=$(yq eval '.operators.nfd.catalogSource // "redhat-operators"' "${CONFIG_FILE}" 2>/dev/null || echo "redhat-operators") GPU_CATALOG_SOURCE=$(yq eval '.operators.gpu.catalogSource // "certified-operators"' "${CONFIG_FILE}" 2>/dev/null || echo "certified-operators") @@ -275,15 +409,28 @@ ${yq_err}" -o jsonpath='{.status.domain}' 2>/dev/null || echo "") fi - log "Configuration loaded: namespace=${AI_NS}, accelerator=${DEFAULT_ACCELERATOR}" + log "Configuration loaded: namespace=${AI_NS}, accelerator=${DEFAULT_ACCELERATOR}, airgap=${AIRGAP_MODE:-false}, modelStaging=${MODEL_STAGING_ENABLED}" +} + +# ====== PLACEHOLDER CREDENTIAL GUARD ====== +# True if objectStore.auth values are still obvious template text. Non-empty +# placeholders pass the length check and get applied into minio-credentials, +# causing SAIA to crash at startup with InvalidAccessKeyId. +object_store_auth_looks_like_placeholder() { + case "${MINIO_ROOT_USER}${MINIO_ROOT_PASSWORD}" in + *\<*|*\>*) return 0 ;; + *CHANGEME*|*changeme*) return 0 ;; + esac + return 1 } # ====== IMAGE HELPERS ====== build_image_url() { local registry="$1" local image_path="$2" - # If the image is already fully qualified (contains a registry host) return as-is - if [[ "$image_path" =~ ^([a-zA-Z0-9.-]+\.[a-zA-Z]{2,}|[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+(:[0-9]+)?)/.*:.+ ]]; then + # Treat as fully-qualified if image starts with a registry host. + # Recognised forms: domain.tld/... , domain.tld:port/... , IP/... , IP:port/... + if [[ "$image_path" =~ ^([a-zA-Z0-9.-]+\.[a-zA-Z]{2,}(:[0-9]+)?|[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+(:[0-9]+)?)/.*:.+ ]]; then echo "$image_path" return 0 fi @@ -400,15 +547,293 @@ configure_images() { log " ✓ Operator image: $operator_full" log " ✓ MODEL_VERSION: $MODEL_VERSION" log " ✓ RAY_VERSION: $RAY_RUNTIME_VERSION" + + # Patch splunk-operator-cluster.yaml with images from config. + # Without this, the Splunk Operator pod and all Splunk Standalone pods use the + # account-specific ECR URLs baked into the committed manifest file. + if [[ -f "${SPLUNK_OPERATOR_FILE}" ]]; then + log "Patching image references in ${SPLUNK_OPERATOR_FILE}..." + if [[ ! -f "${SPLUNK_OPERATOR_FILE}.original" ]]; then + cp "$SPLUNK_OPERATOR_FILE" "${SPLUNK_OPERATOR_FILE}.original" + fi + cp "${SPLUNK_OPERATOR_FILE}.original" "$SPLUNK_OPERATOR_FILE" + + local splunk_full splunk_esc splunk_op_full splunk_op_esc + splunk_full=$(build_image_url "$IMAGE_REGISTRY" "$SPLUNK_IMAGE") + splunk_op_full=$(build_image_url "$IMAGE_REGISTRY" "$SPLUNK_OPERATOR_IMAGE") + splunk_esc=$(echo "$splunk_full" | sed 's/[\/&]/\\&/g') + splunk_op_esc=$(echo "$splunk_op_full" | sed 's/[\/&]/\\&/g') + + "${SED_INPLACE[@]}" "/name: RELATED_IMAGE_SPLUNK_ENTERPRISE/,/value:/ s|value:.*|value: ${splunk_esc}|" "$SPLUNK_OPERATOR_FILE" + "${SED_INPLACE[@]}" "s|image: .*splunk.*operator.*|image: ${splunk_op_esc}|I" "$SPLUNK_OPERATOR_FILE" + + log " ✓ RELATED_IMAGE_SPLUNK_ENTERPRISE: $splunk_full" + log " ✓ Splunk Operator image: $splunk_op_full" + else + warn "Splunk Operator manifest not found at ${SPLUNK_OPERATOR_FILE} — skipping image patch" + fi +} + +# ====== PREFLIGHT HELPER PRINTERS ====== +pf_header() { echo -e "\n\033[1;34m ── $* ──\033[0m" >&2; } +pf_ok() { echo -e " \033[1;32m✔\033[0m $*" >&2; } +pf_warn() { echo -e " \033[1;33m⚠\033[0m $*" >&2; } +pf_fail() { echo -e " \033[1;31m✖\033[0m $*" >&2; PREFLIGHT_FAILURES=$(( ${PREFLIGHT_FAILURES:-0} + 1 )); } + +# ====== PREFLIGHT: REGISTRY REACHABILITY CHECK ====== +# Verifies the configured image registry is reachable and credentials work +# BEFORE any install work begins, so a bad registry config fails fast with a +# clear message instead of surfacing as ImagePullBackOff minutes later. +# +# Strategy: hit the OCI /v2/ ping endpoint (all standard registries implement it), +# then attempt a manifest HEAD for one representative image to confirm auth works +# end-to-end. Uses only curl — no Docker/crane/skopeo required. +# +# Auth dispatch: +# ECR → aws ecr get-login-password (Bearer token) +# DockerHub → imagePullSecrets.dockerHub creds +# GCR → imagePullSecrets.gcr.jsonKey (_json_key) +# ACR → imagePullSecrets.acr creds +# Custom → imagePullSecrets.custom creds (or unauthenticated if none) +# No registry set → public DockerHub (no auth needed) +preflight_check_registry() { + pf_header "Image registry reachability" + + # No registry configured → images pull from Docker Hub / public — nothing to check + if [[ -z "${IMAGE_REGISTRY}" || "${IMAGE_REGISTRY}" == "null" ]]; then + pf_ok "No private registry configured — images pull from public registries (Docker Hub etc.)" + return + fi + + # Determine protocol scheme for the ping URL + local scheme="https" + [[ "${IMAGE_REGISTRY_INSECURE:-false}" == "true" ]] && scheme="http" + + local ping_url="${scheme}://${IMAGE_REGISTRY}/v2/" + # curl_opts: no -f so HTTP 4xx is not treated as a curl error; we read the status code ourselves. + local curl_opts=(--silent --connect-timeout 10 --max-time 15) + [[ "${IMAGE_REGISTRY_INSECURE:-false}" == "true" ]] && curl_opts+=(--insecure) + + # ---- Step 1: TCP/TLS reachability ---- + local http_code + http_code=$(curl "${curl_opts[@]}" -o /dev/null -w "%{http_code}" "${ping_url}" 2>/dev/null) + case "${http_code}" in + 200|401|403) + pf_ok "Registry reachable: ${ping_url} (HTTP ${http_code})" + ;; + 000) + pf_fail "Registry unreachable: cannot connect to ${ping_url} (connection refused / DNS failure / firewall). Fix images.registry or ensure network path to registry is open." + return + ;; + *) + pf_warn "Registry ${IMAGE_REGISTRY} answered HTTP ${http_code} on /v2/ ping (not a standard OCI response, but host is reachable). Proceeding to manifest check." + ;; + esac + + # ---- Step 2: Auth + manifest pull for one representative image ---- + local _fq_re='^([a-zA-Z0-9.-]+\.[a-zA-Z]{2,}(:[0-9]+)?|[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+(:[0-9]+)?)/' + + _image_targets_registry() { + local raw="$1" + [[ -z "${raw}" || "${raw}" == "null" ]] && return 1 + local full + full=$(build_image_url "${IMAGE_REGISTRY}" "${raw}") + if [[ "${full}" == ${IMAGE_REGISTRY}/* ]]; then + echo "${full#${IMAGE_REGISTRY}/}" + return 0 + fi + return 1 + } + + local probe_ref="" + local probe_source="" + for _candidate_var in SAIA_API_IMAGE RAY_HEAD_IMAGE SAIA_API_V2_IMAGE SAIA_DATALOADER_IMAGE RAY_WORKER_IMAGE WEAVIATE_IMAGE FLUENT_BIT_IMAGE OTEL_COLLECTOR_IMAGE NGINX_IMAGE SPLUNK_IMAGE SPLUNK_OPERATOR_IMAGE OPERATOR_IMAGE; do + local _val="${!_candidate_var:-}" + local _ref + if _ref=$(_image_targets_registry "${_val}"); then + probe_ref="${_ref}" + probe_source="${_candidate_var}" + break + fi + done + + if [[ -z "${probe_ref}" ]]; then + pf_warn "No images in config target IMAGE_REGISTRY (${IMAGE_REGISTRY}) — all images appear to be fully qualified to other registries. Skipping auth check." + return + fi + + pf_ok "Probing registry with ${probe_source} image: ${IMAGE_REGISTRY}/${probe_ref}" + + # Split repo and tag/digest + local probe_repo probe_tag + if [[ "${probe_ref}" == *"@"* ]]; then + probe_repo="${probe_ref%%@*}" + probe_tag="${probe_ref##*@}" + local manifest_url="${scheme}://${IMAGE_REGISTRY}/v2/${probe_repo}/manifests/${probe_tag}" + else + probe_repo="${probe_ref%%:*}" + probe_tag="${probe_ref##*:}" + [[ "${probe_tag}" == "${probe_ref}" ]] && probe_tag="latest" + local manifest_url="${scheme}://${IMAGE_REGISTRY}/v2/${probe_repo}/manifests/${probe_tag}" + fi + + local auth_header="" + + # Resolve auth credentials by registry type + if [[ "${IMAGE_REGISTRY}" == *.dkr.ecr.*.amazonaws.com ]]; then + local _ecr_region="${ECR_REGION:-${REGION:-us-east-2}}" + if ! command -v aws &>/dev/null; then + pf_warn "ECR registry configured but aws CLI not found — skipping auth check." + return + fi + local _ecr_token + if ! _ecr_token=$(aws ecr get-login-password --region "${_ecr_region}" 2>/dev/null); then + pf_fail "Cannot obtain ECR token (aws ecr get-login-password failed). Check AWS credentials and IAM permissions (ecr:GetAuthorizationToken)." + return + fi + auth_header="Authorization: Basic $(printf 'AWS:%s' "${_ecr_token}" | base64 | tr -d '\n')" + + elif [[ "${IMAGE_PULL_SECRETS_DOCKERHUB_ENABLED}" == "true" && "${IMAGE_REGISTRY}" == *"docker.io"* ]]; then + local _dh_user _dh_pass + _dh_user=$(yq eval '.imagePullSecrets.dockerHub.username' "${CONFIG_FILE}" 2>/dev/null) + _dh_pass=$(yq eval '.imagePullSecrets.dockerHub.password' "${CONFIG_FILE}" 2>/dev/null) + if [[ -n "${_dh_user}" && -n "${_dh_pass}" && "${_dh_user}" != "null" && "${_dh_pass}" != "null" ]]; then + auth_header="Authorization: Basic $(printf '%s:%s' "${_dh_user}" "${_dh_pass}" | base64 | tr -d '\n')" + fi + + elif [[ "${IMAGE_PULL_SECRETS_ACR_ENABLED}" == "true" && "${IMAGE_REGISTRY}" == *".azurecr.io"* ]]; then + local _acr_user _acr_pass + _acr_user=$(yq eval '.imagePullSecrets.acr.username' "${CONFIG_FILE}" 2>/dev/null) + _acr_pass=$(yq eval '.imagePullSecrets.acr.password' "${CONFIG_FILE}" 2>/dev/null) + if [[ -n "${_acr_user}" && -n "${_acr_pass}" && "${_acr_user}" != "null" && "${_acr_pass}" != "null" ]]; then + auth_header="Authorization: Basic $(printf '%s:%s' "${_acr_user}" "${_acr_pass}" | base64 | tr -d '\n')" + fi + + elif [[ "${IMAGE_PULL_SECRETS_CUSTOM_ENABLED}" == "true" ]]; then + local _custom_user _custom_pass + _custom_user=$(yq eval '.imagePullSecrets.custom.username' "${CONFIG_FILE}" 2>/dev/null) + _custom_pass=$(yq eval '.imagePullSecrets.custom.password' "${CONFIG_FILE}" 2>/dev/null) + if [[ -n "${_custom_user}" && -n "${_custom_pass}" && "${_custom_user}" != "null" && "${_custom_pass}" != "null" ]]; then + auth_header="Authorization: Basic $(printf '%s:%s' "${_custom_user}" "${_custom_pass}" | base64 | tr -d '\n')" + fi + fi + + # Bearer token exchange helper + _bearer_exchange() { + local _cur_code="$1" _murl="$2" _ahdr="${3:-}" + [[ "${_cur_code}" != "401" ]] && { echo "${_cur_code}"; return; } + + local _www_auth _realm _service _scope _tok _tok_code + local _basic_opts=("${curl_opts[@]}") + [[ -n "${_ahdr}" ]] && _basic_opts+=(-H "${_ahdr}") + + _www_auth=$(curl "${_basic_opts[@]}" -o /dev/null -D - "${_murl}" 2>/dev/null \ + | grep -i '^Www-Authenticate:' | head -1) + _realm=$(echo "${_www_auth}" | grep -oP 'realm="[^"]+"' | cut -d'"' -f2) + _service=$(echo "${_www_auth}" | grep -oP 'service="[^"]+"' | cut -d'"' -f2) + _scope=$(echo "${_www_auth}" | grep -oP 'scope="[^"]+"' | cut -d'"' -f2) + + [[ -z "${_realm}" ]] && { echo "401"; return; } + + local _turl="${_realm}?service=${_service}&scope=${_scope}" + _tok=$(curl "${_basic_opts[@]}" "${_turl}" 2>/dev/null \ + | python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('token') or d.get('access_token',''))" \ + 2>/dev/null || true) + + [[ -z "${_tok}" ]] && { echo "401"; return; } + + _tok_code=$(curl "${curl_opts[@]}" \ + -H "Authorization: Bearer ${_tok}" \ + -H "Accept: application/vnd.docker.distribution.manifest.v2+json, application/vnd.oci.image.manifest.v1+json" \ + -o /dev/null -w "%{http_code}" \ + "${_murl}" 2>/dev/null) + echo "${_tok_code}" + } + + local manifest_http_code + if [[ -n "${auth_header}" ]]; then + manifest_http_code=$(curl "${curl_opts[@]}" \ + -H "${auth_header}" \ + -H "Accept: application/vnd.docker.distribution.manifest.v2+json, application/vnd.oci.image.manifest.v1+json" \ + -o /dev/null -w "%{http_code}" \ + "${manifest_url}" 2>/dev/null) + + if [[ "${manifest_http_code}" == "401" ]]; then + manifest_http_code=$(_bearer_exchange "401" "${manifest_url}" "${auth_header}") + fi + else + manifest_http_code=$(curl "${curl_opts[@]}" \ + -H "Accept: application/vnd.docker.distribution.manifest.v2+json, application/vnd.oci.image.manifest.v1+json" \ + -o /dev/null -w "%{http_code}" \ + "${manifest_url}" 2>/dev/null) + + if [[ "${manifest_http_code}" == "401" ]]; then + manifest_http_code=$(_bearer_exchange "401" "${manifest_url}" "") + fi + fi + + case "${manifest_http_code}" in + 200|206) + pf_ok "Registry auth OK: manifest reachable for ${probe_repo}:${probe_tag}" + ;; + 401|403) + pf_fail "Registry credentials rejected (HTTP ${manifest_http_code}) for ${IMAGE_REGISTRY}. Check imagePullSecrets config — images will fail to pull at deploy time." + ;; + 404) + pf_fail "Image not found in registry (HTTP 404): ${IMAGE_REGISTRY}/${probe_repo}:${probe_tag}. Check that images.operator.image tag exists in the registry." + ;; + 000) + pf_fail "Registry manifest check failed: no response from ${manifest_url}. Check network path and registry health." + ;; + *) + pf_warn "Registry manifest check returned HTTP ${manifest_http_code} for ${probe_repo}:${probe_tag} — verify registry is healthy." + ;; + esac } # ====== PREFLIGHT CHECKS ====== preflight_checks() { log "Running preflight checks..." - for tool in oc yq helm aws curl jq base64 tar; do + local _aws_needed="false" + [[ "${ECR_ENABLED:-false}" == "true" ]] && _aws_needed="true" + [[ "${OBJ_STORE_TYPE:-}" == "aws" ]] && _aws_needed="true" + + for tool in oc yq helm curl jq base64 tar; do command -v "$tool" >/dev/null 2>&1 && log " ✓ $tool found" || err "Missing required tool: $tool" done + if [[ "${_aws_needed}" == "true" ]]; then + command -v aws >/dev/null 2>&1 && log " ✓ aws found" || err "Missing required tool: aws (needed for ECR/S3 — install from https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html)" + else + log " – aws CLI not required (ECR disabled, object store is not AWS)" + fi + + # python3 is used by preflight_check_registry() to parse Bearer token JSON. + if command -v python3 >/dev/null 2>&1; then + log " ✓ python3 found" + else + warn " python3 not found — image registry auth check will be skipped (install python3 to enable it)." + fi + + # Object-store CLI tools — used for the model staging pre-check. + case "${OBJ_STORE_TYPE:-}" in + aws) + if command -v aws >/dev/null 2>&1; then + log " ✓ aws CLI found" + else + warn " aws CLI not found — model staging pre-check will be skipped. Install aws CLI to enable it: https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html" + fi + ;; + minio|seaweedfs|s3compat) + if command -v mc >/dev/null 2>&1; then + log " ✓ mc (MinIO client) found" + else + warn " mc (MinIO client) not found — model staging pre-check will be skipped. Install mc to enable it: https://min.io/docs/minio/linux/reference/minio-mc.html" + fi + ;; + esac + + preflight_check_registry # Verify we are connected to the cluster if ! oc whoami &>/dev/null; then @@ -425,6 +850,11 @@ preflight_checks() { [[ -f "${SPLUNK_AI_FILE}" ]] && log " ✓ Manifest: ${SPLUNK_AI_FILE}" || err "Manifest not found: ${SPLUNK_AI_FILE}" + if object_store_auth_looks_like_placeholder; then + err "objectStore.auth still contains template placeholders (e.g. <...> or CHANGEME). Replace with real credentials in ${CONFIG_FILE}" + fi + log " ✓ Object store credentials look real" + log "Preflight checks passed" } @@ -491,6 +921,7 @@ grant_privileged_scc() { install_nfd() { log "Installing Node Feature Discovery Operator (NFD)..." + # Step 1: Subscription + OperatorGroup — idempotent, skip only if already present. if oc get subscription nfd -n openshift-nfd &>/dev/null; then log " ✓ NFD subscription already exists, skipping" return 0 @@ -538,8 +969,13 @@ EOF log " Waiting for NFD CSV... (${retries}/36, phase=${phase:-pending})" done - # Create the NodeFeatureDiscovery CR to start labeling nodes - if ! oc get nodefeaturediscovery nfd-instance -n openshift-nfd &>/dev/null; then + # Step 2: NodeFeatureDiscovery CR — always ensure it exists, regardless of whether + # the Subscription was just created or was already present from a prior run. + # Without this CR the NFD operand never starts and nodes are never labeled with + # nvidia.com/gpu.present, breaking GPU-node auto-detection. + if oc get nodefeaturediscovery nfd-instance -n openshift-nfd &>/dev/null; then + log " ✓ NodeFeatureDiscovery CR already exists, skipping" + else log "Creating NodeFeatureDiscovery CR..." oc apply -f - <<'EOF' apiVersion: nfd.openshift.io/v1 @@ -573,6 +1009,7 @@ EOF install_nvidia_gpu_operator() { log "Installing NVIDIA GPU Operator..." + # Step 1: Subscription + OperatorGroup — idempotent, skip only if already present. if oc get subscription gpu-operator-certified -n nvidia-gpu-operator &>/dev/null; then log " ✓ GPU Operator subscription already exists, skipping" return 0 @@ -620,8 +1057,13 @@ EOF log " Waiting for GPU Operator CSV... (${retries}/36, phase=${phase:-pending})" done - # Create ClusterPolicy to trigger driver + toolkit + device-plugin rollout - if ! oc get clusterpolicy gpu-cluster-policy &>/dev/null; then + # Step 2: ClusterPolicy — always ensure it exists, regardless of whether the + # Subscription was just created or pre-existing. Without this CR the driver, + # toolkit, and device-plugin DaemonSets are never deployed, leaving Ray worker + # pods requesting nvidia.com/gpu unschedulable. + if oc get clusterpolicy gpu-cluster-policy &>/dev/null; then + log " ✓ ClusterPolicy already exists, skipping" + else log "Creating ClusterPolicy CR..." oc apply -f - <<'EOF' apiVersion: nvidia.com/v1 @@ -786,9 +1228,10 @@ install_cert_manager() { fi fi - local cert_manager_url="${CERT_MANAGER_MANIFEST_URL:-https://github.com/cert-manager/cert-manager/releases/download/v1.13.0/cert-manager.yaml}" - local cert_manager_apply="${cert_manager_url#file://}" - oc apply -f "${cert_manager_apply}" + local _cm_url="${CERT_MANAGER_MANIFEST_URL:-https://github.com/cert-manager/cert-manager/releases/download/v1.13.0/cert-manager.yaml}" + # oc apply -f does not understand file:// — strip it to a bare path + [[ "${_cm_url}" == file://* ]] && _cm_url="${_cm_url#file://}" + oc apply -f "${_cm_url}" log "Waiting for cert-manager to be ready..." oc wait --for=condition=ready pod \ @@ -848,9 +1291,9 @@ install_local_path_provisioner() { fi log "Installing local-path-provisioner (no default storage class found)..." - local local_path_url="${LOCAL_PATH_MANIFEST_URL:-https://raw.githubusercontent.com/rancher/local-path-provisioner/v0.0.26/deploy/local-path-storage.yaml}" - local local_path_apply="${local_path_url#file://}" - oc apply -f "${local_path_apply}" + local _lp_url="${LOCAL_PATH_MANIFEST_URL:-https://raw.githubusercontent.com/rancher/local-path-provisioner/v0.0.26/deploy/local-path-storage.yaml}" + [[ "${_lp_url}" == file://* ]] && _lp_url="${_lp_url#file://}" + oc apply -f "${_lp_url}" log "Waiting for local-path-provisioner to be ready..." oc rollout status deployment local-path-provisioner -n local-path-storage --timeout=120s || true @@ -876,7 +1319,7 @@ install_local_path_provisioner() { # container_file_t allows any container to read/write regardless of MCS categories. # Without this, local-path-provisioner creates dirs with var_t which blocks all containers. local _patch_file; _patch_file=$(mktemp /tmp/local-path-patch-XXXXXX.json) - printf '%s' '{"data":{"setup":"#!/bin/sh\nset -eu\nmkdir -m 0777 -p \"$VOL_DIR\"\nif command -v chcon >/dev/null 2>&1; then\n chcon -Rt container_file_t -l s0 \"$VOL_DIR\" 2>/dev/null || true\nfi\n"}}' \ + printf '%s' '{"data":{"helperPod.yaml":"apiVersion: v1\nkind: Pod\nmetadata:\n name: helper-pod\nspec:\n priorityClassName: system-node-critical\n tolerations:\n - key: node.kubernetes.io/disk-pressure\n operator: Exists\n effect: NoSchedule\n containers:\n - name: helper-pod\n image: registry.access.redhat.com/ubi8/ubi-minimal\n imagePullPolicy: IfNotPresent\n securityContext:\n privileged: true\n","setup":"#!/bin/sh\nset -eu\nmkdir -m 0777 -p \"$VOL_DIR\"\nif command -v chcon >/dev/null 2>&1; then\n chcon -Rt container_file_t -l s0 \"$VOL_DIR\" 2>/dev/null || true\nfi\n"}}' \ > "${_patch_file}" oc patch configmap local-path-config -n local-path-storage --type=merge --patch-file="${_patch_file}" rm -f "${_patch_file}" @@ -895,8 +1338,9 @@ install_local_path_provisioner() { relabel_worker_nodes_for_selinux() { log "Relabeling /opt/local-path-provisioner on worker nodes for SELinux..." local workers - workers=$(oc get nodes -l '!node-role.kubernetes.io/master,!node-role.kubernetes.io/control-plane' \ - -o jsonpath='{.items[*].metadata.name}' 2>/dev/null) + # Relabel all nodes — control-plane nodes may also carry workload labels and + # receive PVCs via local-path-provisioner (WaitForFirstConsumer). + workers=$(oc get nodes -o jsonpath='{.items[*].metadata.name}' 2>/dev/null) for node in ${workers}; do log " Relabeling node ${node}..." oc debug "node/${node}" --image=registry.access.redhat.com/ubi8/ubi-minimal -- \ @@ -1232,7 +1676,7 @@ install_splunk_standalone() { # Derive S3 endpoint for Splunk appRepo (endpoint is required by the Splunk Operator) local minio_endpoint="${OBJ_STORE_ENDPOINT}" if [[ -z "${minio_endpoint}" && "${OBJ_STORE_TYPE}" == "aws" ]]; then - minio_endpoint="https://s3.${ECR_REGION}.amazonaws.com" + minio_endpoint="https://s3.${OBJ_STORE_REGION}.amazonaws.com" log " type=aws: using S3 endpoint ${minio_endpoint}" fi [[ -z "${minio_endpoint}" ]] && err "storage.objectStore.endpoint must be set for type=${OBJ_STORE_TYPE}" @@ -1269,6 +1713,19 @@ metadata: namespace: ${AI_NS} spec: replicas: 1 + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: splunk.ai/workload-type + operator: In + values: + - cpu + tolerations: + - key: "node-role.kubernetes.io/master" + operator: "Exists" + effect: "NoSchedule" volumes: - name: defaults configMap: @@ -1287,7 +1744,7 @@ spec: provider: aws storageType: s3 endpoint: ${minio_endpoint} - region: ${ECR_REGION} + region: ${OBJ_STORE_REGION} path: ${OBJ_STORE_BUCKET} secretRef: minio-credentials YAML @@ -1305,12 +1762,16 @@ install_ai_platform_cr() { oc delete jobs -n "${AI_NS}" --field-selector status.successful=0 --wait=false 2>/dev/null || true oc delete pods -n "${AI_NS}" --field-selector status.phase=Failed --wait=false 2>/dev/null || true - # Build imagePullSecrets block + # Build imagePullSecrets block — include every docker-registry secret that + # exists in the namespace, covering ECR, DockerHub, GCR, ACR, and custom + # registries. This avoids hard-coding a fixed list of names. local secrets_yaml="" - for secret_name in ecr-registry-secret; do - oc get secret "${secret_name}" -n "${AI_NS}" &>/dev/null && \ - secrets_yaml+=" - name: ${secret_name}"$'\n' - done + while IFS= read -r secret_name; do + [[ -z "${secret_name}" ]] && continue + secrets_yaml+=" - name: ${secret_name}"$'\n' + done < <(oc get secrets -n "${AI_NS}" \ + -o jsonpath='{range .items[?(@.type=="kubernetes.io/dockerconfigjson")]}{.metadata.name}{"\n"}{end}' \ + 2>/dev/null || true) local image_pull_secrets="" [[ -n "${secrets_yaml}" ]] && image_pull_secrets=" imagePullSecrets:"$'\n'"${secrets_yaml}" @@ -1331,11 +1792,19 @@ install_ai_platform_cr() { if [[ "${feature_count}" -gt 0 ]]; then local i=0 while [[ $i -lt $feature_count ]]; do - local fname fver + local fname fver fsa fscale fname=$(yq eval ".aiPlatform.features[$i].name" "${CONFIG_FILE}") fver=$(yq eval ".aiPlatform.features[$i].version // \"1.0.0\"" "${CONFIG_FILE}") - [[ -n "$fname" && "$fname" != "null" ]] && \ - features_yaml+=" - name: ${fname}"$'\n'" version: \"${fver}\""$'\n' + fsa=$(yq eval ".aiPlatform.features[$i].serviceAccountName // \"\"" "${CONFIG_FILE}") + fscale=$(yq eval ".aiPlatform.features[$i].scaleFactor // \"\"" "${CONFIG_FILE}") + if [[ -n "$fname" && "$fname" != "null" ]]; then + features_yaml+=" - name: ${fname}"$'\n' + features_yaml+=" version: \"${fver}\""$'\n' + [[ -n "$fsa" && "$fsa" != "null" ]] && \ + features_yaml+=" serviceAccountName: ${fsa}"$'\n' + [[ -n "$fscale" && "$fscale" != "null" ]] && \ + features_yaml+=" scaleFactor: ${fscale}"$'\n' + fi i=$((i + 1)) done else @@ -1435,7 +1904,7 @@ metadata: spec: objectStorage: path: ${obj_path} - region: ${ECR_REGION} + region: ${OBJ_STORE_REGION} $( [[ -n "${obj_endpoint}" ]] && echo "endpoint: \"${obj_endpoint}\"" ) secretRef: minio-credentials images: @@ -1454,7 +1923,7 @@ ${svc_template_yaml}${storage_yaml} nodeSelector: splunk.ai/ai-tier-node: "true" splunkConfiguration: - endpoint: http://${AI_STANDALONE_NAME}-standalone-service.${AI_NS}.svc.cluster.local:8089 + endpoint: http://splunk-${AI_STANDALONE_NAME}-standalone-service.${AI_NS}.svc.cluster.local:8088 secretRef: name: ${splunk_ns_secret} namespace: ${AI_NS} @@ -1524,19 +1993,381 @@ EOF log " Use this URL in Splunk AI setup: http://${route_host}" } +# ====== ALL MODELS STAGED PRE-CHECK ====== +# all_models_staged +# Checks whether every artifact already has a staging_complete marker with matching hf_url. +# Returns 0 (all staged) or 1 (one or more missing). +# Fails open: returns 1 if store is unreachable or tool is missing. +all_models_staged() { + local staging_dir="$1" + local accel="$2" + local config_file + + case "${accel}" in + h100) config_file="${staging_dir}/model_artifacts_configs_h100.yaml" ;; + *) config_file="${staging_dir}/model_artifacts_configs.yaml" ;; + esac + + if [[ ! -f "${config_file}" ]]; then + warn "all_models_staged: config file not found: ${config_file} — skipping pre-check." + return 1 + fi + + local ids=() hf_urls=() + local raw_ids raw_urls + raw_ids=$(yq eval '.artifact-configs[].artifact-id' "${config_file}" 2>/dev/null) || { + warn "all_models_staged: could not read artifact IDs from ${config_file} — skipping pre-check." + return 1 + } + raw_urls=$(yq eval '.artifact-configs[].hf-url' "${config_file}" 2>/dev/null) || { + warn "all_models_staged: could not read hf-url fields from ${config_file} — skipping pre-check." + return 1 + } + while IFS= read -r id; do + [[ -n "${id}" ]] && ids+=("${id}") + done <<< "${raw_ids}" + while IFS= read -r url; do + hf_urls+=("${url}") + done <<< "${raw_urls}" + + if [[ ${#ids[@]} -eq 0 ]]; then + warn "all_models_staged: no artifact IDs found in ${config_file} — skipping pre-check." + return 1 + fi + + _marker_matches_url() { + local content="$1" expected_url="$2" + echo "${content}" | grep -q "^hf_url=${expected_url}$" + } + + local missing=() + + case "${OBJ_STORE_TYPE}" in + aws) + if ! command -v aws &>/dev/null; then + warn "all_models_staged: aws CLI not found — skipping pre-check." + return 1 + fi + for i in "${!ids[@]}"; do + local id="${ids[$i]}" hf_url="${hf_urls[$i]:-}" + local marker_path="s3://${OBJ_STORE_BUCKET}/staging_state/${id}/.staging_complete" + local content + content=$(AWS_ACCESS_KEY_ID="${MINIO_ROOT_USER}" \ + AWS_SECRET_ACCESS_KEY="${MINIO_ROOT_PASSWORD}" \ + aws s3 cp "${marker_path}" - --region "${REGION:-us-east-2}" 2>/dev/null) || { missing+=("${id}"); continue; } + _marker_matches_url "${content}" "${hf_url}" || missing+=("${id}") + done + ;; + minio|seaweedfs) + if ! command -v mc &>/dev/null; then + warn "all_models_staged: mc not found — skipping pre-check." + return 1 + fi + if [[ -z "${OBJ_STORE_ENDPOINT}" ]]; then + warn "all_models_staged: OBJ_STORE_ENDPOINT not set — skipping pre-check." + return 1 + fi + local _alias="installer_precheck" + mc alias set "${_alias}" "${OBJ_STORE_ENDPOINT}" \ + "${MINIO_ROOT_USER}" "${MINIO_ROOT_PASSWORD}" --api S3v4 &>/dev/null || { + warn "all_models_staged: could not configure mc alias — skipping pre-check." + return 1 + } + for i in "${!ids[@]}"; do + local id="${ids[$i]}" hf_url="${hf_urls[$i]:-}" + local marker_path="${_alias}/${OBJ_STORE_BUCKET}/staging_state/${id}/.staging_complete" + local content + content=$(mc cat "${marker_path}" 2>/dev/null) || { missing+=("${id}"); continue; } + _marker_matches_url "${content}" "${hf_url}" || missing+=("${id}") + done + ;; + *) + warn "all_models_staged: unsupported store type '${OBJ_STORE_TYPE}' — skipping pre-check." + return 1 + ;; + esac + + if [[ ${#missing[@]} -eq 0 ]]; then + log "✓ All ${#ids[@]} models already staged in object store (${OBJ_STORE_TYPE}) — skipping download and upload." + return 0 + fi + + log "Model staging needed: ${#missing[@]}/${#ids[@]} model(s) not yet staged." + for _m in "${missing[@]}"; do + log " MISSING: ${_m} (${OBJ_STORE_BUCKET}/staging_state/${_m}/.staging_complete not found or hf_url changed)" + done + return 1 +} + +# ====== MODEL ARTIFACT STAGING ====== +# Downloads model artifacts from HuggingFace and uploads them to the configured +# object store. Controlled by storage.modelStaging.enabled in config (default: false). +# Skipped when AIRGAP_MODE=true — models must be pre-staged in that case. +stage_model_artifacts() { + if [[ "${MODEL_STAGING_ENABLED}" != "true" ]]; then + log "Model staging disabled (storage.modelStaging.enabled=false), skipping" + return 0 + fi + + if [[ "${AIRGAP_MODE:-false}" == "true" ]]; then + log "AIRGAP_MODE=true — skipping model staging (models must be pre-staged in object store)" + return 0 + fi + + if object_store_auth_looks_like_placeholder; then + err "Refusing to stage artifacts: objectStore.auth still contains template placeholders; fix ${CONFIG_FILE}" + return 1 + fi + + local staging_dir + staging_dir="$(cd "$(dirname "$0")/../artifacts_download_upload_scripts" && pwd)" \ + || { err "Cannot locate artifacts_download_upload_scripts directory (expected sibling of cluster_setup/)"; return 1; } + + log "Model staging directory: ${staging_dir}" + + # ---- Resolve accelerator (normalize to lowercase) ---- + local _accel + _accel=$(printf '%s' "${DEFAULT_ACCELERATOR}" | tr '[:upper:]' '[:lower:]') + + local _skip_staged="${SKIP_IF_STAGED:-1}" + + # ---- Fast-path: skip everything if all models already staged ---- + if [[ "${_skip_staged}" != "0" ]] && all_models_staged "${staging_dir}" "${_accel}"; then + return 0 + fi + + wait_for_dependency \ + "HuggingFace (huggingface.co) — required for model weight download" \ + "curl -sf --connect-timeout 10 --max-time 15 https://huggingface.co >/dev/null 2>&1" \ + 300 + + log "Downloading model artifacts from Hugging Face (accelerator: ${_accel}, skip-if-staged: ${_skip_staged})..." + ( cd "${staging_dir}" && \ + ACCELERATOR="${_accel}" \ + SKIP_IF_EXISTS="${SKIP_IF_EXISTS:-0}" \ + SKIP_IF_STAGED="${_skip_staged}" \ + OBJ_STORE_TYPE="${OBJ_STORE_TYPE}" \ + OBJ_STORE_BUCKET="${OBJ_STORE_BUCKET}" \ + OBJ_STORE_ENDPOINT="${OBJ_STORE_ENDPOINT}" \ + OBJ_STORE_ACCESS_KEY="${MINIO_ROOT_USER}" \ + OBJ_STORE_SECRET_KEY="${MINIO_ROOT_PASSWORD}" \ + S3_REGION="${REGION:-us-east-2}" \ + S3_PREFIX="model_artifacts" \ + bash ./download_from_huggingface.sh ) \ + || { err "HuggingFace download failed — see output above"; return 1; } + + log "Uploading model artifacts to object store (type=${OBJ_STORE_TYPE})..." + if [[ "${OBJ_STORE_TYPE}" == "minio" || "${OBJ_STORE_TYPE}" == "seaweedfs" ]]; then + [[ -n "${OBJ_STORE_ENDPOINT}" ]] || { err "storage.objectStore.endpoint is required for ${OBJ_STORE_TYPE} model staging"; return 1; } + fi + + case "${OBJ_STORE_TYPE}" in + aws) + ( cd "${staging_dir}" && \ + S3_BUCKET="${OBJ_STORE_BUCKET}" \ + S3_REGION="${OBJ_STORE_REGION:-us-east-2}" \ + AWS_ACCESS_KEY_ID="${MINIO_ROOT_USER}" \ + AWS_SECRET_ACCESS_KEY="${MINIO_ROOT_PASSWORD}" \ + SKIP_IF_STAGED="${_skip_staged}" \ + bash ./upload_to_s3.sh ) \ + || { err "Upload to S3 failed"; return 1; } + ;; + minio) + ( cd "${staging_dir}" && \ + OBJECT_STORE_ENDPOINT="${OBJ_STORE_ENDPOINT}" \ + OBJECT_STORE_BUCKET="${OBJ_STORE_BUCKET}" \ + OBJECT_STORE_ACCESS_KEY="${MINIO_ROOT_USER}" \ + OBJECT_STORE_SECRET_KEY="${MINIO_ROOT_PASSWORD}" \ + SKIP_IF_STAGED="${_skip_staged}" \ + bash ./upload_to_minio.sh ) \ + || { err "Upload to MinIO failed"; return 1; } + ;; + seaweedfs) + ( cd "${staging_dir}" && \ + OBJECT_STORE_ENDPOINT="${OBJ_STORE_ENDPOINT}" \ + OBJECT_STORE_BUCKET="${OBJ_STORE_BUCKET}" \ + OBJECT_STORE_ACCESS_KEY="${MINIO_ROOT_USER}" \ + OBJECT_STORE_SECRET_KEY="${MINIO_ROOT_PASSWORD}" \ + SKIP_IF_STAGED="${_skip_staged}" \ + bash ./upload_to_seaweedfs_upload_only.sh ) \ + || { err "Upload to SeaweedFS failed"; return 1; } + ;; + *) + err "Unsupported objectStore.type for model staging: '${OBJ_STORE_TYPE}' (expected: aws | minio | seaweedfs)" + return 1 + ;; + esac + + # ---- Post-stage verification ---- + log "Verifying model staging completeness..." + if all_models_staged "${staging_dir}" "${_accel}"; then + log "✓ Post-stage verification passed — all models confirmed staged." + else + warn "Post-stage verification: some models may not have been staged successfully." + warn "Check the upload logs above. You can re-run with SKIP_IF_STAGED=0 to force re-upload." + fi + + log "✓ Model artifact staging complete (type=${OBJ_STORE_TYPE}, bucket=${OBJ_STORE_BUCKET})" +} + +# ====== CREATE IMAGE PULL SECRETS ====== +# Creates pull secrets for all enabled registries (ECR, DockerHub, GCR, ACR, custom) +# in the given namespace. Uses --dry-run=client | apply so it is idempotent. +create_image_pull_secrets() { + local ns="$1" + ensure_namespace "${ns}" + + log "Creating image pull secrets in ${ns}..." + local secrets_created=() + + # ECR + if [[ "${IMAGE_PULL_SECRETS_ECR_ENABLED}" == "true" ]]; then + local ecr_region="${ECR_REGION:-us-east-2}" + local ecr_account="${ECR_ACCOUNT:-}" + if ! aws sts get-caller-identity &>/dev/null; then + warn "AWS credentials not available — skipping ECR secret creation" + else + [[ -z "${ecr_account}" ]] && ecr_account=$(aws sts get-caller-identity --query Account --output text 2>/dev/null || echo "") + local ecr_password + if ecr_password=$(aws ecr get-login-password --region "${ecr_region}" 2>/dev/null); then + oc create secret docker-registry ecr-registry-secret \ + --docker-server="${ecr_account}.dkr.ecr.${ecr_region}.amazonaws.com" \ + --docker-username=AWS \ + --docker-password="${ecr_password}" \ + --namespace="${ns}" \ + --dry-run=client -o yaml | oc apply -f - + log " ✓ ECR secret created: ecr-registry-secret" + secrets_created+=("ecr-registry-secret") + else + warn "Failed to get ECR token — skipping ECR secret" + fi + fi + fi + + # DockerHub + if [[ "${IMAGE_PULL_SECRETS_DOCKERHUB_ENABLED}" == "true" ]]; then + local dh_user dh_pass dh_email + dh_user=$(yq eval '.imagePullSecrets.dockerHub.username // ""' "${CONFIG_FILE}" 2>/dev/null || echo "") + dh_pass=$(yq eval '.imagePullSecrets.dockerHub.password // ""' "${CONFIG_FILE}" 2>/dev/null || echo "") + dh_email=$(yq eval '.imagePullSecrets.dockerHub.email // ""' "${CONFIG_FILE}" 2>/dev/null || echo "") + if [[ -n "${dh_user}" && -n "${dh_pass}" ]]; then + local email_arg="" + [[ -n "${dh_email}" ]] && email_arg="--docker-email=${dh_email}" + oc create secret docker-registry docker-hub-secret \ + --docker-server=docker.io \ + --docker-username="${dh_user}" \ + --docker-password="${dh_pass}" \ + ${email_arg} \ + --namespace="${ns}" \ + --dry-run=client -o yaml | oc apply -f - + log " ✓ DockerHub secret created: docker-hub-secret" + secrets_created+=("docker-hub-secret") + else + warn "DockerHub credentials not configured — skipping" + fi + fi + + # GCR + if [[ "${IMAGE_PULL_SECRETS_GCR_ENABLED}" == "true" ]]; then + local gcr_key + gcr_key=$(yq eval '.imagePullSecrets.gcr.jsonKey // ""' "${CONFIG_FILE}" 2>/dev/null || echo "") + if [[ -n "${gcr_key}" && "${gcr_key}" != "null" ]]; then + oc create secret docker-registry gcr-secret \ + --docker-server=gcr.io \ + --docker-username=_json_key \ + --docker-password="${gcr_key}" \ + --namespace="${ns}" \ + --dry-run=client -o yaml | oc apply -f - + log " ✓ GCR secret created: gcr-secret" + secrets_created+=("gcr-secret") + else + warn "GCR JSON key not configured — skipping" + fi + fi + + # ACR + if [[ "${IMAGE_PULL_SECRETS_ACR_ENABLED}" == "true" ]]; then + local acr_reg acr_user acr_pass + acr_reg=$(yq eval '.imagePullSecrets.acr.registry // ""' "${CONFIG_FILE}" 2>/dev/null || echo "") + acr_user=$(yq eval '.imagePullSecrets.acr.username // ""' "${CONFIG_FILE}" 2>/dev/null || echo "") + acr_pass=$(yq eval '.imagePullSecrets.acr.password // ""' "${CONFIG_FILE}" 2>/dev/null || echo "") + if [[ -n "${acr_reg}" && -n "${acr_user}" && -n "${acr_pass}" ]]; then + oc create secret docker-registry acr-secret \ + --docker-server="${acr_reg}" \ + --docker-username="${acr_user}" \ + --docker-password="${acr_pass}" \ + --namespace="${ns}" \ + --dry-run=client -o yaml | oc apply -f - + log " ✓ ACR secret created: acr-secret" + secrets_created+=("acr-secret") + else + warn "ACR credentials not configured — skipping" + fi + fi + + # Custom registry + if [[ "${IMAGE_PULL_SECRETS_CUSTOM_ENABLED}" == "true" ]]; then + local c_name c_server c_user c_pass c_email + c_name=$(yq eval '.imagePullSecrets.custom.name // "custom-registry-secret"' "${CONFIG_FILE}" 2>/dev/null || echo "custom-registry-secret") + c_server=$(yq eval '.imagePullSecrets.custom.server // ""' "${CONFIG_FILE}" 2>/dev/null || echo "") + c_user=$(yq eval '.imagePullSecrets.custom.username // ""' "${CONFIG_FILE}" 2>/dev/null || echo "") + c_pass=$(yq eval '.imagePullSecrets.custom.password // ""' "${CONFIG_FILE}" 2>/dev/null || echo "") + c_email=$(yq eval '.imagePullSecrets.custom.email // ""' "${CONFIG_FILE}" 2>/dev/null || echo "") + if [[ -n "${c_server}" && -n "${c_user}" && -n "${c_pass}" ]]; then + local email_arg="" + [[ -n "${c_email}" ]] && email_arg="--docker-email=${c_email}" + oc create secret docker-registry "${c_name}" \ + --docker-server="${c_server}" \ + --docker-username="${c_user}" \ + --docker-password="${c_pass}" \ + ${email_arg} \ + --namespace="${ns}" \ + --dry-run=client -o yaml | oc apply -f - + log " ✓ Custom registry secret created: ${c_name}" + secrets_created+=("${c_name}") + else + warn "Custom registry credentials not configured — skipping" + fi + fi + + if [[ ${#secrets_created[@]} -gt 0 ]]; then + log " Pull secrets created in ${ns}: ${secrets_created[*]}" + else + log " No additional pull secrets configured" + fi +} + # ====== MAIN INSTALL ====== main_install() { log "============================================" log " Splunk AI Platform — OpenShift Install" log "============================================" + # Sync SILENT_INSTALL ↔ AUTO_APPROVE + [[ "${AUTO_APPROVE:-false}" == "true" ]] && SILENT_INSTALL=true + SILENT_INSTALL="${SILENT_INSTALL:-false}" + [[ "${SILENT_INSTALL}" == "true" ]] && AUTO_APPROVE=true + load_config validate_image_config configure_images + resolve_accelerator_type + resolve_model_staging + show_install_plan + phase_start "Model Staging" + step_start "Model artifact staging" + stage_model_artifacts + step_ok + phase_end "Model Staging" + phase_start "Preflight" + step_start "Model artifact staging" + stage_model_artifacts + step_ok + step_start "Preflight checks" preflight_checks step_ok @@ -1574,8 +2405,11 @@ main_install() { install_ray_operator step_ok - step_start "ECR pull secrets" + step_start "Image pull secrets" ensure_ecr_pull_secret + create_image_pull_secrets "${AI_NS}" + create_image_pull_secrets "splunk-ai-operator-system" + create_image_pull_secrets "splunk-operator" step_ok step_start "Splunk AI Operator" @@ -1802,24 +2636,47 @@ diagnose() { oc get pvc --all-namespaces > "${bundle_dir}/pvcs.txt" 2>&1 || true oc get svc --all-namespaces > "${bundle_dir}/services.txt" 2>&1 || true oc describe nodes > "${bundle_dir}/node-details.txt" 2>&1 || true - - # Per-namespace pod logs for failing pods - log "Collecting logs from non-Running pods..." - local ns pod + oc get deployments --all-namespaces -o wide > "${bundle_dir}/deployments.txt" 2>&1 || true + oc get statefulsets --all-namespaces -o wide > "${bundle_dir}/statefulsets.txt" 2>&1 || true + oc get daemonsets --all-namespaces -o wide > "${bundle_dir}/daemonsets.txt" 2>&1 || true + oc describe deployments --all-namespaces > "${bundle_dir}/deployment-details.txt" 2>&1 || true + oc describe statefulsets --all-namespaces > "${bundle_dir}/statefulset-details.txt" 2>&1 || true + oc describe daemonsets --all-namespaces > "${bundle_dir}/daemonset-details.txt" 2>&1 || true + + # Per-pod log collection: ALL pods, with conditional tail size (300 for failing, 100 for running) + log "Collecting pod logs (all pods)..." + local ns pod phase while IFS= read -r line; do ns=$(echo "${line}" | awk '{print $1}') pod=$(echo "${line}" | awk '{print $2}') + phase=$(echo "${line}" | awk '{print $4}') mkdir -p "${bundle_dir}/pod-logs/${ns}" - oc logs "${pod}" -n "${ns}" --tail=200 \ + local tail_lines=100 + if [[ "${phase}" != "Running" && "${phase}" != "Completed" ]]; then + tail_lines=300 + fi + oc logs "${pod}" -n "${ns}" --all-containers=true --tail="${tail_lines}" \ > "${bundle_dir}/pod-logs/${ns}/${pod}.log" 2>&1 || true - oc logs "${pod}" -n "${ns}" --previous --tail=100 \ + oc logs "${pod}" -n "${ns}" --all-containers=true --previous --tail=100 \ > "${bundle_dir}/pod-logs/${ns}/${pod}.previous.log" 2>&1 || true + done < <(oc get pods --all-namespaces --no-headers 2>/dev/null) + + # Describe unhealthy pods + log "Describing unhealthy pods..." + while IFS= read -r line; do + ns=$(echo "${line}" | awk '{print $1}') + pod=$(echo "${line}" | awk '{print $2}') + mkdir -p "${bundle_dir}/pod-describe/${ns}" + oc describe pod "${pod}" -n "${ns}" \ + > "${bundle_dir}/pod-describe/${ns}/${pod}.txt" 2>&1 || true done < <(oc get pods --all-namespaces --no-headers 2>/dev/null \ | awk '$4 != "Running" && $4 != "Completed" {print $1, $2}') # AI Platform specific resources oc describe aiplatform --all -n "${AI_NS:-ai-platform}" > "${bundle_dir}/aiplatform-cr.txt" 2>&1 || true oc describe aiservice --all -n "${AI_NS:-ai-platform}" > "${bundle_dir}/aiservice-cr.txt" 2>&1 || true + oc describe raycluster --all-namespaces > "${bundle_dir}/raycluster-cr.txt" 2>&1 || true + oc describe rayservice --all-namespaces > "${bundle_dir}/rayservice-cr.txt" 2>&1 || true # Operator logs oc logs -n splunk-ai-operator-system -l control-plane=controller-manager --tail=500 \ @@ -1832,7 +2689,7 @@ diagnose() { # 3. Config file (redact credentials) if [[ -f "${CONFIG_FILE}" ]]; then log "Including config file (credentials redacted)..." - sed 's/\(rootUser\|rootPassword\|AWS_ACCESS_KEY_ID\|AWS_SECRET_ACCESS_KEY\|accessKey\|secretKey\):.*/\1: /g' \ + sed 's/\(rootUser\|rootPassword\|hf-token\|hf-username\|AWS_ACCESS_KEY_ID\|AWS_SECRET_ACCESS_KEY\|accessKey\|secretKey\|password\):.*/\1: /g' \ "${CONFIG_FILE}" > "${bundle_dir}/cluster-config-redacted.yaml" fi @@ -1848,27 +2705,40 @@ diagnose() { # 5. Pack into tar.gz local bundle_tar="${bundle_dir}.tar.gz" + mkdir -p "${LOG_DIR}" tar -czf "${bundle_tar}" -C "$(dirname "${bundle_dir}")" "$(basename "${bundle_dir}")" 2>/dev/null rm -rf "${bundle_dir}" - log "=== Support bundle ready: ${bundle_tar} ===" - log "Attach this file to your support ticket or share with the team." + echo -e "\n\033[1;34m╔══════════════════════════════════════════════════════════╗\033[0m" >&2 + echo -e "\033[1;34m║ SUPPORT BUNDLE READY ║\033[0m" >&2 + echo -e "\033[1;34m╚══════════════════════════════════════════════════════════╝\033[0m" >&2 + log " Bundle: ${bundle_tar}" + log " Attach this file to your support ticket or share with the team." } # ====== USAGE ====== usage() { cat < @@ -1877,9 +2747,35 @@ Prerequisites: EOF } +# ====== VERIFY ALL PODS HEALTHY ====== +verify_all_pods_healthy() { + load_config 2>/dev/null || true + log "Verifying all pods are healthy in namespace ${AI_NS:-ai-platform}..." + local unhealthy + unhealthy=$(oc get pods --all-namespaces --no-headers 2>/dev/null \ + | awk '$4 != "Running" && $4 != "Completed" && $4 != "Succeeded" {print $1, $2, $4}') + if [[ -z "${unhealthy}" ]]; then + log "✓ All pods are healthy" + return 0 + fi + warn "Unhealthy pods detected:" + echo "${unhealthy}" | while read -r ns pod status; do + warn " ${ns}/${pod} — ${status}" + done + return 1 +} + # ====== MAIN ====== -case "${1:-install}" in +_CMD="${1:-install}" +shift 2>/dev/null || true +case "${_CMD}" in install) + while [[ $# -gt 0 ]]; do + case "$1" in + --silent|-s) SILENT_INSTALL=true; shift ;; + *) echo "Unknown install option: $1" >&2; usage >&2; exit 1 ;; + esac + done main_install ;; delete) @@ -1888,6 +2784,20 @@ case "${1:-install}" in diagnose) diagnose ;; + stage-artifacts) + load_config + resolve_accelerator_type + stage_model_artifacts + ;; + verify) + _vpc_rc=0 + verify_all_pods_healthy || _vpc_rc=$? + if (( _vpc_rc != 0 )) && [[ "${AUTO_DIAGNOSE:-true}" != "false" ]]; then + log "Auto-collecting support bundle (set AUTO_DIAGNOSE=false to suppress)..." + diagnose || true + fi + exit "${_vpc_rc}" + ;; *) usage exit 1 From 57beae4d24780b2113c0bb6c9bbdb83ff439daec Mon Sep 17 00:00:00 2001 From: kbhos Date: Fri, 10 Jul 2026 19:55:44 +0530 Subject: [PATCH 18/80] resolve codex comments --- tools/cluster_setup/openshift_with_stack.sh | 83 +++++++++++---------- 1 file changed, 44 insertions(+), 39 deletions(-) diff --git a/tools/cluster_setup/openshift_with_stack.sh b/tools/cluster_setup/openshift_with_stack.sh index 74b74597..bd2e3386 100755 --- a/tools/cluster_setup/openshift_with_stack.sh +++ b/tools/cluster_setup/openshift_with_stack.sh @@ -195,7 +195,7 @@ resolve_model_staging() { } # ====== SUPPORTED ACCELERATOR TYPES ====== -readonly SUPPORTED_ACCELERATORS=("L40S" "H100") +readonly SUPPORTED_ACCELERATORS=("L40S" "H100" "RTX_PRO_6000_BLACKWELL") # ====== RESOLVE ACCELERATOR TYPE ====== # Normalizes and validates DEFAULT_ACCELERATOR. Prompts interactively if missing @@ -1009,13 +1009,13 @@ EOF install_nvidia_gpu_operator() { log "Installing NVIDIA GPU Operator..." - # Step 1: Subscription + OperatorGroup — idempotent, skip only if already present. + # Step 1: Subscription + OperatorGroup — idempotent, skip creation if already present. + # Do NOT early-return here: a prior run may have created the Subscription but never + # the ClusterPolicy below, so we must always fall through to Step 2. if oc get subscription gpu-operator-certified -n nvidia-gpu-operator &>/dev/null; then - log " ✓ GPU Operator subscription already exists, skipping" - return 0 - fi - - oc apply -f - </dev/null || echo "") - if [[ "${phase}" == "Succeeded" ]]; then - log " ✓ GPU Operator CSV ready" - break - fi - sleep 10 - retries=$(( retries + 1 )) - log " Waiting for GPU Operator CSV... (${retries}/36, phase=${phase:-pending})" - done + log "Waiting for GPU Operator CSV to succeed..." + local retries=0 + while (( retries < 36 )); do + local phase + phase=$(oc get csv -n nvidia-gpu-operator -o jsonpath='{.items[0].status.phase}' 2>/dev/null || echo "") + if [[ "${phase}" == "Succeeded" ]]; then + log " ✓ GPU Operator CSV ready" + break + fi + sleep 10 + retries=$(( retries + 1 )) + log " Waiting for GPU Operator CSV... (${retries}/36, phase=${phase:-pending})" + done + fi # Step 2: ClusterPolicy — always ensure it exists, regardless of whether the # Subscription was just created or pre-existing. Without this CR the driver, @@ -1143,14 +1144,12 @@ label_nodes() { ;; manual) - local node_count - node_count=$(yq eval '.openshift.nodes | length' "${CONFIG_FILE}" 2>/dev/null || echo "0") - local i=0 - while [[ $i -lt $node_count ]]; do - local n; n=$(yq eval ".openshift.nodes[$i]" "${CONFIG_FILE}" 2>/dev/null || echo "") - [[ -n "$n" && "$n" != "null" ]] && ai_nodes+=("$n") - i=$((i+1)) - done + # openshift.nodes has cpu/gpu sub-arrays. Both CPU and GPU hosts get the single + # splunk.ai/ai-tier-node=true label (GPU workers are further constrained by their + # nvidia.com/gpu resource request), so collect hosts from both lists. + while IFS= read -r node; do + [[ -n "$node" && "$node" != "null" ]] && ai_nodes+=("$node") + done < <(yq eval '.openshift.nodes.cpu[], .openshift.nodes.gpu[]' "${CONFIG_FILE}" 2>/dev/null) ;; *) @@ -1618,16 +1617,16 @@ install_splunk_operator() { fi fi - if oc create -f "${SPLUNK_OPERATOR_FILE}" 2>/dev/null; then - log " Splunk Operator resources created" - else - log " Resources already exist, updating..." - oc replace --force -f "${SPLUNK_OPERATOR_FILE}" 2>&1 | grep -v "Warning: --force is deprecated" || true - fi + # Server-side apply: idempotent create-or-update that NEVER deletes existing objects. + # This bundle includes CRDs and the splunk-operator Namespace; `oc replace --force` is + # delete-then-recreate, so recreating the CRDs would cascade-delete every Splunk custom + # resource (Standalones, etc.). Server-side apply patches in place and preserves them. + oc apply --server-side --force-conflicts -f "${SPLUNK_OPERATOR_FILE}" 2>&1 || true + log " Splunk Operator resources applied" # Grant privileged SCC to the whole namespace group — this is the pattern OCP SCC admission # actually honours. The operator pod adds NET_BIND_SERVICE which anyuid blocks; privileged - # covers both. group-based grant survives replace --force (which recreates the namespace). + # covers both. group-based grant is namespace-scoped and survives operator manifest updates. oc adm policy add-scc-to-group privileged \ "system:serviceaccounts:${splunk_operator_ns}" 2>/dev/null || true # Force pod recreation so it picks up the new SCC grant @@ -1718,10 +1717,10 @@ spec: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - - key: splunk.ai/workload-type + - key: splunk.ai/ai-tier-node operator: In values: - - cpu + - "true" tolerations: - key: "node-role.kubernetes.io/master" operator: "Exists" @@ -1772,8 +1771,11 @@ install_ai_platform_cr() { done < <(oc get secrets -n "${AI_NS}" \ -o jsonpath='{range .items[?(@.type=="kubernetes.io/dockerconfigjson")]}{.metadata.name}{"\n"}{end}' \ 2>/dev/null || true) + # Emit the whole images: block only when secrets exist. spec.images is an object in + # the CRD, so a bare "images:" with no children would serialize to null and fail + # validation. imagePullSecrets are optional, so omit the key entirely when empty. local image_pull_secrets="" - [[ -n "${secrets_yaml}" ]] && image_pull_secrets=" imagePullSecrets:"$'\n'"${secrets_yaml}" + [[ -n "${secrets_yaml}" ]] && image_pull_secrets=" images:"$'\n'" imagePullSecrets:"$'\n'"${secrets_yaml}" # Object storage path and endpoint local obj_path obj_endpoint @@ -1907,7 +1909,6 @@ spec: region: ${OBJ_STORE_REGION} $( [[ -n "${obj_endpoint}" ]] && echo "endpoint: \"${obj_endpoint}\"" ) secretRef: minio-credentials - images: ${image_pull_secrets} defaultAcceleratorType: ${DEFAULT_ACCELERATOR} features: @@ -2786,6 +2787,10 @@ case "${_CMD}" in ;; stage-artifacts) load_config + # Running this subcommand IS an explicit request to stage, so force staging on + # regardless of storage.modelStaging.enabled (which only gates install-time staging). + # The airgap guard inside stage_model_artifacts still applies. + MODEL_STAGING_ENABLED="true" resolve_accelerator_type stage_model_artifacts ;; From 733b576ef485854bf9d673594a49b01ac86cbd84 Mon Sep 17 00:00:00 2001 From: kbhos Date: Fri, 10 Jul 2026 20:07:33 +0530 Subject: [PATCH 19/80] resolve few more codex commnets --- .../install_from_airgap_bundle_openshift.sh | 10 ++- tools/cluster_setup/openshift_with_stack.sh | 64 +++++++++++-------- .../prepare_airgap_bundle_openshift.sh | 36 ++++++++++- 3 files changed, 79 insertions(+), 31 deletions(-) diff --git a/tools/cluster_setup/install_from_airgap_bundle_openshift.sh b/tools/cluster_setup/install_from_airgap_bundle_openshift.sh index 47f062e3..67f767a4 100755 --- a/tools/cluster_setup/install_from_airgap_bundle_openshift.sh +++ b/tools/cluster_setup/install_from_airgap_bundle_openshift.sh @@ -138,9 +138,13 @@ log "Extracting bundle..." mkdir -p "${EXTRACT_DIR}" tar -xzf "${BUNDLE_TARBALL}" -C "${EXTRACT_DIR}" -# Find the extracted bundle directory (named airgap-bundle-openshift-) -BUNDLE_DIR="$(find "${EXTRACT_DIR}" -maxdepth 1 -mindepth 1 -type d -name 'airgap-bundle-openshift-*' | sort | tail -1)" -[[ -n "${BUNDLE_DIR}" ]] || err "Could not find extracted bundle directory in ${EXTRACT_DIR}" +# Resolve the bundle directory from the tarball's own top-level entry rather than +# globbing EXTRACT_DIR — otherwise a stale airgap-bundle-openshift-* directory from a +# previous run could be selected instead of the bundle the user just passed via --bundle. +BUNDLE_TOP="$(tar -tzf "${BUNDLE_TARBALL}" 2>/dev/null | sed 's#/.*##' | grep -m1 '^airgap-bundle-openshift-')" +[[ -n "${BUNDLE_TOP}" ]] || err "Could not find airgap-bundle-openshift-* directory inside ${BUNDLE_TARBALL}" +BUNDLE_DIR="${EXTRACT_DIR}/${BUNDLE_TOP}" +[[ -d "${BUNDLE_DIR}" ]] || err "Expected extracted bundle directory not found: ${BUNDLE_DIR}" log "Bundle extracted to: ${BUNDLE_DIR}" # ── Verify checksums ─────────────────────────────────────────────────────────── diff --git a/tools/cluster_setup/openshift_with_stack.sh b/tools/cluster_setup/openshift_with_stack.sh index bd2e3386..00744fee 100755 --- a/tools/cluster_setup/openshift_with_stack.sh +++ b/tools/cluster_setup/openshift_with_stack.sh @@ -921,13 +921,13 @@ grant_privileged_scc() { install_nfd() { log "Installing Node Feature Discovery Operator (NFD)..." - # Step 1: Subscription + OperatorGroup — idempotent, skip only if already present. + # Step 1: Subscription + OperatorGroup — idempotent, skip creation if already present. + # Do NOT early-return here: a prior run may have created the Subscription but never + # the NodeFeatureDiscovery CR below, so we must always fall through to Step 2. if oc get subscription nfd -n openshift-nfd &>/dev/null; then - log " ✓ NFD subscription already exists, skipping" - return 0 - fi - - oc apply -f - </dev/null || echo "") - if [[ "${phase}" == "Succeeded" ]]; then - log " ✓ NFD operator ready" - break - fi - sleep 10 - retries=$(( retries + 1 )) - log " Waiting for NFD CSV... (${retries}/36, phase=${phase:-pending})" - done + log "Waiting for NFD CSV to succeed..." + local retries=0 + while (( retries < 36 )); do + local phase + phase=$(oc get csv -n openshift-nfd -o jsonpath='{.items[0].status.phase}' 2>/dev/null || echo "") + if [[ "${phase}" == "Succeeded" ]]; then + log " ✓ NFD operator ready" + break + fi + sleep 10 + retries=$(( retries + 1 )) + log " Waiting for NFD CSV... (${retries}/36, phase=${phase:-pending})" + done + fi # Step 2: NodeFeatureDiscovery CR — always ensure it exists, regardless of whether # the Subscription was just created or was already present from a prior run. @@ -1283,13 +1284,30 @@ EOF # k0s installs this as part of cluster setup. OpenShift has no default storage # class on bare-metal, so we install local-path-provisioner the same way. install_local_path_provisioner() { - if oc get storageclass 2>/dev/null | grep -q "(default)"; then + # When the config requests a specific storage class, skip only if THAT class already + # exists — not merely because the cluster has some other default. The AIPlatform CR + # emits storageClassName: ${STORAGE_CLASS}, so skipping on an unrelated default would + # leave VectorDB PVCs bound to a class that was never created and stuck Pending. + if [[ -n "${STORAGE_CLASS}" && "${STORAGE_CLASS}" != "null" ]]; then + if oc get storageclass "${STORAGE_CLASS}" &>/dev/null; then + log " ✓ Requested storage class '${STORAGE_CLASS}' already exists, skipping local-path install" + oc get storageclass + return 0 + fi + if [[ "${STORAGE_CLASS}" != "local-path" ]]; then + warn "Configured storage.storageClass='${STORAGE_CLASS}' does not exist and is not 'local-path'. + This installer only provisions local-path; create '${STORAGE_CLASS}' manually or set storage.storageClass: local-path in the config." + return 0 + fi + # STORAGE_CLASS is 'local-path' and missing — fall through to install it. + elif oc get storageclass 2>/dev/null | grep -q "(default)"; then + # No specific class requested: rely on the cluster default if one already exists. log " ✓ Default storage class already exists, skipping local-path install" oc get storageclass return 0 fi - log "Installing local-path-provisioner (no default storage class found)..." + log "Installing local-path-provisioner..." local _lp_url="${LOCAL_PATH_MANIFEST_URL:-https://raw.githubusercontent.com/rancher/local-path-provisioner/v0.0.26/deploy/local-path-storage.yaml}" [[ "${_lp_url}" == file://* ]] && _lp_url="${_lp_url#file://}" oc apply -f "${_lp_url}" @@ -2365,10 +2383,6 @@ main_install() { phase_end "Model Staging" phase_start "Preflight" - step_start "Model artifact staging" - stage_model_artifacts - step_ok - step_start "Preflight checks" preflight_checks step_ok diff --git a/tools/cluster_setup/prepare_airgap_bundle_openshift.sh b/tools/cluster_setup/prepare_airgap_bundle_openshift.sh index 909ae1f3..e21b0b25 100755 --- a/tools/cluster_setup/prepare_airgap_bundle_openshift.sh +++ b/tools/cluster_setup/prepare_airgap_bundle_openshift.sh @@ -276,6 +276,8 @@ docker.io/semitechnologies/weaviate:stable-v1.28-007846a quay.io/kuberay/operator:v1.2.2 # ── OpenTelemetry ───────────────────────────────────────────────────────────── +# Collector image (injected into OpenTelemetryCollector CRs). The operator's own +# controller-manager + kube-rbac-proxy images are appended below from the bundled chart. docker.io/otel/opentelemetry-collector-contrib:0.122.1 # ── Fluent Bit ──────────────────────────────────────────────────────────────── @@ -292,10 +294,30 @@ grep -oP '(?<=image: )[^\s]+' "${STAGE_DIR}/manifests/cert-manager.yaml" 2>/dev/ cat >> "${STAGE_DIR}/container-images.txt" <<'IMGEOF' # ── local-path-provisioner ──────────────────────────────────────────────────── -# The helper pod image is overridden to ubi-minimal; mirror both. -registry.access.redhat.com/ubi9/ubi-minimal:latest +# The helper pod and `oc debug` relabeling image is hard-coded to ubi8/ubi-minimal +# in openshift_with_stack.sh (no tag → :latest); mirror the exact image it pulls. +registry.access.redhat.com/ubi8/ubi-minimal:latest IMGEOF +# opentelemetry-operator controller images — extracted from the bundled chart so they +# always match the version pulled above (chart version is resolved dynamically). The +# installer only overrides manager.collectorImage; the manager (+ any rbac-proxy) image +# comes from chart defaults, so a disconnected install that mirrors only this list would +# otherwise ImagePullBackOff on the operator pod. Rendered images are inline-quoted, e.g. +# image: "ghcr.io/.../opentelemetry-operator:0.154.0" +# Exclude helm test-hook images (busybox:latest) — they never deploy during install. +{ + echo "" + echo "# ── OpenTelemetry Operator (controller images from bundled chart) ─────────────" + helm template opentelemetry-operator \ + "${STAGE_DIR}/charts/opentelemetry-operator-${OTEL_CHART_VERSION}.tgz" \ + 2>/dev/null \ + | grep -oE 'image:[[:space:]]*"?[^[:space:]"]+' \ + | sed -E 's/^image:[[:space:]]*"?//' \ + | grep -v '^busybox:' \ + | sort -u +} >> "${STAGE_DIR}/container-images.txt" || true + grep -oP '(?<=image: )[^\s]+' "${STAGE_DIR}/manifests/local-path-storage.yaml" 2>/dev/null | sort -u >> "${STAGE_DIR}/container-images.txt" || true cat >> "${STAGE_DIR}/container-images.txt" <<'IMGEOF' @@ -304,14 +326,22 @@ cat >> "${STAGE_DIR}/container-images.txt" <<'IMGEOF' # These are NOT direct image references — they are OLM Subscriptions backed by # OperatorHub catalog content. Mirror them with oc mirror: # +# The GPU Operator ships in the CERTIFIED catalog (certified-operators / +# certified-operator-index) while NFD ships in the redhat-operators catalog +# (redhat-operator-index). They must be mirrored from their respective catalogs, +# matching the installer defaults: operators.gpu.catalogSource=certified-operators, +# operators.nfd.catalogSource=redhat-operators. +# # imageset-config.yaml example: # kind: ImageSetConfiguration # apiVersion: mirror.openshift.io/v1alpha2 # mirror: # operators: -# - catalog: registry.redhat.io/redhat/redhat-operator-index:v4.14 +# - catalog: registry.redhat.io/redhat/certified-operator-index:v4.14 # packages: # - name: gpu-operator-certified +# - catalog: registry.redhat.io/redhat/redhat-operator-index:v4.14 +# packages: # - name: nfd # # Then apply the generated ImageContentSourcePolicy and CatalogSource. From f102933cdb1f6516bcb6092dae11136178a2132c Mon Sep 17 00:00:00 2001 From: Kumar Pratyush Date: Mon, 13 Jul 2026 17:47:11 +0530 Subject: [PATCH 20/80] feat(infra): add k0s AWS CloudFormation provisioner Creates EC2 infrastructure consumed by k0s_cluster_with_stack.sh: - VPC + public subnet + IGW in a single AZ - Security group with self-referencing rule (private-IP k0s comms) - RHEL 9 instances: controller(s), CPU workers, GPU workers, installer - Installer gets an EIP; k0s nodes stay on private IPs only - Separate EBS volumes for /var/lib/k0s (GPU) and /data/minio (installer) - Auto-creates or reuses AWS key pair, downloads .pem locally - Optional MinIO install via install_minio_ec2.sh after provisioning - Generates my-k0s-config.yaml on the installer with private IPs pre-filled - Commands: provision / output / status / destroy / validate / dry-run - Test config (t3.medium, ~$1-2) for Level 2 validation - Design + test plan in K0S_AWS_PROVISION.md Co-Authored-By: Claude Sonnet 4.6 --- tools/cluster_setup/K0S_AWS_PROVISION.md | 292 +++++ .../k0s-aws-provision-config-test.yaml | 50 + .../k0s-aws-provision-config.yaml | 91 ++ tools/cluster_setup/k0s_aws_provision.sh | 1104 +++++++++++++++++ 4 files changed, 1537 insertions(+) create mode 100644 tools/cluster_setup/K0S_AWS_PROVISION.md create mode 100644 tools/cluster_setup/k0s-aws-provision-config-test.yaml create mode 100644 tools/cluster_setup/k0s-aws-provision-config.yaml create mode 100755 tools/cluster_setup/k0s_aws_provision.sh diff --git a/tools/cluster_setup/K0S_AWS_PROVISION.md b/tools/cluster_setup/K0S_AWS_PROVISION.md new file mode 100644 index 00000000..a0427478 --- /dev/null +++ b/tools/cluster_setup/K0S_AWS_PROVISION.md @@ -0,0 +1,292 @@ +# k0s AWS Provisioner — Design & Test Plan + +## Purpose + +`k0s_aws_provision.sh` creates the EC2 infrastructure consumed by +`k0s_cluster_with_stack.sh`. It is a standalone helper: it creates machines, +wires them into a single security group (all traffic on private IPs), optionally +installs MinIO, and prints a ready-to-paste block for `k0s-cluster-config.yaml`. +No changes to the k0s installer itself. + +--- + +## Architecture + +``` +VPC (10.10.0.0/16) +└── Public Subnet (10.10.0.0/24) + Internet Gateway + │ + ├── Installer (t3.large, 50 GB root, optional 500 GB /data/minio) + │ └── Elastic IP ← SSH from your laptop + │ + ├── k0s Controller(s) (100 GB root) + ├── k0s CPU Worker(s) (200 GB root) + └── k0s GPU Worker(s) (100 GB root + 500 GB /var/lib/k0s, separate EBS) + +Security Group k0s--sg + Inbound: all traffic from same SG (self-referencing) ← private-IP k0s comms + Inbound: TCP 22 from sshAllowedCidr ← your laptop → installer + Outbound: all 0.0.0.0/0 + +airgap: false → k0s nodes get public IPs (internet for k0s binary, NVIDIA, images) +airgap: true → k0s nodes get no public IP (pre-provisioned; script SSHes via private IP) +``` + +All inter-node communication uses private IPs. The k0s installer is run **on +the installer machine** (not from your laptop), so no public IP is needed for +k0s nodes. + +--- + +## Script Commands + +```bash +# Deploy stack +./k0s_aws_provision.sh provision [--config k0s-aws-provision-config.yaml] + +# Print copy-paste block for k0s-cluster-config.yaml +./k0s_aws_provision.sh output [--config ...] + +# Show instance states and MinIO health +./k0s_aws_provision.sh status [--config ...] + +# Tear down everything +./k0s_aws_provision.sh destroy [--config ...] + +# Validate CFN template locally without deploying +./k0s_aws_provision.sh validate [--config ...] + +# Dry-run: generate + validate template, print what would be created +./k0s_aws_provision.sh dry-run [--config ...] +``` + +--- + +## Config File: `k0s-aws-provision-config.yaml` + +```yaml +stackName: my-k0s-infra # CloudFormation stack name; must be unique per region +region: us-east-2 +availabilityZone: us-east-2a # all nodes in same AZ (avoids cross-AZ data transfer cost) +airgap: false # true = k0s nodes get no public IP + +# Your IP for SSH access to the installer machine. Use x.x.x.x/32 to lock down. +sshAllowedCidr: "0.0.0.0/0" + +keyPair: + name: "" # existing AWS keypair name; empty = auto-create "-key" + localPath: "" # local .pem path; empty = save to ~/.ssh/.pem + +nodes: + controller: + count: 1 + instanceType: m6i.2xlarge # 8 vCPU, 32 GB RAM + diskGb: 100 + + cpuWorker: + count: 1 + instanceType: m6i.4xlarge # 16 vCPU, 64 GB RAM + diskGb: 200 + + gpuWorker: + count: 2 + instanceType: g6e.12xlarge # 4× NVIDIA L40S (48 GB each), 192 GB RAM + diskGb: 100 # root volume + dataDiskGb: 500 # separate EBS, mounted at /var/lib/k0s + # capacityReservationId: cr-xxxx # optional, for pre-reserved capacity + +installer: + instanceType: t3.large # 2 vCPU, 8 GB RAM + diskGb: 50 + +minio: + enabled: false + dataDiskGb: 500 # separate EBS on installer, mounted at /data/minio + bucket: ai-platform + rootUser: minioadmin + rootPassword: "" # empty = auto-generate (printed in output) + port: 9000 +``` + +--- + +## Output Block (copy-paste into k0s-cluster-config.yaml) + +``` +=== Paste into your k0s-cluster-config.yaml === + + existingIPs: + controllers: + - 10.10.0.12 + workers: + - 10.10.0.45 # cpu-worker-0 + - 10.10.0.67 # gpu-worker-0 + - 10.10.0.89 # gpu-worker-1 + + cluster: + sshKeyPath: ~/.ssh/my-k0s-infra.pem + sshUser: ec2-user + + storage: + objectStore: + endpoint: "http://10.10.0.100:9000" + auth: + rootUser: minioadmin + rootPassword: + +SSH to installer: + ssh -i ~/.ssh/my-k0s-infra.pem ec2-user@ +``` + +--- + +## What `provision` Does Internally + +1. Validate AWS credentials (`aws sts get-caller-identity`) +2. Key pair: create if `keyPair.name` empty, save .pem to `keyPair.localPath` +3. Auto-discover RHEL 9 AMI (`describe-images --owners 309956199498`) +4. Generate CloudFormation template in `/tmp/-cfn.yaml` +5. `aws cloudformation deploy` — idempotent; deletes ROLLBACK stacks first +6. Wait for `CREATE_COMPLETE`; surface `InsufficientInstanceCapacity` clearly +7. Allocate EIP + associate to installer instance +8. Wait for all instances SSH-reachable (port 22, up to 10 min) +9. SCP private key to installer at `~/.ssh/id_rsa` + set `chmod 600` +10. If `minio.enabled`: SCP `install_minio_ec2.sh`, SSH + run it with `--data-dir /data/minio` +11. Wait for MinIO health check on installer +12. Print output block (private IPs, EIP, MinIO endpoint) + +--- + +## What `destroy` Does + +1. Confirmation prompt: type stack name to confirm +2. `aws cloudformation delete-stack` → wait for completion +3. Release EIP +4. Delete orphan EBS volumes tagged with stack name +5. Delete AWS key pair if it was auto-created by this script +6. Offer to delete local `.pem` file + +--- + +## OS: RHEL 9 + +- AMI owner: `309956199498` (Red Hat official marketplace) +- Default user: `ec2-user` +- Includes RHUI subscription (Red Hat Update Infrastructure) — ~$0.10/hr/instance +- `install_minio_ec2.sh` already handles RHEL 9 (SELinux `restorecon`, `firewall-cmd`) +- k0s installer already supports RHEL/CentOS/Amazon Linux via `dnf` + +--- + +## Test Plan + +### Level 1 — Free, local (run before any deployment) + +```bash +# CloudFormation template syntax +aws cloudformation validate-template --template-body file:///tmp/-cfn.yaml + +# Deep lint (install once: pip install cfn-lint) +cfn-lint /tmp/-cfn.yaml +``` + +Use `./k0s_aws_provision.sh dry-run` to generate the template without deploying. + +### Level 2 — Cheap end-to-end (~$2–4, ~20 min) + +Use `k0s-aws-provision-config-test.yaml` (committed alongside the main config): + +```yaml +stackName: k0s-infra-test +nodes: + controller: + instanceType: t3.medium # $0.04/hr — same provisioning flow, no GPU + diskGb: 8 + cpuWorker: + instanceType: t3.medium + diskGb: 8 + gpuWorker: + count: 1 + instanceType: t3.medium + diskGb: 8 + dataDiskGb: 8 # tests EBS attach + mount logic +installer: + instanceType: t3.small + diskGb: 8 +minio: + enabled: true + dataDiskGb: 8 +``` + +Validates: +- VPC / subnet / SG / IGW / route table creation +- RHEL 9 AMI lookup per region +- Key pair auto-create + .pem download +- All instances boot, SSH reachable via EIP +- EBS attach + XFS format + `/var/lib/k0s` mount (UserData) +- MinIO install + health check on RHEL 9 +- `output` prints correct private IPs +- `status` shows all running +- `destroy` tears down everything cleanly + +Run: +```bash +./k0s_aws_provision.sh provision --config k0s-aws-provision-config-test.yaml +./k0s_aws_provision.sh status --config k0s-aws-provision-config-test.yaml +./k0s_aws_provision.sh output --config k0s-aws-provision-config-test.yaml +./k0s_aws_provision.sh destroy --config k0s-aws-provision-config-test.yaml +``` + +### Level 3 — Real GPU integration (~$50–100, ~4 hr) + +Full config with `g6e.12xlarge` GPU workers. After provision: + +```bash +# 1. Provision +./k0s_aws_provision.sh provision --config k0s-aws-provision-config.yaml + +# 2. SSH to installer (script prints exact command) +ssh -i ~/.ssh/.pem ec2-user@ + +# 3. On installer: run k0s install with auto-generated config +CONFIG_FILE=./my-k0s-config.yaml ./k0s_cluster_with_stack.sh install +``` + +`my-k0s-config.yaml` is auto-generated by `output` with private IPs pre-filled +and SCP'd to the installer. No manual IP copying needed. + +--- + +## AWS Authentication + +Login before running any command: +```bash +eval "$(okta-aws-login -a splunkcloud-ai-dev \ + --role-arn arn:aws:iam::658391232643:role/splunkcloud_account_admin)" +``` + +Verify: +```bash +aws sts get-caller-identity +``` + +The provision script runs `aws sts get-caller-identity` at startup and exits +with a clear message if credentials are not configured. + +--- + +## Cost Summary (Level 2 test) + +| Resource | $/hr | 2-hr test | +|----------|------|-----------| +| 4× t3.medium/small | $0.04–0.05 | ~$0.40 | +| 4× EBS gp3 8 GB | negligible | <$0.01 | +| 1× EIP (associated) | free | free | +| RHEL 9 subscription | $0.10/instance | ~$0.80 | +| Data transfer | negligible | <$0.10 | +| **Total** | | **~$1.30** | + +Level 3 with `g6e.12xlarge` (4 hr): +- 2× g6e.12xlarge: ~$16/hr × 2 × 4 hr = ~$128 +- Other instances: ~$2 +- **Total: ~$130** diff --git a/tools/cluster_setup/k0s-aws-provision-config-test.yaml b/tools/cluster_setup/k0s-aws-provision-config-test.yaml new file mode 100644 index 00000000..c5c82195 --- /dev/null +++ b/tools/cluster_setup/k0s-aws-provision-config-test.yaml @@ -0,0 +1,50 @@ +# =================================================================== +# k0s AWS Provisioner — CHEAP TEST CONFIG (Level 2) +# =================================================================== +# Uses t3.small/medium instances instead of GPU boxes. +# Validates the full provision/output/destroy cycle for ~$1-2. +# Run: +# ./k0s_aws_provision.sh provision --config k0s-aws-provision-config-test.yaml +# ./k0s_aws_provision.sh output --config k0s-aws-provision-config-test.yaml +# ./k0s_aws_provision.sh status --config k0s-aws-provision-config-test.yaml +# ./k0s_aws_provision.sh destroy --config k0s-aws-provision-config-test.yaml +# =================================================================== + +stackName: k0s-infra-test +region: us-east-2 +availabilityZone: us-east-2a +airgap: false +sshAllowedCidr: "0.0.0.0/0" + +keyPair: + name: "" + localPath: "" + +nodes: + controller: + count: 1 + instanceType: t3.medium + diskGb: 8 + + cpuWorker: + count: 1 + instanceType: t3.medium + diskGb: 8 + + gpuWorker: + count: 1 + instanceType: t3.medium # no GPU — tests provisioning + EBS mount flow only + diskGb: 8 + dataDiskGb: 8 # small, tests attach + XFS format + mount + +installer: + instanceType: t3.small + diskGb: 8 + +minio: + enabled: true + dataDiskGb: 8 + bucket: ai-platform-test + rootUser: minioadmin + rootPassword: "" + port: 9000 diff --git a/tools/cluster_setup/k0s-aws-provision-config.yaml b/tools/cluster_setup/k0s-aws-provision-config.yaml new file mode 100644 index 00000000..07d2aa48 --- /dev/null +++ b/tools/cluster_setup/k0s-aws-provision-config.yaml @@ -0,0 +1,91 @@ +# =================================================================== +# k0s AWS Provisioner Configuration +# =================================================================== +# Creates EC2 infrastructure for k0s_cluster_with_stack.sh. +# Run: ./k0s_aws_provision.sh provision --config k0s-aws-provision-config.yaml +# +# After provisioning, run: +# ./k0s_aws_provision.sh output --config k0s-aws-provision-config.yaml +# to get the ready-to-paste block for k0s-cluster-config.yaml. +# =================================================================== + +# CloudFormation stack name — must be unique per AWS account + region. +# All resources are tagged with this name for easy cleanup. +stackName: my-k0s-infra # CHANGE THIS + +# AWS region for all resources. +region: us-east-2 # CHANGE THIS + +# Single availability zone for all nodes (avoids cross-AZ data transfer costs). +availabilityZone: us-east-2a # CHANGE THIS to a zone in your region + +# Set to true for air-gapped clusters. When true, k0s nodes are launched without +# a public IP; the k0s installer is expected to run on the installer machine +# (which does have a public IP via EIP) and reaches k0s nodes via private IP. +# When false, k0s nodes get public IPs for internet access during k0s/NVIDIA +# driver install. +airgap: false + +# CIDR allowed to SSH to the installer machine. Use x.x.x.x/32 for your IP. +# k0s nodes are not directly SSH-reachable from the internet — only via the +# installer machine or via their private IP from within the VPC. +sshAllowedCidr: "0.0.0.0/0" # CHANGE THIS: use your-ip/32 + +# ---------- SSH Key Pair ---------- +keyPair: + # Name of an existing AWS key pair. Leave empty to auto-create one named + # "-key". The auto-created key is saved locally (see localPath). + name: "" + + # Local path where the .pem file is (if name is set) or will be saved + # (if auto-created). Leave empty to save to ~/.ssh/.pem. + localPath: "" + +# ---------- Node Configuration ---------- +nodes: + controller: + count: 1 + instanceType: m6i.2xlarge # 8 vCPU, 32 GB RAM + diskGb: 100 # root volume (/var/lib/k0s control plane data) + + cpuWorker: + count: 1 + instanceType: m6i.4xlarge # 16 vCPU, 64 GB RAM + diskGb: 200 # root volume + + gpuWorker: + count: 2 + instanceType: g6e.12xlarge # 4× NVIDIA L40S (48 GB each), 192 GB RAM + diskGb: 100 # root volume + dataDiskGb: 500 # separate EBS mounted at /var/lib/k0s (model weights) + # Uncomment if you have a capacity reservation for this instance type: + # capacityReservationId: "" + +# ---------- Installer Machine ---------- +# This machine runs the k0s install script and optionally hosts MinIO. +# It lives in the same security group as k0s nodes so all communication +# uses private IPs. It gets an Elastic IP for SSH access from your laptop. +installer: + instanceType: t3.large # 2 vCPU, 8 GB RAM + diskGb: 50 # root volume (OS + tools + scripts) + +# ---------- MinIO Configuration ---------- +# When enabled, the provisioner installs MinIO on the installer machine +# using tools/artifacts_download_upload_scripts/install_minio_ec2.sh. +# MinIO data is stored on a separate EBS volume at /data/minio. +minio: + enabled: false # CHANGE TO true to install MinIO + + # Separate EBS data volume for MinIO storage, mounted at /data/minio. + dataDiskGb: 500 + + # MinIO bucket name (created automatically after install). + bucket: ai-platform + + # MinIO credentials. Leave rootPassword empty to auto-generate a secure + # password (it will be printed in the output block). + rootUser: minioadmin + rootPassword: "" # leave empty to auto-generate + + # MinIO server port. + port: 9000 diff --git a/tools/cluster_setup/k0s_aws_provision.sh b/tools/cluster_setup/k0s_aws_provision.sh new file mode 100755 index 00000000..efc43480 --- /dev/null +++ b/tools/cluster_setup/k0s_aws_provision.sh @@ -0,0 +1,1104 @@ +#!/usr/bin/env bash +# ============================================================================= +# k0s AWS Provisioner +# Creates EC2 infrastructure (VPC, instances, EBS, MinIO) consumed by +# k0s_cluster_with_stack.sh. See K0S_AWS_PROVISION.md for full docs. +# +# Usage: +# ./k0s_aws_provision.sh provision [--config FILE] +# ./k0s_aws_provision.sh output [--config FILE] +# ./k0s_aws_provision.sh status [--config FILE] +# ./k0s_aws_provision.sh destroy [--config FILE] +# ./k0s_aws_provision.sh validate [--config FILE] # template only, no deploy +# ./k0s_aws_provision.sh dry-run [--config FILE] # generate + validate, no deploy +# ============================================================================= +set -euo pipefail +export AWS_PAGER="" AWS_DEFAULT_OUTPUT=json PAGER=cat LANG=C LC_ALL=C + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +DEFAULT_CONFIG="${SCRIPT_DIR}/k0s-aws-provision-config.yaml" +CONFIG_FILE="${DEFAULT_CONFIG}" +MINIO_INSTALL_SCRIPT="${SCRIPT_DIR}/../artifacts_download_upload_scripts/install_minio_ec2.sh" + +# ── Logging ────────────────────────────────────────────────────────────────── +log() { echo "[k0s-provision] $*" >&2; } +warn() { echo "[k0s-provision] WARN: $*" >&2; } +err() { echo "[k0s-provision] ERROR: $*" >&2; exit 1; } + +# ── Arg parsing ────────────────────────────────────────────────────────────── +COMMAND="${1:-}" +[[ -z "$COMMAND" ]] && { echo "Usage: $0 [--config FILE]"; exit 1; } +shift +while [[ $# -gt 0 ]]; do + case "$1" in + --config) CONFIG_FILE="$2"; shift 2 ;; + *) err "Unknown option: $1" ;; + esac +done +[[ -f "$CONFIG_FILE" ]] || err "Config file not found: $CONFIG_FILE" + +# ── yq helper ──────────────────────────────────────────────────────────────── +need_yq() { command -v yq &>/dev/null || err "yq is required. Install: brew install yq OR wget -qO /usr/local/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 && chmod +x /usr/local/bin/yq"; } +cfg() { yq eval "${1}" "${CONFIG_FILE}"; } +cfg_default() { yq eval "${1} // \"${2}\"" "${CONFIG_FILE}"; } + +# ── Load config ─────────────────────────────────────────────────────────────── +load_config() { + need_yq + STACK_NAME="$(cfg '.stackName')" + REGION="$(cfg '.region')" + AZ="$(cfg '.availabilityZone')" + AIRGAP="$(cfg_default '.airgap' 'false')" + SSH_CIDR="$(cfg_default '.sshAllowedCidr' '0.0.0.0/0')" + + KEY_NAME="$(cfg_default '.keyPair.name' '')" + KEY_LOCAL="$(cfg_default '.keyPair.localPath' '')" + [[ "$KEY_NAME" == "null" ]] && KEY_NAME="" + [[ "$KEY_LOCAL" == "null" ]] && KEY_LOCAL="" + [[ -z "$KEY_LOCAL" ]] && KEY_LOCAL="${HOME}/.ssh/${STACK_NAME}.pem" + + # Nodes + CTRL_COUNT="$(cfg_default '.nodes.controller.count' '1')" + CTRL_TYPE="$(cfg_default '.nodes.controller.instanceType' 'm6i.2xlarge')" + CTRL_DISK="$(cfg_default '.nodes.controller.diskGb' '100')" + + CPU_COUNT="$(cfg_default '.nodes.cpuWorker.count' '1')" + CPU_TYPE="$(cfg_default '.nodes.cpuWorker.instanceType' 'm6i.4xlarge')" + CPU_DISK="$(cfg_default '.nodes.cpuWorker.diskGb' '200')" + + GPU_COUNT="$(cfg_default '.nodes.gpuWorker.count' '2')" + GPU_TYPE="$(cfg_default '.nodes.gpuWorker.instanceType' 'g6e.12xlarge')" + GPU_DISK="$(cfg_default '.nodes.gpuWorker.diskGb' '100')" + GPU_DATA_DISK="$(cfg_default '.nodes.gpuWorker.dataDiskGb' '500')" + GPU_CAP_RES="$(cfg_default '.nodes.gpuWorker.capacityReservationId' '')" + [[ "$GPU_CAP_RES" == "null" ]] && GPU_CAP_RES="" + + INST_TYPE="$(cfg_default '.installer.instanceType' 't3.large')" + INST_DISK="$(cfg_default '.installer.diskGb' '50')" + + MINIO_ENABLED="$(cfg_default '.minio.enabled' 'false')" + MINIO_DATA_DISK="$(cfg_default '.minio.dataDiskGb' '500')" + MINIO_BUCKET="$(cfg_default '.minio.bucket' 'ai-platform')" + MINIO_USER="$(cfg_default '.minio.rootUser' 'minioadmin')" + MINIO_PASS="$(cfg_default '.minio.rootPassword' '')" + MINIO_PORT="$(cfg_default '.minio.port' '9000')" + [[ "$MINIO_PASS" == "null" || -z "$MINIO_PASS" ]] && MINIO_PASS="" + + # Derived + CFN_TEMPLATE="/tmp/${STACK_NAME}-cfn.yaml" + TAG_KEY="k0s-provision-stack" +} + +# ── AWS auth check ──────────────────────────────────────────────────────────── +check_aws_auth() { + if ! aws sts get-caller-identity --region "${REGION}" &>/dev/null; then + err "AWS credentials not configured or expired. +Run: eval \"\$(okta-aws-login -a splunkcloud-ai-dev --role-arn arn:aws:iam::658391232643:role/splunkcloud_account_admin)\"" + fi + local identity + identity=$(aws sts get-caller-identity --region "${REGION}" --output json) + log "AWS identity: $(echo "$identity" | jq -r '.Arn')" +} + +# ── AMI lookup: RHEL 9 ──────────────────────────────────────────────────────── +get_rhel9_ami() { + local ami + # Red Hat official marketplace AMIs (owner 309956199498), RHEL 9, x86_64, latest + ami=$(aws ec2 describe-images \ + --owners 309956199498 \ + --filters \ + "Name=name,Values=RHEL-9.*_HVM-*-x86_64-*" \ + "Name=state,Values=available" \ + "Name=architecture,Values=x86_64" \ + --query 'sort_by(Images, &CreationDate)[-1].ImageId' \ + --output text \ + --region "${REGION}" 2>/dev/null) + [[ -z "$ami" || "$ami" == "None" ]] && err "Could not find RHEL 9 AMI in region ${REGION}. Check your region or AWS account marketplace access." + log "RHEL 9 AMI: ${ami}" + echo "$ami" +} + +# ── Key pair management ─────────────────────────────────────────────────────── +ensure_key_pair() { + if [[ -z "$KEY_NAME" ]]; then + KEY_NAME="${STACK_NAME}-key" + log "No key pair specified — auto-creating: ${KEY_NAME}" + if aws ec2 describe-key-pairs --key-names "${KEY_NAME}" --region "${REGION}" &>/dev/null; then + warn "Key pair '${KEY_NAME}' already exists in AWS. If you don't have the .pem, delete it first or set keyPair.name in config." + else + aws ec2 create-key-pair \ + --key-name "${KEY_NAME}" \ + --query 'KeyMaterial' \ + --output text \ + --region "${REGION}" > "${KEY_LOCAL}" + chmod 600 "${KEY_LOCAL}" + log "Key pair created, saved to: ${KEY_LOCAL}" + # Tag for cleanup tracking + local kp_id + kp_id=$(aws ec2 describe-key-pairs --key-names "${KEY_NAME}" --region "${REGION}" \ + --query 'KeyPairs[0].KeyPairId' --output text) + aws ec2 create-tags --resources "${kp_id}" \ + --tags "Key=${TAG_KEY},Value=${STACK_NAME}" "Key=auto-created,Value=true" \ + --region "${REGION}" 2>/dev/null || true + fi + else + log "Using existing key pair: ${KEY_NAME}" + if [[ ! -f "$KEY_LOCAL" ]]; then + warn "Key file not found at ${KEY_LOCAL}. Set keyPair.localPath in config if it is elsewhere." + fi + fi +} + +# ── CloudFormation template generation ─────────────────────────────────────── +generate_cfn_template() { + local ami_id="$1" + log "Generating CloudFormation template: ${CFN_TEMPLATE}" + + # Build GPU capacity reservation snippet + local cap_res_snippet="" + if [[ -n "$GPU_CAP_RES" ]]; then + cap_res_snippet=" CapacityReservationSpecification: + CapacityReservationTarget: + CapacityReservationId: ${GPU_CAP_RES}" + fi + + # Build node public IP setting (no public IP if airgap=true for k0s nodes) + local k0s_node_public_ip="true" + [[ "$AIRGAP" == "true" ]] && k0s_node_public_ip="false" + + # UserData: common setup for all RHEL9 nodes + # - passwordless sudo for ec2-user (required by k0s installer) + # - disable host key checking for inter-node SSH + local userdata_common + userdata_common=$(cat <<'UDEOF' +#!/bin/bash +set -e +# Passwordless sudo for ec2-user (required by k0s_cluster_with_stack.sh) +echo 'ec2-user ALL=(ALL) NOPASSWD:ALL' > /etc/sudoers.d/ec2-user-nopasswd +chmod 440 /etc/sudoers.d/ec2-user-nopasswd +# Disable strict host key checking for inter-node SSH (k0s installer uses ssh -o StrictHostKeyChecking=no) +mkdir -p /home/ec2-user/.ssh +printf 'Host *\n StrictHostKeyChecking no\n UserKnownHostsFile /dev/null\n' \ + >> /home/ec2-user/.ssh/config +chown -R ec2-user:ec2-user /home/ec2-user/.ssh +chmod 700 /home/ec2-user/.ssh +chmod 600 /home/ec2-user/.ssh/config +UDEOF +) + + # UserData: mount data disk at a given mount point + # $1 = mount point (e.g. /var/lib/k0s) + # Device name for second disk on Nitro instances is /dev/nvme1n1 + userdata_mount_disk() { + local mp="$1" + cat <&2 + return + fi + # Format only if no filesystem + if ! blkid "\$dev" &>/dev/null; then + mkfs.xfs -f "\$dev" + fi + mkdir -p "${mp}" + grep -q "\$dev" /etc/fstab || echo "\$dev ${mp} xfs defaults,nofail 0 2" >> /etc/fstab + mountpoint -q "${mp}" || mount "${mp}" + echo "Mounted \$dev at ${mp}" +} +mount_data_disk +MDEOF + } + + # Base64-encode a heredoc for CFN UserData + encode_userdata() { + printf '%s' "$1" | base64 | tr -d '\n' + } + + # k0s node UserData (controller + workers) + local ud_k0s_node + ud_k0s_node="${userdata_common}" + + # GPU worker UserData (adds /var/lib/k0s mount) + local ud_gpu_worker + ud_gpu_worker="${userdata_common}$(userdata_mount_disk '/var/lib/k0s')" + + # Installer UserData (adds /data/minio mount if minio enabled) + local ud_installer + ud_installer="${userdata_common}" + if [[ "$MINIO_ENABLED" == "true" ]]; then + ud_installer="${userdata_common}$(userdata_mount_disk '/data/minio')" + fi + + # Generate GPU worker resources + local gpu_instances="" gpu_data_volumes="" gpu_vol_attachments="" + local i + for ((i=0; i "${CFN_TEMPLATE}" </dev/null || echo "NOT_EXISTS") + + if [[ "$existing_status" == "CREATE_COMPLETE" || "$existing_status" == "UPDATE_COMPLETE" ]]; then + log "Stack already exists and is healthy (${existing_status})." + log "To reprovision, run 'destroy' first." + return 0 + elif [[ "$existing_status" != "NOT_EXISTS" ]]; then + log "Existing stack in state ${existing_status} — deleting before retry..." + aws cloudformation delete-stack --stack-name "${STACK_NAME}" --region "${REGION}" + aws cloudformation wait stack-delete-complete --stack-name "${STACK_NAME}" --region "${REGION}" \ + || warn "Wait for stack delete timed out; proceeding anyway" + fi + + aws cloudformation deploy \ + --template-file "${CFN_TEMPLATE}" \ + --stack-name "${STACK_NAME}" \ + --region "${REGION}" \ + --no-fail-on-empty-changeset \ + --capabilities CAPABILITY_IAM \ + 2>&1 | grep -v "^$" | while IFS= read -r line; do log "$line"; done || true + + local final_status + final_status=$(aws cloudformation describe-stacks \ + --stack-name "${STACK_NAME}" --region "${REGION}" \ + --query 'Stacks[0].StackStatus' --output text 2>/dev/null || echo "UNKNOWN") + + if [[ "$final_status" != "CREATE_COMPLETE" && "$final_status" != "UPDATE_COMPLETE" ]]; then + log "Stack events (last 10):" + aws cloudformation describe-stack-events \ + --stack-name "${STACK_NAME}" --region "${REGION}" \ + --query 'StackEvents[0:10].[ResourceStatus,ResourceType,ResourceStatusReason]' \ + --output table 2>/dev/null || true + err "CloudFormation stack failed: ${final_status}" + fi + log "Stack deployed successfully (${final_status})" +} + +# ── Get instance private IPs by tag ────────────────────────────────────────── +get_private_ips_by_role() { + local role="$1" + aws ec2 describe-instances \ + --region "${REGION}" \ + --filters \ + "Name=tag:${TAG_KEY},Values=${STACK_NAME}" \ + "Name=tag:k0s-role,Values=${role}" \ + "Name=instance-state-name,Values=running,pending" \ + --query 'Reservations[].Instances[].PrivateIpAddress' \ + --output text 2>/dev/null | tr '\t' '\n' | grep -v '^$' | sort +} + +get_installer_eip() { + aws cloudformation describe-stacks \ + --stack-name "${STACK_NAME}" --region "${REGION}" \ + --query "Stacks[0].Outputs[?OutputKey=='InstallerPublicIP'].OutputValue" \ + --output text 2>/dev/null || echo "" +} + +get_installer_private_ip() { + aws cloudformation describe-stacks \ + --stack-name "${STACK_NAME}" --region "${REGION}" \ + --query "Stacks[0].Outputs[?OutputKey=='InstallerPrivateIP'].OutputValue" \ + --output text 2>/dev/null || echo "" +} + +# ── Wait for SSH ────────────────────────────────────────────────────────────── +wait_for_ssh() { + local host="$1" label="$2" timeout=600 elapsed=0 + log "Waiting for SSH on ${label} (${host})..." + while ! ssh -i "${KEY_LOCAL}" \ + -o StrictHostKeyChecking=no \ + -o ConnectTimeout=5 \ + -o BatchMode=yes \ + "ec2-user@${host}" 'true' 2>/dev/null; do + sleep 10; elapsed=$((elapsed+10)) + [[ $elapsed -ge $timeout ]] && err "SSH to ${label} (${host}) timed out after ${timeout}s" + echo -n "." + done + echo "" + log "SSH ready: ${label} (${host})" +} + +# ── Wait for all nodes ──────────────────────────────────────────────────────── +wait_all_nodes_ssh() { + local eip; eip=$(get_installer_eip) + [[ -z "$eip" || "$eip" == "None" ]] && err "Could not get installer EIP from stack outputs" + wait_for_ssh "${eip}" "installer" + + local ctrl_ips; readarray -t ctrl_ips < <(get_private_ips_by_role controller) + local cpu_ips; readarray -t cpu_ips < <(get_private_ips_by_role cpu-worker) + local gpu_ips; readarray -t gpu_ips < <(get_private_ips_by_role gpu-worker) + + # SSH to k0s nodes via installer (jump host) + local all_ips=("${ctrl_ips[@]:-}" "${cpu_ips[@]:-}" "${gpu_ips[@]:-}") + for ip in "${all_ips[@]:-}"; do + [[ -z "$ip" ]] && continue + log "Waiting for SSH on k0s node ${ip} (via installer jump host)..." + local elapsed=0 + while ! ssh -i "${KEY_LOCAL}" \ + -o StrictHostKeyChecking=no \ + -o ConnectTimeout=5 \ + -o BatchMode=yes \ + -J "ec2-user@${eip}" \ + "ec2-user@${ip}" 'true' 2>/dev/null; do + sleep 10; elapsed=$((elapsed+10)) + [[ $elapsed -ge 600 ]] && err "SSH to ${ip} timed out" + echo -n "." + done + echo "" + log "SSH ready: ${ip}" + done +} + +# ── Setup installer machine ─────────────────────────────────────────────────── +setup_installer() { + local eip; eip=$(get_installer_eip) + + log "Copying SSH key to installer machine..." + scp -i "${KEY_LOCAL}" \ + -o StrictHostKeyChecking=no \ + -o ConnectTimeout=10 \ + "${KEY_LOCAL}" "ec2-user@${eip}:~/.ssh/id_rsa" + ssh -i "${KEY_LOCAL}" -o StrictHostKeyChecking=no "ec2-user@${eip}" \ + 'chmod 600 ~/.ssh/id_rsa' + + log "Copying k0s cluster scripts to installer..." + scp -i "${KEY_LOCAL}" -o StrictHostKeyChecking=no -r \ + "${SCRIPT_DIR}/"* "ec2-user@${eip}:~/cluster_setup/" + + # Install prerequisites on installer (yq, kubectl, helm, jq) + log "Installing prerequisites on installer machine..." + ssh -i "${KEY_LOCAL}" -o StrictHostKeyChecking=no "ec2-user@${eip}" 'bash -s' <<'PREREQ' +set -e +export PATH="$PATH:/usr/local/bin" +sudo dnf install -y git jq curl unzip 2>/dev/null || sudo yum install -y git jq curl unzip + +# yq +if ! command -v yq &>/dev/null; then + sudo curl -sSL -o /usr/local/bin/yq \ + "https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64" + sudo chmod +x /usr/local/bin/yq +fi + +# kubectl +if ! command -v kubectl &>/dev/null; then + K8S_VER=$(curl -sSL https://dl.k8s.io/release/stable.txt 2>/dev/null || echo v1.32.0) + sudo curl -sSL -o /usr/local/bin/kubectl \ + "https://dl.k8s.io/release/${K8S_VER}/bin/linux/amd64/kubectl" + sudo chmod +x /usr/local/bin/kubectl +fi + +# helm +if ! command -v helm &>/dev/null; then + curl -sSL https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash +fi + +echo "Prerequisites ready: yq=$(yq --version 2>/dev/null) kubectl=$(kubectl version --client -o json 2>/dev/null | jq -r '.clientVersion.gitVersion' || echo unknown)" +PREREQ +} + +# ── MinIO install ───────────────────────────────────────────────────────────── +install_minio() { + local eip; eip=$(get_installer_eip) + local installer_priv_ip; installer_priv_ip=$(get_installer_private_ip) + + [[ ! -f "$MINIO_INSTALL_SCRIPT" ]] && err "MinIO install script not found: ${MINIO_INSTALL_SCRIPT}" + + # Generate password if not set + if [[ -z "$MINIO_PASS" ]]; then + MINIO_PASS=$(openssl rand -base64 18 2>/dev/null | tr -d '=+/' | head -c 24) + log "Auto-generated MinIO password (save it): ${MINIO_PASS}" + fi + + log "Copying MinIO install script to installer..." + scp -i "${KEY_LOCAL}" -o StrictHostKeyChecking=no \ + "${MINIO_INSTALL_SCRIPT}" "ec2-user@${eip}:~/install_minio_ec2.sh" + + log "Running MinIO install on installer machine..." + ssh -i "${KEY_LOCAL}" -o StrictHostKeyChecking=no "ec2-user@${eip}" \ + "sudo bash ~/install_minio_ec2.sh \ + --bucket '${MINIO_BUCKET}' \ + --user '${MINIO_USER}' \ + --password '${MINIO_PASS}' \ + --data-dir /data/minio \ + --port ${MINIO_PORT}" + + log "MinIO install complete at http://${installer_priv_ip}:${MINIO_PORT}" +} + +# ── Generate my-k0s-config.yaml on installer ───────────────────────────────── +push_k0s_config() { + local eip; eip=$(get_installer_eip) + local installer_priv_ip; installer_priv_ip=$(get_installer_private_ip) + + readarray -t ctrl_ips < <(get_private_ips_by_role controller) + readarray -t cpu_ips < <(get_private_ips_by_role cpu-worker) + readarray -t gpu_ips < <(get_private_ips_by_role gpu-worker) + + # Build existingIPs YAML + local ctrl_yaml=""; for ip in "${ctrl_ips[@]:-}"; do [[ -n "$ip" ]] && ctrl_yaml+=" - ${ip}"$'\n'; done + local worker_yaml="" + for ip in "${cpu_ips[@]:-}"; do [[ -n "$ip" ]] && worker_yaml+=" - ${ip} # cpu-worker"$'\n'; done + for ip in "${gpu_ips[@]:-}"; do [[ -n "$ip" ]] && worker_yaml+=" - ${ip} # gpu-worker"$'\n'; done + + local minio_endpoint=""; local minio_block="" + if [[ "$MINIO_ENABLED" == "true" ]]; then + minio_endpoint="http://${installer_priv_ip}:${MINIO_PORT}" + minio_block=" endpoint: \"${minio_endpoint}\" + auth: + rootUser: \"${MINIO_USER}\" + rootPassword: \"${MINIO_PASS}\"" + fi + + # Write config to installer + ssh -i "${KEY_LOCAL}" -o StrictHostKeyChecking=no "ec2-user@${eip}" \ + "cat > ~/cluster_setup/my-k0s-config.yaml" </dev/null || date) +# Stack: ${STACK_NAME} / Region: ${REGION} +# Run on installer: CONFIG_FILE=~/cluster_setup/my-k0s-config.yaml ~/cluster_setup/k0s_cluster_with_stack.sh install + +cluster: + name: ${STACK_NAME}-cluster + region: ${REGION} + sshKeyPath: ~/.ssh/id_rsa + sshUser: ec2-user + +nodes: + existingIPs: + controllers: +${ctrl_yaml} workers: +${worker_yaml} +storage: + storageClass: local-path + vectorDbSize: 50Gi + objectStore: + type: minio + bucket: ${MINIO_BUCKET} +${minio_block} + +images: + registry: "" + registryInsecure: false + operator: + image: "docker.io/splunk/splunk-ai-operator:0.2.0" + splunk: + image: "docker.io/splunk/splunk:10.2-rhel9" + operatorImage: "docker.io/splunk/splunk-operator:3.0.0" + ray: + headImage: "splunk/ray-head-build-preview:latest" + workerImage: "splunk/ray-worker-gpu-build-preview:latest" + weaviate: + image: "docker.io/semitechnologies/weaviate:stable-v1.28-007846a" + saia: + apiImage: "splunk/saia-api-build-preview:latest" + apiV2Image: "splunk/saia-api-v2-build-preview:latest" + dataLoaderImage: "splunk/saia-data-loader-build-preview:latest" + fluentBit: + image: "docker.io/fluent/fluent-bit:1.9.6" + otelCollector: + image: "docker.io/otel/opentelemetry-collector-contrib:0.122.1" + nginx: + image: "docker.io/library/nginx:1.27-alpine" + +operators: + ray: + version: "v1.2.2" + modelVersion: "v0.3.14-36-g1549f5a" + rayVersion: "2.53.0" + certManager: + installCRDs: true + nvidia: + devicePluginVersion: "v0.17.3" + +kubernetes: + namespace: ai-platform + +files: + splunkOperator: "./splunk-operator-cluster.yaml" + aiPlatform: "./artifacts.yaml" + +splunk: + enabled: false + +aiPlatform: + name: "splunk-ai-stack" + defaultAcceleratorType: "L40S" + workerGroupConfig: + imageRegistry: "" + serviceTemplate: + type: NodePort + nodePort: 30080 + features: + - name: "saia" + version: "1.1.0" + cpuScheduling: + nodeSelector: {} + tolerations: [] + gpuScheduling: + nodeSelector: {} + tolerations: + - key: "nvidia.com/gpu" + operator: "Equal" + value: "true" + effect: "NoSchedule" + +metallb: + install: false + chartVersion: "0.14.8" + namespace: "metallb-system" + pool: + name: "saia-pool" + addresses: + - "10.10.0.200-10.10.0.210" + +imagePullSecrets: + secrets: + - ecr-registry-secret + autoCreateECR: false + +ecr: + account: "" + region: ${REGION} +KCEOF + log "k0s config written to installer: ~/cluster_setup/my-k0s-config.yaml" +} + +# ── Output command ──────────────────────────────────────────────────────────── +cmd_output() { + load_config + + local stack_status + stack_status=$(aws cloudformation describe-stacks \ + --stack-name "${STACK_NAME}" --region "${REGION}" \ + --query 'Stacks[0].StackStatus' --output text 2>/dev/null || echo "NOT_EXISTS") + [[ "$stack_status" == "NOT_EXISTS" ]] && err "Stack '${STACK_NAME}' not found. Run 'provision' first." + + local eip; eip=$(get_installer_eip) + local installer_priv_ip; installer_priv_ip=$(get_installer_private_ip) + + readarray -t ctrl_ips < <(get_private_ips_by_role controller) + readarray -t cpu_ips < <(get_private_ips_by_role cpu-worker) + readarray -t gpu_ips < <(get_private_ips_by_role gpu-worker) + + echo "" + echo "================================================================" + echo " k0s AWS Provision — Output" + echo " Stack: ${STACK_NAME} Region: ${REGION}" + echo "================================================================" + echo "" + echo "=== Paste into your k0s-cluster-config.yaml ===" + echo "" + echo " cluster:" + echo " sshKeyPath: ${KEY_LOCAL}" + echo " sshUser: ec2-user" + echo "" + echo " nodes:" + echo " existingIPs:" + echo " controllers:" + for ip in "${ctrl_ips[@]:-}"; do [[ -n "$ip" ]] && echo " - ${ip}"; done + echo " workers:" + for ip in "${cpu_ips[@]:-}"; do [[ -n "$ip" ]] && echo " - ${ip} # cpu-worker"; done + for ip in "${gpu_ips[@]:-}"; do [[ -n "$ip" ]] && echo " - ${ip} # gpu-worker"; done + if [[ "$MINIO_ENABLED" == "true" ]]; then + echo "" + echo " storage:" + echo " objectStore:" + echo " type: minio" + echo " bucket: ${MINIO_BUCKET}" + echo " endpoint: \"http://${installer_priv_ip}:${MINIO_PORT}\"" + echo " auth:" + echo " rootUser: ${MINIO_USER}" + echo " rootPassword: ${MINIO_PASS}" + fi + echo "" + echo "================================================================" + echo " SSH to installer (run k0s install from here):" + echo " ssh -i ${KEY_LOCAL} ec2-user@${eip}" + echo "" + echo " Auto-generated k0s config on installer:" + echo " ~/cluster_setup/my-k0s-config.yaml" + echo "" + echo " Run install:" + echo " CONFIG_FILE=~/cluster_setup/my-k0s-config.yaml \\" + echo " ~/cluster_setup/k0s_cluster_with_stack.sh install" + echo "================================================================" + echo "" +} + +# ── Status command ──────────────────────────────────────────────────────────── +cmd_status() { + load_config + + local stack_status + stack_status=$(aws cloudformation describe-stacks \ + --stack-name "${STACK_NAME}" --region "${REGION}" \ + --query 'Stacks[0].StackStatus' --output text 2>/dev/null || echo "NOT_EXISTS") + + echo "" + echo "Stack: ${STACK_NAME} Status: ${stack_status} Region: ${REGION}" + echo "" + + if [[ "$stack_status" == "NOT_EXISTS" ]]; then + echo " (stack not found)"; return + fi + + echo "Instances:" + aws ec2 describe-instances \ + --region "${REGION}" \ + --filters "Name=tag:${TAG_KEY},Values=${STACK_NAME}" \ + --query 'Reservations[].Instances[].[Tags[?Key==`Name`].Value|[0], PrivateIpAddress, PublicIpAddress, State.Name, InstanceType]' \ + --output table 2>/dev/null || true + + if [[ "$MINIO_ENABLED" == "true" ]]; then + local eip; eip=$(get_installer_eip) + local installer_priv_ip; installer_priv_ip=$(get_installer_private_ip) + echo "" + echo -n "MinIO health (http://${installer_priv_ip}:${MINIO_PORT}): " + local code + code=$(ssh -i "${KEY_LOCAL}" -o StrictHostKeyChecking=no -o ConnectTimeout=5 \ + "ec2-user@${eip}" \ + "curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:${MINIO_PORT}/minio/health/live" 2>/dev/null || echo "unreachable") + if [[ "$code" == "200" ]]; then echo "OK"; else echo "FAIL (${code})"; fi + fi + echo "" +} + +# ── Destroy command ─────────────────────────────────────────────────────────── +cmd_destroy() { + load_config + + local stack_status + stack_status=$(aws cloudformation describe-stacks \ + --stack-name "${STACK_NAME}" --region "${REGION}" \ + --query 'Stacks[0].StackStatus' --output text 2>/dev/null || echo "NOT_EXISTS") + + if [[ "$stack_status" == "NOT_EXISTS" ]]; then + log "Stack '${STACK_NAME}' does not exist — nothing to destroy."; return 0 + fi + + echo "" + echo "WARNING: This will permanently destroy all resources in stack '${STACK_NAME}':" + echo " Region: ${REGION}" + echo " Instances: controllers(${CTRL_COUNT}), cpu-workers(${CPU_COUNT}), gpu-workers(${GPU_COUNT}), installer" + echo " Volumes, EIPs, VPC, and all data will be deleted." + echo "" + read -r -p "Type the stack name to confirm destruction: " confirm + [[ "$confirm" != "${STACK_NAME}" ]] && { log "Confirmation mismatch — aborting."; return 1; } + + # EBS volumes tagged with stack name (orphan-safe delete) + log "Finding and deleting EBS data volumes..." + local vol_ids + readarray -t vol_ids < <(aws ec2 describe-volumes \ + --region "${REGION}" \ + --filters "Name=tag:${TAG_KEY},Values=${STACK_NAME}" "Name=status,Values=available,in-use" \ + --query 'Volumes[].VolumeId' --output text 2>/dev/null | tr '\t' '\n' | grep -v '^$') + + log "Deleting CloudFormation stack: ${STACK_NAME}..." + aws cloudformation delete-stack --stack-name "${STACK_NAME}" --region "${REGION}" + log "Waiting for stack deletion (this can take 5-10 minutes)..." + aws cloudformation wait stack-delete-complete --stack-name "${STACK_NAME}" --region "${REGION}" \ + || warn "Stack delete wait timed out; check AWS console for stragglers" + + # Delete any volumes that survived (e.g. if attach was stuck) + for vol_id in "${vol_ids[@]:-}"; do + [[ -z "$vol_id" ]] && continue + log "Deleting orphan volume: ${vol_id}" + aws ec2 delete-volume --volume-id "${vol_id}" --region "${REGION}" 2>/dev/null || true + done + + # Delete auto-created key pair + if [[ -z "$(cfg_default '.keyPair.name' '')" || "$(cfg_default '.keyPair.name' '')" == "null" ]]; then + local auto_key="${STACK_NAME}-key" + local kp_auto_created + kp_auto_created=$(aws ec2 describe-key-pairs \ + --key-names "${auto_key}" --region "${REGION}" \ + --query 'KeyPairs[0].Tags[?Key==`auto-created`].Value|[0]' \ + --output text 2>/dev/null || echo "") + if [[ "$kp_auto_created" == "true" ]]; then + log "Deleting auto-created key pair: ${auto_key}" + aws ec2 delete-key-pair --key-name "${auto_key}" --region "${REGION}" 2>/dev/null || true + if [[ -f "${KEY_LOCAL}" ]]; then + read -r -p "Delete local key file ${KEY_LOCAL}? [y/N]: " del_local + [[ "${del_local,,}" == "y" ]] && rm -f "${KEY_LOCAL}" && log "Deleted ${KEY_LOCAL}" + fi + fi + fi + + log "Destroy complete." +} + +# ── Validate / dry-run ──────────────────────────────────────────────────────── +cmd_validate() { + load_config + local ami_id; ami_id=$(get_rhel9_ami) + ensure_key_pair + generate_cfn_template "${ami_id}" + log "Validating template with AWS..." + aws cloudformation validate-template \ + --template-body "file://${CFN_TEMPLATE}" \ + --region "${REGION}" && log "Template is valid." + if command -v cfn-lint &>/dev/null; then + log "Running cfn-lint..." + cfn-lint "${CFN_TEMPLATE}" && log "cfn-lint: no issues." + else + warn "cfn-lint not installed (pip install cfn-lint) — skipping deep lint." + fi +} + +cmd_dryrun() { + cmd_validate + echo "" + echo "=== Dry-run complete. Template at: ${CFN_TEMPLATE} ===" + echo "Would create:" + echo " 1 VPC, 1 Subnet, 1 IGW, 1 Security Group" + echo " 1 Installer (${INST_TYPE}, ${INST_DISK}GB) + EIP" + echo " ${CTRL_COUNT} Controller(s) (${CTRL_TYPE}, ${CTRL_DISK}GB)" + echo " ${CPU_COUNT} CPU Worker(s) (${CPU_TYPE}, ${CPU_DISK}GB)" + echo " ${GPU_COUNT} GPU Worker(s) (${GPU_TYPE}, ${GPU_DISK}GB root + ${GPU_DATA_DISK}GB /var/lib/k0s)" + if [[ "$MINIO_ENABLED" == "true" ]]; then + echo " MinIO on installer (${MINIO_DATA_DISK}GB /data/minio)" + fi + echo " Region: ${REGION} AZ: ${AZ}" +} + +# ── Provision command ───────────────────────────────────────────────────────── +cmd_provision() { + load_config + check_aws_auth + + log "=== k0s AWS Provisioner ===" + log "Stack: ${STACK_NAME} Region: ${REGION} AZ: ${AZ}" + + local ami_id; ami_id=$(get_rhel9_ami) + ensure_key_pair + generate_cfn_template "${ami_id}" + + # Validate before deploying + log "Validating template..." + aws cloudformation validate-template \ + --template-body "file://${CFN_TEMPLATE}" \ + --region "${REGION}" &>/dev/null || err "Template validation failed. Run 'validate' for details." + + deploy_stack + + log "Waiting for all nodes to be SSH-reachable..." + wait_all_nodes_ssh + + setup_installer + + if [[ "$MINIO_ENABLED" == "true" ]]; then + # Wait for /data/minio mount to complete (UserData runs async on RHEL 9) + local eip; eip=$(get_installer_eip) + log "Waiting for /data/minio mount on installer..." + local elapsed=0 + while ! ssh -i "${KEY_LOCAL}" -o StrictHostKeyChecking=no "ec2-user@${eip}" \ + 'mountpoint -q /data/minio' 2>/dev/null; do + sleep 10; elapsed=$((elapsed+10)) + [[ $elapsed -ge 300 ]] && err "/data/minio not mounted after 5 minutes. Check UserData logs: sudo cat /var/log/cloud-init-output.log" + echo -n "." + done; echo "" + install_minio + fi + + push_k0s_config + cmd_output +} + +# ── Dispatch ────────────────────────────────────────────────────────────────── +case "$COMMAND" in + provision) cmd_provision ;; + output) load_config; cmd_output ;; + status) cmd_status ;; + destroy) cmd_destroy ;; + validate) cmd_validate ;; + dry-run) load_config; cmd_dryrun ;; + *) err "Unknown command: ${COMMAND}. Valid: provision output status destroy validate dry-run" ;; +esac From 7b93a8ba534f9b64d2f5227ef8952261d2d02eb4 Mon Sep 17 00:00:00 2001 From: Kumar Pratyush Date: Mon, 13 Jul 2026 23:39:04 +0530 Subject: [PATCH 21/80] feat(infra): rewrite k0s AWS provisioner with SCP compliance and EBS mount fix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rewrites CloudFormation-based provisioner to direct AWS CLI calls, bypassing SCP p-m68tib3s which blocks ec2:RunInstances via CloudFormation. Key changes: - IMDSv2 required (HttpTokens=required) on all instances — required by SCP - EBS encryption on all volumes — required by SCP - Existing VPC only (vpc-09b191e89c83d588e) — no new VPC creation - Separate subnets: k0s nodes in private subnet, installer in public subnet with EIP - SSH ProxyCommand instead of -J flag (fixes key forwarding on macOS) - mount_disk_via_ssh: mount EBS data disks via SSH after attach, not UserData (UserData runs before EBS volumes are attached by the provisioner) - destroy --yes / -y flag for non-interactive teardown - Fix _INSTANCE grep pattern to include INSTALLER_INSTANCE (was missing) - Fix ${var,,} bash 3.2 incompatibility with tr [:upper:] [:lower:] Validated end-to-end: provision → status → output → destroy all pass. All 4 instances, EBS volumes, EIP, SG, key pair created and destroyed cleanly. Co-Authored-By: Claude Sonnet 4.6 --- .../k0s-aws-provision-config-test.yaml | 27 +- .../k0s-aws-provision-config.yaml | 18 +- tools/cluster_setup/k0s_aws_provision.sh | 1306 ++++++++--------- 3 files changed, 613 insertions(+), 738 deletions(-) diff --git a/tools/cluster_setup/k0s-aws-provision-config-test.yaml b/tools/cluster_setup/k0s-aws-provision-config-test.yaml index c5c82195..ec484308 100644 --- a/tools/cluster_setup/k0s-aws-provision-config-test.yaml +++ b/tools/cluster_setup/k0s-aws-provision-config-test.yaml @@ -1,8 +1,8 @@ # =================================================================== -# k0s AWS Provisioner — CHEAP TEST CONFIG (Level 2) +# k0s AWS Provisioner -- CHEAP TEST CONFIG (Level 2) # =================================================================== # Uses t3.small/medium instances instead of GPU boxes. -# Validates the full provision/output/destroy cycle for ~$1-2. +# Validates the full provision/output/status/destroy cycle for ~$1-2. # Run: # ./k0s_aws_provision.sh provision --config k0s-aws-provision-config-test.yaml # ./k0s_aws_provision.sh output --config k0s-aws-provision-config-test.yaml @@ -11,11 +11,16 @@ # =================================================================== stackName: k0s-infra-test -region: us-east-2 -availabilityZone: us-east-2a -airgap: false +region: us-west-2 +availabilityZone: us-west-2a sshAllowedCidr: "0.0.0.0/0" +# Existing ai-platform VPC -- required by SCP (account blocks creating new VPCs for RunInstances) +network: + vpcId: vpc-09b191e89c83d588e + subnetId: subnet-0f10872b190a44521 # private us-west-2a (k0s nodes, NAT outbound) + installerSubnetId: subnet-0561d78f4f4744596 # public us-west-2a (installer, IGW for SSH via EIP) + keyPair: name: "" localPath: "" @@ -24,22 +29,22 @@ nodes: controller: count: 1 instanceType: t3.medium - diskGb: 8 + diskGb: 10 cpuWorker: count: 1 instanceType: t3.medium - diskGb: 8 + diskGb: 10 gpuWorker: count: 1 - instanceType: t3.medium # no GPU — tests provisioning + EBS mount flow only - diskGb: 8 - dataDiskGb: 8 # small, tests attach + XFS format + mount + instanceType: t3.medium # no GPU -- tests provisioning + EBS mount flow only + diskGb: 10 + dataDiskGb: 8 # tests attach + XFS format + mount installer: instanceType: t3.small - diskGb: 8 + diskGb: 10 minio: enabled: true diff --git a/tools/cluster_setup/k0s-aws-provision-config.yaml b/tools/cluster_setup/k0s-aws-provision-config.yaml index 07d2aa48..8f286f12 100644 --- a/tools/cluster_setup/k0s-aws-provision-config.yaml +++ b/tools/cluster_setup/k0s-aws-provision-config.yaml @@ -27,10 +27,24 @@ availabilityZone: us-east-2a # CHANGE THIS to a zone in your region airgap: false # CIDR allowed to SSH to the installer machine. Use x.x.x.x/32 for your IP. -# k0s nodes are not directly SSH-reachable from the internet — only via the -# installer machine or via their private IP from within the VPC. sshAllowedCidr: "0.0.0.0/0" # CHANGE THIS: use your-ip/32 +# ---------- Network (existing VPC) ---------- +# SCP requires instances be launched in the ai-platform VPC. +# Do NOT create a new VPC — it will be blocked. +network: + vpcId: vpc-09b191e89c83d588e # ai-platform-us-west-2-vpc + subnetId: "" # k0s nodes: leave empty to auto-select private subnet in AZ + installerSubnetId: "" # installer: leave empty to auto-select public subnet in AZ + # Available subnets: + # Private (k0s nodes — NAT gateway outbound): + # subnet-0f10872b190a44521 us-west-2a + # subnet-0451eb662c568bfdf us-west-2b + # subnet-0289f96100d496522 us-west-2c + # subnet-0a44f4ec8e746d598 us-west-2d + # Public (installer — IGW, EIP SSH access): + # subnet-0561d78f4f4744596 us-west-2a + # ---------- SSH Key Pair ---------- keyPair: # Name of an existing AWS key pair. Leave empty to auto-create one named diff --git a/tools/cluster_setup/k0s_aws_provision.sh b/tools/cluster_setup/k0s_aws_provision.sh index efc43480..16e18b0e 100755 --- a/tools/cluster_setup/k0s_aws_provision.sh +++ b/tools/cluster_setup/k0s_aws_provision.sh @@ -1,16 +1,21 @@ #!/usr/bin/env bash # ============================================================================= # k0s AWS Provisioner -# Creates EC2 infrastructure (VPC, instances, EBS, MinIO) consumed by -# k0s_cluster_with_stack.sh. See K0S_AWS_PROVISION.md for full docs. +# Creates EC2 instances (with EBS) in an existing VPC/subnet for use by +# k0s_cluster_with_stack.sh. Uses direct AWS CLI calls with required SCP flags: +# - IMDSv2 (HttpTokens=required) +# - EBS encryption +# - Existing VPC (vpc-09b191e89c83d588e, ai-platform-us-west-2-vpc) +# +# The VPC has private subnets + NAT gateway. All nodes get private IPs. +# One node (installer) also gets an EIP for SSH access from your laptop. # # Usage: # ./k0s_aws_provision.sh provision [--config FILE] # ./k0s_aws_provision.sh output [--config FILE] # ./k0s_aws_provision.sh status [--config FILE] -# ./k0s_aws_provision.sh destroy [--config FILE] -# ./k0s_aws_provision.sh validate [--config FILE] # template only, no deploy -# ./k0s_aws_provision.sh dry-run [--config FILE] # generate + validate, no deploy +# ./k0s_aws_provision.sh destroy [--config FILE] [--yes] +# ./k0s_aws_provision.sh dry-run [--config FILE] # ============================================================================= set -euo pipefail export AWS_PAGER="" AWS_DEFAULT_OUTPUT=json PAGER=cat LANG=C LC_ALL=C @@ -20,58 +25,71 @@ DEFAULT_CONFIG="${SCRIPT_DIR}/k0s-aws-provision-config.yaml" CONFIG_FILE="${DEFAULT_CONFIG}" MINIO_INSTALL_SCRIPT="${SCRIPT_DIR}/../artifacts_download_upload_scripts/install_minio_ec2.sh" -# ── Logging ────────────────────────────────────────────────────────────────── +# -- Logging -- log() { echo "[k0s-provision] $*" >&2; } warn() { echo "[k0s-provision] WARN: $*" >&2; } err() { echo "[k0s-provision] ERROR: $*" >&2; exit 1; } -# ── Arg parsing ────────────────────────────────────────────────────────────── +# -- Arg parsing -- COMMAND="${1:-}" -[[ -z "$COMMAND" ]] && { echo "Usage: $0 [--config FILE]"; exit 1; } +[[ -z "$COMMAND" ]] && { echo "Usage: $0 [--config FILE]"; exit 1; } shift +FORCE_DESTROY=false while [[ $# -gt 0 ]]; do case "$1" in --config) CONFIG_FILE="$2"; shift 2 ;; + --yes|-y) FORCE_DESTROY=true; shift ;; *) err "Unknown option: $1" ;; esac done [[ -f "$CONFIG_FILE" ]] || err "Config file not found: $CONFIG_FILE" -# ── yq helper ──────────────────────────────────────────────────────────────── -need_yq() { command -v yq &>/dev/null || err "yq is required. Install: brew install yq OR wget -qO /usr/local/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 && chmod +x /usr/local/bin/yq"; } -cfg() { yq eval "${1}" "${CONFIG_FILE}"; } +# -- yq helper -- +need_yq() { command -v yq &>/dev/null || err "yq is required. Install: brew install yq"; } +cfg() { yq eval "${1}" "${CONFIG_FILE}"; } cfg_default() { yq eval "${1} // \"${2}\"" "${CONFIG_FILE}"; } -# ── Load config ─────────────────────────────────────────────────────────────── +# -- State file: tracks IDs of all created resources for destroy -- +STATE_FILE="" # set in load_config + +save_state() { local key="$1" val="$2"; echo "${key}=${val}" >> "${STATE_FILE}"; } +load_state() { [[ -f "${STATE_FILE}" ]] && source "${STATE_FILE}" || true; } + +# -- Load config -- load_config() { need_yq STACK_NAME="$(cfg '.stackName')" REGION="$(cfg '.region')" AZ="$(cfg '.availabilityZone')" - AIRGAP="$(cfg_default '.airgap' 'false')" SSH_CIDR="$(cfg_default '.sshAllowedCidr' '0.0.0.0/0')" + # Existing VPC + subnet — required by SCP (no creating new VPCs) + VPC_ID="$(cfg_default '.network.vpcId' 'vpc-09b191e89c83d588e')" + SUBNET_ID="$(cfg_default '.network.subnetId' '')" + [[ "$SUBNET_ID" == "null" || -z "$SUBNET_ID" ]] && SUBNET_ID="" + # Installer goes in the public subnet (needs inbound SSH via EIP) + # k0s nodes go in the private subnet (outbound only via NAT) + INSTALLER_SUBNET_ID="$(cfg_default '.network.installerSubnetId' '')" + [[ "$INSTALLER_SUBNET_ID" == "null" || -z "$INSTALLER_SUBNET_ID" ]] && INSTALLER_SUBNET_ID="" + KEY_NAME="$(cfg_default '.keyPair.name' '')" KEY_LOCAL="$(cfg_default '.keyPair.localPath' '')" [[ "$KEY_NAME" == "null" ]] && KEY_NAME="" [[ "$KEY_LOCAL" == "null" ]] && KEY_LOCAL="" [[ -z "$KEY_LOCAL" ]] && KEY_LOCAL="${HOME}/.ssh/${STACK_NAME}.pem" - # Nodes CTRL_COUNT="$(cfg_default '.nodes.controller.count' '1')" - CTRL_TYPE="$(cfg_default '.nodes.controller.instanceType' 'm6i.2xlarge')" - CTRL_DISK="$(cfg_default '.nodes.controller.diskGb' '100')" + CTRL_TYPE="$(cfg_default '.nodes.controller.instanceType' 'm6i.2xlarge')" + CTRL_DISK="$(cfg_default '.nodes.controller.diskGb' '100')" CPU_COUNT="$(cfg_default '.nodes.cpuWorker.count' '1')" - CPU_TYPE="$(cfg_default '.nodes.cpuWorker.instanceType' 'm6i.4xlarge')" - CPU_DISK="$(cfg_default '.nodes.cpuWorker.diskGb' '200')" + CPU_TYPE="$(cfg_default '.nodes.cpuWorker.instanceType' 'm6i.4xlarge')" + CPU_DISK="$(cfg_default '.nodes.cpuWorker.diskGb' '200')" GPU_COUNT="$(cfg_default '.nodes.gpuWorker.count' '2')" - GPU_TYPE="$(cfg_default '.nodes.gpuWorker.instanceType' 'g6e.12xlarge')" - GPU_DISK="$(cfg_default '.nodes.gpuWorker.diskGb' '100')" + GPU_TYPE="$(cfg_default '.nodes.gpuWorker.instanceType' 'g6e.12xlarge')" + GPU_DISK="$(cfg_default '.nodes.gpuWorker.diskGb' '100')" GPU_DATA_DISK="$(cfg_default '.nodes.gpuWorker.dataDiskGb' '500')" - GPU_CAP_RES="$(cfg_default '.nodes.gpuWorker.capacityReservationId' '')" - [[ "$GPU_CAP_RES" == "null" ]] && GPU_CAP_RES="" INST_TYPE="$(cfg_default '.installer.instanceType' 't3.large')" INST_DISK="$(cfg_default '.installer.diskGb' '50')" @@ -84,47 +102,85 @@ load_config() { MINIO_PORT="$(cfg_default '.minio.port' '9000')" [[ "$MINIO_PASS" == "null" || -z "$MINIO_PASS" ]] && MINIO_PASS="" - # Derived - CFN_TEMPLATE="/tmp/${STACK_NAME}-cfn.yaml" TAG_KEY="k0s-provision-stack" + STATE_FILE="${HOME}/.k0s-provision-${STACK_NAME}.state" } -# ── AWS auth check ──────────────────────────────────────────────────────────── +# -- AWS auth check -- check_aws_auth() { - if ! aws sts get-caller-identity --region "${REGION}" &>/dev/null; then - err "AWS credentials not configured or expired. -Run: eval \"\$(okta-aws-login -a splunkcloud-ai-dev --role-arn arn:aws:iam::658391232643:role/splunkcloud_account_admin)\"" - fi local identity - identity=$(aws sts get-caller-identity --region "${REGION}" --output json) + identity=$(aws sts get-caller-identity --region "${REGION}" --output json 2>/dev/null) \ + || err "AWS credentials not configured or expired. +Run: eval \"\$(okta-aws-login -a splunkcloud-ai-dev --role-arn arn:aws:iam::658391232643:role/splunkcloud_account_admin)\"" log "AWS identity: $(echo "$identity" | jq -r '.Arn')" } -# ── AMI lookup: RHEL 9 ──────────────────────────────────────────────────────── +# -- Pick subnets: k0s nodes in private, installer in public -- +pick_subnet() { + # k0s node subnet + if [[ -n "$SUBNET_ID" ]]; then + log "Using configured k0s subnet: ${SUBNET_ID}" + else + SUBNET_ID=$(aws ec2 describe-subnets --region "${REGION}" \ + --filters "Name=vpc-id,Values=${VPC_ID}" \ + "Name=availabilityZone,Values=${AZ}" \ + --query 'Subnets[?Tags[?Key==`Name`] | [?contains(Value, `private`)] | [0]] | [0].SubnetId' \ + --output text 2>/dev/null || echo "") + if [[ -z "$SUBNET_ID" || "$SUBNET_ID" == "None" ]]; then + SUBNET_ID=$(aws ec2 describe-subnets --region "${REGION}" \ + --filters "Name=vpc-id,Values=${VPC_ID}" \ + "Name=availabilityZone,Values=${AZ}" \ + --query 'Subnets[0].SubnetId' --output text) + fi + [[ -z "$SUBNET_ID" || "$SUBNET_ID" == "None" ]] && \ + err "No k0s subnet found in VPC ${VPC_ID} / AZ ${AZ}. Set network.subnetId in config." + log "Auto-selected k0s subnet: ${SUBNET_ID} (${AZ})" + fi + + # Installer subnet — must be public (has IGW route) for inbound SSH via EIP + if [[ -n "$INSTALLER_SUBNET_ID" ]]; then + log "Using configured installer subnet: ${INSTALLER_SUBNET_ID}" + else + INSTALLER_SUBNET_ID=$(aws ec2 describe-subnets --region "${REGION}" \ + --filters "Name=vpc-id,Values=${VPC_ID}" \ + "Name=availabilityZone,Values=${AZ}" \ + --query 'Subnets[?Tags[?Key==`Name`] | [?contains(Value, `public`)] | [0]] | [0].SubnetId' \ + --output text 2>/dev/null || echo "") + if [[ -z "$INSTALLER_SUBNET_ID" || "$INSTALLER_SUBNET_ID" == "None" ]]; then + # Fallback: use same subnet as k0s nodes and warn + INSTALLER_SUBNET_ID="$SUBNET_ID" + warn "No public subnet found in ${AZ}; installer will use the same private subnet as k0s nodes." + warn "EIP may not be SSH-reachable. Set network.installerSubnetId to the public subnet." + else + log "Auto-selected installer subnet (public): ${INSTALLER_SUBNET_ID} (${AZ})" + fi + fi +} + +# -- RHEL 9 AMI -- get_rhel9_ami() { local ami - # Red Hat official marketplace AMIs (owner 309956199498), RHEL 9, x86_64, latest ami=$(aws ec2 describe-images \ --owners 309956199498 \ - --filters \ - "Name=name,Values=RHEL-9.*_HVM-*-x86_64-*" \ - "Name=state,Values=available" \ - "Name=architecture,Values=x86_64" \ + --filters "Name=name,Values=RHEL-9.*_HVM-*-x86_64-*" \ + "Name=state,Values=available" \ + "Name=architecture,Values=x86_64" \ --query 'sort_by(Images, &CreationDate)[-1].ImageId' \ - --output text \ - --region "${REGION}" 2>/dev/null) - [[ -z "$ami" || "$ami" == "None" ]] && err "Could not find RHEL 9 AMI in region ${REGION}. Check your region or AWS account marketplace access." + --output text --region "${REGION}" 2>/dev/null) + [[ -z "$ami" || "$ami" == "None" ]] && \ + err "Could not find RHEL 9 AMI in region ${REGION}." log "RHEL 9 AMI: ${ami}" echo "$ami" } -# ── Key pair management ─────────────────────────────────────────────────────── +# -- Key pair -- ensure_key_pair() { if [[ -z "$KEY_NAME" ]]; then KEY_NAME="${STACK_NAME}-key" - log "No key pair specified — auto-creating: ${KEY_NAME}" - if aws ec2 describe-key-pairs --key-names "${KEY_NAME}" --region "${REGION}" &>/dev/null; then - warn "Key pair '${KEY_NAME}' already exists in AWS. If you don't have the .pem, delete it first or set keyPair.name in config." + if aws ec2 describe-key-pairs --key-names "${KEY_NAME}" \ + --region "${REGION}" &>/dev/null; then + warn "Key pair '${KEY_NAME}' already exists in AWS — reusing." + if [[ ! -f "$KEY_LOCAL" ]]; then warn "Local key file not found at ${KEY_LOCAL}. You may need to set keyPair.localPath."; fi else aws ec2 create-key-pair \ --key-name "${KEY_NAME}" \ @@ -133,576 +189,269 @@ ensure_key_pair() { --region "${REGION}" > "${KEY_LOCAL}" chmod 600 "${KEY_LOCAL}" log "Key pair created, saved to: ${KEY_LOCAL}" - # Tag for cleanup tracking local kp_id - kp_id=$(aws ec2 describe-key-pairs --key-names "${KEY_NAME}" --region "${REGION}" \ - --query 'KeyPairs[0].KeyPairId' --output text) - aws ec2 create-tags --resources "${kp_id}" \ - --tags "Key=${TAG_KEY},Value=${STACK_NAME}" "Key=auto-created,Value=true" \ - --region "${REGION}" 2>/dev/null || true + kp_id=$(aws ec2 describe-key-pairs --key-names "${KEY_NAME}" \ + --region "${REGION}" --query 'KeyPairs[0].KeyPairId' --output text 2>/dev/null || echo "") + if [[ -n "$kp_id" ]]; then + aws ec2 create-tags \ + --resources "${kp_id}" \ + --tags "Key=${TAG_KEY},Value=${STACK_NAME}" "Key=auto-created,Value=true" \ + --region "${REGION}" 2>/dev/null || true + fi fi else log "Using existing key pair: ${KEY_NAME}" - if [[ ! -f "$KEY_LOCAL" ]]; then - warn "Key file not found at ${KEY_LOCAL}. Set keyPair.localPath in config if it is elsewhere." - fi + if [[ ! -f "$KEY_LOCAL" ]]; then warn "Key file not found at ${KEY_LOCAL}. Set keyPair.localPath in config."; fi fi } -# ── CloudFormation template generation ─────────────────────────────────────── -generate_cfn_template() { - local ami_id="$1" - log "Generating CloudFormation template: ${CFN_TEMPLATE}" - - # Build GPU capacity reservation snippet - local cap_res_snippet="" - if [[ -n "$GPU_CAP_RES" ]]; then - cap_res_snippet=" CapacityReservationSpecification: - CapacityReservationTarget: - CapacityReservationId: ${GPU_CAP_RES}" +# -- Create or reuse a security group for this stack -- +ensure_security_group() { + # Check if our SG already exists + local existing + existing=$(aws ec2 describe-security-groups --region "${REGION}" \ + --filters "Name=vpc-id,Values=${VPC_ID}" \ + "Name=group-name,Values=${STACK_NAME}-sg" \ + --query 'SecurityGroups[0].GroupId' --output text 2>/dev/null || echo "") + if [[ -n "$existing" && "$existing" != "None" ]]; then + SG_ID="$existing" + log "Reusing existing security group: ${SG_ID}" + return fi - # Build node public IP setting (no public IP if airgap=true for k0s nodes) - local k0s_node_public_ip="true" - [[ "$AIRGAP" == "true" ]] && k0s_node_public_ip="false" + log "Creating security group..." + SG_ID=$(aws ec2 create-security-group \ + --group-name "${STACK_NAME}-sg" \ + --description "k0s cluster ${STACK_NAME} - all private-IP traffic + SSH" \ + --vpc-id "${VPC_ID}" \ + --region "${REGION}" \ + --query 'GroupId' --output text) + + # Self-referencing: all intra-cluster traffic + aws ec2 authorize-security-group-ingress \ + --group-id "${SG_ID}" \ + --ip-permissions "[{\"IpProtocol\":\"-1\",\"UserIdGroupPairs\":[{\"GroupId\":\"${SG_ID}\"}]}]" \ + --region "${REGION}" &>/dev/null + + # SSH from allowed CIDR (for EIP-bound installer) + aws ec2 authorize-security-group-ingress \ + --group-id "${SG_ID}" \ + --protocol tcp --port 22 \ + --cidr "${SSH_CIDR}" \ + --region "${REGION}" &>/dev/null + + aws ec2 create-tags --resources "${SG_ID}" \ + --tags "Key=Name,Value=${STACK_NAME}-sg" "Key=${TAG_KEY},Value=${STACK_NAME}" \ + --region "${REGION}" + log "Security group: ${SG_ID}" +} - # UserData: common setup for all RHEL9 nodes - # - passwordless sudo for ec2-user (required by k0s installer) - # - disable host key checking for inter-node SSH - local userdata_common - userdata_common=$(cat <<'UDEOF' +# -- UserData builders -- +userdata_base() { + cat <<'UD' #!/bin/bash set -e -# Passwordless sudo for ec2-user (required by k0s_cluster_with_stack.sh) echo 'ec2-user ALL=(ALL) NOPASSWD:ALL' > /etc/sudoers.d/ec2-user-nopasswd chmod 440 /etc/sudoers.d/ec2-user-nopasswd -# Disable strict host key checking for inter-node SSH (k0s installer uses ssh -o StrictHostKeyChecking=no) mkdir -p /home/ec2-user/.ssh printf 'Host *\n StrictHostKeyChecking no\n UserKnownHostsFile /dev/null\n' \ >> /home/ec2-user/.ssh/config chown -R ec2-user:ec2-user /home/ec2-user/.ssh chmod 700 /home/ec2-user/.ssh chmod 600 /home/ec2-user/.ssh/config -UDEOF -) - - # UserData: mount data disk at a given mount point - # $1 = mount point (e.g. /var/lib/k0s) - # Device name for second disk on Nitro instances is /dev/nvme1n1 - userdata_mount_disk() { - local mp="$1" - cat <&2 - return - fi - # Format only if no filesystem - if ! blkid "\$dev" &>/dev/null; then - mkfs.xfs -f "\$dev" - fi + for c in /dev/nvme1n1 /dev/xvdb /dev/sdb; do [ -b "\$c" ] && dev="\$c" && break; done + [ -z "\$dev" ] && { echo "WARNING: data disk not found, skipping ${mp} mount" >&2; return; } + blkid "\$dev" &>/dev/null || mkfs.xfs -f "\$dev" mkdir -p "${mp}" grep -q "\$dev" /etc/fstab || echo "\$dev ${mp} xfs defaults,nofail 0 2" >> /etc/fstab mountpoint -q "${mp}" || mount "${mp}" - echo "Mounted \$dev at ${mp}" } mount_data_disk -MDEOF - } - - # Base64-encode a heredoc for CFN UserData - encode_userdata() { - printf '%s' "$1" | base64 | tr -d '\n' - } - - # k0s node UserData (controller + workers) - local ud_k0s_node - ud_k0s_node="${userdata_common}" - - # GPU worker UserData (adds /var/lib/k0s mount) - local ud_gpu_worker - ud_gpu_worker="${userdata_common}$(userdata_mount_disk '/var/lib/k0s')" - - # Installer UserData (adds /data/minio mount if minio enabled) - local ud_installer - ud_installer="${userdata_common}" - if [[ "$MINIO_ENABLED" == "true" ]]; then - ud_installer="${userdata_common}$(userdata_mount_disk '/data/minio')" - fi - - # Generate GPU worker resources - local gpu_instances="" gpu_data_volumes="" gpu_vol_attachments="" - local i - for ((i=0; i "${CFN_TEMPLATE}" </dev/null || echo "NOT_EXISTS") - - if [[ "$existing_status" == "CREATE_COMPLETE" || "$existing_status" == "UPDATE_COMPLETE" ]]; then - log "Stack already exists and is healthy (${existing_status})." - log "To reprovision, run 'destroy' first." - return 0 - elif [[ "$existing_status" != "NOT_EXISTS" ]]; then - log "Existing stack in state ${existing_status} — deleting before retry..." - aws cloudformation delete-stack --stack-name "${STACK_NAME}" --region "${REGION}" - aws cloudformation wait stack-delete-complete --stack-name "${STACK_NAME}" --region "${REGION}" \ - || warn "Wait for stack delete timed out; proceeding anyway" - fi +make_userdata() { + local mp="${1:-}" + local ud + ud="$(userdata_base)" + [[ -n "$mp" ]] && ud+=$'\n'"$(userdata_mount "$mp")" + printf '%s' "$ud" | base64 | tr -d '\n' +} - aws cloudformation deploy \ - --template-file "${CFN_TEMPLATE}" \ - --stack-name "${STACK_NAME}" \ +# -- Launch one EC2 instance (SCP-compliant: IMDSv2 + encrypted EBS) -- +# Usage: launch_instance