diff --git a/api/v1/aiplatform_types.go b/api/v1/aiplatform_types.go index a77c89a3..b05b33db 100644 --- a/api/v1/aiplatform_types.go +++ b/api/v1/aiplatform_types.go @@ -177,8 +177,8 @@ type VectorDBStorageSpec struct { // FeatureSpec defines the features to enable in the AIPlatform type FeatureSpec struct { - // Name of the feature, e.g. "saia" or "seca" - // +kubebuilder:validation:Enum=saia;seca + // Name of the feature, e.g. "saia", "seca" or "slim" + // +kubebuilder:validation:Enum=saia;seca;slim Name string `json:"name,omitempty"` // ServiceAccountName is the name of the service account to use for the feature ServiceAccountName string `json:"serviceAccountName,omitempty"` diff --git a/config/configs/applications.yaml b/config/configs/applications.yaml index 82751907..d9f51017 100644 --- a/config/configs/applications.yaml +++ b/config/configs/applications.yaml @@ -34,6 +34,7 @@ applications: SERVICE_NAME: "ai_platform_models" SKIP_VERIFICATION: "true" USE_SYSTEM_PERMISSIONS: "true" +{{- if .Replicas.Gemma431bIt }} - args: application_name: Gemma431bIt deployment_configs: @@ -170,6 +171,8 @@ applications: # Airgap k0s has no Redis; cloud sets this to "False" and wires # RESPONSES_REDIS_ADDRESS to its in-namespace Redis StatefulSet. DISABLE_RESPONSES_API_REDIS: "True" +{{- end }} +{{- if .Replicas.GptOss20b }} - args: application_name: GptOss20b deployment_configs: @@ -274,6 +277,8 @@ applications: # See Gemma431bIt above for rationale. Must be "True" in airgap (no # Redis) so vLLM uses NoOpOpenAIServingResponses. DISABLE_RESPONSES_API_REDIS: "True" +{{- end }} +{{- if .Replicas.UaeLarge }} - args: application_name: UaeLarge deployment_configs: @@ -352,6 +357,8 @@ applications: SERVICE_NAME: "ai_platform_models" SKIP_VERIFICATION: "true" USE_SYSTEM_PERMISSIONS: "true" +{{- end }} +{{- if .Replicas.AllMinilmL6V2 }} - args: application_name: AllMinilmL6V2 deployment_configs: @@ -424,6 +431,8 @@ applications: SERVICE_NAME: "ai_platform_models" SKIP_VERIFICATION: "true" USE_SYSTEM_PERMISSIONS: "true" +{{- end }} +{{- if .Replicas.BiEncoder }} - args: application_name: BiEncoder deployment_configs: @@ -493,6 +502,8 @@ applications: SERVICE_NAME: "ai_platform_models" SKIP_VERIFICATION: "true" USE_SYSTEM_PERMISSIONS: "true" +{{- end }} +{{- if .Replicas.MbartTranslator }} - args: application_name: MbartTranslator custom_deployment_import_path: mbart_translator:MbartTranslatorDeployment @@ -553,6 +564,8 @@ applications: SERVICE_NAME: "ai_platform_models" SKIP_VERIFICATION: "true" USE_SYSTEM_PERMISSIONS: "true" +{{- end }} +{{- if .Replicas.XlmRobertaLanguageClassifier }} - args: application_name: XlmRobertaLanguageClassifier deployment_configs: @@ -631,6 +644,7 @@ applications: SERVICE_NAME: "ai_platform_models" SKIP_VERIFICATION: "true" USE_SYSTEM_PERMISSIONS: "true" +{{- end }} - args: application_name: PromptInjectionTfidf custom_deployment_import_path: prompt_injection_tfidf:PromptInjectionTfidfDeployment @@ -670,6 +684,84 @@ applications: SERVICE_NAME: "ai_platform_models" SKIP_VERIFICATION: "true" USE_SYSTEM_PERMISSIONS: "true" +{{- if .Replicas.FmTimeseries }} + # FmTimeseries is a slim-feature model (see config/configs/features/slim.yaml). + # .Replicas.FmTimeseries is only populated when the "slim" feature is enabled, + # so this whole application is rendered only then — with saia alone it would + # otherwise emit "" for the replica counts and break the serve config. + - args: + application_name: FmTimeseries + # Cisco Time Series Model (CTSM) — a 250M-param decoder-only time-series + # forecaster served as a custom PyTorch deployment (NOT vLLM), so it has + # no engine_args / tensor_parallel_size / gpu_memory_utilization. It shares + # a GPU (num_gpus: 0.1). Weights are staged from HuggingFace + # (cisco-ai/cisco-time-series-model-1.0) to model_artifacts/fm_timeseries; + # CHECKPOINT_FILE names the checkpoint inside that prefix. + custom_deployment_import_path: fm_timeseries.main:FMTimeseriesDeployment + deployment_configs: + FMTimeseriesDeployment: + options: + autoscaling_config: + max_replicas: {{.Replicas.FmTimeseries}} + min_replicas: {{.Replicas.FmTimeseries}} + ray_actor_options: + num_gpus: 0.1 + deployment_type: custom_deployment + model_definition: + model_config: + num_layers: 25 + model_id: fm_timeseries + model_loader: + blob_storage: + # Only the checkpoint is needed — CiscoTsmMR reads its model config + # from inside torch_model.pt (loaded_checkpoint["config"]), not from + # config.json. artifacts_list restricts the download to exactly these + # files under blob_prefix, so this must name a file that exists in + # the bucket (verified: model_artifacts/fm_timeseries/torch_model.pt). + artifacts_list: + - torch_model.pt + blob_prefix: model_artifacts/fm_timeseries + name: FmTimeseries + import_path: main:create_serve_app + route_prefix: /fm_timeseries + runtime_env: + working_dir: "file:///home/ray/ray/applications/generic_application.zip" + env_vars: + API_VERSION: "v1" + APPLICATION_NAME: fm_timeseries + # Checkpoint filename inside blob_prefix. The public HF repo ships + # torch_model.pt (the deployment builds the path as + # /CHECKPOINT_FILE), so this must match a staged artifact. + CHECKPOINT_FILE: torch_model.pt + ARTIFACTS_S3_BUCKET: "{{.ArtifactBucketName}}" + S3_BUCKET: "{{.ArtifactBucketName}}" + ARTIFACTS_PROVIDER: "{{.ArtifactsProvider}}" + CLOUD_PROVIDER: "{{.CloudProvider}}" + S3COMPAT_OBJECT_STORE_ENDPOINT_URL: "{{.S3CompatObjectStoreEndpointUrl}}" + S3COMPAT_OBJECT_STORE_ACCESS_KEY: "{{.S3CompatObjectStoreAccessKey}}" + S3COMPAT_OBJECT_STORE_SECRET_KEY: "{{.S3CompatObjectStoreSecretKey}}" + # AWS / boto3 standard credential names — populated whenever the + # operator can load credentials from spec.objectStorage.secretRef. For + # CLOUD_PROVIDER=aws these are the values boto3 reads (the S3COMPAT_* + # names above are only consumed by the s3compat shim). Both code paths + # share the same source-of-truth Secret keys (s3_access_key / + # s3_secret_key) so emitting both pairs is safe — each provider only + # reads its own. AWS_REGION lets boto3 resolve the default regional S3 + # endpoint when no AWS_ENDPOINT_URL is set; required for any AWS S3 + # bucket outside us-east-1 to avoid PermanentRedirect on the first call. + AWS_ACCESS_KEY_ID: "{{.S3CompatObjectStoreAccessKey}}" + AWS_SECRET_ACCESS_KEY: "{{.S3CompatObjectStoreSecretKey}}" + AWS_REGION: "{{.Region}}" + AWS_DEFAULT_REGION: "{{.Region}}" + ENABLE_AUTHN: "false" + ENABLE_AUTHZ: "false" + SERVICE_EXTERNAL_NAME: "ai-platform-models" + SERVICE_INTERNAL_NAME: "ai_platform_models" + SERVICE_NAME: "ai_platform_models" + SKIP_VERIFICATION: "true" + USE_SYSTEM_PERMISSIONS: "true" +{{- end }} +{{- if .Replicas.CrossEncoder }} - args: application_name: CrossEncoder deployment_configs: @@ -743,6 +835,8 @@ applications: SERVICE_NAME: "ai_platform_models" SKIP_VERIFICATION: "true" USE_SYSTEM_PERMISSIONS: "true" +{{- end }} +{{- if .Replicas.E5LanguageClassifier }} - args: application_name: E5LanguageClassifier deployment_configs: @@ -821,6 +915,8 @@ applications: SERVICE_NAME: "ai_platform_models" SKIP_VERIFICATION: "true" USE_SYSTEM_PERMISSIONS: "true" +{{- end }} +{{- if .Replicas.PromptInjectionCrossEncoder }} - args: application_name: PromptInjectionCrossEncoder deployment_configs: @@ -886,6 +982,7 @@ applications: SERVICE_NAME: "ai_platform_models" SKIP_VERIFICATION: "true" USE_SYSTEM_PERMISSIONS: "true" +{{- end }} - args: application_name: PromptInjectionClassifier deployment_type: classification_model_deployment diff --git a/config/configs/features/slim.yaml b/config/configs/features/slim.yaml new file mode 100644 index 00000000..172423ba --- /dev/null +++ b/config/configs/features/slim.yaml @@ -0,0 +1,11 @@ +applicationScale: + FmTimeseries: 1 +instanceScale: + L40S: + l40s-1-gpu: 1 + H100: + h100-1-gpu: 1 + H100_NVL: + h100-nvl-1-gpu: 1 + RTX_PRO_6000_BLACKWELL: + rtx-pro-6000-blackwell-1-gpu: 1 diff --git a/config/crd/bases/ai.splunk.com_aiplatforms.yaml b/config/crd/bases/ai.splunk.com_aiplatforms.yaml index 94dafa27..372dcbf6 100644 --- a/config/crd/bases/ai.splunk.com_aiplatforms.yaml +++ b/config/crd/bases/ai.splunk.com_aiplatforms.yaml @@ -1054,10 +1054,11 @@ spec: description: FeatureSpec defines the features to enable in the AIPlatform properties: name: - description: Name of the feature, e.g. "saia" or "seca" + description: Name of the feature, e.g. "saia", "seca" or "slim" enum: - saia - seca + - slim type: string scaleFactor: description: ScaleFactor is the desired fixed number of replicas diff --git a/config/crd/bases/ai.splunk.com_aiservices.yaml b/config/crd/bases/ai.splunk.com_aiservices.yaml index 88c63d61..6b1d779a 100644 --- a/config/crd/bases/ai.splunk.com_aiservices.yaml +++ b/config/crd/bases/ai.splunk.com_aiservices.yaml @@ -1045,10 +1045,11 @@ spec: description: Feature defines the features to be enabled for the AIService properties: name: - description: Name of the feature, e.g. "saia" or "seca" + description: Name of the feature, e.g. "saia", "seca" or "slim" enum: - saia - seca + - slim type: string scaleFactor: description: ScaleFactor is the desired fixed number of replicas diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index 6cf00492..cee8fd19 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -13,6 +13,7 @@ patches: \ - name: RELATED_IMAGE_RAY_WORKER\n value: \"667741767953.dkr.ecr.us-west-2.amazonaws.com/ml-platform/ray/ray-worker-gpu:build-17\"\n \ - name: RELATED_IMAGE_WEAVIATE\n value: \"semitechnologies/weaviate:stable-v1.28-007846a\"\n \ - name: RELATED_IMAGE_SAIA_API\n value: \"667741767953.dkr.ecr.us-west-2.amazonaws.com/ml-platform/saia/saia-api:build-1\"\n + \ - name: RELATED_IMAGE_SLIM_API\n value: \"667741767953.dkr.ecr.us-west-2.amazonaws.com/ml-platform/slim/slim-api:build-1\"\n \ - name: RELATED_IMAGE_POST_INSTALL_HOOK\n value: \"667741767953.dkr.ecr.us-west-2.amazonaws.com/ml-platform/saia/saia-data-loader:build-1\"\n \ - name: RELATED_IMAGE_FLUENT_BIT\n value: \"fluent/fluent-bit:1.9.6\"\n - name: RELATED_IMAGE_OTEL_COLLECTOR\n value: \"otel/opentelemetry-collector-contrib:0.122.1\"\n diff --git a/helm-chart/splunk-ai-operator/crds/ai.splunk.com_aiplatforms.yaml b/helm-chart/splunk-ai-operator/crds/ai.splunk.com_aiplatforms.yaml index 94dafa27..372dcbf6 100644 --- a/helm-chart/splunk-ai-operator/crds/ai.splunk.com_aiplatforms.yaml +++ b/helm-chart/splunk-ai-operator/crds/ai.splunk.com_aiplatforms.yaml @@ -1054,10 +1054,11 @@ spec: description: FeatureSpec defines the features to enable in the AIPlatform properties: name: - description: Name of the feature, e.g. "saia" or "seca" + description: Name of the feature, e.g. "saia", "seca" or "slim" enum: - saia - seca + - slim type: string scaleFactor: description: ScaleFactor is the desired fixed number of replicas diff --git a/helm-chart/splunk-ai-operator/crds/ai.splunk.com_aiservices.yaml b/helm-chart/splunk-ai-operator/crds/ai.splunk.com_aiservices.yaml index 3cc6cc0e..6b1d779a 100644 --- a/helm-chart/splunk-ai-operator/crds/ai.splunk.com_aiservices.yaml +++ b/helm-chart/splunk-ai-operator/crds/ai.splunk.com_aiservices.yaml @@ -1019,6 +1019,14 @@ spec: type: string type: object x-kubernetes-map-type: atomic + aiPlatformScheme: + default: http + description: AIPlatformScheme specifies the URL scheme for the AI + Platform service ("http" or "https") + enum: + - http + - https + type: string aiPlatformUrl: description: AIPlatformUrl specifies the URL for the AI Platform (deprecated, use AIPlatformRef) @@ -1037,10 +1045,11 @@ spec: description: Feature defines the features to be enabled for the AIService properties: name: - description: Name of the feature, e.g. "saia" or "seca" + description: Name of the feature, e.g. "saia", "seca" or "slim" enum: - saia - seca + - slim type: string scaleFactor: description: ScaleFactor is the desired fixed number of replicas diff --git a/helm-chart/splunk-ai-operator/templates/deployment.yaml b/helm-chart/splunk-ai-operator/templates/deployment.yaml index 4f17067f..f3bb0796 100644 --- a/helm-chart/splunk-ai-operator/templates/deployment.yaml +++ b/helm-chart/splunk-ai-operator/templates/deployment.yaml @@ -78,6 +78,8 @@ spec: value: {{ .Values.saiaApiImage }} - name: RELATED_IMAGE_SAIA_API_V2 value: {{ .Values.saiaApiV2Image }} + - name: RELATED_IMAGE_SLIM_API + value: {{ .Values.slimApiImage }} - name: RELATED_IMAGE_POST_INSTALL_HOOK value: {{ .Values.saiaSchemaImage }} - name: RELATED_IMAGE_OTEL_COLLECTOR diff --git a/helm-chart/splunk-ai-operator/values.yaml b/helm-chart/splunk-ai-operator/values.yaml index 32d1c764..28db7ab3 100644 --- a/helm-chart/splunk-ai-operator/values.yaml +++ b/helm-chart/splunk-ai-operator/values.yaml @@ -110,6 +110,9 @@ saiaApiImage: "docker.io/splunk/saia-api:1.1.0" saiaApiV2Image: "docker.io/splunk/saia-api-v2:1.1.0" saiaSchemaImage: "docker.io/splunk/saia-data-loader:1.1.0" +# SLIM API image +slimApiImage: "docker.io/splunk/slim-api:1.0.0" + # OpenTelemetry Collector sidecar image otelCollectorImage: "docker.io/otel/opentelemetry-collector-contrib:0.122.1" diff --git a/helm-chart/splunk-ai-platform/values.yaml b/helm-chart/splunk-ai-platform/values.yaml index d094750d..66659a52 100644 --- a/helm-chart/splunk-ai-platform/values.yaml +++ b/helm-chart/splunk-ai-platform/values.yaml @@ -56,7 +56,7 @@ serviceAccountName: "" gpuInstanceType: "g6.24xlarge" # List of defined features to enable in the AI platform -# Options include: saia, seca +# Options include: saia, seca, slim features: [] # Example: # - name: "saia" diff --git a/pkg/ai/features/registry.go b/pkg/ai/features/registry.go index 58b238f3..c0016b34 100644 --- a/pkg/ai/features/registry.go +++ b/pkg/ai/features/registry.go @@ -4,9 +4,11 @@ import ( "github.com/splunk/splunk-ai-operator/pkg/ai/features/common" "github.com/splunk/splunk-ai-operator/pkg/ai/features/saia" "github.com/splunk/splunk-ai-operator/pkg/ai/features/seca" + "github.com/splunk/splunk-ai-operator/pkg/ai/features/slim" ) var FeatureFactories = map[string]common.FeatureFactory{ "saia": &saia.SaiaFactory{}, "seca": &seca.SecaFactory{}, + "slim": &slim.SlimFactory{}, } diff --git a/pkg/ai/features/slim/factory.go b/pkg/ai/features/slim/factory.go new file mode 100644 index 00000000..a1fde30d --- /dev/null +++ b/pkg/ai/features/slim/factory.go @@ -0,0 +1,21 @@ +package slim + +import ( + "context" + + aiv1 "github.com/splunk/splunk-ai-operator/api/v1" + "github.com/splunk/splunk-ai-operator/pkg/ai/features/common" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type SlimFactory struct{} + +func (f *SlimFactory) New(ctx context.Context, c client.Client, scheme *runtime.Scheme, ai *aiv1.AIService, recorder record.EventRecorder) (common.FeatureHandler, error) { + return &SlimReconciler{ + Client: c, + Scheme: scheme, + Recorder: recorder, + }, nil +} diff --git a/pkg/ai/features/slim/factory_test.go b/pkg/ai/features/slim/factory_test.go new file mode 100644 index 00000000..8fb5bb6b --- /dev/null +++ b/pkg/ai/features/slim/factory_test.go @@ -0,0 +1,44 @@ +package slim + +import ( + "context" + "testing" + + aiv1 "github.com/splunk/splunk-ai-operator/api/v1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func TestSlimFactory_New(t *testing.T) { + s := runtime.NewScheme() + _ = scheme.AddToScheme(s) + _ = aiv1.AddToScheme(s) + + factory := &SlimFactory{} + fakeClient := fake.NewClientBuilder().WithScheme(s).Build() + recorder := record.NewFakeRecorder(10) + + aiService := &aiv1.AIService{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-service", + Namespace: "default", + }, + } + + handler, err := factory.New(context.Background(), fakeClient, s, aiService, recorder) + + require.NoError(t, err) + require.NotNil(t, handler) + + // Verify it returns a SlimReconciler + reconciler, ok := handler.(*SlimReconciler) + assert.True(t, ok, "Expected handler to be *SlimReconciler") + assert.NotNil(t, reconciler.Client) + assert.NotNil(t, reconciler.Scheme) + assert.NotNil(t, reconciler.Recorder) +} diff --git a/pkg/ai/features/slim/impl.go b/pkg/ai/features/slim/impl.go new file mode 100644 index 00000000..2c4701df --- /dev/null +++ b/pkg/ai/features/slim/impl.go @@ -0,0 +1,800 @@ +package slim + +import ( + "context" + "crypto/sha256" + "fmt" + "os" + "reflect" + "sort" + "strings" + + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + + certmanagerv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" + cmmeta "github.com/cert-manager/cert-manager/pkg/apis/meta/v1" + monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + + aiv1 "github.com/splunk/splunk-ai-operator/api/v1" + common "github.com/splunk/splunk-ai-operator/pkg/ai/features/common" + "github.com/splunk/splunk-ai-operator/pkg/splunkutils" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" +) + +type SlimReconciler struct { + client.Client + Scheme *runtime.Scheme + Recorder record.EventRecorder +} + +func (r *SlimReconciler) Reconcile(ctx context.Context, aiservice *aiv1.AIService) error { + log := log.FromContext(ctx) + + var conditions []metav1.Condition + defer func() { + aiservice.Status.Conditions = conditions + aiservice.Status.ObservedGeneration = aiservice.Generation + _ = r.Status().Update(ctx, aiservice) + }() + + stages := []struct { + name string + fn func(context.Context, *aiv1.AIService) error + }{ + {"Validate", r.validateAIService}, + {"ServiceAccount", r.reconcileServiceAccount}, + {"SlimConfigMap", r.reconcileSlimConfigMap}, + {"FeatureConfigMap", r.reconcileFeatureConfigMap}, + {"Certificate", r.reconcileCertificate}, + {"SlimDeployment", r.reconcileSlimDeployment}, + {"SlimService", r.reconcileSlimService}, + {"SlimMetricsService", r.reconcileSlimMetricsService}, + {"ServiceMonitor", r.reconcileServiceMonitor}, + } + + for _, stage := range stages { + err := stage.fn(ctx, aiservice) + + cond := metav1.Condition{ + Type: stage.name + "Ready", + Status: metav1.ConditionTrue, + Reason: "Reconciled", + Message: "stage succeeded", + LastTransitionTime: metav1.Now(), + } + if err != nil { + cond.Status = metav1.ConditionFalse + cond.Reason = "Error" + cond.Message = err.Error() + } + conditions = append(conditions, cond) + if err != nil { + log.Error(err, "stage failed", "stage", stage.name) + return err + } + } + + conditions = append(conditions, metav1.Condition{ + Type: "Ready", + Status: metav1.ConditionTrue, + Reason: "AllReconciled", + Message: "all stages completed successfully", + LastTransitionTime: metav1.Now(), + }) + + return nil +} + +func (r *SlimReconciler) validateAIService(ctx context.Context, ai *aiv1.AIService) error { + cleanServiceTemplate(&ai.Spec.ServiceTemplate) + + if os.Getenv("RELATED_IMAGE_SLIM_API") == "" { + r.Recorder.Event(ai, corev1.EventTypeWarning, "InvalidSpec", "RELATED_IMAGE_SLIM_API must be set") + return fmt.Errorf("RELATED_IMAGE_SLIM_API must be set") + } + + if ai.Spec.AIPlatformRef.Name == "" && ai.Spec.AIPlatformUrl == "" { + r.Recorder.Event(ai, corev1.EventTypeWarning, "InvalidSpec", "AIPlatformRef.Name or AIPlatformUrl must be set") + return fmt.Errorf("either AIPlatformRef.Name or AIPlatformUrl must be set") + } + + if ai.Spec.AIPlatformRef.Name != "" { + aiPlatform, err := r.getAIPlatform(ctx, ai.Spec.AIPlatformRef) + if err != nil { + r.Recorder.Event(ai, corev1.EventTypeWarning, "InvalidSpec", "fetching AIPlatform failed") + return fmt.Errorf("fetching AIPlatform: %w", err) + } + + if err := r.validateAIPlatformReady(ctx, aiPlatform); err != nil { + return fmt.Errorf("AIPlatform infrastructure not ready: %w", err) + } + + clusterDomain := ai.Spec.ClusterDomain + if clusterDomain == "" { + clusterDomain = "cluster.local" + } + if ai.Spec.AIPlatformUrl == "" { + scheme := ai.Spec.AIPlatformScheme + if scheme == "" { + scheme = "http" + } + ai.Spec.AIPlatformUrl = fmt.Sprintf("%s://%s.%s.svc.%s:8000", + scheme, aiPlatform.Status.RayServiceName, ai.Spec.AIPlatformRef.Namespace, clusterDomain) + } + } + + if ai.Spec.AIPlatformUrl == "" { + r.Recorder.Event(ai, corev1.EventTypeWarning, "InvalidSpec", "AIPlatformUrl is not set") + return fmt.Errorf("AIPlatformUrl must be set (either from AIPlatformRef or explicitly)") + } + + if ai.Spec.Resources.Requests == nil { + ai.Spec.Resources.Requests = corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("4"), + corev1.ResourceMemory: resource.MustParse("5Gi"), + } + } + if ai.Spec.Resources.Limits == nil { + ai.Spec.Resources.Limits = corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("4"), + corev1.ResourceMemory: resource.MustParse("5Gi"), + } + } + if ai.Spec.Replicas == 0 { + ai.Spec.Replicas = 1 + } + + return nil +} + +func (r *SlimReconciler) getAIPlatform(ctx context.Context, ref corev1.ObjectReference) (*aiv1.AIPlatform, error) { + var aiPlatform aiv1.AIPlatform + key := types.NamespacedName{ + Name: ref.Name, + Namespace: ref.Namespace, + } + if err := r.Client.Get(ctx, key, &aiPlatform); err != nil { + return nil, err + } + return &aiPlatform, nil +} + +func (r *SlimReconciler) validateAIPlatformReady(ctx context.Context, aiPlatform *aiv1.AIPlatform) error { + if !common.IsConditionTrue(aiPlatform.Status.Conditions, "RayServiceStatusReady") { + return fmt.Errorf("RayService is not ready") + } + if aiPlatform.Status.RayServiceName == "" { + return fmt.Errorf("RayServiceName not populated in AIPlatform status") + } + return nil +} + +func (r *SlimReconciler) reconcileServiceAccount(ctx context.Context, ai *aiv1.AIService) error { + if ai.Spec.ServiceAccountName == "" { + cleanServiceTemplate(&ai.Spec.ServiceTemplate) + + ai.Spec.ServiceAccountName = ai.Name + "-sa" + if err := r.Update(ctx, ai); err != nil { + return fmt.Errorf("updating SA name in spec: %w", err) + } + sa := &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: ai.Spec.ServiceAccountName, + Namespace: ai.Namespace, + }, + } + if err := controllerutil.SetControllerReference(ai, sa, r.Scheme); err != nil { + r.Recorder.Event(ai, corev1.EventTypeWarning, "InvalidSpec", "ownerref on SA failed") + return fmt.Errorf("ownerref on SA: %w", err) + } + if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, sa, func() error { + return nil + }); err != nil { + r.Recorder.Event(ai, corev1.EventTypeWarning, "InvalidSpec", "create/update SA failed") + return fmt.Errorf("create/update SA: %w", err) + } + } + return nil +} + +// buildSplunkIssuersVal computes the comma-separated SPLUNK_ISSUERS value from the AIService spec. +// The JWT issuer is the Splunk management endpoint (port 8089). +// Priority: CRRef-derived service FQDN → explicit Endpoint → TrustedIssuers only. +// +// This mirrors saia.buildSplunkIssuersVal; slim keeps its own copy so the two +// feature packages stay decoupled. +func buildSplunkIssuersVal(ai *aiv1.AIService) string { + var issuers []string + sc := ai.Spec.SplunkConfiguration + switch { + case sc.SplunkCustomResourceRef.Name != "": + kind := sc.SplunkCustomResourceRef.Kind + if kind == "" { + kind = "Standalone" + } + instanceType := "standalone" + if kind == "IndexerCluster" { + instanceType = "indexer" + } + refNS := sc.SplunkCustomResourceRef.Namespace + if refNS == "" { + refNS = ai.Namespace + } + clusterDomain := ai.Spec.ClusterDomain + if clusterDomain == "" { + clusterDomain = "cluster.local" + } + svc := fmt.Sprintf("splunk-%s-%s-service.%s.svc.%s", sc.SplunkCustomResourceRef.Name, instanceType, refNS, clusterDomain) + issuers = append(issuers, fmt.Sprintf("https://%s:%d", svc, splunkutils.SplunkMgmtPort)) + case sc.Endpoint != "": + issuers = append(issuers, sc.Endpoint) + } + issuers = append(issuers, sc.TrustedIssuers...) + return strings.Join(issuers, ",") +} + +// reconcileSlimConfigMap manages the ConfigMap holding the slim-api boot +// configuration. The keys mirror the AITIER-mode env vars from the slim-api +// repo's `run-slim-aipod` Makefile target, trimmed to only those the service +// actually reads in an in-cluster (non-SCS) deployment: +// +// - DEPLOYMENT_TYPE=AITIER bypasses Vault/Consul/SCS auth in the entrypoint +// and flips the app to CMP (interactive-token) authorization. +// - SPLUNK_ISSUERS is the JWT issuer whitelist used by CMP auth; it is derived +// from the AIService spec (CRRef → endpoint → trustedIssuers), matching saia. +// - SERVICE_NAME / SERVICE_INTERNAL_NAME identify the service. +// +// PLATFORM_URL is injected on the Deployment (not here) because it is derived +// per-AIService from the AIPlatform Ray endpoint. +func (r *SlimReconciler) reconcileSlimConfigMap(ctx context.Context, ai *aiv1.AIService) error { + cmName := fmt.Sprintf("%s-slim-config", ai.Name) + + splunkIssuersVal := buildSplunkIssuersVal(ai) + + defaults := map[string]string{ + "DEPLOYMENT_TYPE": "AITIER", + "SERVICE_NAME": "slim-api", + "SERVICE_INTERNAL_NAME": "SLIM", + "SPLUNK_ISSUERS": splunkIssuersVal, + "ENABLE_AUTHZ": "false", + "API_VERSION": "v1alpha1", + } + + found := &corev1.ConfigMap{} + err := r.Get(ctx, types.NamespacedName{Name: cmName, Namespace: ai.Namespace}, found) + if apierrors.IsNotFound(err) { + return r.createOrUpdateConfigMap(ctx, cmName, defaults, ai) + } else if err != nil { + r.Recorder.Event(ai, corev1.EventTypeWarning, "InvalidSpec", + fmt.Sprintf("failed to retrieve ConfigMap %q: %v", cmName, err)) + return fmt.Errorf("fetching Slim ConfigMap %q: %w", cmName, err) + } + + if found.Data == nil { + found.Data = map[string]string{} + } + // Merge defaults: SPLUNK_ISSUERS is always recomputed from spec so that + // removing or changing a CRRef/endpoint/trustedIssuers is reflected immediately + // (including clearing to empty when Splunk is disabled). + // All other keys use fill-if-missing to preserve manual overrides. + needsUpdate := false + for k, v := range defaults { + if k == "SPLUNK_ISSUERS" { + if found.Data[k] != v { + found.Data[k] = v + needsUpdate = true + } + continue + } + if _, ok := found.Data[k]; !ok || found.Data[k] == "" { + found.Data[k] = v + needsUpdate = true + } + } + if needsUpdate { + if err := controllerutil.SetControllerReference(ai, found, r.Scheme); err != nil { + return fmt.Errorf("ownerref on ConfigMap: %w", err) + } + return r.Update(ctx, found) + } + return nil +} + +func (r *SlimReconciler) reconcileFeatureConfigMap(ctx context.Context, ai *aiv1.AIService) error { + cmName := fmt.Sprintf("splunk-%s-feature-config", ai.Name) + + found := &corev1.ConfigMap{} + err := r.Get(ctx, types.NamespacedName{Name: cmName, Namespace: ai.Namespace}, found) + + if err == nil { + if !hasOwnerReference(found, ai) { + if err := controllerutil.SetControllerReference(ai, found, r.Scheme); err != nil { + r.Recorder.Event(ai, corev1.EventTypeWarning, "FeatureConfigMapError", + fmt.Sprintf("Failed to set owner reference on ConfigMap %q", cmName)) + return fmt.Errorf("failed to set owner reference on ConfigMap %q: %w", cmName, err) + } + if err := r.Update(ctx, found); err != nil { + return fmt.Errorf("failed to update owner reference on ConfigMap %q: %w", cmName, err) + } + r.Recorder.Event(ai, corev1.EventTypeNormal, "FeatureConfigMapUpdated", + fmt.Sprintf("Added owner reference to existing ConfigMap %q", cmName)) + } + return nil + } + + if !apierrors.IsNotFound(err) { + r.Recorder.Event(ai, corev1.EventTypeWarning, "FeatureConfigMapError", + fmt.Sprintf("Failed to retrieve ConfigMap %q", cmName)) + return fmt.Errorf("failed to get ConfigMap %q: %w", cmName, err) + } + + defaultData := map[string]string{ + "features_config.yaml": `customization: + enabled_by_default: true +`, + } + + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: cmName, + Namespace: ai.Namespace, + }, + Data: defaultData, + } + + if err := controllerutil.SetControllerReference(ai, cm, r.Scheme); err != nil { + r.Recorder.Event(ai, corev1.EventTypeWarning, "FeatureConfigMapError", + fmt.Sprintf("Failed to set owner reference on ConfigMap %q", cmName)) + return fmt.Errorf("failed to set owner reference on ConfigMap %q: %w", cmName, err) + } + + if err := r.Create(ctx, cm); err != nil { + r.Recorder.Event(ai, corev1.EventTypeWarning, "FeatureConfigMapError", + fmt.Sprintf("Failed to create ConfigMap %q", cmName)) + return fmt.Errorf("failed to create ConfigMap %q: %w", cmName, err) + } + + r.Recorder.Event(ai, corev1.EventTypeNormal, "FeatureConfigMapCreated", + fmt.Sprintf("Created feature-config ConfigMap %q with default content", cmName)) + + return nil +} + +func hasOwnerReference(obj metav1.Object, owner metav1.Object) bool { + for _, ref := range obj.GetOwnerReferences() { + if ref.UID == owner.GetUID() { + return true + } + } + return false +} + +func (r *SlimReconciler) reconcileCertificate(ctx context.Context, ai *aiv1.AIService) error { + if !ai.Spec.MTLS.Enabled || ai.Spec.MTLS.Termination != "operator" { + return nil + } + + certExists := true + existingCert := &certmanagerv1.Certificate{} + certKey := types.NamespacedName{Name: ai.Name + "-tls", Namespace: ai.Namespace} + if err := r.Get(ctx, certKey, existingCert); err != nil { + if apierrors.IsNotFound(err) { + certExists = false + r.Recorder.Event(ai, corev1.EventTypeNormal, "MTLSCertificateCreating", "Creating mTLS certificate") + } + } + + cert := &certmanagerv1.Certificate{ + ObjectMeta: metav1.ObjectMeta{ + Name: ai.Name + "-tls", + Namespace: ai.Namespace, + }, + Spec: certmanagerv1.CertificateSpec{ + SecretName: ai.Spec.MTLS.SecretName, + IssuerRef: ai.Spec.MTLS.IssuerRef, + DNSNames: ai.Spec.MTLS.DNSNames, + Usages: []certmanagerv1.KeyUsage{ + certmanagerv1.UsageServerAuth, + certmanagerv1.UsageClientAuth, + }, + }, + } + if err := controllerutil.SetControllerReference(ai, cert, r.Scheme); err != nil { + r.Recorder.Event(ai, corev1.EventTypeWarning, "MTLSCertificateError", "Failed to set owner reference on Certificate") + return fmt.Errorf("ownerref on Certificate: %w", err) + } + if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, cert, func() error { + cert.Spec = certmanagerv1.CertificateSpec{ + SecretName: ai.Spec.MTLS.SecretName, + IssuerRef: ai.Spec.MTLS.IssuerRef, + DNSNames: ai.Spec.MTLS.DNSNames, + Usages: []certmanagerv1.KeyUsage{ + certmanagerv1.UsageServerAuth, + certmanagerv1.UsageClientAuth, + }, + } + return nil + }); err != nil { + r.Recorder.Eventf(ai, corev1.EventTypeWarning, "MTLSCertificateCreationFailed", "Failed to create/update Certificate: %v", err) + return fmt.Errorf("create/update Certificate: %w", err) + } + + if !certExists { + r.Recorder.Event(ai, corev1.EventTypeNormal, "MTLSCertificateCreated", "mTLS Certificate created successfully") + } + + certReady := false + for _, cond := range cert.Status.Conditions { + if cond.Type == certmanagerv1.CertificateConditionReady && cond.Status == cmmeta.ConditionTrue { + certReady = true + break + } + } + + if !certReady { + r.Recorder.Event(ai, corev1.EventTypeWarning, "MTLSCertificateNotReady", "Waiting for cert-manager to issue certificate") + return fmt.Errorf("waiting for Certificate %q to become Ready", cert.Name) + } + + r.Recorder.Event(ai, corev1.EventTypeNormal, "MTLSCertificateReady", "mTLS certificate issued successfully") + return nil +} + +func (r *SlimReconciler) reconcileSlimDeployment(ctx context.Context, ai *aiv1.AIService) error { + featureConfigName := fmt.Sprintf("splunk-%s-feature-config", ai.Name) + + volumes := []corev1.Volume{ + { + Name: "config-volume", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{Name: featureConfigName}, + }, + }, + }, + } + + ports := []corev1.ContainerPort{ + {Name: "http", ContainerPort: 8080}, + {Name: "metrics", ContainerPort: 8088}, + } + mounts := []corev1.VolumeMount{ + {Name: "config-volume", MountPath: "/etc/config"}, + } + + // slim-api (AITIER) expects PLATFORM_URL to be a fully-qualified URL + // pointing at the ML-Platform models API served by the Ray head. + platformURL := strings.TrimRight(ai.Spec.AIPlatformUrl, "/") + if !strings.HasSuffix(platformURL, "/ai-platform-models/v1") { + platformURL += "/ai-platform-models/v1" + } + + env := []corev1.EnvVar{ + {Name: "PLATFORM_URL", Value: platformURL}, + } + + if ai.Spec.MTLS.Enabled && ai.Spec.MTLS.Termination == "operator" { + volumes = append(volumes, corev1.Volume{ + Name: "tls", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{SecretName: ai.Spec.MTLS.SecretName}, + }, + }) + mounts = append(mounts, corev1.VolumeMount{Name: "tls", MountPath: "/etc/tls", ReadOnly: true}) + env = append(env, + corev1.EnvVar{Name: "TLS_CERT_FILE", Value: "/etc/tls/tls.crt"}, + corev1.EnvVar{Name: "TLS_KEY_FILE", Value: "/etc/tls/tls.key"}, + ) + ports = append(ports, corev1.ContainerPort{Name: "https", ContainerPort: 8443}) + } else { + env = append(env, corev1.EnvVar{Name: "TLS_DISABLED", Value: "true"}) + } + + envFrom := []corev1.EnvFromSource{ + { + ConfigMapRef: &corev1.ConfigMapEnvSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: fmt.Sprintf("%s-slim-config", ai.Name), + }, + }, + }, + } + + sort.Slice(env, func(i, j int) bool { return env[i].Name < env[j].Name }) + + deployment := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: ai.Name + "-slim-deployment", + Namespace: ai.Namespace, + }, + } + + labels := map[string]string{ + "app": ai.Name, + "component": ai.Name, + "area": "ml", + "team": "ml", + } + for k, v := range ai.Labels { + labels[k] = v + } + + // The issuers hash rolls the pods whenever the derived SPLUNK_ISSUERS value + // changes, so CMP auth picks up spec-driven issuer changes (matches saia). + issuersVal := buildSplunkIssuersVal(ai) + issuersChecksum := fmt.Sprintf("%x", sha256.Sum256([]byte(issuersVal))) + annotations := map[string]string{ + "prometheus.io/port": "8088", + "prometheus.io/path": "/metrics", + "prometheus.io/scheme": "http", + "splunk-ai-operator/splunk-issuers-hash": issuersChecksum, + } + for k, v := range ai.Annotations { + if k == "kubectl.kubernetes.io/last-applied-configuration" || k == "kubectl.kubernetes.io/restartedAt" { + continue + } + annotations[k] = v + } + + if err := controllerutil.SetControllerReference(ai, deployment, r.Scheme); err != nil { + r.Recorder.Event(ai, corev1.EventTypeWarning, "InvalidSpec", "ownerref on Deployment failed") + return fmt.Errorf("ownerref on Deployment: %w", err) + } + + containers := []corev1.Container{{ + Name: ai.Name, + Image: os.Getenv("RELATED_IMAGE_SLIM_API"), + ImagePullPolicy: corev1.PullIfNotPresent, + Ports: ports, + VolumeMounts: mounts, + Resources: ai.Spec.Resources, + Env: env, + EnvFrom: envFrom, + LivenessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{Path: "/health", Port: intstr.FromInt(8080)}, + }, + PeriodSeconds: 30, + FailureThreshold: 10, + }, + ReadinessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{Path: "/health", Port: intstr.FromInt(8080)}, + }, + PeriodSeconds: 30, + FailureThreshold: 10, + }, + StartupProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{Path: "/health", Port: intstr.FromInt(8080)}, + }, + InitialDelaySeconds: 30, + PeriodSeconds: 30, + FailureThreshold: 10, + }, + }} + + if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, deployment, func() error { + deployment.ObjectMeta.Labels = labels + deployment.ObjectMeta.Annotations = annotations + deployment.Spec.Replicas = &ai.Spec.Replicas + + if deployment.Spec.Selector == nil { + deployment.Spec.Selector = &metav1.LabelSelector{ + MatchLabels: map[string]string{"app": ai.Name, "component": ai.Name}, + } + } + + deployment.Spec.Template = corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{"app": ai.Name, "component": ai.Name}, + Annotations: annotations, + }, + Spec: corev1.PodSpec{ + ServiceAccountName: ai.Spec.ServiceAccountName, + Containers: containers, + Volumes: volumes, + Affinity: &ai.Spec.Affinity, + Tolerations: ai.Spec.Tolerations, + ImagePullSecrets: ai.Spec.ImagePullSecrets, + }, + } + return nil + }); err != nil { + r.Recorder.Event(ai, corev1.EventTypeWarning, "InvalidSpec", "create/update Deployment failed") + return fmt.Errorf("create/update Deployment: %w", err) + } + return nil +} + +// reconcileSlimService ensures the public-facing Service, which may be exposed via +// NodePort/LoadBalancer per ServiceTemplate. It intentionally excludes the metrics +// port so that enabling public exposure never publishes /metrics or consumes an +// extra NodePort; metrics are served by reconcileSlimMetricsService instead. +func (r *SlimReconciler) reconcileSlimService(ctx context.Context, ai *aiv1.AIService) error { + serviceTemplate := ai.Spec.ServiceTemplate.DeepCopy() + cleanServiceTemplate(serviceTemplate) + + ports := []corev1.ServicePort{ + {Name: "http", Port: 8080, TargetPort: intstr.FromInt(8080)}, + } + if ai.Spec.MTLS.Enabled && ai.Spec.MTLS.Termination == "operator" { + ports = append(ports, corev1.ServicePort{ + Name: "https", Port: 8443, TargetPort: intstr.FromInt(8443), + }) + } + + svcType := corev1.ServiceTypeClusterIP + switch serviceTemplate.Spec.Type { + case corev1.ServiceTypeLoadBalancer: + svcType = corev1.ServiceTypeLoadBalancer + case corev1.ServiceTypeNodePort: + svcType = corev1.ServiceTypeNodePort + for i, port := range ports { + for _, tplPort := range serviceTemplate.Spec.Ports { + if port.Name == tplPort.Name && tplPort.NodePort != 0 { + ports[i].NodePort = tplPort.NodePort + } + } + } + } + + labels := map[string]string{"app": ai.Name} + for k, v := range ai.Labels { + labels[k] = v + } + annotations := map[string]string{} + for k, v := range ai.Annotations { + if k == "kubectl.kubernetes.io/last-applied-configuration" || k == "kubectl.kubernetes.io/restartedAt" { + continue + } + annotations[k] = v + } + + svc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: ai.Name + "-slim-service", + Namespace: ai.Namespace, + }, + } + if err := controllerutil.SetControllerReference(ai, svc, r.Scheme); err != nil { + r.Recorder.Event(ai, corev1.EventTypeWarning, "InvalidSpec", "ownerref on Service failed") + return fmt.Errorf("ownerref on Service: %w", err) + } + // Type, Ports, and Selector are all set inside the mutate callback because + // CreateOrUpdate reloads the live Service from the cluster before invoking it, + // so any values set on svc beforehand are discarded on updates. + if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, svc, func() error { + svc.ObjectMeta.Labels = labels + svc.ObjectMeta.Annotations = annotations + svc.Spec.Selector = map[string]string{"app": ai.Name, "component": ai.Name} + svc.Spec.Ports = ports + svc.Spec.Type = svcType + return nil + }); err != nil { + r.Recorder.Event(ai, corev1.EventTypeWarning, "InvalidSpec", "create/update Service failed") + return fmt.Errorf("create/update Service: %w", err) + } + return nil +} + +// reconcileSlimMetricsService creates an internal ClusterIP-only Service carrying the +// metrics port, kept separate from the public Service so that NodePort/LoadBalancer +// exposure never publishes /metrics. It carries the "component" label so the +// ServiceMonitor's selector (app+component) matches it. +func (r *SlimReconciler) reconcileSlimMetricsService(ctx context.Context, ai *aiv1.AIService) error { + svc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: ai.Name + "-slim-metrics-service", + Namespace: ai.Namespace, + }, + } + if err := controllerutil.SetControllerReference(ai, svc, r.Scheme); err != nil { + r.Recorder.Event(ai, corev1.EventTypeWarning, "InvalidSpec", "ownerref on metrics Service failed") + return fmt.Errorf("ownerref on metrics Service: %w", err) + } + if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, svc, func() error { + svc.ObjectMeta.Labels = map[string]string{"app": ai.Name, "component": ai.Name} + svc.Spec.Selector = map[string]string{"app": ai.Name, "component": ai.Name} + svc.Spec.Ports = []corev1.ServicePort{ + {Name: "metrics", Port: 8088, TargetPort: intstr.FromInt(8088)}, + } + svc.Spec.Type = corev1.ServiceTypeClusterIP + return nil + }); err != nil { + r.Recorder.Event(ai, corev1.EventTypeWarning, "InvalidSpec", "create/update metrics Service failed") + return fmt.Errorf("create/update metrics Service: %w", err) + } + return nil +} + +func (r *SlimReconciler) reconcileServiceMonitor(ctx context.Context, ai *aiv1.AIService) error { + if !ai.Spec.Metrics.Enabled { + return nil + } + + endpoints := []monitoringv1.Endpoint{ + {Port: "metrics", Path: ai.Spec.Metrics.Path, Scheme: "http"}, + } + + sm := &monitoringv1.ServiceMonitor{ + ObjectMeta: metav1.ObjectMeta{Name: ai.Name + "-metrics", Namespace: ai.Namespace}, + Spec: monitoringv1.ServiceMonitorSpec{ + Selector: metav1.LabelSelector{ + MatchLabels: map[string]string{"app": ai.Name, "component": ai.Name}, + }, + Endpoints: endpoints, + }, + } + if err := controllerutil.SetControllerReference(ai, sm, r.Scheme); err != nil { + return err + } + _, err := controllerutil.CreateOrUpdate(ctx, r.Client, sm, func() error { + sm.Spec = monitoringv1.ServiceMonitorSpec{ + Selector: metav1.LabelSelector{ + MatchLabels: map[string]string{"app": ai.Name, "component": ai.Name}, + }, + Endpoints: endpoints, + } + return nil + }) + return err +} + +func (r *SlimReconciler) createOrUpdateConfigMap( + ctx context.Context, + name string, + data map[string]string, + ai *aiv1.AIService, +) error { + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: ai.Namespace, + }, + Data: data, + } + if err := controllerutil.SetControllerReference(ai, cm, r.Scheme); err != nil { + return err + } + + found := &corev1.ConfigMap{} + err := r.Get(ctx, types.NamespacedName{Name: name, Namespace: ai.Namespace}, found) + if apierrors.IsNotFound(err) { + return r.Create(ctx, cm) + } else if err != nil { + return err + } + + if !reflect.DeepEqual(found.Data, data) { + found.Data = data + return r.Update(ctx, found) + } + return nil +} + +func cleanServiceTemplate(template *corev1.Service) { + if template == nil { + return + } + template.ObjectMeta.CreationTimestamp = metav1.Time{} + template.ObjectMeta.DeletionTimestamp = nil + template.ObjectMeta.DeletionGracePeriodSeconds = nil + template.ObjectMeta.UID = "" + template.ObjectMeta.ResourceVersion = "" + template.ObjectMeta.Generation = 0 + template.ObjectMeta.SelfLink = "" + template.ObjectMeta.ManagedFields = nil + template.Status = corev1.ServiceStatus{} +} diff --git a/pkg/ai/features/slim/impl_test.go b/pkg/ai/features/slim/impl_test.go new file mode 100644 index 00000000..8b6a9f15 --- /dev/null +++ b/pkg/ai/features/slim/impl_test.go @@ -0,0 +1,155 @@ +package slim + +import ( + "context" + "strings" + "testing" + + aiv1 "github.com/splunk/splunk-ai-operator/api/v1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +// buildTestScheme creates a scheme with the types reconcileSlimService needs. +func buildTestScheme(t *testing.T) *runtime.Scheme { + s := runtime.NewScheme() + require.NoError(t, aiv1.AddToScheme(s)) + require.NoError(t, corev1.AddToScheme(s)) + require.NoError(t, appsv1.AddToScheme(s)) + return s +} + +func Test_reconcileSlimService_ServiceTypeVariations(t *testing.T) { + // Slim mirrors SAIA's contract: the customer's k0s-cluster-config.yaml can + // omit / empty / explicitly-set serviceTemplate and get the expected + // Service.Type. Slim has no v1/v2/nginx split, so its public Service is the + // only Service the reconciler renders. The k0s installer gives slim a + // DISTINCT NodePort (30081) from SAIA's (30080), so the "explicit NodePort" + // case pins 30081 to lock in that no-collision contract. + scheme := buildTestScheme(t) + + cases := []struct { + name string + template corev1.Service + wantType corev1.ServiceType + wantNodePort int32 // 0 = don't check + }{ + { + name: "omitted/empty template → ClusterIP", + template: corev1.Service{}, // zero value, what yq-absent produces + wantType: corev1.ServiceTypeClusterIP, + }, + { + name: "explicit ClusterIP → ClusterIP", + template: corev1.Service{ + Spec: corev1.ServiceSpec{Type: corev1.ServiceTypeClusterIP}, + }, + wantType: corev1.ServiceTypeClusterIP, + }, + { + name: "NodePort without explicit port → NodePort auto-allocated", + template: corev1.Service{ + Spec: corev1.ServiceSpec{Type: corev1.ServiceTypeNodePort}, + }, + wantType: corev1.ServiceTypeNodePort, + // wantNodePort == 0 means we don't assert a specific value + }, + { + name: "NodePort with explicit 30081 → NodePort 30081", + template: corev1.Service{ + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeNodePort, + Ports: []corev1.ServicePort{ + {Name: "http", NodePort: 30081}, + }, + }, + }, + wantType: corev1.ServiceTypeNodePort, + wantNodePort: 30081, + }, + { + name: "LoadBalancer → LoadBalancer", + template: corev1.Service{ + Spec: corev1.ServiceSpec{Type: corev1.ServiceTypeLoadBalancer}, + }, + wantType: corev1.ServiceTypeLoadBalancer, + }, + { + name: "Unknown garbage type → ClusterIP (safe default)", + template: corev1.Service{ + Spec: corev1.ServiceSpec{Type: corev1.ServiceType("Bogus")}, + }, + wantType: corev1.ServiceTypeClusterIP, + }, + } + + for _, tc := range cases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + ai := &aiv1.AIService{ + ObjectMeta: metav1.ObjectMeta{ + Name: "svctype-" + sanitize(tc.name), + Namespace: "default", + UID: "uid-123", + }, + Spec: aiv1.AIServiceSpec{ + ServiceTemplate: tc.template, + }, + } + + fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(ai).Build() + r := &SlimReconciler{Client: fakeClient, Scheme: scheme, Recorder: record.NewFakeRecorder(10)} + + require.NoError(t, r.reconcileSlimService(context.Background(), ai)) + + svc := &corev1.Service{} + require.NoError(t, fakeClient.Get(context.Background(), + types.NamespacedName{Name: ai.Name + "-slim-service", Namespace: "default"}, svc)) + + assert.Equal(t, tc.wantType, svc.Spec.Type) + if tc.wantNodePort != 0 { + require.NotEmpty(t, svc.Spec.Ports) + // http is the first port slim renders; the installer patches its + // NodePort. Find it by name to stay robust to port ordering. + var httpPort *corev1.ServicePort + for i := range svc.Spec.Ports { + if svc.Spec.Ports[i].Name == "http" { + httpPort = &svc.Spec.Ports[i] + break + } + } + require.NotNil(t, httpPort, "expected an http port on the slim service") + assert.Equal(t, tc.wantNodePort, httpPort.NodePort) + } + }) + } +} + +// sanitize turns a free-form subtest name into a valid k8s resource name. +func sanitize(s string) string { + s = strings.ToLower(s) + out := make([]byte, 0, len(s)) + for i := 0; i < len(s); i++ { + c := s[i] + switch { + case c >= 'a' && c <= 'z', c >= '0' && c <= '9': + out = append(out, c) + default: + if len(out) > 0 && out[len(out)-1] != '-' { + out = append(out, '-') + } + } + } + // Trim trailing hyphen so the name is a valid RFC-1123 label. + for len(out) > 0 && out[len(out)-1] == '-' { + out = out[:len(out)-1] + } + return string(out) +} diff --git a/pkg/ai/raybuilder/configmap_apps_test.go b/pkg/ai/raybuilder/configmap_apps_test.go index 4beb1b4e..c56c313c 100644 --- a/pkg/ai/raybuilder/configmap_apps_test.go +++ b/pkg/ai/raybuilder/configmap_apps_test.go @@ -28,11 +28,20 @@ func readApplicationsYAMLFromRepo(t *testing.T) string { return string(raw) } -// maskGoTemplates replaces `{{ ... }}` tokens with a plain string so the -// result parses as valid YAML. applications.yaml interpolates Go template -// variables at runtime (see Builder.ReconcileApplicationsConfigMap) — during -// unit testing we never render them, so a syntactic mask is sufficient. +// maskGoTemplates rewrites the Go template so the result parses as valid YAML. +// applications.yaml interpolates Go template variables at runtime (see +// Builder.ReconcileApplicationsConfigMap) — during unit testing we never render +// them, so a syntactic mask is sufficient. Two kinds of directive exist: +// +// - Standalone control-flow lines (`{{- if ... }}`, `{{- end }}`, `{{ range }}`, +// `{{ else }}`) occupy a whole line and would become a bare `PLACEHOLDER` +// scalar at column 0, which is invalid YAML. These lines are dropped +// entirely — since we never render, keeping both branches' bodies is fine. +// - Inline value interpolations (`{{.Replicas.X}}`) are replaced with a plain +// scalar so the surrounding key/value stays well formed. func maskGoTemplates(s string) string { + controlLine := regexp.MustCompile(`(?m)^[ \t]*\{\{-?\s*(if|else|end|range|with)\b[^}]*\}\}[ \t]*\n`) + s = controlLine.ReplaceAllString(s, "") return regexp.MustCompile(`\{\{[^}]+\}\}`).ReplaceAllString(s, "PLACEHOLDER") } diff --git a/pkg/config/config.go b/pkg/config/config.go index 292686aa..a45908f7 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -33,4 +33,8 @@ type OperatorConfig struct { // DebugSaiaEndpoint overrides SAIA service endpoint in debug mode. DebugSaiaEndpoint string + + // DebugSlimEndpoint overrides SLIM API service endpoint in debug mode. + // Example: "http://localhost:8080" + DebugSlimEndpoint string } diff --git a/pkg/service/slim/factory.go b/pkg/service/slim/factory.go new file mode 100644 index 00000000..786ef53e --- /dev/null +++ b/pkg/service/slim/factory.go @@ -0,0 +1,43 @@ +package slim + +import ( + "context" + "fmt" + + "github.com/go-logr/logr" + manager "github.com/splunk/splunk-ai-operator/pkg/service" +) + +type slimManagerFactory struct { + log logr.Logger +} + +// NewManagerFactory creates a new manager factory to create manager interface. +// Returns nil if initialization fails - callers should check for nil. +func NewManagerFactory() manager.Factory { + factory := slimManagerFactory{} + err := factory.init() + if err != nil { + // Log the error since we can't return it from this signature + // In production, consider using a logger + panic(fmt.Sprintf("failed to initialize SLIM manager factory: %v", err)) + } + return &factory +} + +func (f *slimManagerFactory) init() error { + return nil +} + +func (f *slimManagerFactory) newManager(ctx context.Context) (manager.Manager, error) { + newManager := &slimManager{ + log: f.log, + } + return newManager, nil +} + +// NewService implements the Factory interface. +// Returns a new SLIM service manager using the provided context. +func (f *slimManagerFactory) NewService(ctx context.Context) (manager.Manager, error) { + return f.newManager(ctx) +} diff --git a/pkg/service/slim/impl.go b/pkg/service/slim/impl.go new file mode 100644 index 00000000..3c521827 --- /dev/null +++ b/pkg/service/slim/impl.go @@ -0,0 +1,25 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package slim + +import "github.com/go-logr/logr" + +// slimManager implements the service.Manager interface. +type slimManager struct { + // a logger configured for this host + log logr.Logger +} diff --git a/tools/artifacts_download_upload_scripts/README.md b/tools/artifacts_download_upload_scripts/README.md index 70538b6e..cbb377aa 100755 --- a/tools/artifacts_download_upload_scripts/README.md +++ b/tools/artifacts_download_upload_scripts/README.md @@ -347,6 +347,7 @@ The following models are pre-configured and ready to download: - `bi-encoder` - BGE small encoder - `cross-encoder` - MS MARCO cross-encoder - `e5-language-classifier` - Multilingual language detection +- `fm_timeseries` - Cisco Time Series Model (CTSM) forecaster - `mbart-translator` - Multilingual translation - `pii-classifier` - PII detection model - `uae-large` - UAE embedding model diff --git a/tools/artifacts_download_upload_scripts/model_artifacts_configs.yaml b/tools/artifacts_download_upload_scripts/model_artifacts_configs.yaml index 621f80e1..65987458 100755 --- a/tools/artifacts_download_upload_scripts/model_artifacts_configs.yaml +++ b/tools/artifacts_download_upload_scripts/model_artifacts_configs.yaml @@ -22,6 +22,15 @@ artifact-configs: hf-url: https://huggingface.co/Mike0307/multilingual-e5-language-detection is-a-gated-model: false + # Cisco Time Series Model (CTSM). artifact-id must equal the blob_prefix + # suffix in config/configs/applications.yaml (model_artifacts/fm_timeseries). + # images/ holds only README PNGs, not inference weights, so exclude them. + - artifact-id: fm_timeseries + hf-url: https://huggingface.co/cisco-ai/cisco-time-series-model-1.0 + is-a-gated-model: false + files-to-exclude: + - images/ + - artifact-id: gpt-oss-20b hf-url: https://huggingface.co/openai/gpt-oss-20b is-a-gated-model: false diff --git a/tools/artifacts_download_upload_scripts/model_artifacts_configs_h100.yaml b/tools/artifacts_download_upload_scripts/model_artifacts_configs_h100.yaml index 8fb986db..e644acc3 100755 --- a/tools/artifacts_download_upload_scripts/model_artifacts_configs_h100.yaml +++ b/tools/artifacts_download_upload_scripts/model_artifacts_configs_h100.yaml @@ -26,6 +26,15 @@ artifact-configs: hf-url: https://huggingface.co/Mike0307/multilingual-e5-language-detection is-a-gated-model: false + # Cisco Time Series Model (CTSM). artifact-id must equal the blob_prefix + # suffix in config/configs/applications.yaml (model_artifacts/fm_timeseries). + # images/ holds only README PNGs, not inference weights, so exclude them. + - artifact-id: fm_timeseries + hf-url: https://huggingface.co/cisco-ai/cisco-time-series-model-1.0 + is-a-gated-model: false + files-to-exclude: + - images/ + - artifact-id: gpt-oss-20b hf-url: https://huggingface.co/openai/gpt-oss-20b is-a-gated-model: false diff --git a/tools/cluster_setup/EKS_README.md b/tools/cluster_setup/EKS_README.md index 00becee0..6e493fd2 100644 --- a/tools/cluster_setup/EKS_README.md +++ b/tools/cluster_setup/EKS_README.md @@ -782,6 +782,7 @@ CONFIG_FILE=./my-cluster-config.yaml ./eks_cluster_with_stack.sh install | `bi-encoder` | BGE small encoder | | `cross-encoder` | MS MARCO cross-encoder | | `e5-language-classifier` | Multilingual language detection | + | `fm_timeseries` | Cisco Time Series Model (CTSM) forecaster | | `mbart-translator` | Multilingual translation | | `pii-classifier` | PII detection | | `uae-large` | Embedding model | diff --git a/tools/cluster_setup/K0S_README.md b/tools/cluster_setup/K0S_README.md index cbb524cf..940db8cd 100644 --- a/tools/cluster_setup/K0S_README.md +++ b/tools/cluster_setup/K0S_README.md @@ -641,6 +641,7 @@ The `install` command executes these steps in order: | `bi-encoder` | BGE small encoder | | `cross-encoder` | MS MARCO cross-encoder | | `e5-language-classifier` | Multilingual language detection | + | `fm_timeseries` | Cisco Time Series Model (CTSM) forecaster | | `mbart-translator` | Multilingual translation | | `pii-classifier` | PII detection | | `uae-large` | Embedding model | diff --git a/tools/cluster_setup/TEST_PLAN.md b/tools/cluster_setup/TEST_PLAN.md index 1a11fb30..4c073df8 100644 --- a/tools/cluster_setup/TEST_PLAN.md +++ b/tools/cluster_setup/TEST_PLAN.md @@ -115,8 +115,9 @@ grep -l "gemma-4-31b-it" \ grep "^ - artifact-id:" tools/artifacts_download_upload_scripts/model_artifacts_configs.yaml \ | awk '{print $3}' | sort -# K0S_README.md model table — rows of the form " | `` |" with hyphens in name -grep -E "^\s+\| \`[a-z][a-z0-9-]+\` \|" tools/cluster_setup/K0S_README.md \ +# K0S_README.md model table — rows of the form " | `` |" with +# hyphens or underscores in the name (e.g. fm_timeseries). +grep -E "^\s+\| \`[a-z][a-z0-9_-]+\` \|" tools/cluster_setup/K0S_README.md \ | awk -F'`' '{print $2}' | sort ``` diff --git a/tools/cluster_setup/artifacts.yaml b/tools/cluster_setup/artifacts.yaml index c6953e7b..66bc9273 100644 --- a/tools/cluster_setup/artifacts.yaml +++ b/tools/cluster_setup/artifacts.yaml @@ -1062,10 +1062,11 @@ spec: description: FeatureSpec defines the features to enable in the AIPlatform properties: name: - description: Name of the feature, e.g. "saia" or "seca" + description: Name of the feature, e.g. "saia", "seca" or "slim" enum: - saia - seca + - slim type: string scaleFactor: description: ScaleFactor is the desired fixed number of replicas @@ -4085,10 +4086,11 @@ spec: description: Feature defines the features to be enabled for the AIService properties: name: - description: Name of the feature, e.g. "saia" or "seca" + description: Name of the feature, e.g. "saia", "seca" or "slim" enum: - saia - seca + - slim type: string scaleFactor: description: ScaleFactor is the desired fixed number of replicas @@ -5693,6 +5695,8 @@ spec: value: 658391232643.dkr.ecr.us-east-2.amazonaws.com/ml-platform/saia/saia-api-v2:build-v2-012 - name: RELATED_IMAGE_POST_INSTALL_HOOK value: 658391232643.dkr.ecr.us-east-2.amazonaws.com/ml-platform/saia/saia-data-loader:build-v2-012 + - name: RELATED_IMAGE_SLIM_API + value: 658391232643.dkr.ecr.us-east-2.amazonaws.com/ml-platform/slim/slim-api:build-1 - name: SPLUNK_METRICS_INDEX_NAME value: _metrics - name: RELATED_IMAGE_FLUENT_BIT diff --git a/tools/cluster_setup/k0s-cluster-config.yaml b/tools/cluster_setup/k0s-cluster-config.yaml index ee689eb3..680052ce 100644 --- a/tools/cluster_setup/k0s-cluster-config.yaml +++ b/tools/cluster_setup/k0s-cluster-config.yaml @@ -134,6 +134,11 @@ images: apiV2Image: "splunk/saia-api-v2-build-preview:latest" dataLoaderImage: "splunk/saia-data-loader-build-preview:latest" + # slim-api service image (consumed via RELATED_IMAGE_SLIM_API). Only needed + # when the "slim" feature is enabled under aiPlatform.features[]. + slim: + apiImage: "splunk/slim/slim-api:latest" + fluentBit: image: "docker.io/fluent/fluent-bit:1.9.6" @@ -250,11 +255,20 @@ aiPlatform: # serviceTemplate block AND set metallb.install: false. serviceTemplate: type: NodePort # ClusterIP | NodePort | LoadBalancer - nodePort: 30080 # Fixed NodePort (30000-32767). Required only for NodePort. + nodePort: 30080 # Fixed NodePort (30000-32767) for the SAIA public service. Required only for NodePort. + # AIPlatform.serviceTemplate propagates identically to EVERY feature's + # AIService, so slim would otherwise inherit the same 30080 and collide with + # SAIA (a NodePort can back only one Service). When the "slim" feature is + # enabled under NodePort, the installer patches slim's public Service to this + # DISTINCT port instead. Reach slim at http://:. + slimNodePort: 30081 # Fixed NodePort for the slim public service. Only used when slim is enabled + type=NodePort. features: - name: "saia" version: "1.1.0" + # Enable the slim-api service (no v1/v2 routing). Exposed on slimNodePort above. + - name: "slim" + version: "1.0.0" cpuScheduling: nodeSelector: {} diff --git a/tools/cluster_setup/k0s_cluster_with_stack.sh b/tools/cluster_setup/k0s_cluster_with_stack.sh index 4cb4b57a..5d7885d8 100755 --- a/tools/cluster_setup/k0s_cluster_with_stack.sh +++ b/tools/cluster_setup/k0s_cluster_with_stack.sh @@ -673,6 +673,7 @@ Run 'yq eval . ${CONFIG_FILE}' for details, then fix the line and retry." SAIA_API_IMAGE="$(yq eval '.images.saia.apiImage' "$CONFIG_FILE" 2>/dev/null || echo "")" SAIA_API_V2_IMAGE="$(yq eval '.images.saia.apiV2Image' "$CONFIG_FILE" 2>/dev/null || echo "")" SAIA_DATALOADER_IMAGE="$(yq eval '.images.saia.dataLoaderImage' "$CONFIG_FILE" 2>/dev/null || echo "")" + SLIM_API_IMAGE="$(yq eval '.images.slim.apiImage' "$CONFIG_FILE" 2>/dev/null || echo "")" FLUENT_BIT_IMAGE="$(yq eval '.images.fluentBit.image' "$CONFIG_FILE" 2>/dev/null || echo "")" OTEL_COLLECTOR_IMAGE="$(yq eval '.images.otelCollector.image' "$CONFIG_FILE" 2>/dev/null || echo "")" NGINX_IMAGE="$(yq eval '.images.nginx.image' "$CONFIG_FILE" 2>/dev/null || echo "")" @@ -778,6 +779,13 @@ validate_image_config() { if [[ -z "$SAIA_DATALOADER_IMAGE" || "$SAIA_DATALOADER_IMAGE" == "null" ]]; then err "REQUIRED: images.saia.dataLoaderImage must be specified in k0s-cluster-config.yaml" fi + # slim-api is only deployed when the "slim" feature is enabled, so require its + # image only in that case (mirrors the feature-gated Splunk image handling). + if k0s_slim_feature_enabled; then + if [[ -z "$SLIM_API_IMAGE" || "$SLIM_API_IMAGE" == "null" ]]; then + err "REQUIRED: images.slim.apiImage must be specified in k0s-cluster-config.yaml when the 'slim' feature is enabled" + fi + fi if [[ -z "$SPLUNK_OPERATOR_IMAGE" || "$SPLUNK_OPERATOR_IMAGE" == "null" ]]; then SPLUNK_OPERATOR_IMAGE="docker.io/splunk/splunk-operator:3.0.0" log "Using default Splunk Operator image: $SPLUNK_OPERATOR_IMAGE" @@ -837,6 +845,10 @@ configure_images() { local saia_api_full=$(build_image_url "$IMAGE_REGISTRY" "$SAIA_API_IMAGE") local saia_api_v2_full=$(build_image_url "$IMAGE_REGISTRY" "$SAIA_API_V2_IMAGE") local saia_dataloader_full=$(build_image_url "$IMAGE_REGISTRY" "$SAIA_DATALOADER_IMAGE") + # slim-api is optional (feature-gated); only build a URL when configured. + local slim_api_full="" + [[ -n "$SLIM_API_IMAGE" && "$SLIM_API_IMAGE" != "null" ]] && \ + slim_api_full=$(build_image_url "$IMAGE_REGISTRY" "$SLIM_API_IMAGE") local fluent_bit_full=$(build_image_url "$IMAGE_REGISTRY" "$FLUENT_BIT_IMAGE") local otel_collector_full=$(build_image_url "$IMAGE_REGISTRY" "$OTEL_COLLECTOR_IMAGE") # Nginx is an upstream image; don't rewrite it to the ECR registry unless the @@ -851,6 +863,7 @@ configure_images() { local saia_api_escaped=$(echo "$saia_api_full" | sed 's/[\/&]/\\&/g') local saia_api_v2_escaped=$(echo "$saia_api_v2_full" | sed 's/[\/&]/\\&/g') local saia_dataloader_escaped=$(echo "$saia_dataloader_full" | sed 's/[\/&]/\\&/g') + local slim_api_escaped=$(echo "$slim_api_full" | sed 's/[\/&]/\\&/g') local fluent_bit_escaped=$(echo "$fluent_bit_full" | sed 's/[\/&]/\\&/g') local otel_collector_escaped=$(echo "$otel_collector_full" | sed 's/[\/&]/\\&/g') local nginx_escaped=$(echo "$nginx_full" | sed 's/[\/&]/\\&/g') @@ -874,6 +887,11 @@ configure_images() { "${SED_INPLACE[@]}" "/name: RELATED_IMAGE_SAIA_API$/,/value:/ s|value:.*|value: ${saia_api_escaped}|" "$SPLUNK_AI_FILE" "${SED_INPLACE[@]}" "/name: RELATED_IMAGE_SAIA_API_V2/,/value:/ s|value:.*|value: ${saia_api_v2_escaped}|" "$SPLUNK_AI_FILE" "${SED_INPLACE[@]}" "/name: RELATED_IMAGE_POST_INSTALL_HOOK/,/value:/ s|value:.*|value: ${saia_dataloader_escaped}|" "$SPLUNK_AI_FILE" + # slim-api is feature-gated; only rewrite when an image was configured so the + # manifest's default value survives untouched on saia-only installs. + if [[ -n "$slim_api_full" ]]; then + "${SED_INPLACE[@]}" "/name: RELATED_IMAGE_SLIM_API/,/value:/ s|value:.*|value: ${slim_api_escaped}|" "$SPLUNK_AI_FILE" + fi "${SED_INPLACE[@]}" "/name: RELATED_IMAGE_FLUENT_BIT/,/value:/ s|value:.*|value: ${fluent_bit_escaped}|" "$SPLUNK_AI_FILE" "${SED_INPLACE[@]}" "/name: RELATED_IMAGE_OTEL_COLLECTOR/,/value:/ s|value:.*|value: ${otel_collector_escaped}|" "$SPLUNK_AI_FILE" "${SED_INPLACE[@]}" "/name: RELATED_IMAGE_NGINX/,/value:/ s|value:.*|value: ${nginx_escaped}|" "$SPLUNK_AI_FILE" @@ -887,6 +905,7 @@ configure_images() { log " ✓ Updated RELATED_IMAGE_SAIA_API: $saia_api_full" log " ✓ Updated RELATED_IMAGE_SAIA_API_V2: $saia_api_v2_full" log " ✓ Updated RELATED_IMAGE_POST_INSTALL_HOOK: $saia_dataloader_full" + [[ -n "$slim_api_full" ]] && log " ✓ Updated RELATED_IMAGE_SLIM_API: $slim_api_full" log " ✓ Updated RELATED_IMAGE_FLUENT_BIT: $fluent_bit_full" log " ✓ Updated RELATED_IMAGE_OTEL_COLLECTOR: $otel_collector_full" log " ✓ Updated RELATED_IMAGE_NGINX: $nginx_full" @@ -1212,7 +1231,7 @@ preflight_check_registry() { local probe_ref="" local probe_source="" - for _candidate_var in SAIA_API_IMAGE RAY_HEAD_IMAGE SAIA_API_V2_IMAGE SAIA_DATALOADER_IMAGE RAY_WORKER_IMAGE WEAVIATE_IMAGE FLUENT_BIT_IMAGE OTEL_COLLECTOR_IMAGE NGINX_IMAGE SPLUNK_IMAGE SPLUNK_OPERATOR_IMAGE OPERATOR_IMAGE; do + for _candidate_var in SAIA_API_IMAGE RAY_HEAD_IMAGE SAIA_API_V2_IMAGE SAIA_DATALOADER_IMAGE SLIM_API_IMAGE RAY_WORKER_IMAGE WEAVIATE_IMAGE FLUENT_BIT_IMAGE OTEL_COLLECTOR_IMAGE NGINX_IMAGE SPLUNK_IMAGE SPLUNK_OPERATOR_IMAGE OPERATOR_IMAGE; do local _val="${!_candidate_var:-}" local _ref if _ref=$(_image_targets_registry "${_val}"); then @@ -4146,7 +4165,14 @@ install_splunk_standalone() { fi # Create splunk-defaults ConfigMap (optional but recommended) - cat <<'YAML' | kubectl -n "${AI_NS}" apply -f - + # + # issuer_uri becomes the "iss" claim on every interactive JWT Splunk mints. + # It MUST byte-for-byte match splunkConfiguration.endpoint below (which feeds + # SPLUNK_ISSUERS on saia/slim) or CMP auth rejects every token with + # "Issuer '' is not allowed". Keep both in the + # https://splunk--standalone-service..svc.:8089 form used by + # the operator's own SplunkCustomResourceRef path (buildSplunkIssuersVal). + cat <-slim-slim-service. +k0s_slim_feature_enabled() { + local names + names=$(yq eval '.aiPlatform.features[].name // ""' "${CONFIG_FILE}" 2>/dev/null || echo "") + while IFS= read -r n; do + [[ "${n}" == "slim" ]] && return 0 + done <<< "${names}" + return 1 +} + +# Expose the slim public Service the same way SAIA is exposed, but on a DISTINCT +# NodePort (aiPlatform.serviceTemplate.slimNodePort, default 30081). This is +# required because the AIPlatform reconciler copies AIPlatform.spec.serviceTemplate +# down to EVERY feature's AIService verbatim, so slim would otherwise inherit +# SAIA's nodePort (30080) and collide — a single NodePort can back only one +# Service. We patch slim's own AIService.spec.serviceTemplate after it exists; +# reconcileSlimService only mutates Selector/Ports on the existing Service and +# the AIPlatform reconciler preserves an admin-patched serviceTemplate +# (pkg/ai/reconciler.go), so this patch survives subsequent reconciles. +patch_k0s_slim_public_service_workaround() { + k0s_slim_feature_enabled || return 0 + + local platform_name="${CLUSTER_NAME}-ai-platform" + local aiservice_name="${platform_name}-slim" + local public_svc_name="${aiservice_name}-slim-service" + local svc_type svc_node_port slim_node_port + + svc_type=$(yq eval '.aiPlatform.serviceTemplate.type // ""' "${CONFIG_FILE}" 2>/dev/null || echo "") + svc_node_port=$(yq eval '.aiPlatform.serviceTemplate.nodePort // ""' "${CONFIG_FILE}" 2>/dev/null || echo "") + # slimNodePort keeps slim off SAIA's port; fall back to nodePort+1 if unset so + # a config that only sets one shared nodePort still avoids a hard collision. + slim_node_port=$(yq eval '.aiPlatform.serviceTemplate.slimNodePort // ""' "${CONFIG_FILE}" 2>/dev/null || echo "") + if [[ ( -z "${slim_node_port}" || "${slim_node_port}" == "null" ) && -n "${svc_node_port}" && "${svc_node_port}" != "null" ]]; then + slim_node_port=$((svc_node_port + 1)) + fi + + wait_for_k0s_aiservice_exists "${aiservice_name}" + + if saia_service_template_enabled_k0s; then + log "Patching AIService/${aiservice_name} with slim public exposure settings (type=${svc_type})..." + if [[ "${svc_type}" == "NodePort" && -n "${slim_node_port}" && "${slim_node_port}" != "null" ]]; then + log "slim exposed via NodePort ${slim_node_port} — reach it at http://:${slim_node_port} (front with a cloud LB on cloud VMs). Distinct from SAIA's ${svc_node_port} to avoid a NodePort collision." >&2 + kubectl -n "${AI_NS}" patch aiservice "${aiservice_name}" --type merge -p "{ + \"spec\": { + \"serviceTemplate\": { + \"spec\": { + \"type\": \"NodePort\", + \"ports\": [ + { + \"name\": \"http\", + \"port\": 8080, + \"targetPort\": 8080, + \"nodePort\": ${slim_node_port} + } + ] + } + } + } +}" + else + kubectl -n "${AI_NS}" patch aiservice "${aiservice_name}" --type merge -p "{ + \"spec\": { + \"serviceTemplate\": { + \"spec\": { + \"type\": \"${svc_type}\" + } + } + } +}" + fi + fi + + apply_k0s_saia_service_annotations "${aiservice_name}" + + kubectl -n "${AI_NS}" annotate aiservice "${aiservice_name}" script-reconcile-ts="$(date +%s)" --overwrite >/dev/null + + if saia_service_template_enabled_k0s; then + log "Recreating slim public Service to ensure patched settings take effect..." + kubectl -n "${AI_NS}" delete svc "${public_svc_name}" --ignore-not-found >/dev/null 2>&1 || true + # Wait briefly for the operator to recreate it before moving on; if it + # doesn't come back in time the next reconcile will render it anyway. + local waited=0 + while ! kubectl -n "${AI_NS}" get svc "${public_svc_name}" >/dev/null 2>&1; do + [[ ${waited} -ge 300 ]] && break + sleep 5 + waited=$((waited + 5)) + done + fi +} + # ====== INSTALL FULL STACK ====== install_ai_platform_stack() { log "Installing complete AI Platform stack..." @@ -4916,6 +5036,8 @@ install_ai_platform_stack() { install_splunk_ai_operator install_ai_platform_cr patch_k0s_saia_public_service_workaround + # Expose slim on its own NodePort when the feature is enabled (no-op otherwise). + patch_k0s_slim_public_service_workaround # Now wait for Splunk Standalone to be ready (likely already done by now) wait_for_splunk_standalone @@ -5968,7 +6090,7 @@ show_platform_access_info() { log " kubectl get pods -n ${AI_NS} -l app.kubernetes.io/instance=splunk-standalone" log " " log " 💡 Access Splunk Web (once ready):" - log " kubectl port-forward -n ${AI_NS} svc/splunk-standalone-standalone-service 8000:8000" + log " kubectl port-forward -n ${AI_NS} svc/splunk-${AI_STANDALONE_NAME}-standalone-service 8000:8000" log " Open: http://localhost:8000" log ""