Allow service tags to be specified on public IP addresses (#261)

Author: johnstairs

cli/internal/install/cloudinstall/cloud-config-pretty.tpl

@@ -106,6 +106,23 @@ cloud:
             # osSku: defaults to AzureLinux
             {{- end }}
           {{- end }}
+
+        {{- if .InboundIpServiceTags }}
+
+        inboundIpServiceTags:
+            {{- range .InboundIpServiceTags }}
+            - type: {{ .Type }}
+              tag: {{ .Tag }}
+            {{- end }}
+        {{- end }}
+        {{- if .OutboundIpServiceTags }}
+
+        outboundIpServiceTags:
+          {{- range .OutboundIpServiceTags }}
+          - type: {{ .Type }}
+            tag: {{ .Tag }}
+          {{- end }}
+        {{- end }}
     {{- end }}
 
     # These are the principals that will have the ability to run `tyger api install`.

cli/internal/install/cloudinstall/cloudconfig.go

@@ -159,18 +159,25 @@ func (c *ComputeConfig) GetApiHostCluster() *ClusterConfig {
 	panic("API host cluster not found - this should have been caught during validation")
 }
 
+type IpServiceTag struct {
+	Type string `yaml:"type"`
+	Tag  string `yaml:"tag"`
+}
+
 type ClusterConfig struct {
-	Name              string                                    `yaml:"name"`
-	ApiHost           bool                                      `yaml:"apiHost"`
-	Location          string                                    `yaml:"location"`
-	Sku               armcontainerservice.ManagedClusterSKUTier `yaml:"sku"`
-	KubernetesVersion string                                    `yaml:"kubernetesVersion,omitempty"`
-	ExistingSubnet    *SubnetReference                          `yaml:"existingSubnet,omitempty"`
-	SystemNodePool    *NodePoolConfig                           `yaml:"systemNodePool"`
-	UserNodePools     []*NodePoolConfig                         `yaml:"userNodePools"`
-	PodCidr           string                                    `yaml:"podCidr,omitempty"`
-	ServiceCidr       string                                    `yaml:"serviceCidr,omitempty"`
-	DnsServiceIp      string                                    `yaml:"dnsServiceIp,omitempty"`
+	Name                  string                                    `yaml:"name"`
+	ApiHost               bool                                      `yaml:"apiHost"`
+	Location              string                                    `yaml:"location"`
+	Sku                   armcontainerservice.ManagedClusterSKUTier `yaml:"sku"`
+	KubernetesVersion     string                                    `yaml:"kubernetesVersion,omitempty"`
+	ExistingSubnet        *SubnetReference                          `yaml:"existingSubnet,omitempty"`
+	SystemNodePool        *NodePoolConfig                           `yaml:"systemNodePool"`
+	UserNodePools         []*NodePoolConfig                         `yaml:"userNodePools"`
+	PodCidr               string                                    `yaml:"podCidr,omitempty"`
+	ServiceCidr           string                                    `yaml:"serviceCidr,omitempty"`
+	DnsServiceIp          string                                    `yaml:"dnsServiceIp,omitempty"`
+	InboundIpServiceTags  []IpServiceTag                            `yaml:"inboundIpServiceTags,omitempty"`
+	OutboundIpServiceTags []IpServiceTag                            `yaml:"outboundIpServiceTags,omitempty"`
 }
 
 type SubnetReference struct {

cli/internal/install/cloudinstall/compute.go

@@ -46,6 +46,11 @@ func (inst *Installer) getCluster(ctx context.Context, clusterConfig *ClusterCon
 }
 
 func (inst *Installer) createCluster(ctx context.Context, clusterConfig *ClusterConfig) (*armcontainerservice.ManagedCluster, error) {
+	outboundIpAddress, err := inst.createOutboundIpAddress(ctx, clusterConfig)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create outbound IP address: %w", err)
+	}
+
 	var tags map[string]*string
 
 	existingCluster, err := inst.getCluster(ctx, clusterConfig)
@@ -122,7 +127,9 @@ func (inst *Installer) createCluster(ctx context.Context, clusterConfig *Cluster
 					Enabled: Ptr(true),
 				},
 			},
-			NetworkProfile: &armcontainerservice.NetworkProfile{},
+			NetworkProfile: &armcontainerservice.NetworkProfile{
+				LoadBalancerProfile: &armcontainerservice.ManagedClusterLoadBalancerProfile{},
+			},
 		},
 		SKU: &armcontainerservice.ManagedClusterSKU{
 			Name: Ptr(armcontainerservice.ManagedClusterSKUNameBase),
@@ -148,6 +155,12 @@ func (inst *Installer) createCluster(ctx context.Context, clusterConfig *Cluster
 		}
 	}
 
+	if outboundIpAddress != nil {
+		cluster.Properties.NetworkProfile.LoadBalancerProfile.OutboundIPs = &armcontainerservice.ManagedClusterLoadBalancerProfileOutboundIPs{
+			PublicIPs: []*armcontainerservice.ResourceReference{{ID: outboundIpAddress.ID}},
+		}
+	}
+
 	if workspace := inst.Config.Cloud.LogAnalyticsWorkspace; workspace != nil {
 		oic, err := armoperationalinsights.NewWorkspacesClient(inst.Config.Cloud.SubscriptionID, inst.Credential, nil)
 		if err != nil {
@@ -459,9 +472,77 @@ func (inst *Installer) createCluster(ctx context.Context, clusterConfig *Cluster
 		}
 	}
 
+	if outboundIpAddress != nil {
+		if err := assignRbacRole(ctx, []string{*createdCluster.Identity.PrincipalID}, false, *outboundIpAddress.ID, "Network Contributor", inst.Config.Cloud.SubscriptionID, inst.Credential); err != nil {
+			return nil, fmt.Errorf("failed to assign RBAC role on outbound IP address: %w", err)
+		}
+	}
+
 	return &createdCluster, nil
 }
 
+func (inst *Installer) createOutboundIpAddress(ctx context.Context, cluster *ClusterConfig) (*armnetwork.PublicIPAddress, error) {
+	if len(cluster.OutboundIpServiceTags) == 0 {
+		// We let AKS manage the IP for us
+		return nil, nil
+	}
+
+	log.Ctx(ctx).Info().Msg("Creating outbound IP address")
+	publicIPAddressesClient, err := armnetwork.NewPublicIPAddressesClient(inst.Config.Cloud.SubscriptionID, inst.Credential, nil)
+	if err != nil {
+		log.Fatal().Err(err).Msg("failed to create public IP addresses client")
+	}
+
+	ipAddressName := fmt.Sprintf("%s-outbound-ip", cluster.Name)
+
+	var tags map[string]*string
+	if resp, err := publicIPAddressesClient.Get(ctx, inst.Config.Cloud.ResourceGroup, ipAddressName, nil); err == nil {
+		if existingTag, ok := resp.Tags[TagKey]; ok {
+			if *existingTag != inst.Config.EnvironmentName {
+				return nil, fmt.Errorf("public IP address '%s' is already in use by environment '%s'", ipAddressName, *existingTag)
+			}
+			tags = resp.Tags
+		}
+	}
+
+	if tags == nil {
+		tags = make(map[string]*string)
+	}
+	tags[TagKey] = &inst.Config.EnvironmentName
+
+	ipTags := []*armnetwork.IPTag{}
+	for _, t := range cluster.OutboundIpServiceTags {
+		ipTags = append(ipTags, &armnetwork.IPTag{
+			IPTagType: &t.Type,
+			Tag:       &t.Tag,
+		})
+	}
+
+	ipAddress := armnetwork.PublicIPAddress{
+		Location: &cluster.Location,
+		SKU: &armnetwork.PublicIPAddressSKU{
+			Name: Ptr(armnetwork.PublicIPAddressSKUNameStandard),
+			Tier: Ptr(armnetwork.PublicIPAddressSKUTierRegional),
+		},
+		Properties: &armnetwork.PublicIPAddressPropertiesFormat{
+			PublicIPAllocationMethod: Ptr(armnetwork.IPAllocationMethodStatic),
+			IPTags:                   ipTags,
+		},
+	}
+
+	poller, err := publicIPAddressesClient.BeginCreateOrUpdate(ctx, inst.Config.Cloud.ResourceGroup, ipAddressName, ipAddress, nil)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create output IP address: %w", err)
+	}
+
+	res, err := poller.PollUntilDone(ctx, nil)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create output IP address: %w", err)
+	}
+
+	return &res.PublicIPAddress, nil
+}
+
 func clusterNeedsUpdating(cluster, existingCluster armcontainerservice.ManagedCluster) (hasChanges bool, onlyScaleDown bool) {
 	if *existingCluster.Properties.ProvisioningState != "Succeeded" {
 		return true, false
@@ -628,6 +709,25 @@ func clusterNeedsUpdating(cluster, existingCluster armcontainerservice.ManagedCl
 		return true, false
 	}
 
+	desiredOutputIpCount := 0
+	if cluster.Properties.NetworkProfile.LoadBalancerProfile.OutboundIPs != nil {
+		desiredOutputIpCount = len(cluster.Properties.NetworkProfile.LoadBalancerProfile.OutboundIPs.PublicIPs)
+	}
+
+	existingOutputIpCount := 0
+	if existingCluster.Properties.NetworkProfile.LoadBalancerProfile.OutboundIPs != nil {
+		existingOutputIpCount = len(existingCluster.Properties.NetworkProfile.LoadBalancerProfile.OutboundIPs.PublicIPs)
+	}
+
+	if desiredOutputIpCount != existingOutputIpCount ||
+		(desiredOutputIpCount > 0 &&
+			!slices.EqualFunc(
+				cluster.Properties.NetworkProfile.LoadBalancerProfile.OutboundIPs.PublicIPs,
+				existingCluster.Properties.NetworkProfile.LoadBalancerProfile.OutboundIPs.PublicIPs,
+				func(a, b *armcontainerservice.ResourceReference) bool { return *a.ID == *b.ID })) {
+		return true, false
+	}
+
 	return hasChanges, onlyScaleDown
 }
 

cli/internal/install/cloudinstall/helm.go

@@ -7,6 +7,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"maps"
 	"net/http"
 	"os"
 	"reflect"
@@ -58,19 +59,28 @@ func (inst *Installer) installTraefik(ctx context.Context, restConfigPromise *in
 		return nil, fmt.Errorf("failed to ensure Traefik dynamic ConfigMap: %w", err)
 	}
 
-	var annotations map[string]any
+	var serviceAnnotations map[string]any
 	if inst.Config.Cloud.PrivateNetworking {
-		annotations = map[string]any{
+		serviceAnnotations = map[string]any{
 			"service.beta.kubernetes.io/azure-load-balancer-internal": "true",
 			"service.beta.kubernetes.io/azure-pls-create":             "true",
 			"service.beta.kubernetes.io/azure-pls-name":               TraefikPrivateLinkServiceName,
 		}
 	} else {
-		annotations = map[string]any{
+		serviceAnnotations = map[string]any{
 			"service.beta.kubernetes.io/azure-dns-label-name": inst.Config.Cloud.Compute.DnsLabel,
 		}
 	}
 
+	if ipServiceTags := inst.Config.Cloud.Compute.GetApiHostCluster().InboundIpServiceTags; len(ipServiceTags) > 0 {
+		pairs := []string{}
+		for _, t := range ipServiceTags {
+			pairs = append(pairs, fmt.Sprintf("%s=%s", t.Type, t.Tag))
+		}
+
+		serviceAnnotations["service.beta.kubernetes.io/azure-pip-ip-tags"] = strings.Join(pairs, ",")
+	}
+
 	traefikConfig := HelmChartConfig{
 		RepoName:    "traefik",
 		Namespace:   TraefikNamespace,
@@ -94,7 +104,7 @@ func (inst *Installer) installTraefik(ctx context.Context, restConfigPromise *in
 				},
 			},
 			"service": map[string]any{
-				"annotations": annotations,
+				"annotations": serviceAnnotations,
 				"spec": map[string]any{
 					"externalTrafficPolicy": "Local", // in order to preserve client IP addresses
 				},
@@ -156,6 +166,41 @@ func (inst *Installer) installTraefik(ctx context.Context, restConfigPromise *in
 		overrides = inst.Config.Cloud.Compute.Helm.Traefik
 	}
 
+	// Check if there is an existing release to see if should be removed first.
+	helmOptions := helmclient.RestConfClientOptions{
+		RestConfig: restConfig,
+		Options: &helmclient.Options{
+			DebugLog: func(format string, v ...any) {
+				log.Debug().Msgf(format, v...)
+			},
+			Namespace: traefikConfig.Namespace,
+		},
+	}
+
+	helmClient, err := helmclient.NewClientFromRestConf(&helmOptions)
+	if err != nil {
+		return false, fmt.Errorf("failed to create helm client: %w", err)
+	}
+	if existingRelease, err := helmClient.GetRelease(traefikConfig.ReleaseName); err != nil {
+		if !errors.Is(err, driver.ErrReleaseNotFound) {
+			return nil, fmt.Errorf("failed to get existing Traefik release: %w", err)
+		}
+	} else {
+		var existingServiceAnnotations map[string]any
+		if svc, _ := existingRelease.Config["service"].(map[string]any); svc != nil {
+			existingServiceAnnotations, _ = svc["annotations"].(map[string]any)
+		}
+
+		if !maps.Equal(existingServiceAnnotations, serviceAnnotations) {
+			log.Ctx(ctx).Warn().Msg("Existing Traefik installation has different service annotations, uninstalling it first")
+			if err = helmClient.UninstallReleaseByName(traefikConfig.ReleaseName); err != nil {
+				log.Warn().Msgf("Failed to uninstall existing Traefik release: %v", err)
+			} else {
+				time.Sleep(2 * time.Minute) // Give some time for Azure resources to be removed
+			}
+		}
+	}
+
 	startTime := time.Now().Add(-10 * time.Second)
 	if _, _, err := installHelmChart(ctx, restConfig, &traefikConfig, overrides, false); err != nil {
 		installErr := err

cli/internal/install/cloudinstall/storage.go

@@ -32,7 +32,7 @@ func (inst *Installer) CreateStorageAccount(ctx context.Context,
 	if resp, err := storageClient.GetProperties(ctx, resourceGroupName, storageAccountConfig.Name, nil); err == nil {
 		if existingTag, ok := resp.Tags[TagKey]; ok {
 			if *existingTag != inst.Config.EnvironmentName {
-				return nil, fmt.Errorf("storage account '%s' is already in use by enrironment '%s'", storageAccountConfig.Name, *existingTag)
+				return nil, fmt.Errorf("storage account '%s' is already in use by environment '%s'", storageAccountConfig.Name, *existingTag)
 			}
 			tags = resp.Tags
 		}

deploy/config/microsoft/cloudconfig-private-link.yml

@@ -214,6 +214,8 @@ organizations:
               objectId: 15bf92f7-9210-480f-a6ad-8d576d0baac7
               displayName: tyger-client-contributor
 
+        miseImage: eminence.azurecr.io/mise-sidecar:dev-1-37-0
+
       buffers:
         # TTL for active buffers before they are automatically soft-deleted (D.HH:MM:SS) (0 = never expire)
         activeLifetime: 0.00:00

deploy/config/microsoft/cloudconfig.yml

@@ -76,6 +76,14 @@ cloud:
             maxCount: 10
             # osSku: defaults to AzureLinux
 
+        inboundIpServiceTags:
+            - type: FirstPartyUsage
+              tag: /NonProd
+
+        outboundIpServiceTags:
+          - type: FirstPartyUsage
+            tag: /NonProd
+
     # These are the principals that will have the ability to run `tyger api install`.
     # They will have access to the "tyger" namespace in each cluster and will have
     # the necessary Azure RBAC role assignments.