Use images published to MCR (#143)

Author: johnstairs

.github/workflows/pr-ci.yml

@@ -282,27 +282,6 @@ jobs:
           set -euo pipefail
           make -s integration-test-no-up
 
-  publish-official-images:
-    needs:
-      - test-azure-needs-hosted-runner
-    runs-on: ${{ fromJson(needs.test-azure-needs-hosted-runner.outputs.AZURE_RUNS_ON_JSON)}}
-    defaults:
-      run:
-        shell: bash
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-
-      - name: Log in to Azure
-        uses: ./.github/actions/login-azure
-
-      - name: Publish official images
-        run: |
-          set -euo pipefail
-          make publish-official-images
-
   build-windows-binaries:
     runs-on: ubuntu-latest
     defaults:
@@ -539,8 +518,10 @@ jobs:
       - name: get container registry
         run: |
           set -euo pipefail
-          official_container_registry=$(echo "$DEVELOPER_CONFIG_BASE64" | base64 -d | jq -r '.officialContainerRegistry.fqdn')
-          echo "OFFICIAL_CONTAINER_REGISTRY=$(echo $official_container_registry)" >> $GITHUB_ENV
+          official_pull_container_registry=$(echo "$DEVELOPER_CONFIG_BASE64" | base64 -d | jq -r '.officialPullContainerRegistry.fqdn')
+          echo "OFFICIAL_PULL_CONTAINER_REGISTRY=$(echo $official_pull_container_registry)" >> $GITHUB_ENV
+          official_pull_container_registry_directory=$(echo "$DEVELOPER_CONFIG_BASE64" | base64 -d | jq -r '.officialPullContainerRegistry.directory // ""')
+          echo "OFFICIAL_PULL_CONTAINER_REGISTRY_DIRECTORY=$(echo $official_pull_container_registry_directory)" >> $GITHUB_ENV
 
       - name: Run GoReleaser
         uses: goreleaser/goreleaser-action@v5

Makefile

@@ -67,9 +67,11 @@ _docker-build:
 	target_arg="--${DOCKER_BUILD_TARGET}"
 
 	tag=$${EXPLICIT_IMAGE_TAG:-dev}
+	dev_config=$$(scripts/get-config.sh --dev -o json)
 
-	registry=$$(scripts/get-config.sh --dev -e .wipContainerRegistry.fqdn)
-	scripts/build-images.sh $$target_arg ${DOCKER_BUILD_ARCH_FLAGS} ${DOCKER_BUILD_PUSH_FLAGS} --tag "$$tag" --registry "$${registry}"
+	registry=$$(echo "$${dev_config}" | jq -r '.wipContainerRegistry.fqdn')
+	directory=$$(echo "$${dev_config}" | jq -r '.wipContainerRegistry.directory // ""')
+	scripts/build-images.sh $$target_arg ${DOCKER_BUILD_ARCH_FLAGS} ${DOCKER_BUILD_PUSH_FLAGS} --tag "$$tag" --registry "$${registry}" --registry-directory "$${directory}"
 
 docker-build-test: login-wip-acr
 	$(MAKE) _docker-build DOCKER_BUILD_TARGET=test-connectivity
@@ -86,11 +88,13 @@ docker-build-worker-waiter: login-wip-acr
 docker-build: docker-build-test docker-build-tyger-server docker-build-buffer-sidecar docker-build-worker-waiter
 
 publish-official-images:
-	registry=$$(scripts/get-config.sh --dev -e .officialContainerRegistry.fqdn)
+	dev_config=$$(scripts/get-config.sh --dev -o json)
+	registry=$$(echo "$${dev_config}" | jq -r '.officialPushContainerRegistry.fqdn')
 	registry_name=$$(echo "$$registry" | cut -d'.' -f1)
 	az acr login --name "$${registry_name}"
+	registry_directory=$$(echo "$${dev_config}" | jq -r '.officialPushContainerRegistry.directory // ""')
 	tag=$$(git describe --tags)
-	scripts/build-images.sh --push --push-force --arch amd64 --arch arm64 --tyger-server --worker-waiter --buffer-sidecar --helm --tag "$${tag}" --registry "$${registry}"
+	scripts/build-images.sh --push --push-force --arch amd64 --arch arm64 --tyger-server --worker-waiter --buffer-sidecar --helm --tag "$${tag}" --registry "$${registry}" --registry-directory "$${registry_directory}"
 
 integration-test-no-up-prereqs:
 
@@ -119,13 +123,13 @@ get-tyger-uri:
 	echo ${TYGER_URI}
 
 install-cli:
-	official_container_registry=$$(scripts/get-config.sh --dev -e .officialContainerRegistry.fqdn)
+	dev_config=$$(scripts/get-config.sh --dev -o json)
+	container_registry=$$(echo "$${dev_config}" | jq -r '.wipContainerRegistry.fqdn')
+	container_registry_directory=$$(echo "$${dev_config}" | jq -r '.wipContainerRegistry.directory // ""')
 	cd cli
 	tag=$$(git describe --tags 2> /dev/null || echo "0.0.0")
 	CGO_ENABLED=0 go install -buildvcs=false -ldflags="-s -w \
-		-X main.version=$${tag} \
-		-X github.com/microsoft/tyger/cli/internal/install.ContainerRegistry=$${official_container_registry} \
-		-X github.com/microsoft/tyger/cli/internal/install.ContainerImageTag=$${tag}" \
+		-X main.version=$${tag}" \
 		./cmd/tyger ./cmd/buffer-sidecar ./cmd/tyger-proxy
 
 cli-ready: install-cli

cli/.goreleaser.yaml

@@ -11,7 +11,8 @@ builds:
     ldflags:
       - -s -w
       - -X main.version={{ .Tag }}
-      - -X github.com/microsoft/tyger/cli/internal/install.ContainerRegistry={{ .Env.OFFICIAL_CONTAINER_REGISTRY }}
+      - -X github.com/microsoft/tyger/cli/internal/install.ContainerRegistry={{ .Env.OFFICIAL_PULL_CONTAINER_REGISTRY }}
+      - -X github.com/microsoft/tyger/cli/internal/install.ContainerRegistryDirectory={{ .Env.OFFICIAL_PULL_CONTAINER_REGISTRY_DIRECTORY }}
       - -X github.com/microsoft/tyger/cli/internal/install.ContainerImageTag={{ .Tag }}
     goos:
       - linux

cli/internal/install/cloudinstall/config.tpl

@@ -111,7 +111,7 @@ api:
   #   tyger:
   #     repoName:
   #     repoUrl: # not set if using `chartRef`
-  #     chartRef: # e.g. oci://tyger.azurecr.io/helm/tyger
+  #     chartRef: # e.g. oci://mcr.microsoft.com/tyger/helm/tyger
   #     namespace:
   #     version:
   #     values:

cli/internal/install/cloudinstall/helm.go

@@ -321,14 +321,6 @@ func (inst *Installer) InstallTyger(ctx context.Context) error {
 }
 
 func (inst *Installer) InstallTygerHelmChart(ctx context.Context, restConfig *rest.Config, dryRun bool) (manifest string, valuesYaml string, err error) {
-	if install.ContainerRegistry == "" {
-		panic("officialContainerRegistry not set during build")
-	}
-
-	if install.ContainerImageTag == "" {
-		panic("officialContainerImageTag not set during build")
-	}
-
 	clustersConfigJson, err := json.Marshal(inst.Config.Cloud.Compute.Clusters)
 	if err != nil {
 		return "", "", fmt.Errorf("failed to marshal cluster configuration: %w", err)
@@ -412,13 +404,13 @@ func (inst *Installer) InstallTygerHelmChart(ctx context.Context, restConfig *re
 	helmConfig := HelmChartConfig{
 		Namespace:   TygerNamespace,
 		ReleaseName: DefaultTygerReleaseName,
-		ChartRef:    fmt.Sprintf("oci://%s/helm/tyger", install.ContainerRegistry),
+		ChartRef:    fmt.Sprintf("oci://%s%shelm/tyger", install.ContainerRegistry, install.GetNormalizedContainerRegistryDirectory()),
 		Version:     install.ContainerImageTag,
 		Values: map[string]any{
-			"image":              fmt.Sprintf("%s/tyger-server:%s", install.ContainerRegistry, install.ContainerImageTag),
-			"bufferSidecarImage": fmt.Sprintf("%s/buffer-sidecar:%s", install.ContainerRegistry, install.ContainerImageTag),
-			"bufferCopierImage":  fmt.Sprintf("%s/buffer-copier:%s", install.ContainerRegistry, install.ContainerImageTag),
-			"workerWaiterImage":  fmt.Sprintf("%s/worker-waiter:%s", install.ContainerRegistry, install.ContainerImageTag),
+			"image":              fmt.Sprintf("%s%styger-server:%s", install.ContainerRegistry, install.GetNormalizedContainerRegistryDirectory(), install.ContainerImageTag),
+			"bufferSidecarImage": fmt.Sprintf("%s%sbuffer-sidecar:%s", install.ContainerRegistry, install.GetNormalizedContainerRegistryDirectory(), install.ContainerImageTag),
+			"bufferCopierImage":  fmt.Sprintf("%s%sbuffer-copier:%s", install.ContainerRegistry, install.GetNormalizedContainerRegistryDirectory(), install.ContainerImageTag),
+			"workerWaiterImage":  fmt.Sprintf("%s%sworker-waiter:%s", install.ContainerRegistry, install.GetNormalizedContainerRegistryDirectory(), install.ContainerImageTag),
 			"hostname":           inst.Config.Api.DomainName,
 			"identity": map[string]any{
 				"tygerServer": map[string]any{

cli/internal/install/dockerinstall/validation.go

@@ -154,7 +154,7 @@ func (inst *Installer) QuickValidateConfig() bool {
 }
 
 func defaultImage(repo string) string {
-	return fmt.Sprintf("%s/%s:%s", install.ContainerRegistry, repo, install.ContainerImageTag)
+	return fmt.Sprintf("%s%s%s:%s", install.ContainerRegistry, install.GetNormalizedContainerRegistryDirectory(), repo, install.ContainerImageTag)
 }
 
 func validationError(success *bool, format string, args ...any) {

cli/internal/install/installer.go

@@ -7,6 +7,8 @@ import (
 	"context"
 	"errors"
 	"io"
+	"strings"
+	"sync"
 )
 
 var (
@@ -15,11 +17,21 @@ var (
 )
 
 var (
-	// Set during build but we provide defaults so that there is some value when debugging.
-	// We will need to update these from time to time. Alternatively, you can set the registry
-	// values using the --set command-line argument.
-	ContainerRegistry string = "tyger.azurecr.io"
-	ContainerImageTag string = "v0.4.0-112-g428a5e8"
+	// Set during build
+	ContainerRegistry          string = ""
+	ContainerRegistryDirectory string = ""
+	ContainerImageTag          string = ""
+
+	GetNormalizedContainerRegistryDirectory = sync.OnceValue(func() string {
+		normalized := ContainerRegistryDirectory
+		if !strings.HasPrefix(normalized, "/") {
+			normalized = "/" + normalized
+		}
+		if !strings.HasSuffix(normalized, "/") {
+			normalized = normalized + "/"
+		}
+		return normalized
+	})
 )
 
 type Installer interface {

deploy/config/microsoft/devconfig.yml

@@ -1,10 +1,12 @@
 subscriptionId: 87d8acb3-5176-4651-b457-6ab9cefd8e3d
 wipContainerRegistry:
-  name: eminence
   fqdn: eminence.azurecr.io
-officialContainerRegistry:
-  name: tyger
-  fqdn: tyger.azurecr.io
+officialPushContainerRegistry:
+  fqdn: tygerprod.azurecr.io
+  directory: /public/tyger
+officialPullContainerRegistry:
+  fqdn: mcr.microsoft.com
+  directory: /tyger
 keyVault: eminence
 testAppUri: api://tyger-test-client
 pemCertSecret:

docs/introduction/installation/cloud-installation.md

@@ -157,7 +157,7 @@ api:
   #   tyger:
   #     repoName:
   #     repoUrl: # not set if using `chartRef`
-  #     chartRef: # e.g. oci://tyger.azurecr.io/helm/tyger
+  #     chartRef: # e.g. oci://mcr.microsoft.com/tyger/helm/tyger
   #     namespace:
   #     version:
   #     values: # Helm values overrides
@@ -175,7 +175,7 @@ command-line with `--set`. For example:
 
 ```bash
 tyger api install -f config.yml \
-  --set api.helm.tyger.chartRef=oci://tyger.azurecr.io/helm/tyger \
+  --set api.helm.tyger.chartRef=oci://mcr.microsoft.com/tyger/helm/tyger \
   --set api.helm.tyger.version=v0.4.0
 ```
 
@@ -216,7 +216,7 @@ tyger api install -f config.yml
 ```
 
 This command installs the Tyger API from the Helm chart in the
-`tyger.azurecr.io/helm/tyger` registry, using a version baked into the `tyger`
+`mcr.microsoft.com/tyger/helm/tyger` registry, using a version baked into the `tyger`
 CLI. Upgrade the server by updating the CLI and rerunning `tyger api install`.
 
 The API's TLS certificate is automatically created using [Let's

scripts/build-images.sh

@@ -19,6 +19,7 @@ Options:
   --worker-waiter                  Build (and optionally push) the worker-waiter image
   --buffer-sidecar                 Build (and optionally push) the buffer-sidecar image
   --helm                           Package and push the Tyger Helm chart
+  --registry-directory             The parent directory of the repositories. e.g. <registry>/<registry-dir>/<repo-name>
   --arch amd64|arm64               The architecture to build for. Can be specified multiple times.
   --push                           Push runtime images (requires --tag or --use-git-hash-as-tag)
   --push-force                     Force runtime images, will overwrite images with same tag (requires --tag or --use-git-hash-as-tag)
@@ -28,6 +29,8 @@ Options:
 EOF
 }
 
+registry_dir="/"
+
 while [[ $# -gt 0 ]]; do
   key="$1"
 
@@ -71,6 +74,10 @@ while [[ $# -gt 0 ]]; do
     push=1
     shift
     ;;
+  --registry-directory)
+    registry_dir="$2"
+    shift 2
+    ;;
   --push-force)
     push=1
     force=1
@@ -96,6 +103,16 @@ while [[ $# -gt 0 ]]; do
   esac
 done
 
+# Ensure registry_dir starts with a /
+if [[ ! "$registry_dir" =~ ^/ ]]; then
+  registry_dir="/$registry_dir"
+fi
+
+# Ensure registry_dir ends with a /
+if [[ ! "$registry_dir" =~ /$ ]]; then
+  registry_dir="$registry_dir/"
+fi
+
 # if nether amd64 nor arm64 is specified, build for both
 if [[ -z "${amd64:-}" && -z "${arm64:-}" ]]; then
   amd64=true
@@ -107,7 +124,7 @@ export DOCKER_BUILDKIT=1
 repo_root_dir="$(dirname "$0")/.."
 
 function build_and_push_platform() {
-  full_image="${container_registry_fqdn}/${repo}:${image_tag_with_platform}"
+  full_image="${container_registry_fqdn}${registry_dir}${repo}:${image_tag_with_platform}"
   echo "Building image ${full_image}..."
 
   set +e
@@ -156,8 +173,8 @@ function build_and_push() {
     return 0
   fi
 
-  manifest_name="${container_registry_fqdn}/${repo}:${image_tag}"
-  docker manifest create --amend "${manifest_name}" "${container_registry_fqdn}/${repo}:${image_tag}-amd64" "${container_registry_fqdn}/${repo}:${image_tag}-arm64" >/dev/null
+  manifest_name="${container_registry_fqdn}${registry_dir}${repo}:${image_tag}"
+  docker manifest create --amend "${manifest_name}" "${container_registry_fqdn}${registry_dir}${repo}:${image_tag}-amd64" "${container_registry_fqdn}${registry_dir}${repo}:${image_tag}-arm64" >/dev/null
 
   # Push manigest
   if [[ -z "${force:-}" ]]; then
@@ -232,7 +249,7 @@ if [[ -n "${helm:-}" ]]; then
   username="00000000-0000-0000-0000-000000000000"
   echo "${token}" | docker login "${container_registry_fqdn}" -u "${username}" --password-stdin
 
-  helm_repo_namespace="oci://${container_registry_fqdn}/helm"
+  helm_repo_namespace="oci://${container_registry_fqdn}${registry_dir}helm"
   chart_dir=${repo_root_dir}/deploy/helm/tyger
   package_dir=$(mktemp -d)