fix race

infra/k8s-master

@@ -29,7 +29,7 @@ if [ $cloud = aws ] && [ $platform != eks ] && [ $platform != ocp4 ]; then
     echo "Applying aws-eks-load-balancer-controller"
     helm repo add eks https://aws.github.io/eks-charts
     helm install aws-load-balancer-controller eks/aws-load-balancer-controller -n kube-system --version 3.2.1 --set clusterName=$name-$cluster --set feature-gates=ServiceTypeLoadBalancerOnly=true --set defaultLoadBalancerScheme=internet-facing --set defaultTags.pxd_uuid=$pxd_uuid --set replicaCount=1
-    kubectl wait --for=condition=ready pod -n kube-system -l app.kubernetes.io/name=aws-load-balancer-controller --timeout 30s
+    kubectl get mutatingwebhookconfiguration aws-load-balancer-webhook -o json | jq '.webhooks[].failurePolicy = "Ignore"' | kubectl apply -f - # Prevent service from blocking Service creation later
     kubectl apply -f - <<EOF
 apiVersion: v1
 kind: ServiceAccount

scripts/purelab

@@ -2,9 +2,9 @@ helm repo add aws-ebs-csi-driver https://kubernetes-sigs.github.io/aws-ebs-csi-d
 helm repo update
 helm install aws-ebs-csi-driver --namespace kube-system aws-ebs-csi-driver/aws-ebs-csi-driver --set controller.replicaCount=1
 
+kubectl rollout status deployment/aws-load-balancer-controller -n kube-system --timeout=300s
 kubectl apply -f /assets/purelab/purelab.yml
 kubectl scale sts node-desktop -n purelab --replicas $nodes
-kubectl wait --for=jsonpath='{.status.currentReplicas}'=$nodes sts/node-desktop -n purelab
 
 kubectl apply -f - <<EOF
 apiVersion: v1
@@ -108,6 +108,7 @@ for i in $logs $files; do
   aws s3 cp s3://peak-purelab/$i .
 done
 
+kubectl rollout status sts/node-desktop -n purelab --timeout=300s
 for i in $(kubectl get pod -n purelab -l app=node-desktop -o jsonpath='{.items[*].metadata.name}'); do (
   echo copying files to pod $i
   kubectl exec -n purelab $i -- mkdir -p /support/Toolkit /support/logs
@@ -122,6 +123,7 @@ for i in $(kubectl get pod -n purelab -l app=node-desktop -o jsonpath='{.items[*
 ) & done
 wait
 echo finished copying files
+kubectl delete pod -n purelab --field-selector=status.phase=Failed
 
 cat <<\EOF >/px-deploy/script-delete/purelab.sh
 #!/usr/bin/bash

scripts/snoop

@@ -5,4 +5,5 @@ EOF
 systemctl restart sshd
 
 install -d -m 777 /var/log/sessions
+kubectl rollout status deployment/aws-load-balancer-controller -n kube-system --timeout=300s
 kubectl apply -f /assets/snoop.yaml

scripts/training-modvirt-cluster → scripts/training-modvirt-ocp-cluster

@@ -9,6 +9,7 @@ for i in $(seq 1 $nodes); do
 done
 
 # Provision MinIO
+kubectl rollout status deployment/aws-load-balancer-controller -n kube-system --timeout=300s
 kubectl apply -f /assets/minio/minio-deployment.yml
 
 # Delete AWS keys

scripts/wetty

@@ -1,6 +1,7 @@
 mkdir /etc/wetty
 rm -f /etc/securetty
 sed -i '/master-1/s/$/ px-training/' /etc/hosts
+kubectl rollout status deployment/aws-load-balancer-controller -n kube-system --timeout=300s
 
 kubectl apply -f - <<EOF
 apiVersion: v1