Merge pull request #606 from Shunpoco/implement-downloader-retry

Implement retry logic on downloader

Author: vdice

.github/workflows/helm-chart-smoketest.yml

@@ -151,6 +151,9 @@ jobs:
             --set rcm.nodeInstallerImage.tag=chart-test \
             --set rcm.shimDownloaderImage.repository=shim-downloader \
             --set rcm.shimDownloaderImage.tag=chart-test \
+            --set rcm.shimDownloaderConfig.configMapName=configmap-test \
+            --set rcm.shimDownloaderConfig.content.numRetry=5 \
+            --set rcm.shimDownloaderConfig.content.sleepDuration=3 \
             deploy/helm
 
       - name: apply Spin shim
@@ -174,6 +177,10 @@ jobs:
         run: |
           timeout 60s bash -c 'until [[ "$(kubectl -n rcm get $(kubectl get pods -n rcm --no-headers -o name | grep install | head -n1) -o jsonpath="{.status.phase}" 2>/dev/null)" == "Succeeded" ]]; do sleep 2; done'
 
+      - name: verify configMap for shim-downloader is added
+        run: |
+          timeout 60s bash -c 'until [[ $(kubectl -n rcm get $(kubectl get pods -n rcm --no-headers -o name | grep install | head -n1) -o jsonpath="{.spec.initContainers[0].envFrom[0].configMapRef.name}" 2>/dev/null) == "configmap-test" ]]; do sleep 2; done'
+
       - name: run Spin App
         run: |
           kubectl apply -f testdata/apps/spin-app.yaml

deploy/helm/templates/deployment.yaml

@@ -49,6 +49,10 @@ spec:
             value: "{{ .Values.rcm.nodeInstallerImage.repository }}:{{ .Values.rcm.nodeInstallerImage.tag | default .Chart.AppVersion }}"
           - name: SHIM_NODE_INSTALLER_JOB_TTL
             value: "{{ .Values.rcm.nodeInstallerJob.ttl | default 0 }}"
+          {{- if .Values.rcm.shimDownloaderConfig }}
+          - name: SHIM_DOWNLOADER_CONFIG_MAP
+            value: "{{ .Values.rcm.shimDownloaderConfig.configMapName }}"
+          {{- end }}
           ports:
             - name: http
               containerPort: {{ .Values.service.port }}
@@ -79,3 +83,13 @@ spec:
       tolerations:
         {{- toYaml . | nindent 8 }}
       {{- end }}
+{{- if .Values.rcm.shimDownloaderConfig }}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ .Values.rcm.shimDownloaderConfig.configMapName }}
+data:
+  NUM_RETRY: "{{ .Values.rcm.shimDownloaderConfig.content.numRetry | default 3 }}"
+  SLEEP_DURATION: "{{ .Values.rcm.shimDownloaderConfig.content.sleepDuration | default 2 }}"
+{{- end }}

deploy/helm/values.yaml

@@ -20,6 +20,12 @@ rcm:
   nodeInstallerJob:
     ttl: 0
   leaderElectEnabled: false
+  # shimDownloaderConfig generates a ConfigMap which sets a downloader container's variables. Since those variables have defaults, you don't need to enable it unless you want to customize them.
+  # shimDownloaderConfig:
+  #   configMapName: shim-downloader-config
+  #   content:
+  #     numRetry: 3
+  #     sleepDuration: 2
 
 imagePullSecrets: []
 nameOverride: ""

images/downloader/download_shim.sh

@@ -23,10 +23,27 @@ log "start downloading shim from  ${SHIM_LOCATION}..." "INFO"
 
 mkdir -p /assets
 
+num_retry=${NUM_RETRY:-3}
+sleep_duration=${SLEEP_DURATION:-2}
+
+for (( i=0; i < $num_retry+1; i++ ))
+do
+    if curl -sLo "containerd-shim-${SHIM_NAME}" "${SHIM_LOCATION}"
+    then
+        ls -lah "containerd-shim-${SHIM_NAME}"
+        break
+    elif [ $i -eq $num_retry ]
+    then
+        log "number of failed downloads reached max retry ${num_retry}" "ERROR"
+        exit 1
+    else
+        log "download failed. retry after sleep... (${i}/${num_retry})" "ERROR"
+        sleep $sleep_duration
+    fi
+done
+
 # overwrite default name of shim binary; use the name of shim resource instead
 # to enable installing multiple versions of the same shim
-curl -sLo "containerd-shim-${SHIM_NAME}" "${SHIM_LOCATION}"
-ls -lah "containerd-shim-${SHIM_NAME}"
 
 log "$(curl --version)" "INFO"
 log "$(tar --version)" "INFO"

internal/controller/shim_controller.go

@@ -434,6 +434,18 @@ func (sr *ShimReconciler) setOperationConfiguration(shim *rcmv1.Shim, opConfig *
 				},
 			},
 		}}
+		if configMapName := os.Getenv("SHIM_DOWNLOADER_CONFIG_MAP"); configMapName != "" {
+			opConfig.initContainer[0].EnvFrom = []corev1.EnvFromSource{
+				{
+					ConfigMapRef: &corev1.ConfigMapEnvSource{
+						LocalObjectReference: corev1.LocalObjectReference{
+							Name: configMapName,
+						},
+					},
+				},
+			}
+		}
+
 		opConfig.args = []string{
 			"install",
 			"-H",