Support upgrade scenario

Signed-off-by: wen.rui <wen.rui@daocloud.io>

Author: Rei1010

.github/workflows/auto-release.yaml

@@ -135,18 +135,19 @@ jobs:
     uses: ./.github/workflows/call-release-website.yaml
     with:
       ref: ${{ needs.ensure-tag.outputs.tag }}
-  # excute a full e2e test when hami release
+  # execute a full e2e test when hami release
   release-e2e:
-    needs: [release-notes]
+    needs: [ensure-tag, release-notes]
     uses: ./.github/workflows/call-e2e.yaml
     with:
       ref: ${{ needs.ensure-tag.outputs.tag }}
       type: "release"
 
-  # excute a compatibility test when hami release
+  # execute a compatibility test when hami release
   release-e2e-upgrade:
-      needs: [release-notes]
+      needs: [ensure-tag, release-notes]
       uses: ./.github/workflows/call-e2e-upgrade.yaml
       with:
         ref: ${{ needs.ensure-tag.outputs.tag }}
+        type: "release"
 

.github/workflows/call-e2e-upgrade.yaml

@@ -6,13 +6,77 @@ on:
       ref:
         required: true
         type: string
-permissions: write-all
+      type:
+        description: 'E2E type'
+        type: string
+        default: pullrequest
 
 jobs:
-  upgrade-e2e:
-    runs-on: ubuntu-latest
+  e2e-upgrade-test:
+    strategy:
+      matrix:
+        include:
+          - device: nvidia
+            type: tesla-p4
+    #          - device: nvidia
+    #            type: rtx-4090
+    #          - device: huawei
+    #            type: ascend-910b
+    runs-on: [ "${{ matrix.device }}", "${{ matrix.type }}" ]
+    environment: ${{ matrix.device }}
+    env:
+      HAMI_VERSION: ${{ inputs.ref }}
+      E2E_TYPE: ${{ inputs.type }}
     steps:
-      - name: e2e upgrade test
-        # https://github.com/actions/virtual-environments/issues/709
+      - name: checkout code
+        uses: actions/checkout@v4
+
+      - name: install Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: "1.21"
+
+      - name: setup e2e env
+        run: |
+          make e2e-env-setup
+
+      - name: download hami helm
+        if: inputs.type == 'pullrequest'
+        uses: actions/download-artifact@v4
+        with:
+          name: chart_package_artifact
+          path: charts/
+
+      - name: download hami image
+        if: inputs.type == 'pullrequest'
+        uses: actions/download-artifact@v4
+        with:
+          name: hami-image
+          path: ./image
+
+      - name: load e2e image
+        if: inputs.type == 'pullrequest'
+        run: |
+          echo "Loading Docker image from image.tar..."
+          if [ -z "${VSPHERE_GPU_VM_IP}" ]; then
+            echo "Error: VSPHERE_GPU_VM_IP is not defined!"
+            exit 1
+          fi
+          scp ./image/image.tar root@$VSPHERE_GPU_VM_IP:/home/
+          ssh root@$VSPHERE_GPU_VM_IP "nerdctl load -i /home/image.tar"
+          ssh root@$VSPHERE_GPU_VM_IP "nerdctl image ls | grep hami"
+
+      - name: deploy previous release hami helm
+        run: |
+          HAMI_VERSION=$(git tag -l --sort=-v:refname | grep -v ${{ env.HAMI_VERSION }}  | head -n3 )
+          E2E_TYPE="upgrade"
+          echo "Previous release tag is: ${HAMI_VERSION}"
+          make helm-deploy
+
+      - name: deploy release hami helm
+        run: |
+          make helm-deploy
+
+      - name: e2e test
         run: |
-          echo "Need to add e2e upgrade test"
+          make e2e-test

.github/workflows/call-e2e.yaml

@@ -9,7 +9,6 @@ on:
         type: string
       type:
         description: 'E2E type'
-        required: true
         type: string
         default: pullrequest
 
@@ -27,8 +26,8 @@ jobs:
     runs-on: [ "${{ matrix.device }}", "${{ matrix.type }}" ]
     environment: ${{ matrix.device }}
     env:
-      E2E_TYPE: ${{ inputs.type }}
       HAMI_VERSION: ${{ inputs.ref }}
+      E2E_TYPE: ${{ inputs.type }}
     steps:
       - name: checkout code
         uses: actions/checkout@v4

.github/workflows/ci.yaml

@@ -251,3 +251,12 @@ jobs:
     with:
       ref: ${{ needs.get_info.outputs.version }}
       type: "pullrequest"
+
+  # execute a full e2e test when hami code merge
+  e2e_upgrade_test:
+    uses: ./.github/workflows/call-e2e-upgrade.yaml
+    needs: [ package_chart, get_info, build ]
+    if: needs.get_info.outputs.e2e_run == 'true'
+    with:
+      ref: ${{ needs.get_info.outputs.version }}
+      type: 'pullrequest'

hack/deploy-helm.sh

@@ -53,7 +53,7 @@ if [ "${E2E_TYPE}" == "pullrequest" ]; then
     echo "Error: No .tgz file found in the charts directory."
     exit 1
   fi
-elif [ "${E2E_TYPE}" == "release" ]; then
+elif [ "${E2E_TYPE}" == "release" ] || [ "${E2E_TYPE}" == "upgrade" ]; then
   HELM_SOURCE="${HELM_NAME}/${HAMI_ALIAS}"
   echo "Using remote chart: ${HELM_SOURCE}"
 else
@@ -88,7 +88,7 @@ echo "Kubeconfig: ${KUBE_CONF}"
 if ! helm --debug upgrade --install --create-namespace --cleanup-on-fail \
   "${HAMI_ALIAS}" "${HELM_SOURCE}" -n "${TARGET_NS}" \
   --set devicePlugin.passDeviceSpecsEnabled=false \
-  --version "${HELM_VER}" --wait --timeout 10m --kubeconfig "${KUBE_CONF}"; then
+  --version "${HELM_VER}" --wait --timeout 15m --kubeconfig "${KUBE_CONF}"; then
   echo "Error: Failed to deploy/upgrade Helm Chart. Please check the Helm logs above for more details."
   exit 1
 fi
@@ -102,4 +102,4 @@ if ! util::check_pods_status "${KUBE_CONF}" ; then
   exit 1
 fi
 
-echo "HAMI Helm Chart deployed successfully."
+echo "HAMI Helm Chart deployed successfully."