Fix environment variables (#21)

Author: Ariouz

.github/scripts/extract_matrix.py

@@ -110,6 +110,8 @@ def system_packages(self) -> list[str]:
         # TODO: support more packages
         system_packages = []
         for k, _ in self.job.get("packages", {}).items():
+            if k in ["mx", "python3"]:
+                continue
             if k.startswith("pip:"):
                 continue
             elif k.startswith("00:") or k.startswith("01:"):

.github/workflows/ci-matrix-gen.yml

@@ -13,9 +13,6 @@ on:
         required: true
         type: string
 
-  schedule:
-    - cron: '0 0 * * 1'
-
 jobs:
   generate-tier1:
     runs-on: ubuntu-latest
@@ -58,13 +55,36 @@ jobs:
   tier1:
     needs: generate-tier1
     runs-on: ${{ matrix.os }}
-    env: ${{ matrix.env }}
+    #env: ${{ matrix.env }}
     strategy:
       fail-fast: false
       matrix:
         include: ${{ fromJson(needs.generate-tier1.outputs.matrix) }}
     steps: &buildsteps
 
+      - name: Setup env
+        shell: bash
+        if: runner.os != 'Windows'
+        run: |
+          matrix_env='${{ toJson(matrix.env) }}'
+          echo $matrix_env | jq -r 'to_entries[] | "\(.key)=\(.value)"' >> tmp.env
+          while IFS= read -r line; do
+            eval "echo \"$line\"" >> $GITHUB_ENV
+          done < tmp.env
+          rm tmp.env
+
+      - name: Setup env (Windows)
+        if: runner.os == 'Windows'
+        shell: pwsh
+        run: |
+          $matrix_env = '${{ toJson(matrix.env) }}'
+          $envs = $matrix_env | ConvertFrom-Json
+          foreach ($pair in $envs.PSObject.Properties) {
+            $full_value = $pair.Value -replace '%([A-Za-z0-9_]+)%', '$env:$1'
+            $value = $ExecutionContext.InvokeCommand.ExpandString($full_value)
+            "$($pair.Name)=$value" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8
+          }
+
       - name: Actions/Checkout
         uses: actions/checkout@main
         with:
@@ -250,7 +270,7 @@ jobs:
     if: ${{ success() || inputs.jobs_to_run }}
     needs: [generate-tier2, tier1]
     runs-on: ${{ matrix.os }}
-    env: ${{ matrix.env }}
+    #env: ${{ matrix.env }}
     strategy:
       fail-fast: false
       matrix:
@@ -261,7 +281,7 @@ jobs:
     if: ${{ success() || inputs.jobs_to_run }}
     needs: [generate-tier3, tier2]
     runs-on: ${{ matrix.os }}
-    env: ${{ matrix.env }}
+    #env: ${{ matrix.env }}
     strategy:
       fail-fast: false
       matrix:

.github/workflows/ci-unittest-retagger.yml

@@ -19,14 +19,18 @@ jobs:
   run-retagger:
     if: success()
     needs: build-standalone-artifacts
+    strategy:
+      fail-fast: false
+      matrix:
+        target: [linux-amd64, linux-aarch64, windows-amd64]
     uses: ./.github/workflows/ci-matrix-gen.yml
+    name: Run retagger on ${{ matrix.target }}
     with:
-      jobs_to_run: ${{ github.event.inputs.jobs_to_run || 'python-unittest-retagger' }}
+      jobs_to_run: ${{ github.event.inputs.jobs_to_run || format('^python-unittest-retagger.*{0}.*', matrix.target) }}
 
   merge_all_reports:
     runs-on: ubuntu-latest
     needs: run-retagger
-    if: ${{ success() }}
     env:
       GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     steps: 
@@ -59,7 +63,7 @@ jobs:
         working-directory: main
         run: |
           mkdir -p ../retagger-reports ../diffs
-          mv retagger-report*.json ../retagger-reports
+          mv retagger-report*.json ../retagger-reports || true
 
           declare -a os_list=("linux-x86_64" "linux-aarch64" "win32-AMD64")
           for os in "${os_list[@]}"; do

ci/python-gate.libsonnet

@@ -151,14 +151,14 @@
         darwin: {
             common: ENV_POSIX + {
                 LC_CTYPE: "en_US.UTF-8",
-                PATH: utils.path_combine(ENVIRONMENT.common.PATH, "$PYTHON3_HOME:$MUSL_TOOLCHAIN/bin:/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin:/opt/homebrew/bin:/opt/homebrew/sbin"),
+                PATH: utils.path_combine(ENVIRONMENT.common.PATH, "$PATH:$PYTHON3_HOME:$MUSL_TOOLCHAIN/bin"),
             },
             amd64: {},
             aarch64: {},
         },
         windows: {
             common: {
-                PATH: "$PATH;C:\\ProgramData\\chocolatey\\bin;C:\\hostedtoolcache\\windows\\Python\\3.12.3\\x64\\Scripts;C:\\hostedtoolcache\\windows\\Python\\3.12.3\\x64;C:\\Program Files\\PowerShell\\7;C:\\windows\\system32;C:\\windows;C:\\windows\\System32\\Wbem;C:\\windows\\System32\\WindowsPowerShell\\v1.0\\;C:\\windows\\System32\\OpenSSH\\;C:\\Program Files\\dotnet\\;C:\\Program Files\\Microsoft SQL Server\\150\\Tools\\Binn\\;C:\\Program Files\\Git\\cmd;C:\\Program Files\\Git\\usr\\bin;C:\\hostedtoolcache\\windows\\Ruby\\3.2.3\\x64\\bin;C:\\hostedtoolcache\\windows\\Go\\1.22.2\\x64\\bin;C:\\hostedtoolcache\\windows\\Node\\20.11.1\\x64\\;C:\\npm\\prefix;C:\\Program Files\\Amazon\\AWSCLIV2\\;C:\\Program Files\\Amazon\\SessionManagerPlugin\\bin\\;C:\\Program Files\\LLVM\\bin;C:\\Program Files\\CMake\\bin;C:\\Program Files\\OpenSSL\\bin;C:\\msys64\\usr\\bin;C:\\hostedtoolcache\\windows\\php\\8.2.15\\x64;C:\\Program Files\\Azure DevOps CLI\\bin;C:\\hostedtoolcache\\windows\\R\\4.3.3\\x64\\bin;C:\\Program Files (x86)\\Microsoft Visual Studio\\2022\\Enterprise\\Common7\\IDE\\Extensions\\Microsoft\\NuGet;",
+                PATH: "%MAVEN_HOME%;%PATH%",
             },
             amd64: {},
             aarch64: {},
@@ -170,7 +170,7 @@
         LD_LIBRARY_PATH: "$LIBGMP/lib:$LLVM/lib:$LD_LIBRARY_PATH",
         FORK_COUNTS_DIRECTORY: "$BUILD_DIR/benchmarking-config/fork-counts",
         RODINIA_DATASET_ZIP: $.overlay_imports.RODINIA_DATASET_ZIP,
-        PATH: utils.path_combine(ENVIRONMENT.common.PATH, "$PYTHON3_HOME:$MUSL_TOOLCHAIN/bin:/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin"),
+        PATH: utils.path_combine(ENVIRONMENT.common.PATH, "$PATH:$PYTHON3_HOME:$MUSL_TOOLCHAIN/bin"),
     },
 
     // This is the diff to 'ENVIRONMENT' and meant to be used on OL8 images.