Merge pull request #253 from UIUCLibrary/fix-ci

Fix ci

Author: henryborchers

Jenkinsfile

@@ -92,7 +92,7 @@ RUN --mount=type=cache,target=${PIP_DOWNLOAD_CACHE} \
     fi && \
     uv run --only-group conan conan install /tmp -pr:b=default --build missing -pr:b=default && \
     uv run --only-group conan conan cache clean "*" -b --source --build --temp && \
-    if [ "$(jq -r '.remotes[0].url' ${CONAN_HOME}/remotes.json )" != "${CONAN_CENTER_PROXY_V2_URL}" ]; then \
+    if [ -f "/tmp/remotes.json" ]; then \
       mv -f /tmp/remotes.json ${CONAN_HOME}/remotes.json; \
     fi && \
     mv -f /tmp/global.conf.original ${CONAN_HOME}/global.conf

ci/docker/linux/jenkins/Dockerfile

@@ -79,7 +79,7 @@ RUN --mount=type=cache,target=${PIP_DOWNLOAD_CACHE} \
     uv run --only-group conan conan install /tmp -pr:b=default --build missing && \
     uv run --only-group conan conan cache clean "*" -b --source --build --temp && \
     rm -rf venv && \
-    if [ "$(jq -r '.remotes[0].url' ${CONAN_HOME}/remotes.json )" != "${CONAN_CENTER_PROXY_V2_URL}" ]; then \
+    if [ -f "/tmp/remotes.json" ]; then \
         mv -f /tmp/remotes.json ${CONAN_HOME}/remotes.json; \
     fi && \
     mv -f /tmp/global.conf.original ${CONAN_HOME}/global.conf

ci/docker/linux/tox/Dockerfile

@@ -59,7 +59,13 @@ generate_wheel(){
             exit 1
     esac
 
+    dockerPurposeLabel='build-wheel'
+    if [[ -v ci ]]; then
+        dockerPurposeLabel='ci'
+    fi
+
     docker build \
+        --label="purpose=$dockerPurposeLabel" \
         -t $docker_image_name_to_use \
         --platform=$platform \
         -f "$DOCKERFILE" \
@@ -115,6 +121,7 @@ generate_wheel(){
             echo 'Done'
             "
     docker run --rm \
+        --label="purpose=$dockerPurposeLabel" \
         --platform=$platform \
         -v "$PROJECT_ROOT":/project:ro \
         -v $OUTPUT_PATH:/dist \

scripts/build_linux_wheels.sh

@@ -12,10 +12,14 @@ function Build-DockerImage {
         [string]$DockerExec = "docker.exe",
         [string]$DockerIsolation = "process"
     )
-
+    $dockerPurpose = "build-wheel"
+    if ($env:CI) {
+        $dockerPurpose = "ci"
+    }
     $projectRootDirectory = (Get-Item $PSScriptRoot).Parent.FullName
     $dockerArgsList = @(
         "build",
+        "--label=purpose=$dockerPurpose",
         "--isolation", $DockerIsolation,
         "--platform windows/amd64",
         "-f", $DockerfilePath,
@@ -55,8 +59,13 @@ function Build-Wheel {
     $UV_TOOL_DIR = "${containerCacheDir}\uvtools"
     $UV_PYTHON_CACHE_DIR = "${containerCacheDir}\uvpython"
 
+    $dockerPurpose = "build-wheel"
+    if ($env:CI) {
+        $dockerPurpose = "ci"
+    }
     $dockerArgsList = @(
         "run",
+        "--label=purpose=$dockerPurpose",
         "--isolation", $DockerIsolation,
         "--platform windows/amd64",
         "--rm",

scripts/build_windows.ps1

@@ -11,7 +11,7 @@ ARG CONAN_CENTER_PROXY_V2_URL=https://center2.conan.io
 # If you want to use a diffrent remote for Conan, such as a proxy. Set the CONAN_CENTER_PROXY_V2_URL
 # Not this is only for building the image. The actual conan center proxy URL is set in the remotes.json file.
 
-ARG manylinux_image=quay.io/pypa/manylinux2014_x86_64
+ARG manylinux_image=quay.io/pypa/manylinux2014
 # Base image to use, this should be of the manylinux family. More info at: https://github.com/pypa/manylinux
 
 ARG UV_CACHE_DIR=/.cache/uv
@@ -33,7 +33,21 @@ RUN --mount=type=cache,target=/path/to/pipx/cache \
 # ==============================================================================
 FROM base AS conan_builder
 RUN --mount=type=cache,target=/var/cache/yum \
-    yum install -y jq
+    case $(uname -m) in \
+      x86_64)  yum install -y jq ;; \
+      aarch64) yum install -y wget gawk; \
+               DOWNLOAD_URL='https://github.com/jqlang/jq/releases/download/jq-1.7.1/jq-linux-arm64' && \
+               EXPECTED_HASH='4dd2d8a0661df0b22f1bb9a1f9830f06b6f3b8f7d91211a1ef5d7c4f06a8b4a5' && \
+               wget $DOWNLOAD_URL -P /tmp && \
+               CALCULATED_HASH=$(sha256sum /tmp/jq-linux-arm64 | awk '{print $1}') && \
+               if [ "$CALCULATED_HASH" != "$EXPECTED_HASH" ]; then \
+                echo "Hash does not match. Expected: $EXPECTED_HASH. Got: $CALCULATED_HASH." && exit 1; \
+               fi && \
+               mv /tmp/jq-linux-arm64 /usr/local/bin/jq && \
+               chmod +x /usr/local/bin/jq ;; \
+    esac; \
+    jq --version && \
+    yum install -y perl-IPC-Cmd perl-Digest-SHA perl-Time-Piece
 
 ARG CONAN_USER_HOME
 ARG CONAN_HOME
@@ -66,7 +80,7 @@ RUN --mount=type=bind,source=pyproject.toml,target=pyproject.toml \
     uv run --only-group conan conan install /tmp --build missing && \
     uv run --only-group conan conan cache clean "*" -b --source --build --temp && \
     uv cache prune && \
-    if [ "$(jq -r '.remotes[0].url' ${CONAN_HOME}/remotes.json )" != "${CONAN_CENTER_PROXY_V2_URL}" ]; then \
+    if [ -f "/tmp/remotes.json" ]; then \
         mv -f /tmp/remotes.json ${CONAN_HOME}/remotes.json; \
     fi && \
     mv -f /tmp/global.conf.original ${CONAN_HOME}/global.conf

scripts/resources/package_for_linux/Dockerfile

@@ -73,20 +73,20 @@ COPY pyproject.toml uv.lock ${UV_PROJECT}/
 #SHELL ["cmd", "/S", "/C" ]
 ENV CONAN_CMAKE_GENERATOR=Ninja
 ARG CONAN_CENTER_PROXY_V2_URL
-RUN uv run --only-group conan conan profile detect --exist-ok ; `
+RUN uv run --only-group=conan conan profile detect --exist-ok ; `
     if (${env:CONAN_CENTER_PROXY_V2_URL} -ne $(Get-Content -Raw -Path ${Env:CONAN_HOME}\remotes.json)) { `
           Copy-Item -Path "${Env:CONAN_HOME}\remotes.json" -Destination "c:\remotes.json" ; `
-          uv run --only-group conan conan remote update conan-center --url ${env:CONAN_CENTER_PROXY_V2_URL}; `
+          uv run --only-group=conan conan remote update conan-center --url ${env:CONAN_CENTER_PROXY_V2_URL}; `
     }; `
-    uv run --only-group conan conan install c:/temp/ -pr:h=default -pr:b=python -of c:/temp/delme; `
+    uv run --only-group=conan conan install c:/temp/ -pr:h=default -pr:b=python -of c:/temp/delme -c tools.graph:skip_test=True; `
     if ($LASTEXITCODE -ne 0) { `
       throw \"Command 'conan install' failed with exit code: $LASTEXITCODE\"`
     } ;`
     Remove-Item -Path "c:\temp\delme" -Recurse -Force ; `
-    uv run --only-group conan conan cache clean "*" -b --source --build --temp ; `
+    uv run --only-group=conan conan cache clean "*" -b --source --build --temp -vdebug; `
     Remove-Item -Path "${env:UV_PROJECT}\.venv" -Recurse -Force ; `
     uv cache clean --no-progress ; `
-    if (${env:CONAN_CENTER_PROXY_V2_URL} -ne $(Get-Content -Raw -Path ${Env:CONAN_HOME}\remotes.json)) { `
+    if (Test-Path -Path "c:\remotes.json" ) { `
       Move-Item -Path "c:\remotes.json" -Destination "${Env:CONAN_HOME}\remotes.json" -Force ;`
     }
 
@@ -114,5 +114,3 @@ WORKDIR C:/src
 ENV DISTUTILS_USE_SDK=1
 
 COPY ["scripts/resources/windows/tox/powershell_modules", "c:/Program Files/WindowsPowerShell/Modules/"]
-# To help mark the image as a CI image so it can be cleaned up more easily
-LABEL purpose=ci

scripts/resources/windows/tox/Dockerfile

@@ -239,7 +239,25 @@ function InstallMSVC{
     }
     Write-Host "Cleaning up Package Cache - Done"
 
-
+    # ======== Wait for Windows Update to finish ========
+    # It seems that installing MSVC build tools triggers Windows
+    # update. This generates a lot of extra unneeded files for
+    # the docker layer and bloats the image significantly.
+    # Waiting for the update to finish and then cleaning up the
+    # generated files seems to be the best solution to keep the
+    # layor size down.
+
+    $timeout = New-TimeSpan -Hours 1
+    $stopwatch = [System.Diagnostics.Stopwatch]::StartNew()
+    $installer = New-Object -ComObject Microsoft.Update.Installer
+    while ($installer.IsBusy) {
+        Write-Host "Windows Update is currently busy installing updates. Waiting..."
+        if ($stopwatch.Elapsed -gt $timeout) {
+            $stopwatch.Stop()
+            throw "Timeout exceeded: Operation took longer than 1 hour."
+        }
+        Start-Sleep -Seconds 30
+    }
     # ======== Cleanup SoftwareDistribution folder ========
     $SoftwareDistributionDownloadPath = "C:\Windows\SoftwareDistribution\Download"
     if (Test-Path "$SoftwareDistributionDownloadPath"){