fix: Fix CVE-2023-44487 in mmlspark/release Docker image (#2520)

- Upgrade base from Ubuntu 20.04 to 22.04 (MCR mirror)
- Bump Spark from 3.5.0 to 3.5.4 (eliminates vulnerable jetty-io)
- Update conda installer URL and add TOS acceptance
- Add -y flag to conda update for non-interactive builds
- Fix PATH to match conda install prefix (/usr/local)
- Upgrade PyJWT >= 2.12.0 (CVE-2026-32597)
- Fix minimal Dockerfile with same security updates
- Restructure pipeline: build always, push only on release

Resolves IcM incident 31000000570827 / MSRC case 110886

Author: BrendanWalsh

pipeline.yaml

@@ -243,9 +243,8 @@ jobs:
         failTaskOnFailedTests: true
       condition: and(eq(variables.runTests, 'True'), succeededOrFailed())
 #
-- job: PublishDocker
-  condition: eq('${{ parameters.publishDockerImages }}', true)
-  displayName: PublishDocker
+- job: BuildDocker
+  displayName: BuildDocker
   pool:
     vmImage: ubuntu-22.04
   steps:
@@ -259,55 +258,53 @@ jobs:
           VERSION=$(sbt "core/version" | tail -1 |  cut -d' ' -f2 | sed 's/\x1b\[[0-9;]*m//g')
           echo '##vso[task.setvariable variable=version]'$VERSION
           echo '##vso[task.setvariable variable=gittag]'$(git tag -l --points-at HEAD)
+    # Build all images (runs on every build to validate Dockerfiles)
     - task: Docker@2
       displayName: Demo Image Build
       inputs:
-        containerRegistry: 'SynapseML MCR MSI'
-        repository: 'public/mmlspark/build-demo'
         command: 'build'
         buildContext: "."
         Dockerfile: 'tools/docker/demo/Dockerfile'
-        tags: $(version)
-        arguments: --build-arg SYNAPSEML_VERSION=$(version)
-    - task: Docker@2
-      displayName: Demo Image Push
-      inputs:
-        containerRegistry: 'SynapseML MCR MSI'
-        repository: 'public/mmlspark/build-demo'
-        command: 'push'
-        tags: $(version)
+        arguments: --build-arg SYNAPSEML_VERSION=$(version) -t mmlspark-demo:$(version)
     - task: Docker@2
       displayName: Minimal Image Build
       inputs:
-        containerRegistry: 'SynapseML MCR MSI'
-        repository: 'public/mmlspark/build-minimal'
         command: 'build'
         buildContext: "."
         Dockerfile: 'tools/docker/minimal/Dockerfile'
-        tags: $(version)
-        arguments: --build-arg SYNAPSEML_VERSION=$(version)
+        arguments: --build-arg SYNAPSEML_VERSION=$(version) -t mmlspark-minimal:$(version)
+    # Push demo and minimal on every master build
+    - bash: |
+        set -e
+        docker tag mmlspark-demo:$(version) mmlsparkmcr.azurecr.io/public/mmlspark/build-demo:$(version)
+        docker tag mmlspark-minimal:$(version) mmlsparkmcr.azurecr.io/public/mmlspark/build-minimal:$(version)
+      condition: and(eq(variables.isMaster, true), eq('${{ parameters.publishDockerImages }}', true))
+      displayName: Tag Dev Images for ACR
     - task: Docker@2
-      displayName: Minimal Image Push
+      condition: and(eq(variables.isMaster, true), eq('${{ parameters.publishDockerImages }}', true))
+      displayName: Demo Image Push
       inputs:
         containerRegistry: 'SynapseML MCR MSI'
-        repository: 'public/mmlspark/build-minimal'
+        repository: 'public/mmlspark/build-demo'
         command: 'push'
         tags: $(version)
     - task: Docker@2
-      condition: and(eq(variables.isMaster, true), startsWith(variables['gittag'], 'v'))
-      displayName: Release Image Build
+      condition: and(eq(variables.isMaster, true), eq('${{ parameters.publishDockerImages }}', true))
+      displayName: Minimal Image Push
       inputs:
         containerRegistry: 'SynapseML MCR MSI'
-        repository: 'public/mmlspark/release'
-        command: 'build'
-        buildContext: "."
-        Dockerfile: 'tools/docker/demo/Dockerfile'
-        tags: |
-          $(version)
-          latest
-        arguments: --build-arg SYNAPSEML_VERSION=$(version)
+        repository: 'public/mmlspark/build-minimal'
+        command: 'push'
+        tags: $(version)
+    # Push release only on tagged releases
+    - bash: |
+        set -e
+        docker tag mmlspark-demo:$(version) mmlsparkmcr.azurecr.io/public/mmlspark/release:$(version)
+        docker tag mmlspark-demo:$(version) mmlsparkmcr.azurecr.io/public/mmlspark/release:latest
+      condition: and(eq('${{ parameters.publishDockerImages }}', true), eq(variables.isMaster, true), startsWith(variables['gittag'], 'v'))
+      displayName: Tag Release Image for ACR
     - task: Docker@2
-      condition: and(eq(variables.isMaster, true), startsWith(variables['gittag'], 'v'))
+      condition: and(eq('${{ parameters.publishDockerImages }}', true), eq(variables.isMaster, true), startsWith(variables['gittag'], 'v'))
       displayName: Release Image Push
       inputs:
         containerRegistry: 'SynapseML MCR MSI'

tools/docker/demo/Dockerfile

@@ -1,9 +1,9 @@
-FROM mcr.microsoft.com/oss/mirror/docker.io/library/ubuntu:20.04
+FROM mcr.microsoft.com/mirror/docker/library/ubuntu:22.04
 
 ARG SYNAPSEML_VERSION=1.1.2
 ARG DEBIAN_FRONTEND=noninteractive
 
-ENV SPARK_VERSION=3.5.0
+ENV SPARK_VERSION=3.5.4
 ENV HADOOP_VERSION=3
 ENV SYNAPSEML_VERSION=${SYNAPSEML_VERSION}
 ENV JAVA_HOME /usr/lib/jvm/java-1.11.0-openjdk-amd64
@@ -23,14 +23,18 @@ RUN apt-get -qq update \
     && rm -rf /var/lib/apt/lists/* /var/log/dpkg.log
 
 # Install conda and required dependencies
-RUN curl -sSL https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -o /tmp/miniconda.sh \
+RUN curl -sSL https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -o /tmp/miniconda.sh \
     && bash /tmp/miniconda.sh -bfp /usr/local \
     && rm -rf /tmp/miniconda.sh \
-    && conda update conda \
+    && conda config --set auto_activate_base false \
+    && conda tos accept --override-channels --channel https://repo.anaconda.com/pkgs/main \
+    && conda tos accept --override-channels --channel https://repo.anaconda.com/pkgs/r \
+    && conda update -y conda \
     && conda install -y python=3 jupyter pyspark \
+    && pip install --upgrade "PyJWT>=2.12.0" \
     && conda clean --all --yes
 
-ENV PATH /opt/conda/bin:$PATH
+ENV PATH /usr/local/bin:$PATH
 
 # Download and install Spark
 RUN wget https://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz \

tools/docker/minimal/Dockerfile

@@ -1,9 +1,9 @@
-FROM mcr.microsoft.com/oss/mirror/docker.io/library/ubuntu:20.04
+FROM mcr.microsoft.com/mirror/docker/library/ubuntu:22.04
 
 ARG SYNAPSEML_VERSION=1.1.2
 ARG DEBIAN_FRONTEND=noninteractive
 
-ENV SPARK_VERSION=3.5.0
+ENV SPARK_VERSION=3.5.4
 ENV HADOOP_VERSION=3
 ENV SYNAPSEML_VERSION=${SYNAPSEML_VERSION}
 ENV JAVA_HOME /usr/lib/jvm/java-1.11.0-openjdk-amd64
@@ -23,14 +23,18 @@ RUN apt-get -qq update \
     && rm -rf /var/lib/apt/lists/* /var/log/dpkg.log
 
 # Install conda and required dependencies
-RUN curl -sSL https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -o /tmp/miniconda.sh \
+RUN curl -sSL https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -o /tmp/miniconda.sh \
     && bash /tmp/miniconda.sh -bfp /usr/local \
     && rm -rf /tmp/miniconda.sh \
-    && conda update conda \
-    && conda install -y python=3 jupyter pyspark \
+    && conda config --set auto_activate_base false \
+    && conda tos accept --override-channels --channel https://repo.anaconda.com/pkgs/main \
+    && conda tos accept --override-channels --channel https://repo.anaconda.com/pkgs/r \
+    && conda update -y conda \
+    && conda install -y python=3 pyspark \
+    && pip install --upgrade "PyJWT>=2.12.0" \
     && conda clean --all --yes
 
-ENV PATH /opt/conda/bin:$PATH
+ENV PATH /usr/local/bin:$PATH
 
 # Download and install Spark
 RUN wget https://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz \