Only pull license for selected tests and publishing container (#37827)

Author: Abacn

.github/actions/gradle-command-self-hosted-action/action.yml

@@ -42,5 +42,5 @@ runs:
           rm ~/.m2/settings.xml
         fi
         ./gradlew ${{ inputs.gradle-command }} --max-workers=${{ inputs.max-workers }} --continue --no-daemon \
-         -Dorg.gradle.jvmargs=-Xms2g -Dorg.gradle.jvmargs=-Xmx6g -Dorg.gradle.vfs.watch=false -Pdocker-pull-licenses \
+         -Dorg.gradle.jvmargs=-Xms2g -Dorg.gradle.jvmargs=-Xmx6g -Dorg.gradle.vfs.watch=false \
         ${{ inputs.arguments }}

.github/workflows/beam_PreCommit_GoPortable.yml

@@ -88,4 +88,5 @@ jobs:
       - name: Run goPortablePreCommit script
         uses: ./.github/actions/gradle-command-self-hosted-action
         with:
-          gradle-command: :goPortablePreCommit
+          gradle-command: :goPortablePreCommit
+          arguments: -Pdocker-pull-licenses

.github/workflows/beam_PreCommit_Java_Examples_Dataflow.yml

@@ -105,6 +105,7 @@ jobs:
           arguments: |
             -PdisableSpotlessCheck=true \
             -PdisableCheckStyle=true \
+            -Pdocker-pull-licenses \
       - name: Archive JUnit Test Results
         uses: actions/upload-artifact@v4
         if: ${{ !success() }}

.github/workflows/beam_PreCommit_Java_Examples_Dataflow_Java21.yml

@@ -119,6 +119,7 @@ jobs:
           -PtestJavaVersion=21 \
           -PskipCheckerFramework \
           -Pjava21Home=$JAVA_HOME_21_X64 \
+          -Pdocker-pull-licenses \
         max-workers: 12
     - name: Archive JUnit Test Results
       uses: actions/upload-artifact@v4

.github/workflows/beam_PreCommit_PythonDocker.yml

@@ -102,4 +102,5 @@ jobs:
           gradle-command: :sdks:python:container:py${{steps.set_py_ver_clean.outputs.py_ver_clean}}:docker
           arguments: |
             -Pposargs=apache_beam/dataframe/ \
-            -PpythonVersion=${{ matrix.python_version }}
+            -PpythonVersion=${{ matrix.python_version }} \
+            -Pdocker-pull-licenses

.github/workflows/beam_Publish_Beam_SDK_Snapshots.yml

@@ -127,3 +127,4 @@ jobs:
             -Pdocker-tag-list=${{ github.sha }},${BEAM_VERSION}${LATEST_TAG} \
             -Pcontainer-architecture-list=arm64,amd64 \
             -Ppush-containers \
+            -Pdocker-pull-licenses

.github/workflows/beam_Publish_Docker_Snapshots.yml

@@ -82,11 +82,13 @@ jobs:
           gradle-command: :runners:spark:3:job-server:container:dockerPush
           arguments: |
             -Pdocker-repository-root=gcr.io/apache-beam-testing/beam_portability \
-            -Pdocker-tag-list=${{ github.sha }}${LATEST_TAG}
+            -Pdocker-tag-list=${{ github.sha }}${LATEST_TAG} \
+            -Pdocker-pull-licenses
       - name: run Publish Docker Snapshots script for Flink 1.17
         uses: ./.github/actions/gradle-command-self-hosted-action
         with:
           gradle-command: :runners:flink:1.17:job-server-container:dockerPush
           arguments: |
             -Pdocker-repository-root=gcr.io/apache-beam-testing/beam_portability \
-            -Pdocker-tag-list=${{ github.sha }}${LATEST_TAG}
+            -Pdocker-tag-list=${{ github.sha }}${LATEST_TAG} \
+            -Pdocker-pull-licenses

sdks/python/container/license_scripts/pull_licenses_py.py

@@ -75,7 +75,7 @@ def copy_license_files(dep):
 @retry(
     reraise=True,
     wait=wait_exponential(multiplier=2),
-    stop=stop_after_attempt(10))
+    stop=stop_after_attempt(8))
 def pull_from_url(dep, configs):
   '''
   :param dep: name of a dependency