Implement Maven Central mirroring in CI environments

Author: shunping

buildSrc/build.gradle.kts

@@ -15,6 +15,18 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+import java.util.Properties
+
+val parentProperties = Properties().apply {
+  val file = file("../gradle.properties")
+  if (file.exists()) {
+    file.inputStream().use { load(it) }
+  }
+}
+
+val mavenCentralMirrorUrl = parentProperties.getProperty("mavenCentralMirrorUrl")
+val isCi = System.getenv("GITHUB_ACTIONS") != null || System.getenv("JENKINS_HOME") != null
+val useMirror = isCi && !mavenCentralMirrorUrl.isNullOrBlank()
 
 // Plugins for configuring _this build_ of the module
 plugins {
@@ -25,6 +37,13 @@ plugins {
 
 // Define the set of repositories required to fetch and enable plugins.
 repositories {
+  if (useMirror) {
+    logger.lifecycle("Running in CI. Mirroring Maven Central repositories via Google Maven Mirror for buildSrc.")
+  }
+
+  if (useMirror) {
+    maven { url = uri(mavenCentralMirrorUrl!!) }
+  }
   maven { url = uri("https://plugins.gradle.org/m2/") }
   maven {
     url = uri("https://repo.spring.io/plugins-release/")

buildSrc/settings.gradle.kts

@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.util.Properties
+
+pluginManagement {
+    val parentProperties = java.util.Properties().apply {
+        val file = file("../gradle.properties")
+        if (file.exists()) {
+            file.inputStream().use { load(it) }
+        }
+    }
+
+    val mavenCentralMirrorUrl = parentProperties.getProperty("mavenCentralMirrorUrl")
+    val isCi = System.getenv("GITHUB_ACTIONS") != null || System.getenv("JENKINS_HOME") != null
+    val useMirror = isCi && !mavenCentralMirrorUrl.isNullOrBlank()
+
+    if (useMirror) {
+        logger.lifecycle("Running in CI. Mirroring Maven Central repositories via Google Maven Mirror for buildSrc plugins.")
+    }
+
+    repositories {
+        if (useMirror) {
+            maven { url = uri(mavenCentralMirrorUrl!!) }
+        }
+        gradlePluginPortal()
+    }
+}

buildSrc/src/main/groovy/org/apache/beam/gradle/Repositories.groovy

@@ -19,10 +19,16 @@
 package org.apache.beam.gradle
 
 import org.gradle.api.Project
-
 class Repositories {
 
   static void register(Project project) {
+    def mirrorUrl = project.findProperty("mavenCentralMirrorUrl")
+    boolean isCi = System.getenv("GITHUB_ACTIONS") != null || System.getenv("JENKINS_HOME") != null
+    def useMirror = isCi && mirrorUrl
+
+    if (useMirror) {
+      project.logger.lifecycle("Running in CI. Mirroring Maven Central repositories via Google Maven Mirror.")
+    }
 
     project.repositories {
       maven { url project.offlineRepositoryRoot }
@@ -36,7 +42,11 @@ class Repositories {
         return
       }
 
-      mavenCentral()
+      if (!useMirror) {
+        mavenCentral()
+      } else {
+        maven { url mirrorUrl }
+      }
       mavenLocal()
 
       // For Confluent Kafka dependencies
@@ -60,15 +70,19 @@ class Repositories {
     // Apply a plugin which provides the 'updateOfflineRepository' task that creates an offline
     // repository. This offline repository satisfies all Gradle build dependencies and Java
     // project dependencies. The offline repository is placed within $rootDir/offline-repo
-    // but can be overridden by specifying '-PofflineRepositoryRoot=/path/to/repo'.
+    // but can be overridden by specifying '-Pareas/offline-repo'.
     // Note that parallel build must be disabled when executing 'updateOfflineRepository'
     // by specifying '--no-parallel', see
     // https://github.com/mdietrichstein/gradle-offline-dependencies-plugin/issues/3
     project.apply plugin: "io.pry.gradle.offline_dependencies"
     project.offlineDependencies {
       repositories {
         mavenLocal()
-        mavenCentral()
+        if (!useMirror) {
+          mavenCentral()
+        } else {
+          maven { url mirrorUrl }
+        }
         maven { url "https://plugins.gradle.org/m2/" }
         maven { url "https://repo.spring.io/plugins-release" }
         maven { url "https://packages.confluent.io/maven/" }

gradle.properties

@@ -44,3 +44,6 @@ flink_versions=1.17,1.18,1.19,1.20,2.0
 spark_versions=3,4
 # supported python versions
 python_versions=3.10,3.11,3.12,3.13,3.14
+
+# Maven Central fallback mirror URL
+mavenCentralMirrorUrl=https://maven-central.storage-download.googleapis.com/maven2/

settings.gradle.kts

@@ -18,6 +18,20 @@
 import com.gradle.enterprise.gradleplugin.internal.extension.BuildScanExtensionWithHiddenFeatures
 
 pluginManagement {
+    val mavenCentralMirrorUrl = settings.providers.gradleProperty("mavenCentralMirrorUrl").orNull
+    val isCi = System.getenv("GITHUB_ACTIONS") != null || System.getenv("JENKINS_HOME") != null
+    val useMirror = isCi && !mavenCentralMirrorUrl.isNullOrBlank()
+
+    if (useMirror) {
+        logger.lifecycle("Running in CI. Mirroring Maven Central repositories via Google Maven Mirror.")
+    }
+
+    repositories {
+        if (useMirror) {
+            maven { url = uri(mavenCentralMirrorUrl!!) }
+        }
+        gradlePluginPortal()
+    }
     plugins {
         id("org.javacc.javacc") version "4.0.3" // enable the JavaCC parser generator
     }