feat: add warnings for public repository downloads in multiple SDKs (#36476)

* feat: add warnings for public repository downloads in multiple SDKs

add  warnings when downloading dependencies from public repositories in Java, Go, and Python SDKs
warn users about potential risks and suggest pre-staging dependencies or using private mirrors

* logger

* added cached_jar_path

Author: liferoad

sdks/go/pkg/beam/core/runtime/xlangx/expansionx/download.go

@@ -21,6 +21,7 @@ import (
 	"archive/zip"
 	"fmt"
 	"io"
+	"log"
 	"net/http"
 	"os"
 	"os/exec"
@@ -106,6 +107,15 @@ func getLocalJar(url string) (string, error) {
 		return jarPath, nil
 	}
 
+	// Issue warning when downloading from public repositories
+	if strings.Contains(url, "repo.maven.apache.org") ||
+		strings.Contains(url, "repo1.maven.org") ||
+		strings.Contains(url, "maven.google.com") ||
+		strings.Contains(url, "maven-central.storage-download.googleapis.com") {
+		log.Printf("WARNING: Downloading JAR file from public repository: %s. "+
+			"This may pose security risks or cause instability due to repository availability. Consider pre-staging dependencies or using private mirrors.", url)
+	}
+
 	resp, err := http.Get(string(url))
 	if err != nil {
 		return "", err
@@ -334,6 +344,16 @@ func (j *jarGetter) getJar(gradleTarget, version string) (string, error) {
 			gradleTarget)
 	}
 
+	// Issue warning when downloading from public repositories
+	fullURLStr := string(fullURL)
+	if strings.Contains(fullURLStr, "repo.maven.apache.org") ||
+		strings.Contains(fullURLStr, "repo1.maven.org") ||
+		strings.Contains(fullURLStr, "maven.google.com") ||
+		strings.Contains(fullURLStr, "maven-central.storage-download.googleapis.com") {
+		log.Printf("WARNING: Downloading JAR file from public repository: %s. "+
+			"This may pose security risks or cause instability due to repository availability. Consider pre-staging dependencies or using private mirrors.", fullURLStr)
+	}
+
 	resp, err := http.Get(string(fullURL))
 	if err != nil {
 		return "", err

sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/JavaUdfLoader.java

@@ -125,6 +125,20 @@ public AggregateFn loadAggregateFunction(List<String> functionPath, String jarPa
    */
   private File downloadFile(String inputPath, String mimeType) throws IOException {
     Preconditions.checkArgument(!inputPath.isEmpty(), "Path cannot be empty.");
+
+    // Issue warning when downloading from public repositories
+    if (inputPath.startsWith("http://") || inputPath.startsWith("https://")) {
+      if (inputPath.contains("repo.maven.apache.org")
+          || inputPath.contains("repo1.maven.org")
+          || inputPath.contains("maven.google.com")
+          || inputPath.contains("maven-central.storage-download.googleapis.com")) {
+        LOG.warn(
+            "WARNING: Downloading JAR file from public repository: {}. "
+                + "This may pose security risks or cause instability due to repository availability. Consider pre-staging dependencies or using private mirrors.",
+            inputPath);
+      }
+    }
+
     ResourceId inputResource = FileSystems.matchNewResource(inputPath, false /* is directory */);
     try (ReadableByteChannel inputChannel = FileSystems.open(inputResource)) {
       File outputFile = File.createTempFile("sql-udf-", inputResource.getFilename());

sdks/python/apache_beam/utils/subprocess_server.py

@@ -280,8 +280,10 @@ def _really_stop_process(process_and_endpoint):
 class JavaJarServer(SubprocessServer):
 
   MAVEN_CENTRAL_REPOSITORY = 'https://repo.maven.apache.org/maven2'
-  MAVEN_STAGING_REPOSITORY = 'https://repository.apache.org/content/groups/staging'  # pylint: disable=line-too-long
-  GOOGLE_MAVEN_MIRROR = 'https://maven-central.storage-download.googleapis.com/maven2'  # pylint: disable=line-too-long
+  MAVEN_STAGING_REPOSITORY = (
+      'https://repository.apache.org/content/groups/staging')
+  GOOGLE_MAVEN_MIRROR = (
+      'https://maven-central.storage-download.googleapis.com/maven2')
   BEAM_GROUP_ID = 'org.apache.beam'
   JAR_CACHE = os.path.expanduser("~/.apache_beam/cache/jars")
 
@@ -431,6 +433,26 @@ def _download_jar_to_cache(
       cached_jar_path (str): The local path where the jar should be cached.
       user_agent (str): The user agent to use when downloading.
     """
+    # Issue warning when downloading from public repositories
+    public_repos = [
+        cls.MAVEN_CENTRAL_REPOSITORY,
+        cls.GOOGLE_MAVEN_MIRROR,
+    ]
+
+    if any(download_url.startswith(repo) for repo in public_repos):
+      _LOGGER.warning(
+          "   WARNING: Apache Beam is downloading dependencies from a "
+          "public repository at runtime.\n"
+          "   This may pose security risks or cause instability due to "
+          "repository availability.\n"
+          "   URL: %s\n"
+          "   Destination: %s\n"
+          "   Consider pre-staging dependencies or using a private repository "
+          "mirror.\n"
+          "   For more information, see: "
+          "https://beam.apache.org/documentation/sdks/python-dependencies/",
+          download_url,
+          cached_jar_path)
     try:
       url_read = FileSystems.open(download_url)
     except ValueError:

sdks/python/apache_beam/yaml/yaml_provider.py

@@ -68,6 +68,8 @@
 from apache_beam.yaml import yaml_utils
 from apache_beam.yaml.yaml_errors import maybe_with_exception_handling_transform_fn
 
+_LOGGER = logging.getLogger(__name__)
+
 
 class NotAvailableWithReason:
   """A False value that provides additional content.
@@ -1322,6 +1324,18 @@ def _create_venv_from_scratch(
         venv_python = os.path.join(venv, 'bin', 'python')
         venv_pip = os.path.join(venv, 'bin', 'pip')
         subprocess.run([venv_python, '-m', 'ensurepip'], check=True)
+        # Issue warning when installing packages from PyPI
+        _LOGGER.warning(
+            " WARNING: Apache Beam is installing Python packages "
+            "from PyPI at runtime.\n"
+            "   This may pose security risks or cause instability due to "
+            "repository availability.\n"
+            "   Packages: %s\n"
+            "   Consider pre-staging dependencies or using a private "
+            "repository mirror.\n"
+            "   For more information, see: "
+            "https://beam.apache.org/documentation/sdks/python-dependencies/",
+            ', '.join(packages))
         subprocess.run([venv_pip, 'install'] + packages, check=True)
         with open(venv + '-requirements.txt', 'w') as fout:
           fout.write('\n'.join(packages))
@@ -1342,6 +1356,18 @@ def _create_venv_from_clone(
         clonable_venv = cls._create_venv_to_clone(base_python)
         clonevirtualenv.clone_virtualenv(clonable_venv, venv)
         venv_pip = os.path.join(venv, 'bin', 'pip')
+        # Issue warning when installing packages from PyPI
+        _LOGGER.warning(
+            "   WARNING: Apache Beam is installing Python packages "
+            "from PyPI at runtime.\n"
+            "   This may pose security risks or cause instability due to "
+            "repository availability.\n"
+            "   Packages: %s\n"
+            "   Consider pre-staging dependencies or using a private "
+            "repository mirror.\n"
+            "   For more information, see: "
+            "https://beam.apache.org/documentation/sdks/python-dependencies/",
+            ', '.join(packages))
         subprocess.run([venv_pip, 'install'] + packages, check=True)
         with open(venv + '-requirements.txt', 'w') as fout:
           fout.write('\n'.join(packages))