From 8a799175a7e5e67cadc74c9edda043411e1c7c50 Mon Sep 17 00:00:00 2001 From: liferoad Date: Fri, 10 Oct 2025 10:48:50 -0400 Subject: [PATCH 1/3] feat: add warnings for public repository downloads in multiple SDKs add warnings when downloading dependencies from public repositories in Java, Go, and Python SDKs warn users about potential risks and suggest pre-staging dependencies or using private mirrors --- .../runtime/xlangx/expansionx/download.go | 20 ++++++++++++++++ .../extensions/sql/impl/JavaUdfLoader.java | 14 +++++++++++ .../apache_beam/utils/subprocess_server.py | 24 +++++++++++++++++-- sdks/python/apache_beam/yaml/yaml_provider.py | 24 +++++++++++++++++++ 4 files changed, 80 insertions(+), 2 deletions(-) diff --git a/sdks/go/pkg/beam/core/runtime/xlangx/expansionx/download.go b/sdks/go/pkg/beam/core/runtime/xlangx/expansionx/download.go index e5fff1039675..0b5eba625023 100644 --- a/sdks/go/pkg/beam/core/runtime/xlangx/expansionx/download.go +++ b/sdks/go/pkg/beam/core/runtime/xlangx/expansionx/download.go @@ -21,6 +21,7 @@ import ( "archive/zip" "fmt" "io" + "log" "net/http" "os" "os/exec" @@ -106,6 +107,15 @@ func getLocalJar(url string) (string, error) { return jarPath, nil } + // Issue warning when downloading from public repositories + if strings.Contains(url, "repo.maven.apache.org") || + strings.Contains(url, "repo1.maven.org") || + strings.Contains(url, "maven.google.com") || + strings.Contains(url, "maven-central.storage-download.googleapis.com") { + log.Printf("WARNING: Downloading JAR file from public repository: %s. "+ + "This may pose security risks or cause instability due to repository availability. Consider pre-staging dependencies or using private mirrors.", url) + } + resp, err := http.Get(string(url)) if err != nil { return "", err @@ -334,6 +344,16 @@ func (j *jarGetter) getJar(gradleTarget, version string) (string, error) { gradleTarget) } + // Issue warning when downloading from public repositories + fullURLStr := string(fullURL) + if strings.Contains(fullURLStr, "repo.maven.apache.org") || + strings.Contains(fullURLStr, "repo1.maven.org") || + strings.Contains(fullURLStr, "maven.google.com") || + strings.Contains(fullURLStr, "maven-central.storage-download.googleapis.com") { + log.Printf("WARNING: Downloading JAR file from public repository: %s. "+ + "This may pose security risks or cause instability due to repository availability. Consider pre-staging dependencies or using private mirrors.", fullURLStr) + } + resp, err := http.Get(string(fullURL)) if err != nil { return "", err diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/JavaUdfLoader.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/JavaUdfLoader.java index 1e584ecdef40..4feae9abf1be 100644 --- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/JavaUdfLoader.java +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/JavaUdfLoader.java @@ -125,6 +125,20 @@ public AggregateFn loadAggregateFunction(List functionPath, String jarPa */ private File downloadFile(String inputPath, String mimeType) throws IOException { Preconditions.checkArgument(!inputPath.isEmpty(), "Path cannot be empty."); + + // Issue warning when downloading from public repositories + if (inputPath.startsWith("http://") || inputPath.startsWith("https://")) { + if (inputPath.contains("repo.maven.apache.org") + || inputPath.contains("repo1.maven.org") + || inputPath.contains("maven.google.com") + || inputPath.contains("maven-central.storage-download.googleapis.com")) { + LOG.warn( + "WARNING: Downloading JAR file from public repository: {}. " + + "This may pose security risks or cause instability due to repository availability. Consider pre-staging dependencies or using private mirrors.", + inputPath); + } + } + ResourceId inputResource = FileSystems.matchNewResource(inputPath, false /* is directory */); try (ReadableByteChannel inputChannel = FileSystems.open(inputResource)) { File outputFile = File.createTempFile("sql-udf-", inputResource.getFilename()); diff --git a/sdks/python/apache_beam/utils/subprocess_server.py b/sdks/python/apache_beam/utils/subprocess_server.py index c1b17bb8ff3b..ebb91719dba0 100644 --- a/sdks/python/apache_beam/utils/subprocess_server.py +++ b/sdks/python/apache_beam/utils/subprocess_server.py @@ -280,8 +280,10 @@ def _really_stop_process(process_and_endpoint): class JavaJarServer(SubprocessServer): MAVEN_CENTRAL_REPOSITORY = 'https://repo.maven.apache.org/maven2' - MAVEN_STAGING_REPOSITORY = 'https://repository.apache.org/content/groups/staging' # pylint: disable=line-too-long - GOOGLE_MAVEN_MIRROR = 'https://maven-central.storage-download.googleapis.com/maven2' # pylint: disable=line-too-long + MAVEN_STAGING_REPOSITORY = ( + 'https://repository.apache.org/content/groups/staging') + GOOGLE_MAVEN_MIRROR = ( + 'https://maven-central.storage-download.googleapis.com/maven2') BEAM_GROUP_ID = 'org.apache.beam' JAR_CACHE = os.path.expanduser("~/.apache_beam/cache/jars") @@ -431,6 +433,24 @@ def _download_jar_to_cache( cached_jar_path (str): The local path where the jar should be cached. user_agent (str): The user agent to use when downloading. """ + # Issue warning when downloading from public repositories + public_repos = [ + cls.MAVEN_CENTRAL_REPOSITORY, + cls.GOOGLE_MAVEN_MIRROR, + ] + + if any(download_url.startswith(repo) for repo in public_repos): + _LOGGER.warning( + " WARNING: Apache Beam is downloading dependencies from a " + "public repository at runtime.\n" + " This may pose security risks or cause instability due to " + "repository availability.\n" + " URL: %s\n" + " Consider pre-staging dependencies or using a private repository " + "mirror.\n" + " For more information, see: " + "https://beam.apache.org/documentation/sdks/python-dependencies/", + download_url) try: url_read = FileSystems.open(download_url) except ValueError: diff --git a/sdks/python/apache_beam/yaml/yaml_provider.py b/sdks/python/apache_beam/yaml/yaml_provider.py index 3af457b7010b..c5576703ed49 100755 --- a/sdks/python/apache_beam/yaml/yaml_provider.py +++ b/sdks/python/apache_beam/yaml/yaml_provider.py @@ -1322,6 +1322,18 @@ def _create_venv_from_scratch( venv_python = os.path.join(venv, 'bin', 'python') venv_pip = os.path.join(venv, 'bin', 'pip') subprocess.run([venv_python, '-m', 'ensurepip'], check=True) + # Issue warning when installing packages from PyPI + _LOGGER.warning( + " WARNING: Apache Beam is installing Python packages " + "from PyPI at runtime.\n" + " This may pose security risks or cause instability due to " + "repository availability.\n" + " Packages: %s\n" + " Consider pre-staging dependencies or using a private " + "repository mirror.\n" + " For more information, see: " + "https://beam.apache.org/documentation/sdks/python-dependencies/", + ', '.join(packages)) subprocess.run([venv_pip, 'install'] + packages, check=True) with open(venv + '-requirements.txt', 'w') as fout: fout.write('\n'.join(packages)) @@ -1342,6 +1354,18 @@ def _create_venv_from_clone( clonable_venv = cls._create_venv_to_clone(base_python) clonevirtualenv.clone_virtualenv(clonable_venv, venv) venv_pip = os.path.join(venv, 'bin', 'pip') + # Issue warning when installing packages from PyPI + _LOGGER.warning( + " WARNING: Apache Beam is installing Python packages " + "from PyPI at runtime.\n" + " This may pose security risks or cause instability due to " + "repository availability.\n" + " Packages: %s\n" + " Consider pre-staging dependencies or using a private " + "repository mirror.\n" + " For more information, see: " + "https://beam.apache.org/documentation/sdks/python-dependencies/", + ', '.join(packages)) subprocess.run([venv_pip, 'install'] + packages, check=True) with open(venv + '-requirements.txt', 'w') as fout: fout.write('\n'.join(packages)) From d5ae674448b2b25e16f711a9e40af24864d844c5 Mon Sep 17 00:00:00 2001 From: liferoad Date: Fri, 10 Oct 2025 11:17:02 -0400 Subject: [PATCH 2/3] logger --- sdks/python/apache_beam/yaml/yaml_provider.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sdks/python/apache_beam/yaml/yaml_provider.py b/sdks/python/apache_beam/yaml/yaml_provider.py index c5576703ed49..0b47cbf2e686 100755 --- a/sdks/python/apache_beam/yaml/yaml_provider.py +++ b/sdks/python/apache_beam/yaml/yaml_provider.py @@ -68,6 +68,8 @@ from apache_beam.yaml import yaml_utils from apache_beam.yaml.yaml_errors import maybe_with_exception_handling_transform_fn +_LOGGER = logging.getLogger(__name__) + class NotAvailableWithReason: """A False value that provides additional content. From a86346c94832964fba576e57738fbf6589b9a14b Mon Sep 17 00:00:00 2001 From: liferoad Date: Fri, 10 Oct 2025 14:50:05 -0400 Subject: [PATCH 3/3] added cached_jar_path --- sdks/python/apache_beam/utils/subprocess_server.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sdks/python/apache_beam/utils/subprocess_server.py b/sdks/python/apache_beam/utils/subprocess_server.py index ebb91719dba0..7fb692e66ea7 100644 --- a/sdks/python/apache_beam/utils/subprocess_server.py +++ b/sdks/python/apache_beam/utils/subprocess_server.py @@ -446,11 +446,13 @@ def _download_jar_to_cache( " This may pose security risks or cause instability due to " "repository availability.\n" " URL: %s\n" + " Destination: %s\n" " Consider pre-staging dependencies or using a private repository " "mirror.\n" " For more information, see: " "https://beam.apache.org/documentation/sdks/python-dependencies/", - download_url) + download_url, + cached_jar_path) try: url_read = FileSystems.open(download_url) except ValueError: