diff --git a/sdks/go/pkg/beam/core/runtime/xlangx/expansionx/download.go b/sdks/go/pkg/beam/core/runtime/xlangx/expansionx/download.go index e5fff1039675..0b5eba625023 100644 --- a/sdks/go/pkg/beam/core/runtime/xlangx/expansionx/download.go +++ b/sdks/go/pkg/beam/core/runtime/xlangx/expansionx/download.go @@ -21,6 +21,7 @@ import ( "archive/zip" "fmt" "io" + "log" "net/http" "os" "os/exec" @@ -106,6 +107,15 @@ func getLocalJar(url string) (string, error) { return jarPath, nil } + // Issue warning when downloading from public repositories + if strings.Contains(url, "repo.maven.apache.org") || + strings.Contains(url, "repo1.maven.org") || + strings.Contains(url, "maven.google.com") || + strings.Contains(url, "maven-central.storage-download.googleapis.com") { + log.Printf("WARNING: Downloading JAR file from public repository: %s. "+ + "This may pose security risks or cause instability due to repository availability. Consider pre-staging dependencies or using private mirrors.", url) + } + resp, err := http.Get(string(url)) if err != nil { return "", err @@ -334,6 +344,16 @@ func (j *jarGetter) getJar(gradleTarget, version string) (string, error) { gradleTarget) } + // Issue warning when downloading from public repositories + fullURLStr := string(fullURL) + if strings.Contains(fullURLStr, "repo.maven.apache.org") || + strings.Contains(fullURLStr, "repo1.maven.org") || + strings.Contains(fullURLStr, "maven.google.com") || + strings.Contains(fullURLStr, "maven-central.storage-download.googleapis.com") { + log.Printf("WARNING: Downloading JAR file from public repository: %s. "+ + "This may pose security risks or cause instability due to repository availability. Consider pre-staging dependencies or using private mirrors.", fullURLStr) + } + resp, err := http.Get(string(fullURL)) if err != nil { return "", err diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/JavaUdfLoader.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/JavaUdfLoader.java index 1e584ecdef40..4feae9abf1be 100644 --- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/JavaUdfLoader.java +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/JavaUdfLoader.java @@ -125,6 +125,20 @@ public AggregateFn loadAggregateFunction(List functionPath, String jarPa */ private File downloadFile(String inputPath, String mimeType) throws IOException { Preconditions.checkArgument(!inputPath.isEmpty(), "Path cannot be empty."); + + // Issue warning when downloading from public repositories + if (inputPath.startsWith("http://") || inputPath.startsWith("https://")) { + if (inputPath.contains("repo.maven.apache.org") + || inputPath.contains("repo1.maven.org") + || inputPath.contains("maven.google.com") + || inputPath.contains("maven-central.storage-download.googleapis.com")) { + LOG.warn( + "WARNING: Downloading JAR file from public repository: {}. " + + "This may pose security risks or cause instability due to repository availability. Consider pre-staging dependencies or using private mirrors.", + inputPath); + } + } + ResourceId inputResource = FileSystems.matchNewResource(inputPath, false /* is directory */); try (ReadableByteChannel inputChannel = FileSystems.open(inputResource)) { File outputFile = File.createTempFile("sql-udf-", inputResource.getFilename()); diff --git a/sdks/python/apache_beam/utils/subprocess_server.py b/sdks/python/apache_beam/utils/subprocess_server.py index c1b17bb8ff3b..7fb692e66ea7 100644 --- a/sdks/python/apache_beam/utils/subprocess_server.py +++ b/sdks/python/apache_beam/utils/subprocess_server.py @@ -280,8 +280,10 @@ def _really_stop_process(process_and_endpoint): class JavaJarServer(SubprocessServer): MAVEN_CENTRAL_REPOSITORY = 'https://repo.maven.apache.org/maven2' - MAVEN_STAGING_REPOSITORY = 'https://repository.apache.org/content/groups/staging' # pylint: disable=line-too-long - GOOGLE_MAVEN_MIRROR = 'https://maven-central.storage-download.googleapis.com/maven2' # pylint: disable=line-too-long + MAVEN_STAGING_REPOSITORY = ( + 'https://repository.apache.org/content/groups/staging') + GOOGLE_MAVEN_MIRROR = ( + 'https://maven-central.storage-download.googleapis.com/maven2') BEAM_GROUP_ID = 'org.apache.beam' JAR_CACHE = os.path.expanduser("~/.apache_beam/cache/jars") @@ -431,6 +433,26 @@ def _download_jar_to_cache( cached_jar_path (str): The local path where the jar should be cached. user_agent (str): The user agent to use when downloading. """ + # Issue warning when downloading from public repositories + public_repos = [ + cls.MAVEN_CENTRAL_REPOSITORY, + cls.GOOGLE_MAVEN_MIRROR, + ] + + if any(download_url.startswith(repo) for repo in public_repos): + _LOGGER.warning( + " WARNING: Apache Beam is downloading dependencies from a " + "public repository at runtime.\n" + " This may pose security risks or cause instability due to " + "repository availability.\n" + " URL: %s\n" + " Destination: %s\n" + " Consider pre-staging dependencies or using a private repository " + "mirror.\n" + " For more information, see: " + "https://beam.apache.org/documentation/sdks/python-dependencies/", + download_url, + cached_jar_path) try: url_read = FileSystems.open(download_url) except ValueError: diff --git a/sdks/python/apache_beam/yaml/yaml_provider.py b/sdks/python/apache_beam/yaml/yaml_provider.py index 3af457b7010b..0b47cbf2e686 100755 --- a/sdks/python/apache_beam/yaml/yaml_provider.py +++ b/sdks/python/apache_beam/yaml/yaml_provider.py @@ -68,6 +68,8 @@ from apache_beam.yaml import yaml_utils from apache_beam.yaml.yaml_errors import maybe_with_exception_handling_transform_fn +_LOGGER = logging.getLogger(__name__) + class NotAvailableWithReason: """A False value that provides additional content. @@ -1322,6 +1324,18 @@ def _create_venv_from_scratch( venv_python = os.path.join(venv, 'bin', 'python') venv_pip = os.path.join(venv, 'bin', 'pip') subprocess.run([venv_python, '-m', 'ensurepip'], check=True) + # Issue warning when installing packages from PyPI + _LOGGER.warning( + " WARNING: Apache Beam is installing Python packages " + "from PyPI at runtime.\n" + " This may pose security risks or cause instability due to " + "repository availability.\n" + " Packages: %s\n" + " Consider pre-staging dependencies or using a private " + "repository mirror.\n" + " For more information, see: " + "https://beam.apache.org/documentation/sdks/python-dependencies/", + ', '.join(packages)) subprocess.run([venv_pip, 'install'] + packages, check=True) with open(venv + '-requirements.txt', 'w') as fout: fout.write('\n'.join(packages)) @@ -1342,6 +1356,18 @@ def _create_venv_from_clone( clonable_venv = cls._create_venv_to_clone(base_python) clonevirtualenv.clone_virtualenv(clonable_venv, venv) venv_pip = os.path.join(venv, 'bin', 'pip') + # Issue warning when installing packages from PyPI + _LOGGER.warning( + " WARNING: Apache Beam is installing Python packages " + "from PyPI at runtime.\n" + " This may pose security risks or cause instability due to " + "repository availability.\n" + " Packages: %s\n" + " Consider pre-staging dependencies or using a private " + "repository mirror.\n" + " For more information, see: " + "https://beam.apache.org/documentation/sdks/python-dependencies/", + ', '.join(packages)) subprocess.run([venv_pip, 'install'] + packages, check=True) with open(venv + '-requirements.txt', 'w') as fout: fout.write('\n'.join(packages))