chore(bigquery-jdbc): dockerized proxy environment to run integration tests

Author: logachev

java-bigquery-jdbc/Dockerfile

@@ -1,4 +1,4 @@
-FROM gcr.io/cloud-devrel-public-resources/java11
+FROM gcr.io/cloud-devrel-public-resources/java11 AS base
 ARG BRANCH=main
 ENV JDBC_DOCKER_ENV=true
 
@@ -17,8 +17,19 @@ RUN bash -c " \
     && install_modules java-bigquery \
     && rm -rf /git"
 
-# This will ensure all deps are present
+# This will ensure all deps are present, including integration test deps, while skipping shade
 WORKDIR /src
-RUN mvn install
+RUN mvn install -Penable-integration-tests -Dtest=NoSuchTest -Dit.test=NoSuchTest -DfailIfNoTests=false -Dsurefire.failIfNoSpecifiedTests=false -Dit.failIfNoSpecifiedTests=false -Dmaven.shade.skip=true -DskipShade=true
 
 ENTRYPOINT []
+
+# Proxy stage: configured squid proxy and iptables to force all traffic through it
+FROM base AS proxy
+RUN apt-get update && apt-get install -y squid iptables iproute2 curl && rm -rf /var/lib/apt/lists/*
+COPY tools/environments/proxy/start-proxy.sh /usr/local/bin/start-proxy.sh
+RUN chmod +x /usr/local/bin/start-proxy.sh
+ENTRYPOINT ["/usr/local/bin/start-proxy.sh"]
+
+# Regular stage: same as base, default stage
+FROM base AS regular
+

java-bigquery-jdbc/Makefile

@@ -1,9 +1,11 @@
 SHELL := /bin/bash # Default 'sh' doesn't support 'source'
 BUILD_BRANCH=main
 CONTAINER_NAME=jdbc
+PROXY_CONTAINER_NAME=$(CONTAINER_NAME)-proxy
 PACKAGE_DESTINATION=$(PWD)/drivers
 SRC="$(PWD)"
 skipSurefire ?= true
+skipShade ?= true
 JDBC_DRIVER_VERSION = $(shell mvn help:evaluate -Dexpression=project.version -q -DforceStdout)
 JDBC_JAR = $(PACKAGE_DESTINATION)/google-cloud-bigquery-jdbc-$(JDBC_DRIVER_VERSION)-all.jar
 
@@ -33,13 +35,14 @@ unittest: |
 		-Dtest=$(test) \
 		test
 
-# Important: By default, this command will skip unittests.
+# Important: By default, this command will skip unittests & uberjar build.
 # To include unit tests, run: make integration-test skipSurefire=false
 integration-test:
 	mvn -B -ntp \
 		-Penable-integration-tests \
 		-DtrimStackTrace=false \
 		-DskipSurefire=$(skipSurefire) \
+		-DskipShade=$(skipShade) \
 		-Dclirr.skip=true \
 		-Denforcer.skip=true \
 		-Dit.failIfNoSpecifiedTests=true \
@@ -76,21 +79,30 @@ run-it-standalone:
 # Commands for dockerized environments
 .docker-run: | 
 	docker run -it \
+		--cap-add=NET_ADMIN \
 		-v $(GOOGLE_APPLICATION_CREDENTIALS):/auth/application_creds.json \
 		-v "$(GOOGLE_APPLICATION_CREDENTIALS).p12":/auth/application_creds.p12 \
 		-e "GOOGLE_APPLICATION_CREDENTIALS=/auth/application_creds.json" \
 		-v $(SRC):/src \
 		-e "SA_EMAIL=test_email" \
 		-e "SA_SECRET=/auth/application_creds.json" \
 		-e "SA_SECRET_P12=/auth/application_creds.p12" \
+		-e "BIGQUERY_BASE_URL=$(BIGQUERY_BASE_URL)" \
+		-e "BIGQUERY_URL_FLAGS=$(BIGQUERY_URL_FLAGS)" \
 		$(CONTAINER_NAME) $(args)
 
 docker-build:
-	docker build -t $(CONTAINER_NAME) -f Dockerfile --build-arg BRANCH=${BUILD_BRANCH} $(SRC)
+	docker build --target regular -t $(CONTAINER_NAME) -f Dockerfile --build-arg BRANCH=${BUILD_BRANCH} $(SRC)
+
+docker-proxy-build:
+	docker build --target proxy -t $(PROXY_CONTAINER_NAME) -f Dockerfile --build-arg BRANCH=${BUILD_BRANCH} $(SRC)
 
 docker-session:
 	$(MAKE) .docker-run args="bash"
 
+docker-proxy-session:
+	$(MAKE) .docker-run-proxy args="bash"
+
 docker-package-all-dependencies: docker-build
 	mkdir -p $(PACKAGE_DESTINATION)
 	docker run \
@@ -134,6 +146,9 @@ docker-unittest: |
 docker-integration-test: .check-env
 	$(MAKE) .docker-run args="make integration-test test=$(test) skipSurefire=$(skipSurefire)"
 
+docker-proxy-integration-test: .check-env docker-proxy-build
+	$(MAKE) docker-integration-test CONTAINER_NAME=$(PROXY_CONTAINER_NAME) BIGQUERY_URL_FLAGS="ProxyHost=127.0.0.1;ProxyPort=3128;"
+
 docker-coverage: 
 	$(MAKE) .docker-run args="make unit-test-coverage"
 	$(MAKE) .docker-run args="make full-coverage"

java-bigquery-jdbc/pom.xml

@@ -31,6 +31,7 @@
     <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
     <github.global.server>github</github.global.server>
     <site.installationModule>google-cloud-bigquery-jdbc</site.installationModule>
+    <skipShade>false</skipShade>
   </properties>
 
   <build>
@@ -96,6 +97,7 @@
                   <goal>shade</goal>
                 </goals>
                 <configuration>
+                  <skip>${skipShade}</skip>
                   <shadedArtifactAttached>true</shadedArtifactAttached>
                   <createSourcesJar>true</createSourcesJar>
                   <shadeSourcesContent>true</shadeSourcesContent>

java-bigquery-jdbc/tools/environments/proxy/start-proxy.sh

@@ -0,0 +1,58 @@
+#!/bin/bash
+# start-proxy.sh
+
+set -e
+
+echo "Starting Squid proxy..."
+# Run squid in background. 
+# On Debian, /usr/sbin/squid is the binary.
+# -s sends errors to syslog. -Y during rebuild. -C do not catch fatal signals.
+/usr/sbin/squid -sYC
+
+# Wait for squid to be ready and listen on 3128
+echo "Waiting for Squid to listen on port 3128..."
+timeout=30
+while ! curl -s -I -x http://127.0.0.1:3128 https://www.google.com >/dev/null; do
+    sleep 1
+    timeout=$((timeout - 1))
+    if [ $timeout -le 0 ]; then
+        echo "Squid failed to start or cannot access the internet."
+        exit 1
+    fi
+done
+echo "Squid is ready and working."
+
+# Configure iptables to restrict network access
+echo "Configuring iptables rules..."
+
+# 1. Allow loopback traffic
+iptables -A OUTPUT -o lo -j ACCEPT
+
+# 2. Allow squid user (proxy) to access the network
+iptables -A OUTPUT -m owner --uid-owner proxy -j ACCEPT
+
+# 3. Allow DNS (port 53) for everyone
+iptables -A OUTPUT -p udp --dport 53 -j ACCEPT
+iptables -A OUTPUT -p tcp --dport 53 -j ACCEPT
+
+# 4. Allow outgoing traffic to port 3128 (proxies) for testing external proxies
+iptables -A OUTPUT -p tcp --dport 3128 -j ACCEPT
+
+# 4.5 Allow raw access to Maven Central (repo.maven.apache.org) for dynamic dependency downloads
+echo "Resolving repo.maven.apache.org and allowing raw access..."
+for ip in $(getent ahosts repo.maven.apache.org | awk '{print $1}' | sort -u); do
+    if [[ $ip =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+        echo "Allowing outbound to $ip"
+        iptables -A OUTPUT -d "$ip" -p tcp --dport 443 -j ACCEPT
+        iptables -A OUTPUT -d "$ip" -p tcp --dport 80 -j ACCEPT
+    fi
+done
+
+# 5. Reject all other outgoing TCP/UDP traffic
+iptables -A OUTPUT -p tcp -j REJECT --reject-with tcp-reset
+iptables -A OUTPUT -p udp -j REJECT
+
+echo "Raw network access is now disabled. All traffic must go through the proxy."
+
+# Execute the main command
+exec "$@"