Run tests in Gitlab (#8656)

* first pass checks

* add some debug

* Revert "add some debug"

This reverts commit 26a7adb.

* Normalize node index and node total

* fix normalization

* actually use the normalized values

* first pass: migrate tests to Gitlab

* less scenarios for now

* bump memory and workers

* actually use task partitions

* Add section start/end

* Allow individual tests to control gradle memory

* remove a little of the reportspam

* use concrete docker image tags

* configure test containers to use mirror

* fix limit

* fix missing container versions

* Better gitlab naming with smarter split

* update caches with test dep cache, save test results

* Cleanup repositories {} blocks

* update proxy url to use internal routing

* no "s"

* allow insecure everywhere

* include apache with confluence

* remove intentional fail

* use gradlePluginCache, mavenCentral after pluginportal

* proxy mule as well

* change BUILD_CACHE_TYPE to CACHE_TYPE since its used for both

* split cache like with circleci. Cache became too big

* Some caches are too big to be saved

* test fixes

* more test fixes

* disable some tests for now

* more fixes

* Use maven proxy with openliberty smoketests

* missed one more mongodb case

* enable some more of the matrix

* Fix lettuce tests

* Fix lettuce tests

* cleanup imports

* create asyncCommands

* codenarc

* Cleanup aggregator tests to try to reduce flakiness

* debug maven logs for now

* try non-ssh url

* Revert "debug maven logs for now"

This reverts commit c5c158f.

* add some debug logging

* codenarc

* try fixed value of parallelism

* enable debug logs for smoketest building only

* limit number of reserved threads

* some debug for why the smoketest is not running

* more debug for gradle launcher failure

* try fixing graal test

* try to force the logging

* codenarc

* set available processors for jetty

* codenarc

* bluntly bump max threads

* some more debug

* remove verbose logging

* try memcached test with new location

* opt out of split by parallel

* Try with 3 gradle workers

* try decreasing reporting interval

* Refractor CrashTrackingSmokeTest to use OutputThreads

* Simple forked memory test

* better ergonomics

* try increasing timeout

* extend agent test runner

* don't override forkedMinHeapSize and forkedMaxHeapSize

* print out requests

* remove logging statement

* Better crash tracking

* remove intentionally failing test

* use a test UDP server to record oom events

* Make crash tracking smoke test more resiliant

* loop through messages until the correct event

* some debug

* configurable dogstatsd port

* enable everything

* use sleep instead of park nanos

* save body data ahead of time

* retry once for system failures

* spotless

* cleanup

* spotless/merge issues

* Skip ExceptionHistogramTest on J9

* add a print statement around the body for debugging

* comment out all but 8, 17, and 21 for now

* fix spring 6 forwarding headers

* revert logging statements

* needs to be set in 3.1 for latestDepTest

* faster OOM

* property was set in incorrect location

* ssi_smoke, flaky, and esoteric jvms on master only

* use normal runner when TestContainers isn't needed

* skip forwarding header filter here too

* tweaks

* limit executor size and fix IPV4 assumptions

* try more workers with less memory

* set max memory for all crashtracking tests

* double retry

* scale up memory, less jobs

* revert memory bump, increase forked tests

* tweaks

* add flag for nondefault jvms and flaky tests

* correct description

* Externalize runtime processors override

* ExceptionHistogramTest exclusions already handled by master

* increase spring native build memory

* bump gradle build timeout

* bump graalvm compile memory

---------

Co-authored-by: Santiago Mola <santiago.mola@datadoghq.com>

Author: 2 people

.gitlab-ci.yml

@@ -94,12 +94,15 @@ default:
   - export NORMALIZED_NODE_TOTAL=${CI_NODE_TOTAL:-1}
   - ONE_INDEXED_NODE_INDEX=${CI_NODE_INDEX:-1}; export NORMALIZED_NODE_INDEX=$((ONE_INDEXED_NODE_INDEX - 1))
   - echo "NORMALIZED_NODE_TOTAL=${NORMALIZED_NODE_TOTAL}, NORMALIZED_NODE_INDEX=$NORMALIZED_NODE_INDEX"
+<<<<<<< HEAD
 
 .cgroup_info: &cgroup_info
   - source .gitlab/gitlab-utils.sh
   - gitlab_section_start "cgroup-info" "cgroup info"
   - .gitlab/cgroup-info.sh
   - gitlab_section_end "cgroup-info"
+=======
+>>>>>>> f494c33b01 (Run tests in Gitlab (#8656))
 
 .gradle_build: &gradle_build
   image: ghcr.io/datadog/dd-trace-java-docker-build:${BUILDER_IMAGE_VERSION_PREFIX}base
@@ -146,8 +149,11 @@ default:
     - mv .gradle-copy .gradle
     - ls -la
     - gitlab_section_end "gradle-dance"
+<<<<<<< HEAD
   after_script:
     - *cgroup_info
+=======
+>>>>>>> f494c33b01 (Run tests in Gitlab (#8656))
 
 build:
   extends: .gradle_build
@@ -224,6 +230,7 @@ populate_dep_cache:
 #        CACHE_TYPE: "latestdep"
 #      - GRADLE_TARGET: ":smokeTest"
 #        CACHE_TYPE: "smoke"
+<<<<<<< HEAD
 
 publish-artifacts-to-s3:
   image: registry.ddbuild.io/images/mirror/amazon/aws-cli:2.4.29
@@ -256,6 +263,8 @@ publish-artifacts-to-s3:
       annotations:
         - links.json
 
+=======
+>>>>>>> f494c33b01 (Run tests in Gitlab (#8656))
 
 spotless:
   extends: .gradle_build
@@ -283,7 +292,10 @@ test_published_artifacts:
     - export GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx1G -Xms1G -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp'"
     - ./gradlew check --info $GRADLE_ARGS
   after_script:
+<<<<<<< HEAD
     - *cgroup_info
+=======
+>>>>>>> f494c33b01 (Run tests in Gitlab (#8656))
     - source .gitlab/gitlab-utils.sh
     - gitlab_section_start "collect-reports" "Collecting reports"
     - .circleci/collect_reports.sh
@@ -302,7 +314,10 @@ test_published_artifacts:
   script:
     - ./gradlew $GRADLE_TARGET -PskipTests -PrunBuildSrcTests -PskipSpotless -PtaskPartitionCount=$NORMALIZED_NODE_TOTAL -PtaskPartition=$NORMALIZED_NODE_INDEX $GRADLE_ARGS
   after_script:
+<<<<<<< HEAD
     - *cgroup_info
+=======
+>>>>>>> f494c33b01 (Run tests in Gitlab (#8656))
     - source .gitlab/gitlab-utils.sh
     - gitlab_section_start "collect-reports" "Collecting reports"
     - .circleci/collect_reports.sh --destination ./check_reports --move
@@ -363,7 +378,10 @@ muzzle:
     - split --number=l/$NORMALIZED_NODE_TOTAL --suffix-length=1 --numeric-suffixes sortedMuzzleTasks muzzleSplit
     - ./gradlew `cat muzzleSplit${NORMALIZED_NODE_INDEX} | xargs` $GRADLE_ARGS
   after_script:
+<<<<<<< HEAD
     - *cgroup_info
+=======
+>>>>>>> f494c33b01 (Run tests in Gitlab (#8656))
     - source .gitlab/gitlab-utils.sh
     - gitlab_section_start "collect-reports" "Collecting reports"
     - .circleci/collect_reports.sh
@@ -409,13 +427,22 @@ muzzle-dep-report:
 
 .test_job:
   extends: .gradle_build
+<<<<<<< HEAD
   image: ghcr.io/datadog/dd-trace-java-docker-build:${BUILDER_IMAGE_VERSION_PREFIX}$testJvm
+=======
+  image: ghcr.io/datadog/dd-trace-java-docker-build:$testJvm
+>>>>>>> f494c33b01 (Run tests in Gitlab (#8656))
   tags: [ "docker-in-docker:amd64" ] # use docker-in-docker runner for testcontainers
   needs: [ build_tests ]
   stage: tests
   variables:
+<<<<<<< HEAD
     KUBERNETES_MEMORY_REQUEST: 17Gi
     KUBERNETES_MEMORY_LIMIT: 17Gi
+=======
+    KUBERNETES_MEMORY_REQUEST: 16Gi
+    KUBERNETES_MEMORY_LIMIT: 16Gi
+>>>>>>> f494c33b01 (Run tests in Gitlab (#8656))
     KUBERNETES_CPU_REQUEST: 10
     GRADLE_WORKERS: 4
     GRADLE_MEM: 3G
@@ -439,21 +466,33 @@ muzzle-dep-report:
       export PROFILER_COMMAND="-XX:StartFlightRecording=settings=profile,filename=/tmp/${CI_JOB_NAME_SLUG}.jfr,dumponexit=true";
       fi
     - *prepare_test_env
+<<<<<<< HEAD
     - export GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xms$GRADLE_MEM -Xmx$GRADLE_MEM $PROFILER_COMMAND -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp' -Ddatadog.forkedMaxHeapSize=1024M -Ddatadog.forkedMinHeapSize=128M"
+=======
+    - export GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xms$GRADLE_MEM -Xmx$GRADLE_MEM $PROFILER_COMMAND -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp' -Ddatadog.forkedMaxHeapSize=768M -Ddatadog.forkedMinHeapSize=128M"
+>>>>>>> f494c33b01 (Run tests in Gitlab (#8656))
     - ./gradlew $GRADLE_TARGET $GRADLE_PARAMS -PtestJvm=$testJvm -PtaskPartitionCount=$NORMALIZED_NODE_TOTAL -PtaskPartition=$NORMALIZED_NODE_INDEX $GRADLE_ARGS --continue || $CONTINUE_ON_FAILURE
   after_script:
     - *restore_pretest_env
     - *set_datadog_api_keys
+<<<<<<< HEAD
     - *cgroup_info
+=======
+>>>>>>> f494c33b01 (Run tests in Gitlab (#8656))
     - source .gitlab/gitlab-utils.sh
     - gitlab_section_start "collect-reports" "Collecting reports"
     - .circleci/collect_reports.sh
     - if [ "$PROFILE_TESTS" == "true" ]; then .circleci/collect_profiles.sh; fi
     - .circleci/collect_results.sh
+<<<<<<< HEAD
     - .circleci/upload_ciapp.sh $CACHE_TYPE $testJvm
     - gitlab_section_end "collect-reports"
     - URL_ENCODED_JOB_NAME=$(jq -rn --arg x "$CI_JOB_NAME" '$x|@uri')
     - echo -e "${TEXT_BOLD}${TEXT_YELLOW}See test results in Datadog:${TEXT_CLEAR} https://app.datadoghq.com/ci/test/runs?query=test_level%3Atest%20%40test.service%3Add-trace-java%20%40ci.pipeline.id%3A${CI_PIPELINE_ID}%20%40ci.job.name%3A%22${URL_ENCODED_JOB_NAME}%22"
+=======
+    - .circleci/upload_ciapp.sh tests $testJvm
+    - gitlab_section_end "collect-reports"
+>>>>>>> f494c33b01 (Run tests in Gitlab (#8656))
   artifacts:
     when: always
     paths:
@@ -597,7 +636,11 @@ test_smoke:
     GRADLE_PARAMS: "-PskipFlakyTests"
     CACHE_TYPE: "smoke"
   parallel:
+<<<<<<< HEAD
     matrix: *test_matrix_4
+=======
+    matrix: *test_matrix_2
+>>>>>>> f494c33b01 (Run tests in Gitlab (#8656))
 
 test_ssi_smoke:
   extends: .test_job
@@ -608,7 +651,11 @@ test_ssi_smoke:
     DD_INJECT_FORCE: "true"
     DD_INJECTION_ENABLED: "tracer"
   parallel:
+<<<<<<< HEAD
     matrix: *test_matrix_4
+=======
+    matrix: *test_matrix_2
+>>>>>>> f494c33b01 (Run tests in Gitlab (#8656))
 
 test_smoke_graalvm:
   extends: .test_job
@@ -630,6 +677,31 @@ test_smoke_semeru8_debugger:
     CACHE_TYPE: "smoke"
     NON_DEFAULT_JVMS: "true"
     testJvm: "semeru8"
+<<<<<<< HEAD
+=======
+
+required:
+  extends: .fan_in
+  needs:
+    - job: spotless
+      optional: true
+    - job: muzzle
+      optional: true
+    - job: test_published_artifacts
+      optional: true
+    - job: agent_integration_tests
+      optional: true
+    - job: check_base
+      optional: true
+    - job: check_inst
+      optional: true
+    - job: check_smoke
+      optional: true
+    - job: check_profiling
+      optional: true
+    - job: check_debugger
+      optional: true
+>>>>>>> f494c33b01 (Run tests in Gitlab (#8656))
 
 deploy_to_profiling_backend:
   stage: publish

.gitlab/gitlab-utils.sh

@@ -24,4 +24,4 @@ export TEXT_BLUE="\e[34m"
 export TEXT_MAGENTA="\e[35m"
 export TEXT_CYAN="\e[36m"
 export TEXT_CLEAR="\e[0m"
-export TEXT_BOLD="\e[1m"
+export TEXT_BOLD="\e[1m"

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/utils/ShellCommandExecutor.java

@@ -161,7 +161,11 @@ private <T> T executeCommand(
         throw new TimeoutException(
             "Timeout while waiting for '"
                 + String.join(" ", command)
-                + "'; "
+                + "'; in "
+                + executionFolder
+                + "\n StdOut: \n"
+                + IOUtils.readFully(inputStreamConsumer.read(), Charset.defaultCharset())
+                + "\n StdErr: \n "
                 + IOUtils.readFully(errorStreamConsumer.read(), Charset.defaultCharset()));
       }
     } catch (InterruptedException e) {

dd-java-agent/agent-ci-visibility/src/test/resources/ci/git/shallow/git/config

@@ -6,7 +6,7 @@
 	ignorecase = true
 	precomposeunicode = true
 [remote "origin"]
-	url = git@github.com:Netflix/zuul.git
+	url = https://github.com/Netflix/zuul.git
 	fetch = +refs/heads/master:refs/remotes/origin/master
 [branch "master"]
 	remote = origin

dd-java-agent/instrumentation/aws-java-dynamodb-2.0/src/test/groovy/DynamoDbClientTest.groovy

@@ -29,7 +29,7 @@ import spock.lang.Shared
 import java.time.Duration
 
 class DynamoDbClientTest extends AgentTestRunner {
-  static final LOCALSTACK = new GenericContainer(DockerImageName.parse("localstack/localstack"))
+  static final LOCALSTACK = new GenericContainer(DockerImageName.parse("localstack/localstack:4.2.0"))
   .withExposedPorts(4566)
   .withEnv("SERVICES", "dynamodb")
   .withReuse(true)

dd-java-agent/instrumentation/aws-java-eventbridge-2.0/src/test/groovy/EventBridgeClientTest.groovy

@@ -21,7 +21,7 @@ import java.time.Duration
 import java.util.concurrent.CompletableFuture
 
 class EventBridgeClientTest extends AgentTestRunner {
-  static final LOCALSTACK = new GenericContainer(DockerImageName.parse("localstack/localstack"))
+  static final LOCALSTACK = new GenericContainer(DockerImageName.parse("localstack/localstack:4.2.0"))
   .withExposedPorts(4566)
   .withEnv("SERVICES", "sns,sqs,events")
   .withReuse(true)

dd-java-agent/instrumentation/aws-java-s3-2.0/src/test/groovy/S3ClientTest.groovy

@@ -18,7 +18,7 @@ import spock.lang.Shared
 import java.time.Duration
 
 class S3ClientTest extends AgentTestRunner {
-  static final LOCALSTACK = new GenericContainer(DockerImageName.parse("localstack/localstack"))
+  static final LOCALSTACK = new GenericContainer(DockerImageName.parse("localstack/localstack:4.2.0"))
   .withExposedPorts(4566)
   .withEnv("SERVICES", "s3")
   .withReuse(true)
@@ -92,8 +92,8 @@ class S3ClientTest extends AgentTestRunner {
             tag "bucketname", bucketName
             tag "http.method", "PUT"
             tag "http.status_code", 200
-            tag "http.url", { it.startsWith("http://localhost") && it.contains("/$key") }
-            tag "peer.hostname", "localhost"
+            tag "http.url", { it.startsWith("http://" + LOCALSTACK.getHost()) && it.contains("/$key") }
+            tag "peer.hostname", LOCALSTACK.getHost()
             tag "peer.port", { it instanceof Integer }
             tag "span.kind", "client"
             tag "aws.requestId", { it != null }
@@ -151,8 +151,8 @@ class S3ClientTest extends AgentTestRunner {
             tag "bucketname", bucketName
             tag "http.method", "PUT"
             tag "http.status_code", 200
-            tag "http.url", { it.startsWith("http://localhost") && it.contains("/$sourceKey") }
-            tag "peer.hostname", "localhost"
+            tag "http.url", { it.startsWith("http://" + LOCALSTACK.getHost()) && it.contains("/$sourceKey") }
+            tag "peer.hostname", LOCALSTACK.getHost()
             tag "peer.port", { it instanceof Integer }
             tag "span.kind", "client"
             tag "aws.requestId", { it != null }
@@ -185,8 +185,8 @@ class S3ClientTest extends AgentTestRunner {
             tag "bucketname", bucketName
             tag "http.method", "PUT"
             tag "http.status_code", 200
-            tag "http.url", { it.startsWith("http://localhost") && it.contains("/$destKey") }
-            tag "peer.hostname", "localhost"
+            tag "http.url", { it.startsWith("http://" + LOCALSTACK.getHost()) && it.contains("/$destKey") }
+            tag "peer.hostname", LOCALSTACK.getHost()
             tag "peer.port", { it instanceof Integer }
             tag "span.kind", "client"
             tag "aws.requestId", { it != null }
@@ -276,8 +276,8 @@ class S3ClientTest extends AgentTestRunner {
             tag "bucketname", bucketName
             tag "http.method", "POST"
             tag "http.status_code", 200
-            tag "http.url", { it.startsWith("http://localhost") && it.contains("/$key") }
-            tag "peer.hostname", "localhost"
+            tag "http.url", { it.startsWith("http://" + LOCALSTACK.getHost()) && it.contains("/$key") }
+            tag "peer.hostname", LOCALSTACK.getHost()
             tag "peer.port", { it instanceof Integer }
             tag "span.kind", "client"
             tag "aws.requestId", { it != null }
@@ -303,8 +303,8 @@ class S3ClientTest extends AgentTestRunner {
             tag "bucketname", bucketName
             tag "http.method", "PUT"
             tag "http.status_code", 200
-            tag "http.url", { it.startsWith("http://localhost") && it.contains("/$key") }
-            tag "peer.hostname", "localhost"
+            tag "http.url", { it.startsWith("http://" + LOCALSTACK.getHost()) && it.contains("/$key") }
+            tag "peer.hostname", LOCALSTACK.getHost()
             tag "peer.port", { it instanceof Integer }
             tag "span.kind", "client"
             tag "aws.requestId", { it != null }
@@ -330,8 +330,8 @@ class S3ClientTest extends AgentTestRunner {
             tag "bucketname", bucketName
             tag "http.method", "PUT"
             tag "http.status_code", 200
-            tag "http.url", { it.startsWith("http://localhost") && it.contains("/$key") }
-            tag "peer.hostname", "localhost"
+            tag "http.url", { it.startsWith("http://" + LOCALSTACK.getHost()) && it.contains("/$key") }
+            tag "peer.hostname", LOCALSTACK.getHost()
             tag "peer.port", { it instanceof Integer }
             tag "span.kind", "client"
             tag "aws.requestId", { it != null }
@@ -365,8 +365,8 @@ class S3ClientTest extends AgentTestRunner {
             tag "bucketname", bucketName
             tag "http.method", "POST"
             tag "http.status_code", 200
-            tag "http.url", { it.startsWith("http://localhost") && it.contains("/$key") }
-            tag "peer.hostname", "localhost"
+            tag "http.url", { it.startsWith("http://" + LOCALSTACK.getHost()) && it.contains("/$key") }
+            tag "peer.hostname", LOCALSTACK.getHost()
             tag "peer.port", { it instanceof Integer }
             tag "span.kind", "client"
             tag "aws.requestId", { it != null }

dd-java-agent/instrumentation/aws-java-sdk-2.2/src/payloadTaggingTest/groovy/PayloadTaggingTest.groovy

@@ -26,7 +26,7 @@ abstract class AbstractPayloadTaggingTest extends AgentTestRunner {
   static final Object NA = {}
 
   static final int DEFAULT_PORT = 4566
-  static final LOCALSTACK = new GenericContainer(DockerImageName.parse("localstack/localstack"))
+  static final LOCALSTACK = new GenericContainer(DockerImageName.parse("localstack/localstack:4.2.0"))
   .withExposedPorts(DEFAULT_PORT)
   .withEnv("SERVICES", "apigateway,events,s3,sns,sqs,kinesis")
   .withReuse(true)

dd-java-agent/instrumentation/aws-java-sfn-2.0/src/test/groovy/SfnClientTest.groovy

@@ -25,7 +25,7 @@ abstract class SfnClientTest extends VersionedNamingTestBase {
   @Shared Object endPoint
 
   def setupSpec() {
-    localStack = new GenericContainer(DockerImageName.parse("localstack/localstack"))
+    localStack = new GenericContainer(DockerImageName.parse("localstack/localstack:4.2.0"))
       .withExposedPorts(4566)
       .withEnv("SERVICES", "stepfunctions")
       .withReuse(true)

dd-java-agent/instrumentation/aws-java-sns-1.0/src/test/groovy/SnsClientTest.groovy

@@ -27,7 +27,7 @@ import java.time.Duration
 
 abstract class SnsClientTest extends VersionedNamingTestBase {
 
-  static final LOCALSTACK = new GenericContainer(DockerImageName.parse("localstack/localstack"))
+  static final LOCALSTACK = new GenericContainer(DockerImageName.parse("localstack/localstack:4.2.0"))
   .withExposedPorts(4566) // Default LocalStack port
   .withEnv("SERVICES", "sns,sqs") // Enable SNS and SQS service
   .withReuse(true)