Support running the end-to-end tests against defined Docker images. Resolves #3567. (#5711)

Signed-off-by: David Venable <dlv@amazon.com>

Author: dlvenable

e2e-test/README.md

@@ -1,3 +1,26 @@
 # Data Prepper End-to-end Tests
 
 This module includes all e2e tests for data-prepper.
+
+## Running against a specific image
+
+The end-to-end tests can run against a specific remote or local image.
+
+Use the following Gradle build parameters:
+
+* `endToEndDataPrepperImage` - specify the image
+* `endToEndDataPrepperTag` - specify the tag
+
+### Running from a released image
+
+This example shows running from the DockerHub image using version 2.11.0.
+
+```shell
+./gradlew -PendToEndDataPrepperImage=opensearchproject/data-prepper -PendToEndDataPrepperTag=2.11.0 :e2e-test:log:basicLogEndToEndTest
+```
+
+This example shows running from the ECR image using version 2.11.0
+
+```shell
+./gradlew -PendToEndDataPrepperImage=public.ecr.aws/opensearchproject/data-prepper -PendToEndDataPrepperTag=2.11.0 :e2e-test:peerforwarder:localAggregateEndToEndTest
+```

e2e-test/build.gradle

@@ -27,6 +27,10 @@ subprojects {
         targetJavaVersion = project.hasProperty('endToEndJavaVersion') ? project.getProperty('endToEndJavaVersion') : 'docker'
         targetOpenTelemetryVersion = project.hasProperty('openTelemetryVersion') ? project.getProperty('openTelemetryVersion') : "${libs.versions.opentelemetry.get()}"
         dataPrepperBaseImage = "eclipse-temurin:${targetJavaVersion}-jre"
+        dataPrepperDefinedDockerImage =
+                project.hasProperty('endToEndDataPrepperImage') && project.hasProperty('endToEndDataPrepperTag') ?
+                        "${project.getProperty('endToEndDataPrepperImage')}:${project.getProperty('endToEndDataPrepperTag')}" :
+                        null;
     }
 
     sourceSets {
@@ -75,9 +79,16 @@ subprojects {
         images.add('e2e-test-data-prepper')
     }
 
+    tasks.register('pullDataPrepperDockerImage', DockerPullImage) {
+        image = "${dataPrepperDefinedDockerImage}"
+    }
 
     tasks.register('dataPrepperDockerImage', DockerProviderTask) {
-        if(targetJavaVersion == 'docker') {
+        if(dataPrepperDefinedDockerImage != null) {
+            dependsOn 'pullDataPrepperDockerImage'
+            imageId = "${dataPrepperDefinedDockerImage}"
+        }
+        else if(targetJavaVersion == 'docker') {
             dependsOn ':release:docker:docker'
             imageId = "${project.rootProject.name}:${project.version}"
         }

e2e-test/log/build.gradle

@@ -108,7 +108,7 @@ logTestConfigurations.each { testConfiguration ->
     }
 
     tasks.register(testConfiguration.testName, Test) {
-        dependsOn build
+        dependsOn compileJava
         dependsOn startOpenSearchDockerContainer
         dependsOn "start${testConfiguration.testName}"
 

e2e-test/trace/build.gradle

@@ -58,12 +58,12 @@ def DATA_PREPPER_CONFIG_STATIC_YAML = 'data_prepper_static.yml'
 def RELEASED_DATA_PREPPER_DOCKER_IMAGE = 'opensearchproject/data-prepper:latest'
 
 
-tasks.register('pullDataPrepperDockerImage', DockerPullImage) {
+tasks.register('pullLatestDataPrepperDockerImage', DockerPullImage) {
     image = RELEASED_DATA_PREPPER_DOCKER_IMAGE
 }
 
 tasks.register('latestDataPrepperDockerImage', DockerProviderTask) {
-    dependsOn 'pullDataPrepperDockerImage'
+    dependsOn 'pullLatestDataPrepperDockerImage'
     imageId = RELEASED_DATA_PREPPER_DOCKER_IMAGE
 }