From b80b12dd7f1613294b79b7d87a1ee4ebb3646766 Mon Sep 17 00:00:00 2001 From: Santhosh Gandhe <1909520+san81@users.noreply.github.com> Date: Wed, 11 Jun 2025 12:29:25 -0700 Subject: [PATCH 1/7] along with the profile, setting additional AWS credentials through environment variables as they are not picked up in different dockers we are packaging. Signed-off-by: Santhosh Gandhe <1909520+san81@users.noreply.github.com> --- e2e-test/log/build.gradle | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/e2e-test/log/build.gradle b/e2e-test/log/build.gradle index 60aa05a181..69c3083f4d 100644 --- a/e2e-test/log/build.gradle +++ b/e2e-test/log/build.gradle @@ -79,12 +79,16 @@ logTestConfigurations.each { testConfiguration -> exposePorts('tcp', [2021, 4900]) hostConfig.portBindings = ['2021:2021', '4900:4900'] hostConfig.binds = [ - "${System.getProperty('user.home')}/.aws" : '/root/.aws', + "${System.getProperty('user.home')}/.aws" : '/root/.aws', (project.file("src/integrationTest/resources/${testConfiguration.pipelineConfiguration}").toString()) : '/usr/share/data-prepper/pipelines/log-pipeline.yaml', (project.file("src/integrationTest/resources/${testConfiguration.dataPrepperConfiguration}").toString()): '/usr/share/data-prepper/config/data-prepper-config.yaml' ] envVars = [ - 'AWS_PROFILE' : System.getenv('AWS_PROFILE') ?: '' + 'AWS_PROFILE' : System.getenv('AWS_PROFILE') ?: '', + 'AWS_ACCESS_KEY_ID' : System.getenv('AWS_ACCESS_KEY_ID') ?: '', + 'AWS_SECRET_ACCESS_KEY': System.getenv('AWS_SECRET_ACCESS_KEY') ?: '', + 'AWS_SESSION_TOKEN' : System.getenv('AWS_SESSION_TOKEN') ?: '', + 'AWS_REGION' : System.getenv('AWS_REGION') ?: 'us-east-2' ] hostConfig.network = createDataPrepperNetwork.getNetworkName() targetImageId dataPrepperDockerImage.imageId From cb91b3519d290edbadff9eec59fcef8766cebd78 Mon Sep 17 00:00:00 2001 From: Santhosh Gandhe <1909520+san81@users.noreply.github.com> Date: Wed, 11 Jun 2025 13:37:48 -0700 Subject: [PATCH 2/7] Trying to pass the credentials in a different approach Signed-off-by: Santhosh Gandhe <1909520+san81@users.noreply.github.com> --- .github/workflows/data-prepper-aws-secrets-e2e-tests.yml | 2 +- e2e-test/log/build.gradle | 8 ++------ 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/.github/workflows/data-prepper-aws-secrets-e2e-tests.yml b/.github/workflows/data-prepper-aws-secrets-e2e-tests.yml index 26247886f8..6b08138aa5 100644 --- a/.github/workflows/data-prepper-aws-secrets-e2e-tests.yml +++ b/.github/workflows/data-prepper-aws-secrets-e2e-tests.yml @@ -50,4 +50,4 @@ jobs: - name: Checkout Data Prepper uses: actions/checkout@v2 - name: Run basic grok with AWS secrets end-to-end tests with Gradle - run: AWS_PROFILE=default ./gradlew -PendToEndJavaVersion=${{ matrix.java }} :e2e-test:log:${{ matrix.test }} + run: AWS_ACCESS_KEY_ID=${{ steps.creds.outputs.aws-access-key-id }} AWS_SECRET_ACCESS_KEY=${{ steps.creds.outputs.aws-secret-access-key }} AWS_SESSION_TOKEN=${{ steps.creds.outputs.aws-session-token }} AWS_REGION=${{ secrets.TEST_REGION }} ./gradlew -PendToEndJavaVersion=${{ matrix.java }} :e2e-test:log:${{ matrix.test }} diff --git a/e2e-test/log/build.gradle b/e2e-test/log/build.gradle index 69c3083f4d..60aa05a181 100644 --- a/e2e-test/log/build.gradle +++ b/e2e-test/log/build.gradle @@ -79,16 +79,12 @@ logTestConfigurations.each { testConfiguration -> exposePorts('tcp', [2021, 4900]) hostConfig.portBindings = ['2021:2021', '4900:4900'] hostConfig.binds = [ - "${System.getProperty('user.home')}/.aws" : '/root/.aws', + "${System.getProperty('user.home')}/.aws" : '/root/.aws', (project.file("src/integrationTest/resources/${testConfiguration.pipelineConfiguration}").toString()) : '/usr/share/data-prepper/pipelines/log-pipeline.yaml', (project.file("src/integrationTest/resources/${testConfiguration.dataPrepperConfiguration}").toString()): '/usr/share/data-prepper/config/data-prepper-config.yaml' ] envVars = [ - 'AWS_PROFILE' : System.getenv('AWS_PROFILE') ?: '', - 'AWS_ACCESS_KEY_ID' : System.getenv('AWS_ACCESS_KEY_ID') ?: '', - 'AWS_SECRET_ACCESS_KEY': System.getenv('AWS_SECRET_ACCESS_KEY') ?: '', - 'AWS_SESSION_TOKEN' : System.getenv('AWS_SESSION_TOKEN') ?: '', - 'AWS_REGION' : System.getenv('AWS_REGION') ?: 'us-east-2' + 'AWS_PROFILE' : System.getenv('AWS_PROFILE') ?: '' ] hostConfig.network = createDataPrepperNetwork.getNetworkName() targetImageId dataPrepperDockerImage.imageId From f1f6cfb6ac4f825a2011fed8eab8143a281a6039 Mon Sep 17 00:00:00 2001 From: Santhosh Gandhe <1909520+san81@users.noreply.github.com> Date: Wed, 11 Jun 2025 14:43:10 -0700 Subject: [PATCH 3/7] Trying again with build time environment variables Signed-off-by: Santhosh Gandhe <1909520+san81@users.noreply.github.com> --- .github/workflows/data-prepper-aws-secrets-e2e-tests.yml | 2 +- e2e-test/log/build.gradle | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/data-prepper-aws-secrets-e2e-tests.yml b/.github/workflows/data-prepper-aws-secrets-e2e-tests.yml index 6b08138aa5..26247886f8 100644 --- a/.github/workflows/data-prepper-aws-secrets-e2e-tests.yml +++ b/.github/workflows/data-prepper-aws-secrets-e2e-tests.yml @@ -50,4 +50,4 @@ jobs: - name: Checkout Data Prepper uses: actions/checkout@v2 - name: Run basic grok with AWS secrets end-to-end tests with Gradle - run: AWS_ACCESS_KEY_ID=${{ steps.creds.outputs.aws-access-key-id }} AWS_SECRET_ACCESS_KEY=${{ steps.creds.outputs.aws-secret-access-key }} AWS_SESSION_TOKEN=${{ steps.creds.outputs.aws-session-token }} AWS_REGION=${{ secrets.TEST_REGION }} ./gradlew -PendToEndJavaVersion=${{ matrix.java }} :e2e-test:log:${{ matrix.test }} + run: AWS_PROFILE=default ./gradlew -PendToEndJavaVersion=${{ matrix.java }} :e2e-test:log:${{ matrix.test }} diff --git a/e2e-test/log/build.gradle b/e2e-test/log/build.gradle index 60aa05a181..69c3083f4d 100644 --- a/e2e-test/log/build.gradle +++ b/e2e-test/log/build.gradle @@ -79,12 +79,16 @@ logTestConfigurations.each { testConfiguration -> exposePorts('tcp', [2021, 4900]) hostConfig.portBindings = ['2021:2021', '4900:4900'] hostConfig.binds = [ - "${System.getProperty('user.home')}/.aws" : '/root/.aws', + "${System.getProperty('user.home')}/.aws" : '/root/.aws', (project.file("src/integrationTest/resources/${testConfiguration.pipelineConfiguration}").toString()) : '/usr/share/data-prepper/pipelines/log-pipeline.yaml', (project.file("src/integrationTest/resources/${testConfiguration.dataPrepperConfiguration}").toString()): '/usr/share/data-prepper/config/data-prepper-config.yaml' ] envVars = [ - 'AWS_PROFILE' : System.getenv('AWS_PROFILE') ?: '' + 'AWS_PROFILE' : System.getenv('AWS_PROFILE') ?: '', + 'AWS_ACCESS_KEY_ID' : System.getenv('AWS_ACCESS_KEY_ID') ?: '', + 'AWS_SECRET_ACCESS_KEY': System.getenv('AWS_SECRET_ACCESS_KEY') ?: '', + 'AWS_SESSION_TOKEN' : System.getenv('AWS_SESSION_TOKEN') ?: '', + 'AWS_REGION' : System.getenv('AWS_REGION') ?: 'us-east-2' ] hostConfig.network = createDataPrepperNetwork.getNetworkName() targetImageId dataPrepperDockerImage.imageId From 5933c856433852cb3b297255784a0ec43a287234 Mon Sep 17 00:00:00 2001 From: Santhosh Gandhe <1909520+san81@users.noreply.github.com> Date: Wed, 11 Jun 2025 15:05:37 -0700 Subject: [PATCH 4/7] enabling debugging Signed-off-by: Santhosh Gandhe <1909520+san81@users.noreply.github.com> --- .github/workflows/data-prepper-aws-secrets-e2e-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/data-prepper-aws-secrets-e2e-tests.yml b/.github/workflows/data-prepper-aws-secrets-e2e-tests.yml index 26247886f8..b92eb4db25 100644 --- a/.github/workflows/data-prepper-aws-secrets-e2e-tests.yml +++ b/.github/workflows/data-prepper-aws-secrets-e2e-tests.yml @@ -50,4 +50,4 @@ jobs: - name: Checkout Data Prepper uses: actions/checkout@v2 - name: Run basic grok with AWS secrets end-to-end tests with Gradle - run: AWS_PROFILE=default ./gradlew -PendToEndJavaVersion=${{ matrix.java }} :e2e-test:log:${{ matrix.test }} + run: AWS_PROFILE=default ./gradlew -PendToEndJavaVersion=${{ matrix.java }} :e2e-test:log:${{ matrix.test }} --debug From 7ca08d38e946811c406a53f648528de5bd2a5713 Mon Sep 17 00:00:00 2001 From: Santhosh Gandhe <1909520+san81@users.noreply.github.com> Date: Wed, 11 Jun 2025 15:30:04 -0700 Subject: [PATCH 5/7] keeping untouched Signed-off-by: Santhosh Gandhe <1909520+san81@users.noreply.github.com> --- .github/workflows/data-prepper-aws-secrets-e2e-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/data-prepper-aws-secrets-e2e-tests.yml b/.github/workflows/data-prepper-aws-secrets-e2e-tests.yml index b92eb4db25..26247886f8 100644 --- a/.github/workflows/data-prepper-aws-secrets-e2e-tests.yml +++ b/.github/workflows/data-prepper-aws-secrets-e2e-tests.yml @@ -50,4 +50,4 @@ jobs: - name: Checkout Data Prepper uses: actions/checkout@v2 - name: Run basic grok with AWS secrets end-to-end tests with Gradle - run: AWS_PROFILE=default ./gradlew -PendToEndJavaVersion=${{ matrix.java }} :e2e-test:log:${{ matrix.test }} --debug + run: AWS_PROFILE=default ./gradlew -PendToEndJavaVersion=${{ matrix.java }} :e2e-test:log:${{ matrix.test }} From 3cfb0eddfcf5624ffca9b91c8c38b1d33906f5fd Mon Sep 17 00:00:00 2001 From: Santhosh Gandhe <1909520+san81@users.noreply.github.com> Date: Wed, 11 Jun 2025 16:18:14 -0700 Subject: [PATCH 6/7] Trying upload test artifacts to better understand the failure reason Signed-off-by: Santhosh Gandhe <1909520+san81@users.noreply.github.com> --- .../workflows/data-prepper-aws-secrets-e2e-tests.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/data-prepper-aws-secrets-e2e-tests.yml b/.github/workflows/data-prepper-aws-secrets-e2e-tests.yml index 26247886f8..534254893e 100644 --- a/.github/workflows/data-prepper-aws-secrets-e2e-tests.yml +++ b/.github/workflows/data-prepper-aws-secrets-e2e-tests.yml @@ -8,7 +8,7 @@ on: pull_request_target: types: [ opened, synchronize, reopened ] workflow_dispatch: -# permission can be added at job level or workflow level +# permission can be added at job level or workflow level permissions: id-token: write # This is required for requesting the JWT contents: read # This is required for actions/checkout @@ -51,3 +51,12 @@ jobs: uses: actions/checkout@v2 - name: Run basic grok with AWS secrets end-to-end tests with Gradle run: AWS_PROFILE=default ./gradlew -PendToEndJavaVersion=${{ matrix.java }} :e2e-test:log:${{ matrix.test }} + - name: Upload Test Reports + if: always() + uses: actions/upload-artifact@v3 + with: + name: test-reports-${{ matrix.java }} + path: | + **/build/reports/tests/ + **/build/test-results/ + retention-days: 7 \ No newline at end of file From e6e2caaab4c1a4be6bb11484c9bb6043d26da50e Mon Sep 17 00:00:00 2001 From: Santhosh Gandhe <1909520+san81@users.noreply.github.com> Date: Wed, 11 Jun 2025 16:55:15 -0700 Subject: [PATCH 7/7] printing the docker inspect to make sure the docker image got created as expected Signed-off-by: Santhosh Gandhe <1909520+san81@users.noreply.github.com> --- e2e-test/build.gradle | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/e2e-test/build.gradle b/e2e-test/build.gradle index cf5feed97b..fa0ca7d64e 100644 --- a/e2e-test/build.gradle +++ b/e2e-test/build.gradle @@ -91,6 +91,13 @@ subprojects { else if(targetJavaVersion == 'docker') { dependsOn ':release:docker:docker' imageId = "${project.rootProject.name}:${project.version}" + doLast { + println "\n=== Inspecting Docker Image: ${project.rootProject.name}:${project.version} ===" + exec { + commandLine 'docker', 'inspect', "${project.rootProject.name}:${project.version}" + } + println "=== End of Docker Inspect ===" + } } else { dependsOn 'createDataPrepperDockerFile' @@ -145,4 +152,13 @@ subprojects { dependencies { testImplementation testLibs.junit.vintage } + + tasks.register('inspectDataPrepperImage') { + dependsOn dataPrepperDockerImage + doLast { + exec { + commandLine 'docker', 'inspect', "${project.rootProject.name}:${project.version}" + } + } + } }