Disable FAIL_ON_UNKNOWN_PROPERTIES in SqsMessageParser ObjectMapper (… #28
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # | |
| # Copyright OpenSearch Contributors | |
| # SPDX-License-Identifier: Apache-2.0 | |
| # | |
| # The OpenSearch Contributors require contributions made to | |
| # this file be licensed under the Apache-2.0 license or a | |
| # compatible open source license. | |
| # | |
| name: Integration Tests - OpenSearch sink against OpenSearch | |
| on: | |
| push: | |
| paths: | |
| - 'data-prepper-plugins/opensearch/**' | |
| - '*gradle*' | |
| pull_request: | |
| paths: | |
| - 'data-prepper-plugins/opensearch/**' | |
| - '*gradle*' | |
| workflow_dispatch: | |
| jobs: | |
| integration-tests: | |
| strategy: | |
| matrix: | |
| java: [11] | |
| opensearch: | |
| - 1.0.1 | |
| - 1.1.0 | |
| - 1.2.4 | |
| - 1.3.20 | |
| - 2.0.1 | |
| - 2.1.0 | |
| - 2.3.0 | |
| - 2.5.0 | |
| - 2.7.0 | |
| - 2.9.0 | |
| - 2.11.1 | |
| - 2.15.0 | |
| - 2.19.5 | |
| - 3.0.0 | |
| - 3.5.0 | |
| include: | |
| - opensearch: 2.0.1 | |
| runner: ubuntu-22.04 | |
| - opensearch: 2.1.0 | |
| runner: ubuntu-22.04 | |
| - opensearch: 2.3.0 | |
| runner: ubuntu-22.04 | |
| fail-fast: false | |
| runs-on: ${{ matrix.runner || 'ubuntu-latest' }} | |
| steps: | |
| - name: Set up JDK ${{ matrix.java }} | |
| uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0 | |
| with: | |
| distribution: temurin | |
| java-version: ${{ matrix.java }} | |
| - name: Checkout Data Prepper | |
| uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 | |
| - name: Set up Gradle | |
| uses: gradle/actions/setup-gradle@50e97c2cd7a37755bbfafc9c5b7cafaece252f6e # v6.1.0 | |
| with: | |
| add-job-summary: on-failure | |
| - name: Set password for OpenSearch version | |
| run: | | |
| if [[ "${{ matrix.opensearch }}" =~ ^(2\.(1[2-9]|[2-9][0-9])|[3-9]\.) ]]; then | |
| echo "PASSWORD=yourStrongPassword123!" >> $GITHUB_ENV | |
| else | |
| echo "PASSWORD=admin" >> $GITHUB_ENV | |
| fi | |
| - name: Run OpenSearch docker | |
| run: | | |
| docker pull opensearchproject/opensearch:${{ matrix.opensearch }} | |
| docker run -p 9200:9200 -p 9600:9600 -e "discovery.type=single-node" -e 'OPENSEARCH_INITIAL_ADMIN_PASSWORD=yourStrongPassword123!' -d opensearchproject/opensearch:${{ matrix.opensearch }} | |
| sleep 1 | |
| - name: Wait for OpenSearch to be available | |
| run: | | |
| ./gradlew :data-prepper-plugins:opensearch:integrationTest --tests "org.opensearch.dataprepper.plugins.sink.opensearch.OpenSearchIT" -Dtests.opensearch.host=localhost:9200 -Dtests.opensearch.user=admin -Dtests.opensearch.password="$PASSWORD" | |
| - name: Run OpenSearch tests | |
| run: | | |
| ./gradlew :data-prepper-plugins:opensearch:integrationTest -Dtests.opensearch.host=localhost:9200 -Dtests.opensearch.user=admin -Dtests.opensearch.password="$PASSWORD" -Dtests.opensearch.bundle=true -Dtests.opensearch.version=opensearch:${{ matrix.opensearch }} | |
| - name: Upload Unit Test Results | |
| if: always() | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 | |
| with: | |
| name: data-prepper-opensearch-integration-tests-opensearch-${{ matrix.opensearch }}-java-${{ matrix.java }} | |
| path: '**/test-results/**/*.xml' | |
| mtls-integration-tests: | |
| strategy: | |
| matrix: | |
| java: [11] | |
| opensearch: | |
| - 1.3.20 | |
| - 2.19.5 | |
| - 3.5.0 | |
| fail-fast: false | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Set up JDK ${{ matrix.java }} | |
| uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0 | |
| with: | |
| distribution: temurin | |
| java-version: ${{ matrix.java }} | |
| - name: Checkout Data Prepper | |
| uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 | |
| - name: Set up Gradle | |
| uses: gradle/actions/setup-gradle@50e97c2cd7a37755bbfafc9c5b7cafaece252f6e # v6.1.0 | |
| with: | |
| add-job-summary: on-failure | |
| - name: Generate client certificates | |
| run: | | |
| openssl req -x509 -newkey rsa:2048 -keyout ca-key.pem -out ca-cert.pem -days 1 -nodes -subj "/CN=TestCA" | |
| openssl req -newkey rsa:2048 -keyout client-key.pem -out client-csr.pem -nodes -subj "/CN=DataPrepper" | |
| openssl x509 -req -in client-csr.pem -CA ca-cert.pem -CAkey ca-key.pem -CAcreateserial -out client-cert.pem -days 1 | |
| rm -f client-csr.pem ca-cert.srl | |
| - name: Set password for OpenSearch version | |
| run: | | |
| if [[ "${{ matrix.opensearch }}" =~ ^(2\.(1[2-9]|[2-9][0-9])|[3-9]\.) ]]; then | |
| echo "PASSWORD=yourStrongPassword123!" >> $GITHUB_ENV | |
| else | |
| echo "PASSWORD=admin" >> $GITHUB_ENV | |
| fi | |
| - name: Start OpenSearch with demo config | |
| run: | | |
| docker run -d --name opensearch-mtls -p 9200:9200 -p 9600:9600 \ | |
| -e "discovery.type=single-node" \ | |
| -e "OPENSEARCH_INITIAL_ADMIN_PASSWORD=yourStrongPassword123!" \ | |
| opensearchproject/opensearch:${{ matrix.opensearch }} | |
| - name: Wait for OpenSearch to be available | |
| run: | | |
| for i in $(seq 1 120); do | |
| if curl -sk -u admin:"$PASSWORD" https://localhost:9200 >/dev/null 2>&1; then | |
| echo "OpenSearch ready after ${i} seconds" | |
| break | |
| fi | |
| sleep 1 | |
| done | |
| - name: Enable mTLS on OpenSearch | |
| run: | | |
| docker cp ca-cert.pem opensearch-mtls:/usr/share/opensearch/config/ | |
| docker exec opensearch-mtls bash -c " | |
| CONFIG=/usr/share/opensearch/config/opensearch.yml | |
| if grep -q 'plugins.security' \$CONFIG; then | |
| PREFIX=plugins.security | |
| else | |
| PREFIX=opendistro_security | |
| fi | |
| if grep -q 'ssl.http.clientauth_mode' \$CONFIG; then | |
| sed -i 's/ssl.http.clientauth_mode:.*/ssl.http.clientauth_mode: REQUIRE/' \$CONFIG | |
| else | |
| echo \"\${PREFIX}.ssl.http.clientauth_mode: REQUIRE\" >> \$CONFIG | |
| fi | |
| if grep -q 'ssl.http.pemtrustedcas_filepath' \$CONFIG; then | |
| sed -i \"s|ssl.http.pemtrustedcas_filepath:.*|ssl.http.pemtrustedcas_filepath: /usr/share/opensearch/config/ca-cert.pem|\" \$CONFIG | |
| else | |
| echo \"\${PREFIX}.ssl.http.pemtrustedcas_filepath: /usr/share/opensearch/config/ca-cert.pem\" >> \$CONFIG | |
| fi | |
| " | |
| docker restart opensearch-mtls | |
| - name: Wait for mTLS OpenSearch to restart | |
| run: | | |
| sleep 10 | |
| for i in $(seq 1 120); do | |
| if curl -sk --cert client-cert.pem --key client-key.pem -u admin:"$PASSWORD" https://localhost:9200 >/dev/null 2>&1; then | |
| echo "mTLS OpenSearch ready after ${i} seconds" | |
| break | |
| fi | |
| sleep 1 | |
| done | |
| - name: Run mTLS integration tests | |
| run: | | |
| ./gradlew :data-prepper-plugins:opensearch:integrationTest \ | |
| --tests "org.opensearch.dataprepper.plugins.sink.opensearch.OpenSearchClientCertIT" \ | |
| -Dtests.opensearch.host=localhost:9200 \ | |
| -Dtests.opensearch.user=admin \ | |
| -Dtests.opensearch.password="$PASSWORD" \ | |
| -Dtests.mtls.client.cert=$PWD/client-cert.pem \ | |
| -Dtests.mtls.client.key=$PWD/client-key.pem | |
| - name: Upload Test Results | |
| if: always() | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 | |
| with: | |
| name: data-prepper-opensearch-mtls-integration-tests-${{ matrix.opensearch }}-java-${{ matrix.java }} | |
| path: '**/test-results/**/*.xml' | |
| publish-test-results: | |
| name: "Publish Unit Tests Results" | |
| needs: [integration-tests, mtls-integration-tests] | |
| runs-on: ubuntu-latest | |
| if: always() | |
| steps: | |
| - name: Download Artifacts | |
| uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 | |
| with: | |
| path: test-results | |
| - name: Publish Unit Test Results | |
| uses: EnricoMi/publish-unit-test-result-action@b9f6c61d965bcaa18acc02d6daf706373a448f02 # v1 | |
| with: | |
| files: "test-results/**/*.xml" |