From ccb23815c85d17636272959ce4d55cf3f73db5ee Mon Sep 17 00:00:00 2001 From: EugeneKisel-sm Date: Wed, 16 Jul 2025 15:42:50 +0200 Subject: [PATCH 01/14] Add test coverage --- .coveragerc | 17 +++++++++++++++++ .github/workflows/pull_request.yml | 11 +++++++++-- Dockerfile | 17 +++++++++++++++++ README.md | 1 + requirements.txt | 3 ++- 5 files changed, 46 insertions(+), 3 deletions(-) create mode 100644 .coveragerc diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 000000000..7eb8d3641 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,17 @@ +[run] +source = src/conductor +omit = + tests/* + examples/* + */__init__.py + +[report] +exclude_lines = + pragma: no cover + def __repr__ + raise NotImplementedError + if __name__ == .__main__.: + pass + raise ImportError + except ImportError: + if TYPE_CHECKING: \ No newline at end of file diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index c637bfa30..67eda9b42 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -8,11 +8,18 @@ jobs: steps: - name: Checkout uses: actions/checkout@v3 - - name: Test + + - name: Build and test with coverage run: > DOCKER_BUILDKIT=1 docker build . --progress=plain --build-arg CONDUCTOR_AUTH_KEY=${{ secrets.CONDUCTOR_AUTH_KEY }} --build-arg CONDUCTOR_AUTH_SECRET=${{ secrets.CONDUCTOR_AUTH_SECRET }} --build-arg CONDUCTOR_SERVER_URL=${{ secrets.CONDUCTOR_SERVER_URL }} - --target=unit_test \ No newline at end of file + --target=coverage_test + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v3 + with: + file: coverage.xml + fail_ci_if_error: true \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index fcde662df..cbe88d660 100644 --- a/Dockerfile +++ b/Dockerfile @@ -29,6 +29,23 @@ RUN python3 -m unittest discover --verbose --start-directory=./tests/serdesertes RUN coverage run --source=./src/conductor/client/orkes -m unittest discover --verbose --start-directory=./tests/integration RUN coverage report -m +FROM python_test_base as coverage_test +ARG CONDUCTOR_AUTH_KEY +ARG CONDUCTOR_AUTH_SECRET +ARG CONDUCTOR_SERVER_URL +ENV CONDUCTOR_AUTH_KEY=${CONDUCTOR_AUTH_KEY} +ENV CONDUCTOR_AUTH_SECRET=${CONDUCTOR_AUTH_SECRET} +ENV CONDUCTOR_SERVER_URL=${CONDUCTOR_SERVER_URL} +RUN ls -ltr +# Run unit, backward compatibility, and serialization tests with coverage +RUN coverage run -m unittest discover --verbose --start-directory=./tests/unit +RUN coverage run --append -m unittest discover --verbose --start-directory=./tests/backwardcompatibility +RUN coverage run --append -m unittest discover --verbose --start-directory=./tests/serdesertest +# Run integration tests with coverage +RUN coverage run --append --source=./src/conductor/client/orkes -m unittest discover --verbose --start-directory=./tests/integration +RUN coverage report -m +RUN coverage xml -o /package/coverage.xml + FROM python_test_base as test ARG CONDUCTOR_AUTH_KEY ARG CONDUCTOR_AUTH_SECRET diff --git a/README.md b/README.md index 3cc3842e3..2745595d8 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ # Conductor OSS Python SDK +[![codecov](https://codecov.io/gh/conductor-oss/python-sdk/branch/main/graph/badge.svg)](https://codecov.io/gh/conductor-oss/python-sdk) Python SDK for working with https://github.com/conductor-oss/conductor. diff --git a/requirements.txt b/requirements.txt index 07134be2a..214fddf7f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,4 +6,5 @@ typing-extensions >= 4.2.0 astor >= 0.8.1 shortuuid >= 1.0.11 dacite >= 1.8.1 -deprecated >= 1.2.14 \ No newline at end of file +deprecated >= 1.2.14 +pytest-cov>=4.1.0 \ No newline at end of file From 1af60220ff08fccbbb9605f47a1261bae34bb7be Mon Sep 17 00:00:00 2001 From: EugeneKisel-sm Date: Wed, 16 Jul 2025 15:58:29 +0200 Subject: [PATCH 02/14] Optimize pr.yml file --- .github/workflows/pull_request.yml | 32 ++++++++++++++++++++---------- Dockerfile | 19 ++++++++---------- 2 files changed, 30 insertions(+), 21 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 67eda9b42..b74b23c34 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -2,24 +2,36 @@ name: Continuous Integration on: [pull_request, workflow_dispatch] +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: unit-test: runs-on: ubuntu-latest + env: + COVERAGE_FILE: coverage.xml steps: - - name: Checkout + - name: Checkout code uses: actions/checkout@v3 - - name: Build and test with coverage - run: > - DOCKER_BUILDKIT=1 docker build . - --progress=plain - --build-arg CONDUCTOR_AUTH_KEY=${{ secrets.CONDUCTOR_AUTH_KEY }} - --build-arg CONDUCTOR_AUTH_SECRET=${{ secrets.CONDUCTOR_AUTH_SECRET }} - --build-arg CONDUCTOR_SERVER_URL=${{ secrets.CONDUCTOR_SERVER_URL }} - --target=coverage_test + - name: Build test image + run: | + DOCKER_BUILDKIT=1 docker build . \ + --target coverage_test \ + -t conductor-sdk-test:latest + + - name: Run tests with coverage + run: | + docker run --rm \ + -e CONDUCTOR_AUTH_KEY=${{ secrets.CONDUCTOR_AUTH_KEY }} \ + -e CONDUCTOR_AUTH_SECRET=${{ secrets.CONDUCTOR_AUTH_SECRET }} \ + -e CONDUCTOR_SERVER_URL=${{ secrets.CONDUCTOR_SERVER_URL }} \ + -v ${{ github.workspace }}/${{ env.COVERAGE_FILE }}:/package/${{ env.COVERAGE_FILE }} \ + conductor-sdk-test:latest - name: Upload coverage to Codecov uses: codecov/codecov-action@v3 with: - file: coverage.xml + file: ${{ env.COVERAGE_FILE }} fail_ci_if_error: true \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index cbe88d660..6b934c076 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,8 +12,8 @@ RUN ls -ltr ENV PYTHONPATH /package/src:/package/tests RUN python3 -m pip install pylint #RUN python3 -m pylint --disable=all ./src -RUN python3 -m pip install coverage -RUN python3 -m pip install -r ./requirements.txt +RUN python3 -m pip install coverage && \ + python3 -m pip install -r ./requirements.txt FROM python_test_base as unit_test ARG CONDUCTOR_AUTH_KEY @@ -36,15 +36,12 @@ ARG CONDUCTOR_SERVER_URL ENV CONDUCTOR_AUTH_KEY=${CONDUCTOR_AUTH_KEY} ENV CONDUCTOR_AUTH_SECRET=${CONDUCTOR_AUTH_SECRET} ENV CONDUCTOR_SERVER_URL=${CONDUCTOR_SERVER_URL} -RUN ls -ltr -# Run unit, backward compatibility, and serialization tests with coverage -RUN coverage run -m unittest discover --verbose --start-directory=./tests/unit -RUN coverage run --append -m unittest discover --verbose --start-directory=./tests/backwardcompatibility -RUN coverage run --append -m unittest discover --verbose --start-directory=./tests/serdesertest -# Run integration tests with coverage -RUN coverage run --append --source=./src/conductor/client/orkes -m unittest discover --verbose --start-directory=./tests/integration -RUN coverage report -m -RUN coverage xml -o /package/coverage.xml +CMD coverage run -m unittest discover --verbose --start-directory=./tests/unit && \ + coverage run --append -m unittest discover --verbose --start-directory=./tests/backwardcompatibility && \ + coverage run --append -m unittest discover --verbose --start-directory=./tests/serdesertest && \ + coverage run --append --source=./src/conductor/client/orkes -m unittest discover --verbose --start-directory=./tests/integration && \ + coverage report -m && \ + coverage xml -o /package/coverage.xml FROM python_test_base as test ARG CONDUCTOR_AUTH_KEY From 4e610117423a01a25895cee865be34f310d6f84b Mon Sep 17 00:00:00 2001 From: EugeneKisel-sm Date: Wed, 16 Jul 2025 16:20:54 +0200 Subject: [PATCH 03/14] Optimize pr.yml and add missing tokens --- .github/workflows/pull_request.yml | 11 +++++++++-- Dockerfile | 14 -------------- README.md | 3 ++- 3 files changed, 11 insertions(+), 17 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index b74b23c34..4e3e5058b 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -18,7 +18,7 @@ jobs: - name: Build test image run: | DOCKER_BUILDKIT=1 docker build . \ - --target coverage_test \ + --target python_test_base \ -t conductor-sdk-test:latest - name: Run tests with coverage @@ -28,10 +28,17 @@ jobs: -e CONDUCTOR_AUTH_SECRET=${{ secrets.CONDUCTOR_AUTH_SECRET }} \ -e CONDUCTOR_SERVER_URL=${{ secrets.CONDUCTOR_SERVER_URL }} \ -v ${{ github.workspace }}/${{ env.COVERAGE_FILE }}:/package/${{ env.COVERAGE_FILE }} \ - conductor-sdk-test:latest + conductor-sdk-test:latest \ + /bin/sh -c "coverage run -m unittest discover --verbose --start-directory=./tests/unit && \ + coverage run --append -m unittest discover --verbose --start-directory=./tests/backwardcompatibility && \ + coverage run --append -m unittest discover --verbose --start-directory=./tests/serdesertest && \ + coverage run --append --source=./src/conductor/client/orkes -m unittest discover --verbose --start-directory=./tests/integration && \ + coverage report -m && \ + coverage xml -o /package/coverage.xml" - name: Upload coverage to Codecov uses: codecov/codecov-action@v3 with: + token: ${{ secrets.CODECOV_TOKEN }} file: ${{ env.COVERAGE_FILE }} fail_ci_if_error: true \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 6b934c076..abc72456f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -29,20 +29,6 @@ RUN python3 -m unittest discover --verbose --start-directory=./tests/serdesertes RUN coverage run --source=./src/conductor/client/orkes -m unittest discover --verbose --start-directory=./tests/integration RUN coverage report -m -FROM python_test_base as coverage_test -ARG CONDUCTOR_AUTH_KEY -ARG CONDUCTOR_AUTH_SECRET -ARG CONDUCTOR_SERVER_URL -ENV CONDUCTOR_AUTH_KEY=${CONDUCTOR_AUTH_KEY} -ENV CONDUCTOR_AUTH_SECRET=${CONDUCTOR_AUTH_SECRET} -ENV CONDUCTOR_SERVER_URL=${CONDUCTOR_SERVER_URL} -CMD coverage run -m unittest discover --verbose --start-directory=./tests/unit && \ - coverage run --append -m unittest discover --verbose --start-directory=./tests/backwardcompatibility && \ - coverage run --append -m unittest discover --verbose --start-directory=./tests/serdesertest && \ - coverage run --append --source=./src/conductor/client/orkes -m unittest discover --verbose --start-directory=./tests/integration && \ - coverage report -m && \ - coverage xml -o /package/coverage.xml - FROM python_test_base as test ARG CONDUCTOR_AUTH_KEY ARG CONDUCTOR_AUTH_SECRET diff --git a/README.md b/README.md index 2745595d8..8120b2029 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,6 @@ # Conductor OSS Python SDK -[![codecov](https://codecov.io/gh/conductor-oss/python-sdk/branch/main/graph/badge.svg)](https://codecov.io/gh/conductor-oss/python-sdk) +[![CI Status](https://github.com/conductor-oss/python-sdk/actions/workflows/pull_request.yml/badge.svg)](https://github.com/conductor-oss/python-sdk/actions/workflows/pull_request.yml) +[![codecov](https://codecov.io/gh/conductor-oss/python-sdk/branch/main/graph/badge.svg?token=K10D161X4R)](https://codecov.io/gh/conductor-oss/python-sdk) Python SDK for working with https://github.com/conductor-oss/conductor. From 438301368ce2c87131e32bd544e6f2592af59c26 Mon Sep 17 00:00:00 2001 From: EugeneKisel-sm Date: Wed, 16 Jul 2025 16:27:52 +0200 Subject: [PATCH 04/14] Update yml to upload coverage even if tests failed --- .github/workflows/pull_request.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 4e3e5058b..0b9ea0fde 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -22,6 +22,8 @@ jobs: -t conductor-sdk-test:latest - name: Run tests with coverage + id: run_tests + continue-on-error: true run: | docker run --rm \ -e CONDUCTOR_AUTH_KEY=${{ secrets.CONDUCTOR_AUTH_KEY }} \ @@ -37,8 +39,13 @@ jobs: coverage xml -o /package/coverage.xml" - name: Upload coverage to Codecov + if: always() + continue-on-error: true uses: codecov/codecov-action@v3 with: token: ${{ secrets.CODECOV_TOKEN }} file: ${{ env.COVERAGE_FILE }} - fail_ci_if_error: true \ No newline at end of file + + - name: Check test results + if: steps.run_tests.outcome == 'failure' + run: exit 1 \ No newline at end of file From 60f51fe5d05ee85b0b387bba5060e14d74419392 Mon Sep 17 00:00:00 2001 From: EugeneKisel-sm Date: Wed, 16 Jul 2025 16:56:09 +0200 Subject: [PATCH 05/14] Fix coverage file --- .github/workflows/pull_request.yml | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 0b9ea0fde..ef869698f 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -13,7 +13,7 @@ jobs: COVERAGE_FILE: coverage.xml steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Build test image run: | @@ -21,6 +21,9 @@ jobs: --target python_test_base \ -t conductor-sdk-test:latest + - name: Prepare coverage file + run: touch ${{ env.COVERAGE_FILE }} + - name: Run tests with coverage id: run_tests continue-on-error: true @@ -29,18 +32,29 @@ jobs: -e CONDUCTOR_AUTH_KEY=${{ secrets.CONDUCTOR_AUTH_KEY }} \ -e CONDUCTOR_AUTH_SECRET=${{ secrets.CONDUCTOR_AUTH_SECRET }} \ -e CONDUCTOR_SERVER_URL=${{ secrets.CONDUCTOR_SERVER_URL }} \ - -v ${{ github.workspace }}/${{ env.COVERAGE_FILE }}:/package/${{ env.COVERAGE_FILE }} \ + -v ${{ github.workspace }}/${{ env.COVERAGE_FILE }}:/package/${{ env.COVERAGE_FILE }}:rw \ conductor-sdk-test:latest \ /bin/sh -c "coverage run -m unittest discover --verbose --start-directory=./tests/unit && \ coverage run --append -m unittest discover --verbose --start-directory=./tests/backwardcompatibility && \ coverage run --append -m unittest discover --verbose --start-directory=./tests/serdesertest && \ coverage run --append --source=./src/conductor/client/orkes -m unittest discover --verbose --start-directory=./tests/integration && \ coverage report -m && \ - coverage xml -o /package/coverage.xml" + coverage xml -o /package/${{ env.COVERAGE_FILE }}" - - name: Upload coverage to Codecov + - name: Verify coverage file + id: verify_coverage if: always() continue-on-error: true + run: | + if [ ! -s "${{ env.COVERAGE_FILE }}" ]; then + echo "Coverage file is empty or does not exist" + exit 1 + fi + echo "Coverage file exists and is not empty" + + - name: Upload coverage to Codecov + if: always() && steps.verify_coverage.outcome == 'success' + continue-on-error: true uses: codecov/codecov-action@v3 with: token: ${{ secrets.CODECOV_TOKEN }} From 95e2e460f781337608f45a8799e8be72b8864334 Mon Sep 17 00:00:00 2001 From: EugeneKisel-sm Date: Wed, 16 Jul 2025 17:17:22 +0200 Subject: [PATCH 06/14] Split tests by categories --- .github/workflows/pull_request.yml | 46 ++++++++++++++++++++++++------ 1 file changed, 37 insertions(+), 9 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index ef869698f..cde9e639a 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -24,8 +24,8 @@ jobs: - name: Prepare coverage file run: touch ${{ env.COVERAGE_FILE }} - - name: Run tests with coverage - id: run_tests + - name: Run unit tests + id: unit_tests continue-on-error: true run: | docker run --rm \ @@ -34,12 +34,40 @@ jobs: -e CONDUCTOR_SERVER_URL=${{ secrets.CONDUCTOR_SERVER_URL }} \ -v ${{ github.workspace }}/${{ env.COVERAGE_FILE }}:/package/${{ env.COVERAGE_FILE }}:rw \ conductor-sdk-test:latest \ - /bin/sh -c "coverage run -m unittest discover --verbose --start-directory=./tests/unit && \ - coverage run --append -m unittest discover --verbose --start-directory=./tests/backwardcompatibility && \ - coverage run --append -m unittest discover --verbose --start-directory=./tests/serdesertest && \ - coverage run --append --source=./src/conductor/client/orkes -m unittest discover --verbose --start-directory=./tests/integration && \ - coverage report -m && \ - coverage xml -o /package/${{ env.COVERAGE_FILE }}" + /bin/sh -c "coverage run -m unittest discover --verbose --start-directory=./tests/unit" + + - name: Run backward compatibility tests + id: bc_tests + continue-on-error: true + run: | + docker run --rm \ + -e CONDUCTOR_AUTH_KEY=${{ secrets.CONDUCTOR_AUTH_KEY }} \ + -e CONDUCTOR_AUTH_SECRET=${{ secrets.CONDUCTOR_AUTH_SECRET }} \ + -e CONDUCTOR_SERVER_URL=${{ secrets.CONDUCTOR_SERVER_URL }} \ + -v ${{ github.workspace }}/${{ env.COVERAGE_FILE }}:/package/${{ env.COVERAGE_FILE }}:rw \ + conductor-sdk-test:latest \ + /bin/sh -c "coverage run --append -m unittest discover --verbose --start-directory=./tests/backwardcompatibility" + + - name: Run serdeser tests + id: serdeser_tests + continue-on-error: true + run: | + docker run --rm \ + -e CONDUCTOR_AUTH_KEY=${{ secrets.CONDUCTOR_AUTH_KEY }} \ + -e CONDUCTOR_AUTH_SECRET=${{ secrets.CONDUCTOR_AUTH_SECRET }} \ + -e CONDUCTOR_SERVER_URL=${{ secrets.CONDUCTOR_SERVER_URL }} \ + -v ${{ github.workspace }}/${{ env.COVERAGE_FILE }}:/package/${{ env.COVERAGE_FILE }}:rw \ + conductor-sdk-test:latest \ + /bin/sh -c "coverage run --append -m unittest discover --verbose --start-directory=./tests/serdesertest" + + - name: Generate coverage report + id: coverage_report + continue-on-error: true + run: | + docker run --rm \ + -v ${{ github.workspace }}/${{ env.COVERAGE_FILE }}:/package/${{ env.COVERAGE_FILE }}:rw \ + conductor-sdk-test:latest \ + /bin/sh -c "coverage report -m && coverage xml -o /package/${{ env.COVERAGE_FILE }}" - name: Verify coverage file id: verify_coverage @@ -61,5 +89,5 @@ jobs: file: ${{ env.COVERAGE_FILE }} - name: Check test results - if: steps.run_tests.outcome == 'failure' + if: steps.unit_tests.outcome == 'failure' || steps.bc_tests.outcome == 'failure' || steps.serdeser_tests.outcome == 'failure' run: exit 1 \ No newline at end of file From 43fe071a1c7d2464fd4baac0fe5dc5ba795360bd Mon Sep 17 00:00:00 2001 From: EugeneKisel-sm Date: Wed, 16 Jul 2025 17:22:57 +0200 Subject: [PATCH 07/14] Fix coverage generation --- .github/workflows/pull_request.yml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index cde9e639a..82f0b280f 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -22,7 +22,9 @@ jobs: -t conductor-sdk-test:latest - name: Prepare coverage file - run: touch ${{ env.COVERAGE_FILE }} + run: | + touch ${{ env.COVERAGE_FILE }} + chmod 666 ${{ env.COVERAGE_FILE }} - name: Run unit tests id: unit_tests @@ -34,7 +36,7 @@ jobs: -e CONDUCTOR_SERVER_URL=${{ secrets.CONDUCTOR_SERVER_URL }} \ -v ${{ github.workspace }}/${{ env.COVERAGE_FILE }}:/package/${{ env.COVERAGE_FILE }}:rw \ conductor-sdk-test:latest \ - /bin/sh -c "coverage run -m unittest discover --verbose --start-directory=./tests/unit" + /bin/sh -c "cd /package && coverage run --source=src/conductor -m unittest discover --verbose --start-directory=./tests/unit" - name: Run backward compatibility tests id: bc_tests @@ -46,7 +48,7 @@ jobs: -e CONDUCTOR_SERVER_URL=${{ secrets.CONDUCTOR_SERVER_URL }} \ -v ${{ github.workspace }}/${{ env.COVERAGE_FILE }}:/package/${{ env.COVERAGE_FILE }}:rw \ conductor-sdk-test:latest \ - /bin/sh -c "coverage run --append -m unittest discover --verbose --start-directory=./tests/backwardcompatibility" + /bin/sh -c "cd /package && coverage run --append --source=src/conductor -m unittest discover --verbose --start-directory=./tests/backwardcompatibility" - name: Run serdeser tests id: serdeser_tests @@ -58,7 +60,7 @@ jobs: -e CONDUCTOR_SERVER_URL=${{ secrets.CONDUCTOR_SERVER_URL }} \ -v ${{ github.workspace }}/${{ env.COVERAGE_FILE }}:/package/${{ env.COVERAGE_FILE }}:rw \ conductor-sdk-test:latest \ - /bin/sh -c "coverage run --append -m unittest discover --verbose --start-directory=./tests/serdesertest" + /bin/sh -c "cd /package && coverage run --append --source=src/conductor -m unittest discover --verbose --start-directory=./tests/serdesertest" - name: Generate coverage report id: coverage_report @@ -67,7 +69,7 @@ jobs: docker run --rm \ -v ${{ github.workspace }}/${{ env.COVERAGE_FILE }}:/package/${{ env.COVERAGE_FILE }}:rw \ conductor-sdk-test:latest \ - /bin/sh -c "coverage report -m && coverage xml -o /package/${{ env.COVERAGE_FILE }}" + /bin/sh -c "cd /package && coverage report -m && coverage xml" - name: Verify coverage file id: verify_coverage @@ -76,6 +78,7 @@ jobs: run: | if [ ! -s "${{ env.COVERAGE_FILE }}" ]; then echo "Coverage file is empty or does not exist" + ls -l ${{ env.COVERAGE_FILE }} exit 1 fi echo "Coverage file exists and is not empty" From 939feb19be602452acd4d688f9bfe26ffd8aa9e7 Mon Sep 17 00:00:00 2001 From: EugeneKisel-sm Date: Wed, 16 Jul 2025 17:30:26 +0200 Subject: [PATCH 08/14] Fix coverage file generation --- .github/workflows/pull_request.yml | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 82f0b280f..fe5806f3c 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -11,6 +11,7 @@ jobs: runs-on: ubuntu-latest env: COVERAGE_FILE: coverage.xml + COVERAGE_DB: .coverage steps: - name: Checkout code uses: actions/checkout@v4 @@ -21,10 +22,12 @@ jobs: --target python_test_base \ -t conductor-sdk-test:latest - - name: Prepare coverage file + - name: Prepare coverage files run: | touch ${{ env.COVERAGE_FILE }} + touch ${{ env.COVERAGE_DB }} chmod 666 ${{ env.COVERAGE_FILE }} + chmod 666 ${{ env.COVERAGE_DB }} - name: Run unit tests id: unit_tests @@ -35,6 +38,7 @@ jobs: -e CONDUCTOR_AUTH_SECRET=${{ secrets.CONDUCTOR_AUTH_SECRET }} \ -e CONDUCTOR_SERVER_URL=${{ secrets.CONDUCTOR_SERVER_URL }} \ -v ${{ github.workspace }}/${{ env.COVERAGE_FILE }}:/package/${{ env.COVERAGE_FILE }}:rw \ + -v ${{ github.workspace }}/${{ env.COVERAGE_DB }}:/package/${{ env.COVERAGE_DB }}:rw \ conductor-sdk-test:latest \ /bin/sh -c "cd /package && coverage run --source=src/conductor -m unittest discover --verbose --start-directory=./tests/unit" @@ -47,6 +51,7 @@ jobs: -e CONDUCTOR_AUTH_SECRET=${{ secrets.CONDUCTOR_AUTH_SECRET }} \ -e CONDUCTOR_SERVER_URL=${{ secrets.CONDUCTOR_SERVER_URL }} \ -v ${{ github.workspace }}/${{ env.COVERAGE_FILE }}:/package/${{ env.COVERAGE_FILE }}:rw \ + -v ${{ github.workspace }}/${{ env.COVERAGE_DB }}:/package/${{ env.COVERAGE_DB }}:rw \ conductor-sdk-test:latest \ /bin/sh -c "cd /package && coverage run --append --source=src/conductor -m unittest discover --verbose --start-directory=./tests/backwardcompatibility" @@ -59,6 +64,7 @@ jobs: -e CONDUCTOR_AUTH_SECRET=${{ secrets.CONDUCTOR_AUTH_SECRET }} \ -e CONDUCTOR_SERVER_URL=${{ secrets.CONDUCTOR_SERVER_URL }} \ -v ${{ github.workspace }}/${{ env.COVERAGE_FILE }}:/package/${{ env.COVERAGE_FILE }}:rw \ + -v ${{ github.workspace }}/${{ env.COVERAGE_DB }}:/package/${{ env.COVERAGE_DB }}:rw \ conductor-sdk-test:latest \ /bin/sh -c "cd /package && coverage run --append --source=src/conductor -m unittest discover --verbose --start-directory=./tests/serdesertest" @@ -68,8 +74,9 @@ jobs: run: | docker run --rm \ -v ${{ github.workspace }}/${{ env.COVERAGE_FILE }}:/package/${{ env.COVERAGE_FILE }}:rw \ + -v ${{ github.workspace }}/${{ env.COVERAGE_DB }}:/package/${{ env.COVERAGE_DB }}:rw \ conductor-sdk-test:latest \ - /bin/sh -c "cd /package && coverage report -m && coverage xml" + /bin/sh -c "cd /package && coverage report && coverage xml && cat coverage.xml" - name: Verify coverage file id: verify_coverage @@ -78,10 +85,11 @@ jobs: run: | if [ ! -s "${{ env.COVERAGE_FILE }}" ]; then echo "Coverage file is empty or does not exist" - ls -l ${{ env.COVERAGE_FILE }} + ls -la ${{ env.COVERAGE_FILE }} ${{ env.COVERAGE_DB }} exit 1 fi echo "Coverage file exists and is not empty" + cat ${{ env.COVERAGE_FILE }} - name: Upload coverage to Codecov if: always() && steps.verify_coverage.outcome == 'success' From 70c393d7125ba6db6ca74dfc63361877fec995bb Mon Sep 17 00:00:00 2001 From: EugeneKisel-sm Date: Wed, 16 Jul 2025 17:36:21 +0200 Subject: [PATCH 09/14] Remove extra logging --- .github/workflows/pull_request.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index fe5806f3c..b497e9156 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -76,7 +76,7 @@ jobs: -v ${{ github.workspace }}/${{ env.COVERAGE_FILE }}:/package/${{ env.COVERAGE_FILE }}:rw \ -v ${{ github.workspace }}/${{ env.COVERAGE_DB }}:/package/${{ env.COVERAGE_DB }}:rw \ conductor-sdk-test:latest \ - /bin/sh -c "cd /package && coverage report && coverage xml && cat coverage.xml" + /bin/sh -c "cd /package && coverage report && coverage xml" - name: Verify coverage file id: verify_coverage @@ -89,7 +89,6 @@ jobs: exit 1 fi echo "Coverage file exists and is not empty" - cat ${{ env.COVERAGE_FILE }} - name: Upload coverage to Codecov if: always() && steps.verify_coverage.outcome == 'success' From bdbc1e945f11825fe016d4e66fa155e235a44571 Mon Sep 17 00:00:00 2001 From: EugeneKisel-sm Date: Wed, 16 Jul 2025 17:45:14 +0200 Subject: [PATCH 10/14] Fix multiple access coverage issue --- .github/workflows/pull_request.yml | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index b497e9156..973206e1f 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest env: COVERAGE_FILE: coverage.xml - COVERAGE_DB: .coverage + COVERAGE_DIR: .coverage-reports steps: - name: Checkout code uses: actions/checkout@v4 @@ -22,12 +22,12 @@ jobs: --target python_test_base \ -t conductor-sdk-test:latest - - name: Prepare coverage files + - name: Prepare coverage directory run: | + mkdir -p ${{ env.COVERAGE_DIR }} + chmod 777 ${{ env.COVERAGE_DIR }} touch ${{ env.COVERAGE_FILE }} - touch ${{ env.COVERAGE_DB }} chmod 666 ${{ env.COVERAGE_FILE }} - chmod 666 ${{ env.COVERAGE_DB }} - name: Run unit tests id: unit_tests @@ -37,10 +37,9 @@ jobs: -e CONDUCTOR_AUTH_KEY=${{ secrets.CONDUCTOR_AUTH_KEY }} \ -e CONDUCTOR_AUTH_SECRET=${{ secrets.CONDUCTOR_AUTH_SECRET }} \ -e CONDUCTOR_SERVER_URL=${{ secrets.CONDUCTOR_SERVER_URL }} \ - -v ${{ github.workspace }}/${{ env.COVERAGE_FILE }}:/package/${{ env.COVERAGE_FILE }}:rw \ - -v ${{ github.workspace }}/${{ env.COVERAGE_DB }}:/package/${{ env.COVERAGE_DB }}:rw \ + -v ${{ github.workspace }}/${{ env.COVERAGE_DIR }}:/package/${{ env.COVERAGE_DIR }}:rw \ conductor-sdk-test:latest \ - /bin/sh -c "cd /package && coverage run --source=src/conductor -m unittest discover --verbose --start-directory=./tests/unit" + /bin/sh -c "cd /package && coverage run --parallel-mode --source=src/conductor -m unittest discover --verbose --start-directory=./tests/unit" - name: Run backward compatibility tests id: bc_tests @@ -50,10 +49,9 @@ jobs: -e CONDUCTOR_AUTH_KEY=${{ secrets.CONDUCTOR_AUTH_KEY }} \ -e CONDUCTOR_AUTH_SECRET=${{ secrets.CONDUCTOR_AUTH_SECRET }} \ -e CONDUCTOR_SERVER_URL=${{ secrets.CONDUCTOR_SERVER_URL }} \ - -v ${{ github.workspace }}/${{ env.COVERAGE_FILE }}:/package/${{ env.COVERAGE_FILE }}:rw \ - -v ${{ github.workspace }}/${{ env.COVERAGE_DB }}:/package/${{ env.COVERAGE_DB }}:rw \ + -v ${{ github.workspace }}/${{ env.COVERAGE_DIR }}:/package/${{ env.COVERAGE_DIR }}:rw \ conductor-sdk-test:latest \ - /bin/sh -c "cd /package && coverage run --append --source=src/conductor -m unittest discover --verbose --start-directory=./tests/backwardcompatibility" + /bin/sh -c "cd /package && coverage run --parallel-mode --source=src/conductor -m unittest discover --verbose --start-directory=./tests/backwardcompatibility" - name: Run serdeser tests id: serdeser_tests @@ -63,20 +61,19 @@ jobs: -e CONDUCTOR_AUTH_KEY=${{ secrets.CONDUCTOR_AUTH_KEY }} \ -e CONDUCTOR_AUTH_SECRET=${{ secrets.CONDUCTOR_AUTH_SECRET }} \ -e CONDUCTOR_SERVER_URL=${{ secrets.CONDUCTOR_SERVER_URL }} \ - -v ${{ github.workspace }}/${{ env.COVERAGE_FILE }}:/package/${{ env.COVERAGE_FILE }}:rw \ - -v ${{ github.workspace }}/${{ env.COVERAGE_DB }}:/package/${{ env.COVERAGE_DB }}:rw \ + -v ${{ github.workspace }}/${{ env.COVERAGE_DIR }}:/package/${{ env.COVERAGE_DIR }}:rw \ conductor-sdk-test:latest \ - /bin/sh -c "cd /package && coverage run --append --source=src/conductor -m unittest discover --verbose --start-directory=./tests/serdesertest" + /bin/sh -c "cd /package && coverage run --parallel-mode --source=src/conductor -m unittest discover --verbose --start-directory=./tests/serdesertest" - name: Generate coverage report id: coverage_report continue-on-error: true run: | docker run --rm \ + -v ${{ github.workspace }}/${{ env.COVERAGE_DIR }}:/package/${{ env.COVERAGE_DIR }}:rw \ -v ${{ github.workspace }}/${{ env.COVERAGE_FILE }}:/package/${{ env.COVERAGE_FILE }}:rw \ - -v ${{ github.workspace }}/${{ env.COVERAGE_DB }}:/package/${{ env.COVERAGE_DB }}:rw \ conductor-sdk-test:latest \ - /bin/sh -c "cd /package && coverage report && coverage xml" + /bin/sh -c "cd /package && coverage combine ${{ env.COVERAGE_DIR }}/.coverage* && coverage report && coverage xml" - name: Verify coverage file id: verify_coverage @@ -85,7 +82,7 @@ jobs: run: | if [ ! -s "${{ env.COVERAGE_FILE }}" ]; then echo "Coverage file is empty or does not exist" - ls -la ${{ env.COVERAGE_FILE }} ${{ env.COVERAGE_DB }} + ls -la ${{ env.COVERAGE_FILE }} ${{ env.COVERAGE_DIR }} exit 1 fi echo "Coverage file exists and is not empty" From 1aa0c0f174495a18cda579284cd0fd2a53913fbb Mon Sep 17 00:00:00 2001 From: EugeneKisel-sm Date: Wed, 16 Jul 2025 18:11:39 +0200 Subject: [PATCH 11/14] Fix report generation --- .github/workflows/pull_request.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 973206e1f..1296d452c 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -39,7 +39,7 @@ jobs: -e CONDUCTOR_SERVER_URL=${{ secrets.CONDUCTOR_SERVER_URL }} \ -v ${{ github.workspace }}/${{ env.COVERAGE_DIR }}:/package/${{ env.COVERAGE_DIR }}:rw \ conductor-sdk-test:latest \ - /bin/sh -c "cd /package && coverage run --parallel-mode --source=src/conductor -m unittest discover --verbose --start-directory=./tests/unit" + /bin/sh -c "cd /package && COVERAGE_FILE=/package/${{ env.COVERAGE_DIR }}/.coverage.unit coverage run --source=src/conductor -m unittest discover --verbose --start-directory=./tests/unit" - name: Run backward compatibility tests id: bc_tests @@ -51,7 +51,7 @@ jobs: -e CONDUCTOR_SERVER_URL=${{ secrets.CONDUCTOR_SERVER_URL }} \ -v ${{ github.workspace }}/${{ env.COVERAGE_DIR }}:/package/${{ env.COVERAGE_DIR }}:rw \ conductor-sdk-test:latest \ - /bin/sh -c "cd /package && coverage run --parallel-mode --source=src/conductor -m unittest discover --verbose --start-directory=./tests/backwardcompatibility" + /bin/sh -c "cd /package && COVERAGE_FILE=/package/${{ env.COVERAGE_DIR }}/.coverage.bc coverage run --source=src/conductor -m unittest discover --verbose --start-directory=./tests/backwardcompatibility" - name: Run serdeser tests id: serdeser_tests @@ -63,7 +63,7 @@ jobs: -e CONDUCTOR_SERVER_URL=${{ secrets.CONDUCTOR_SERVER_URL }} \ -v ${{ github.workspace }}/${{ env.COVERAGE_DIR }}:/package/${{ env.COVERAGE_DIR }}:rw \ conductor-sdk-test:latest \ - /bin/sh -c "cd /package && coverage run --parallel-mode --source=src/conductor -m unittest discover --verbose --start-directory=./tests/serdesertest" + /bin/sh -c "cd /package && COVERAGE_FILE=/package/${{ env.COVERAGE_DIR }}/.coverage.serdeser coverage run --source=src/conductor -m unittest discover --verbose --start-directory=./tests/serdesertest" - name: Generate coverage report id: coverage_report @@ -73,7 +73,7 @@ jobs: -v ${{ github.workspace }}/${{ env.COVERAGE_DIR }}:/package/${{ env.COVERAGE_DIR }}:rw \ -v ${{ github.workspace }}/${{ env.COVERAGE_FILE }}:/package/${{ env.COVERAGE_FILE }}:rw \ conductor-sdk-test:latest \ - /bin/sh -c "cd /package && coverage combine ${{ env.COVERAGE_DIR }}/.coverage* && coverage report && coverage xml" + /bin/sh -c "cd /package && coverage combine /package/${{ env.COVERAGE_DIR }}/.coverage.* && coverage report && coverage xml" - name: Verify coverage file id: verify_coverage From 9c9c44d33060727755c9e482d577b633ef3e08d8 Mon Sep 17 00:00:00 2001 From: EugeneKisel-sm Date: Wed, 16 Jul 2025 18:34:23 +0200 Subject: [PATCH 12/14] Create requirements.dev.txt --- Dockerfile | 6 ++---- requirements.dev.txt | 3 +++ requirements.txt | 3 +-- 3 files changed, 6 insertions(+), 6 deletions(-) create mode 100644 requirements.dev.txt diff --git a/Dockerfile b/Dockerfile index abc72456f..37e9a6616 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,10 +10,8 @@ WORKDIR /package RUN pwd RUN ls -ltr ENV PYTHONPATH /package/src:/package/tests -RUN python3 -m pip install pylint -#RUN python3 -m pylint --disable=all ./src -RUN python3 -m pip install coverage && \ - python3 -m pip install -r ./requirements.txt +RUN python3 -m pip install -r ./requirements.txt && \ + python3 -m pip install -r ./requirements.dev.txt FROM python_test_base as unit_test ARG CONDUCTOR_AUTH_KEY diff --git a/requirements.dev.txt b/requirements.dev.txt new file mode 100644 index 000000000..b9570bda0 --- /dev/null +++ b/requirements.dev.txt @@ -0,0 +1,3 @@ +pylint>=2.17.5 +coverage>=7.3.2 +pytest-cov>=4.1.0 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 214fddf7f..07134be2a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,5 +6,4 @@ typing-extensions >= 4.2.0 astor >= 0.8.1 shortuuid >= 1.0.11 dacite >= 1.8.1 -deprecated >= 1.2.14 -pytest-cov>=4.1.0 \ No newline at end of file +deprecated >= 1.2.14 \ No newline at end of file From 622ed5f66a28c7cfe6251f6731ffcb07ead7e319 Mon Sep 17 00:00:00 2001 From: EugeneKisel-sm Date: Wed, 16 Jul 2025 18:41:51 +0200 Subject: [PATCH 13/14] Remove extra spaces --- requirements.txt | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/requirements.txt b/requirements.txt index 07134be2a..4f9f96be4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,9 @@ -certifi >= 14.05.14 -prometheus-client >= 0.13.1 -six >= 1.10 -requests >= 2.31.0 -typing-extensions >= 4.2.0 -astor >= 0.8.1 -shortuuid >= 1.0.11 -dacite >= 1.8.1 -deprecated >= 1.2.14 \ No newline at end of file +certifi>=14.05.14 +prometheus-client>=0.13.1 +six>=1.10 +requests>=2.31.0 +typing-extensions>=4.2.0 +astor>=0.8.1 +shortuuid>=1.0.11 +dacite>=1.8.1 +deprecated>=1.2.14 \ No newline at end of file From 325a2655b1fc376511e0d2a7e54bce940c1e9ac2 Mon Sep 17 00:00:00 2001 From: EugeneKisel-sm Date: Wed, 16 Jul 2025 18:45:11 +0200 Subject: [PATCH 14/14] Fix build issue --- requirements.dev.txt | 6 +++--- requirements.txt | 18 +++++++++--------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/requirements.dev.txt b/requirements.dev.txt index b9570bda0..5884736cb 100644 --- a/requirements.dev.txt +++ b/requirements.dev.txt @@ -1,3 +1,3 @@ -pylint>=2.17.5 -coverage>=7.3.2 -pytest-cov>=4.1.0 \ No newline at end of file +pylint >= 2.17.5 +coverage >= 7.2.7 +pytest-cov >= 4.1.0 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 4f9f96be4..07134be2a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,9 @@ -certifi>=14.05.14 -prometheus-client>=0.13.1 -six>=1.10 -requests>=2.31.0 -typing-extensions>=4.2.0 -astor>=0.8.1 -shortuuid>=1.0.11 -dacite>=1.8.1 -deprecated>=1.2.14 \ No newline at end of file +certifi >= 14.05.14 +prometheus-client >= 0.13.1 +six >= 1.10 +requests >= 2.31.0 +typing-extensions >= 4.2.0 +astor >= 0.8.1 +shortuuid >= 1.0.11 +dacite >= 1.8.1 +deprecated >= 1.2.14 \ No newline at end of file