diff --git a/.github/workflows/cpu-tests.yml b/.github/workflows/cpu-tests.yml index 431e82ccc7..c3f0127887 100644 --- a/.github/workflows/cpu-tests.yml +++ b/.github/workflows/cpu-tests.yml @@ -49,7 +49,7 @@ jobs: raw=$(git branch -r --contains ${{ github.ref_name }}) branch=${raw/origin\/} fi - tox -e test-cpu -- $branch + MERLIN_BRANCH=$branch tox -e test-cpu # Build docs, treat warnings as errors - name: Building docs diff --git a/.github/workflows/gpu-tests.yml b/.github/workflows/gpu-tests.yml index a12bdad57f..5b93fd9756 100644 --- a/.github/workflows/gpu-tests.yml +++ b/.github/workflows/gpu-tests.yml @@ -31,4 +31,4 @@ jobs: raw=$(git branch -r --contains ${{ github.ref_name }}) branch=${raw/origin\/} fi - cd ${{ github.workspace }}; tox -e test-gpu -- $branch + cd ${{ github.workspace }}; MERLIN_BRANCH=$branch tox -e test-gpu diff --git a/requirements/test.txt b/requirements/test.txt index 0a29deeed9..93b974e66a 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -19,7 +19,7 @@ s3fs>=2021.4 aiobotocore>=1.3.3 # required for synthetic data `merlin.datasets` and notebook tests using merlin models -merlin-models[tensorflow]@git+https://github.com/NVIDIA-Merlin/models.git +merlin-models[tensorflow] # needed to run notebook tests nest-asyncio diff --git a/tox.ini b/tox.ini index da0c1239a3..fd532cb1d1 100644 --- a/tox.ini +++ b/tox.ini @@ -16,13 +16,13 @@ setenv = ; Runs in: Github Actions ; Runs all CPU-based tests. NOTE: if you are using an M1 mac, this will fail. You need to ; change the tensorflow dependency to `tensorflow-macos` in requirements/test-cpu.txt. -deps = -rrequirements/test.txt +deps = + -rrequirements/test.txt + git+https://github.com/NVIDIA-Merlin/core.git@{env:MERLIN_BRANCH} + git+https://github.com/NVIDIA-Merlin/dataloader.git@{env:MERLIN_BRANCH} + git+https://github.com/NVIDIA-Merlin/models.git@{env:MERLIN_BRANCH} commands = - python -m pip install --upgrade git+https://github.com/NVIDIA-Merlin/core.git@{posargs:main} - python -m pip install --upgrade git+https://github.com/NVIDIA-Merlin/dataloader.git@{posargs:main} - python -m pip install --upgrade git+https://github.com/NVIDIA-Merlin/models.git@{posargs:main} - python -m pip install --upgrade git+https://github.com/NVIDIA-Merlin/core.git@{posargs:main} - python -m pytest --cov-report term --cov=merlin -rxs tests/unit + python -m pytest --cov-report term --cov=merlin -rxs {posargs:tests/unit} [testenv:test-gpu] setenv = @@ -32,21 +32,19 @@ passenv = NR_USER CUDA_VISIBLE_DEVICES sitepackages=true -; Runs in: Internal Jenkins +; Runs in: GitHub Actions ; Runs GPU-based tests. ; The jenkins jobs run on an image based on merlin-hugectr. This will include all cudf configuration ; and other gpu-specific libraries that we can enxpect will always exist. Thus, we don't need ; to install requirements.txt yet. As we get better at python environment isolation, we will ; need to add some back. deps = - pytest - pytest-cov + -rrequirements/test.txt + git+https://github.com/NVIDIA-Merlin/core.git@{env:MERLIN_BRANCH} + git+https://github.com/NVIDIA-Merlin/dataloader.git@{env:MERLIN_BRANCH} + git+https://github.com/NVIDIA-Merlin/models.git@{env:MERLIN_BRANCH} commands = - python -m pip install --upgrade git+https://github.com/NVIDIA-Merlin/core.git - python -m pip install --upgrade git+https://github.com/NVIDIA-Merlin/dataloader.git - python -m pip install --upgrade git+https://github.com/NVIDIA-Merlin/models.git - python -m pip install --upgrade git+https://github.com/NVIDIA-Merlin/core.git@{posargs:main} - python -m pytest --cov-report term --cov merlin -rxs tests/unit + python -m pytest --cov-report term --cov merlin -rxs {posargs:tests/unit} [testenv:test-merlin] ; Runs in: Internal Jenkins @@ -57,16 +55,13 @@ sitepackages=true allowlist_externals = git deps = -rrequirements/gpu.txt + git+https://github.com/NVIDIA-Merlin/core.git@{env:MERLIN_BRANCH} + git+https://github.com/NVIDIA-Merlin/dataloader.git@{env:MERLIN_BRANCH} + git+https://github.com/NVIDIA-Merlin/models.git@{env:MERLIN_BRANCH} + git+https://github.com/NVIDIA-Merlin/systems.git@{env:MERLIN_BRANCH} commands = ; the GIT_COMMIT env is the current commit of the systems repo ; NOTE!!!! We must clean this up in the jenkins configuration with `rm -rf "Merlin-$GIT_COMMIT"` - python -m pip install --upgrade git+https://github.com/NVIDIA-Merlin/core.git - python -m pip install --upgrade git+https://github.com/NVIDIA-Merlin/dataloader.git - python -m pip install --upgrade git+https://github.com/NVIDIA-Merlin/models.git - python -m pip install --upgrade git+https://github.com/NVIDIA-Merlin/systems.git - python -m pip install --upgrade git+https://github.com/NVIDIA-Merlin/core.git@{posargs:main} - python -m pip install . - git clone https://github.com/NVIDIA-Merlin/Merlin.git Merlin-{env:GIT_COMMIT} ; this runs the tests then removes the Merlin repo directory whether the tests work or fail @@ -77,17 +72,18 @@ commands = ; Generates documentation with sphinx. There are other steps in the Github Actions workflow ; to publish the documentation on release. changedir = {toxinidir} -deps = -rrequirements/docs.txt +deps = + -rrequirements/docs.txt + git+https://github.com/NVIDIA-Merlin/core.git commands = - python -m pip install --upgrade git+https://github.com/NVIDIA-Merlin/core.git python -m sphinx.cmd.build -P -b {posargs:html} docs/source docs/build/{posargs:html} [testenv:docs-multi] ; Run the multi-version build that is shown on GitHub Pages. changedir = {toxinidir} -deps = -rrequirements/docs.txt +deps = + -rrequirements/docs.txt + git+https://github.com/NVIDIA-Merlin/core.git commands = - python -m pip install --upgrade git+https://github.com/NVIDIA-Merlin/core.git sphinx-multiversion --dump-metadata docs/source docs/build/html | jq "keys" sphinx-multiversion docs/source docs/build/html -