Skip to content

Commit 9b881fc

Browse files
karthiknadigkarthiknadig
authored
Move coverage to nightly and simplify reporting (#15776)
* Enable coverage for venv and single workspace tests * Upload coverage for single workspace * Try with spawn wrap * Fix coverage artifact upload * Try running tests with coverage directly * Run coverage separately * Remove start page from coverage * Add back codecov * No need to move codecov.yml * Ensure nightly coverage runs on schedule only * Nightly should only run coverage * Address comments * Remove setting node version env for each job
1 parent 6bb8c69 commit 9b881fc

File tree

8 files changed

+375
-218
lines changed

8 files changed

+375
-218
lines changed

‎.github/codecov.yml‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,3 @@ comment:
2929
layout: 'diff, files'
3030
behavior: default
3131
require_changes: no
32-
33-
fixes:
34-
- 'path with spaces/::'

‎.github/workflows/insiders.yml‎

Lines changed: 13 additions & 120 deletions
Original file line numberDiff line numberDiff line change
@@ -9,11 +9,8 @@ env:
99
NODE_VERSION: 12.15.0
1010
PYTHON_VERSION: 3.9
1111
MOCHA_REPORTER_JUNIT: true # Use the mocha-multi-reporters and send output to both console (spec) and JUnit (mocha-junit-reporter). Also enables a reporter which exits the process running the tests if it haven't already.
12-
# Key for the cache created at the end of the the 'Cache ./pythonFiles/lib/python' step.
13-
CACHE_PYTHONFILES: cache-pvsc-pythonFiles
1412
ARTIFACT_NAME_VSIX: ms-python-insiders-vsix
1513
VSIX_NAME: ms-python-insiders.vsix
16-
COVERAGE_REPORTS: tests-coverage-reports
1714
TEST_RESULTS_DIRECTORY: .
1815
# Force a path with spaces and to test extension works in these scenarios
1916
# Unicode characters are causing 2.7 failures so skip that for now.
@@ -100,12 +97,6 @@ jobs:
10097
name: Tests
10198
# The value of runs-on is the OS of the current job (specified in the strategy matrix below) instead of being hardcoded.
10299
runs-on: ${{ matrix.os }}
103-
env:
104-
# Something in Node 12.16.0 breaks the TS debug adapter, and ubuntu-latest bundles Node 12.16.1.
105-
# We can remove this when we switch over to the python-based DA in https://github.com/microsoft/vscode-python/issues/7136.
106-
# See https://github.com/microsoft/ptvsd/issues/2068
107-
# At this point pinning is only needed for consistency. We no longer have TS debug adapter.
108-
NODE_VERSION: 12.15.0
109100
defaults:
110101
run:
111102
working-directory: ${{env.special-working-directory}}
@@ -170,9 +161,7 @@ jobs:
170161
if: matrix.test-suite == 'debugger' && matrix.python == 2.7
171162

172163
- name: Install functional test requirements
173-
run: |
174-
python -m pip install numpy
175-
python -m pip install --upgrade -r ./build/functional-test-requirements.txt
164+
run: python -m pip install --upgrade -r ./build/functional-test-requirements.txt
176165
if: matrix.test-suite == 'functional'
177166

178167
- name: Prepare pipenv for venv tests
@@ -247,16 +236,7 @@ jobs:
247236

248237
# Run TypeScript unit tests only for Python 3.X.
249238
- name: Run TypeScript unit tests
250-
run: npm run test:unittests:cover
251-
if: matrix.test-suite == 'ts-unit' && startsWith(matrix.python, 3.)
252-
253-
# Upload unit test coverage reports for later use in the "reports" job.
254-
- name: Upload unit test coverage reports
255-
uses: actions/upload-artifact@v2
256-
with:
257-
name: ${{runner.os}}-${{env.COVERAGE_REPORTS}}
258-
path: ${{ env.special-working-directory }}/.nyc_output
259-
retention-days: 1
239+
run: npm run test:unittests
260240
if: matrix.test-suite == 'ts-unit' && startsWith(matrix.python, 3.)
261241

262242
# Run the Python tests in our codebase.
@@ -310,16 +290,7 @@ jobs:
310290

311291
# Run TypeScript functional tests
312292
- name: Run TypeScript functional tests
313-
run: npm run test:functional:cover
314-
if: matrix.test-suite == 'functional'
315-
316-
# Upload functional test coverage reports for later use in the "reports" job.
317-
- name: Upload functional test coverage reports
318-
uses: actions/upload-artifact@v2
319-
with:
320-
name: ${{runner.os}}-${{env.COVERAGE_REPORTS}}
321-
path: ${{ env.special-working-directory }}/.nyc_output
322-
retention-days: 1
293+
run: npm run test:functional
323294
if: matrix.test-suite == 'functional'
324295

325296
smoke-tests:
@@ -444,92 +415,14 @@ jobs:
444415
echo "CI_DISABLE_AUTO_SELECTION=1" >> $GITHUB_ENV
445416
shell: bash
446417

447-
# - name: Run insider tests
448-
# env:
449-
# DISPLAY: 10
450-
# INSTALL_JUPYTER_EXTENSION: true
451-
# INSTALL_PYLANCE_EXTENSION: true
452-
# VSC_PYTHON_CI_TEST_VSC_CHANNEL: insiders
453-
# TEST_FILES_SUFFIX: insiders.test
454-
# CODE_TESTS_WORKSPACE: ./src/testMultiRootWkspc/smokeTests
455-
# uses: GabrielBB/xvfb-action@v1.4
456-
# with:
457-
# run: node --no-force-async-hooks-checks ./out/test/standardTest.js
458-
459-
coverage:
460-
name: Coverage reports upload
461-
runs-on: ubuntu-latest
462-
if: github.repository == 'microsoft/vscode-python'
463-
needs: [tests, smoke-tests]
464-
defaults:
465-
run:
466-
working-directory: ${{env.special-working-directory}}
467-
steps:
468-
# Checkout the repo in the same way as how the tests are run,
469-
# or nyc won't know how to reconcile the reports generated by the tests.
470-
- name: Checkout
471-
uses: actions/checkout@v2
472-
with:
473-
path: ${{env.special-working-directory-relative}}
474-
475-
- name: Install Node
476-
uses: actions/setup-node@v2.1.5
477-
with:
478-
node-version: ${{env.NODE_VERSION}}
479-
480-
- name: Install dependencies (npm ci)
481-
run: npm ci --prefer-offline
482-
483-
# It isn't possible to specify a regex for artifact names, so we have to download each artifact manually.
484-
# The name pattern is ${{runner.os}}-${{env.COVERAGE_REPORTS}}, and possible values for runner.os are `Linux`, `Windows`, or `macOS`.
485-
# See https://help.github.com/en/actions/reference/contexts-and-expression-syntax-for-github-actions#runner-context
486-
- name: Download Ubuntu test coverage artifacts
487-
uses: actions/download-artifact@v2
488-
with:
489-
name: Linux-${{ env.COVERAGE_REPORTS }}
490-
path: ${{env.special-working-directory}}/Linux-${{ env.COVERAGE_REPORTS }}
491-
492-
- name: Extract Ubuntu coverage artifacts to ./nyc_output
493-
run: |
494-
mkdir .nyc_output
495-
mv Linux-${{ env.COVERAGE_REPORTS }}/* .nyc_output
496-
rm -r Linux-${{ env.COVERAGE_REPORTS }}
497-
498-
- name: Generate coverage reports
499-
run: npm run test:cover:report
500-
continue-on-error: true
501-
502-
# The only location supported by the codecov action is the workspace root.
503-
# See https://github.com/codecov/codecov-action#arguments
504-
- name: Move codecov.yml to root
505-
run: |
506-
mkdir $GITHUB_WORKSPACE/.github
507-
mv .github/codecov.yml $GITHUB_WORKSPACE
508-
509-
- name: Upload coverage to codecov
510-
uses: codecov/codecov-action@v1
511-
with:
512-
token: ${{ secrets.CODECOV_TOKEN }}
513-
file: ${{env.special-working-directory}}/coverage/cobertura-coverage.xml
514-
515-
upload:
516-
name: Upload VSIX to Azure Blob Storage
517-
if: github.repository == 'microsoft/vscode-python'
518-
runs-on: ubuntu-latest
519-
needs: [tests, smoke-tests, build-vsix]
520-
env:
521-
BLOB_CONTAINER_NAME: extension-builds
522-
BLOB_NAME: ms-python-insiders.vsix
523-
steps:
524-
- name: Download VSIX
525-
uses: actions/download-artifact@v2
526-
with:
527-
name: ${{ env.ARTIFACT_NAME_VSIX }}
528-
- name: Azure Login
529-
uses: azure/login@v1
418+
- name: Run insider tests
419+
env:
420+
DISPLAY: 10
421+
INSTALL_JUPYTER_EXTENSION: true
422+
INSTALL_PYLANCE_EXTENSION: true
423+
VSC_PYTHON_CI_TEST_VSC_CHANNEL: insiders
424+
TEST_FILES_SUFFIX: insiders.test
425+
CODE_TESTS_WORKSPACE: ./src/testMultiRootWkspc/smokeTests
426+
uses: GabrielBB/xvfb-action@v1.4
530427
with:
531-
creds: ${{ secrets.AZURE_CREDENTIALS }}
532-
- name: Upload to Blob Storage
533-
run: az storage blob upload --file ${{ env.VSIX_NAME }} --account-name pvsc --container-name ${{ env.BLOB_CONTAINER_NAME }} --name ${{ env.BLOB_NAME }} --auth-mode login
534-
- name: Get URL to uploaded VSIX
535-
run: az storage blob url --account-name pvsc --container-name ${{ env.BLOB_CONTAINER_NAME }} --name ${{ env.BLOB_NAME }} --auth-mode login
428+
run: node --no-force-async-hooks-checks ./out/test/standardTest.js

‎.github/workflows/nightly-coverage.yml‎

Lines changed: 188 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,188 @@
1+
name: Coverage
2+
3+
on:
4+
schedule:
5+
# Run daily at 2:00 AM
6+
- cron: '0 2 * * *'
7+
8+
env:
9+
NODE_VERSION: 12.15.0
10+
PYTHON_VERSION: 3.9
11+
MOCHA_REPORTER_JUNIT: true # Use the mocha-multi-reporters and send output to both console (spec) and JUnit (mocha-junit-reporter). Also enables a reporter which exits the process running the tests if it haven't already.
12+
ARTIFACT_NAME_VSIX: ms-python-insiders-vsix
13+
VSIX_NAME: ms-python-insiders.vsix
14+
TEST_RESULTS_DIRECTORY: .
15+
16+
jobs:
17+
tests:
18+
name: Tests with Coverage
19+
# The value of runs-on is the OS of the current job (specified in the strategy matrix below) instead of being hardcoded.
20+
runs-on: ${{ matrix.os }}
21+
22+
strategy:
23+
fail-fast: false
24+
matrix:
25+
# We're not running CI on macOS for now because it's one less matrix entry to lower the number of runners used,
26+
# macOS runners are expensive, and we assume that Ubuntu is enough to cover the Unix case.
27+
os: [ubuntu-latest, windows-latest]
28+
29+
steps:
30+
- name: Checkout
31+
uses: actions/checkout@v2
32+
33+
- name: Install Node
34+
uses: actions/setup-node@v2.1.5
35+
with:
36+
node-version: ${{env.NODE_VERSION}}
37+
38+
- name: Install dependencies (npm ci)
39+
run: npm ci
40+
41+
- name: Compile
42+
run: npx gulp prePublishNonBundle
43+
44+
- name: Use Python ${{env.PYTHON_VERSION}}
45+
uses: actions/setup-python@v2
46+
with:
47+
python-version: ${{env.PYTHON_VERSION}}
48+
49+
- name: Install Python requirements
50+
run: |
51+
python -m pip --disable-pip-version-check install -t ./pythonFiles/lib/python --no-cache-dir --implementation py --no-deps --upgrade -r requirements.txt --no-user
52+
# We need to have debugpy so that tests relying on it keep passing, but we don't need install_debugpy's logic in the test phase.
53+
python -m pip --disable-pip-version-check install -t ./pythonFiles/lib/python --no-cache-dir --implementation py --no-deps --upgrade --pre debugpy
54+
55+
- name: Install test requirements
56+
run: python -m pip install --upgrade -r build/test-requirements.txt
57+
58+
- name: Install functional test requirements
59+
run: python -m pip install --upgrade -r ./build/functional-test-requirements.txt
60+
61+
- name: Prepare pipenv for venv tests
62+
env:
63+
TEST_FILES_SUFFIX: testvirtualenvs
64+
PYTHON_VIRTUAL_ENVS_LOCATION: './src/tmp/envPaths.json'
65+
shell: pwsh
66+
run: |
67+
python -m pip install pipenv
68+
python -m pipenv run python ./build/ci/addEnvPath.py ${{env.PYTHON_VIRTUAL_ENVS_LOCATION}} pipenvPath
69+
70+
- name: Prepare virtualenv for venv tests
71+
env:
72+
TEST_FILES_SUFFIX: testvirtualenvs
73+
PYTHON_VIRTUAL_ENVS_LOCATION: './src/tmp/envPaths.json'
74+
shell: pwsh
75+
run: |
76+
python -m pip install virtualenv
77+
python -m virtualenv .virtualenv/
78+
if ('${{matrix.os}}' -match 'windows-latest') {
79+
& ".virtualenv/Scripts/python.exe" ./build/ci/addEnvPath.py ${{env.PYTHON_VIRTUAL_ENVS_LOCATION}} virtualEnvPath
80+
} else {
81+
& ".virtualenv/bin/python" ./build/ci/addEnvPath.py ${{env.PYTHON_VIRTUAL_ENVS_LOCATION}} virtualEnvPath
82+
}
83+
84+
- name: Prepare venv for venv tests
85+
env:
86+
TEST_FILES_SUFFIX: testvirtualenvs
87+
PYTHON_VIRTUAL_ENVS_LOCATION: './src/tmp/envPaths.json'
88+
shell: pwsh
89+
run: |
90+
python -m venv .venv
91+
if ('${{matrix.os}}' -match 'windows-latest') {
92+
& ".venv/Scripts/python.exe" ./build/ci/addEnvPath.py ${{env.PYTHON_VIRTUAL_ENVS_LOCATION}} venvPath
93+
} else {
94+
& ".venv/bin/python" ./build/ci/addEnvPath.py ${{env.PYTHON_VIRTUAL_ENVS_LOCATION}} venvPath
95+
}
96+
97+
- name: Prepare conda for venv tests
98+
env:
99+
TEST_FILES_SUFFIX: testvirtualenvs
100+
PYTHON_VIRTUAL_ENVS_LOCATION: './src/tmp/envPaths.json'
101+
shell: pwsh
102+
run: |
103+
# 1. For `terminalActivation.testvirtualenvs.test.ts`
104+
if ('${{matrix.os}}' -match 'windows-latest') {
105+
$condaPythonPath = Join-Path -Path $Env:CONDA -ChildPath python.exe
106+
$condaExecPath = Join-Path -Path $Env:CONDA -ChildPath Scripts | Join-Path -ChildPath conda
107+
} else{
108+
$condaPythonPath = Join-Path -Path $Env:CONDA -ChildPath bin | Join-Path -ChildPath python
109+
$condaExecPath = Join-Path -Path $Env:CONDA -ChildPath bin | Join-Path -ChildPath conda
110+
}
111+
& $condaPythonPath ./build/ci/addEnvPath.py ${{env.PYTHON_VIRTUAL_ENVS_LOCATION}} condaExecPath $condaExecPath
112+
& $condaPythonPath ./build/ci/addEnvPath.py ${{env.PYTHON_VIRTUAL_ENVS_LOCATION}} condaPath
113+
114+
# 2. For `interpreterLocatorService.testvirtualenvs.ts`
115+
116+
& $condaExecPath create -n "test_env1" -y python
117+
& $condaExecPath create -p "./test_env2" -y python
118+
& $condaExecPath create -p "~/test_env3" -y python
119+
120+
- name: Run TypeScript unit tests
121+
run: npm run test:unittests:cover
122+
123+
- name: Run Python unit tests
124+
run: |
125+
python pythonFiles/tests/run_all.py
126+
127+
# The virtual environment based tests use the `testSingleWorkspace` set of tests
128+
# with the environment variable `TEST_FILES_SUFFIX` set to `testvirtualenvs`,
129+
# which is set in the "Prepare environment for venv tests" step.
130+
# We also use a third-party GitHub Action to install xvfb on Linux,
131+
# run tests and then clean up the process once the tests ran.
132+
# See https://github.com/GabrielBB/xvfb-action
133+
- name: Run venv tests
134+
env:
135+
TEST_FILES_SUFFIX: testvirtualenvs
136+
CI_PYTHON_VERSION: ${{env.PYTHON_VERSION}}
137+
CI_DISABLE_AUTO_SELECTION: 1
138+
uses: GabrielBB/xvfb-action@v1.4
139+
with:
140+
run: npm run testSingleWorkspace:cover
141+
142+
- name: Run single-workspace tests
143+
env:
144+
CI_PYTHON_VERSION: ${{env.PYTHON_VERSION}}
145+
CI_DISABLE_AUTO_SELECTION: 1
146+
uses: GabrielBB/xvfb-action@v1.4
147+
with:
148+
run: npm run testSingleWorkspace:cover
149+
150+
# Enable these tests when coverage is setup for multiroot workspace tests
151+
# - name: Run multi-workspace tests
152+
# env:
153+
# CI_PYTHON_VERSION: ${{env.PYTHON_VERSION}}
154+
# CI_DISABLE_AUTO_SELECTION: 1
155+
# uses: GabrielBB/xvfb-action@v1.4
156+
# with:
157+
# run: npm run testMultiWorkspace:cover
158+
159+
# Enable these tests when coverage is setup for debugger tests
160+
# - name: Run debugger tests
161+
# env:
162+
# CI_PYTHON_VERSION: ${{env.PYTHON_VERSION}}
163+
# CI_DISABLE_AUTO_SELECTION: 1
164+
# uses: GabrielBB/xvfb-action@v1.4
165+
# with:
166+
# run: npm run testDebugger:cover
167+
168+
- name: Run TypeScript functional tests
169+
env:
170+
CI_PYTHON_VERSION: ${{env.PYTHON_VERSION}}
171+
CI_DISABLE_AUTO_SELECTION: 1
172+
run: npm run test:functional:cover
173+
174+
- name: Generate coverage reports
175+
run: npm run test:cover:report
176+
177+
- name: Upload HTML report
178+
uses: actions/upload-artifact@v2
179+
with:
180+
name: ${{ runner.os }}-coverage-report-html
181+
path: ./coverage
182+
retention-days: 1
183+
184+
- name: Upload coverage to codecov
185+
uses: codecov/codecov-action@v1
186+
with:
187+
token: ${{ secrets.CODECOV_TOKEN }}
188+
file: ./coverage/cobertura-coverage.xml

0 commit comments

Comments
 (0)