Skip to content

Commit 3fa3b4f

Browse files
mvdbeekmvdbeek
authored
Merge pull request #593 from bernt-matthias/topic/bump-sync-workflows
switch workflows to 21.05
2 parents ee5ebb4 + e466f89 commit 3fa3b4f

2 files changed

Lines changed: 69 additions & 99 deletions

File tree

‎.github/workflows/ci.yaml‎

Lines changed: 58 additions & 89 deletions
Original file line numberDiff line numberDiff line change
@@ -6,23 +6,24 @@ on:
66
repository_dispatch:
77
types: [run-all-tool-tests-command]
88
env:
9-
GALAXY_RELEASE: release_21.01
109
GALAXY_FORK: galaxyproject
10+
GALAXY_BRANCH: release_21.05
1111
MAX_CHUNKS: 40
1212
jobs:
1313
setup:
14-
name: Setup cache
14+
name: Setup cache and determine changed repositories
1515
if: github.repository_owner == 'galaxyproject'
1616
runs-on: ubuntu-latest
1717
outputs:
18-
branch: ${{ steps.get-fork-branch.outputs.branch }}
18+
galaxy-head-sha: ${{ steps.get-galaxy-sha.outputs.galaxy-head-sha }}
1919
fork: ${{ steps.get-fork-branch.outputs.fork }}
20-
galaxy_head_sha: ${{ steps.get-galaxy-sha.outputs.galaxy_head_sha }}
21-
nchunks: ${{ steps.get-chunks.outputs.nchunks }}
22-
chunk_list: ${{ steps.get-chunks.outputs.chunk_list }}
20+
branch: ${{ steps.get-fork-branch.outputs.branch }}
21+
repository-list: ${{ steps.discover.outputs.repository-list }}
22+
chunk-count: ${{ steps.discover.outputs.chunk-count }}
23+
chunk-list: ${{ steps.discover.outputs.chunk-list }}
2324
strategy:
2425
matrix:
25-
python-version: [3.7]
26+
python-version: ['3.7']
2627
steps:
2728
- name: Add reaction
2829
if: ${{ github.event.client_payload.slash_command.command == 'run-all-tool-tests' }}
@@ -38,10 +39,10 @@ jobs:
3839
TMP="${{ github.event.client_payload.slash_command.args.named.fork }}"
3940
echo "::set-output name=fork::${TMP:-$GALAXY_FORK}"
4041
TMP="${{ github.event.client_payload.slash_command.args.named.branch }}"
41-
echo "::set-output name=branch::${TMP:-$GALAXY_RELEASE}"
42-
- name: Determine latest galaxy commit
42+
echo "::set-output name=branch::${TMP:-$GALAXY_BRANCH}"
43+
- name: Determine latest commit in the Galaxy repo
4344
id: get-galaxy-sha
44-
run: echo "::set-output name=galaxy_head_sha::$(git ls-remote https://github.com/${{ steps.get-fork-branch.outputs.fork }}/galaxy refs/heads/${{ steps.get-fork-branch.outputs.branch }} | cut -f1)"
45+
run: echo "::set-output name=galaxy-head-sha::$(git ls-remote https://github.com/${{ steps.get-fork-branch.outputs.fork }}/galaxy refs/heads/${{ steps.get-fork-branch.outputs.branch }} | cut -f1)"
4546
- uses: actions/setup-python@v1
4647
with:
4748
python-version: ${{ matrix.python-version }}
@@ -50,46 +51,40 @@ jobs:
5051
id: cache-pip
5152
with:
5253
path: ~/.cache/pip
53-
key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ steps.get-galaxy-sha.outputs.galaxy_head_sha }}
54+
key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ steps.get-galaxy-sha.outputs.galaxy-head-sha }}
5455
# Install the `wheel` package so that when installing other packages which
5556
# are not available as wheels, pip will build a wheel for them, which can be cached.
5657
- name: Install wheel
5758
run: pip install wheel
58-
- name: Install Planemo
59-
run: pip install planemo
60-
- name: Fake a Planemo run to update cache
61-
if: steps.cache-pip.outputs.cache-hit != 'true'
62-
run: |
63-
touch tool.xml
64-
PIP_QUIET=1 planemo test --galaxy_python_version ${{ matrix.python-version }} --no_conda_auto_init --galaxy_source 'https://github.com/${{ steps.get-fork-branch.outputs.fork }}/galaxy' --galaxy_branch "${{ steps.get-fork-branch.outputs.branch }}" --galaxy_python_version ${{ matrix.python-version }}
6559
- uses: actions/checkout@v2
6660
with:
6761
fetch-depth: 1
68-
- name: Get approximate number of tools and compute number of chunks
69-
id: get-chunks
70-
run: |
71-
nchunks=$(find data_managers/ tools/ tool_collections/ -iname '*.xml' | grep -v 'data_manager_conf.xml\|datatypes_conf.xml\|macro\|repository_dependencies.xml\|test-data/\|tool_dependencies.xml' | xargs grep -l '<tool ' | wc -l)
72-
if [ "$nchunks" -gt "$MAX_CHUNKS" ]; then
73-
nchunks=$MAX_CHUNKS
74-
elif [ "$nchunks" -eq 0 ]; then
75-
nchunks=1
76-
fi
77-
echo "::set-output name=nchunks::$nchunks"
78-
echo "::set-output name=chunk_list::[$(seq -s ", " 0 $(($nchunks - 1)))]"
62+
- name: Fake a Planemo run to update cache and determine commit range, repositories, and chunks
63+
uses: galaxyproject/planemo-ci-action@v1
64+
id: discover
65+
with:
66+
create-cache: ${{ steps.cache-pip.outputs.cache-hit != 'true' || steps.cache-planemo.outputs.cache-hit != 'true' }}
67+
galaxy-fork: ${{ steps.get-fork-branch.outputs.fork }}
68+
galaxy-branch: ${{ steps.get-fork-branch.outputs.branch }}
69+
max-chunks: ${{ env.MAX_CHUNKS }}
70+
python-version: ${{ matrix.python-version }}
71+
- name: Show repository list
72+
run: echo '${{ steps.discover.outputs.repository-list }}'
7973
- name: Show chunks
8074
run: |
81-
echo 'Using ${{ steps.get-chunks.outputs.nchunks }} chunks (${{ steps.get-chunks.outputs.chunk_list }})'
75+
echo 'Using ${{ steps.discover.outputs.chunk-count }} chunks (${{ steps.discover.outputs.chunk-list }})'
8276
8377
test:
8478
name: Test tools
8579
# This job runs on Linux
8680
runs-on: ubuntu-latest
8781
needs: setup
82+
if: needs.setup.outputs.repository-list != ''
8883
strategy:
8984
fail-fast: false
9085
matrix:
91-
chunk: ${{ fromJson(needs.setup.outputs.chunk_list) }}
92-
python-version: [3.7]
86+
chunk: ${{ fromJson(needs.setup.outputs.chunk-list) }}
87+
python-version: ['3.7']
9388
services:
9489
postgres:
9590
image: postgres:11
@@ -113,46 +108,33 @@ jobs:
113108
id: cache-pip
114109
with:
115110
path: ~/.cache/pip
116-
key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ needs.setup.outputs.galaxy_head_sha }}
117-
- name: Install Planemo
118-
run: pip install planemo
119-
- name: Planemo ci_find_tools, chunked
120-
run: planemo ci_find_tools --chunk_count ${{ needs.setup.outputs.nchunks }} --chunk ${{ matrix.chunk }} --exclude test_repositories --exclude packages --exclude deprecated --exclude_from .tt_skip --group_tools --output tool.list
121-
- name: Show tools chunk list
122-
run: cat tool.list
123-
- name: Planemo test tools
124-
run: |
125-
mkdir json_output
126-
while read -r TOOL_GROUP; do
127-
# Check if any of the lines in .tt_biocontainer_skip is a substring of $TOOL
128-
if echo $TOOL_GROUP | grep -qf .tt_biocontainer_skip; then
129-
PLANEMO_OPTIONS=""
130-
else
131-
PLANEMO_OPTIONS="--biocontainers --no_dependency_resolution --no_conda_auto_init"
132-
fi
133-
json=$(mktemp -u -p json_output --suff .json)
134-
PIP_QUIET=1 planemo test --database_connection postgresql://postgres:postgres@localhost:5432/galaxy $PLANEMO_OPTIONS --galaxy_source "https://github.com/${{ needs.setup.outputs.fork }}/galaxy" --galaxy_branch "${{ needs.setup.outputs.branch }}" --galaxy_python_version ${{ matrix.python-version }} --test_output_json "$json" $TOOL_GROUP || true
135-
docker system prune --all --force --volumes || true
136-
done < tool.list
137-
- name: Merge tool_test_output.json files
138-
run: planemo merge_test_reports json_output/*.json tool_test_output.json
139-
- name: Create tool_test_output.html
140-
run: planemo test_reports tool_test_output.json --test_output tool_test_output.html
141-
- name: Copy artifacts into place
142-
run: |
143-
mkdir upload
144-
mv tool_test_output.json tool_test_output.html upload/
111+
key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ needs.setup.outputs.galaxy-head-sha }}
112+
- name: Planemo test
113+
uses: galaxyproject/planemo-ci-action@v1
114+
id: test
115+
with:
116+
mode: test
117+
repository-list: ${{ needs.setup.outputs.repository-list }}
118+
galaxy-fork: ${{ needs.setup.outputs.fork }}
119+
galaxy-branch: ${{ needs.setup.outputs.branch }}
120+
chunk: ${{ matrix.chunk }}
121+
chunk-count: ${{ needs.setup.outputs.chunk-count }}
145122
- uses: actions/upload-artifact@v2
146123
with:
147-
name: 'Tool test output ${{ matrix.chunk }}'
124+
name: 'Tool test output ${{ matrix.chunk }}'
148125
path: upload
149126

127+
# - combine the results of the test chunks (which will never fail due
128+
# to `|| true`) and create a global test report as json and html which
129+
# is provided as artifact
130+
# - check if any tool test actually failed (by lookup in the combined json)
131+
# and fail this step if this is the case
150132
combine_outputs:
151133
name: Combine chunked test results
152134
needs: [setup, test]
153135
strategy:
154136
matrix:
155-
python-version: [3.7]
137+
python-version: ['3.7']
156138
# This job runs on Linux
157139
runs-on: ubuntu-latest
158140
steps:
@@ -167,33 +149,21 @@ jobs:
167149
id: cache-pip
168150
with:
169151
path: ~/.cache/pip
170-
key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ needs.setup.outputs.galaxy_head_sha }}
171-
- name: Install Planemo
172-
run: pip install planemo
173-
- name: Install jq
174-
run: sudo apt-get install jq
152+
key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ needs.setup.outputs.galaxy-head-sha }}
175153
- name: Combine outputs
176-
run: find artifacts/ -name tool_test_output.json -exec sh -c 'planemo merge_test_reports "$@" tool_test_output.json' sh {} +
177-
- name: Create tool_test_output.html
178-
run: planemo test_reports tool_test_output.json --test_output tool_test_output.html
179-
- name: Copy artifacts into place
180-
run: |
181-
mkdir upload
182-
mv tool_test_output.json tool_test_output.html upload/
154+
uses: galaxyproject/planemo-ci-action@v1
155+
id: combine
156+
with:
157+
mode: combine
158+
html-report: true
183159
- uses: actions/upload-artifact@v2
184160
with:
185161
name: 'All tool test results'
186162
path: upload
187-
188163
- name: Create URL to the run output
189164
if: ${{ github.event.client_payload.slash_command.command == 'run-all-tool-tests' }}
190165
id: vars
191-
run: echo ::set-output name=run-url::https://github.com/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID
192-
193-
- name: Gather statistics
194-
if: ${{ github.event.client_payload.slash_command.command == 'run-all-tool-tests' }}
195-
id: stats
196-
run: echo ::set-output name=statistics::$(jq '.["tests"][]["data"]["status"]' upload/tool_test_output.json | sed 's/"//g' | sort | uniq -c)
166+
run: echo "::set-output name=run-url::https://github.com/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID"
197167

198168
- name: Create comment
199169
if: ${{ github.event.client_payload.slash_command.command == 'run-all-tool-tests' }}
@@ -205,14 +175,13 @@ jobs:
205175
body: |
206176
Summary:
207177
208-
${{ steps.stats.outputs.statistics }}
178+
${{ steps.combine.outputs.statistics }}
209179
210180
[Find all tool test results here][1]
211181
212182
[1]: ${{ steps.vars.outputs.run-url }}
213-
- name: Check status of combined outputs
214-
run: |
215-
if jq '.["tests"][]["data"]["status"]' upload/tool_test_output.json | grep -v "success"; then
216-
echo "Unsuccessful tests found, inspect the 'All tool test results' artifact for details."
217-
exit 1
218-
fi
183+
- name: Check outputs
184+
uses: galaxyproject/planemo-ci-action@v1
185+
id: check
186+
with:
187+
mode: check

‎.github/workflows/pr.yaml‎

Lines changed: 11 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ name: Galaxy Tool Linting and Tests for push and PR
22
on: [push, pull_request]
33
env:
44
GALAXY_FORK: galaxyproject
5-
GALAXY_BRANCH: release_21.01
5+
GALAXY_BRANCH: release_21.05
66
MAX_CHUNKS: 4
77
jobs:
88
# the setup job does two things:
@@ -23,7 +23,7 @@ jobs:
2323
chunk-list: ${{ steps.discover.outputs.chunk-list }}
2424
strategy:
2525
matrix:
26-
python-version: [3.7]
26+
python-version: ['3.7']
2727
steps:
2828
- name: Print github context properties
2929
run: |
@@ -89,7 +89,7 @@ jobs:
8989
strategy:
9090
fail-fast: false
9191
matrix:
92-
python-version: [3.7]
92+
python-version: ['3.7']
9393
steps:
9494
# checkout the repository
9595
# and use it as the current working directory
@@ -122,7 +122,7 @@ jobs:
122122
strategy:
123123
fail-fast: false
124124
matrix:
125-
python-version: [3.7]
125+
python-version: ['3.7']
126126
steps:
127127
# checkout the repository to master
128128
# and use it as the current working directory
@@ -147,10 +147,11 @@ jobs:
147147
name: Lint R scripts
148148
needs: setup
149149
if: needs.setup.outputs.repository-list != ''
150-
runs-on: ubuntu-latest
150+
runs-on: ${{ matrix.os }}
151151
strategy:
152152
matrix:
153-
r-version: [4.0.1]
153+
os: [ubuntu-20.04]
154+
r-version: ['4.0.1']
154155
steps:
155156
# checkout the repository to master
156157
# and use it as the current working directory
@@ -164,7 +165,7 @@ jobs:
164165
uses: actions/cache@v2
165166
with:
166167
path: ${{ env.R_LIBS_USER }}
167-
key: r_cache_${{ matrix.r-version }}
168+
key: r_cache_${{ matrix.os }}_${{ matrix.r-version }}
168169
- name: Install non-R lintr dependencies
169170
run: sudo apt-get install libcurl4-openssl-dev
170171
- name: Install lintr
@@ -209,7 +210,7 @@ jobs:
209210
fail-fast: false
210211
matrix:
211212
chunk: ${{ fromJson(needs.setup.outputs.chunk-list) }}
212-
python-version: [3.7]
213+
python-version: ['3.7']
213214
services:
214215
postgres:
215216
image: postgres:11
@@ -265,7 +266,7 @@ jobs:
265266
needs: [setup, test]
266267
strategy:
267268
matrix:
268-
python-version: [3.7]
269+
python-version: ['3.7']
269270
# This job runs on Linux
270271
runs-on: ubuntu-latest
271272
steps:
@@ -303,7 +304,7 @@ jobs:
303304
needs: [setup, lint, flake8, lintr, combine_outputs]
304305
strategy:
305306
matrix:
306-
python-version: [3.7]
307+
python-version: ['3.7']
307308
runs-on: ubuntu-latest
308309
if: github.ref == 'refs/heads/master' && github.repository_owner == 'galaxyproject'
309310
steps:

0 commit comments

Comments
 (0)