Skip to content

Commit d762373

Browse files
StuartWheaterStuartWheater
authored
Update to align with V6.3.5 dev changes
Update to align with V6.3.5 dev changes
2 parents 92e2d59 + 78d3b4d commit d762373

479 files changed

Lines changed: 13301 additions & 10452 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.Rbuildignore‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,17 +3,23 @@
33
^\.Rproj\.user$
44
^LICENSE.md$
55
^azure-pipelines\.yml$
6+
^opal_azure-pipelines\.yml$
7+
^armadillo_azure-pipelines\.yml$
68
^azure-pipelines_site\.pp$
79
^azure-pipelines_site-dsdanger\.pp$
810
^azure-pipelines_check\.Rout$
911
^azure-pipelines_test\.Rout$
12+
^tests/docker/armadillo/standard/log/.gitkeep$
1013
^checkDocumentationUpdated\.sh$
1114
^docker-compose_armadillo\.yml$
1215
^docker-compose_opal\.yml$
1316
^docker-compose\.yml$
1417
^R/secure.global.ranking.md$
1518
^_pkgdown\.yml$
1619
^docs$
17-
^dsBase_6.3.0.tar.gz$
18-
^dsBase_6.3.0-permissive.tar.gz$
19-
^dsDanger_6.3.0.tar.gz$
20+
^dsBase_6.3.4.tar.gz$
21+
^dsBase_6.3.4-permissive.tar.gz$
22+
^dsDanger_6.3.4.tar.gz$
23+
^\.circleci$
24+
^\.circleci/config\.yml$
25+
^\.github$

‎.circleci/config.yml‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
# Copyright (c) 2024 Arjuna Technologies, Newcastle upon Tyne. All rights reserved.
2+
3+
version: 2.1
4+
5+
jobs:
6+
dsbaseclient:
7+
docker:
8+
# - image: cimg/base:current
9+
- image: cimg/base:2024.11
10+
resource_class: small
11+
steps:
12+
- checkout
13+
- setup_remote_docker:
14+
docker_layout_caching: true
15+
- run: |
16+
echo "Building"
17+
echo " Repo Name: " $CIRCLE_PROJECT_REPONAME
18+
echo " Branch: " $CIRCLE_BRANCH
19+
echo " Tag: " $CIRCLE_TAG
20+
- run:
21+
command: |
22+
sudo apt-get install --no-install-recommends software-properties-common dirmngr
23+
wget -qO- https://cloud.r-project.org/bin/linux/ubuntu/marutter_pubkey.asc | sudo tee -a /etc/apt/trusted.gpg.d/cran_ubuntu_key.asc
24+
sudo add-apt-repository -y "deb https://cloud.r-project.org/bin/linux/ubuntu $(lsb_release -cs)-cran40/"
25+
26+
sudo apt-get update -y
27+
sudo apt-get install -y r-base-core cmake
28+
- run:
29+
command: |
30+
sudo apt-get install -y libxml2-dev
31+
sudo apt-get install -y libfontconfig1-dev
32+
sudo apt-get install -y libudunits2-dev
33+
sudo apt-get install -y libharfbuzz-dev
34+
sudo apt-get install -y libfribidi-dev
35+
sudo apt-get install -y libfreetype6-dev
36+
sudo apt-get install -y libpng-dev
37+
sudo apt-get install -y libtiff5-dev
38+
sudo apt-get install -y libjpeg-dev
39+
- run:
40+
command: |
41+
sudo Rscript -e "install.packages('devtools', dependencies=TRUE)"
42+
sudo Rscript -e "install.packages('covr', dependencies=TRUE)"
43+
44+
sudo Rscript -e "install.packages('fields', dependencies=TRUE)"
45+
sudo Rscript -e "install.packages('metafor', dependencies=TRUE)"
46+
sudo Rscript -e "install.packages('meta', dependencies=TRUE)"
47+
sudo Rscript -e "install.packages('ggplot2', dependencies=TRUE)"
48+
sudo Rscript -e "install.packages('gridExtra', dependencies=TRUE)"
49+
sudo Rscript -e "install.packages('data.table', dependencies=TRUE)"
50+
sudo Rscript -e "install.packages('methods', dependencies=TRUE)"
51+
sudo Rscript -e "install.packages('dplyr', dependencies=TRUE)"
52+
53+
sudo Rscript -e "install.packages('DSI', dependencies=TRUE)"
54+
sudo Rscript -e "install.packages('DSOpal', dependencies=TRUE)"
55+
sudo Rscript -e "install.packages('DSLite', dependencies=TRUE)"
56+
sudo Rscript -e "install.packages('MolgenisAuth', dependencies=TRUE)"
57+
sudo Rscript -e "install.packages('MolgenisArmadillo', dependencies=TRUE)"
58+
sudo Rscript -e "install.packages('DSMolgenisArmadillo', dependencies=TRUE)"
59+
sudo Rscript -e "install.packages('DescTools', dependencies=TRUE)"
60+
sudo Rscript -e "install.packages('e1071', dependencies=TRUE)"
61+
- run:
62+
command: |
63+
sudo Rscript -e 'library(covr); covr::codecov(token = "'$CODECOV_TOKEN'", quiet=FALSE)'
64+
workflows:
65+
build:
66+
jobs:
67+
- dsbaseclient

‎.github/.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
*.html

‎.github/workflows/dsBaseClient_test_suite.yaml‎

Lines changed: 245 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,245 @@
1+
################################################################################
2+
# DataSHIELD GHA test suite - dsBaseClient
3+
# Adapted from `armadillo_azure-pipelines.yml` by Roberto Villegas-Diaz
4+
#
5+
# Inside the root directory $(Pipeline.Workspace) will be a file tree like:
6+
# /dsBaseClient <- Checked out version of datashield/dsBaseClient
7+
# /dsBaseClient/logs <- Where results of tests and logs are collated
8+
# /testStatus <- Checked out version of datashield/testStatus
9+
#
10+
# As of Sept. 2025 this takes ~ 95 mins to run.
11+
################################################################################
12+
name: dsBaseClient tests' suite
13+
14+
on:
15+
push:
16+
schedule:
17+
- cron: '0 0 * * 0' # Weekly
18+
- cron: '0 1 * * *' # Nightly
19+
20+
jobs:
21+
dsBaseClient_test_suite:
22+
runs-on: ubuntu-latest
23+
timeout-minutes: 120
24+
permissions:
25+
contents: read
26+
27+
# These should all be constant, except TEST_FILTER. This can be used to test
28+
# subsets of test files in the testthat directory. Options are like:
29+
# '*' <- Run all tests.
30+
# 'asNumericDS*' <- Run all asNumericDS tests, i.e. all the arg, etc. tests.
31+
# '*_smk_*' <- Run all the smoke tests for all functions.
32+
env:
33+
TEST_FILTER: '_-|datachk-|smk-|arg-|disc-|perf-|smk_expt-|expt-|math-'
34+
_r_check_system_clock_: 0
35+
WORKFLOW_ID: ${{ github.run_id }}-${{ github.run_attempt }}
36+
PROJECT_NAME: dsBaseClient
37+
BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
38+
REPO_OWNER: ${{ github.repository_owner }}
39+
R_KEEP_PKG_SOURCE: yes
40+
GITHUB_TOKEN: ${{ github.token || 'placeholder-token' }}
41+
42+
steps:
43+
- name: Checkout dsBaseClient
44+
uses: actions/checkout@v4
45+
with:
46+
path: dsBaseClient
47+
48+
- name: Checkout testStatus
49+
if: ${{ github.actor != 'nektos/act' }} # for local deployment only
50+
uses: actions/checkout@v4
51+
with:
52+
repository: ${{ env.REPO_OWNER }}/testStatus
53+
ref: master
54+
path: testStatus
55+
persist-credentials: false
56+
token: ${{ env.GITHUB_TOKEN }}
57+
58+
- name: Uninstall default MySQL
59+
run: |
60+
curl https://bazel.build/bazel-release.pub.gpg | sudo apt-key add -
61+
sudo service mysql stop || true
62+
sudo apt-get update
63+
sudo apt-get remove --purge mysql-client mysql-server mysql-common -y
64+
sudo apt-get autoremove -y
65+
sudo apt-get autoclean -y
66+
sudo rm -rf /var/lib/mysql/
67+
68+
- uses: r-lib/actions/setup-pandoc@v2
69+
70+
- uses: r-lib/actions/setup-r@v2
71+
with:
72+
r-version: release
73+
http-user-agent: release
74+
use-public-rspm: true
75+
76+
- name: Install R and dependencies
77+
run: |
78+
sudo apt-get install --no-install-recommends software-properties-common dirmngr -y
79+
wget -qO- https://cloud.r-project.org/bin/linux/ubuntu/marutter_pubkey.asc | sudo tee -a /etc/apt/trusted.gpg.d/cran_ubuntu_key.asc
80+
sudo add-apt-repository "deb https://cloud.r-project.org/bin/linux/ubuntu $(lsb_release -cs)-cran40/"
81+
sudo apt-get update -qq
82+
sudo apt-get upgrade -y
83+
sudo apt-get install -qq libxml2-dev libcurl4-openssl-dev libssl-dev libgsl-dev libgit2-dev r-base -y
84+
sudo apt-get install -qq libharfbuzz-dev libfribidi-dev libmagick++-dev xml-twig-tools -y
85+
sudo R -q -e "install.packages(c('devtools','covr','fields','meta','metafor','ggplot2','gridExtra','data.table','DSI','DSOpal','DSLite','MolgenisAuth','MolgenisArmadillo','DSMolgenisArmadillo','DescTools','e1071'), repos='https://cloud.r-project.org')"
86+
sudo R -q -e "devtools::install_github(repo='datashield/dsDangerClient', ref=Sys.getenv('BRANCH_NAME'))"
87+
88+
- uses: r-lib/actions/setup-r-dependencies@v2
89+
with:
90+
dependencies: 'c("Imports")'
91+
extra-packages: |
92+
any::rcmdcheck
93+
cran::devtools
94+
cran::git2r
95+
cran::RCurl
96+
cran::readr
97+
cran::magrittr
98+
cran::xml2
99+
cran::purrr
100+
cran::dplyr
101+
cran::stringr
102+
cran::tidyr
103+
cran::quarto
104+
cran::knitr
105+
cran::kableExtra
106+
cran::rmarkdown
107+
cran::downlit
108+
needs: check
109+
110+
- name: Check manual updated
111+
run: |
112+
orig_sum=$(find man -type f | sort -u | xargs cat | md5sum)
113+
R -q -e "devtools::document()"
114+
new_sum=$(find man -type f | sort -u | xargs cat | md5sum)
115+
if [ "$orig_sum" != "$new_sum" ]; then
116+
echo "Your committed man/*.Rd files are out of sync with the R headers."
117+
exit 1
118+
fi
119+
working-directory: dsBaseClient
120+
continue-on-error: true
121+
122+
- name: Devtools checks
123+
run: |
124+
R -q -e "devtools::check(args = c('--no-examples', '--no-tests'))" | tee azure-pipelines_check.Rout
125+
grep --quiet "^0 errors" azure-pipelines_check.Rout && grep --quiet " 0 warnings" azure-pipelines_check.Rout && grep --quiet " 0 notes" azure-pipelines_check.Rout
126+
working-directory: dsBaseClient
127+
continue-on-error: true
128+
129+
- name: Start Armadillo docker-compose
130+
run: docker compose -f docker-compose_armadillo.yml up -d --build
131+
working-directory: dsBaseClient
132+
133+
- name: Install test datasets
134+
run: |
135+
sleep 60
136+
R -q -f "molgenis_armadillo-upload_testing_datasets.R"
137+
working-directory: dsBaseClient/tests/testthat/data_files
138+
139+
- name: Install dsBase to Armadillo
140+
run: |
141+
curl -u admin:admin -X GET http://localhost:8080/packages
142+
curl -u admin:admin -H 'Content-Type: multipart/form-data' -F "file=@dsBase_6.3.4-permissive.tar.gz" -X POST http://localhost:8080/install-package
143+
sleep 60
144+
docker restart dsbaseclient-armadillo-1
145+
sleep 30
146+
curl -u admin:admin -X POST http://localhost:8080/whitelist/dsBase
147+
working-directory: dsBaseClient
148+
149+
- name: Run tests with coverage & JUnit report
150+
run: |
151+
mkdir -p logs
152+
R -q -e "devtools::reload();"
153+
R -q -e '
154+
write.csv(
155+
covr::coverage_to_list(
156+
covr::package_coverage(
157+
type = c("none"),
158+
code = c('"'"'
159+
output_file <- file("test_console_output.txt");
160+
sink(output_file);
161+
sink(output_file, type = "message");
162+
junit_rep <- testthat::JunitReporter$new(file = file.path(getwd(), "test_results.xml"));
163+
progress_rep <- testthat::ProgressReporter$new(max_failures = 999999);
164+
multi_rep <- testthat::MultiReporter$new(reporters = list(progress_rep, junit_rep));
165+
options("datashield.return_errors" = FALSE, "default_driver" = "ArmadilloDriver");
166+
testthat::test_package("${{ env.PROJECT_NAME }}", filter = "${{ env.TEST_FILTER }}", reporter = multi_rep, stop_on_failure = FALSE)'"'"'
167+
)
168+
)
169+
),
170+
"coveragelist.csv"
171+
)'
172+
173+
mv coveragelist.csv logs/
174+
mv test_* logs/
175+
working-directory: dsBaseClient
176+
177+
- name: Check for JUnit errors
178+
run: |
179+
issue_count=$(sed 's/failures="0" errors="0"//' test_results.xml | grep -c errors= || true)
180+
echo "Number of testsuites with issues: $issue_count"
181+
sed 's/failures="0" errors="0"//' test_results.xml | grep errors= > issues.log || true
182+
cat issues.log || true
183+
# continue with workflow even when some tests fail
184+
exit 0
185+
working-directory: dsBaseClient/logs
186+
187+
- name: Write versions to file
188+
run: |
189+
echo "branch:${{ env.BRANCH_NAME }}" > ${{ env.WORKFLOW_ID }}.txt
190+
echo "os:$(lsb_release -ds)" >> ${{ env.WORKFLOW_ID }}.txt
191+
echo "R:$(R --version | head -n1)" >> ${{ env.WORKFLOW_ID }}.txt
192+
working-directory: dsBaseClient/logs
193+
194+
- name: Parse results from testthat and covr
195+
run: |
196+
Rscript --verbose --vanilla ../testStatus/source/parse_test_report.R logs/
197+
working-directory: dsBaseClient
198+
199+
- name: Render report
200+
run: |
201+
cd testStatus
202+
203+
mkdir -p new/logs/${{ env.PROJECT_NAME }}/${{ env.BRANCH_NAME }}/${{ env.WORKFLOW_ID }}/
204+
mkdir -p new/docs/${{ env.PROJECT_NAME }}/${{ env.BRANCH_NAME }}/${{ env.WORKFLOW_ID }}/
205+
mkdir -p new/docs/${{ env.PROJECT_NAME }}/${{ env.BRANCH_NAME }}/latest/
206+
207+
# Copy logs to new logs directory location
208+
cp -rv ../dsBaseClient/logs/* new/logs/${{ env.PROJECT_NAME }}/${{ env.BRANCH_NAME }}/${{ env.WORKFLOW_ID }}/
209+
cp -rv ../dsBaseClient/logs/${{ env.WORKFLOW_ID }}.txt new/logs/${{ env.PROJECT_NAME }}/${{ env.BRANCH_NAME }}/${{ env.WORKFLOW_ID }}/
210+
211+
R -e 'input_dir <- file.path("../new/logs", Sys.getenv("PROJECT_NAME"), Sys.getenv("BRANCH_NAME"), Sys.getenv("WORKFLOW_ID")); quarto::quarto_render("source/test_report.qmd", execute_params = list(input_dir = input_dir))'
212+
mv source/test_report.html new/docs/${{ env.PROJECT_NAME }}/${{ env.BRANCH_NAME }}/${{ env.WORKFLOW_ID }}/index.html
213+
cp -r new/docs/${{ env.PROJECT_NAME }}/${{ env.BRANCH_NAME }}/${{ env.WORKFLOW_ID }}/* new/docs/${{ env.PROJECT_NAME }}/${{ env.BRANCH_NAME }}/latest
214+
215+
env:
216+
PROJECT_NAME: ${{ env.PROJECT_NAME }}
217+
BRANCH_NAME: ${{ env.BRANCH_NAME }}
218+
WORKFLOW_ID: ${{ env.WORKFLOW_ID }}
219+
220+
- name: Upload test logs
221+
uses: actions/upload-artifact@v4
222+
with:
223+
name: dsbaseclient-logs
224+
path: testStatus/new
225+
226+
- name: Dump environment info
227+
run: |
228+
echo -e "\n#############################"
229+
echo -e "ls /: ######################"
230+
ls -al .
231+
echo -e "\n#############################"
232+
echo -e "lscpu: ######################"
233+
lscpu
234+
echo -e "\n#############################"
235+
echo -e "memory: #####################"
236+
free -m
237+
echo -e "\n#############################"
238+
echo -e "env: ########################"
239+
env
240+
echo -e "\n#############################"
241+
echo -e "R sessionInfo(): ############"
242+
R -e 'sessionInfo()'
243+
sudo apt install tree -y
244+
tree .
245+

‎.github/workflows/pkgdown.yaml‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
# Workflow derived from https://github.com/r-lib/actions/tree/v2/examples
2+
# Need help debugging build failures? Start at https://github.com/r-lib/actions#where-to-find-help
3+
on:
4+
push:
5+
branches: [main, master]
6+
pull_request:
7+
release:
8+
types: [published]
9+
workflow_dispatch:
10+
11+
name: pkgdown.yaml
12+
13+
permissions: read-all
14+
15+
jobs:
16+
pkgdown:
17+
runs-on: ubuntu-latest
18+
# Only restrict concurrency for non-PR jobs
19+
concurrency:
20+
group: pkgdown-${{ github.event_name != 'pull_request' || github.run_id }}
21+
env:
22+
GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
23+
permissions:
24+
contents: write
25+
steps:
26+
- uses: actions/checkout@v4
27+
28+
- uses: r-lib/actions/setup-pandoc@v2
29+
30+
- uses: r-lib/actions/setup-r@v2
31+
with:
32+
use-public-rspm: true
33+
34+
- uses: r-lib/actions/setup-r-dependencies@v2
35+
with:
36+
extra-packages: any::pkgdown, local::.
37+
needs: website
38+
39+
- name: Build site
40+
run: pkgdown::build_site_github_pages(new_process = FALSE, install = FALSE)
41+
shell: Rscript {0}
42+
43+
- name: Deploy to GitHub pages 🚀
44+
if: github.event_name != 'pull_request'
45+
uses: JamesIves/github-pages-deploy-action@v4.5.0
46+
with:
47+
clean: false
48+
branch: gh-pages
49+
folder: docs

‎.gitignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,3 +10,5 @@
1010
inst/tests/test-template.R
1111
azure-pipelines.Rout
1212
tests/testthat/connection_to_datasets/local_settings.csv
13+
tests/docker/armadillo/standard/logs/
14+
tests/docker/armadillo/standard/data/

0 commit comments

Comments
 (0)