Skip to content

Commit 9af0c8c

Browse files
veravera
committed
Merge branch 'refs/heads/develop' into feat/non-superuser-dataverse-linking
2 parents a1e4225 + 199057d commit 9af0c8c

194 files changed

Lines changed: 9406 additions & 1665 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/add_bugs_to_project.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ jobs:
1111
name: Add bug to project
1212
runs-on: ubuntu-latest
1313
steps:
14-
- uses: actions/add-to-project@v1.0.2
14+
- uses: actions/add-to-project@v2.0.0
1515
with:
1616
project-url: https://github.com/orgs/IQSS/projects/34
1717
github-token: ${{ secrets.ADD_TO_PROJECT_PAT }}

‎.github/workflows/container_integration_tests.yml‎

Lines changed: 285 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,285 @@
1+
name: Container Integration Tests Workflow
2+
3+
on:
4+
workflow_dispatch:
5+
push:
6+
branches:
7+
- develop
8+
- master
9+
paths-ignore:
10+
- "doc/**"
11+
- "**/*.md"
12+
- ".github/ISSUE_TEMPLATE/**"
13+
- ".github/*.md"
14+
pull_request:
15+
branches:
16+
- develop
17+
- master
18+
paths-ignore:
19+
- "doc/**"
20+
- "**/*.md"
21+
- ".github/ISSUE_TEMPLATE/**"
22+
- ".github/*.md"
23+
24+
concurrency:
25+
group: "container-integration-tests-${{ github.ref }}"
26+
cancel-in-progress: true
27+
28+
jobs:
29+
main-integration-tests-workflow:
30+
runs-on: ubuntu-latest
31+
timeout-minutes: 60
32+
33+
defaults:
34+
run:
35+
shell: bash
36+
37+
permissions:
38+
contents: read
39+
checks: write
40+
pull-requests: write
41+
42+
steps:
43+
44+
# ---------------------------
45+
# CHECKOUT
46+
# ---------------------------
47+
- name: Checkout repository
48+
uses: actions/checkout@v6
49+
50+
# ---------------------------
51+
# VERIFY DOCKER
52+
# ---------------------------
53+
- name: Verify Docker
54+
run: |
55+
set -euo pipefail
56+
docker version
57+
58+
# ---------------------------
59+
# SETUP JAVA + MAVEN
60+
# ---------------------------
61+
- name: Setup Java
62+
uses: actions/setup-java@v5
63+
with:
64+
distribution: "temurin"
65+
java-version: "21"
66+
cache: "maven"
67+
68+
- name: Verify Maven
69+
run: |
70+
set -euo pipefail
71+
mvn -version
72+
73+
# ---------------------------
74+
# BUILD IMAGES (Dataverse-native)
75+
# ---------------------------
76+
- name: Build Dataverse containers via Maven
77+
run: |
78+
set -euo pipefail
79+
mvn -Pct -T 1C package
80+
81+
# ---------------------------
82+
# START CONTAINERS (BACKGROUND)
83+
# ---------------------------
84+
- name: Start Dataverse stack
85+
run: |
86+
set -euo pipefail
87+
mvn -Pct docker:start \
88+
-Ddataverse.feature.index-harvested-metadata-source=true \
89+
-Ddataverse.oai.server.maxidentifiers=2 \
90+
-Ddataverse.oai.server.maxrecords=2
91+
92+
# ---------------------------
93+
# WAIT FOR API READINESS
94+
# ---------------------------
95+
- name: Wait for Dataverse API readiness
96+
run: |
97+
set -euo pipefail
98+
URL="http://localhost:8080/api/info/version"
99+
MAX_ATTEMPTS=10
100+
SLEEP_TIME=15
101+
echo "Waiting for Dataverse readiness..."
102+
for attempt in $(seq 1 $MAX_ATTEMPTS); do
103+
echo "Attempt $attempt..."
104+
RESPONSE=$(curl -s --max-time 15 "$URL" || true)
105+
STATUS=$(echo "$RESPONSE" | jq -r '.status' 2>/dev/null || echo "NOT_READY")
106+
if [ "$STATUS" = "OK" ]; then
107+
echo "Dataverse endpoint is READY."
108+
echo "Dataverse waiting for full readiness. Waiting 30 more seconds."
109+
sleep 30
110+
echo "Response: $RESPONSE"
111+
exit 0
112+
fi
113+
echo "Not ready. Sleeping ${SLEEP_TIME}s..."
114+
sleep $SLEEP_TIME
115+
if [ $SLEEP_TIME -lt 60 ]; then
116+
SLEEP_TIME=$((SLEEP_TIME * 2))
117+
if [ $SLEEP_TIME -gt 60 ]; then
118+
SLEEP_TIME=60
119+
fi
120+
fi
121+
done
122+
echo "Dataverse failed to become ready."
123+
docker ps
124+
CONTAINERS="$(docker ps -aq)"
125+
if [ -n "$CONTAINERS" ]; then
126+
for cid in $CONTAINERS; do
127+
echo "===== Logs for container $cid ====="
128+
docker logs "$cid" || true
129+
done
130+
else
131+
echo "No running containers to show logs for."
132+
fi
133+
exit 1
134+
135+
# ---------------------------
136+
# MAP LOCALSTACK TO LOCALHOST
137+
# ---------------------------
138+
- name: Map localstack to localhost for Maven tests
139+
run: echo "127.0.0.1 localstack" | sudo tee -a /etc/hosts
140+
141+
# ---------------------------
142+
# CONFIGURE DATAVERSE FOR TESTS
143+
# ---------------------------
144+
- name: Configure Dataverse API Settings
145+
run: |
146+
set -euo pipefail
147+
148+
echo "Setting API Database Settings via internal container curl..."
149+
150+
# We define the settings in an array
151+
declare -A settings=(
152+
[":BuiltinUsersKey"]="burrito"
153+
[":ProvCollectionEnabled"]="true"
154+
[":AllowApiTokenLookupViaApi"]="true"
155+
[":AllowSignUp"]="true"
156+
)
157+
# We run curl INSIDE the container so the source IP is 127.0.0.1
158+
for key in "${!settings[@]}"; do
159+
echo "Setting $key..."
160+
docker exec dev_dataverse curl --fail-with-body -sS -X PUT -d "${settings[$key]}" "http://localhost:8080/api/admin/settings/$key"
161+
echo ""
162+
done
163+
164+
# ---------------------------
165+
# PRE-TEST INJECTIONS
166+
# ---------------------------
167+
- name: Put SUSHI config file in place
168+
run: |
169+
set -euo pipefail
170+
171+
SOURCE_FILE="${{ github.workspace }}/src/test/java/edu/harvard/iq/dataverse/makedatacount/sushi_sample_logs.json"
172+
173+
echo "Injecting local file into container..."
174+
# This reads the local file and writes it inside the container using standard input
175+
docker exec -i dev_dataverse sh -c "cat > /tmp/sushi_sample_logs.json" < "$SOURCE_FILE"
176+
177+
# Verify the content is actually there and has size
178+
docker exec dev_dataverse ls -l /tmp/sushi_sample_logs.json
179+
docker exec dev_dataverse head -n 5 /tmp/sushi_sample_logs.json
180+
181+
# ---------------------------
182+
# RUN MAVEN INTEGRATION TESTS
183+
# ---------------------------
184+
- name: Run Maven Integration Tests
185+
env:
186+
DVAPIKEY: "burrito"
187+
DV_APIKEY: "burrito"
188+
DV_API_KEY: "burrito"
189+
run: |
190+
set -euo pipefail
191+
TEST_SUITE=$(cat tests/integration-tests.txt)
192+
193+
echo "Running suite: $TEST_SUITE"
194+
195+
mvn test \
196+
-Dtest="$TEST_SUITE" \
197+
-Dmaven.test.failure.ignore=true \
198+
-Ddataverse.test.baseurl=http://localhost:8080 \
199+
-DcompilerArgument=-Xlint:unchecked
200+
201+
# ---------------------------
202+
# UPLOAD SUREFIRE/FAILSAFE REPORTS
203+
# ---------------------------
204+
- name: Upload Test Failure Reports
205+
if: always()
206+
uses: actions/upload-artifact@v7
207+
with:
208+
name: maven-test-reports
209+
path: |
210+
target/surefire-reports/
211+
target/failsafe-reports/
212+
retention-days: 14
213+
214+
# ---------------------------
215+
# PUBLISH TEST DASHBOARD IN GITHUB PR
216+
# ---------------------------
217+
- name: Publish Test Results Dashboard
218+
uses: EnricoMi/publish-unit-test-result-action@v2
219+
if: always()
220+
with:
221+
files: |
222+
target/failsafe-reports/TEST-*.xml
223+
target/surefire-reports/TEST-*.xml
224+
225+
# ---------------------------
226+
# FAIL WORKFLOW IF TESTS FAILED
227+
# ---------------------------
228+
- name: Check for Test Failures
229+
if: always()
230+
run: |
231+
echo "Checking Surefire/Failsafe reports for failures..."
232+
if grep -q "<failure" target/surefire-reports/*.xml target/failsafe-reports/*.xml 2>/dev/null; then
233+
echo "Tests failed! Failing the workflow."
234+
exit 1
235+
fi
236+
echo "All tests passed."
237+
238+
# ---------------------------
239+
# COLLECT DOCKER LOGS (ALWAYS, WITH MAPPING)
240+
# ---------------------------
241+
- name: Collect Docker logs (mapped)
242+
if: always()
243+
run: |
244+
mkdir -p docker-logs
245+
echo "Gathering container metadata..."
246+
docker ps -a --format '{{.Names}}|{{.Image}}|{{.Status}}' > docker-logs/container-summary.txt
247+
while IFS='|' read -r name image status; do
248+
# Create a readable label
249+
label="$name"
250+
case "$name" in
251+
*dataverse*)
252+
label="dataverse-app"
253+
;;
254+
*postgres*)
255+
label="postgres-db"
256+
;;
257+
*solr*)
258+
label="solr-index"
259+
;;
260+
*localstack*)
261+
label="localstack-s3"
262+
;;
263+
esac
264+
echo "Collecting logs for $name ($label)"
265+
{
266+
echo "===== CONTAINER: $name ====="
267+
echo "Label: $label"
268+
echo "Image: $image"
269+
echo "Status: $status"
270+
echo ""
271+
echo "===== LOGS ====="
272+
docker logs --timestamps "$name" 2>&1 || true
273+
} > "docker-logs/${label}__${name}.log"
274+
done < docker-logs/container-summary.txt
275+
276+
# ---------------------------
277+
# UPLOAD DOCKER LOGS (ALWAYS)
278+
# ---------------------------
279+
- name: Upload Docker logs
280+
if: always()
281+
uses: actions/upload-artifact@v7
282+
with:
283+
name: docker-logs
284+
path: docker-logs/
285+
retention-days: 14

‎.github/workflows/container_maintenance.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -173,7 +173,7 @@ jobs:
173173
with:
174174
platforms: ${{ env.PLATFORMS }}
175175
- name: Setup Trivy binary for vulnerability scanning
176-
uses: aquasecurity/setup-trivy@v0.2.6
176+
uses: aquasecurity/setup-trivy@v0.3.1
177177
with:
178178
version: v0.69.3
179179

‎.github/workflows/copy_labels.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,6 @@ jobs:
1414
name: Copy labels from linked issues
1515
steps:
1616
- name: copy-labels
17-
uses: michalvankodev/copy-issue-labels@v1.3.0
17+
uses: michalvankodev/copy-issue-labels@v2.0.0
1818
with:
1919
repo-token: ${{ secrets.GITHUB_TOKEN }}

‎.readthedocs.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,9 @@ formats:
55
- pdf
66

77
build:
8-
os: ubuntu-22.04
8+
os: ubuntu-24.04
99
tools:
10-
python: "3.10"
10+
python: "3.12"
1111
apt_packages:
1212
- graphviz
1313

‎conf/keycloak/builtin-users-spi/pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,7 @@
100100
</build>
101101

102102
<properties>
103-
<keycloak.version>26.5.5</keycloak.version>
103+
<keycloak.version>26.6.0</keycloak.version>
104104
<java.version>17</java.version>
105105
<jakarta.persistence.version>3.2.0</jakarta.persistence.version>
106106
<mindrot.jbcrypt.version>0.4</mindrot.jbcrypt.version>

‎conf/mdc/counter_weekly.sh‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ case $HTTP_STATUS in
3131
# Successfully queued
3232
# Extract status from the nested data object
3333
STATUS=$(echo "$RESPONSE_BODY" | jq -r '.data.status')
34-
34+
3535
# Extract message from the nested data object
3636
if echo "$RESPONSE_BODY" | jq -e '.data.message' > /dev/null 2>&1 && [ "$(echo "$RESPONSE_BODY" | jq -r '.data.message')" != "null" ]; then
3737
MESSAGE=$(echo "$RESPONSE_BODY" | jq -r '.data.message')
@@ -89,4 +89,5 @@ done
8989
}
9090

9191
# Call the function on the root dataverse to start processing
92-
processDV 1
92+
processDV 1
93+
echo "Processing Dataverse Complete: $(date)"

0 commit comments

Comments
 (0)