Skip to content

Commit 473a453

Browse files
pjenveypjenvey
authored
test: fix test metric filenames (#2291)
to produce a "test_suite" of unit/integration per: https://mozilla.github.io/ecosystem-test-scripts/reference-guides/project_onboarding_procedure.html#1-setup-cicd-to-push-test-result-and-coverage-data-to-gcs the parser's lax about the final __results/coverage components: https://github.com/mozilla/ecosystem-test-scripts/blob/65674d8e28e27d3d1753c75d8f74c22f0e0a5896/scripts/metric_reporter/parser/base_parser.py#L16 so follow Kat's other patterns of e.g. "__<extra>-<metadata>-results.xml" and kill tokenserver_utils_results.xml/some unused env vars Issue STOR-548
1 parent e0ea07d commit 473a453

1 file changed

Lines changed: 30 additions & 37 deletions

File tree

‎Makefile‎

Lines changed: 30 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -24,36 +24,32 @@ LOAD_TEST_DIR := $(TOOLS_DIR)/tokenserver/loadtests
2424
SYNCSTORAGE_LOAD_TEST_DIR := $(TOOLS_DIR)/syncstorage-loadtest
2525
RUST_LOG ?= debug
2626

27+
SYNC_SYNCSTORAGE__DATABASE_URL ?= mysql://sample_user:sample_password@localhost/syncstorage_rs
28+
SYNC_TOKENSERVER__DATABASE_URL ?= mysql://sample_user:sample_password@localhost/tokenserver_rs
29+
30+
PYTHON_SITE_PACKAGES = $(shell poetry run python -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())")
31+
2732
# In order to be consumed by the ETE Test Metric Pipeline, files need to follow a strict naming
2833
# convention: {job_number}__{utc_epoch_datetime}__{repository}__{workflow}__{test_suite}__results{-index}.xml
2934
EPOCH_TIME := $(shell date +"%s")
3035
WORKFLOW := $(shell echo "$(GITHUB_WORKFLOW)" | tr ' ' '-' | tr '[:upper:]' '[:lower:]')
3136
TEST_RESULTS_DIR ?= workflow/test-results
3237
TEST_PROFILE := $(if $(or $(CIRCLECI),$(GITHUB_ACTIONS)),ci,default)
3338
TEST_FILE_PREFIX := $(if $(GITHUB_ACTIONS),$(GITHUB_RUN_NUMBER)__$(EPOCH_TIME)__$(notdir $(GITHUB_REPOSITORY))__$(WORKFLOW)__)
34-
UNIT_JUNIT_XML := $(TEST_RESULTS_DIR)/$(TEST_FILE_PREFIX)unit__results.xml
35-
MYSQL_UNIT_JUNIT_XML := $(TEST_RESULTS_DIR)/$(TEST_FILE_PREFIX)mysql_unit__results.xml
36-
POSTGRES_UNIT_JUNIT_XML := $(TEST_RESULTS_DIR)/$(TEST_FILE_PREFIX)postgres_unit__results.xml
37-
SPANNER_UNIT_JUNIT_XML := $(TEST_RESULTS_DIR)/$(TEST_FILE_PREFIX)spanner_unit__results.xml
38-
MYSQL_COVERAGE_JSON := $(TEST_RESULTS_DIR)/$(TEST_FILE_PREFIX)mysql_unit__coverage.json
39-
POSTGRES_COVERAGE_JSON := $(TEST_RESULTS_DIR)/$(TEST_FILE_PREFIX)postgres_unit__coverage.json
40-
SPANNER_COVERAGE_JSON := $(TEST_RESULTS_DIR)/$(TEST_FILE_PREFIX)spanner_unit__coverage.json
41-
UNIT_COVERAGE_JSON := $(TEST_RESULTS_DIR)/$(TEST_FILE_PREFIX)unit__coverage.json
42-
43-
SPANNER_INT_JUNIT_XML := $(TEST_RESULTS_DIR)/$(TEST_FILE_PREFIX)spanner_integration__results.xml
44-
SPANNER_NO_JWK_INT_JUNIT_XML := $(TEST_RESULTS_DIR)/$(TEST_FILE_PREFIX)spanner_no_jwk_integration__results.xml
45-
POSTGRES_INT_JUNIT_XML := $(TEST_RESULTS_DIR)/$(TEST_FILE_PREFIX)postgres_integration__results.xml
46-
POSTGRES_NO_JWK_INT_JUNIT_XML := $(TEST_RESULTS_DIR)/$(TEST_FILE_PREFIX)postgres_no_jwk_integration__results.xml
47-
MYSQL_INT_JUNIT_XML := $(TEST_RESULTS_DIR)/$(TEST_FILE_PREFIX)mysql_integration__results.xml
48-
MYSQL_NO_JWK_INT_JUNIT_XML := $(TEST_RESULTS_DIR)/$(TEST_FILE_PREFIX)mysql_no_jwk_integration__results.xml
49-
TOKENSERVER_UTILS_JUNIT_XML := $(TEST_RESULTS_DIR)/$(TEST_FILE_PREFIX)tokenserver_utils__results.xml
50-
51-
LOCAL_INTEGRATION_JUNIT_XML := $(TEST_RESULTS_DIR)/$(TEST_FILE_PREFIX)local_integration__results.xml
52-
SYNC_SYNCSTORAGE__DATABASE_URL ?= mysql://sample_user:sample_password@localhost/syncstorage_rs
53-
SYNC_TOKENSERVER__DATABASE_URL ?= mysql://sample_user:sample_password@localhost/tokenserver_rs
5439

55-
SRC_ROOT = $(shell pwd)
56-
PYTHON_SITE_PACKAGES = $(shell poetry run python -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())")
40+
UNIT_MYSQL_JUNIT_XML := $(TEST_RESULTS_DIR)/$(TEST_FILE_PREFIX)unit__mysql-results.xml
41+
UNIT_POSTGRES_JUNIT_XML := $(TEST_RESULTS_DIR)/$(TEST_FILE_PREFIX)unit__postgres-results.xml
42+
UNIT_SPANNER_JUNIT_XML := $(TEST_RESULTS_DIR)/$(TEST_FILE_PREFIX)unit__spanner-results.xml
43+
UNIT_COVERAGE_MYSQL_JSON := $(TEST_RESULTS_DIR)/$(TEST_FILE_PREFIX)unit__mysql-coverage.json
44+
UNIT_COVERAGE_POSTGRES_JSON := $(TEST_RESULTS_DIR)/$(TEST_FILE_PREFIX)unit__postgres-coverage.json
45+
UNIT_COVERAGE_SPANNER_JSON := $(TEST_RESULTS_DIR)/$(TEST_FILE_PREFIX)unit__spanner-coverage.json
46+
47+
INT_SPANNER_JUNIT_XML := $(TEST_RESULTS_DIR)/$(TEST_FILE_PREFIX)integration__spanner-results.xml
48+
INT_SPANNER_NO_JWK_JUNIT_XML := $(TEST_RESULTS_DIR)/$(TEST_FILE_PREFIX)integration__spanner-no-jwk-results.xml
49+
INT_POSTGRES_JUNIT_XML := $(TEST_RESULTS_DIR)/$(TEST_FILE_PREFIX)integration__postgres-results.xml
50+
INT_POSTGRES_NO_JWK_JUNIT_XML := $(TEST_RESULTS_DIR)/$(TEST_FILE_PREFIX)integration__postgres-no-jwk-results.xml
51+
INT_MYSQL_JUNIT_XML := $(TEST_RESULTS_DIR)/$(TEST_FILE_PREFIX)integration__mysql-results.xml
52+
INT_MYSQL_NO_JWK_JUNIT_XML := $(TEST_RESULTS_DIR)/$(TEST_FILE_PREFIX)integration__mysql-no-jwk-results.xml
5753

5854
clippy_mysql:
5955
# Matches what's run in circleci
@@ -117,16 +113,15 @@ docker_run_mysql_e2e_tests:
117113
up \
118114
--exit-code-from e2e-tests \
119115
--abort-on-container-exit || exit_code=$$?
120-
docker cp mysql-e2e-tests:/mysql_no_jwk_integration_results.xml ${MYSQL_NO_JWK_INT_JUNIT_XML}
116+
docker cp mysql-e2e-tests:/mysql_no_jwk_integration_results.xml ${INT_MYSQL_NO_JWK_JUNIT_XML}
121117
RESULTS_FILENAME=mysql_integration_results.xml docker compose \
122118
-f docker/docker-compose.mysql.yaml \
123119
-f docker/docker-compose.e2e.mysql.yaml \
124120
-f docker/docker-compose.e2e.jwk-cache.yaml \
125121
up \
126122
--exit-code-from e2e-tests \
127123
--abort-on-container-exit || exit_code=$$?
128-
docker cp mysql-e2e-tests:/mysql_integration_results.xml ${MYSQL_INT_JUNIT_XML}
129-
docker cp mysql-e2e-tests:/tokenserver_utils_results.xml ${TOKENSERVER_UTILS_JUNIT_XML}
124+
docker cp mysql-e2e-tests:/mysql_integration_results.xml ${INT_MYSQL_JUNIT_XML}
130125
docker compose \
131126
-f docker/docker-compose.mysql.yaml \
132127
-f docker/docker-compose.e2e.mysql.yaml \
@@ -151,16 +146,15 @@ docker_run_postgres_e2e_tests:
151146
up \
152147
--exit-code-from e2e-tests \
153148
--abort-on-container-exit || exit_code=$$?
154-
docker cp postgres-e2e-tests:/postgres_no_jwk_integration_results.xml ${POSTGRES_NO_JWK_INT_JUNIT_XML}
149+
docker cp postgres-e2e-tests:/postgres_no_jwk_integration_results.xml ${INT_POSTGRES_NO_JWK_JUNIT_XML}
155150
RESULTS_FILENAME=postgres_integration_results.xml docker compose \
156151
-f docker/docker-compose.postgres.yaml \
157152
-f docker/docker-compose.e2e.postgres.yaml \
158153
-f docker/docker-compose.e2e.jwk-cache.yaml \
159154
up \
160155
--exit-code-from e2e-tests \
161156
--abort-on-container-exit || exit_code=$$?
162-
docker cp postgres-e2e-tests:/postgres_integration_results.xml ${POSTGRES_INT_JUNIT_XML}
163-
docker cp postgres-e2e-tests:/tokenserver_utils_results.xml ${TOKENSERVER_UTILS_JUNIT_XML}
157+
docker cp postgres-e2e-tests:/postgres_integration_results.xml ${INT_POSTGRES_JUNIT_XML}
164158
docker compose \
165159
-f docker/docker-compose.postgres.yaml \
166160
-f docker/docker-compose.e2e.postgres.yaml \
@@ -185,16 +179,15 @@ docker_run_spanner_e2e_tests:
185179
up \
186180
--exit-code-from e2e-tests \
187181
--abort-on-container-exit || exit_code=$$?
188-
docker cp spanner-e2e-tests:/spanner_no_jwk_integration_results.xml ${SPANNER_NO_JWK_INT_JUNIT_XML}
182+
docker cp spanner-e2e-tests:/spanner_no_jwk_integration_results.xml ${INT_SPANNER_NO_JWK_JUNIT_XML}
189183
RESULTS_FILENAME=spanner_integration_results.xml docker compose \
190184
-f docker/docker-compose.spanner.yaml \
191185
-f docker/docker-compose.e2e.spanner.yaml \
192186
-f docker/docker-compose.e2e.jwk-cache.yaml \
193187
up \
194188
--exit-code-from e2e-tests \
195189
--abort-on-container-exit || exit_code=$$?
196-
docker cp spanner-e2e-tests:/spanner_integration_results.xml ${SPANNER_INT_JUNIT_XML}
197-
docker cp spanner-e2e-tests:/tokenserver_utils_results.xml ${TOKENSERVER_UTILS_JUNIT_XML}
190+
docker cp spanner-e2e-tests:/spanner_integration_results.xml ${INT_SPANNER_JUNIT_XML}
198191
docker compose \
199192
-f docker/docker-compose.spanner.yaml \
200193
-f docker/docker-compose.e2e.spanner.yaml \
@@ -231,23 +224,23 @@ test_with_coverage:
231224
SYNC_SYNCSTORAGE__DATABASE_URL=${SYNC_SYNCSTORAGE__DATABASE_URL} \
232225
SYNC_TOKENSERVER__DATABASE_URL=${SYNC_TOKENSERVER__DATABASE_URL} \
233226
RUST_TEST_THREADS=1 \
234-
cargo llvm-cov --summary-only --json --output-path ${MYSQL_COVERAGE_JSON} \
227+
cargo llvm-cov --summary-only --json --output-path ${UNIT_COVERAGE_MYSQL_JSON} \
235228
nextest --workspace --profile ${TEST_PROFILE}; exit_code=$$?
236-
mv target/nextest/${TEST_PROFILE}/junit.xml ${MYSQL_UNIT_JUNIT_XML}
229+
mv target/nextest/${TEST_PROFILE}/junit.xml ${UNIT_MYSQL_JUNIT_XML}
237230
exit $$exit_code
238231

239232
.ONESHELL:
240233
spanner_test_with_coverage:
241-
cargo llvm-cov --summary-only --json --output-path ${SPANNER_COVERAGE_JSON} \
234+
cargo llvm-cov --summary-only --json --output-path ${UNIT_COVERAGE_SPANNER_JSON} \
242235
nextest --workspace --no-default-features --features=syncstorage-db/spanner --features=py_verifier --profile ${TEST_PROFILE}; exit_code=$$?
243-
mv target/nextest/${TEST_PROFILE}/junit.xml ${SPANNER_UNIT_JUNIT_XML}
236+
mv target/nextest/${TEST_PROFILE}/junit.xml ${UNIT_SPANNER_JUNIT_XML}
244237
exit $$exit_code
245238

246239
.ONESHELL:
247240
postgres_test_with_coverage:
248-
cargo llvm-cov --summary-only --json --output-path ${POSTGRES_COVERAGE_JSON} \
241+
cargo llvm-cov --summary-only --json --output-path ${UNIT_COVERAGE_POSTGRES_JSON} \
249242
nextest --workspace --no-default-features --features=syncstorage-db/postgres --features=tokenserver-db/postgres --features=py_verifier --profile ${TEST_PROFILE}; exit_code=$$?
250-
mv target/nextest/${TEST_PROFILE}/junit.xml ${POSTGRES_UNIT_JUNIT_XML}
243+
mv target/nextest/${TEST_PROFILE}/junit.xml ${UNIT_POSTGRES_JUNIT_XML}
251244
exit $$exit_code
252245

253246
.ONESHELL:

0 commit comments

Comments
 (0)