Merge branch 'master' into chore/cargo-upgrade-stor-436

Author: chenba

.circleci/config.yml

@@ -46,12 +46,11 @@ commands:
           name: Setup python
           command: |
             sudo apt-get update && sudo apt-get install -y python3-dev python3-pip python3-venv
-            pip3 install hawkauthlib konfig pyramid pyramid_hawkauth requests simplejson unittest2 WebTest WSGIProxy2 poetry
+            pip3 install hawkauthlib konfig pyramid pyramid_hawkauth requests simplejson unittest2 WebTest WSGIProxy2 poetry poetry-plugin-export
             # NOTE: Python3.12 requires `--break-system-packages`.
             # This command is run on the cimg/rust image, which is running python 3.10
             python3 -mvenv venv
             . venv/bin/activate
-            poetry self add poetry-plugin-export
             poetry install --with tokenserver-unit-tests,dev --no-interaction --no-ansi
   rust-check:
     steps:
@@ -202,6 +201,7 @@ commands:
           environment:
             SYNC_SYNCSTORAGE__DATABASE_URL: spanner://projects/test-project/instances/test-instance/databases/test-database
             SYNC_SYNCSTORAGE__SPANNER_EMULATOR_HOST: 127.0.0.1:9010
+            SYNC_TOKENSERVER__DATABASE_URL: mysql://test:test@127.0.0.1/tokenserver
             RUST_TEST_THREADS: 1
           command: make spanner_test_with_coverage
 
@@ -412,11 +412,22 @@ jobs:
           # XXX: begin_test_transaction doesn't play nice over threaded tests
           RUST_TEST_THREADS: 1
       - image: gcr.io/cloud-spanner-emulator/emulator
+      - image: cimg/mysql:8.0
+        auth:
+          username: $DOCKER_USER
+          password: $DOCKER_PASS
+        environment:
+          MYSQL_ROOT_PASSWORD: password
+          MYSQL_USER: test
+          MYSQL_PASSWORD: test
+          MYSQL_DATABASE: syncstorage
     resource_class: xlarge
     steps:
       - checkout
       - display-versions
       - setup-python
+      - setup-mysql
+      - create-tokenserver-database-mysql
       # XXX: currently the time needed to setup-sccache negates its savings
       # - setup-sccache
       # - restore-sccache-cache

.github/workflows/mozcloud-publish.yaml

@@ -18,6 +18,9 @@ on:
   workflow_dispatch: {}
 
 jobs:
+  # Note: we are moving towards renaming all images `syncserver`, the union of sync and tokenserver.
+  # This presently remains for the time being to simplify deploys by maintaining `image_name: syncstorage-rs`.
+  # Historical context: <https://github.com/mozilla-services/syncstorage-rs/pull/1306>
   build-and-push-syncstorage-rs:
     if: >
       github.event_name == 'workflow_dispatch' ||
@@ -44,7 +47,7 @@ jobs:
         MYSQLCLIENT_PKG=libmysqlclient-dev
       should_tag_ghcr: true
 
-  build-and-push-syncstorage-rs-postgres:
+  build-and-push-syncserver-postgres:
     if: >
       github.event_name == 'workflow_dispatch' ||
       (
@@ -62,14 +65,17 @@ jobs:
       packages: write
     uses: mozilla-it/deploy-actions/.github/workflows/build-and-push.yml@4784cb70739a4f32ce010921f60fb1ebbc791a38 # v6.2.2
     with:
-      image_name: syncstorage-rs-postgres
+      image_name: syncserver-postgres
       gar_name: sync-prod
       project_id: moz-fx-sync-prod
       docker_build_args: |
         SYNCSTORAGE_DATABASE_BACKEND=postgres
         TOKENSERVER_DATABASE_BACKEND=postgres
       should_tag_ghcr: true
 
+  # Note: we are moving towards renaming all images `syncserver`, the union of sync and tokenserver.
+  # This presently remains for the time being to simplify deploys by maintaining `image_name: syncstorage-rs-spanner-python-utils`.
+  # Historical context: <https://github.com/mozilla-services/syncstorage-rs/pull/1306>
   build-and-push-syncstorage-rs-spanner-python-utils:
     if: >
       github.event_name == 'workflow_dispatch' ||
@@ -95,7 +101,7 @@ jobs:
       image_build_context: tools/spanner
       should_tag_ghcr: true
 
-  build-and-push-syncstorage-rs-postgres-python-utils:
+  build-and-push-syncserver-postgres-python-utils:
     if: >
       github.event_name == 'workflow_dispatch' ||
       (
@@ -113,14 +119,14 @@ jobs:
       packages: write
     uses: mozilla-it/deploy-actions/.github/workflows/build-and-push.yml@4784cb70739a4f32ce010921f60fb1ebbc791a38 # v6.2.2
     with:
-      image_name: syncstorage-rs-postgres-python-utils
+      image_name: syncserver-postgres-python-utils
       gar_name: sync-prod
       project_id: moz-fx-sync-prod
       dockerfile_path: tools/postgres/Dockerfile
       image_build_context: tools/postgres
       should_tag_ghcr: true
 
-  build-and-push-syncstorage-rs-mysql:
+  build-and-push-syncserver-mysql:
     if: >
       github.event_name == 'workflow_dispatch' ||
       (
@@ -138,7 +144,7 @@ jobs:
       packages: write
     uses: mozilla-it/deploy-actions/.github/workflows/build-and-push.yml@4784cb70739a4f32ce010921f60fb1ebbc791a38 # v6.2.2
     with:
-      image_name: syncstorage-rs-mysql
+      image_name: syncserver-mysql
       gar_name: sync-prod
       project_id: moz-fx-sync-prod
       docker_build_args: |

.github/workflows/spanner.yml

@@ -25,6 +25,21 @@ jobs:
           - 9010:9010
           - 9020:9020
 
+      mysql:
+        image: mysql:8.0
+        env:
+          MYSQL_ROOT_PASSWORD: password
+          MYSQL_USER: test
+          MYSQL_PASSWORD: test
+          MYSQL_DATABASE: syncstorage
+        ports:
+          - 3306:3306
+        options: >-
+          --health-cmd="mysqladmin ping"
+          --health-interval=10s
+          --health-timeout=5s
+          --health-retries=3
+
     env:
       # The code expects a spanner URL like:
       SYNC_SYNCSTORAGE__DATABASE_URL: spanner://projects/test-project/instances/test-instance/databases/test-database
@@ -86,10 +101,16 @@ jobs:
           # prepare-spanner.sh uses the REST API (port 9020)
           scripts/prepare-spanner.sh
 
+      - name: Create Tokenserver database
+        run: |
+          mysql -u root -ppassword -h 127.0.0.1 -e 'CREATE DATABASE tokenserver;'
+          mysql -u root -ppassword -h 127.0.0.1 -e "GRANT ALL ON tokenserver.* to 'test'@'%';"
+
       - name: Run Spanner unit tests with coverage
         env:
           SYNC_SYNCSTORAGE__DATABASE_URL: spanner://projects/test-project/instances/test-instance/databases/test-database
           SYNC_SYNCSTORAGE__SPANNER_EMULATOR_HOST: localhost:9010
+          SYNC_TOKENSERVER__DATABASE_URL: mysql://test:test@127.0.0.1/tokenserver
           RUST_TEST_THREADS: 1
         run: make spanner_test_with_coverage
 

Makefile

@@ -163,7 +163,7 @@ test_with_coverage:
 .ONESHELL:
 spanner_test_with_coverage:
 	cargo llvm-cov --summary-only --json --output-path ${SPANNER_COVERAGE_JSON} \
-		nextest --workspace --no-default-features --features=syncstorage-db/spanner --features=py_verifier --profile ${TEST_PROFILE} || true; exit_code=$$?
+		nextest --workspace --no-default-features --features=syncstorage-db/spanner --features=py_verifier --profile ${TEST_PROFILE}; exit_code=$$?
 	mv target/nextest/${TEST_PROFILE}/junit.xml ${SPANNER_UNIT_JUNIT_XML}
 	exit $$exit_code