Skip to content

Commit d253073

Browse files
taddestaddes
committed
spanner
1 parent 99188cb commit d253073

1 file changed

Lines changed: 288 additions & 1 deletion

File tree

‎.github/workflows/checks.yml‎

Lines changed: 288 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -253,6 +253,9 @@ jobs:
253253
~/.cache/pypoetry/virtualenvs
254254
key: ${{ runner.os }}-python-${{ hashFiles('pyproject.toml', 'poetry.lock') }}
255255

256+
- name: Create test results directory
257+
run: mkdir -p workflow/test-results
258+
256259
- name: Install PostgreSQL client
257260
run: sudo apt-get update && sudo apt-get install -y postgresql-client
258261

@@ -457,7 +460,7 @@ jobs:
457460
process_gcloudignore: false
458461

459462
# MySQL
460-
build-and-test-mysql:
463+
build-and-unit-test-mysql:
461464
runs-on: ubuntu-latest
462465
needs: [rust-env, python-env]
463466
permissions:
@@ -508,6 +511,9 @@ jobs:
508511
~/.cache/pypoetry/virtualenvs
509512
key: ${{ runner.os }}-python-${{ hashFiles('pyproject.toml', 'poetry.lock') }}
510513

514+
- name: Create test results directory
515+
run: mkdir -p workflow/test-results
516+
511517
- name: Install MySQL client
512518
run: sudo apt-get update && sudo apt-get install -y default-mysql-client
513519

@@ -699,3 +705,284 @@ jobs:
699705
glob: "*.xml"
700706
parent: false
701707
process_gcloudignore: false
708+
709+
# Spanner
710+
build-and-unit-test-spanner:
711+
runs-on: ubuntu-latest
712+
needs: [rust-env, python-env]
713+
permissions:
714+
contents: read
715+
checks: write
716+
717+
services:
718+
spanner-emulator:
719+
image: gcr.io/cloud-spanner-emulator/emulator:1.4.0
720+
ports:
721+
- 9010:9010
722+
- 9020:9020
723+
724+
mysql:
725+
image: mysql:8.0
726+
env:
727+
MYSQL_ROOT_PASSWORD: password
728+
MYSQL_USER: test
729+
MYSQL_PASSWORD: test
730+
MYSQL_DATABASE: syncstorage
731+
ports:
732+
- 3306:3306
733+
options: >-
734+
--health-cmd="mysqladmin ping"
735+
--health-interval=10s
736+
--health-timeout=5s
737+
--health-retries=3
738+
739+
env:
740+
# The code expects a spanner URL like:
741+
SYNC_SYNCSTORAGE__DATABASE_URL: spanner://projects/test-project/instances/test-instance/databases/test-database
742+
RUST_BACKTRACE: 1
743+
RUST_TEST_THREADS: 1
744+
745+
steps:
746+
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
747+
with:
748+
persist-credentials: false
749+
750+
- name: Restore Rust toolchain
751+
uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5
752+
with:
753+
path: |
754+
~/.rustup/toolchains
755+
~/.rustup/update-hashes
756+
key: ${{ runner.os }}-rust-toolchain-${{ env.RUST_VERSION }}
757+
758+
- name: Restore pip and Poetry virtualenv
759+
uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5
760+
with:
761+
path: |
762+
~/.cache/pip
763+
~/.cache/pypoetry/virtualenvs
764+
key: ${{ runner.os }}-python-${{ hashFiles('pyproject.toml', 'poetry.lock') }}
765+
766+
- name: Create test results directory
767+
run: mkdir -p workflow/test-results
768+
769+
- name: Create version.json
770+
run: |
771+
printf '{"commit":"%s","version":"%s","source":"https://github.com/%s/%s","build":"%s"}\n' \
772+
"${GITHUB_SHA}" \
773+
"${GITHUB_REF_NAME}" \
774+
"${GITHUB_REPOSITORY_OWNER}" \
775+
"${GITHUB_REPOSITORY_NAME}" \
776+
"${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \
777+
> version.json
778+
env:
779+
GITHUB_REPOSITORY_NAME: ${{ github.event.repository.name }}
780+
781+
- name: Install cargo-nextest
782+
run: curl -LsSf https://get.nexte.st/latest/linux | tar zxf - -C ${CARGO_HOME:-~/.cargo}/bin
783+
784+
- name: Install cargo-llvm-cov
785+
run: cargo install --locked cargo-llvm-cov
786+
787+
- name: Build workspace (spanner feature)
788+
run: |
789+
# Build with the spanner feature so any compile-time issues surface early
790+
cargo build --workspace --no-default-features --features=syncstorage-db/spanner --features=py_verifier
791+
792+
- name: Wait for Spanner Emulator to be ready
793+
run: |
794+
echo "Waiting for Spanner emulator to be ready..."
795+
for i in {1..30}; do
796+
if curl -s http://localhost:9020/ > /dev/null 2>&1; then
797+
echo "Spanner emulator is ready (REST port 9020 responding)"
798+
break
799+
fi
800+
echo "Attempt $i/30: Spanner emulator not ready yet, waiting..."
801+
sleep 2
802+
done
803+
# Verify both ports are accessible
804+
if ! curl -s http://localhost:9020/ > /dev/null 2>&1; then
805+
echo "ERROR: Cannot connect to Spanner emulator REST API at localhost:9020"
806+
exit 1
807+
fi
808+
echo "Spanner emulator is fully ready"
809+
810+
- name: Setup Spanner schema & instance (prepare-spanner.sh)
811+
env:
812+
SYNC_SYNCSTORAGE__DATABASE_URL: spanner://projects/test-project/instances/test-instance/databases/test-database
813+
SYNC_SYNCSTORAGE__SPANNER_EMULATOR_HOST: http://localhost:9020
814+
run: |
815+
# prepare-spanner.sh uses the REST API (port 9020)
816+
scripts/prepare-spanner.sh
817+
818+
- name: Create Tokenserver database
819+
run: |
820+
mysql -u root -ppassword -h 127.0.0.1 -e 'CREATE DATABASE tokenserver;'
821+
mysql -u root -ppassword -h 127.0.0.1 -e "GRANT ALL ON tokenserver.* to 'test'@'%';"
822+
823+
- name: Run Spanner unit tests with coverage
824+
env:
825+
SYNC_SYNCSTORAGE__DATABASE_URL: spanner://projects/test-project/instances/test-instance/databases/test-database
826+
SYNC_SYNCSTORAGE__SPANNER_EMULATOR_HOST: localhost:9010
827+
SYNC_TOKENSERVER__DATABASE_URL: mysql://test:test@127.0.0.1/tokenserver
828+
SYNC_TOKENSERVER__NODE_TYPE: spanner
829+
RUST_TEST_THREADS: 1
830+
run: make spanner_test_with_coverage
831+
832+
- name: Publish Test Report
833+
uses: dorny/test-reporter@a810f9bf83f2344124a920a7a0a85a6716e791f0
834+
if: always()
835+
with:
836+
name: Spanner Unit Tests
837+
path: workflow/test-results/*.xml
838+
reporter: java-junit
839+
fail-on-error: false
840+
841+
- name: Upload test results
842+
if: always()
843+
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
844+
with:
845+
name: spanner-test-results
846+
path: workflow/test-results/
847+
848+
# Upload to GCS on master
849+
- name: Authenticate to Google Cloud
850+
if: github.ref == 'refs/heads/master'
851+
uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3
852+
with:
853+
credentials_json: ${{ secrets.ETE_GCLOUD_SERVICE_KEY }}
854+
855+
- name: Upload JUnit results to GCS
856+
if: github.ref == 'refs/heads/master'
857+
uses: google-github-actions/upload-cloud-storage@c0f6160ff80057923ff50e5e567695cea181ec23 # v2
858+
with:
859+
path: workflow/test-results
860+
destination: ecosystem-test-eng-metrics/syncstorage-rs/junit
861+
glob: "*.xml"
862+
parent: false
863+
process_gcloudignore: false
864+
865+
- name: Upload coverage results to GCS
866+
if: github.ref == 'refs/heads/master'
867+
uses: google-github-actions/upload-cloud-storage@c0f6160ff80057923ff50e5e567695cea181ec23 # v2
868+
with:
869+
path: workflow/test-results
870+
destination: ecosystem-test-eng-metrics/syncstorage-rs/coverage
871+
glob: "*.json"
872+
parent: false
873+
process_gcloudignore: false
874+
875+
build-spanner-image:
876+
runs-on: ubuntu-latest
877+
needs: [rust-env, python-env]
878+
permissions:
879+
contents: read
880+
actions: write
881+
882+
steps:
883+
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
884+
with:
885+
persist-credentials: false
886+
887+
- name: Create version.json
888+
run: |
889+
printf '{"commit":"%s","version":"%s","source":"https://github.com/%s/%s","build":"%s"}\n' \
890+
"${GITHUB_SHA}" \
891+
"${GITHUB_REF_NAME}" \
892+
"${GITHUB_REPOSITORY_OWNER}" \
893+
"${GITHUB_REPOSITORY_NAME}" \
894+
"${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \
895+
> version.json
896+
env:
897+
GITHUB_REPOSITORY_NAME: ${{ github.event.repository.name }}
898+
899+
- name: Cache Docker image tar
900+
id: cache-spanner-image
901+
uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5
902+
with:
903+
path: /tmp/spanner-image.tar
904+
key: ${{ runner.os }}-spanner-image-${{ hashFiles('Dockerfile', 'Cargo.lock', '**/*.rs', '**/Cargo.toml', 'tools/**', 'scripts/**') }}
905+
- name: Set up Docker Buildx
906+
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
907+
908+
- name: Build Spanner Docker image (local artifact)
909+
uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6
910+
with:
911+
context: .
912+
push: false
913+
tags: app:build
914+
build-args: |
915+
SYNCSTORAGE_DATABASE_BACKEND=spanner
916+
MYSQLCLIENT_PKG=libmysqlclient-dev
917+
outputs: type=docker,dest=/tmp/spanner-image.tar
918+
cache-from: type=gha
919+
cache-to: type=gha,mode=max
920+
921+
- name: Upload Docker image artifact
922+
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
923+
with:
924+
name: spanner-docker-image
925+
path: /tmp/spanner-image.tar
926+
retention-days: 1
927+
928+
spanner-e2e-tests:
929+
runs-on: ubuntu-latest
930+
needs: build-spanner-image
931+
permissions:
932+
contents: read
933+
checks: write
934+
935+
steps:
936+
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
937+
with:
938+
persist-credentials: false
939+
940+
- name: Download Docker image
941+
uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6
942+
with:
943+
name: spanner-docker-image
944+
path: /tmp
945+
946+
- name: Load Docker image
947+
run: docker load --input /tmp/spanner-image.tar
948+
949+
- name: Create test results directory
950+
run: mkdir -p workflow/test-results
951+
952+
- name: Run Spanner e2e tests
953+
run: make docker_run_spanner_e2e_tests
954+
env:
955+
SYNCSTORAGE_RS_IMAGE: app:build
956+
957+
- name: Publish E2E Test Report
958+
uses: dorny/test-reporter@a810f9bf83f2344124a920a7a0a85a6716e791f0
959+
if: always()
960+
with:
961+
name: Spanner E2E Tests
962+
path: workflow/test-results/*.xml
963+
reporter: java-junit
964+
fail-on-error: false
965+
966+
- name: Upload e2e test results
967+
if: always()
968+
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
969+
with:
970+
name: spanner-e2e-test-results
971+
path: workflow/test-results/
972+
973+
# Upload to GCS on master
974+
- name: Authenticate to Google Cloud
975+
if: github.ref == 'refs/heads/master'
976+
uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3
977+
with:
978+
credentials_json: ${{ secrets.ETE_GCLOUD_SERVICE_KEY }}
979+
980+
- name: Upload e2e test results to GCS
981+
if: github.ref == 'refs/heads/master'
982+
uses: google-github-actions/upload-cloud-storage@c0f6160ff80057923ff50e5e567695cea181ec23 # v2
983+
with:
984+
path: workflow/test-results
985+
destination: ecosystem-test-eng-metrics/syncstorage-rs/junit
986+
glob: "*.xml"
987+
parent: false
988+
process_gcloudignore: false

0 commit comments

Comments
 (0)