diff --git a/.ci/complement_package.gotpl b/.ci/complement_package.gotpl index 2dd5e5e0e6..e71ed9f616 100644 --- a/.ci/complement_package.gotpl +++ b/.ci/complement_package.gotpl @@ -29,7 +29,7 @@ which is under the Unlicense licence. {{- with .TestCases -}} {{- /* Passing tests are first */ -}} {{- range . -}} - {{- if eq .Result "PASS" -}} + {{- if and (eq .Result "PASS") (not $settings.HideSuccessfulTests) -}} ::group::{{ "\033" }}[0;32m✅{{ " " }}{{- .Name -}} {{- "\033" -}}[0;37m ({{if $settings.ShowTestStatus}}{{.Result}}; {{end}}{{ .Duration -}} {{- with .Coverage -}} @@ -49,7 +49,8 @@ which is under the Unlicense licence. {{- /* Then skipped tests are second */ -}} {{- range . -}} - {{- if eq .Result "SKIP" -}} + {{- /* Skipped tests are also purposefully hidden if "HideSuccessfulTests" is enabled */ -}} + {{- if and (eq .Result "SKIP") (not $settings.HideSuccessfulTests) -}} ::group::{{ "\033" }}[0;33m🚧{{ " " }}{{- .Name -}} {{- "\033" -}}[0;37m ({{if $settings.ShowTestStatus}}{{.Result}}; {{end}}{{ .Duration -}} {{- with .Coverage -}} diff --git a/.ci/scripts/calculate_builds.py b/.ci/scripts/calculate_builds.py index 7358f3ba86..3ef31fa601 100755 --- a/.ci/scripts/calculate_builds.py +++ b/.ci/scripts/calculate_builds.py @@ -77,11 +77,13 @@ def set_output(key: str, value: str) -> None: "sta-version": "0.13.0", "fcs-version": "0.0.2", }, + "mod026": {"sic-version": "0.2.2", "sta-version": "0.13.1", "fcs-version": "0.0.2"}, + "mod027": {"sic-version": "0.5.0", "sta-version": "0.13.1", "fcs-version": "0.0.2"}, } # Adjust this section to decide what gets built and layered on top # THIS IS THE SECTION TO EDIT, after you have added the new versions above -current_mod_packs_to_build = ["mod024", "mod025"] +current_mod_packs_to_build = ["mod026", "mod027"] generated_jobs: list[dict[str, Any]] = [] for mod_pack_job in current_mod_packs_to_build: diff --git a/.ci/scripts/gotestfmt b/.ci/scripts/gotestfmt deleted file mode 100755 index 83e0ec6361..0000000000 --- a/.ci/scripts/gotestfmt +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash -# -# wraps `gotestfmt`, hiding output from successful packages unless -# all tests passed. - -set -o pipefail -set -e - -# tee the test results to a log, whilst also piping them into gotestfmt, -# telling it to hide successful results, so that we can clearly see -# unsuccessful results. -tee complement.log | gotestfmt -hide successful-packages - -# gotestfmt will exit non-zero if there were any failures, so if we got to this -# point, we must have had a successful result. -echo "All tests successful; showing all test results" - -# Pipe the test results back through gotestfmt, showing all results. -# The log file consists of JSON lines giving the test results, interspersed -# with regular stdout lines (including reports of downloaded packages). -grep '^{"Time":' complement.log | gotestfmt diff --git a/.ci/scripts/setup_complement_prerequisites.sh b/.ci/scripts/setup_complement_prerequisites.sh index 47a3ff8e69..1fdaf5e631 100755 --- a/.ci/scripts/setup_complement_prerequisites.sh +++ b/.ci/scripts/setup_complement_prerequisites.sh @@ -10,7 +10,6 @@ alias block='{ set +x; } 2>/dev/null; func() { echo "::group::$*"; set -x; }; fu alias endblock='{ set +x; } 2>/dev/null; func() { echo "::endgroup::"; set -x; }; func' block Install Complement Dependencies - sudo apt-get -qq update && sudo apt-get install -qqy libolm3 libolm-dev go install -v github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@latest endblock diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index b1dd8a75e7..051c66ebba 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -28,7 +28,7 @@ jobs: steps: - name: Set up Docker Buildx id: buildx - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 + uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -41,20 +41,20 @@ jobs: echo "SYNAPSE_VERSION=$(grep "^version" pyproject.toml | sed -E 's/version\s*=\s*["]([^"]*)["]/\1/')" >> $GITHUB_ENV - name: Log in to DockerHub - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 + uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Log in to GHCR - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 + uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - name: Tailscale - uses: tailscale/github-action@53acf823325fe9ca47f4cdaa951f90b4b0de5bb9 # v4.1.1 + uses: tailscale/github-action@306e68a486fd2350f2bfc3b19fcd143891a4a2d8 # v4.1.2 with: oauth-client-id: ${{ secrets.TS_OAUTH_CLIENT_ID }} audience: ${{ secrets.TS_AUDIENCE }} @@ -79,7 +79,7 @@ jobs: services/backend-repositories/secret/data/oci.element.io password | OCI_PASSWORD ; - name: Login to Element OCI Registry - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 + uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0 with: registry: oci-push.vpn.infra.element.io username: ${{ steps.import-secrets.outputs.OCI_USERNAME }} @@ -87,7 +87,7 @@ jobs: - name: Build and push by digest id: build - uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2 + uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7.0.0 with: push: true labels: | @@ -108,7 +108,7 @@ jobs: touch "${{ runner.temp }}/digests/${digest#sha256:}" - name: Upload digest - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: digests-${{ matrix.suffix }} path: ${{ runner.temp }}/digests/* @@ -129,21 +129,21 @@ jobs: - build steps: - name: Download digests - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: path: ${{ runner.temp }}/digests pattern: digests-* merge-multiple: true - name: Log in to DockerHub - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 + uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0 if: ${{ startsWith(matrix.repository, 'docker.io') }} with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Log in to GHCR - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 + uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0 if: ${{ startsWith(matrix.repository, 'ghcr.io') }} with: registry: ghcr.io @@ -151,7 +151,7 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Tailscale - uses: tailscale/github-action@53acf823325fe9ca47f4cdaa951f90b4b0de5bb9 # v4.1.1 + uses: tailscale/github-action@306e68a486fd2350f2bfc3b19fcd143891a4a2d8 # v4.1.2 with: oauth-client-id: ${{ secrets.TS_OAUTH_CLIENT_ID }} audience: ${{ secrets.TS_AUDIENCE }} @@ -176,20 +176,20 @@ jobs: services/backend-repositories/secret/data/oci.element.io password | OCI_PASSWORD ; - name: Login to Element OCI Registry - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 + uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0 with: registry: oci-push.vpn.infra.element.io username: ${{ steps.import-secrets.outputs.OCI_USERNAME }} password: ${{ steps.import-secrets.outputs.OCI_PASSWORD }} - name: Set up Docker Buildx - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 + uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 - name: Install Cosign uses: sigstore/cosign-installer@faadad0cce49287aee09b3a48701e75088a2c6ad # v4.0.0 - name: Calculate docker image tag - uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # v5.10.0 + uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v6.0.0 with: images: ${{ matrix.repository }} flavor: | diff --git a/.github/workflows/docs-pr.yaml b/.github/workflows/docs-pr.yaml index 57f69f2524..ba5baf160a 100644 --- a/.github/workflows/docs-pr.yaml +++ b/.github/workflows/docs-pr.yaml @@ -45,7 +45,7 @@ jobs: cp book/welcome_and_overview.html book/index.html - name: Upload Artifact - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: book path: book diff --git a/.github/workflows/famedly-tests.yml b/.github/workflows/famedly-tests.yml index 65ab394dff..b2ce55fe0d 100644 --- a/.github/workflows/famedly-tests.yml +++ b/.github/workflows/famedly-tests.yml @@ -507,7 +507,7 @@ jobs: - name: Run token-authenticator tests working-directory: synapse-token-authenticator - run: hatch run pip install -r synapse-requirements.txt && hatch run cov + run: hatch run pip install -r synapse-requirements.txt && hatch test -p -c - name: Display Hatch Environment Info if: always() diff --git a/.github/workflows/fix_lint.yaml b/.github/workflows/fix_lint.yaml index 4752b6afeb..aea55fe0ce 100644 --- a/.github/workflows/fix_lint.yaml +++ b/.github/workflows/fix_lint.yaml @@ -25,7 +25,7 @@ jobs: with: toolchain: ${{ env.RUST_VERSION }} components: clippy, rustfmt - - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2.8.2 + - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 - name: Setup Poetry uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 diff --git a/.github/workflows/latest_deps.yml b/.github/workflows/latest_deps.yml index 2d945c2096..746223ef4a 100644 --- a/.github/workflows/latest_deps.yml +++ b/.github/workflows/latest_deps.yml @@ -47,7 +47,7 @@ jobs: uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master with: toolchain: ${{ env.RUST_VERSION }} - - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2.8.2 + - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 # The dev dependencies aren't exposed in the wheel metadata (at least with current # poetry-core versions), so we install with poetry. @@ -83,7 +83,7 @@ jobs: uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master with: toolchain: ${{ env.RUST_VERSION }} - - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2.8.2 + - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 - run: sudo apt-get -qq install xmlsec1 - name: Set up PostgreSQL ${{ matrix.postgres-version }} @@ -157,7 +157,7 @@ jobs: uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master with: toolchain: ${{ env.RUST_VERSION }} - - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2.8.2 + - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 - name: Ensure sytest runs `pip install` # Delete the lockfile so sytest will `pip install` rather than `poetry install` @@ -172,7 +172,7 @@ jobs: if: ${{ always() }} run: /sytest/scripts/tap_to_gha.pl /logs/results.tap - name: Upload SyTest logs - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 if: ${{ always() }} with: name: Sytest Logs - ${{ job.status }} - (${{ join(matrix.*, ', ') }}) @@ -207,7 +207,7 @@ jobs: - name: Prepare Complement's Prerequisites run: synapse/.ci/scripts/setup_complement_prerequisites.sh - - uses: actions/setup-go@7a3fe6cf4cb3a834922a1244abfce67bcef6a0c5 # v6.2.0 + - uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0 with: cache-dependency-path: complement/go.sum go-version-file: complement/go.mod diff --git a/.github/workflows/push_complement_image.yml b/.github/workflows/push_complement_image.yml index 7793172e55..3931dbebb3 100644 --- a/.github/workflows/push_complement_image.yml +++ b/.github/workflows/push_complement_image.yml @@ -47,15 +47,19 @@ jobs: if: github.event_name == 'push' with: ref: master + # We use `poetry` in `complement.sh` + - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 + with: + poetry-version: "2.2.1" - name: Login to registry - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 + uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Work out labels for complement image id: meta - uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # v5.10.0 + uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v6.0.0 with: images: ghcr.io/${{ github.repository }}/complement-synapse tags: | diff --git a/.github/workflows/release-artifacts.yml b/.github/workflows/release-artifacts.yml index 4343cb482f..c313a64250 100644 --- a/.github/workflows/release-artifacts.yml +++ b/.github/workflows/release-artifacts.yml @@ -61,7 +61,7 @@ jobs: - name: Set up Docker Buildx id: buildx - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 + uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 - name: Set up docker layer caching uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 @@ -99,7 +99,7 @@ jobs: echo "ARTIFACT_NAME=${DISTRO#*:}" >> "$GITHUB_OUTPUT" - name: Upload debs as artifacts - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: debs-${{ steps.artifact-name.outputs.ARTIFACT_NAME }} path: debs/* @@ -150,7 +150,7 @@ jobs: # musl: (TODO: investigate). CIBW_TEST_SKIP: pp3*-* *musl* - - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: Wheel-${{ matrix.os }} path: ./wheelhouse/*.whl @@ -171,7 +171,7 @@ jobs: - name: Build sdist run: python -m build --sdist - - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: Sdist path: dist/*.tar.gz @@ -187,7 +187,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Download all workflow run artifacts - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - name: Build a tarball for the debs # We need to merge all the debs uploads into one folder, then compress # that. diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a03d27472d..488208291f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -26,7 +26,7 @@ jobs: linting: ${{ !startsWith(github.ref, 'refs/pull/') || steps.filter.outputs.linting }} linting_readme: ${{ !startsWith(github.ref, 'refs/pull/') || steps.filter.outputs.linting_readme }} steps: - - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 + - uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1 id: filter # We only check on PRs if: startsWith(github.ref, 'refs/pull/') @@ -91,7 +91,7 @@ jobs: uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master with: toolchain: ${{ env.RUST_VERSION }} - - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2.8.2 + - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 with: python-version: "3.x" @@ -157,7 +157,7 @@ jobs: uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master with: toolchain: ${{ env.RUST_VERSION }} - - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2.8.2 + - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 - name: Setup Poetry uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 @@ -221,7 +221,7 @@ jobs: with: components: clippy toolchain: ${{ env.RUST_VERSION }} - - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2.8.2 + - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 - run: cargo clippy -- -D warnings @@ -240,7 +240,7 @@ jobs: with: toolchain: nightly-2026-02-01 components: clippy - - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2.8.2 + - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 - run: cargo clippy --all-features -- -D warnings @@ -257,7 +257,7 @@ jobs: uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master with: toolchain: ${{ env.RUST_VERSION }} - - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2.8.2 + - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 - name: Setup Poetry uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 @@ -296,7 +296,7 @@ jobs: # `.rustfmt.toml`. toolchain: nightly-2025-04-23 components: rustfmt - - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2.8.2 + - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 - run: cargo fmt --check @@ -393,7 +393,7 @@ jobs: uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master with: toolchain: ${{ env.RUST_VERSION }} - - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2.8.2 + - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 with: @@ -437,7 +437,7 @@ jobs: uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master with: toolchain: ${{ env.RUST_VERSION }} - - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2.8.2 + - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 # There aren't wheels for some of the older deps, so we need to install # their build dependencies @@ -552,7 +552,7 @@ jobs: uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master with: toolchain: ${{ env.RUST_VERSION }} - - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2.8.2 + - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 - name: Run SyTest run: /bootstrap.sh synapse @@ -561,7 +561,7 @@ jobs: if: ${{ always() }} run: /sytest/scripts/tap_to_gha.pl /logs/results.tap - name: Upload SyTest logs - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 if: ${{ always() }} with: name: Sytest Logs - ${{ job.status }} - (${{ join(matrix.job.*, ', ') }}) @@ -658,7 +658,7 @@ jobs: PGPASSWORD: postgres PGDATABASE: postgres - name: "Upload schema differences" - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 if: ${{ failure() && !cancelled() && steps.run_tester_script.outcome == 'failure' }} with: name: Schema dumps @@ -703,7 +703,7 @@ jobs: uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master with: toolchain: ${{ env.RUST_VERSION }} - - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2.8.2 + - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 # We use `poetry` in `complement.sh` - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 @@ -715,7 +715,7 @@ jobs: - name: Prepare Complement's Prerequisites run: synapse/.ci/scripts/setup_complement_prerequisites.sh - - uses: actions/setup-go@7a3fe6cf4cb3a834922a1244abfce67bcef6a0c5 # v6.2.0 + - uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0 with: cache-dependency-path: complement/go.sum go-version-file: complement/go.mod @@ -744,6 +744,12 @@ jobs: - name: Formatted sanity check Complement test logs # Always run this step if we attempted to run the Complement tests. if: always() && steps.run_sanity_check_complement_image_test.outcome != 'skipped' + # We do not hide successful tests in `gotestfmt` here as the list of sanity + # check tests is so short. Feel free to change this when we get more tests. + # + # Note that the `-hide` argument is interpreted by `gotestfmt`. From it, + # it derives several values under `$settings` and passes them to our + # custom `.ci/complement_package.gotpl` template to render the output. run: cat /tmp/gotest-sanity-check-complement.log | gotestfmt -hide "successful-downloads,empty-packages" - name: Run Complement Tests @@ -764,10 +770,15 @@ jobs: POSTGRES: ${{ (matrix.database == 'Postgres') && 1 || '' }} WORKERS: ${{ (matrix.arrangement == 'workers') && 1 || '' }} - - name: Formatted Complement test logs + - name: Formatted Complement test logs (only failing are shown) # Always run this step if we attempted to run the Complement tests. if: always() && steps.run_complement_tests.outcome != 'skipped' - run: cat /tmp/gotest-complement.log | gotestfmt -hide "successful-downloads,empty-packages" + # Hide successful tests in order to reduce the verbosity of the otherwise very large output. + # + # Note that the `-hide` argument is interpreted by `gotestfmt`. From it, + # it derives several values under `$settings` and passes them to our + # custom `.ci/complement_package.gotpl` template to render the output. + run: cat /tmp/gotest-complement.log | gotestfmt -hide "successful-downloads,successful-tests,empty-packages" - name: Run in-repo Complement Tests id: run_in_repo_complement_tests @@ -787,10 +798,15 @@ jobs: POSTGRES: ${{ (matrix.database == 'Postgres') && 1 || '' }} WORKERS: ${{ (matrix.arrangement == 'workers') && 1 || '' }} - - name: Formatted in-repo Complement test logs + - name: Formatted in-repo Complement test logs (only failing are shown) # Always run this step if we attempted to run the Complement tests. if: always() && steps.run_in_repo_complement_tests.outcome != 'skipped' - run: cat /tmp/gotest-in-repo-complement.log | gotestfmt -hide "successful-downloads,empty-packages" + # Hide successful tests in order to reduce the verbosity of the otherwise very large output. + # + # Note that the `-hide` argument is interpreted by `gotestfmt`. From it, + # it derives several values under `$settings` and passes them to our + # custom `.ci/complement_package.gotpl` template to render the output. + run: cat /tmp/gotest-in-repo-complement.log | gotestfmt -hide "successful-downloads,successful-tests,empty-packages" cargo-test: if: ${{ needs.changes.outputs.rust == 'true' }} @@ -806,7 +822,7 @@ jobs: uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master with: toolchain: ${{ env.RUST_VERSION }} - - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2.8.2 + - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 - run: cargo test @@ -826,7 +842,7 @@ jobs: uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master with: toolchain: nightly-2022-12-01 - - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2.8.2 + - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 - run: cargo bench --no-run diff --git a/.github/workflows/twisted_trunk.yml b/.github/workflows/twisted_trunk.yml index d38e38ebcb..be73e5283b 100644 --- a/.github/workflows/twisted_trunk.yml +++ b/.github/workflows/twisted_trunk.yml @@ -48,7 +48,7 @@ jobs: uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master with: toolchain: ${{ env.RUST_VERSION }} - - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2.8.2 + - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 with: @@ -76,7 +76,7 @@ jobs: uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master with: toolchain: ${{ env.RUST_VERSION }} - - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2.8.2 + - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 with: @@ -121,7 +121,7 @@ jobs: uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master with: toolchain: ${{ env.RUST_VERSION }} - - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2.8.2 + - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 - name: Patch dependencies # Note: The poetry commands want to create a virtualenv in /src/.venv/, @@ -145,7 +145,7 @@ jobs: if: ${{ always() }} run: /sytest/scripts/tap_to_gha.pl /logs/results.tap - name: Upload SyTest logs - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 if: ${{ always() }} with: name: Sytest Logs - ${{ job.status }} - (${{ join(matrix.*, ', ') }}) @@ -180,7 +180,7 @@ jobs: - name: Prepare Complement's Prerequisites run: synapse/.ci/scripts/setup_complement_prerequisites.sh - - uses: actions/setup-go@7a3fe6cf4cb3a834922a1244abfce67bcef6a0c5 # v6.2.0 + - uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0 with: cache-dependency-path: complement/go.sum go-version-file: complement/go.mod diff --git a/CHANGES.md b/CHANGES.md index 162f51dc93..51122e5097 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,54 @@ +# Synapse 1.151.0 (2026-04-07) + +## Bugfixes + +- Fix `KNOWN_ROOM_VERSIONS.__contains__` raising `TypeError` for non-string keys, which could cause `/sync` to fail for rooms with a `NULL` room version in the database. Bug introduced in [#19589](https://github.com/element-hq/synapse/pull/19589) as part of v1.151.0rc1. ([\#19649](https://github.com/element-hq/synapse/issues/19649)) + +### Famedly additions for v1.151.0_1 + +- bump: Synapse Token Authenticator to `v0.13.1` in the Famedly docker image +- bump: Synapse Invite Checker to `0.5.0` in the Famedly docker image + + +# Synapse 1.151.0rc1 (2026-03-31) + +## Features + +- Add stable support for [MSC4284](https://github.com/matrix-org/matrix-spec-proposals/pull/4284) Policy Servers. ([\#19503](https://github.com/element-hq/synapse/issues/19503)) +- Update and stabilize support for [MSC2666](https://github.com/matrix-org/matrix-spec-proposals/pull/2666): Get rooms in common with another user. Contributed by @tulir @ Beeper. ([\#19511](https://github.com/element-hq/synapse/issues/19511)) +- Updated experimental support for [MSC4388: Secure out-of-band channel for sign in with QR](https://github.com/matrix-org/matrix-spec-proposals/pull/4388). ([\#19573](https://github.com/element-hq/synapse/issues/19573)) +- Stabilize `room_version` and `encryption` fields in the space/room `/hierarchy` API (part of [MSC3266](https://github.com/matrix-org/matrix-spec-proposals/pull/3266)). ([\#19576](https://github.com/element-hq/synapse/issues/19576)) +- Introduce a [configuration option](https://element-hq.github.io/synapse/latest/usage/configuration/config_documentation.html#matrix_authentication_service) to allow using HTTP/2 over plaintext when Synapse connects to Matrix Authentication Service. ([\#19586](https://github.com/element-hq/synapse/issues/19586)) + +## Bugfixes + +- Fix [MSC4284](https://github.com/matrix-org/matrix-spec-proposals/pull/4284) Policy Servers implementation to skip signing `org.matrix.msc4284.policy` and `m.room.policy` state events. ([\#19503](https://github.com/element-hq/synapse/issues/19503)) +- Correctly apply [MSC4284](https://github.com/matrix-org/matrix-spec-proposals/pull/4284) Policy Server signatures to events when the sender and policy server have the same server name. ([\#19503](https://github.com/element-hq/synapse/issues/19503)) +- Allow Synapse to start up even when discovery fails for an OpenID Connect provider. ([\#19509](https://github.com/element-hq/synapse/issues/19509)) +- Fix quarantine media admin APIs sometimes returning inaccurate counts for remote media. ([\#19559](https://github.com/element-hq/synapse/issues/19559)) +- Fix `Build and push complement image` CI job not having `poetry` available for the Complement runner script. ([\#19578](https://github.com/element-hq/synapse/issues/19578)) +- Increase timeout for policy server requests to avoid repeated requests for checking media. ([\#19629](https://github.com/element-hq/synapse/issues/19629)) + +## Deprecations and Removals + +- Remove support for [MSC3852: Expose user agent information on Device](https://github.com/matrix-org/matrix-spec-proposals/pull/3852) as the MSC was closed. ([\#19430](https://github.com/element-hq/synapse/issues/19430)) + +## Internal Changes + +- Fix small comment typo in config output from the `demo/start.sh` script. ([\#19538](https://github.com/element-hq/synapse/issues/19538)) +- Add MSC3820 comment context to `RoomVersion` attributes. ([\#19577](https://github.com/element-hq/synapse/issues/19577)) +- Remove `redacted_because` from internal unsigned. ([\#19581](https://github.com/element-hq/synapse/issues/19581)) +- Prevent sending registration emails if registration is disabled. ([\#19585](https://github.com/element-hq/synapse/issues/19585)) +- Port `RoomVersion` to Rust. ([\#19589](https://github.com/element-hq/synapse/issues/19589)) +- Only show failing Complement tests in the formatted output in CI. ([\#19590](https://github.com/element-hq/synapse/issues/19590)) +- Ensure old Complement test files are removed when downloading a Complement checkout via `./scripts-dev/complement.sh`. ([\#19592](https://github.com/element-hq/synapse/issues/19592)) +- Update `HomeserverTestCase.pump()` docstring to demystify behavior (Twisted reactor/clock). ([\#19602](https://github.com/element-hq/synapse/issues/19602)) +- Deprecate `HomeserverTestCase.pump()` in favor of more direct `HomeserverTestCase.reactor.advance(...)` usage. ([\#19602](https://github.com/element-hq/synapse/issues/19602)) +- Lower the Postgres database `statement_timeout` to 10m (previously 1h). ([\#19604](https://github.com/element-hq/synapse/issues/19604)) + + + + # Synapse 1.150.0 (2026-03-24) No significant changes since 1.150.0rc1. diff --git a/Cargo.lock b/Cargo.lock index d6945bfbb7..5cb2ab58af 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -748,9 +748,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.21.3" +version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "openssl-probe" @@ -1116,9 +1116,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.4" +version = "0.103.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a17884ae0c1b773f1ccd2bd4a8c72f16da897310a98b0e84bf349ad5ead92fc" +checksum = "df33b2b81ac578cabaf06b89b0631153a3f416b0a886e8a7a1707fb51abbd1ef" dependencies = [ "ring", "rustls-pki-types", @@ -1410,9 +1410,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.49.0" +version = "1.50.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72a2903cd7736441aac9df9d7688bd0ce48edccaadf181c3b90be801e81d3d86" +checksum = "27ad5e34374e03cfffefc301becb44e9dc3c17584f414349ebe29ed26661822d" dependencies = [ "bytes", "libc", diff --git a/complement/go.mod b/complement/go.mod index 2a9adb9b95..716aabdf32 100644 --- a/complement/go.mod +++ b/complement/go.mod @@ -10,7 +10,7 @@ require ( ) require ( - github.com/docker/docker v28.3.3+incompatible // indirect + github.com/docker/docker v28.3.3+incompatible github.com/docker/go-connections v0.5.0 // indirect github.com/hashicorp/go-set/v3 v3.0.0 // indirect github.com/moby/sys/atomicwriter v0.1.0 // indirect diff --git a/complement/tests/internal/dockerutil/files.go b/complement/tests/internal/dockerutil/files.go new file mode 100644 index 0000000000..e19f684368 --- /dev/null +++ b/complement/tests/internal/dockerutil/files.go @@ -0,0 +1,68 @@ +package dockerutil + +import ( + "archive/tar" + "bytes" + "fmt" + "testing" + + "github.com/docker/docker/api/types/container" + "github.com/docker/docker/client" +) + +// Write `data` as a file into a container at the given `path`. +// +// Internally, produces an uncompressed single-file tape archive (tar) that is sent to the docker +// daemon to be unpacked into the container filesystem. +func WriteFileIntoContainer( + t *testing.T, + docker *client.Client, + containerID string, + path string, + data []byte, +) error { + // Create a fake/virtual tar file in memory that we can copy to the container + // via https://stackoverflow.com/a/52131297/796832 + var buf bytes.Buffer + tw := tar.NewWriter(&buf) + err := tw.WriteHeader(&tar.Header{ + Name: path, + Mode: 0777, + Size: int64(len(data)), + }) + if err != nil { + return fmt.Errorf( + "WriteIntoContainer: failed to write tarball header for %s: %v", + path, + err, + ) + } + _, err = tw.Write([]byte(data)) + if err != nil { + return fmt.Errorf("WriteIntoContainer: failed to write tarball data for %s: %w", path, err) + } + + err = tw.Close() + if err != nil { + return fmt.Errorf( + "WriteIntoContainer: failed to close tarball writer for %s: %w", + path, + err, + ) + } + + // Put our new fake file in the container volume + err = docker.CopyToContainer( + t.Context(), + containerID, + "/", + &buf, + container.CopyToContainerOptions{ + AllowOverwriteDirWithFile: false, + }, + ) + if err != nil { + return fmt.Errorf("WriteIntoContainer: failed to copy: %s", err) + } + return nil +} diff --git a/complement/tests/oidc_test.go b/complement/tests/oidc_test.go new file mode 100644 index 0000000000..deabf49950 --- /dev/null +++ b/complement/tests/oidc_test.go @@ -0,0 +1,120 @@ +// This file is licensed under the Affero General Public License (AGPL) version 3. +// +// Copyright (C) 2026 Element Creations Ltd +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// See the GNU Affero General Public License for more details: +// . + +package synapse_tests + +import ( + "net/http" + "net/url" + "strings" + "testing" + + dockerClient "github.com/docker/docker/client" + "github.com/element-hq/synapse/tests/internal/dockerutil" + "github.com/matrix-org/complement" + "github.com/matrix-org/complement/client" + "github.com/matrix-org/complement/match" + "github.com/matrix-org/complement/must" +) + +const OIDC_HOMESERVER_CONFIG string = ` +oidc_providers: + - idp_id: "test_provider" + idp_name: "Test OIDC Provider" + issuer: "https://example.invalid" + client_id: "test_client_id" + client_secret: "test_secret" + scopes: ["openid"] + discover: true + user_mapping_provider: + module: "synapse.handlers.oidc.JinjaOidcMappingProvider" + config: + display_name_template: "{{ user.given_name }}" + email_template: "{{ user.email }}" +` + +// Test that Synapse still starts up when configured with an OIDC provider that is unavailable. +// +// This is a regression test: Synapse previously would fail to start up +// at all if the OIDC provider was down on startup. +// https://github.com/element-hq/synapse/issues/8088 +// +// Now instead of failing to start, Synapse will produce a 503 response on the +// `/_matrix/client/v3/login/sso/redirect/oidc-test_provider` endpoint. +func TestOIDCProviderUnavailable(t *testing.T) { + // Deploy a single homeserver + deployment := complement.Deploy(t, 1) + defer deployment.Destroy(t) + + // Get Docker client to manipulate container + dc, err := dockerClient.NewClientWithOpts( + dockerClient.FromEnv, + dockerClient.WithAPIVersionNegotiation(), + ) + must.NotError(t, "failed creating docker client", err) + + // Configure the OIDC Provider by writing a config fragment + err = dockerutil.WriteFileIntoContainer( + t, + dc, + deployment.ContainerID(t, "hs1"), + "/conf/homeserver.d/oidc_provider.yaml", + []byte(OIDC_HOMESERVER_CONFIG), + ) + if err != nil { + t.Fatalf("Failed to write updated config to container: %v", err) + } + + // Restart the homeserver to apply the new config + deployment.StopServer(t, "hs1") + // Careful: port number changes here + deployment.StartServer(t, "hs1") + // Must get after the restart so the port number is correct + unauthedClient := deployment.UnauthenticatedClient(t, "hs1") + + // Test that trying to log in with an OIDC provider that is down + // causes an HTML error page to be shown to the user. + // (This replaces the redirect that would happen if the provider was + // up.) + // + // More importantly, implicitly tests that Synapse can start up + // and answer requests even though an OIDC provider is down. + t.Run("/login/sso/redirect shows HTML error", func(t *testing.T) { + // Build a request to the /redirect/ endpoint, that would normally be navigated to + // by the user's browser in order to start the login flow. + queryParams := url.Values{} + queryParams.Add("redirectUrl", "http://redirect.invalid/redirect") + res := unauthedClient.Do( + t, + "GET", + []string{"_matrix", "client", "v3", "login", "sso", "redirect", "oidc-test_provider"}, + client.WithQueries(queryParams), + ) + + body := must.MatchResponse(t, res, match.HTTPResponse{ + // Should get a 503 + StatusCode: http.StatusServiceUnavailable, + + Headers: map[string]string{ + // Should get an HTML page explaining the problem to the user + "Content-Type": "text/html; charset=utf-8", + }, + }) + + bodyText := string(body) + + // The HTML page contains phrases from the template we expect + if !strings.Contains(bodyText, "login provider is unavailable right now") { + t.Fatalf("Keyword not found in HTML error page, got %s", bodyText) + } + }) +} diff --git a/debian/changelog b/debian/changelog index c8146764de..f3dc1eb0c2 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,15 @@ +matrix-synapse-py3 (1.151.0) stable; urgency=medium + + * New synapse release 1.151.0. + + -- Synapse Packaging team Tue, 07 Apr 2026 12:15:10 +0000 + +matrix-synapse-py3 (1.151.0~rc1) stable; urgency=medium + + * New synapse release 1.151.0rc1. + + -- Synapse Packaging team Tue, 31 Mar 2026 12:23:34 +0000 + matrix-synapse-py3 (1.150.0) stable; urgency=medium * New synapse release 1.150.0. diff --git a/demo/start.sh b/demo/start.sh index 0b61ac9991..471e881f46 100755 --- a/demo/start.sh +++ b/demo/start.sh @@ -49,7 +49,7 @@ for port in 8080 8081 8082; do # Please don't accidentally bork me with your fancy settings. listeners=$(cat <<-PORTLISTENERS # Configure server to listen on both $https_port and $port - # This overides some of the default settings above + # This overrides some of the default settings above listeners: - port: $https_port type: http diff --git a/docker/complement/conf/start_for_complement.sh b/docker/complement/conf/start_for_complement.sh index da1b26a283..e0d30abed3 100755 --- a/docker/complement/conf/start_for_complement.sh +++ b/docker/complement/conf/start_for_complement.sh @@ -128,6 +128,10 @@ openssl x509 -req -in /conf/server.tls.csr \ export SYNAPSE_TLS_CERT=/conf/server.tls.crt export SYNAPSE_TLS_KEY=/conf/server.tls.key +# Add a directory for tests to add config overrides if they want +mkdir --parents /conf/homeserver.d +export _SYNAPSE_COMPLEMENT_EXTRA_CONFIG_DIR=/conf/homeserver.d + # Run the script that writes the necessary config files and starts supervisord, which in turn # starts everything else exec /configure_workers_and_start.py "$@" diff --git a/docker/conf-workers/synapse.supervisord.conf.j2 b/docker/conf-workers/synapse.supervisord.conf.j2 index 4fb11b259e..9388b1ff0e 100644 --- a/docker/conf-workers/synapse.supervisord.conf.j2 +++ b/docker/conf-workers/synapse.supervisord.conf.j2 @@ -5,10 +5,16 @@ command=/usr/local/bin/python -m synapse.app.complement_fork_starter {{ main_config_path }} synapse.app.homeserver --config-path="{{ main_config_path }}" + {%- if extra_config_dir %} + --config-path="{{ extra_config_dir }}" + {%- endif %} --config-path=/conf/workers/shared.yaml {%- for worker in workers %} -- {{ worker.app }} --config-path="{{ main_config_path }}" + {%- if extra_config_dir %} + --config-path="{{ extra_config_dir }}" + {%- endif %} --config-path=/conf/workers/shared.yaml --config-path=/conf/workers/{{ worker.name }}.yaml {%- endfor %} @@ -24,6 +30,9 @@ exitcodes=0 environment=http_proxy="%(ENV_SYNAPSE_HTTP_PROXY)s",https_proxy="%(ENV_SYNAPSE_HTTPS_PROXY)s",no_proxy="%(ENV_SYNAPSE_NO_PROXY)s" command=/usr/local/bin/prefix-log /usr/local/bin/python -m synapse.app.homeserver --config-path="{{ main_config_path }}" + {%- if extra_config_dir %} + --config-path="{{ extra_config_dir }}" + {%- endif %} --config-path=/conf/workers/shared.yaml priority=10 # Log startup failures to supervisord's stdout/err diff --git a/docker/configure_workers_and_start.py b/docker/configure_workers_and_start.py index ed15e8efef..1b8d4f9989 100755 --- a/docker/configure_workers_and_start.py +++ b/docker/configure_workers_and_start.py @@ -856,6 +856,7 @@ def parse_worker_types( def generate_worker_files( environ: Mapping[str, str], config_path: str, + extra_config_dir: str | None, data_dir: str, requested_workers: list[Worker], ) -> None: @@ -865,6 +866,7 @@ def generate_worker_files( Args: environ: os.environ instance. config_path: The location of the generated Synapse main worker config file. + extra_config_dir: A directory in which extra Synapse configuration files will be loaded. data_dir: The location of the synapse data directory. Where log and user-facing config files live. requested_workers: A list of requested workers @@ -1258,6 +1260,7 @@ def generate_worker_files( "/etc/supervisor/conf.d/synapse.conf", workers=worker_descriptors, main_config_path=config_path, + extra_config_dir=extra_config_dir, use_forking_launcher=environ.get("SYNAPSE_USE_EXPERIMENTAL_FORKING_LAUNCHER"), ) @@ -1318,6 +1321,9 @@ def main(args: list[str], environ: MutableMapping[str, str]) -> None: config_dir = environ.get("SYNAPSE_CONFIG_DIR", "/data") config_path = environ.get("SYNAPSE_CONFIG_PATH", config_dir + "/homeserver.yaml") + # All files in this directory will be loaded as `homeserver.yaml` snippets + # Currently only intended for use by Complement + extra_config_dir = environ.get("_SYNAPSE_COMPLEMENT_EXTRA_CONFIG_DIR", None) data_dir = environ.get("SYNAPSE_DATA_DIR", "/data") # override SYNAPSE_NO_TLS, we don't support TLS in worker mode, @@ -1352,6 +1358,7 @@ def main(args: list[str], environ: MutableMapping[str, str]) -> None: generate_worker_files( environ=environ, config_path=config_path, + extra_config_dir=extra_config_dir, data_dir=data_dir, requested_workers=requested_workers, ) diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index 085e43a842..963f1631e0 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -687,6 +687,8 @@ This setting has the following sub-options: * `endpoint` (string): The URL where Synapse can reach MAS. This *must* have the `discovery` and `oauth` resources mounted. Defaults to `"http://localhost:8080"`. +* `force_http2` (boolean): Force HTTP/2 over plaintext (H2C) when connecting to MAS. MAS supports this natively, but a reverse proxy between Synapse and MAS may not. Defaults to `false`. + * `secret` (string|null): A shared secret that will be used to authenticate requests from and to MAS. * `secret_path` (string|null): Alternative to `secret`, reading the shared secret from a file. The file should be a plain text file, containing only the secret. Synapse reads the secret from the given file once at startup. diff --git a/poetry.lock b/poetry.lock index ecf62936ad..4ed2b96bf9 100644 --- a/poetry.lock +++ b/poetry.lock @@ -453,61 +453,61 @@ files = [ [[package]] name = "cryptography" -version = "46.0.5" +version = "46.0.6" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." optional = false python-versions = "!=3.9.0,!=3.9.1,>=3.8" groups = ["main", "dev"] files = [ - {file = "cryptography-46.0.5-cp311-abi3-macosx_10_9_universal2.whl", hash = "sha256:351695ada9ea9618b3500b490ad54c739860883df6c1f555e088eaf25b1bbaad"}, - {file = "cryptography-46.0.5-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c18ff11e86df2e28854939acde2d003f7984f721eba450b56a200ad90eeb0e6b"}, - {file = "cryptography-46.0.5-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4d7e3d356b8cd4ea5aff04f129d5f66ebdc7b6f8eae802b93739ed520c47c79b"}, - {file = "cryptography-46.0.5-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:50bfb6925eff619c9c023b967d5b77a54e04256c4281b0e21336a130cd7fc263"}, - {file = "cryptography-46.0.5-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:803812e111e75d1aa73690d2facc295eaefd4439be1023fefc4995eaea2af90d"}, - {file = "cryptography-46.0.5-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:3ee190460e2fbe447175cda91b88b84ae8322a104fc27766ad09428754a618ed"}, - {file = "cryptography-46.0.5-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:f145bba11b878005c496e93e257c1e88f154d278d2638e6450d17e0f31e558d2"}, - {file = "cryptography-46.0.5-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:e9251e3be159d1020c4030bd2e5f84d6a43fe54b6c19c12f51cde9542a2817b2"}, - {file = "cryptography-46.0.5-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:47fb8a66058b80e509c47118ef8a75d14c455e81ac369050f20ba0d23e77fee0"}, - {file = "cryptography-46.0.5-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:4c3341037c136030cb46e4b1e17b7418ea4cbd9dd207e4a6f3b2b24e0d4ac731"}, - {file = "cryptography-46.0.5-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:890bcb4abd5a2d3f852196437129eb3667d62630333aacc13dfd470fad3aaa82"}, - {file = "cryptography-46.0.5-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:80a8d7bfdf38f87ca30a5391c0c9ce4ed2926918e017c29ddf643d0ed2778ea1"}, - {file = "cryptography-46.0.5-cp311-abi3-win32.whl", hash = "sha256:60ee7e19e95104d4c03871d7d7dfb3d22ef8a9b9c6778c94e1c8fcc8365afd48"}, - {file = "cryptography-46.0.5-cp311-abi3-win_amd64.whl", hash = "sha256:38946c54b16c885c72c4f59846be9743d699eee2b69b6988e0a00a01f46a61a4"}, - {file = "cryptography-46.0.5-cp314-cp314t-macosx_10_9_universal2.whl", hash = "sha256:94a76daa32eb78d61339aff7952ea819b1734b46f73646a07decb40e5b3448e2"}, - {file = "cryptography-46.0.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5be7bf2fb40769e05739dd0046e7b26f9d4670badc7b032d6ce4db64dddc0678"}, - {file = "cryptography-46.0.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:fe346b143ff9685e40192a4960938545c699054ba11d4f9029f94751e3f71d87"}, - {file = "cryptography-46.0.5-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:c69fd885df7d089548a42d5ec05be26050ebcd2283d89b3d30676eb32ff87dee"}, - {file = "cryptography-46.0.5-cp314-cp314t-manylinux_2_28_ppc64le.whl", hash = "sha256:8293f3dea7fc929ef7240796ba231413afa7b68ce38fd21da2995549f5961981"}, - {file = "cryptography-46.0.5-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:1abfdb89b41c3be0365328a410baa9df3ff8a9110fb75e7b52e66803ddabc9a9"}, - {file = "cryptography-46.0.5-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:d66e421495fdb797610a08f43b05269e0a5ea7f5e652a89bfd5a7d3c1dee3648"}, - {file = "cryptography-46.0.5-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:4e817a8920bfbcff8940ecfd60f23d01836408242b30f1a708d93198393a80b4"}, - {file = "cryptography-46.0.5-cp314-cp314t-manylinux_2_34_ppc64le.whl", hash = "sha256:68f68d13f2e1cb95163fa3b4db4bf9a159a418f5f6e7242564fc75fcae667fd0"}, - {file = "cryptography-46.0.5-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:a3d1fae9863299076f05cb8a778c467578262fae09f9dc0ee9b12eb4268ce663"}, - {file = "cryptography-46.0.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:c4143987a42a2397f2fc3b4d7e3a7d313fbe684f67ff443999e803dd75a76826"}, - {file = "cryptography-46.0.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:7d731d4b107030987fd61a7f8ab512b25b53cef8f233a97379ede116f30eb67d"}, - {file = "cryptography-46.0.5-cp314-cp314t-win32.whl", hash = "sha256:c3bcce8521d785d510b2aad26ae2c966092b7daa8f45dd8f44734a104dc0bc1a"}, - {file = "cryptography-46.0.5-cp314-cp314t-win_amd64.whl", hash = "sha256:4d8ae8659ab18c65ced284993c2265910f6c9e650189d4e3f68445ef82a810e4"}, - {file = "cryptography-46.0.5-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:4108d4c09fbbf2789d0c926eb4152ae1760d5a2d97612b92d508d96c861e4d31"}, - {file = "cryptography-46.0.5-cp38-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7d1f30a86d2757199cb2d56e48cce14deddf1f9c95f1ef1b64ee91ea43fe2e18"}, - {file = "cryptography-46.0.5-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:039917b0dc418bb9f6edce8a906572d69e74bd330b0b3fea4f79dab7f8ddd235"}, - {file = "cryptography-46.0.5-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:ba2a27ff02f48193fc4daeadf8ad2590516fa3d0adeeb34336b96f7fa64c1e3a"}, - {file = "cryptography-46.0.5-cp38-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:61aa400dce22cb001a98014f647dc21cda08f7915ceb95df0c9eaf84b4b6af76"}, - {file = "cryptography-46.0.5-cp38-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:3ce58ba46e1bc2aac4f7d9290223cead56743fa6ab94a5d53292ffaac6a91614"}, - {file = "cryptography-46.0.5-cp38-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:420d0e909050490d04359e7fdb5ed7e667ca5c3c402b809ae2563d7e66a92229"}, - {file = "cryptography-46.0.5-cp38-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:582f5fcd2afa31622f317f80426a027f30dc792e9c80ffee87b993200ea115f1"}, - {file = "cryptography-46.0.5-cp38-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:bfd56bb4b37ed4f330b82402f6f435845a5f5648edf1ad497da51a8452d5d62d"}, - {file = "cryptography-46.0.5-cp38-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:a3d507bb6a513ca96ba84443226af944b0f7f47dcc9a399d110cd6146481d24c"}, - {file = "cryptography-46.0.5-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:9f16fbdf4da055efb21c22d81b89f155f02ba420558db21288b3d0035bafd5f4"}, - {file = "cryptography-46.0.5-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:ced80795227d70549a411a4ab66e8ce307899fad2220ce5ab2f296e687eacde9"}, - {file = "cryptography-46.0.5-cp38-abi3-win32.whl", hash = "sha256:02f547fce831f5096c9a567fd41bc12ca8f11df260959ecc7c3202555cc47a72"}, - {file = "cryptography-46.0.5-cp38-abi3-win_amd64.whl", hash = "sha256:556e106ee01aa13484ce9b0239bca667be5004efb0aabbed28d353df86445595"}, - {file = "cryptography-46.0.5-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:3b4995dc971c9fb83c25aa44cf45f02ba86f71ee600d81091c2f0cbae116b06c"}, - {file = "cryptography-46.0.5-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:bc84e875994c3b445871ea7181d424588171efec3e185dced958dad9e001950a"}, - {file = "cryptography-46.0.5-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:2ae6971afd6246710480e3f15824ed3029a60fc16991db250034efd0b9fb4356"}, - {file = "cryptography-46.0.5-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:d861ee9e76ace6cf36a6a89b959ec08e7bc2493ee39d07ffe5acb23ef46d27da"}, - {file = "cryptography-46.0.5-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:2b7a67c9cd56372f3249b39699f2ad479f6991e62ea15800973b956f4b73e257"}, - {file = "cryptography-46.0.5-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:8456928655f856c6e1533ff59d5be76578a7157224dbd9ce6872f25055ab9ab7"}, - {file = "cryptography-46.0.5.tar.gz", hash = "sha256:abace499247268e3757271b2f1e244b36b06f8515cf27c4d49468fc9eb16e93d"}, + {file = "cryptography-46.0.6-cp311-abi3-macosx_10_9_universal2.whl", hash = "sha256:64235194bad039a10bb6d2d930ab3323baaec67e2ce36215fd0952fad0930ca8"}, + {file = "cryptography-46.0.6-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:26031f1e5ca62fcb9d1fcb34b2b60b390d1aacaa15dc8b895a9ed00968b97b30"}, + {file = "cryptography-46.0.6-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9a693028b9cbe51b5a1136232ee8f2bc242e4e19d456ded3fa7c86e43c713b4a"}, + {file = "cryptography-46.0.6-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:67177e8a9f421aa2d3a170c3e56eca4e0128883cf52a071a7cbf53297f18b175"}, + {file = "cryptography-46.0.6-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:d9528b535a6c4f8ff37847144b8986a9a143585f0540fbcb1a98115b543aa463"}, + {file = "cryptography-46.0.6-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:22259338084d6ae497a19bae5d4c66b7ca1387d3264d1c2c0e72d9e9b6a77b97"}, + {file = "cryptography-46.0.6-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:760997a4b950ff00d418398ad73fbc91aa2894b5c1db7ccb45b4f68b42a63b3c"}, + {file = "cryptography-46.0.6-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:3dfa6567f2e9e4c5dceb8ccb5a708158a2a871052fa75c8b78cb0977063f1507"}, + {file = "cryptography-46.0.6-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:cdcd3edcbc5d55757e5f5f3d330dd00007ae463a7e7aa5bf132d1f22a4b62b19"}, + {file = "cryptography-46.0.6-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:d4e4aadb7fc1f88687f47ca20bb7227981b03afaae69287029da08096853b738"}, + {file = "cryptography-46.0.6-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:2b417edbe8877cda9022dde3a008e2deb50be9c407eef034aeeb3a8b11d9db3c"}, + {file = "cryptography-46.0.6-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:380343e0653b1c9d7e1f55b52aaa2dbb2fdf2730088d48c43ca1c7c0abb7cc2f"}, + {file = "cryptography-46.0.6-cp311-abi3-win32.whl", hash = "sha256:bcb87663e1f7b075e48c3be3ecb5f0b46c8fc50b50a97cf264e7f60242dca3f2"}, + {file = "cryptography-46.0.6-cp311-abi3-win_amd64.whl", hash = "sha256:6739d56300662c468fddb0e5e291f9b4d084bead381667b9e654c7dd81705124"}, + {file = "cryptography-46.0.6-cp314-cp314t-macosx_10_9_universal2.whl", hash = "sha256:2ef9e69886cbb137c2aef9772c2e7138dc581fad4fcbcf13cc181eb5a3ab6275"}, + {file = "cryptography-46.0.6-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7f417f034f91dcec1cb6c5c35b07cdbb2ef262557f701b4ecd803ee8cefed4f4"}, + {file = "cryptography-46.0.6-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d24c13369e856b94892a89ddf70b332e0b70ad4a5c43cf3e9cb71d6d7ffa1f7b"}, + {file = "cryptography-46.0.6-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:aad75154a7ac9039936d50cf431719a2f8d4ed3d3c277ac03f3339ded1a5e707"}, + {file = "cryptography-46.0.6-cp314-cp314t-manylinux_2_28_ppc64le.whl", hash = "sha256:3c21d92ed15e9cfc6eb64c1f5a0326db22ca9c2566ca46d845119b45b4400361"}, + {file = "cryptography-46.0.6-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:4668298aef7cddeaf5c6ecc244c2302a2b8e40f384255505c22875eebb47888b"}, + {file = "cryptography-46.0.6-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:8ce35b77aaf02f3b59c90b2c8a05c73bac12cea5b4e8f3fbece1f5fddea5f0ca"}, + {file = "cryptography-46.0.6-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:c89eb37fae9216985d8734c1afd172ba4927f5a05cfd9bf0e4863c6d5465b013"}, + {file = "cryptography-46.0.6-cp314-cp314t-manylinux_2_34_ppc64le.whl", hash = "sha256:ed418c37d095aeddf5336898a132fba01091f0ac5844e3e8018506f014b6d2c4"}, + {file = "cryptography-46.0.6-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:69cf0056d6947edc6e6760e5f17afe4bea06b56a9ac8a06de9d2bd6b532d4f3a"}, + {file = "cryptography-46.0.6-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8e7304c4f4e9490e11efe56af6713983460ee0780f16c63f219984dab3af9d2d"}, + {file = "cryptography-46.0.6-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:b928a3ca837c77a10e81a814a693f2295200adb3352395fad024559b7be7a736"}, + {file = "cryptography-46.0.6-cp314-cp314t-win32.whl", hash = "sha256:97c8115b27e19e592a05c45d0dd89c57f81f841cc9880e353e0d3bf25b2139ed"}, + {file = "cryptography-46.0.6-cp314-cp314t-win_amd64.whl", hash = "sha256:c797e2517cb7880f8297e2c0f43bb910e91381339336f75d2c1c2cbf811b70b4"}, + {file = "cryptography-46.0.6-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:12cae594e9473bca1a7aceb90536060643128bb274fcea0fc459ab90f7d1ae7a"}, + {file = "cryptography-46.0.6-cp38-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:639301950939d844a9e1c4464d7e07f902fe9a7f6b215bb0d4f28584729935d8"}, + {file = "cryptography-46.0.6-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ed3775295fb91f70b4027aeba878d79b3e55c0b3e97eaa4de71f8f23a9f2eb77"}, + {file = "cryptography-46.0.6-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:8927ccfbe967c7df312ade694f987e7e9e22b2425976ddbf28271d7e58845290"}, + {file = "cryptography-46.0.6-cp38-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:b12c6b1e1651e42ab5de8b1e00dc3b6354fdfd778e7fa60541ddacc27cd21410"}, + {file = "cryptography-46.0.6-cp38-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:063b67749f338ca9c5a0b7fe438a52c25f9526b851e24e6c9310e7195aad3b4d"}, + {file = "cryptography-46.0.6-cp38-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:02fad249cb0e090b574e30b276a3da6a149e04ee2f049725b1f69e7b8351ec70"}, + {file = "cryptography-46.0.6-cp38-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:7e6142674f2a9291463e5e150090b95a8519b2fb6e6aaec8917dd8d094ce750d"}, + {file = "cryptography-46.0.6-cp38-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:456b3215172aeefb9284550b162801d62f5f264a081049a3e94307fe20792cfa"}, + {file = "cryptography-46.0.6-cp38-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:341359d6c9e68834e204ceaf25936dffeafea3829ab80e9503860dcc4f4dac58"}, + {file = "cryptography-46.0.6-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:9a9c42a2723999a710445bc0d974e345c32adfd8d2fac6d8a251fa829ad31cfb"}, + {file = "cryptography-46.0.6-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:6617f67b1606dfd9fe4dbfa354a9508d4a6d37afe30306fe6c101b7ce3274b72"}, + {file = "cryptography-46.0.6-cp38-abi3-win32.whl", hash = "sha256:7f6690b6c55e9c5332c0b59b9c8a3fb232ebf059094c17f9019a51e9827df91c"}, + {file = "cryptography-46.0.6-cp38-abi3-win_amd64.whl", hash = "sha256:79e865c642cfc5c0b3eb12af83c35c5aeff4fa5c672dc28c43721c2c9fdd2f0f"}, + {file = "cryptography-46.0.6-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:2ea0f37e9a9cf0df2952893ad145fd9627d326a59daec9b0802480fa3bcd2ead"}, + {file = "cryptography-46.0.6-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:a3e84d5ec9ba01f8fd03802b2147ba77f0c8f2617b2aff254cedd551844209c8"}, + {file = "cryptography-46.0.6-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:12f0fa16cc247b13c43d56d7b35287ff1569b5b1f4c5e87e92cc4fcc00cd10c0"}, + {file = "cryptography-46.0.6-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:50575a76e2951fe7dbd1f56d181f8c5ceeeb075e9ff88e7ad997d2f42af06e7b"}, + {file = "cryptography-46.0.6-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:90e5f0a7b3be5f40c3a0a0eafb32c681d8d2c181fc2a1bdabe9b3f611d9f6b1a"}, + {file = "cryptography-46.0.6-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:6728c49e3b2c180ef26f8e9f0a883a2c585638db64cf265b49c9ba10652d430e"}, + {file = "cryptography-46.0.6.tar.gz", hash = "sha256:27550628a518c5c6c903d84f637fbecf287f6cb9ced3804838a1295dc1fd0759"}, ] [package.dependencies] @@ -521,7 +521,7 @@ nox = ["nox[uv] (>=2024.4.15)"] pep8test = ["check-sdist", "click (>=8.0.1)", "mypy (>=1.14)", "ruff (>=0.11.11)"] sdist = ["build (>=1.0.0)"] ssh = ["bcrypt (>=3.1.5)"] -test = ["certifi (>=2024)", "cryptography-vectors (==46.0.5)", "pretend (>=0.7)", "pytest (>=7.4.0)", "pytest-benchmark (>=4.0)", "pytest-cov (>=2.10.1)", "pytest-xdist (>=3.5.0)"] +test = ["certifi (>=2024)", "cryptography-vectors (==46.0.6)", "pretend (>=0.7)", "pytest (>=7.4.0)", "pytest-benchmark (>=4.0)", "pytest-cov (>=2.10.1)", "pytest-xdist (>=3.5.0)"] test-randomorder = ["pytest-randomly"] [[package]] @@ -2159,14 +2159,14 @@ tests = ["pytest (>=9)", "typing-extensions (>=4.15)"] [[package]] name = "phonenumbers" -version = "9.0.25" +version = "9.0.26" description = "Python version of Google's common library for parsing, formatting, storing and validating international phone numbers." optional = false python-versions = ">=2.5" groups = ["main"] files = [ - {file = "phonenumbers-9.0.25-py2.py3-none-any.whl", hash = "sha256:b1fd6c20d588f5bcd40af3899d727a9f536364211ec6eac554fcd75ca58992a3"}, - {file = "phonenumbers-9.0.25.tar.gz", hash = "sha256:a5f236fa384c6a77378d7836c8e486ade5f984ad2e8e6cc0dbe5124315cdc81b"}, + {file = "phonenumbers-9.0.26-py2.py3-none-any.whl", hash = "sha256:ff473da5712965b6c7f7a31cbff8255864df694eb48243771133ecb761e807c1"}, + {file = "phonenumbers-9.0.26.tar.gz", hash = "sha256:9e582c827f0f5503cddeebef80099475a52ffa761551d8384099c7ec71298cbf"}, ] [[package]] @@ -2368,14 +2368,14 @@ psycopg2 = "*" [[package]] name = "pyasn1" -version = "0.6.2" +version = "0.6.3" description = "Pure-Python implementation of ASN.1 types and DER/BER/CER codecs (X.208)" optional = false python-versions = ">=3.8" groups = ["main"] files = [ - {file = "pyasn1-0.6.2-py3-none-any.whl", hash = "sha256:1eb26d860996a18e9b6ed05e7aae0e9fc21619fcee6af91cca9bad4fbea224bf"}, - {file = "pyasn1-0.6.2.tar.gz", hash = "sha256:9b59a2b25ba7e4f8197db7686c09fb33e658b98339fadb826e9512629017833b"}, + {file = "pyasn1-0.6.3-py3-none-any.whl", hash = "sha256:a80184d120f0864a52a073acc6fc642847d0be408e7c7252f31390c0f4eadcde"}, + {file = "pyasn1-0.6.3.tar.gz", hash = "sha256:697a8ecd6d98891189184ca1fa05d1bb00e2f84b5977c481452050549c8a72cf"}, ] [[package]] @@ -2966,25 +2966,26 @@ typing-extensions = {version = ">=4.4.0", markers = "python_version < \"3.13\""} [[package]] name = "requests" -version = "2.32.5" +version = "2.33.0" description = "Python HTTP for Humans." optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main", "dev"] files = [ - {file = "requests-2.32.5-py3-none-any.whl", hash = "sha256:2462f94637a34fd532264295e186976db0f5d453d1cdd31473c85a6a161affb6"}, - {file = "requests-2.32.5.tar.gz", hash = "sha256:dbba0bac56e100853db0ea71b82b4dfd5fe2bf6d3754a8893c3af500cec7d7cf"}, + {file = "requests-2.33.0-py3-none-any.whl", hash = "sha256:3324635456fa185245e24865e810cecec7b4caf933d7eb133dcde67d48cee69b"}, + {file = "requests-2.33.0.tar.gz", hash = "sha256:c7ebc5e8b0f21837386ad0e1c8fe8b829fa5f544d8df3b2253bff14ef29d7652"}, ] [package.dependencies] -certifi = ">=2017.4.17" +certifi = ">=2023.5.7" charset_normalizer = ">=2,<4" idna = ">=2.5,<4" -urllib3 = ">=1.21.1,<3" +urllib3 = ">=1.26,<3" [package.extras] socks = ["PySocks (>=1.5.6,!=1.5.7)"] -use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] +test = ["PySocks (>=1.5.6,!=1.5.7)", "pytest (>=3)", "pytest-cov", "pytest-httpbin (==2.1.0)", "pytest-mock", "pytest-xdist"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<8)"] [[package]] name = "requests-toolbelt" diff --git a/pyproject.toml b/pyproject.toml index 7e1909b708..ab44627039 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "matrix-synapse" -version = "1.150.0" +version = "1.151.0" description = "Homeserver for the Matrix decentralised comms protocol" readme = "README.rst" authors = [ diff --git a/rust/src/events/internal_metadata.rs b/rust/src/events/internal_metadata.rs index 0233623e80..be71eef126 100644 --- a/rust/src/events/internal_metadata.rs +++ b/rust/src/events/internal_metadata.rs @@ -261,6 +261,11 @@ pub struct EventInternalMetadata { #[pyo3(get, set)] instance_name: Option, + /// The event ID of the redaction event, if this event has been redacted. + /// This is set dynamically at load time and is not persisted to the database. + #[pyo3(get, set)] + redacted_by: Option, + /// whether this event is an outlier (ie, whether we have the state at that /// point in the DAG) #[pyo3(get, set)] @@ -289,6 +294,7 @@ impl EventInternalMetadata { data, stream_ordering: None, instance_name: None, + redacted_by: None, outlier: false, }) } diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 83b8de7b64..a6e01fce64 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -15,6 +15,7 @@ pub mod matrix_const; pub mod msc4388_rendezvous; pub mod push; pub mod rendezvous; +pub mod room_versions; pub mod segmenter; lazy_static! { @@ -58,6 +59,7 @@ fn synapse_rust(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { rendezvous::register_module(py, m)?; msc4388_rendezvous::register_module(py, m)?; segmenter::register_module(py, m)?; + room_versions::register_module(py, m)?; Ok(()) } diff --git a/rust/src/room_versions.rs b/rust/src/room_versions.rs new file mode 100644 index 0000000000..fbcc32516a --- /dev/null +++ b/rust/src/room_versions.rs @@ -0,0 +1,835 @@ +/* + * This file is licensed under the Affero General Public License (AGPL) version 3. + * + * Copyright (C) 2026 Element Creations Ltd. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * See the GNU Affero General Public License for more details: + * . + */ + +//! Rust implementation of room version definitions. + +use std::sync::{Arc, LazyLock, RwLock}; + +use pyo3::{ + exceptions::{PyKeyError, PyRuntimeError}, + prelude::*, + types::{PyFrozenSet, PyIterator, PyModule, PyModuleMethods}, + Bound, IntoPyObjectExt, PyResult, Python, +}; + +/// Internal enum for tracking the version of the event format, +/// independently of the room version. +/// +/// To reduce confusion, the event format versions are named after the room +/// versions that they were used or introduced in. +/// The concept of an 'event format version' is specific to Synapse (the +/// specification does not mention this term.) +#[pyclass(frozen)] +pub struct EventFormatVersions {} + +#[pymethods] +impl EventFormatVersions { + /// $id:server event id format: used for room v1 and v2 + #[classattr] + const ROOM_V1_V2: i32 = 1; + /// MSC1659-style $hash event id format: used for room v3 + #[classattr] + const ROOM_V3: i32 = 2; + /// MSC1884-style $hash format: introduced for room v4 + #[classattr] + const ROOM_V4_PLUS: i32 = 3; + /// MSC4291 room IDs as hashes: introduced for room HydraV11 + #[classattr] + const ROOM_V11_HYDRA_PLUS: i32 = 4; +} + +/// Enum to identify the state resolution algorithms. +#[pyclass(frozen)] +pub struct StateResolutionVersions {} + +#[pymethods] +impl StateResolutionVersions { + /// Room v1 state res + #[classattr] + const V1: i32 = 1; + /// MSC1442 state res: room v2 and later + #[classattr] + const V2: i32 = 2; + /// MSC4297 state res + #[classattr] + const V2_1: i32 = 3; +} + +/// Room disposition constants. +#[pyclass(frozen)] +pub struct RoomDisposition {} + +#[pymethods] +impl RoomDisposition { + #[classattr] + const STABLE: &'static str = "stable"; + #[classattr] + const UNSTABLE: &'static str = "unstable"; +} + +/// Enum for listing possible MSC3931 room version feature flags, for push rules. +#[pyclass(frozen)] +pub struct PushRuleRoomFlag {} + +#[pymethods] +impl PushRuleRoomFlag { + /// MSC3932: Room version supports MSC1767 Extensible Events. + #[classattr] + const EXTENSIBLE_EVENTS: &'static str = "org.matrix.msc3932.extensible_events"; +} + +/// An object which describes the unique attributes of a room version. +#[pyclass(frozen, eq, hash, get_all)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct RoomVersion { + /// The identifier for this version. + pub identifier: &'static str, + /// One of the RoomDisposition constants. + pub disposition: &'static str, + /// One of the EventFormatVersions constants. + pub event_format: i32, + /// One of the StateResolutionVersions constants. + pub state_res: i32, + pub enforce_key_validity: bool, + /// Before MSC2432, m.room.aliases had special auth rules and redaction rules. + pub special_case_aliases_auth: bool, + /// Strictly enforce canonicaljson, do not allow: + /// * Integers outside the range of [-2^53 + 1, 2^53 - 1] + /// * Floats + /// * NaN, Infinity, -Infinity + pub strict_canonicaljson: bool, + /// MSC2209: Check 'notifications' key while verifying + /// m.room.power_levels auth rules. + pub limit_notifications_power_levels: bool, + /// MSC3820: No longer include the creator in m.room.create events (room version 11). + pub implicit_room_creator: bool, + /// MSC3820: Apply updated redaction rules algorithm from room version 11. + pub updated_redaction_rules: bool, + /// Support the 'restricted' join rule. + pub restricted_join_rule: bool, + /// Support for the proper redaction rules for the restricted join rule. + /// This requires restricted_join_rule to be enabled. + pub restricted_join_rule_fix: bool, + /// Support the 'knock' join rule. + pub knock_join_rule: bool, + /// MSC3389: Protect relation information from redaction. + pub msc3389_relation_redactions: bool, + /// Support the 'knock_restricted' join rule. + pub knock_restricted_join_rule: bool, + /// Enforce integer power levels. + pub enforce_int_power_levels: bool, + /// MSC3931: Adds a push rule condition for "room version feature flags", making + /// some push rules room version dependent. Note that adding a flag to this list + /// is not enough to mark it "supported": the push rule evaluator also needs to + /// support the flag. Unknown flags are ignored by the evaluator, making conditions + /// fail if used. Values from PushRuleRoomFlag. + pub msc3931_push_features: &'static [&'static str], + /// MSC3757: Restricting who can overwrite a state event. + pub msc3757_enabled: bool, + /// MSC4289: Creator power enabled. + pub msc4289_creator_power_enabled: bool, + /// MSC4291: Room IDs as hashes of the create event. + pub msc4291_room_ids_as_hashes: bool, + /// Set of room versions where Synapse strictly enforces event key size limits + /// in bytes, rather than in codepoints. + /// + /// In these room versions, we are stricter with event size validation. + pub strict_event_byte_limits_room_versions: bool, +} + +const ROOM_VERSION_V1: RoomVersion = RoomVersion { + identifier: "1", + disposition: RoomDisposition::STABLE, + event_format: EventFormatVersions::ROOM_V1_V2, + state_res: StateResolutionVersions::V1, + enforce_key_validity: false, + special_case_aliases_auth: true, + strict_canonicaljson: false, + limit_notifications_power_levels: false, + implicit_room_creator: false, + updated_redaction_rules: false, + restricted_join_rule: false, + restricted_join_rule_fix: false, + knock_join_rule: false, + msc3389_relation_redactions: false, + knock_restricted_join_rule: false, + enforce_int_power_levels: false, + msc3931_push_features: &[], + msc3757_enabled: false, + msc4289_creator_power_enabled: false, + msc4291_room_ids_as_hashes: false, + strict_event_byte_limits_room_versions: false, +}; + +const ROOM_VERSION_V2: RoomVersion = RoomVersion { + identifier: "2", + disposition: RoomDisposition::STABLE, + event_format: EventFormatVersions::ROOM_V1_V2, + state_res: StateResolutionVersions::V2, + enforce_key_validity: false, + special_case_aliases_auth: true, + strict_canonicaljson: false, + limit_notifications_power_levels: false, + implicit_room_creator: false, + updated_redaction_rules: false, + restricted_join_rule: false, + restricted_join_rule_fix: false, + knock_join_rule: false, + msc3389_relation_redactions: false, + knock_restricted_join_rule: false, + enforce_int_power_levels: false, + msc3931_push_features: &[], + msc3757_enabled: false, + msc4289_creator_power_enabled: false, + msc4291_room_ids_as_hashes: false, + strict_event_byte_limits_room_versions: false, +}; + +const ROOM_VERSION_V3: RoomVersion = RoomVersion { + identifier: "3", + disposition: RoomDisposition::STABLE, + event_format: EventFormatVersions::ROOM_V3, + state_res: StateResolutionVersions::V2, + enforce_key_validity: false, + special_case_aliases_auth: true, + strict_canonicaljson: false, + limit_notifications_power_levels: false, + implicit_room_creator: false, + updated_redaction_rules: false, + restricted_join_rule: false, + restricted_join_rule_fix: false, + knock_join_rule: false, + msc3389_relation_redactions: false, + knock_restricted_join_rule: false, + enforce_int_power_levels: false, + msc3931_push_features: &[], + msc3757_enabled: false, + msc4289_creator_power_enabled: false, + msc4291_room_ids_as_hashes: false, + strict_event_byte_limits_room_versions: false, +}; + +const ROOM_VERSION_V4: RoomVersion = RoomVersion { + identifier: "4", + disposition: RoomDisposition::STABLE, + event_format: EventFormatVersions::ROOM_V4_PLUS, + state_res: StateResolutionVersions::V2, + enforce_key_validity: false, + special_case_aliases_auth: true, + strict_canonicaljson: false, + limit_notifications_power_levels: false, + implicit_room_creator: false, + updated_redaction_rules: false, + restricted_join_rule: false, + restricted_join_rule_fix: false, + knock_join_rule: false, + msc3389_relation_redactions: false, + knock_restricted_join_rule: false, + enforce_int_power_levels: false, + msc3931_push_features: &[], + msc3757_enabled: false, + msc4289_creator_power_enabled: false, + msc4291_room_ids_as_hashes: false, + strict_event_byte_limits_room_versions: false, +}; + +const ROOM_VERSION_V5: RoomVersion = RoomVersion { + identifier: "5", + disposition: RoomDisposition::STABLE, + event_format: EventFormatVersions::ROOM_V4_PLUS, + state_res: StateResolutionVersions::V2, + enforce_key_validity: true, + special_case_aliases_auth: true, + strict_canonicaljson: false, + limit_notifications_power_levels: false, + implicit_room_creator: false, + updated_redaction_rules: false, + restricted_join_rule: false, + restricted_join_rule_fix: false, + knock_join_rule: false, + msc3389_relation_redactions: false, + knock_restricted_join_rule: false, + enforce_int_power_levels: false, + msc3931_push_features: &[], + msc3757_enabled: false, + msc4289_creator_power_enabled: false, + msc4291_room_ids_as_hashes: false, + strict_event_byte_limits_room_versions: false, +}; + +const ROOM_VERSION_V6: RoomVersion = RoomVersion { + identifier: "6", + disposition: RoomDisposition::STABLE, + event_format: EventFormatVersions::ROOM_V4_PLUS, + state_res: StateResolutionVersions::V2, + enforce_key_validity: true, + special_case_aliases_auth: false, + strict_canonicaljson: true, + limit_notifications_power_levels: true, + implicit_room_creator: false, + updated_redaction_rules: false, + restricted_join_rule: false, + restricted_join_rule_fix: false, + knock_join_rule: false, + msc3389_relation_redactions: false, + knock_restricted_join_rule: false, + enforce_int_power_levels: false, + msc3931_push_features: &[], + msc3757_enabled: false, + msc4289_creator_power_enabled: false, + msc4291_room_ids_as_hashes: false, + strict_event_byte_limits_room_versions: false, +}; + +const ROOM_VERSION_V7: RoomVersion = RoomVersion { + identifier: "7", + disposition: RoomDisposition::STABLE, + event_format: EventFormatVersions::ROOM_V4_PLUS, + state_res: StateResolutionVersions::V2, + enforce_key_validity: true, + special_case_aliases_auth: false, + strict_canonicaljson: true, + limit_notifications_power_levels: true, + implicit_room_creator: false, + updated_redaction_rules: false, + restricted_join_rule: false, + restricted_join_rule_fix: false, + knock_join_rule: true, + msc3389_relation_redactions: false, + knock_restricted_join_rule: false, + enforce_int_power_levels: false, + msc3931_push_features: &[], + msc3757_enabled: false, + msc4289_creator_power_enabled: false, + msc4291_room_ids_as_hashes: false, + strict_event_byte_limits_room_versions: false, +}; + +const ROOM_VERSION_V8: RoomVersion = RoomVersion { + identifier: "8", + disposition: RoomDisposition::STABLE, + event_format: EventFormatVersions::ROOM_V4_PLUS, + state_res: StateResolutionVersions::V2, + enforce_key_validity: true, + special_case_aliases_auth: false, + strict_canonicaljson: true, + limit_notifications_power_levels: true, + implicit_room_creator: false, + updated_redaction_rules: false, + restricted_join_rule: true, + restricted_join_rule_fix: false, + knock_join_rule: true, + msc3389_relation_redactions: false, + knock_restricted_join_rule: false, + enforce_int_power_levels: false, + msc3931_push_features: &[], + msc3757_enabled: false, + msc4289_creator_power_enabled: false, + msc4291_room_ids_as_hashes: false, + strict_event_byte_limits_room_versions: false, +}; + +const ROOM_VERSION_V9: RoomVersion = RoomVersion { + identifier: "9", + disposition: RoomDisposition::STABLE, + event_format: EventFormatVersions::ROOM_V4_PLUS, + state_res: StateResolutionVersions::V2, + enforce_key_validity: true, + special_case_aliases_auth: false, + strict_canonicaljson: true, + limit_notifications_power_levels: true, + implicit_room_creator: false, + updated_redaction_rules: false, + restricted_join_rule: true, + restricted_join_rule_fix: true, + knock_join_rule: true, + msc3389_relation_redactions: false, + knock_restricted_join_rule: false, + enforce_int_power_levels: false, + msc3931_push_features: &[], + msc3757_enabled: false, + msc4289_creator_power_enabled: false, + msc4291_room_ids_as_hashes: false, + strict_event_byte_limits_room_versions: false, +}; + +const ROOM_VERSION_V10: RoomVersion = RoomVersion { + identifier: "10", + disposition: RoomDisposition::STABLE, + event_format: EventFormatVersions::ROOM_V4_PLUS, + state_res: StateResolutionVersions::V2, + enforce_key_validity: true, + special_case_aliases_auth: false, + strict_canonicaljson: true, + limit_notifications_power_levels: true, + implicit_room_creator: false, + updated_redaction_rules: false, + restricted_join_rule: true, + restricted_join_rule_fix: true, + knock_join_rule: true, + msc3389_relation_redactions: false, + knock_restricted_join_rule: true, + enforce_int_power_levels: true, + msc3931_push_features: &[], + msc3757_enabled: false, + msc4289_creator_power_enabled: false, + msc4291_room_ids_as_hashes: false, + strict_event_byte_limits_room_versions: false, +}; + +/// MSC3389 (Redaction changes for events with a relation) based on room version "10". +const ROOM_VERSION_MSC3389V10: RoomVersion = RoomVersion { + identifier: "org.matrix.msc3389.10", + disposition: RoomDisposition::UNSTABLE, + event_format: EventFormatVersions::ROOM_V4_PLUS, + state_res: StateResolutionVersions::V2, + enforce_key_validity: true, + special_case_aliases_auth: false, + strict_canonicaljson: true, + limit_notifications_power_levels: true, + implicit_room_creator: false, + updated_redaction_rules: false, + restricted_join_rule: true, + restricted_join_rule_fix: true, + knock_join_rule: true, + msc3389_relation_redactions: true, // Changed from v10 + knock_restricted_join_rule: true, + enforce_int_power_levels: true, + msc3931_push_features: &[], + msc3757_enabled: false, + msc4289_creator_power_enabled: false, + msc4291_room_ids_as_hashes: false, + strict_event_byte_limits_room_versions: true, +}; + +/// MSC1767 (Extensible Events) based on room version "10". +const ROOM_VERSION_MSC1767V10: RoomVersion = RoomVersion { + identifier: "org.matrix.msc1767.10", + disposition: RoomDisposition::UNSTABLE, + event_format: EventFormatVersions::ROOM_V4_PLUS, + state_res: StateResolutionVersions::V2, + enforce_key_validity: true, + special_case_aliases_auth: false, + strict_canonicaljson: true, + limit_notifications_power_levels: true, + implicit_room_creator: false, + updated_redaction_rules: false, + restricted_join_rule: true, + restricted_join_rule_fix: true, + knock_join_rule: true, + msc3389_relation_redactions: false, + knock_restricted_join_rule: true, + enforce_int_power_levels: true, + msc3931_push_features: &[PushRuleRoomFlag::EXTENSIBLE_EVENTS], + msc3757_enabled: false, + msc4289_creator_power_enabled: false, + msc4291_room_ids_as_hashes: false, + strict_event_byte_limits_room_versions: false, +}; + +/// MSC3757 (Restricting who can overwrite a state event) based on room version "10". +const ROOM_VERSION_MSC3757V10: RoomVersion = RoomVersion { + identifier: "org.matrix.msc3757.10", + disposition: RoomDisposition::UNSTABLE, + event_format: EventFormatVersions::ROOM_V4_PLUS, + state_res: StateResolutionVersions::V2, + enforce_key_validity: true, + special_case_aliases_auth: false, + strict_canonicaljson: true, + limit_notifications_power_levels: true, + implicit_room_creator: false, + updated_redaction_rules: false, + restricted_join_rule: true, + restricted_join_rule_fix: true, + knock_join_rule: true, + msc3389_relation_redactions: false, + knock_restricted_join_rule: true, + enforce_int_power_levels: true, + msc3931_push_features: &[], + msc3757_enabled: true, + msc4289_creator_power_enabled: false, + msc4291_room_ids_as_hashes: false, + strict_event_byte_limits_room_versions: false, +}; + +const ROOM_VERSION_V11: RoomVersion = RoomVersion { + identifier: "11", + disposition: RoomDisposition::STABLE, + event_format: EventFormatVersions::ROOM_V4_PLUS, + state_res: StateResolutionVersions::V2, + enforce_key_validity: true, + special_case_aliases_auth: false, + strict_canonicaljson: true, + limit_notifications_power_levels: true, + implicit_room_creator: true, // Used by MSC3820 + updated_redaction_rules: true, // Used by MSC3820 + restricted_join_rule: true, + restricted_join_rule_fix: true, + knock_join_rule: true, + msc3389_relation_redactions: false, + knock_restricted_join_rule: true, + enforce_int_power_levels: true, + msc3931_push_features: &[], + msc3757_enabled: false, + msc4289_creator_power_enabled: false, + msc4291_room_ids_as_hashes: false, + strict_event_byte_limits_room_versions: true, // Changed from v10 +}; + +/// MSC3757 (Restricting who can overwrite a state event) based on room version "11". +const ROOM_VERSION_MSC3757V11: RoomVersion = RoomVersion { + identifier: "org.matrix.msc3757.11", + disposition: RoomDisposition::UNSTABLE, + event_format: EventFormatVersions::ROOM_V4_PLUS, + state_res: StateResolutionVersions::V2, + enforce_key_validity: true, + special_case_aliases_auth: false, + strict_canonicaljson: true, + limit_notifications_power_levels: true, + implicit_room_creator: true, // Used by MSC3820 + updated_redaction_rules: true, // Used by MSC3820 + restricted_join_rule: true, + restricted_join_rule_fix: true, + knock_join_rule: true, + msc3389_relation_redactions: false, + knock_restricted_join_rule: true, + enforce_int_power_levels: true, + msc3931_push_features: &[], + msc3757_enabled: true, + msc4289_creator_power_enabled: false, + msc4291_room_ids_as_hashes: false, + strict_event_byte_limits_room_versions: true, +}; + +const ROOM_VERSION_HYDRA_V11: RoomVersion = RoomVersion { + identifier: "org.matrix.hydra.11", + disposition: RoomDisposition::UNSTABLE, + event_format: EventFormatVersions::ROOM_V11_HYDRA_PLUS, + state_res: StateResolutionVersions::V2_1, // Changed from v11 + enforce_key_validity: true, + special_case_aliases_auth: false, + strict_canonicaljson: true, + limit_notifications_power_levels: true, + implicit_room_creator: true, // Used by MSC3820 + updated_redaction_rules: true, // Used by MSC3820 + restricted_join_rule: true, + restricted_join_rule_fix: true, + knock_join_rule: true, + msc3389_relation_redactions: false, + knock_restricted_join_rule: true, + enforce_int_power_levels: true, + msc3931_push_features: &[], + msc3757_enabled: false, + msc4289_creator_power_enabled: true, // Changed from v11 + msc4291_room_ids_as_hashes: true, // Changed from v11 + strict_event_byte_limits_room_versions: true, +}; + +const ROOM_VERSION_V12: RoomVersion = RoomVersion { + identifier: "12", + disposition: RoomDisposition::STABLE, + event_format: EventFormatVersions::ROOM_V11_HYDRA_PLUS, + state_res: StateResolutionVersions::V2_1, // Changed from v11 + enforce_key_validity: true, + special_case_aliases_auth: false, + strict_canonicaljson: true, + limit_notifications_power_levels: true, + implicit_room_creator: true, // Used by MSC3820 + updated_redaction_rules: true, // Used by MSC3820 + restricted_join_rule: true, + restricted_join_rule_fix: true, + knock_join_rule: true, + msc3389_relation_redactions: false, + knock_restricted_join_rule: true, + enforce_int_power_levels: true, + msc3931_push_features: &[], + msc3757_enabled: false, + msc4289_creator_power_enabled: true, // Changed from v11 + msc4291_room_ids_as_hashes: true, // Changed from v11 + strict_event_byte_limits_room_versions: true, +}; + +/// Helper class for managing the known room versions, and providing dict-like +/// access to them for Python. +/// +/// Note: this is not necessarily all room versions, as we may not want to +/// support all experimental room versions. +/// +/// Note: room versions can be added to this mapping at startup (allowing +/// support for experimental room versions to be behind experimental feature +/// flags). +#[pyclass(frozen, mapping)] +#[derive(Clone)] +pub struct KnownRoomVersionsMapping { + // Note we use a Vec here to ensure that the order of keys is + // deterministic. We rely on this when generating parameterized tests in + // Python, as Python will generate the tests names including the room + // version and index (and we need test names to be stable to e.g. support + // running tests in parallel). + pub versions: Arc>>, +} + +#[pymethods] +impl KnownRoomVersionsMapping { + /// Add a new room version to the mapping, indicating that this instance + /// supports it. + fn add_room_version(&self, version: RoomVersion) -> PyResult<()> { + let mut versions = self + .versions + .write() + .map_err(|_| PyRuntimeError::new_err("KnownRoomVersionsMapping lock poisoned"))?; + + if versions.iter().any(|v| v.identifier == version.identifier) { + // We already have this room version, so we don't add it again (as + // otherwise we'd end up with duplicates). + return Ok(()); + } + + versions.push(version); + Ok(()) + } + + fn __getitem__(&self, key: &Bound<'_, PyAny>) -> PyResult { + // We need to accept anything as the key, but we know that only strings + // are valid keys, so if it's not a string we just raise a KeyError. + let Ok(key) = key.extract::<&str>() else { + return Err(PyKeyError::new_err(key.to_string())); + }; + let versions = self.versions.read().unwrap(); + versions + .iter() + .find(|v| v.identifier == key) + .copied() + .ok_or_else(|| PyKeyError::new_err(key.to_string())) + } + + fn __contains__(&self, key: &Bound<'_, PyAny>) -> bool { + // We need to accept anything as the key, but we know that only strings + // are valid keys, so if it's not a string we just return false. + let Ok(key) = key.extract::<&str>() else { + return false; + }; + let versions = self.versions.read().unwrap(); + versions.iter().any(|v| v.identifier == key) + } + + fn keys(&self) -> PyResult> { + // Note that technically we should return a view here (that also acts + // like a Set *and* has a stable ordering). We don't depend on this, so + // for simplicity we just return a list of the keys. + let versions = self.versions.read().unwrap(); + Ok(versions.iter().map(|v| v.identifier).collect()) + } + + fn values(&self) -> PyResult> { + let versions = self.versions.read().unwrap(); + Ok(versions.clone()) + } + + fn items(&self) -> PyResult> { + // Note that technically we should return a view here (that also acts + // like a Set *and* has a stable ordering). We don't depend on this, so + // for simplicity we just return a list of the items. + let versions = self.versions.read().unwrap(); + Ok(versions.iter().map(|v| (v.identifier, *v)).collect()) + } + + #[pyo3(signature = (key, default=None))] + fn get<'py>( + &self, + py: Python<'py>, + key: Bound<'py, PyAny>, + default: Option>, + ) -> PyResult>> { + // We need to accept anything as the key, but we know that only strings + // are valid keys, so if it's not a string we just return the default. + let Ok(key) = key.extract::<&str>() else { + return Ok(default); + }; + + let versions = self.versions.read().unwrap(); + if let Some(version) = versions.iter().find(|v| v.identifier == key).copied() { + return Ok(Some(version.into_bound_py_any(py)?)); + } + + Ok(default) + } + + fn __len__(&self) -> PyResult { + let versions = self + .versions + .read() + .map_err(|_| PyRuntimeError::new_err("KnownRoomVersionsMapping lock poisoned"))?; + Ok(versions.len()) + } + + fn __iter__<'py>(&self, py: Python<'py>) -> PyResult> { + let key_list = self.keys()?; + + let bound_key_list = key_list.into_bound_py_any(py)?; + PyIterator::from_object(&bound_key_list) + } +} + +impl<'py> IntoPyObject<'py> for &KnownRoomVersionsMapping { + type Target = KnownRoomVersionsMapping; + + type Output = Bound<'py, Self::Target>; + + type Error = PyErr; + + fn into_pyobject(self, py: Python<'py>) -> Result { + self.clone().into_pyobject(py) + } +} + +/// All room versions this instance knows about, used to build the +/// `KNOWN_ROOM_VERSIONS` dict. +/// +/// Note: this is not necessarily all room versions, as we may not want to +/// support all experimental room versions. +static KNOWN_ROOM_VERSIONS: LazyLock = LazyLock::new(|| { + let vec = vec![ + ROOM_VERSION_V1, + ROOM_VERSION_V2, + ROOM_VERSION_V3, + ROOM_VERSION_V4, + ROOM_VERSION_V5, + ROOM_VERSION_V6, + ROOM_VERSION_V7, + ROOM_VERSION_V8, + ROOM_VERSION_V9, + ROOM_VERSION_V10, + ROOM_VERSION_V11, + ROOM_VERSION_V12, + ROOM_VERSION_MSC3757V10, + ROOM_VERSION_MSC3757V11, + ROOM_VERSION_HYDRA_V11, + ]; + + KnownRoomVersionsMapping { + versions: Arc::new(RwLock::new(vec)), + } +}); + +/// Container class for room version constants. +/// +/// This should contain all room versions that we know about. +#[pyclass(frozen)] +pub struct RoomVersions {} + +#[pymethods] +#[allow(non_snake_case)] +impl RoomVersions { + #[classattr] + fn V1(py: Python<'_>) -> PyResult> { + ROOM_VERSION_V1.into_py_any(py) + } + #[classattr] + fn V2(py: Python<'_>) -> PyResult> { + ROOM_VERSION_V2.into_py_any(py) + } + #[classattr] + fn V3(py: Python<'_>) -> PyResult> { + ROOM_VERSION_V3.into_py_any(py) + } + #[classattr] + fn V4(py: Python<'_>) -> PyResult> { + ROOM_VERSION_V4.into_py_any(py) + } + #[classattr] + fn V5(py: Python<'_>) -> PyResult> { + ROOM_VERSION_V5.into_py_any(py) + } + #[classattr] + fn V6(py: Python<'_>) -> PyResult> { + ROOM_VERSION_V6.into_py_any(py) + } + #[classattr] + fn V7(py: Python<'_>) -> PyResult> { + ROOM_VERSION_V7.into_py_any(py) + } + #[classattr] + fn V8(py: Python<'_>) -> PyResult> { + ROOM_VERSION_V8.into_py_any(py) + } + #[classattr] + fn V9(py: Python<'_>) -> PyResult> { + ROOM_VERSION_V9.into_py_any(py) + } + #[classattr] + fn V10(py: Python<'_>) -> PyResult> { + ROOM_VERSION_V10.into_py_any(py) + } + #[classattr] + fn MSC1767v10(py: Python<'_>) -> PyResult> { + ROOM_VERSION_MSC1767V10.into_py_any(py) + } + #[classattr] + fn MSC3389v10(py: Python<'_>) -> PyResult> { + ROOM_VERSION_MSC3389V10.into_py_any(py) + } + #[classattr] + fn MSC3757v10(py: Python<'_>) -> PyResult> { + ROOM_VERSION_MSC3757V10.into_py_any(py) + } + #[classattr] + fn V11(py: Python<'_>) -> PyResult> { + ROOM_VERSION_V11.into_py_any(py) + } + #[classattr] + fn MSC3757v11(py: Python<'_>) -> PyResult> { + ROOM_VERSION_MSC3757V11.into_py_any(py) + } + #[classattr] + fn HydraV11(py: Python<'_>) -> PyResult> { + ROOM_VERSION_HYDRA_V11.into_py_any(py) + } + #[classattr] + fn V12(py: Python<'_>) -> PyResult> { + ROOM_VERSION_V12.into_py_any(py) + } +} + +/// Called when registering modules with python. +pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { + let child_module = PyModule::new(py, "room_versions")?; + child_module.add_class::()?; + child_module.add_class::()?; + child_module.add_class::()?; + child_module.add_class::()?; + child_module.add_class::()?; + child_module.add_class::()?; + child_module.add_class::()?; + + // Build KNOWN_EVENT_FORMAT_VERSIONS as a frozenset + let known_ef: [i32; 4] = [ + EventFormatVersions::ROOM_V1_V2, + EventFormatVersions::ROOM_V3, + EventFormatVersions::ROOM_V4_PLUS, + EventFormatVersions::ROOM_V11_HYDRA_PLUS, + ]; + let known_event_format_versions = PyFrozenSet::new(py, known_ef)?; + child_module.add("KNOWN_EVENT_FORMAT_VERSIONS", known_event_format_versions)?; + child_module.add("KNOWN_ROOM_VERSIONS", &*KNOWN_ROOM_VERSIONS)?; + + m.add_submodule(&child_module)?; + + // Register in sys.modules + py.import("sys")? + .getattr("modules")? + .set_item("synapse.synapse_rust.room_versions", child_module)?; + + Ok(()) +} diff --git a/schema/synapse-config.schema.yaml b/schema/synapse-config.schema.yaml index f8194ad726..be765ac5ed 100644 --- a/schema/synapse-config.schema.yaml +++ b/schema/synapse-config.schema.yaml @@ -1,5 +1,5 @@ $schema: https://famedly.github.io/synapse/latest/schema/v1/meta.schema.json -$id: https://famedly.github.io/synapse/schema/synapse/v1.150/synapse-config.schema.json +$id: https://famedly.github.io/synapse/schema/synapse/v1.151/synapse-config.schema.json type: object properties: famedly_maximum_refresh_token_lifetime: @@ -693,6 +693,13 @@ properties: and `oauth` resources mounted. default: http://localhost:8080 + force_http2: + type: boolean + description: >- + Force HTTP/2 over plaintext (H2C) when connecting to MAS. MAS supports + this natively, but a reverse proxy between Synapse and MAS may not. + default: false + secret: type: ["string", "null"] description: >- diff --git a/scripts-dev/complement.sh b/scripts-dev/complement.sh index a8a361fd4a..cca87d42a9 100755 --- a/scripts-dev/complement.sh +++ b/scripts-dev/complement.sh @@ -160,8 +160,18 @@ main() { if [[ -z "$COMPLEMENT_DIR" ]]; then COMPLEMENT_REF=${COMPLEMENT_REF:-main} echo "COMPLEMENT_DIR not set. Fetching Complement checkout from ${COMPLEMENT_REF}..." - wget -Nq https://github.com/matrix-org/complement/archive/${COMPLEMENT_REF}.tar.gz + + # Download the Complement checkout at the specified ref. + wget -q https://github.com/matrix-org/complement/archive/${COMPLEMENT_REF}.tar.gz + + # Delete the existing complement checkout. Otherwise we'll end up with stale + # test files after they're deleted server-side, and `tar` will not delete + # old files. + rm -rf complement-${COMPLEMENT_REF} + + # Extract the checkout. tar -xzf ${COMPLEMENT_REF}.tar.gz + COMPLEMENT_DIR=complement-${COMPLEMENT_REF} echo "Checkout available at 'complement-${COMPLEMENT_REF}'" fi diff --git a/synapse/_scripts/synapse_port_db.py b/synapse/_scripts/synapse_port_db.py index 79b2a0c528..eedceb170e 100755 --- a/synapse/_scripts/synapse_port_db.py +++ b/synapse/_scripts/synapse_port_db.py @@ -122,7 +122,7 @@ "presence_stream": ["currently_active"], "public_room_list_stream": ["visibility"], "pushers": ["enabled"], - "redactions": ["have_censored"], + "redactions": ["have_censored", "recheck"], "remote_media_cache": ["authenticated"], "room_memberships": ["participant"], "room_stats_state": ["is_federatable"], diff --git a/synapse/api/auth/mas.py b/synapse/api/auth/mas.py index 79c15a5329..95c6d62a9d 100644 --- a/synapse/api/auth/mas.py +++ b/synapse/api/auth/mas.py @@ -111,6 +111,7 @@ def __init__(self, hs: "HomeServer"): self._rust_http_client = HttpClient( reactor=hs.get_reactor(), user_agent=self._http_client.user_agent.decode("utf8"), + http2_only=self._config.force_http2, ) self._server_metadata = RetryOnExceptionCachedCall[ServerMetadata]( self._load_metadata diff --git a/synapse/api/constants.py b/synapse/api/constants.py index 21139a3867..eb9e6cc39b 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -158,6 +158,8 @@ class EventTypes: PollStart: Final = "m.poll.start" + RoomPolicy: Final = "m.room.policy" + class ToDeviceEventTypes: RoomKeyRequest: Final = "m.room_key_request" diff --git a/synapse/api/room_versions.py b/synapse/api/room_versions.py index 2f98d7a8a8..0035d7a58b 100644 --- a/synapse/api/room_versions.py +++ b/synapse/api/room_versions.py @@ -18,480 +18,22 @@ # # - -import attr - - -class EventFormatVersions: - """This is an internal enum for tracking the version of the event format, - independently of the room version. - - To reduce confusion, the event format versions are named after the room - versions that they were used or introduced in. - The concept of an 'event format version' is specific to Synapse (the - specification does not mention this term.) - """ - - ROOM_V1_V2 = 1 # $id:server event id format: used for room v1 and v2 - ROOM_V3 = 2 # MSC1659-style $hash event id format: used for room v3 - ROOM_V4_PLUS = 3 # MSC1884-style $hash format: introduced for room v4 - ROOM_V11_HYDRA_PLUS = 4 # MSC4291 room IDs as hashes: introduced for room HydraV11 - - -KNOWN_EVENT_FORMAT_VERSIONS = { - EventFormatVersions.ROOM_V1_V2, - EventFormatVersions.ROOM_V3, - EventFormatVersions.ROOM_V4_PLUS, - EventFormatVersions.ROOM_V11_HYDRA_PLUS, -} - - -class StateResolutionVersions: - """Enum to identify the state resolution algorithms""" - - V1 = 1 # room v1 state res - V2 = 2 # MSC1442 state res: room v2 and later - V2_1 = 3 # MSC4297 state res - - -class RoomDisposition: - STABLE = "stable" - UNSTABLE = "unstable" - - -class PushRuleRoomFlag: - """Enum for listing possible MSC3931 room version feature flags, for push rules""" - - # MSC3932: Room version supports MSC1767 Extensible Events. - EXTENSIBLE_EVENTS = "org.matrix.msc3932.extensible_events" - - -@attr.s(slots=True, frozen=True, auto_attribs=True) -class RoomVersion: - """An object which describes the unique attributes of a room version.""" - - identifier: str # the identifier for this version - disposition: str # one of the RoomDispositions - event_format: int # one of the EventFormatVersions - state_res: int # one of the StateResolutionVersions - enforce_key_validity: bool - - # Before MSC2432, m.room.aliases had special auth rules and redaction rules - special_case_aliases_auth: bool - # Strictly enforce canonicaljson, do not allow: - # * Integers outside the range of [-2 ^ 53 + 1, 2 ^ 53 - 1] - # * Floats - # * NaN, Infinity, -Infinity - strict_canonicaljson: bool - # MSC2209: Check 'notifications' key while verifying - # m.room.power_levels auth rules. - limit_notifications_power_levels: bool - # No longer include the creator in m.room.create events. - implicit_room_creator: bool - # Apply updated redaction rules algorithm from room version 11. - updated_redaction_rules: bool - # Support the 'restricted' join rule. - restricted_join_rule: bool - # Support for the proper redaction rules for the restricted join rule. This requires - # restricted_join_rule to be enabled. - restricted_join_rule_fix: bool - # Support the 'knock' join rule. - knock_join_rule: bool - # MSC3389: Protect relation information from redaction. - msc3389_relation_redactions: bool - # Support the 'knock_restricted' join rule. - knock_restricted_join_rule: bool - # Enforce integer power levels - enforce_int_power_levels: bool - # MSC3931: Adds a push rule condition for "room version feature flags", making - # some push rules room version dependent. Note that adding a flag to this list - # is not enough to mark it "supported": the push rule evaluator also needs to - # support the flag. Unknown flags are ignored by the evaluator, making conditions - # fail if used. - msc3931_push_features: tuple[str, ...] # values from PushRuleRoomFlag - # MSC3757: Restricting who can overwrite a state event - msc3757_enabled: bool - # MSC4289: Creator power enabled - msc4289_creator_power_enabled: bool - # MSC4291: Room IDs as hashes of the create event - msc4291_room_ids_as_hashes: bool - - -class RoomVersions: - V1 = RoomVersion( - "1", - RoomDisposition.STABLE, - EventFormatVersions.ROOM_V1_V2, - StateResolutionVersions.V1, - enforce_key_validity=False, - special_case_aliases_auth=True, - strict_canonicaljson=False, - limit_notifications_power_levels=False, - implicit_room_creator=False, - updated_redaction_rules=False, - restricted_join_rule=False, - restricted_join_rule_fix=False, - knock_join_rule=False, - msc3389_relation_redactions=False, - knock_restricted_join_rule=False, - enforce_int_power_levels=False, - msc3931_push_features=(), - msc3757_enabled=False, - msc4289_creator_power_enabled=False, - msc4291_room_ids_as_hashes=False, - ) - V2 = RoomVersion( - "2", - RoomDisposition.STABLE, - EventFormatVersions.ROOM_V1_V2, - StateResolutionVersions.V2, - enforce_key_validity=False, - special_case_aliases_auth=True, - strict_canonicaljson=False, - limit_notifications_power_levels=False, - implicit_room_creator=False, - updated_redaction_rules=False, - restricted_join_rule=False, - restricted_join_rule_fix=False, - knock_join_rule=False, - msc3389_relation_redactions=False, - knock_restricted_join_rule=False, - enforce_int_power_levels=False, - msc3931_push_features=(), - msc3757_enabled=False, - msc4289_creator_power_enabled=False, - msc4291_room_ids_as_hashes=False, - ) - V3 = RoomVersion( - "3", - RoomDisposition.STABLE, - EventFormatVersions.ROOM_V3, - StateResolutionVersions.V2, - enforce_key_validity=False, - special_case_aliases_auth=True, - strict_canonicaljson=False, - limit_notifications_power_levels=False, - implicit_room_creator=False, - updated_redaction_rules=False, - restricted_join_rule=False, - restricted_join_rule_fix=False, - knock_join_rule=False, - msc3389_relation_redactions=False, - knock_restricted_join_rule=False, - enforce_int_power_levels=False, - msc3931_push_features=(), - msc3757_enabled=False, - msc4289_creator_power_enabled=False, - msc4291_room_ids_as_hashes=False, - ) - V4 = RoomVersion( - "4", - RoomDisposition.STABLE, - EventFormatVersions.ROOM_V4_PLUS, - StateResolutionVersions.V2, - enforce_key_validity=False, - special_case_aliases_auth=True, - strict_canonicaljson=False, - limit_notifications_power_levels=False, - implicit_room_creator=False, - updated_redaction_rules=False, - restricted_join_rule=False, - restricted_join_rule_fix=False, - knock_join_rule=False, - msc3389_relation_redactions=False, - knock_restricted_join_rule=False, - enforce_int_power_levels=False, - msc3931_push_features=(), - msc3757_enabled=False, - msc4289_creator_power_enabled=False, - msc4291_room_ids_as_hashes=False, - ) - V5 = RoomVersion( - "5", - RoomDisposition.STABLE, - EventFormatVersions.ROOM_V4_PLUS, - StateResolutionVersions.V2, - enforce_key_validity=True, - special_case_aliases_auth=True, - strict_canonicaljson=False, - limit_notifications_power_levels=False, - implicit_room_creator=False, - updated_redaction_rules=False, - restricted_join_rule=False, - restricted_join_rule_fix=False, - knock_join_rule=False, - msc3389_relation_redactions=False, - knock_restricted_join_rule=False, - enforce_int_power_levels=False, - msc3931_push_features=(), - msc3757_enabled=False, - msc4289_creator_power_enabled=False, - msc4291_room_ids_as_hashes=False, - ) - V6 = RoomVersion( - "6", - RoomDisposition.STABLE, - EventFormatVersions.ROOM_V4_PLUS, - StateResolutionVersions.V2, - enforce_key_validity=True, - special_case_aliases_auth=False, - strict_canonicaljson=True, - limit_notifications_power_levels=True, - implicit_room_creator=False, - updated_redaction_rules=False, - restricted_join_rule=False, - restricted_join_rule_fix=False, - knock_join_rule=False, - msc3389_relation_redactions=False, - knock_restricted_join_rule=False, - enforce_int_power_levels=False, - msc3931_push_features=(), - msc3757_enabled=False, - msc4289_creator_power_enabled=False, - msc4291_room_ids_as_hashes=False, - ) - V7 = RoomVersion( - "7", - RoomDisposition.STABLE, - EventFormatVersions.ROOM_V4_PLUS, - StateResolutionVersions.V2, - enforce_key_validity=True, - special_case_aliases_auth=False, - strict_canonicaljson=True, - limit_notifications_power_levels=True, - implicit_room_creator=False, - updated_redaction_rules=False, - restricted_join_rule=False, - restricted_join_rule_fix=False, - knock_join_rule=True, - msc3389_relation_redactions=False, - knock_restricted_join_rule=False, - enforce_int_power_levels=False, - msc3931_push_features=(), - msc3757_enabled=False, - msc4289_creator_power_enabled=False, - msc4291_room_ids_as_hashes=False, - ) - V8 = RoomVersion( - "8", - RoomDisposition.STABLE, - EventFormatVersions.ROOM_V4_PLUS, - StateResolutionVersions.V2, - enforce_key_validity=True, - special_case_aliases_auth=False, - strict_canonicaljson=True, - limit_notifications_power_levels=True, - implicit_room_creator=False, - updated_redaction_rules=False, - restricted_join_rule=True, - restricted_join_rule_fix=False, - knock_join_rule=True, - msc3389_relation_redactions=False, - knock_restricted_join_rule=False, - enforce_int_power_levels=False, - msc3931_push_features=(), - msc3757_enabled=False, - msc4289_creator_power_enabled=False, - msc4291_room_ids_as_hashes=False, - ) - V9 = RoomVersion( - "9", - RoomDisposition.STABLE, - EventFormatVersions.ROOM_V4_PLUS, - StateResolutionVersions.V2, - enforce_key_validity=True, - special_case_aliases_auth=False, - strict_canonicaljson=True, - limit_notifications_power_levels=True, - implicit_room_creator=False, - updated_redaction_rules=False, - restricted_join_rule=True, - restricted_join_rule_fix=True, - knock_join_rule=True, - msc3389_relation_redactions=False, - knock_restricted_join_rule=False, - enforce_int_power_levels=False, - msc3931_push_features=(), - msc3757_enabled=False, - msc4289_creator_power_enabled=False, - msc4291_room_ids_as_hashes=False, - ) - V10 = RoomVersion( - "10", - RoomDisposition.STABLE, - EventFormatVersions.ROOM_V4_PLUS, - StateResolutionVersions.V2, - enforce_key_validity=True, - special_case_aliases_auth=False, - strict_canonicaljson=True, - limit_notifications_power_levels=True, - implicit_room_creator=False, - updated_redaction_rules=False, - restricted_join_rule=True, - restricted_join_rule_fix=True, - knock_join_rule=True, - msc3389_relation_redactions=False, - knock_restricted_join_rule=True, - enforce_int_power_levels=True, - msc3931_push_features=(), - msc3757_enabled=False, - msc4289_creator_power_enabled=False, - msc4291_room_ids_as_hashes=False, - ) - MSC1767v10 = RoomVersion( - # MSC1767 (Extensible Events) based on room version "10" - "org.matrix.msc1767.10", - RoomDisposition.UNSTABLE, - EventFormatVersions.ROOM_V4_PLUS, - StateResolutionVersions.V2, - enforce_key_validity=True, - special_case_aliases_auth=False, - strict_canonicaljson=True, - limit_notifications_power_levels=True, - implicit_room_creator=False, - updated_redaction_rules=False, - restricted_join_rule=True, - restricted_join_rule_fix=True, - knock_join_rule=True, - msc3389_relation_redactions=False, - knock_restricted_join_rule=True, - enforce_int_power_levels=True, - msc3931_push_features=(PushRuleRoomFlag.EXTENSIBLE_EVENTS,), - msc3757_enabled=False, - msc4289_creator_power_enabled=False, - msc4291_room_ids_as_hashes=False, - ) - MSC3757v10 = RoomVersion( - # MSC3757 (Restricting who can overwrite a state event) based on room version "10" - "org.matrix.msc3757.10", - RoomDisposition.UNSTABLE, - EventFormatVersions.ROOM_V4_PLUS, - StateResolutionVersions.V2, - enforce_key_validity=True, - special_case_aliases_auth=False, - strict_canonicaljson=True, - limit_notifications_power_levels=True, - implicit_room_creator=False, - updated_redaction_rules=False, - restricted_join_rule=True, - restricted_join_rule_fix=True, - knock_join_rule=True, - msc3389_relation_redactions=False, - knock_restricted_join_rule=True, - enforce_int_power_levels=True, - msc3931_push_features=(), - msc3757_enabled=True, - msc4289_creator_power_enabled=False, - msc4291_room_ids_as_hashes=False, - ) - V11 = RoomVersion( - "11", - RoomDisposition.STABLE, - EventFormatVersions.ROOM_V4_PLUS, - StateResolutionVersions.V2, - enforce_key_validity=True, - special_case_aliases_auth=False, - strict_canonicaljson=True, - limit_notifications_power_levels=True, - implicit_room_creator=True, # Used by MSC3820 - updated_redaction_rules=True, # Used by MSC3820 - restricted_join_rule=True, - restricted_join_rule_fix=True, - knock_join_rule=True, - msc3389_relation_redactions=False, - knock_restricted_join_rule=True, - enforce_int_power_levels=True, - msc3931_push_features=(), - msc3757_enabled=False, - msc4289_creator_power_enabled=False, - msc4291_room_ids_as_hashes=False, - ) - MSC3757v11 = RoomVersion( - # MSC3757 (Restricting who can overwrite a state event) based on room version "11" - "org.matrix.msc3757.11", - RoomDisposition.UNSTABLE, - EventFormatVersions.ROOM_V4_PLUS, - StateResolutionVersions.V2, - enforce_key_validity=True, - special_case_aliases_auth=False, - strict_canonicaljson=True, - limit_notifications_power_levels=True, - implicit_room_creator=True, # Used by MSC3820 - updated_redaction_rules=True, # Used by MSC3820 - restricted_join_rule=True, - restricted_join_rule_fix=True, - knock_join_rule=True, - msc3389_relation_redactions=False, - knock_restricted_join_rule=True, - enforce_int_power_levels=True, - msc3931_push_features=(), - msc3757_enabled=True, - msc4289_creator_power_enabled=False, - msc4291_room_ids_as_hashes=False, - ) - HydraV11 = RoomVersion( - "org.matrix.hydra.11", - RoomDisposition.UNSTABLE, - EventFormatVersions.ROOM_V11_HYDRA_PLUS, - StateResolutionVersions.V2_1, # Changed from v11 - enforce_key_validity=True, - special_case_aliases_auth=False, - strict_canonicaljson=True, - limit_notifications_power_levels=True, - implicit_room_creator=True, # Used by MSC3820 - updated_redaction_rules=True, # Used by MSC3820 - restricted_join_rule=True, - restricted_join_rule_fix=True, - knock_join_rule=True, - msc3389_relation_redactions=False, - knock_restricted_join_rule=True, - enforce_int_power_levels=True, - msc3931_push_features=(), - msc3757_enabled=False, - msc4289_creator_power_enabled=True, # Changed from v11 - msc4291_room_ids_as_hashes=True, # Changed from v11 - ) - V12 = RoomVersion( - "12", - RoomDisposition.STABLE, - EventFormatVersions.ROOM_V11_HYDRA_PLUS, - StateResolutionVersions.V2_1, # Changed from v11 - enforce_key_validity=True, - special_case_aliases_auth=False, - strict_canonicaljson=True, - limit_notifications_power_levels=True, - implicit_room_creator=True, # Used by MSC3820 - updated_redaction_rules=True, # Used by MSC3820 - restricted_join_rule=True, - restricted_join_rule_fix=True, - knock_join_rule=True, - msc3389_relation_redactions=False, - knock_restricted_join_rule=True, - enforce_int_power_levels=True, - msc3931_push_features=(), - msc3757_enabled=False, - msc4289_creator_power_enabled=True, # Changed from v11 - msc4291_room_ids_as_hashes=True, # Changed from v11 - ) - - -KNOWN_ROOM_VERSIONS: dict[str, RoomVersion] = { - v.identifier: v - for v in ( - RoomVersions.V1, - RoomVersions.V2, - RoomVersions.V3, - RoomVersions.V4, - RoomVersions.V5, - RoomVersions.V6, - RoomVersions.V7, - RoomVersions.V8, - RoomVersions.V9, - RoomVersions.V10, - RoomVersions.V11, - RoomVersions.V12, - RoomVersions.MSC3757v10, - RoomVersions.MSC3757v11, - RoomVersions.HydraV11, - ) -} +from synapse.synapse_rust.room_versions import ( + KNOWN_EVENT_FORMAT_VERSIONS, + KNOWN_ROOM_VERSIONS, + EventFormatVersions, + PushRuleRoomFlag, + RoomVersion, + RoomVersions, + StateResolutionVersions, +) + +__all__ = [ + "EventFormatVersions", + "KNOWN_EVENT_FORMAT_VERSIONS", + "KNOWN_ROOM_VERSIONS", + "PushRuleRoomFlag", + "RoomVersion", + "RoomVersions", + "StateResolutionVersions", +] diff --git a/synapse/app/_base.py b/synapse/app/_base.py index b10efefd46..a9a3411b89 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -706,12 +706,10 @@ async def start(hs: "HomeServer", *, freeze: bool = True) -> None: # Load the OIDC provider metadatas, if OIDC is enabled. if hs.config.oidc.oidc_enabled: oidc = hs.get_oidc_handler() + # Preload the provider metadata. + # This will spawn fire-and-forget background processes. # Loading the provider metadata also ensures the provider config is valid. - # - # FIXME: It feels a bit strange to validate and block on startup as one of these - # OIDC providers could be temporarily unavailable and cause Synapse to be unable - # to start. - await oidc.load_metadata() + oidc.preload_metadata() # Load the certificate from disk. refresh_certificate(hs) diff --git a/synapse/appservice/api.py b/synapse/appservice/api.py index 2bbf77a352..d4e9d50b96 100644 --- a/synapse/appservice/api.py +++ b/synapse/appservice/api.py @@ -32,7 +32,7 @@ from prometheus_client import Counter from typing_extensions import ParamSpec, TypeGuard -from synapse.api.constants import EventTypes, Membership, ThirdPartyEntityKind +from synapse.api.constants import ThirdPartyEntityKind from synapse.api.errors import CodeMessageException, HttpResponseException from synapse.appservice import ( ApplicationService, @@ -40,7 +40,7 @@ TransactionUnusedFallbackKeys, ) from synapse.events import EventBase -from synapse.events.utils import SerializeEventConfig, serialize_event +from synapse.events.utils import SerializeEventConfig from synapse.http.client import SimpleHttpClient, is_unknown_endpoint from synapse.logging import opentracing from synapse.metrics import SERVER_NAME_LABEL @@ -128,6 +128,7 @@ def __init__(self, hs: "HomeServer"): self.server_name = hs.hostname self.clock = hs.get_clock() self.config = hs.config.appservice + self._event_serializer = hs.get_event_client_serializer() self.protocol_meta_cache: ResponseCache[tuple[str, str]] = ResponseCache( clock=hs.get_clock(), @@ -343,7 +344,7 @@ async def push_bulk( # This is required by the configuration. assert service.hs_token is not None - serialized_events = self._serialize(service, events) + serialized_events = await self._serialize(service, events) if txn_id is None: logger.warning( @@ -539,30 +540,23 @@ async def query_keys( return response - def _serialize( + async def _serialize( self, service: "ApplicationService", events: Iterable[EventBase] ) -> list[JsonDict]: time_now = self.clock.time_msec() - return [ - serialize_event( - e, - time_now, - config=SerializeEventConfig( - as_client_event=True, - # If this is an invite or a knock membership event, and we're interested - # in this user, then include any stripped state alongside the event. - include_stripped_room_state=( - e.type == EventTypes.Member - and ( - e.membership == Membership.INVITE - or e.membership == Membership.KNOCK - ) - and service.is_interested_in_user(e.state_key) - ), - # Appservices are considered 'trusted' by the admin and should have - # applicable metadata on their events. - include_admin_metadata=True, - ), - ) - for e in events - ] + return await self._event_serializer.serialize_events( + list(events), + time_now, + config=SerializeEventConfig( + as_client_event=True, + # If this is an invite or a knock membership event, then include + # any stripped state alongside the event. We could narrow this + # down to only users the appservice is "interested in", however + # it's not worth the complexity of doing so, and it's simpler to + # just include it for all users. + include_stripped_room_state=True, + # Appservices are considered 'trusted' by the admin and should have + # applicable metadata on their events. + include_admin_metadata=True, + ), + ) diff --git a/synapse/config/experimental.py b/synapse/config/experimental.py index b31c0baebf..91bef558ed 100644 --- a/synapse/config/experimental.py +++ b/synapse/config/experimental.py @@ -425,9 +425,6 @@ def read_config( # previously calculated push actions. self.msc2654_enabled: bool = experimental.get("msc2654_enabled", False) - # MSC2666: Query mutual rooms between two users. - self.msc2666_enabled: bool = experimental.get("msc2666_enabled", False) - # MSC2815 (allow room moderators to view redacted event content) self.msc2815_enabled: bool = experimental.get("msc2815_enabled", False) @@ -448,9 +445,6 @@ def read_config( # MSC3848: Introduce errcodes for specific event sending failures self.msc3848_enabled: bool = experimental.get("msc3848_enabled", False) - # MSC3852: Expose last seen user agent field on /_matrix/client/v3/devices. - self.msc3852_enabled: bool = experimental.get("msc3852_enabled", False) - # MSC3866: M_USER_AWAITING_APPROVAL error code raw_msc3866_config = experimental.get("msc3866", {}) self.msc3866 = MSC3866Config(**raw_msc3866_config) @@ -486,8 +480,7 @@ def read_config( self.msc1767_enabled: bool = experimental.get("msc1767_enabled", False) if self.msc1767_enabled: # Enable room version (and thus applicable push rules from MSC3931/3932) - version_id = RoomVersions.MSC1767v10.identifier - KNOWN_ROOM_VERSIONS[version_id] = RoomVersions.MSC1767v10 + KNOWN_ROOM_VERSIONS.add_room_version(RoomVersions.MSC1767v10) # MSC3391: Removing account data. self.msc3391_enabled = experimental.get("msc3391_enabled", False) @@ -547,9 +540,9 @@ def read_config( # See: https://github.com/element-hq/synapse/issues/19433 msc4388_mode = experimental.get("msc4388_mode", "off") - if msc4388_mode not in ["off", "public", "authenticated"]: + if msc4388_mode not in ["off", "open", "authenticated"]: raise ConfigError( - "msc4388_mode must be one of 'off', 'public' or 'authenticated'", + "msc4388_mode must be one of 'off', 'open' or 'authenticated'", ("experimental", "msc4388_mode"), ) self.msc4388_enabled: bool = msc4388_mode != "off" diff --git a/synapse/config/mas.py b/synapse/config/mas.py index 104ba17ab7..6973e9ae58 100644 --- a/synapse/config/mas.py +++ b/synapse/config/mas.py @@ -36,6 +36,7 @@ class MasConfigModel(ParseModel): enabled: StrictBool = False endpoint: AnyHttpUrl = AnyHttpUrl("http://localhost:8080") + force_http2: StrictBool = False secret: StrictStr | None = Field(default=None) # We set `strict=False` to allow `str` instances. secret_path: FilePath | None = Field(default=None, strict=False) @@ -82,6 +83,7 @@ def read_config( self.enabled = parsed.enabled self.endpoint = parsed.endpoint + self.force_http2 = parsed.force_http2 self._secret = parsed.secret self._secret_path = parsed.secret_path diff --git a/synapse/event_auth.py b/synapse/event_auth.py index 7098843742..f39f55b472 100644 --- a/synapse/event_auth.py +++ b/synapse/event_auth.py @@ -60,7 +60,6 @@ KNOWN_ROOM_VERSIONS, EventFormatVersions, RoomVersion, - RoomVersions, ) from synapse.events import is_creator from synapse.state import CREATE_KEY @@ -383,25 +382,6 @@ def check_state_dependent_auth_rules( logger.debug("Allowing! %s", event) -# Set of room versions where Synapse did not apply event key size limits -# in bytes, but rather in codepoints. -# In these room versions, we are more lenient with event size validation. -LENIENT_EVENT_BYTE_LIMITS_ROOM_VERSIONS = { - RoomVersions.V1, - RoomVersions.V2, - RoomVersions.V3, - RoomVersions.V4, - RoomVersions.V5, - RoomVersions.V6, - RoomVersions.V7, - RoomVersions.V8, - RoomVersions.V9, - RoomVersions.V10, - RoomVersions.MSC1767v10, - RoomVersions.MSC3757v10, -} - - def _check_size_limits(event: "EventBase") -> None: """ Checks the size limits in a PDU. @@ -440,9 +420,7 @@ def _check_size_limits(event: "EventBase") -> None: if len(event.event_id) > 255: raise EventSizeError("'event_id' too large", unpersistable=True) - strict_byte_limits = ( - event.room_version not in LENIENT_EVENT_BYTE_LIMITS_ROOM_VERSIONS - ) + strict_byte_limits = event.room_version.strict_event_byte_limits_room_versions # Byte size check: if these fail, then be lenient to avoid breaking rooms. if len(event.user_id.encode("utf-8")) > 255: diff --git a/synapse/events/utils.py b/synapse/events/utils.py index 89eb2182af..76ebac8b17 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -88,6 +88,7 @@ def prune_event(event: EventBase) -> EventBase: ) pruned_event.internal_metadata.instance_name = event.internal_metadata.instance_name pruned_event.internal_metadata.outlier = event.internal_metadata.outlier + pruned_event.internal_metadata.redacted_by = event.internal_metadata.redacted_by # Mark the event as redacted pruned_event.internal_metadata.redacted = True @@ -123,6 +124,7 @@ def clone_event(event: EventBase) -> EventBase: ) new_event.internal_metadata.instance_name = event.internal_metadata.instance_name new_event.internal_metadata.outlier = event.internal_metadata.outlier + new_event.internal_metadata.redacted_by = event.internal_metadata.redacted_by return new_event @@ -423,7 +425,7 @@ class SerializeEventConfig: # the transaction_id and delay_id in the unsigned section of the event. requester: Requester | None = None # List of event fields to include. If empty, all fields will be returned. - only_event_fields: list[str] | None = None + only_event_fields: list[str] | None = attr.ib(default=None) # Some events can have stripped room state stored in the `unsigned` field. # This is required for invite and knock functionality. If this option is # False, that state will be removed from the event before it is returned. @@ -434,6 +436,16 @@ class SerializeEventConfig: # whether an event was soft failed by the server. include_admin_metadata: bool = False + @only_event_fields.validator + def _validate_only_event_fields( + self, attribute: attr.Attribute, value: Any + ) -> None: + if value is None: + return + + if not isinstance(value, list) or not all(isinstance(f, str) for f in value): + raise TypeError("only_event_fields must be a list of strings") + _DEFAULT_SERIALIZE_EVENT_CONFIG = SerializeEventConfig() @@ -444,7 +456,7 @@ def make_config_for_admin(existing: SerializeEventConfig) -> SerializeEventConfi return attr.evolve(existing, include_admin_metadata=True) -def serialize_event( +def _serialize_event( e: JsonDict | EventBase, time_now_ms: int, *, @@ -476,13 +488,6 @@ def serialize_event( d["unsigned"]["age"] = time_now_ms - d["unsigned"]["age_ts"] del d["unsigned"]["age_ts"] - if "redacted_because" in e.unsigned: - d["unsigned"]["redacted_because"] = serialize_event( - e.unsigned["redacted_because"], - time_now_ms, - config=config, - ) - # If we have applicable fields saved in the internal_metadata, include them in the # unsigned section of the event if the event was sent by the same session (or when # appropriate, just the same sender) as the one requesting the event. @@ -559,14 +564,6 @@ def serialize_event( if e.internal_metadata.policy_server_spammy: d["unsigned"]["io.element.synapse.policy_server_spammy"] = True - only_event_fields = config.only_event_fields - if only_event_fields: - if not isinstance(only_event_fields, list) or not all( - isinstance(f, str) for f in only_event_fields - ): - raise TypeError("only_event_fields must be a list of strings") - d = only_fields(d, only_event_fields) - return d @@ -591,6 +588,7 @@ async def serialize_event( *, config: SerializeEventConfig = _DEFAULT_SERIALIZE_EVENT_CONFIG, bundle_aggregations: dict[str, "BundledAggregations"] | None = None, + redaction_map: Mapping[str, "EventBase"] | None = None, ) -> JsonDict: """Serializes a single event. @@ -600,6 +598,8 @@ async def serialize_event( config: Event serialization config bundle_aggregations: A map from event_id to the aggregations to be bundled into the event. + redaction_map: Optional pre-fetched map from redaction event_id to event, + used to avoid per-event DB lookups when serializing many events. Returns: The serialized event @@ -617,7 +617,34 @@ async def serialize_event( ): config = make_config_for_admin(config) - serialized_event = serialize_event(event, time_now, config=config) + serialized_event = _serialize_event(event, time_now, config=config) + + # If the event was redacted, fetch the redaction event from the database + # and include it in the serialized event's unsigned section. + redacted_by: str | None = event.internal_metadata.redacted_by + if redacted_by is not None: + serialized_event.setdefault("unsigned", {})["redacted_by"] = redacted_by + if redaction_map is not None: + redaction_event: EventBase | None = redaction_map.get(redacted_by) + else: + redaction_event = await self._store.get_event( + redacted_by, + allow_none=True, + ) + if redaction_event is not None: + serialized_redaction = _serialize_event( + redaction_event, time_now, config=config + ) + serialized_event.setdefault("unsigned", {})["redacted_because"] = ( + serialized_redaction + ) + # format_event_for_client_v1 copies redacted_because to the + # top level, but since we add it after that runs, do it here. + if ( + config.as_client_event + and config.event_format is format_event_for_client_v1 + ): + serialized_event["redacted_because"] = serialized_redaction new_unsigned = {} for callback in self._add_extra_fields_to_unsigned_client_event_callbacks: @@ -630,6 +657,13 @@ async def serialize_event( new_unsigned.update(serialized_event["unsigned"]) serialized_event["unsigned"] = new_unsigned + # Only include fields that the client has requested. + # + # Note: we always return bundled aggregations, though it is unclear why. + only_event_fields = config.only_event_fields + if only_event_fields: + serialized_event = only_fields(serialized_event, only_event_fields) + # Check if there are any bundled aggregations to include with the event. if bundle_aggregations: if event.event_id in bundle_aggregations: @@ -745,12 +779,23 @@ async def serialize_events( str(len(events)), ) + # Batch-fetch all redaction events in one go rather than one per event. + redaction_ids = { + e.internal_metadata.redacted_by + for e in events + if isinstance(e, EventBase) and e.internal_metadata.redacted_by is not None + } + redaction_map = ( + await self._store.get_events(redaction_ids) if redaction_ids else {} + ) + return [ await self.serialize_event( event, time_now, config=config, bundle_aggregations=bundle_aggregations, + redaction_map=redaction_map, ) for event in events ] diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index ba738ad65e..55151ca549 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -72,7 +72,6 @@ from synapse.logging.opentracing import SynapseTags, log_kv, set_tag, tag_args, trace from synapse.metrics import SERVER_NAME_LABEL from synapse.types import JsonDict, StrCollection, UserID, get_domain_from_id -from synapse.types.handlers.policy_server import RECOMMENDATION_OK, RECOMMENDATION_SPAM from synapse.util.async_helpers import concurrently_execute from synapse.util.caches.expiringcache import ExpiringCache from synapse.util.duration import Duration @@ -438,72 +437,16 @@ async def _record_failure_callback( return None - @trace - @tag_args - async def get_pdu_policy_recommendation( - self, destination: str, pdu: EventBase, timeout: int | None = None - ) -> str: - """Requests that the destination server (typically a policy server) - check the event and return its recommendation on how to handle the - event. - - If the policy server could not be contacted or the policy server - returned an unknown recommendation, this returns an OK recommendation. - This type fixing behaviour is done because the typical caller will be - in a critical call path and would generally interpret a `None` or similar - response as "weird value; don't care; move on without taking action". We - just frontload that logic here. - - - Args: - destination: The remote homeserver to ask (a policy server) - pdu: The event to check - timeout: How long to try (in ms) the destination for before - giving up. None indicates no timeout. - - Returns: - The policy recommendation, or RECOMMENDATION_OK if the policy server was - uncontactable or returned an unknown recommendation. - """ - - logger.debug( - "get_pdu_policy_recommendation for event_id=%s from %s", - pdu.event_id, - destination, - ) - - try: - res = await self.transport_layer.get_policy_recommendation_for_pdu( - destination, pdu, timeout=timeout - ) - recommendation = res.get("recommendation") - if not isinstance(recommendation, str): - raise InvalidResponseError("recommendation is not a string") - if recommendation not in (RECOMMENDATION_OK, RECOMMENDATION_SPAM): - logger.warning( - "get_pdu_policy_recommendation: unknown recommendation: %s", - recommendation, - ) - return RECOMMENDATION_OK - return recommendation - except Exception as e: - logger.warning( - "get_pdu_policy_recommendation: server %s responded with error, assuming OK recommendation: %s", - destination, - e, - ) - return RECOMMENDATION_OK - @trace @tag_args async def ask_policy_server_to_sign_event( self, destination: str, pdu: EventBase, timeout: int | None = None - ) -> JsonDict | None: + ) -> JsonDict: """Requests that the destination server (typically a policy server) sign the event as not spam. If the policy server could not be contacted or the policy server - returned an error, this returns no signature. + returned an error, that error is raised. Args: destination: The remote homeserver to ask (a policy server) @@ -519,17 +462,9 @@ async def ask_policy_server_to_sign_event( pdu.event_id, destination, ) - try: - return await self.transport_layer.ask_policy_server_to_sign_event( - destination, pdu, timeout=timeout - ) - except Exception as e: - logger.warning( - "ask_policy_server_to_sign_event: server %s responded with error: %s", - destination, - e, - ) - return None + return await self.transport_layer.ask_policy_server_to_sign_event( + destination, pdu, timeout=timeout + ) @trace @tag_args diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py index 35d3c30c69..5d5212ef96 100644 --- a/synapse/federation/transport/client.py +++ b/synapse/federation/transport/client.py @@ -47,6 +47,7 @@ ) from synapse.events import EventBase, make_event_from_dict from synapse.federation.units import Transaction +from synapse.http.client import is_unknown_endpoint from synapse.http.matrixfederationclient import ByteParser, LegacyJsonSendParser from synapse.http.types import QueryParams from synapse.types import JsonDict, UserID @@ -141,33 +142,6 @@ async def get_event( destination, path=path, timeout=timeout, try_trailing_slash_on_400=True ) - async def get_policy_recommendation_for_pdu( - self, destination: str, event: EventBase, timeout: int | None = None - ) -> JsonDict: - """Requests the policy recommendation for the given pdu from the given policy server. - - Args: - destination: The host name of the remote homeserver checking the event. - event: The event to check. - timeout: How long to try (in ms) the destination for before giving up. - None indicates no timeout. - - Returns: - The full recommendation object from the remote server. - """ - logger.debug( - "get_policy_recommendation_for_pdu dest=%s, event_id=%s", - destination, - event.event_id, - ) - return await self.client.post_json( - destination=destination, - path=f"/_matrix/policy/unstable/org.matrix.msc4284/event/{event.event_id}/check", - data=event.get_pdu_json(), - ignore_backoff=True, - timeout=timeout, - ) - async def ask_policy_server_to_sign_event( self, destination: str, event: EventBase, timeout: int | None = None ) -> JsonDict: @@ -186,13 +160,28 @@ async def ask_policy_server_to_sign_event( The signature from the policy server, structured in the same was as the 'signatures' JSON in the event e.g { "$policy_server_via_domain" : { "ed25519:policy_server": "signature_base64" }} """ - return await self.client.post_json( - destination=destination, - path="/_matrix/policy/unstable/org.matrix.msc4284/sign", - data=event.get_pdu_json(), - ignore_backoff=True, - timeout=timeout, - ) + # Try stable first, then fall back to unstable if unsupported. All other errors + # are just errors. + try: + return await self.client.post_json( + destination=destination, + path="/_matrix/policy/v1/sign", + data=event.get_pdu_json(), + ignore_backoff=True, + timeout=timeout, + ) + except HttpResponseException as ex: + if is_unknown_endpoint(ex): + # TODO: Remove unstable MSC4284 support + # https://github.com/element-hq/synapse/issues/19502 + return await self.client.post_json( + destination=destination, + path="/_matrix/policy/unstable/org.matrix.msc4284/sign", + data=event.get_pdu_json(), + ignore_backoff=True, + timeout=timeout, + ) + raise async def backfill( self, destination: str, room_id: str, event_tuples: Collection[str], limit: int diff --git a/synapse/handlers/device.py b/synapse/handlers/device.py index 29074f5e20..9a371651fb 100644 --- a/synapse/handlers/device.py +++ b/synapse/handlers/device.py @@ -129,7 +129,6 @@ def __init__(self, hs: "HomeServer"): self._auth_handler = hs.get_auth_handler() self._account_data_handler = hs.get_account_data_handler() self._event_sources = hs.get_event_sources() - self._msc3852_enabled = hs.config.experimental.msc3852_enabled self._query_appservices_for_keys = ( hs.config.experimental.msc3984_appservice_key_query ) diff --git a/synapse/handlers/oidc.py b/synapse/handlers/oidc.py index 429a739380..09b19056a7 100644 --- a/synapse/handlers/oidc.py +++ b/synapse/handlers/oidc.py @@ -49,18 +49,20 @@ MacaroonInvalidSignatureException, ) +from twisted.internet import defer +from twisted.internet.defer import Deferred from twisted.web.client import readBody from twisted.web.http_headers import Headers from synapse.api.errors import SynapseError from synapse.config import ConfigError from synapse.config.oidc import OidcProviderClientSecretJwtKey, OidcProviderConfig -from synapse.handlers.sso import MappingException, UserAttributes +from synapse.handlers.sso import MappingException, SsoSetupError, UserAttributes from synapse.http.server import finish_request from synapse.http.servlet import parse_string from synapse.http.site import SynapseRequest from synapse.logging.context import make_deferred_yieldable -from synapse.module_api import ModuleApi +from synapse.metrics.background_process_metrics import wrap_as_background_process from synapse.types import JsonDict, UserID, map_username_to_mxid_localpart from synapse.util.caches.cached_call import RetryOnExceptionCachedCall from synapse.util.clock import Clock @@ -69,6 +71,7 @@ from synapse.util.templates import _localpart_from_email_filter if TYPE_CHECKING: + from synapse.module_api import ModuleApi from synapse.server import HomeServer logger = logging.getLogger(__name__) @@ -123,6 +126,8 @@ class OidcHandler: def __init__(self, hs: "HomeServer"): self._sso_handler = hs.get_sso_handler() + # Needed for wrap_as_background_process + self.hs = hs provider_confs = hs.config.oidc.oidc_providers # we should not have been instantiated if there is no configured provider. @@ -134,20 +139,47 @@ def __init__(self, hs: "HomeServer"): for p in provider_confs } - async def load_metadata(self) -> None: - """Validate the config and load the metadata from the remote endpoint. + @wrap_as_background_process("preload_oidc_metadata") + async def _preload_metadata_one_provider( + self, idp_id: str, p: "OidcProvider" + ) -> None: + """Attempt to preload the metadata from a single OIDC provider's remote endpoint + in the background. - Called at startup to ensure we have everything we need. + Will not raise exceptions, but will log at CRITICAL if an OIDC provider is broken. """ + + logger.info("Preloading OIDC provider %r", idp_id) + try: + await p.load_metadata() + if not p._uses_userinfo: + await p.load_jwks() + except Exception: + logger.critical( + # Include 'login' keyword for searchability. + "Error while preloading OIDC provider %r. Login may be broken!", + idp_id, + exc_info=True, + ) + + def preload_metadata(self) -> "Deferred[list[tuple[bool, None]]]": + """Attempt to preload the metadata from all the OIDC providers' remote endpoints + in the background. + + Will not raise exceptions, but will log at CRITICAL if an OIDC provider is broken. + + Can be **optionally** awaited in which case it will resolve when all + preloads are finished. + """ + + to_wait = [] for idp_id, p in self._providers.items(): - try: - await p.load_metadata() - if not p._uses_userinfo: - await p.load_jwks() - except Exception as e: - raise Exception( - "Error while initialising OIDC provider %r" % (idp_id,) - ) from e + to_wait.append(self._preload_metadata_one_provider(idp_id, p)) + + return defer.DeferredList( + to_wait, + consumeErrors=True, + ) async def handle_oidc_callback(self, request: SynapseRequest) -> None: """Handle an incoming request to /_synapse/client/oidc/callback @@ -359,6 +391,18 @@ def __str__(self) -> str: return self.error +class OidcDiscoveryError(SsoSetupError): + """ + Used to catch and mark errors when performing OIDC discovery. + """ + + +class OidcMetadataError(SsoSetupError): + """ + Used to catch and mark errors in the OIDC metadata configuration. + """ + + class OidcProvider: """Wraps the config for a single OIDC IdentityProvider @@ -372,6 +416,9 @@ def __init__( macaroon_generator: MacaroonGenerator, provider: OidcProviderConfig, ): + # Needed here to break import loops + from synapse.module_api import ModuleApi + self._store = hs.get_datastores().main self._clock = hs.get_clock() @@ -644,9 +691,15 @@ async def _load_metadata(self) -> OpenIDProviderMetadata: # load any data from the discovery endpoint, if enabled if self._config.discover: - url = get_well_known_url(self._config.issuer, external=True) - metadata_response = await self._http_client.get_json(url) - metadata.update(metadata_response) + try: + url = get_well_known_url(self._config.issuer, external=True) + metadata_response = await self._http_client.get_json(url) + metadata.update(metadata_response) + except Exception as e: + # This `Exception` bound is a bit broad, but at least expecting + # `twisted.internet.error.ConnectionRefusedError` + # and likely many other network or JSON errors. + raise OidcDiscoveryError() from e # override any discovered data with any settings in our config if self._config.authorization_endpoint: @@ -671,7 +724,12 @@ async def _load_metadata(self) -> OpenIDProviderMetadata: self._config.id_token_signing_alg_values_supported ) - self._validate_metadata(metadata) + try: + self._validate_metadata(metadata) + except ValueError as e: + # Wrap this error so that we can special-case it higher up + # Pass through `str(e)` as the message so tests can match it. + raise OidcMetadataError(str(e)) from e return metadata @@ -1685,7 +1743,7 @@ class JinjaOidcMappingProvider(OidcMappingProvider[JinjaOidcMappingConfig]): This is the default mapping provider. """ - def __init__(self, config: JinjaOidcMappingConfig, module_api: ModuleApi): + def __init__(self, config: JinjaOidcMappingConfig, module_api: "ModuleApi"): self._config = config @staticmethod diff --git a/synapse/handlers/room_policy.py b/synapse/handlers/room_policy.py index 0663a36714..fee3c6cfaf 100644 --- a/synapse/handlers/room_policy.py +++ b/synapse/handlers/room_policy.py @@ -17,13 +17,14 @@ import logging from typing import TYPE_CHECKING +import attr from signedjson.key import decode_verify_key_bytes from unpaddedbase64 import decode_base64 -from synapse.api.errors import SynapseError +from synapse.api.constants import EventTypes +from synapse.api.errors import Codes, HttpResponseException, SynapseError from synapse.crypto.keyring import VerifyJsonRequest from synapse.events import EventBase -from synapse.types.handlers.policy_server import RECOMMENDATION_OK from synapse.util.stringutils import parse_and_validate_server_name if TYPE_CHECKING: @@ -31,10 +32,18 @@ logger = logging.getLogger(__name__) -POLICY_SERVER_EVENT_TYPE = "org.matrix.msc4284.policy" POLICY_SERVER_KEY_ID = "ed25519:policy_server" +@attr.s(slots=True, auto_attribs=True) +class PolicyServerInfo: + # name of the server. + server_name: str + + # the unpadded base64-encoded Ed25519 public key of the server. + public_key: str + + class RoomPolicyHandler: def __init__(self, hs: "HomeServer"): self._hs = hs @@ -43,77 +52,131 @@ def __init__(self, hs: "HomeServer"): self._event_auth_handler = hs.get_event_auth_handler() self._federation_client = hs.get_federation_client() - async def is_event_allowed(self, event: EventBase) -> bool: - """Check if the given event is allowed in the room by the policy server. + def _is_policy_server_state_event(self, event: EventBase) -> bool: + state_key = event.get_state_key() + if state_key is not None and state_key == "": + # TODO: Remove unstable MSC4284 support + # https://github.com/element-hq/synapse/issues/19502 + # Note: we can probably drop this whole function when we remove unstable support + return event.type in [EventTypes.RoomPolicy, "org.matrix.msc4284.policy"] + return False - Note: This will *always* return True if the room's policy server is Synapse - itself. This is because Synapse can't be a policy server (currently). + async def _get_policy_server(self, room_id: str) -> PolicyServerInfo | None: + """Get the policy server's name and Ed25519 public key for the room, if set. - If no policy server is configured in the room, this returns True. Similarly, if - the policy server is invalid in any way (not joined, not a server, etc), this - returns True. + If the `m.room.policy` state event is invalid, then a policy server is not set. It + can be invalid if: + - The room doesn't have an `m.room.policy` state event with empty state key. + - The policy state event is missing the `via` or `public_keys` field. + - The policy state event's public keys is missing an `ed25519` key. + - The via server is not a valid server name. + - The via server is not in the room. + - The via server is Synapse itself. - If a valid and contactable policy server is configured in the room, this returns - True if that server suggests the event is not spammy, and False otherwise. + TODO: Remove unstable MSC4284 support - https://github.com/element-hq/synapse/issues/19502 + This function also checks for the unstable `org.matrix.msc4284.policy` state event. Args: - event: The event to check. This should be a fully-formed PDU. + room_id: The room ID to get the policy server for. Returns: - bool: True if the event is allowed in the room, False otherwise. + A tuple of policy server name and its Ed25519 public key (unpadded base64). + Both values will be None if no policy server is configured or the configration + is invalid. """ - if event.type == POLICY_SERVER_EVENT_TYPE and event.state_key is not None: - return True # always allow policy server change events - policy_event = await self._storage_controllers.state.get_current_state_event( - event.room_id, POLICY_SERVER_EVENT_TYPE, "" + room_id, EventTypes.RoomPolicy, "" ) + public_key = None if not policy_event: - return True # no policy server == default allow + # TODO: Remove unstable MSC4284 support + # https://github.com/element-hq/synapse/issues/19502 + policy_event = ( + await self._storage_controllers.state.get_current_state_event( + room_id, "org.matrix.msc4284.policy", "" + ) + ) + if not policy_event: + return None # neither stable or unstable configured + + # Unstable configured, grab its public key + public_key = policy_event.content.get("public_key", None) + else: + # Stable configured, grab its public key + public_keys = policy_event.content.get("public_keys") + if isinstance(public_keys, dict): + ed25519_key = public_keys.get("ed25519") + if isinstance(ed25519_key, str): + public_key = ed25519_key + + if public_key is None or not isinstance(public_key, str): + return None # no public key means no policy server policy_server = policy_event.content.get("via", "") if policy_server is None or not isinstance(policy_server, str): - return True # no policy server == default allow + return None # no policy server if policy_server == self._hs.hostname: - return True # Synapse itself can't be a policy server (currently) + return None # Synapse itself can't be a policy server (currently) try: parse_and_validate_server_name(policy_server) except ValueError: - return True # invalid policy server == default allow + return None # invalid policy server is_in_room = await self._event_auth_handler.is_host_in_room( - event.room_id, policy_server + room_id, policy_server ) if not is_in_room: - return True # policy server not in room == default allow - - # Check if the event has been signed with the public key in the policy server state event. - # If it is, we can save an HTTP hit. - # We actually want to get the policy server state event BEFORE THE EVENT rather than - # the current state value, else changing the public key will cause all of these checks to fail. - # However, if we are checking outlier events (which we will due to is_event_allowed being called - # near the edges at _check_sigs_and_hash) we won't know the state before the event, so the - # only safe option is to use the current state - public_key = policy_event.content.get("public_key", None) - if public_key is not None and isinstance(public_key, str): - valid = await self._verify_policy_server_signature( - event, policy_server, public_key - ) - if valid: - return True - # fallthrough to hit /check manually - - # At this point, the server appears valid and is in the room, so ask it to check - # the event. - recommendation = await self._federation_client.get_pdu_policy_recommendation( - policy_server, event + return None # policy server not in room + + return PolicyServerInfo(policy_server, public_key) + + async def is_event_allowed(self, event: EventBase) -> bool: + """Check if the given event is allowed in the room by the policy server. + + Note: This will *always* return True if the room's policy server is Synapse + itself. This is because Synapse can't be a policy server (currently). + + If no policy server is configured in the room, this returns True. Similarly, if + the policy server is invalid in any way (not joined, not a server, etc), this + returns True. + + If a valid and contactable policy server is configured in the room, this returns + True if that server suggests the event is not spammy, and False otherwise. + + Args: + event: The event to check. This should be a fully-formed PDU. + + Returns: + bool: True if the event is allowed in the room, False otherwise. + """ + if self._is_policy_server_state_event(event): + return True # always allow policy server change events + + policy_server = await self._get_policy_server(event.room_id) + if policy_server is None: + return True # no policy server configured, so allow + + # Check if the event has been signed with the public key in the policy server + # state event. If it is, the event is valid according to the policy server and + # we don't need to request a fresh signature. + valid = await self._verify_policy_server_signature( + event, policy_server.server_name, policy_server.public_key ) - if recommendation != RECOMMENDATION_OK: + if valid: + return True # valid signature == allow + + # We couldn't save the HTTP hit, so do that hit. + try: + await self.ask_policy_server_to_sign_event(event, verify=True) + except Exception as ex: + # We probably caught either a refusal to sign, an invalid signature, or + # some other transient or network error. These are all rejection cases. + logger.warning("Failed to get a signature from the policy server: %s", ex) return False - return True # default allow + return True # passed all verifications and checks, so allow async def _verify_policy_server_signature( self, event: EventBase, policy_server: str, public_key: str @@ -140,47 +203,71 @@ async def ask_policy_server_to_sign_event( """Ask the policy server to sign this event. The signature is added to the event signatures block. Does nothing if there is no policy server state event in the room. If the policy server - refuses to sign the event (as it's marked as spam) does nothing. + refuses to sign the event (as it's marked as spam), an error is raised. Args: - event: The event to sign - verify: If True, verify that the signature is correctly signed by the public_key in the - policy server state event. + event: The event to sign. + verify: If True, verify that the signature is correctly signed by the policy server's + defined public key. Raises: - if verify=True and the policy server signed the event with an invalid signature. Does - not raise if the policy server refuses to sign the event. + When the policy server refuses to sign the event, or when verify is True and the + signature is invalid. """ - policy_event = await self._storage_controllers.state.get_current_state_event( - event.room_id, POLICY_SERVER_EVENT_TYPE, "" - ) - if not policy_event: + if self._is_policy_server_state_event(event): + # per spec, policy servers aren't asked to sign `m.room.policy` state events + # with empty state keys return - policy_server = policy_event.content.get("via", None) - if policy_server is None or not isinstance(policy_server, str): - return - # Only ask to sign events if the policy state event has a public_key (so they can be subsequently verified) - public_key = policy_event.content.get("public_key", None) - if public_key is None or not isinstance(public_key, str): + + policy_server = await self._get_policy_server(event.room_id) + if policy_server is None: return # Ask the policy server to sign this event. # We set a smallish timeout here as we don't want to block event sending too long. - signature = await self._federation_client.ask_policy_server_to_sign_event( - policy_server, - event, - timeout=3000, - ) - if ( - # the policy server returns {} if it refuses to sign the event. - signature and len(signature) > 0 - ): - event.signatures.update(signature) - if verify: - is_valid = await self._verify_policy_server_signature( - event, policy_server, public_key + try: + signature = await self._federation_client.ask_policy_server_to_sign_event( + policy_server.server_name, + event, + timeout=30000, + ) + # TODO: We can *probably* remove this when we remove unstable MSC4284 support. + # The server *should* be returning either a signature or an error, but there could + # also be implementation bugs. Whoever reads this when removing unstable MSC4284 + # stuff, make a decision on whether to remove this bit. + # https://github.com/element-hq/synapse/issues/19502 + if not signature or len(signature) == 0: + raise SynapseError( + 403, + "This event has been rejected as probable spam by the policy server", + Codes.FORBIDDEN, + ) + + # Note: if the policy server and event sender are the same server, the sender + # might not have added policy server signatures to the event for whatever reason. + # When this happens, we don't want to obliterate the event's existing signatures + # because the event will fail authorization. This is why we add defaults rather + # than simply `update` the signatures on the event. + # + # This situation can happen if the homeserver and policy server parts are + # logically the same server, but run by different software. For example, Synapse + # will not ask "itself" for a policy server signature, even if its server name + # is the designated policy server, so it could send an event outwards that other + # servers need to manually fetch signatures for. This is the code that allows + # those events to continue working (because they're legally sent, even if missing + # the policy server signature). + event.signatures.setdefault(policy_server.server_name, {}).update( + signature.get(policy_server.server_name, {}) + ) + except HttpResponseException as ex: + # re-wrap HTTP errors as `SynapseError` so they can be proxied to clients directly + raise ex.to_synapse_error() from ex + + if verify: + is_valid = await self._verify_policy_server_signature( + event, policy_server.server_name, policy_server.public_key + ) + if not is_valid: + raise SynapseError( + 500, + f"policy server {policy_server.server_name} failed to sign event correctly", ) - if not is_valid: - raise SynapseError( - 500, - f"policy server {policy_server} failed to sign event correctly", - ) diff --git a/synapse/handlers/room_summary.py b/synapse/handlers/room_summary.py index 9ec0d33f11..bbcdc0877e 100644 --- a/synapse/handlers/room_summary.py +++ b/synapse/handlers/room_summary.py @@ -128,7 +128,6 @@ def __init__(self, hs: "HomeServer"): name="get_room_hierarchy", server_name=self.server_name, ) - self._msc3266_enabled = hs.config.experimental.msc3266_enabled async def get_room_hierarchy( self, @@ -791,6 +790,7 @@ async def _build_room_entry(self, room_id: str, for_federation: bool) -> JsonDic entry: JsonDict = { "room_id": stats.room_id, + "room_version": stats.version, "name": stats.name, "topic": stats.topic, "canonical_alias": stats.canonical_alias, @@ -802,12 +802,9 @@ async def _build_room_entry(self, room_id: str, for_federation: bool) -> JsonDic ), "guest_can_join": stats.guest_access == "can_join", "room_type": stats.room_type, + "encryption": stats.encryption, } - if self._msc3266_enabled: - entry["im.nheko.summary.version"] = stats.version - entry["im.nheko.summary.encryption"] = stats.encryption - # Federation requests need to provide additional information so the # requested server is able to filter the response appropriately. if for_federation: diff --git a/synapse/handlers/sso.py b/synapse/handlers/sso.py index 0b2587f94f..bb5ca329e0 100644 --- a/synapse/handlers/sso.py +++ b/synapse/handlers/sso.py @@ -70,6 +70,15 @@ class MappingException(Exception): """ +class SsoSetupError(Exception): + """ + Used to catch and tag errors relating to an SSO setup's. + + Used as the superclass of specific error classes, + as note we can't e.g. import the OIDC module unless OIDC dependencies are installed. + """ + + class SsoIdentityProvider(Protocol): """Abstract base class to be implemented by SSO Identity Providers diff --git a/synapse/res/templates/sso_error.html b/synapse/res/templates/sso_error.html index 6fa36c11c9..e7e34c7293 100644 --- a/synapse/res/templates/sso_error.html +++ b/synapse/res/templates/sso_error.html @@ -20,6 +20,27 @@

You are not allowed to log in here.

+{% elif error == "provider_unavailable" %} +
+

This login provider is unavailable right now.

+

+ There has been a problem which means that it's not currently possible + to log in with that provider. +

+

+ If your account has a password, or is connected to other login providers, + try those methods to log in. +

+

+ This issue could be temporary, so please try again later. + If the problem persists, please contact the server's administrator. +

+ + If you are the administrator of this server, please check the server logs for more information. + This could be caused by a server misconfiguration or an issue with the provider. + +
+ {% include "sso_footer.html" without context %} {% else %}

There was an error

diff --git a/synapse/rest/admin/events.py b/synapse/rest/admin/events.py index 1c39d5caf3..8da7a67820 100644 --- a/synapse/rest/admin/events.py +++ b/synapse/rest/admin/events.py @@ -5,7 +5,6 @@ from synapse.events.utils import ( SerializeEventConfig, format_event_raw, - serialize_event, ) from synapse.http.servlet import RestServlet from synapse.http.site import SynapseRequest @@ -40,6 +39,7 @@ def __init__(self, hs: "HomeServer"): self._auth = hs.get_auth() self._store = hs.get_datastores().main self._clock = hs.get_clock() + self._event_serializer = hs.get_event_client_serializer() async def on_GET( self, request: SynapseRequest, event_id: str @@ -64,6 +64,10 @@ async def on_GET( include_stripped_room_state=True, include_admin_metadata=True, ) - res = {"event": serialize_event(event, self._clock.time_msec(), config=config)} + res = { + "event": await self._event_serializer.serialize_event( + event, self._clock.time_msec(), config=config + ) + } return HTTPStatus.OK, res diff --git a/synapse/rest/client/devices.py b/synapse/rest/client/devices.py index b39ca6a483..4b84131d32 100644 --- a/synapse/rest/client/devices.py +++ b/synapse/rest/client/devices.py @@ -55,7 +55,6 @@ def __init__(self, hs: "HomeServer"): self.hs = hs self.auth = hs.get_auth() self.device_handler = hs.get_device_handler() - self._msc3852_enabled = hs.config.experimental.msc3852_enabled async def on_GET(self, request: SynapseRequest) -> tuple[int, JsonDict]: requester = await self.auth.get_user_by_req(request, allow_guest=True) @@ -63,17 +62,10 @@ async def on_GET(self, request: SynapseRequest) -> tuple[int, JsonDict]: requester.user.to_string() ) - # If MSC3852 is disabled, then the "last_seen_user_agent" field will be - # removed from each device. If it is enabled, then the field name will - # be replaced by the unstable identifier. - # - # When MSC3852 is accepted, this block of code can just be removed to - # expose "last_seen_user_agent" to clients. for device in devices: - last_seen_user_agent = device["last_seen_user_agent"] + # This field is only for admin access and should not be exposed to clients. + # (MSC3852, which is closed, did propose to expose it.). del device["last_seen_user_agent"] - if self._msc3852_enabled: - device["org.matrix.msc3852.last_seen_user_agent"] = last_seen_user_agent return 200, {"devices": devices} @@ -144,7 +136,6 @@ def __init__(self, hs: "HomeServer"): handler = hs.get_device_handler() self.device_handler = handler self.auth_handler = hs.get_auth_handler() - self._msc3852_enabled = hs.config.experimental.msc3852_enabled self._auth_delegation_enabled = ( hs.config.mas.enabled or hs.config.experimental.msc3861.enabled ) @@ -159,16 +150,9 @@ async def on_GET( if device is None: raise NotFoundError("No device found") - # If MSC3852 is disabled, then the "last_seen_user_agent" field will be - # removed from each device. If it is enabled, then the field name will - # be replaced by the unstable identifier. - # - # When MSC3852 is accepted, this block of code can just be removed to - # expose "last_seen_user_agent" to clients. - last_seen_user_agent = device["last_seen_user_agent"] + # This field is only for admin access and should not be exposed to clients. + # (MSC3852, which is closed, did propose to expose it.) del device["last_seen_user_agent"] - if self._msc3852_enabled: - device["org.matrix.msc3852.last_seen_user_agent"] = last_seen_user_agent return 200, device diff --git a/synapse/rest/client/login.py b/synapse/rest/client/login.py index f7d54bec6f..3e1bc0e369 100644 --- a/synapse/rest/client/login.py +++ b/synapse/rest/client/login.py @@ -18,7 +18,6 @@ # [This file includes modifications made by New Vector Limited] # # - import logging import re from typing import ( @@ -42,7 +41,7 @@ from synapse.api.ratelimiting import Ratelimiter from synapse.api.urls import CLIENT_API_PREFIX from synapse.appservice import ApplicationService -from synapse.handlers.sso import SsoIdentityProvider +from synapse.handlers.sso import SsoIdentityProvider, SsoSetupError from synapse.http import get_request_uri from synapse.http.server import HttpServer, finish_request from synapse.http.servlet import ( @@ -697,11 +696,26 @@ async def on_GET(self, request: SynapseRequest, idp_id: str | None = None) -> No args: dict[bytes, list[bytes]] = request.args # type: ignore client_redirect_url = parse_bytes_from_args(args, "redirectUrl", required=True) - sso_url = await self._sso_handler.handle_redirect_request( - request, - client_redirect_url, - idp_id, - ) + try: + sso_url = await self._sso_handler.handle_redirect_request( + request, + client_redirect_url, + idp_id, + ) + except SsoSetupError: + logger.exception( + "Login redirect failed because SSO/identity provider %r unavailable", + idp_id, + ) + # Show an error page that is slightly more friendly than JSON + self._sso_handler.render_error( + request, + "provider_unavailable", + "This login provider is currently unavailable on this homeserver.", + code=503, + ) + return + logger.info("Redirecting to %s", sso_url) request.redirect(sso_url) finish_request(request) diff --git a/synapse/rest/client/mutual_rooms.py b/synapse/rest/client/mutual_rooms.py index a6a913db34..d6783c15da 100644 --- a/synapse/rest/client/mutual_rooms.py +++ b/synapse/rest/client/mutual_rooms.py @@ -29,7 +29,7 @@ from synapse.http.server import HttpServer from synapse.http.servlet import RestServlet, parse_strings_from_args from synapse.http.site import SynapseRequest -from synapse.types import JsonDict +from synapse.types import JsonDict, UserID from ._base import client_patterns @@ -65,13 +65,10 @@ def _parse_mutual_rooms_batch_token_args(args: dict[bytes, list[bytes]]) -> str class UserMutualRoomsServlet(RestServlet): """ - GET /uk.half-shot.msc2666/user/mutual_rooms?user_id={user_id}&from={token} HTTP/1.1 + GET /mutual_rooms?user_id={user_id}&from={token} HTTP/1.1 """ - PATTERNS = client_patterns( - "/uk.half-shot.msc2666/user/mutual_rooms$", - releases=(), # This is an unstable feature - ) + PATTERNS = [*client_patterns("/mutual_rooms$", releases=("v1",))] def __init__(self, hs: "HomeServer"): super().__init__() @@ -82,7 +79,9 @@ async def on_GET(self, request: SynapseRequest) -> tuple[int, JsonDict]: # twisted.web.server.Request.args is incorrectly defined as Any | None args: dict[bytes, list[bytes]] = request.args # type: ignore - user_ids = parse_strings_from_args(args, "user_id", required=True) + user_ids = parse_strings_from_args( + args, "user_id", required=True, encoding="utf-8" + ) from_batch = _parse_mutual_rooms_batch_token_args(args) if len(user_ids) > 1: @@ -93,13 +92,19 @@ async def on_GET(self, request: SynapseRequest) -> tuple[int, JsonDict]: ) user_id = user_ids[0] + if not UserID.is_valid_strict(user_id): + raise SynapseError( + HTTPStatus.BAD_REQUEST, + "Invalid user_id query parameter", + errcode=Codes.INVALID_PARAM, + ) requester = await self.auth.get_user_by_req(request) if user_id == requester.user.to_string(): raise SynapseError( HTTPStatus.BAD_REQUEST, "You cannot request a list of shared rooms with yourself", - errcode=Codes.UNKNOWN, + errcode=Codes.INVALID_PARAM, ) # Sort here instead of the database function, so that we don't expose @@ -109,6 +114,7 @@ async def on_GET(self, request: SynapseRequest) -> tuple[int, JsonDict]: frozenset((requester.user.to_string(), user_id)) ) ) + total_count = len(rooms) if from_batch: # A from_batch token was provided, so cut off any rooms where the ID is @@ -123,7 +129,7 @@ async def on_GET(self, request: SynapseRequest) -> tuple[int, JsonDict]: if len(rooms) <= MUTUAL_ROOMS_BATCH_LIMIT: # We've reached the end of the list, don't return a batch token - return 200, {"joined": rooms} + return 200, {"joined": rooms, "count": total_count} rooms = rooms[:MUTUAL_ROOMS_BATCH_LIMIT] # We use urlsafe unpadded base64 encoding for the batch token in order to @@ -135,11 +141,14 @@ async def on_GET(self, request: SynapseRequest) -> tuple[int, JsonDict]: # in the room ID. In the event that some silly user does that, don't let # them paginate further. if next_batch == from_batch: - return 200, {"joined": rooms} + return 200, {"joined": rooms, "count": total_count} - return 200, {"joined": list(rooms), "next_batch": next_batch} + return 200, { + "joined": rooms, + "next_batch": next_batch, + "count": total_count, + } def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: - if hs.config.experimental.msc2666_enabled: - UserMutualRoomsServlet(hs).register(http_server) + UserMutualRoomsServlet(hs).register(http_server) diff --git a/synapse/rest/client/register.py b/synapse/rest/client/register.py index fdd2f1985a..73832ba7a8 100644 --- a/synapse/rest/client/register.py +++ b/synapse/rest/client/register.py @@ -86,6 +86,7 @@ def __init__(self, hs: "HomeServer"): self.server_name = hs.hostname self.identity_handler = hs.get_identity_handler() self.config = hs.config + self._registration_enabled = hs.config.registration.enable_registration if self.hs.config.email.can_verify_email: self.registration_mailer = Mailer( @@ -109,6 +110,14 @@ async def on_POST(self, request: SynapseRequest) -> tuple[int, JsonDict]: raise SynapseError( 400, "Email-based registration has been disabled on this server" ) + + if not self._registration_enabled: + raise SynapseError( + 403, + "Registration is disabled on this homeserver", + Codes.FORBIDDEN, + ) + body = parse_json_object_from_request(request) assert_params_in_dict(body, ["client_secret", "email", "send_attempt"]) diff --git a/synapse/rest/client/rendezvous.py b/synapse/rest/client/rendezvous.py index bd9205fc5f..246788609e 100644 --- a/synapse/rest/client/rendezvous.py +++ b/synapse/rest/client/rendezvous.py @@ -20,6 +20,7 @@ # import logging +from http import HTTPStatus from http.client import TEMPORARY_REDIRECT from typing import TYPE_CHECKING, Any @@ -81,6 +82,12 @@ def __init__(self, hs: "HomeServer") -> None: hs.config.experimental.msc4388_requires_authentication ) + async def on_GET(self, request: SynapseRequest) -> tuple[int, Any]: + if self.require_authentication: + # This will raise if the user is not authenticated + await self.auth.get_user_by_req(request) + return HTTPStatus.OK, {"create_available": True} + async def on_POST(self, request: SynapseRequest) -> tuple[int, Any]: if self.require_authentication: # This will raise if the user is not authenticated diff --git a/synapse/rest/client/room.py b/synapse/rest/client/room.py index 9172bfcb4e..65d9c130ef 100644 --- a/synapse/rest/client/room.py +++ b/synapse/rest/client/room.py @@ -55,7 +55,6 @@ EventClientSerializer, SerializeEventConfig, format_event_for_client_v2, - serialize_event, ) from synapse.handlers.pagination import GetMessagesResult from synapse.http.server import HttpServer @@ -214,6 +213,7 @@ def __init__(self, hs: "HomeServer"): self.delayed_events_handler = hs.get_delayed_events_handler() self.auth = hs.get_auth() self.clock = hs.get_clock() + self._event_serializer = hs.get_event_client_serializer() self._max_event_delay_ms = hs.config.server.max_event_delay_ms self._spam_checker_module_callbacks = hs.get_module_api_callbacks().spam_checker self._msc4354_enabled = hs.config.experimental.msc4354_enabled @@ -285,7 +285,7 @@ async def on_GET( raise SynapseError(404, "Event not found.", errcode=Codes.NOT_FOUND) if format == "event": - event = serialize_event( + event = await self._event_serializer.serialize_event( data, self.clock.time_msec(), config=SerializeEventConfig( diff --git a/synapse/rest/client/versions.py b/synapse/rest/client/versions.py index f8d7a1a4d9..7bf4b12e8b 100644 --- a/synapse/rest/client/versions.py +++ b/synapse/rest/client/versions.py @@ -144,7 +144,7 @@ async def on_GET(self, request: SynapseRequest) -> tuple[int, JsonDict]: # Implements additional endpoints as described in MSC2432 "org.matrix.msc2432": True, # Implements additional endpoints as described in MSC2666 - "uk.half-shot.msc2666.query_mutual_rooms": self.config.experimental.msc2666_enabled, + "uk.half-shot.msc2666.query_mutual_rooms.stable": True, # Whether new rooms will be set to encrypted or not (based on presets). "io.element.e2ee_forced.public": self.e2ee_forced_public, "io.element.e2ee_forced.private": self.e2ee_forced_private, @@ -189,8 +189,6 @@ async def on_GET(self, request: SynapseRequest) -> tuple[int, JsonDict]: is not None ) ), - # MSC4388: Secure out-of-band channel for sign in with QR - "io.element.msc4388": (self.config.experimental.msc4388_enabled), # MSC4140: Delayed events "org.matrix.msc4140": bool(self.config.server.max_event_delay_ms), # Simplified sliding sync diff --git a/synapse/storage/databases/main/events.py b/synapse/storage/databases/main/events.py index cb452dbc9b..941a5f9f3a 100644 --- a/synapse/storage/databases/main/events.py +++ b/synapse/storage/databases/main/events.py @@ -2944,6 +2944,7 @@ def _store_redaction(self, txn: LoggingTransaction, event: EventBase) -> None: values={ "redacts": event.redacts, "received_ts": self._clock.time_msec(), + "recheck": event.internal_metadata.need_to_check_redaction(), }, ) diff --git a/synapse/storage/databases/main/events_bg_updates.py b/synapse/storage/databases/main/events_bg_updates.py index f8300e016b..934cd157ca 100644 --- a/synapse/storage/databases/main/events_bg_updates.py +++ b/synapse/storage/databases/main/events_bg_updates.py @@ -159,6 +159,11 @@ def __init__( "redactions_received_ts", self._redactions_received_ts ) + self.db_pool.updates.register_background_update_handler( + _BackgroundUpdates.REDACTIONS_RECHECK_BG_UPDATE, + self._redactions_recheck_bg_update, + ) + # This index gets deleted in `event_fix_redactions_bytes` update self.db_pool.updates.register_background_index_update( "event_fix_redactions_bytes_create_index", @@ -747,6 +752,66 @@ def _redactions_received_ts_txn(txn: LoggingTransaction) -> int: return count + async def _redactions_recheck_bg_update( + self, progress: JsonDict, batch_size: int + ) -> int: + """Fills in the `recheck` column of the `redactions` table based on + the `recheck_redaction` field in each event's internal metadata.""" + last_event_id = progress.get("last_event_id", "") + + def _txn(txn: LoggingTransaction) -> int: + sql = """ + SELECT r.event_id, ej.internal_metadata + FROM redactions AS r + LEFT JOIN event_json AS ej USING (event_id) + WHERE r.event_id > ? + ORDER BY r.event_id ASC + LIMIT ? + """ + txn.execute(sql, (last_event_id, batch_size)) + rows = txn.fetchall() + if not rows: + return 0 + + updates = [] + for event_id, internal_metadata_json in rows: + if internal_metadata_json is not None: + internal_metadata = db_to_json(internal_metadata_json) + recheck = bool(internal_metadata.get("recheck_redaction", False)) + else: + recheck = False + if not recheck: + # Column defaults to true, so we only need to update rows + # where recheck should be false. + updates.append((event_id, recheck)) + + self.db_pool.simple_update_many_txn( + txn, + table="redactions", + key_names=("event_id",), + key_values=[(event_id,) for event_id, _ in updates], + value_names=("recheck",), + value_values=[(recheck,) for _, recheck in updates], + ) + + upper_event_id = rows[-1][0] + self.db_pool.updates._background_update_progress_txn( + txn, + _BackgroundUpdates.REDACTIONS_RECHECK_BG_UPDATE, + {"last_event_id": upper_event_id}, + ) + + return len(rows) + + count = await self.db_pool.runInteraction("_redactions_recheck_bg_update", _txn) + + if not count: + await self.db_pool.updates._end_background_update( + _BackgroundUpdates.REDACTIONS_RECHECK_BG_UPDATE + ) + + return count + async def _event_fix_redactions_bytes( self, progress: JsonDict, batch_size: int ) -> int: diff --git a/synapse/storage/databases/main/events_worker.py b/synapse/storage/databases/main/events_worker.py index cb0764feb8..cc79b8042b 100644 --- a/synapse/storage/databases/main/events_worker.py +++ b/synapse/storage/databases/main/events_worker.py @@ -179,7 +179,11 @@ class _EventRow: rejected_reason: if the event was rejected, the reason why. - redactions: a list of event-ids which (claim to) redact this event. + unconfirmed_redactions: a list of event-ids which (claim to) redact this event + and need to be rechecked. + + confirmed_redactions: a list of event-ids which redact this event and have been + confirmed as valid redactions. outlier: True if this event is an outlier. """ @@ -192,7 +196,8 @@ class _EventRow: format_version: int | None room_version_id: str | None rejected_reason: str | None - redactions: list[str] + unconfirmed_redactions: list[str] + confirmed_redactions: list[str] outlier: bool @@ -1359,14 +1364,20 @@ async def _fetch_event_ids_and_get_outstanding_redactions( ) row_map = await self._enqueue_events(event_ids_to_fetch) - # we need to recursively fetch any redactions of those events + # we need to recursively fetch redaction events that require + # rechecking, so we can validate them redaction_ids: set[str] = set() for event_id in event_ids_to_fetch: row = row_map.get(event_id) fetched_event_ids.add(event_id) if row: fetched_events[event_id] = row - redaction_ids.update(row.redactions) + + # If this event only has unconfirmed redactions we fetch + # them from the DB so that we check them to see if any are + # valid. + if not row.confirmed_redactions: + redaction_ids.update(row.unconfirmed_redactions) event_ids_to_fetch = redaction_ids.difference(fetched_event_ids) return event_ids_to_fetch @@ -1510,9 +1521,12 @@ async def _fetch_event_ids_and_get_outstanding_redactions( # the cache entries. result_map: dict[str, EventCacheEntry] = {} for event_id, original_ev in event_map.items(): - redactions = fetched_events[event_id].redactions + row = fetched_events[event_id] redacted_event = self._maybe_redact_event_row( - original_ev, redactions, event_map + original_ev, + row.unconfirmed_redactions, + row.confirmed_redactions, + event_map, ) cache_entry = EventCacheEntry( @@ -1606,21 +1620,25 @@ def _fetch_event_rows( format_version=row[5], room_version_id=row[6], rejected_reason=row[7], - redactions=[], + unconfirmed_redactions=[], + confirmed_redactions=[], outlier=bool(row[8]), # This is an int in SQLite3 ) # check for redactions - redactions_sql = "SELECT event_id, redacts FROM redactions WHERE " + redactions_sql = "SELECT event_id, redacts, recheck FROM redactions WHERE " clause, args = make_in_list_sql_clause(txn.database_engine, "redacts", evs) txn.execute(redactions_sql + clause, args) - for redacter, redacted in txn: + for redacter, redacted, recheck in txn: d = event_dict.get(redacted) if d: - d.redactions.append(redacter) + if recheck: + d.unconfirmed_redactions.append(redacter) + else: + d.confirmed_redactions.append(redacter) # check for MSC4293 redactions to_check = [] @@ -1669,24 +1687,28 @@ def _fetch_event_rows( # backfilled events, as they have a negative stream ordering if e_row.stream_ordering >= redact_end_ordering: continue - e_row.redactions.append(redacting_event_id) + e_row.unconfirmed_redactions.append(redacting_event_id) return event_dict def _maybe_redact_event_row( self, original_ev: EventBase, - redactions: Iterable[str], + unconfirmed_redactions: Iterable[str], + confirmed_redactions: Iterable[str], event_map: dict[str, EventBase], ) -> EventBase | None: - """Given an event object and a list of possible redacting event ids, + """Given an event object and lists of possible redacting event ids, determine whether to honour any of those redactions and if so return a redacted event. Args: original_ev: The original event. - redactions: list of event ids of potential redaction events + unconfirmed_redactions: list of event ids of redaction events that need + domain rechecking (room v3+). + confirmed_redactions: list of event ids of redaction events that have + already been validated and do not need rechecking. event_map: other events which have been fetched, in which we can - look up the redaaction events. Map from event id to event. + look up the redaction events. Map from event id to event. Returns: If the event should be redacted, a pruned event object. Otherwise, None. @@ -1695,7 +1717,12 @@ def _maybe_redact_event_row( # we choose to ignore redactions of m.room.create events. return None - for redaction_id in redactions: + for redaction_id in confirmed_redactions: + redacted_event = prune_event(original_ev) + redacted_event.internal_metadata.redacted_by = redaction_id + return redacted_event + + for redaction_id in unconfirmed_redactions: redaction_event = event_map.get(redaction_id) if not redaction_event or redaction_event.rejected_reason: # we don't have the redaction event, or the redaction event was not @@ -1736,12 +1763,10 @@ def _maybe_redact_event_row( # we found a good redaction event. Redact! redacted_event = prune_event(original_ev) - redacted_event.unsigned["redacted_by"] = redaction_id - - # It's fine to add the event directly, since get_pdu_json - # will serialise this field correctly - redacted_event.unsigned["redacted_because"] = redaction_event + redacted_event.internal_metadata.redacted_by = redaction_id + # Note: The `redacted_because` field will later be populated by + # `EventClientSerializer.serialize_event`. return redacted_event # no valid redaction found for this event diff --git a/synapse/storage/databases/main/room.py b/synapse/storage/databases/main/room.py index 633df07736..7ac88e4c2a 100644 --- a/synapse/storage/databases/main/room.py +++ b/synapse/storage/databases/main/room.py @@ -1217,7 +1217,6 @@ def _quarantine_remote_media_txn( txn.execute(sql, [quarantined_by] + sql_args) total_media_quarantined += txn.rowcount if txn.rowcount > 0 else 0 - total_media_quarantined = 0 if hashes: sql_many_clause_sql, sql_many_clause_args = make_in_list_sql_clause( txn.database_engine, "sha256", hashes diff --git a/synapse/storage/engines/postgres.py b/synapse/storage/engines/postgres.py index 03ecff27f0..7cd50fb8f1 100644 --- a/synapse/storage/engines/postgres.py +++ b/synapse/storage/engines/postgres.py @@ -31,6 +31,7 @@ IsolationLevel, ) from synapse.storage.types import Cursor +from synapse.util.duration import Duration if TYPE_CHECKING: from synapse.storage.database import LoggingDatabaseConnection @@ -54,14 +55,15 @@ def _disable_bytes_adapter(_: bytes) -> NoReturn: psycopg2.extensions.register_adapter(bytes, _disable_bytes_adapter) self.synchronous_commit: bool = database_config.get("synchronous_commit", True) - # Set the statement timeout to 1 hour by default. - # Any query taking more than 1 hour should probably be considered a bug; + # Set the statement timeout to 10 minutes by default. + # + # Any query taking more than 10 minutes should probably be considered a bug; # most of the time this is a sign that work needs to be split up or that # some degenerate query plan has been created and the client has probably # timed out/walked off anyway. # This is in milliseconds. self.statement_timeout: int | None = database_config.get( - "statement_timeout", 60 * 60 * 1000 + "statement_timeout", Duration(minutes=10).as_millis() ) self._version: int | None = None # unknown as yet diff --git a/synapse/storage/schema/__init__.py b/synapse/storage/schema/__init__.py index 7d65bc742f..c16e492cc7 100644 --- a/synapse/storage/schema/__init__.py +++ b/synapse/storage/schema/__init__.py @@ -19,7 +19,7 @@ # # -SCHEMA_VERSION = 93 # remember to update the list below when updating +SCHEMA_VERSION = 94 # remember to update the list below when updating """Represents the expectations made by the codebase about the database schema This should be incremented whenever the codebase changes its requirements on the @@ -171,6 +171,9 @@ Changes in SCHEMA_VERSION = 93 - MSC4140: Set delayed events to be uniquely identifiable by their delay ID. + +Changes in SCHEMA_VERSION = 94 + - Add `recheck` column (boolean, default true) to the `redactions` table. """ diff --git a/synapse/storage/schema/main/delta/94/01_redactions_recheck.sql b/synapse/storage/schema/main/delta/94/01_redactions_recheck.sql new file mode 100644 index 0000000000..e99aa52f1f --- /dev/null +++ b/synapse/storage/schema/main/delta/94/01_redactions_recheck.sql @@ -0,0 +1,15 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2026 Element Creations, Ltd +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + + +ALTER TABLE redactions ADD COLUMN recheck boolean NOT NULL DEFAULT true; diff --git a/synapse/storage/schema/main/delta/94/02_redactions_recheck_bg_update.sql b/synapse/storage/schema/main/delta/94/02_redactions_recheck_bg_update.sql new file mode 100644 index 0000000000..6367afd318 --- /dev/null +++ b/synapse/storage/schema/main/delta/94/02_redactions_recheck_bg_update.sql @@ -0,0 +1,15 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2026 Element Creations, Ltd +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + +INSERT INTO background_updates (ordering, update_name, progress_json) VALUES + (9402, 'redactions_recheck', '{}'); diff --git a/synapse/synapse_rust/events.pyi b/synapse/synapse_rust/events.pyi index eac5699a77..73a9b75ca0 100644 --- a/synapse/synapse_rust/events.pyi +++ b/synapse/synapse_rust/events.pyi @@ -21,6 +21,8 @@ class EventInternalMetadata: """the stream ordering of this event. None, until it has been persisted.""" instance_name: str | None """the instance name of the server that persisted this event. None, until it has been persisted.""" + redacted_by: str | None + """the event ID of the redaction event, if this event has been redacted. Set dynamically at load time, not persisted.""" outlier: bool """whether this event is an outlier (ie, whether we have the state at that diff --git a/synapse/synapse_rust/push.pyi b/synapse/synapse_rust/push.pyi index 9d8f0389e8..ef0d5f94f4 100644 --- a/synapse/synapse_rust/push.pyi +++ b/synapse/synapse_rust/push.pyi @@ -65,7 +65,7 @@ class PushRuleEvaluator: notification_power_levels: Mapping[str, int], related_events_flattened: Mapping[str, Mapping[str, JsonValue]], related_event_match_enabled: bool, - room_version_feature_flags: tuple[str, ...], + room_version_feature_flags: list[str], msc3931_enabled: bool, msc4210_enabled: bool, msc4306_enabled: bool, diff --git a/synapse/synapse_rust/room_versions.pyi b/synapse/synapse_rust/room_versions.pyi new file mode 100644 index 0000000000..909e3a1c26 --- /dev/null +++ b/synapse/synapse_rust/room_versions.pyi @@ -0,0 +1,142 @@ +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . + +from collections.abc import Mapping +from typing import Iterator + +class EventFormatVersions: + """Internal enum for tracking the version of the event format, + independently of the room version. + + To reduce confusion, the event format versions are named after the room + versions that they were used or introduced in. + The concept of an 'event format version' is specific to Synapse (the + specification does not mention this term.) + """ + + ROOM_V1_V2: int + """:server event id format: used for room v1 and v2""" + ROOM_V3: int + """MSC1659-style event id format: used for room v3""" + ROOM_V4_PLUS: int + """MSC1884-style format: introduced for room v4""" + ROOM_V11_HYDRA_PLUS: int + """MSC4291 room IDs as hashes: introduced for room HydraV11""" + +KNOWN_EVENT_FORMAT_VERSIONS: frozenset[int] + +class StateResolutionVersions: + """Enum to identify the state resolution algorithms.""" + + V1: int + """Room v1 state res""" + V2: int + """MSC1442 state res: room v2 and later""" + V2_1: int + """MSC4297 state res""" + +class RoomDisposition: + """Room disposition constants.""" + + STABLE: str + UNSTABLE: str + +class PushRuleRoomFlag: + """Enum for listing possible MSC3931 room version feature flags, for push rules.""" + + EXTENSIBLE_EVENTS: str + """MSC3932: Room version supports MSC1767 Extensible Events.""" + +class RoomVersion: + """An object which describes the unique attributes of a room version.""" + + identifier: str + """The identifier for this version.""" + disposition: str + """One of the RoomDisposition constants.""" + event_format: int + """One of the EventFormatVersions constants.""" + state_res: int + """One of the StateResolutionVersions constants.""" + enforce_key_validity: bool + special_case_aliases_auth: bool + """Before MSC2432, m.room.aliases had special auth rules and redaction rules.""" + strict_canonicaljson: bool + """Strictly enforce canonicaljson, do not allow: + * Integers outside the range of [-2^53 + 1, 2^53 - 1] + * Floats + * NaN, Infinity, -Infinity + """ + limit_notifications_power_levels: bool + """MSC2209: Check 'notifications' key while verifying + m.room.power_levels auth rules.""" + implicit_room_creator: bool + """MSC3820: No longer include the creator in m.room.create events (room version 11).""" + updated_redaction_rules: bool + """MSC3820: Apply updated redaction rules algorithm from room version 11.""" + restricted_join_rule: bool + """Support the 'restricted' join rule.""" + restricted_join_rule_fix: bool + """Support for the proper redaction rules for the restricted join rule. + This requires restricted_join_rule to be enabled.""" + knock_join_rule: bool + """Support the 'knock' join rule.""" + msc3389_relation_redactions: bool + """MSC3389: Protect relation information from redaction.""" + knock_restricted_join_rule: bool + """Support the 'knock_restricted' join rule.""" + enforce_int_power_levels: bool + """Enforce integer power levels.""" + msc3931_push_features: list[str] + """MSC3931: Adds a push rule condition for "room version feature flags", making + some push rules room version dependent. Note that adding a flag to this list + is not enough to mark it "supported": the push rule evaluator also needs to + support the flag. Unknown flags are ignored by the evaluator, making conditions + fail if used. Values from PushRuleRoomFlag.""" + msc3757_enabled: bool + """MSC3757: Restricting who can overwrite a state event.""" + msc4289_creator_power_enabled: bool + """MSC4289: Creator power enabled.""" + msc4291_room_ids_as_hashes: bool + """MSC4291: Room IDs as hashes of the create event.""" + strict_event_byte_limits_room_versions: bool + """Whether this room version strictly enforces event key size limits in bytes, + rather than in codepoints. + + If true, this room version uses stricter event size validation.""" + +class RoomVersions: + V1: RoomVersion + V2: RoomVersion + V3: RoomVersion + V4: RoomVersion + V5: RoomVersion + V6: RoomVersion + V7: RoomVersion + V8: RoomVersion + V9: RoomVersion + V10: RoomVersion + MSC1767v10: RoomVersion + MSC3389v10: RoomVersion + MSC3757v10: RoomVersion + V11: RoomVersion + MSC3757v11: RoomVersion + HydraV11: RoomVersion + V12: RoomVersion + +class KnownRoomVersionsMapping(Mapping[str, RoomVersion]): + def add_room_version(self, room_version: RoomVersion) -> None: ... + def __getitem__(self, key: str) -> RoomVersion: ... + def __len__(self) -> int: ... + def __iter__(self) -> Iterator[str]: ... + +KNOWN_ROOM_VERSIONS: KnownRoomVersionsMapping diff --git a/synapse/types/__init__.py b/synapse/types/__init__.py index fb1f1192b7..02889795bb 100644 --- a/synapse/types/__init__.py +++ b/synapse/types/__init__.py @@ -343,7 +343,7 @@ def is_valid(cls: type[DS], s: str) -> bool: # possible for invalid data to exist in room-state, etc. parse_and_validate_server_name(obj.domain) return True - except Exception: + except (SynapseError, ValueError): return False __repr__ = to_string @@ -355,6 +355,29 @@ class UserID(DomainSpecificString): SIGIL = "@" + @classmethod + def is_valid_strict(cls, s: str) -> bool: + """ + Parses the input string and attempts to ensure it is a valid and compliant user + ID according to https://spec.matrix.org/v1.17/appendices/#historical-user-ids. + + This should be used with care: there are existing non-compliant user IDs in the + wild with empty or non-ASCII localparts, which will be rejected by this method. + """ + if len(s.encode("utf-8")) > 255: + return False + try: + obj = cls.from_string(s) + if not is_compliant_user_id_localpart(obj.localpart): + return False + # Apply additional validation to the domain. This is only done + # during is_valid (and not part of from_string) since it is + # possible for invalid data to exist in room-state, etc. + parse_and_validate_server_name(obj.domain) + return True + except (SynapseError, ValueError): + return False + @attr.s(slots=True, frozen=True, repr=False) class RoomAlias(DomainSpecificString): @@ -453,12 +476,17 @@ class EventID(DomainSpecificString): def contains_invalid_mxid_characters(localpart: str) -> bool: - """Check for characters not allowed in an mxid or groupid localpart + """ + Check for characters not allowed in a modern user ID localpart. + + This is primarily used for new registrations and MUST NOT be used to validate + existing user IDs, as there are real users whose user IDs don't follow this + character set. + + See https://spec.matrix.org/v1.17/appendices/#user-identifiers Args: localpart: the localpart to be checked - use_extended_character_set: True to use the extended allowed characters - from MSC4009. Returns: True if there are any naughty characters @@ -466,6 +494,28 @@ def contains_invalid_mxid_characters(localpart: str) -> bool: return any(c not in MXID_LOCALPART_ALLOWED_CHARACTERS for c in localpart) +def is_compliant_user_id_localpart(localpart: str) -> bool: + """ + Validates that the given user ID localpart is within the "compliant" range, + i.e. not empty and all characters are between U+0021 and U+007E inclusive. + See https://spec.matrix.org/v1.17/appendices/#historical-user-ids + + To check if a localpart is non-historical, use contains_invalid_mxid_characters instead. + + This should be used with care: there are existing non-compliant user IDs in the + wild with empty or non-ASCII localparts, which will be rejected by this method. + + Args: + localpart: the localpart to be checked + + Returns: + True if the localpart is compliant, False otherwise + """ + if not localpart: + return False + return all(0x21 <= ord(c) <= 0x7E for c in localpart) + + UPPER_CASE_PATTERN = re.compile(b"[A-Z_]") # the following is a pattern which matches '=', and bytes which are not allowed in a mxid diff --git a/synapse/types/handlers/policy_server.py b/synapse/types/handlers/policy_server.py deleted file mode 100644 index bfc09dabf4..0000000000 --- a/synapse/types/handlers/policy_server.py +++ /dev/null @@ -1,16 +0,0 @@ -# -# This file is licensed under the Affero General Public License (AGPL) version 3. -# -# Copyright (C) 2025 New Vector, Ltd -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# See the GNU Affero General Public License for more details: -# . -# - -RECOMMENDATION_OK = "ok" -RECOMMENDATION_SPAM = "spam" diff --git a/synapse/types/storage/__init__.py b/synapse/types/storage/__init__.py index b01653246a..992c36caba 100644 --- a/synapse/types/storage/__init__.py +++ b/synapse/types/storage/__init__.py @@ -64,3 +64,5 @@ class _BackgroundUpdates: ) FIXUP_MAX_DEPTH_CAP = "fixup_max_depth_cap" + + REDACTIONS_RECHECK_BG_UPDATE = "redactions_recheck" diff --git a/tests/events/test_utils.py b/tests/events/test_utils.py index 9ea015e138..af44b5dec1 100644 --- a/tests/events/test_utils.py +++ b/tests/events/test_utils.py @@ -20,9 +20,8 @@ # import unittest as stdlib_unittest -from typing import Any, Mapping +from typing import TYPE_CHECKING, Any, Mapping -import attr from parameterized import parameterized from synapse.api.constants import EventContentFields @@ -38,11 +37,15 @@ make_config_for_admin, maybe_upsert_event_field, prune_event, - serialize_event, ) from synapse.types import JsonDict, create_requester from synapse.util.frozenutils import freeze +from tests.unittest import HomeserverTestCase + +if TYPE_CHECKING: + from synapse.server import HomeServer + def MockEvent(**kwargs: Any) -> EventBase: if "event_id" not in kwargs: @@ -553,11 +556,6 @@ def test_relations(self) -> None: room_version=RoomVersions.V10, ) - # Create a new room version. - msc3389_room_ver = attr.evolve( - RoomVersions.V10, msc3389_relation_redactions=True - ) - self.run_test( { "type": "m.room.message", @@ -581,7 +579,7 @@ def test_relations(self) -> None: "signatures": {}, "unsigned": {}, }, - room_version=msc3389_room_ver, + room_version=RoomVersions.MSC3389v10, ) # If the field is not an object, redact it. @@ -599,7 +597,7 @@ def test_relations(self) -> None: "signatures": {}, "unsigned": {}, }, - room_version=msc3389_room_ver, + room_version=RoomVersions.MSC3389v10, ) # If the m.relates_to property would be empty, redact it. @@ -614,7 +612,7 @@ def test_relations(self) -> None: "signatures": {}, "unsigned": {}, }, - room_version=msc3389_room_ver, + room_version=RoomVersions.MSC3389v10, ) @@ -644,19 +642,27 @@ def test_unsigned_is_copied(self) -> None: self.assertEqual(cloned.internal_metadata.txn_id, "txn") -class SerializeEventTestCase(stdlib_unittest.TestCase): +class SerializeEventTestCase(HomeserverTestCase): + def prepare(self, reactor: Any, clock: Any, hs: "HomeServer") -> None: + self._event_serializer = hs.get_event_client_serializer() + def serialize( self, ev: EventBase, fields: list[str] | None, include_admin_metadata: bool = False, + redaction_map: Mapping[str, EventBase] | None = None, ) -> JsonDict: - return serialize_event( - ev, - 1479807801915, - config=SerializeEventConfig( - only_event_fields=fields, include_admin_metadata=include_admin_metadata - ), + return self.get_success( + self._event_serializer.serialize_event( + ev, + 1479807801915, + config=SerializeEventConfig( + only_event_fields=fields, + include_admin_metadata=include_admin_metadata, + ), + redaction_map=redaction_map, + ) ) def test_event_fields_works_with_keys(self) -> None: @@ -770,9 +776,8 @@ def test_event_fields_all_fields_if_empty(self) -> None: def test_event_fields_fail_if_fields_not_str(self) -> None: with self.assertRaises(TypeError): - self.serialize( - MockEvent(room_id="!foo:bar", content={"foo": "bar"}), - ["room_id", 4], # type: ignore[list-item] + SerializeEventConfig( + only_event_fields=["room_id", 4], # type: ignore[list-item] ) def test_default_serialize_config_excludes_admin_metadata(self) -> None: @@ -873,6 +878,52 @@ def test_make_serialize_config_for_admin_retains_other_fields(self) -> None: ) self.assertTrue(admin_config.include_admin_metadata) + def test_redacted_because_is_filtered_out(self) -> None: + """If an event's unsigned dict has a `redacted_by` field, then the + `redacted_because` should be filtered out if not specified in + `only_event_fields`.""" + + redaction_id = "$redaction_event_id" + + event = MockEvent( + type="foo", + event_id="test", + room_id="!foo:bar", + content={"foo": "bar"}, + ) + event.internal_metadata.redacted_by = redaction_id + + redaction_event = MockEvent( + type="m.room.redaction", + event_id=redaction_id, + content={"redacts": "test"}, + ) + + self.assertEqual( + self.serialize( + event, + ["content.foo"], + redaction_map={redaction_id: redaction_event}, + ), + { + "content": {"foo": "bar"}, + }, + ) + + self.assertEqual( + self.serialize( + event, + ["content.foo", "unsigned.redacted_because"], + redaction_map={redaction_id: redaction_event}, + ), + { + "content": {"foo": "bar"}, + "unsigned": { + "redacted_because": self.serialize(redaction_event, fields=None), + }, + }, + ) + class CopyPowerLevelsContentTestCase(stdlib_unittest.TestCase): def setUp(self) -> None: diff --git a/tests/handlers/test_oidc.py b/tests/handlers/test_oidc.py index 4583afb625..62b84c77a4 100644 --- a/tests/handlers/test_oidc.py +++ b/tests/handlers/test_oidc.py @@ -19,7 +19,7 @@ # # import os -from typing import Any, Awaitable, ContextManager +from typing import Any, ContextManager from unittest.mock import ANY, AsyncMock, Mock, patch from urllib.parse import parse_qs, urlparse @@ -29,6 +29,7 @@ from synapse.handlers.sso import MappingException from synapse.http.site import SynapseRequest +from synapse.logging.context import make_deferred_yieldable from synapse.server import HomeServer from synapse.types import JsonDict, UserID from synapse.util.clock import Clock @@ -44,7 +45,7 @@ from authlib.oidc.core import UserInfo from authlib.oidc.discovery import OpenIDProviderMetadata - from synapse.handlers.oidc import Token, UserAttributeDict + from synapse.handlers.oidc import OidcMetadataError, Token, UserAttributeDict HAS_OIDC = True except ImportError: @@ -264,6 +265,29 @@ def test_discovery(self) -> None: self.get_success(self.provider.load_metadata()) self.fake_server.get_metadata_handler.assert_not_called() + @override_config({"oidc_config": {**DEFAULT_CONFIG, "discover": True}}) + def test_startup_metadata_preload_failure_is_logged(self) -> None: + """ + The metadata preload on startup causes logs to be emitted, but no + exceptions. + """ + with self.fake_server.buggy_endpoint(metadata=True): + with self.assertLogs("synapse.handlers.oidc", level="CRITICAL") as logs: + self.get_success( + make_deferred_yieldable(self.handler.preload_metadata()) + ) + + self.assertIn( + "CRITICAL:synapse.handlers.oidc:Error while preloading OIDC provider 'oidc'. Login may be broken!", + [line for record in logs.output for line in record.split("\n")], + ) + + # Even though the preload failed, if we try to load the metadata later, + # the attempt still goes through. + self.fake_server.get_metadata_handler.assert_not_called() + self.get_success(self.provider.load_metadata()) + self.fake_server.get_metadata_handler.assert_called_once() + @override_config({"oidc_config": {**EXPLICIT_ENDPOINT_CONFIG, "discover": True}}) def test_discovery_with_explicit_config(self) -> None: """ @@ -351,49 +375,53 @@ def test_validate_config(self) -> None: """Provider metadatas are extensively validated.""" h = self.provider - def force_load_metadata() -> Awaitable[None]: + def force_load_metadata() -> None: async def force_load() -> "OpenIDProviderMetadata": return await h.load_metadata(force=True) - return get_awaitable_result(force_load()) + get_awaitable_result(force_load()) # Default test config does not throw force_load_metadata() with self.metadata_edit({"issuer": None}): - self.assertRaisesRegex(ValueError, "issuer", force_load_metadata) + self.assertRaisesRegex(OidcMetadataError, "issuer", force_load_metadata) with self.metadata_edit({"issuer": "http://insecure/"}): - self.assertRaisesRegex(ValueError, "issuer", force_load_metadata) + self.assertRaisesRegex(OidcMetadataError, "issuer", force_load_metadata) with self.metadata_edit({"issuer": "https://invalid/?because=query"}): - self.assertRaisesRegex(ValueError, "issuer", force_load_metadata) + self.assertRaisesRegex(OidcMetadataError, "issuer", force_load_metadata) with self.metadata_edit({"authorization_endpoint": None}): self.assertRaisesRegex( - ValueError, "authorization_endpoint", force_load_metadata + OidcMetadataError, "authorization_endpoint", force_load_metadata ) with self.metadata_edit({"authorization_endpoint": "http://insecure/auth"}): self.assertRaisesRegex( - ValueError, "authorization_endpoint", force_load_metadata + OidcMetadataError, "authorization_endpoint", force_load_metadata ) with self.metadata_edit({"token_endpoint": None}): - self.assertRaisesRegex(ValueError, "token_endpoint", force_load_metadata) + self.assertRaisesRegex( + OidcMetadataError, "token_endpoint", force_load_metadata + ) with self.metadata_edit({"token_endpoint": "http://insecure/token"}): - self.assertRaisesRegex(ValueError, "token_endpoint", force_load_metadata) + self.assertRaisesRegex( + OidcMetadataError, "token_endpoint", force_load_metadata + ) with self.metadata_edit({"jwks_uri": None}): - self.assertRaisesRegex(ValueError, "jwks_uri", force_load_metadata) + self.assertRaisesRegex(OidcMetadataError, "jwks_uri", force_load_metadata) with self.metadata_edit({"jwks_uri": "http://insecure/jwks.json"}): - self.assertRaisesRegex(ValueError, "jwks_uri", force_load_metadata) + self.assertRaisesRegex(OidcMetadataError, "jwks_uri", force_load_metadata) with self.metadata_edit({"response_types_supported": ["id_token"]}): self.assertRaisesRegex( - ValueError, "response_types_supported", force_load_metadata + OidcMetadataError, "response_types_supported", force_load_metadata ) with self.metadata_edit( @@ -406,7 +434,7 @@ async def force_load() -> "OpenIDProviderMetadata": {"token_endpoint_auth_methods_supported": ["client_secret_post"]} ): self.assertRaisesRegex( - ValueError, + OidcMetadataError, "token_endpoint_auth_methods_supported", force_load_metadata, ) @@ -423,7 +451,9 @@ async def force_load() -> "OpenIDProviderMetadata": h._scopes = [] self.assertTrue(h._uses_userinfo) with self.metadata_edit({"userinfo_endpoint": None}): - self.assertRaisesRegex(ValueError, "userinfo_endpoint", force_load_metadata) + self.assertRaisesRegex( + OidcMetadataError, "userinfo_endpoint", force_load_metadata + ) with self.metadata_edit({"jwks_uri": None}): # Shouldn't raise with a valid userinfo, even without jwks diff --git a/tests/handlers/test_room_policy.py b/tests/handlers/test_room_policy.py index ff212ab06e..9adf0e38c2 100644 --- a/tests/handlers/test_room_policy.py +++ b/tests/handlers/test_room_policy.py @@ -19,7 +19,8 @@ from twisted.internet.testing import MemoryReactor -from synapse.api.errors import SynapseError +from synapse.api.constants import EventTypes +from synapse.api.errors import HttpResponseException, SynapseError from synapse.crypto.event_signing import compute_event_signature from synapse.events import EventBase, make_event_from_dict from synapse.handlers.room_policy import POLICY_SERVER_KEY_ID @@ -27,7 +28,6 @@ from synapse.rest.client import filter, login, room, sync from synapse.server import HomeServer from synapse.types import JsonDict, UserID -from synapse.types.handlers.policy_server import RECOMMENDATION_OK, RECOMMENDATION_SPAM from synapse.util.clock import Clock from tests import unittest @@ -49,13 +49,9 @@ def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: # mock out the federation transport client self.mock_federation_transport_client = mock.Mock( spec=[ - "get_policy_recommendation_for_pdu", "ask_policy_server_to_sign_event", ] ) - self.mock_federation_transport_client.get_policy_recommendation_for_pdu = ( - mock.AsyncMock() - ) self.mock_federation_transport_client.ask_policy_server_to_sign_event = ( mock.AsyncMock() ) @@ -106,25 +102,6 @@ def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: }, ) - # Prepare the policy server mock to decide spam vs not spam on those events - self.call_count = 0 - - async def get_policy_recommendation_for_pdu( - destination: str, - pdu: EventBase, - timeout: int | None = None, - ) -> JsonDict: - self.call_count += 1 - self.assertEqual(destination, self.OTHER_SERVER_NAME) - if pdu.event_id == self.spammy_event.event_id: - return {"recommendation": RECOMMENDATION_SPAM} - elif pdu.event_id == self.not_spammy_event.event_id: - return {"recommendation": RECOMMENDATION_OK} - else: - self.fail("Unexpected event ID") - - self.mock_federation_transport_client.get_policy_recommendation_for_pdu.side_effect = get_policy_recommendation_for_pdu - # Mock policy server actions on signing events async def policy_server_signs_event( destination: str, pdu: EventBase, timeout: int | None = None @@ -157,7 +134,9 @@ async def policy_server_refuses_to_sign_event( async def policy_server_event_sign_error( destination: str, pdu: EventBase, timeout: int | None = None ) -> JsonDict | None: - return None + raise HttpResponseException( + 500, "Internal Server Error", b'{"errcode": "M_UNKNOWN"}' + ) self.policy_server_signs_event = policy_server_signs_event self.policy_server_refuses_to_sign_event = policy_server_refuses_to_sign_event @@ -174,14 +153,16 @@ def _add_policy_server_to_room(self, public_key: str | None = None) -> None: self.hs, self.room_id, policy_user_id, "join" ) ) - content = { + content: JsonDict = { "via": self.OTHER_SERVER_NAME, } if public_key is not None: - content["public_key"] = public_key + content["public_keys"] = { + "ed25519": public_key, + } self.helper.send_state( self.room_id, - "org.matrix.msc4284.policy", + EventTypes.RoomPolicy, content, tok=self.creator_token, state_key="", @@ -192,12 +173,11 @@ def test_no_policy_event_set(self) -> None: # case where a room doesn't use a policy server. ok = self.get_success(self.handler.is_event_allowed(self.spammy_event)) self.assertEqual(ok, True) - self.assertEqual(self.call_count, 0) def test_empty_policy_event_set(self) -> None: self.helper.send_state( self.room_id, - "org.matrix.msc4284.policy", + EventTypes.RoomPolicy, { # empty content (no `via`) }, @@ -207,12 +187,11 @@ def test_empty_policy_event_set(self) -> None: ok = self.get_success(self.handler.is_event_allowed(self.spammy_event)) self.assertEqual(ok, True) - self.assertEqual(self.call_count, 0) def test_nonstring_policy_event_set(self) -> None: self.helper.send_state( self.room_id, - "org.matrix.msc4284.policy", + EventTypes.RoomPolicy, { "via": 42, # should be a server name }, @@ -222,12 +201,11 @@ def test_nonstring_policy_event_set(self) -> None: ok = self.get_success(self.handler.is_event_allowed(self.spammy_event)) self.assertEqual(ok, True) - self.assertEqual(self.call_count, 0) def test_self_policy_event_set(self) -> None: self.helper.send_state( self.room_id, - "org.matrix.msc4284.policy", + EventTypes.RoomPolicy, { # We ignore events when the policy server is ourselves (for now?) "via": (UserID.from_string(self.creator)).domain, @@ -238,12 +216,11 @@ def test_self_policy_event_set(self) -> None: ok = self.get_success(self.handler.is_event_allowed(self.spammy_event)) self.assertEqual(ok, True) - self.assertEqual(self.call_count, 0) def test_invalid_server_policy_event_set(self) -> None: self.helper.send_state( self.room_id, - "org.matrix.msc4284.policy", + EventTypes.RoomPolicy, { "via": "|this| is *not* a (valid) server name.com", }, @@ -253,12 +230,11 @@ def test_invalid_server_policy_event_set(self) -> None: ok = self.get_success(self.handler.is_event_allowed(self.spammy_event)) self.assertEqual(ok, True) - self.assertEqual(self.call_count, 0) def test_not_in_room_policy_event_set(self) -> None: self.helper.send_state( self.room_id, - "org.matrix.msc4284.policy", + EventTypes.RoomPolicy, { "via": f"x.{self.OTHER_SERVER_NAME}", }, @@ -268,14 +244,30 @@ def test_not_in_room_policy_event_set(self) -> None: ok = self.get_success(self.handler.is_event_allowed(self.spammy_event)) self.assertEqual(ok, True) - self.assertEqual(self.call_count, 0) + + def test_missing_public_key_event_set(self) -> None: + """ + Tests that a missing public key in the `m.room.policy` state event (an invalid + configuration) is treated as though there is no policy server configured, thus + allowing all events. + """ + self._add_policy_server_to_room() # no public_key + + ok = self.get_success(self.handler.is_event_allowed(self.spammy_event)) + self.assertEqual(ok, True) def test_spammy_event_is_spam(self) -> None: - self._add_policy_server_to_room() + verify_key_str = encode_verify_key_base64(get_verify_key(self.signing_key)) + self._add_policy_server_to_room(public_key=verify_key_str) + + # Explicitly configure the policy server mock to refuse to sign the event. + self.mock_federation_transport_client.ask_policy_server_to_sign_event.return_value = False ok = self.get_success(self.handler.is_event_allowed(self.spammy_event)) self.assertEqual(ok, False) - self.assertEqual(self.call_count, 1) + + # Ensure we actually contacted the policy server once for this event. + self.mock_federation_transport_client.ask_policy_server_to_sign_event.assert_awaited_once() def test_signed_event_is_not_spam(self) -> None: verify_key_str = encode_verify_key_base64(get_verify_key(self.signing_key)) @@ -306,8 +298,6 @@ def test_signed_event_is_not_spam(self) -> None: ok = self.get_success(self.handler.is_event_allowed(event)) self.assertEqual(ok, True) - # Make sure we did not make an HTTP hit to get_policy_recommendation_for_pdu - self.assertEqual(self.call_count, 0) def test_ask_policy_server_to_sign_event_ok(self) -> None: verify_key_str = encode_verify_key_base64(get_verify_key(self.signing_key)) @@ -348,8 +338,15 @@ def test_ask_policy_server_to_sign_event_refuses(self) -> None: }, ) self.mock_federation_transport_client.ask_policy_server_to_sign_event.side_effect = self.policy_server_refuses_to_sign_event - self.get_success( - self.handler.ask_policy_server_to_sign_event(event, verify=True) + fail = self.get_failure( + self.handler.ask_policy_server_to_sign_event(event, verify=True), + SynapseError, + ) + self.assertIsInstance(fail.value, SynapseError) + self.assertEqual(fail.value.code, 403) + self.assertEqual( + fail.value.msg, + "This event has been rejected as probable spam by the policy server", ) self.assertEqual(len(event.signatures), 0) @@ -370,9 +367,12 @@ def test_ask_policy_server_to_sign_event_cannot_reach(self) -> None: }, ) self.mock_federation_transport_client.ask_policy_server_to_sign_event.side_effect = self.policy_server_event_sign_error - self.get_success( - self.handler.ask_policy_server_to_sign_event(event, verify=True) + fail = self.get_failure( + self.handler.ask_policy_server_to_sign_event(event, verify=True), + SynapseError, ) + self.assertIsInstance(fail.value, SynapseError) + self.assertEqual(fail.value.code, 500) self.assertEqual(len(event.signatures), 0) def test_ask_policy_server_to_sign_event_wrong_sig(self) -> None: diff --git a/tests/replication/storage/test_events.py b/tests/replication/storage/test_events.py index 28bfb8b8ea..b7b94482ef 100644 --- a/tests/replication/storage/test_events.py +++ b/tests/replication/storage/test_events.py @@ -101,11 +101,10 @@ def test_redactions(self) -> None: msg_dict = msg.get_dict() msg_dict["content"] = {} - msg_dict["unsigned"]["redacted_by"] = redaction.event_id - msg_dict["unsigned"]["redacted_because"] = redaction redacted = make_event_from_dict( msg_dict, internal_metadata_dict=msg.internal_metadata.get_dict() ) + redacted.internal_metadata.redacted_by = redaction.event_id self.check( "get_event", [msg.event_id], redacted, asserter=self.assertEventsEqual ) @@ -125,11 +124,10 @@ def test_backfilled_redactions(self) -> None: msg_dict = msg.get_dict() msg_dict["content"] = {} - msg_dict["unsigned"]["redacted_by"] = redaction.event_id - msg_dict["unsigned"]["redacted_because"] = redaction redacted = make_event_from_dict( msg_dict, internal_metadata_dict=msg.internal_metadata.get_dict() ) + redacted.internal_metadata.redacted_by = redaction.event_id self.check( "get_event", [msg.event_id], redacted, asserter=self.assertEventsEqual ) diff --git a/tests/rest/client/test_msc4388_rendezvous.py b/tests/rest/client/test_msc4388_rendezvous.py index f16bb3f344..913a41dedb 100644 --- a/tests/rest/client/test_msc4388_rendezvous.py +++ b/tests/rest/client/test_msc4388_rendezvous.py @@ -131,6 +131,10 @@ def test_disabled(self) -> None: } ) def test_off(self) -> None: + # Discovery endpoint should return 404 + channel = self.make_request("GET", rz_endpoint, {}, access_token=None) + self.assertEqual(channel.code, 404) + # Create session should also fail channel = self.make_request("POST", rz_endpoint, {}, access_token=None) self.assertEqual(channel.code, 404) @@ -143,7 +147,7 @@ def test_off(self) -> None: "endpoint": "https://issuer", }, "experimental_features": { - "msc4388_mode": "public", + "msc4388_mode": "open", }, } ) @@ -156,6 +160,11 @@ def test_rendezvous_public(self) -> None: - Deleting the data - Sequence token handling """ + # Discovery should return 200 + channel = self.make_request("GET", rz_endpoint, {}, access_token=None) + self.assertEqual(channel.code, 200) + self.assertTrue(channel.json_body.get("create_available")) + # We can post arbitrary data to the endpoint channel = self.make_request( "POST", @@ -268,7 +277,11 @@ def test_rendezvous_requires_authentication(self) -> None: self.setup_mock_oauth() alice_token = self.register_oauth_user("alice", "device1") - # This should fail without authentication: + # Discovery should fail due to lack of authentication + channel = self.make_request("GET", rz_endpoint, {}, access_token=None) + self.assertEqual(channel.code, 401) + + # Creating a session should fail without authentication: channel = self.make_request( "POST", rz_endpoint, @@ -277,6 +290,11 @@ def test_rendezvous_requires_authentication(self) -> None: ) self.assertEqual(channel.code, 401) + # Discovery should succeed with authentication + channel = self.make_request("GET", rz_endpoint, {}, access_token=alice_token) + self.assertEqual(channel.code, 200) + self.assertTrue(channel.json_body.get("create_available")) + # This should work as we are now authenticated channel = self.make_request( "POST", @@ -288,7 +306,7 @@ def test_rendezvous_requires_authentication(self) -> None: rendezvous_id = channel.json_body["id"] sequence_token = channel.json_body["sequence_token"] expires_in_ms = channel.json_body["expires_in_ms"] - self.assertGreater(expires_in_ms, 0) + self.assertEqual(expires_in_ms, 120000) session_endpoint = rz_endpoint + f"/{rendezvous_id}" @@ -302,7 +320,7 @@ def test_rendezvous_requires_authentication(self) -> None: self.assertEqual(channel.code, 200) self.assertEqual(channel.json_body["data"], "foo=bar") self.assertEqual(channel.json_body["sequence_token"], sequence_token) - self.assertEqual(channel.json_body["expires_in_ms"], expires_in_ms) + self.assertEqual(channel.json_body["expires_in_ms"], expires_in_ms - 100) # We can update the data without authentication channel = self.make_request( @@ -325,7 +343,7 @@ def test_rendezvous_requires_authentication(self) -> None: self.assertEqual(channel.code, 200) self.assertEqual(channel.json_body["data"], "foo=baz") self.assertEqual(channel.json_body["sequence_token"], new_sequence_token) - self.assertEqual(channel.json_body["expires_in_ms"], expires_in_ms - 200) + self.assertEqual(channel.json_body["expires_in_ms"], expires_in_ms - 300) # We can delete the data without authentication channel = self.make_request( @@ -355,7 +373,7 @@ def test_rendezvous_requires_authentication(self) -> None: "endpoint": "https://issuer", }, "experimental_features": { - "msc4388_mode": "public", + "msc4388_mode": "open", }, } ) @@ -402,7 +420,7 @@ def test_expiration(self) -> None: "endpoint": "https://issuer", }, "experimental_features": { - "msc4388_mode": "public", + "msc4388_mode": "open", }, } ) @@ -473,7 +491,7 @@ def test_capacity(self) -> None: "endpoint": "https://issuer", }, "experimental_features": { - "msc4388_mode": "public", + "msc4388_mode": "open", }, } ) @@ -531,7 +549,7 @@ def test_hard_capacity(self) -> None: "endpoint": "https://issuer", }, "experimental_features": { - "msc4388_mode": "public", + "msc4388_mode": "open", }, } ) @@ -585,7 +603,7 @@ def test_data_type(self) -> None: "endpoint": "https://issuer", }, "experimental_features": { - "msc4388_mode": "public", + "msc4388_mode": "open", }, } ) @@ -648,7 +666,7 @@ def test_max_length(self) -> None: "endpoint": "https://issuer", }, "experimental_features": { - "msc4388_mode": "public", + "msc4388_mode": "open", }, } ) @@ -698,7 +716,7 @@ def test_rendezvous_rejects_unsafe_get_requests( "endpoint": "https://issuer", }, "experimental_features": { - "msc4388_mode": "public", + "msc4388_mode": "open", }, } ) diff --git a/tests/rest/client/test_mutual_rooms.py b/tests/rest/client/test_mutual_rooms.py index f78c67fcd9..cbaca17cdd 100644 --- a/tests/rest/client/test_mutual_rooms.py +++ b/tests/rest/client/test_mutual_rooms.py @@ -43,12 +43,6 @@ class UserMutualRoomsTest(unittest.HomeserverTestCase): mutual_rooms.register_servlets, ] - def default_config(self) -> dict: - config = super().default_config() - experimental = config.setdefault("experimental_features", {}) - experimental.setdefault("msc2666_enabled", True) - return config - def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: config = self.default_config() return self.setup_test_homeserver(config=config) @@ -62,27 +56,12 @@ def _get_mutual_rooms( ) -> FakeChannel: return self.make_request( "GET", - "/_matrix/client/unstable/uk.half-shot.msc2666/user/mutual_rooms" + "/_matrix/client/v1/mutual_rooms" f"?user_id={quote(other_user)}" + (f"&from={quote(since_token)}" if since_token else ""), access_token=token, ) - @unittest.override_config({"experimental_features": {"msc2666_enabled": False}}) - def test_mutual_rooms_no_experimental_flag(self) -> None: - """ - The endpoint should 404 if the experimental flag is not enabled. - """ - # Register a user. - u1 = self.register_user("user1", "pass") - u1_token = self.login(u1, "pass") - - # Check that we're unable to query the endpoint due to the endpoint - # being unrecognised. - channel = self._get_mutual_rooms(u1_token, "@not-used:test") - self.assertEqual(404, channel.code, channel.result) - self.assertEqual("M_UNRECOGNIZED", channel.json_body["errcode"], channel.result) - def test_shared_room_list_public(self) -> None: """ A room should show up in the shared list of rooms between two users @@ -129,6 +108,7 @@ def _check_mutual_rooms_with( channel = self._get_mutual_rooms(u1_token, u2) self.assertEqual(200, channel.code, channel.result) self.assertEqual(len(channel.json_body["joined"]), 1) + self.assertEqual(channel.json_body["count"], 1) self.assertEqual(channel.json_body["joined"][0], room_id_one) # Create another room and invite user2 to it @@ -142,6 +122,7 @@ def _check_mutual_rooms_with( channel = self._get_mutual_rooms(u1_token, u2) self.assertEqual(200, channel.code, channel.result) self.assertEqual(len(channel.json_body["joined"]), 2) + self.assertEqual(channel.json_body["count"], 2) for room_id_id in channel.json_body["joined"]: self.assertIn(room_id_id, [room_id_one, room_id_two]) @@ -167,11 +148,13 @@ def test_shared_room_list_pagination_two_pages(self) -> None: channel = self._get_mutual_rooms(u1_token, u2) self.assertEqual(200, channel.code, channel.result) self.assertEqual(channel.json_body["joined"], room_ids[0:10]) + self.assertEqual(channel.json_body["count"], 15) self.assertIn("next_batch", channel.json_body) channel = self._get_mutual_rooms(u1_token, u2, channel.json_body["next_batch"]) self.assertEqual(200, channel.code, channel.result) self.assertEqual(channel.json_body["joined"], room_ids[10:20]) + self.assertEqual(channel.json_body["count"], 15) self.assertNotIn("next_batch", channel.json_body) def test_shared_room_list_pagination_one_page(self) -> None: @@ -180,6 +163,7 @@ def test_shared_room_list_pagination_one_page(self) -> None: channel = self._get_mutual_rooms(u1_token, u2) self.assertEqual(200, channel.code, channel.result) self.assertEqual(channel.json_body["joined"], room_ids) + self.assertEqual(channel.json_body["count"], 10) self.assertNotIn("next_batch", channel.json_body) def test_shared_room_list_pagination_invalid_token(self) -> None: @@ -209,6 +193,7 @@ def test_shared_room_list_after_leave(self) -> None: channel = self._get_mutual_rooms(u1_token, u2) self.assertEqual(200, channel.code, channel.result) self.assertEqual(len(channel.json_body["joined"]), 1) + self.assertEqual(channel.json_body["count"], 1) self.assertEqual(channel.json_body["joined"][0], room) self.helper.leave(room, user=u1, tok=u1_token) @@ -217,11 +202,13 @@ def test_shared_room_list_after_leave(self) -> None: channel = self._get_mutual_rooms(u1_token, u2) self.assertEqual(200, channel.code, channel.result) self.assertEqual(len(channel.json_body["joined"]), 0) + self.assertEqual(channel.json_body["count"], 0) # Check user2's view of shared rooms with user1 channel = self._get_mutual_rooms(u2_token, u1) self.assertEqual(200, channel.code, channel.result) self.assertEqual(len(channel.json_body["joined"]), 0) + self.assertEqual(channel.json_body["count"], 0) def test_shared_room_list_nonexistent_user(self) -> None: u1 = self.register_user("user1", "pass") @@ -232,4 +219,27 @@ def test_shared_room_list_nonexistent_user(self) -> None: channel = self._get_mutual_rooms(u1_token, "@meow:example.com") self.assertEqual(200, channel.code, channel.result) self.assertEqual(len(channel.json_body["joined"]), 0) + self.assertEqual(channel.json_body["count"], 0) self.assertNotIn("next_batch", channel.json_body) + + def test_shared_room_list_invalid_user(self) -> None: + u1 = self.register_user("user1", "pass") + u1_token = self.login(u1, "pass") + + channel = self._get_mutual_rooms(u1_token, "@:example.com") + self.assertEqual(400, channel.code, channel.result) + self.assertEqual( + "M_INVALID_PARAM", channel.json_body["errcode"], channel.result + ) + + channel = self._get_mutual_rooms(u1_token, "@" + "a" * 255 + ":example.com") + self.assertEqual(400, channel.code, channel.result) + self.assertEqual( + "M_INVALID_PARAM", channel.json_body["errcode"], channel.result + ) + + channel = self._get_mutual_rooms(u1_token, "@🐈️:example.com") + self.assertEqual(400, channel.code, channel.result) + self.assertEqual( + "M_INVALID_PARAM", channel.json_body["errcode"], channel.result + ) diff --git a/tests/rest/client/test_register.py b/tests/rest/client/test_register.py index 2c0396a3de..f66c56a6b1 100644 --- a/tests/rest/client/test_register.py +++ b/tests/rest/client/test_register.py @@ -22,7 +22,7 @@ import datetime import importlib.resources as importlib_resources import os -from typing import Any +from typing import Any, cast from unittest.mock import AsyncMock from twisted.internet.testing import MemoryReactor @@ -66,6 +66,17 @@ def make_homeserver( hs.get_send_email_handler()._sendmail = AsyncMock() return hs + def _get_sendmail_mock(self) -> AsyncMock: + """ + Cast the homeserver's `_sendmail` object as an `AsyncMock`. + + `_sendmail` is an `AsyncMock` (see `make_homeserver`) but this type + information doesn't make it through the test harness. Thus we need to + cast the object again. + """ + sendmail = self.hs.get_send_email_handler()._sendmail + return cast(AsyncMock, sendmail) + def test_POST_appservice_registration_valid(self) -> None: user_id = "@as_user_kermit:test" as_token = "i_am_an_app_service" @@ -747,6 +758,33 @@ def test_request_token_existing_email_inhibit_error(self) -> None: self.assertIsNotNone(channel.json_body.get("sid")) + @unittest.override_config( + { + "public_baseurl": "https://test_server", + "email": { + "smtp_host": "mail_server", + "smtp_port": 2525, + "notif_from": "sender@host", + }, + } + ) + def test_request_token_allowed_when_email_flow_is_advertised(self) -> None: + sendmail = self._get_sendmail_mock() + sendmail.reset_mock() + + channel = self.make_request( + "POST", + b"register/email/requestToken", + { + "client_secret": "foobar", + "email": "test@example.com", + "send_attempt": 1, + }, + ) + self.assertEqual(200, channel.code, channel.result) + self.assertIsNotNone(channel.json_body.get("sid")) + sendmail.assert_awaited_once() + @unittest.override_config( { "public_baseurl": "https://test_server", diff --git a/tests/rest/client/test_rooms.py b/tests/rest/client/test_rooms.py index f85c9939ce..221121007d 100644 --- a/tests/rest/client/test_rooms.py +++ b/tests/rest/client/test_rooms.py @@ -4746,10 +4746,10 @@ def test_banning_remote_member_with_flag_redacts_their_events(self) -> None: original = self.get_success(self.store.get_event(message.event_id)) if not original: self.fail("Expected to find remote message in DB") - redacted_because = original.unsigned.get("redacted_because") - if not redacted_because: - self.fail("Did not find redacted_because field") - self.assertEqual(redacted_because.event_id, ban_event_id) + redacted_by = original.internal_metadata.redacted_by + if not redacted_by: + self.fail("Did not find redacted_by field") + self.assertEqual(redacted_by, ban_event_id) def test_unbanning_remote_user_stops_redaction_action(self) -> None: bad_user = "@remote_bad_user:" + self.OTHER_SERVER_NAME @@ -5111,7 +5111,7 @@ def test_kicking_remote_member_with_flag_redacts_their_events(self) -> None: original = self.get_success(self.store.get_event(message.event_id)) if not original: self.fail("Expected to find remote message in DB") - self.assertEqual(original.unsigned["redacted_by"], ban_event_id) + self.assertEqual(original.internal_metadata.redacted_by, ban_event_id) def test_rejoining_kicked_remote_user_stops_redaction_action(self) -> None: bad_user = "@remote_bad_user:" + self.OTHER_SERVER_NAME diff --git a/tests/server.py b/tests/server.py index 001d11e5e8..1b6e8633fc 100644 --- a/tests/server.py +++ b/tests/server.py @@ -1358,6 +1358,16 @@ def thread_pool() -> threadpool.ThreadPool: hs.get_media_sender_thread_pool = thread_pool # type: ignore[method-assign] + # Load the OIDC provider metadatas, if OIDC is enabled. + # This matches `start` in synapse/app/_base.py + # + # TODO: Extract common startup logic somewhere cleaner + if hs.config.oidc.oidc_enabled: + oidc = hs.get_oidc_handler() + # Preload the provider metadata. + # This will spawn fire-and-forget background processes. + oidc.preload_metadata() + # Load any configured modules into the homeserver module_api = hs.get_module_api() for module, module_config in hs.config.modules.loaded_modules: diff --git a/tests/storage/test_events_bg_updates.py b/tests/storage/test_events_bg_updates.py index d1a794c5a1..a5b53de77f 100644 --- a/tests/storage/test_events_bg_updates.py +++ b/tests/storage/test_events_bg_updates.py @@ -14,11 +14,14 @@ # +from canonicaljson import encode_canonical_json + from twisted.internet.testing import MemoryReactor from synapse.api.constants import MAX_DEPTH from synapse.api.room_versions import RoomVersion, RoomVersions from synapse.server import HomeServer +from synapse.types.storage import _BackgroundUpdates from synapse.util.clock import Clock from tests.unittest import HomeserverTestCase @@ -154,3 +157,133 @@ def test_fixup_max_depth_cap_bg_update_old_room_version(self) -> None: # Assert that the topological_ordering of events has not been changed # from their depth. self.assertDictEqual(event_id_to_depth, dict(rows)) + + +class TestRedactionsRecheckBgUpdate(HomeserverTestCase): + """Test the background update that backfills the `recheck` column in redactions.""" + + def prepare( + self, reactor: MemoryReactor, clock: Clock, homeserver: HomeServer + ) -> None: + self.store = self.hs.get_datastores().main + self.db_pool = self.store.db_pool + + # Re-insert the background update, since it already ran during setup. + self.get_success( + self.db_pool.simple_insert( + table="background_updates", + values={ + "update_name": _BackgroundUpdates.REDACTIONS_RECHECK_BG_UPDATE, + "progress_json": "{}", + }, + ) + ) + self.db_pool.updates._all_done = False + + def _insert_redaction( + self, + event_id: str, + redacts: str, + recheck_redaction: bool | None = None, + insert_event_json: bool = True, + ) -> None: + """Insert a row into `redactions` and optionally a matching `event_json` row. + + Args: + event_id: The event ID of the redaction event. + redacts: The event ID being redacted. + recheck_redaction: The value of `recheck_redaction` in internal metadata. + If None, the key is omitted from internal metadata. + insert_event_json: Whether to insert a corresponding row in `event_json`. + """ + self.get_success( + self.db_pool.simple_insert( + table="redactions", + values={ + "event_id": event_id, + "redacts": redacts, + "have_censored": False, + "received_ts": 0, + }, + ) + ) + + if insert_event_json: + internal_metadata: dict = {} + if recheck_redaction is not None: + internal_metadata["recheck_redaction"] = recheck_redaction + + self.get_success( + self.db_pool.simple_insert( + table="event_json", + values={ + "event_id": event_id, + "room_id": "!room:test", + "internal_metadata": encode_canonical_json( + internal_metadata + ).decode("utf-8"), + "json": "{}", + "format_version": 3, + }, + ) + ) + + def _get_recheck(self, event_id: str) -> bool: + row = self.get_success( + self.db_pool.simple_select_one( + table="redactions", + keyvalues={"event_id": event_id}, + retcols=["recheck"], + ) + ) + return bool(row[0]) + + def test_recheck_true(self) -> None: + """A redaction with recheck_redaction=True in internal metadata gets recheck=True.""" + self._insert_redaction("$redact1:test", "$target1:test", recheck_redaction=True) + + self.wait_for_background_updates() + + self.assertTrue(self._get_recheck("$redact1:test")) + + def test_recheck_false(self) -> None: + """A redaction with recheck_redaction=False in internal metadata gets recheck=False.""" + self._insert_redaction( + "$redact2:test", "$target2:test", recheck_redaction=False + ) + + self.wait_for_background_updates() + + self.assertFalse(self._get_recheck("$redact2:test")) + + def test_recheck_absent_from_metadata(self) -> None: + """A redaction with no recheck_redaction key in internal metadata gets recheck=False.""" + self._insert_redaction("$redact3:test", "$target3:test", recheck_redaction=None) + + self.wait_for_background_updates() + + self.assertFalse(self._get_recheck("$redact3:test")) + + def test_recheck_no_event_json(self) -> None: + """A redaction with no event_json row gets recheck=False.""" + self._insert_redaction( + "$redact4:test", "$target4:test", insert_event_json=False + ) + + self.wait_for_background_updates() + + self.assertFalse(self._get_recheck("$redact4:test")) + + def test_batching(self) -> None: + """The update processes rows in batches, completing when all are done.""" + self._insert_redaction("$redact5:test", "$target5:test", recheck_redaction=True) + self._insert_redaction( + "$redact6:test", "$target6:test", recheck_redaction=False + ) + self._insert_redaction("$redact7:test", "$target7:test", recheck_redaction=True) + + self.wait_for_background_updates() + + self.assertTrue(self._get_recheck("$redact5:test")) + self.assertFalse(self._get_recheck("$redact6:test")) + self.assertTrue(self._get_recheck("$redact7:test")) diff --git a/tests/storage/test_redaction.py b/tests/storage/test_redaction.py index 92eb99f1d5..c82ccf1600 100644 --- a/tests/storage/test_redaction.py +++ b/tests/storage/test_redaction.py @@ -158,7 +158,7 @@ def test_redact(self) -> None: event, ) - self.assertFalse("redacted_because" in event.unsigned) + self.assertIsNone(event.internal_metadata.redacted_by) # Redact event reason = "Because I said so" @@ -168,7 +168,7 @@ def test_redact(self) -> None: self.assertEqual(msg_event.event_id, event.event_id) - self.assertTrue("redacted_because" in event.unsigned) + self.assertIsNotNone(event.internal_metadata.redacted_by) self.assertObjectHasAttributes( { @@ -179,15 +179,6 @@ def test_redact(self) -> None: event, ) - self.assertObjectHasAttributes( - { - "type": EventTypes.Redaction, - "user_id": self.u_alice.to_string(), - "content": {"reason": reason}, - }, - event.unsigned["redacted_because"], - ) - def test_redact_join(self) -> None: self.inject_room_member(self.room1, self.u_alice, Membership.JOIN) @@ -206,7 +197,7 @@ def test_redact_join(self) -> None: event, ) - self.assertFalse(hasattr(event, "redacted_because")) + self.assertIsNone(event.internal_metadata.redacted_by) # Redact event reason = "Because I said so" @@ -216,7 +207,7 @@ def test_redact_join(self) -> None: event = self.get_success(self.store.get_event(msg_event.event_id)) - self.assertTrue("redacted_because" in event.unsigned) + self.assertIsNotNone(event.internal_metadata.redacted_by) self.assertObjectHasAttributes( { @@ -227,15 +218,6 @@ def test_redact_join(self) -> None: event, ) - self.assertObjectHasAttributes( - { - "type": EventTypes.Redaction, - "user_id": self.u_alice.to_string(), - "content": {"reason": reason}, - }, - event.unsigned["redacted_because"], - ) - def test_circular_redaction(self) -> None: redaction_event_id1 = "$redaction1_id:test" redaction_event_id2 = "$redaction2_id:test" @@ -331,10 +313,7 @@ def internal_metadata(self) -> EventInternalMetadata: fetched = self.get_success(self.store.get_event(redaction_event_id1)) # it should have been redacted - self.assertEqual(fetched.unsigned["redacted_by"], redaction_event_id2) - self.assertEqual( - fetched.unsigned["redacted_because"].event_id, redaction_event_id2 - ) + self.assertEqual(fetched.internal_metadata.redacted_by, redaction_event_id2) def test_redact_censor(self) -> None: """Test that a redacted event gets censored in the DB after a month""" @@ -355,7 +334,7 @@ def test_redact_censor(self) -> None: event, ) - self.assertFalse("redacted_because" in event.unsigned) + self.assertIsNone(event.internal_metadata.redacted_by) # Redact event reason = "Because I said so" @@ -363,7 +342,7 @@ def test_redact_censor(self) -> None: event = self.get_success(self.store.get_event(msg_event.event_id)) - self.assertTrue("redacted_because" in event.unsigned) + self.assertIsNotNone(event.internal_metadata.redacted_by) self.assertObjectHasAttributes( { diff --git a/tests/synapse_rust/test_room_versions.py b/tests/synapse_rust/test_room_versions.py new file mode 100644 index 0000000000..aff1ab2783 --- /dev/null +++ b/tests/synapse_rust/test_room_versions.py @@ -0,0 +1,45 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . + +from synapse.api.room_versions import KNOWN_ROOM_VERSIONS + +from tests import unittest + + +class KnownRoomVersionsMappingTestCase(unittest.TestCase): + """Tests for the Rust-backed KNOWN_ROOM_VERSIONS mapping.""" + + def test_contains_none(self) -> None: + """Test that `None in KNOWN_ROOM_VERSIONS` returns False + rather than raising a TypeError, matching Python dict semantics.""" + self.assertFalse(None in KNOWN_ROOM_VERSIONS) + + def test_contains_non_string(self) -> None: + """Test that non-string keys return False from __contains__.""" + self.assertFalse(42 in KNOWN_ROOM_VERSIONS) # type: ignore[comparison-overlap] + self.assertFalse(3.14 in KNOWN_ROOM_VERSIONS) # type: ignore[comparison-overlap] + self.assertFalse([] in KNOWN_ROOM_VERSIONS) # type: ignore[comparison-overlap] + + def test_contains_known_version(self) -> None: + self.assertTrue("1" in KNOWN_ROOM_VERSIONS) + + def test_contains_unknown_version(self) -> None: + self.assertFalse("unknown" in KNOWN_ROOM_VERSIONS) + + def test_getitem_non_string_raises_key_error(self) -> None: + """Test that KNOWN_ROOM_VERSIONS[non_string] raises KeyError, + not TypeError, matching Python dict semantics.""" + with self.assertRaises(KeyError): + KNOWN_ROOM_VERSIONS[42] # type: ignore[index] + with self.assertRaises(KeyError): + KNOWN_ROOM_VERSIONS[None] # type: ignore[index] diff --git a/tests/unittest.py b/tests/unittest.py index 0220548d7b..ee9bfcf7e6 100644 --- a/tests/unittest.py +++ b/tests/unittest.py @@ -704,8 +704,43 @@ async def run_bg_updates() -> None: def pump(self, by: float = 0.0) -> None: """ - Pump the reactor enough that Deferreds will fire. + XXX: Deprecated: This method is deprecated. Use `self.reactor.advance(...)` + directly instead. + + Pump the reactor enough that `clock.call_later` scheduled callbacks will fire. + + To demystify this function, it simply advances time by the number of seconds + specified (defaults to `0`, we also multiply by 100, so `pump(1)` is 100 seconds + in 1 second steps/increments) whilst calling any pending callbacks, allowing any + queued/pending tasks to run because enough time has passed. + + So for example, if you have some Synapse code that does + `clock.call_later(Duration(seconds=2), callback)`, then calling + `self.pump(by=0.02)` will advance time by 2 seconds, which is enough for that + callback to be ready to run now. Same for `clock.sleep(...)` , + `clock.looping_call(...)`, and whatever other clock utilities that use + `clock.call_later` under the hood for scheduling tasks. Trying to use + `pump(by=...)` with exact math to meet a specific deadline feels pretty dirty + though which is why we recommend using `self.reactor.advance(...)` directly + nowadays. + + We don't have any exact historical context for why `pump()` was introduced into + the codebase beyond the code itself. We assume that we multiply by 100 so that + when you use the clock to schedule something that schedules more things, it + tries to run the whole chain to completion. + + XXX: If you're having to call this function, please call out in comments, which + scheduled thing you're aiming to trigger. Please also check whether the + `pump(...)` is even necessary as it was often misused. + + Args: + by: The time increment in seconds to advance time by. We will advance time + in 100 steps, each step by this value. """ + # We multiply by 100, so `pump(1)` actually advances time by 100 seconds in 1 + # second steps/increments. We assume this was done so that when you use the + # clock to schedule something that schedules more things, it tries to run the + # whole chain to completion. self.reactor.pump([by] * 100) def get_success(self, d: Awaitable[TV], by: float = 0.0) -> TV: