MSC4311: send full PDUs in invite_room_state/knock_room_state over

Author: FrenchGithubUser

changelog.d/19749.feature

@@ -0,0 +1,2 @@
+Partial [MSC4311](https://github.com/matrix-org/matrix-spec-proposals/pull/4311) implementation: `invite_room_state` and `knock_room_state` are now exchanged
+  as full PDUs over federation, allowing receiving servers to verify room IDs. Contributed by @FrenchGithubUser @Famedly.

synapse/events/utils.py

@@ -1019,15 +1019,6 @@ def strip_event(event: EventBase) -> JsonDict:
     Stripped state events can only have the `sender`, `type`, `state_key` and `content`
     properties present.
     """
-    # MSC4311: Ensure the create event is available on invites and knocks.
-    # TODO: Implement the rest of MSC4311
-    if (
-        event.room_version.msc4291_room_ids_as_hashes
-        and event.type == EventTypes.Create
-        and event.get_state_key() == ""
-    ):
-        return event.get_pdu_json()
-
     return {
         "type": event.type,
         "state_key": event.state_key,
@@ -1036,6 +1027,22 @@ def strip_event(event: EventBase) -> JsonDict:
     }
 
 
+def strip_event_dict(pdu_dict: JsonDict) -> JsonDict:
+    """Strip a PDU dict to stripped state format (4 fields only).
+
+    Used to convert full PDUs received over federation (MSC4311) to the
+    stripped state format expected by the Client-Server API.
+
+    Callers must pre-filter to ensure pdu_dict has a non-empty "type" key.
+    """
+    return {
+        "type": pdu_dict["type"],
+        "state_key": pdu_dict.get("state_key", ""),
+        "content": pdu_dict.get("content", {}),
+        "sender": pdu_dict.get("sender", ""),
+    }
+
+
 def parse_stripped_state_event(raw_stripped_event: Any) -> StrippedStateEvent | None:
     """
     Given a raw value from an event's `unsigned` field, attempt to parse it into a

synapse/federation/federation_client.py

@@ -72,6 +72,7 @@
 from synapse.logging.opentracing import SynapseTags, log_kv, set_tag, tag_args, trace
 from synapse.metrics import SERVER_NAME_LABEL
 from synapse.types import JsonDict, StrCollection, UserID, get_domain_from_id
+from synapse.types.state import StateFilter
 from synapse.util.async_helpers import concurrently_execute
 from synapse.util.caches.expiringcache import ExpiringCache
 from synapse.util.duration import Duration
@@ -1350,6 +1351,27 @@ async def _do_send_invite(
         """
         time_now = self._clock.time_msec()
 
+        stripped = pdu.unsigned.get("invite_room_state", [])
+        if stripped and pdu.room_version.msc4291_room_ids_as_hashes:
+            # MSC4311: upgrade stripped state to full PDUs so the receiving server
+            # can verify the room ID. Use the stripped state as a guide for which
+            # events to look up.
+            types = [
+                (e.get("type", ""), e.get("state_key", ""))
+                for e in stripped
+                if isinstance(e, dict) and e.get("type")
+            ]
+            state_filter = StateFilter.from_types(types)
+            state_ids = await self.store.get_partial_filtered_current_state_ids(
+                pdu.room_id, state_filter
+            )
+            state_events = await self.store.get_events(state_ids.values())
+            invite_room_state_pdus: list = [
+                e.get_pdu_json() for e in state_events.values()
+            ]
+        else:
+            invite_room_state_pdus = stripped
+
         try:
             return await self.transport_layer.send_invite_v2(
                 destination=destination,
@@ -1358,7 +1380,7 @@ async def _do_send_invite(
                 content={
                     "event": pdu.get_pdu_json(time_now),
                     "room_version": room_version.identifier,
-                    "invite_room_state": pdu.unsigned.get("invite_room_state", []),
+                    "invite_room_state": invite_room_state_pdus,
                 },
             )
         except HttpResponseException as e:

synapse/federation/federation_server.py

@@ -957,19 +957,22 @@ async def on_send_knock_request(
         Returns:
             The stripped room state.
         """
-        _, context = await self._on_send_membership_event(
+        event, context = await self._on_send_membership_event(
             origin, content, Membership.KNOCK, room_id
         )
 
-        # Retrieve stripped state events from the room and send them back to the remote
-        # server. This will allow the remote server's clients to display information
-        # related to the room while the knock request is pending.
-        stripped_room_state = (
-            await self.store.get_stripped_room_state_from_event_context(
+        if event.room_version.msc4291_room_ids_as_hashes:
+            # MSC4311: return full PDUs so the knocking server can validate the room ID.
+            knock_room_state = await self.store.get_room_state_pdus_from_event_context(
                 context, self._room_prejoin_state_types
             )
-        )
-        return {"knock_room_state": stripped_room_state}
+        else:
+            knock_room_state = (
+                await self.store.get_stripped_room_state_from_event_context(
+                    context, self._room_prejoin_state_types
+                )
+            )
+        return {"knock_room_state": knock_room_state}
 
     async def _on_send_membership_event(
         self, origin: str, content: JsonDict, membership_type: str, room_id: str

synapse/federation/transport/server/federation.py

@@ -28,10 +28,11 @@
     Sequence,
 )
 
-from synapse.api.constants import Direction, EduTypes
+from synapse.api.constants import Direction, EduTypes, EventTypes
 from synapse.api.errors import Codes, SynapseError
-from synapse.api.room_versions import RoomVersions
+from synapse.api.room_versions import KNOWN_ROOM_VERSIONS, RoomVersions
 from synapse.api.urls import FEDERATION_UNSTABLE_PREFIX, FEDERATION_V2_PREFIX
+from synapse.events.utils import strip_event_dict
 from synapse.federation.transport.server._base import (
     Authenticator,
     BaseFederationServlet,
@@ -522,9 +523,29 @@ async def on_PUT(
         if not isinstance(invite_room_state, list):
             invite_room_state = []
 
-        # Synapse expects invite_room_state to be in unsigned, as it is in v1
-        # API
-
+        room_version_obj = KNOWN_ROOM_VERSIONS.get(room_version)
+        if room_version_obj and room_version_obj.msc4291_room_ids_as_hashes:
+            # MSC4311: invite_room_state contains full PDUs. Warn if m.room.create
+            # is absent, then strip to 4-field format for C-S API storage.
+            create_event_present = any(
+                isinstance(e, dict)
+                and e.get("type") == EventTypes.Create
+                and e.get("state_key") == ""
+                for e in invite_room_state
+            )
+            if not create_event_present:
+                logger.warning(
+                    "invite_room_state from %s for room %s is missing m.room.create",
+                    origin,
+                    room_id,
+                )
+            invite_room_state = [
+                strip_event_dict(e)
+                for e in invite_room_state
+                if isinstance(e, dict) and e.get("type")
+            ]
+
+        # Synapse expects invite_room_state to be in unsigned, as it is in v1 API
         event.setdefault("unsigned", {})["invite_room_state"] = invite_room_state
 
         result = await self.handler.on_invite_request(

synapse/handlers/federation.py

@@ -59,6 +59,7 @@
 from synapse.event_auth import validate_event_for_room_version
 from synapse.events import EventBase
 from synapse.events.snapshot import EventContext, UnpersistedEventContextBase
+from synapse.events.utils import strip_event_dict
 from synapse.events.validator import EventValidator
 from synapse.federation.federation_client import InvalidResponseError
 from synapse.handlers.pagination import PURGE_PAGINATION_LOCK_NAME
@@ -865,19 +866,37 @@ async def do_knock(
         # further information about the room in the form of partial state events
         knock_response = await self.federation_client.send_knock(target_hosts, event)
 
-        # Store any stripped room state events in the "unsigned" key of the event.
+        # Store room state events in the "unsigned" key of the event.
         # This is a bit of a hack and is cribbing off of invites. Basically we
         # store the room state here and retrieve it again when this event appears
         # in the invitee's sync stream. It is stripped out for all other local users.
-        stripped_room_state = knock_response.get("knock_room_state")
+        knock_room_state = knock_response.get("knock_room_state")
 
-        if stripped_room_state is None:
+        if knock_room_state is None:
             raise KeyError("Missing 'knock_room_state' field in send_knock response")
 
-        if not isinstance(stripped_room_state, list):
+        if not isinstance(knock_room_state, list):
             raise TypeError("'knock_room_state' has wrong type")
 
-        event.unsigned["knock_room_state"] = stripped_room_state
+        # MSC4311: knock_room_state contains full PDUs over federation.
+        # Validate that m.room.create is present, then strip to stripped state for clients.
+        create_event_present = any(
+            isinstance(e, dict)
+            and e.get("type") == EventTypes.Create
+            and e.get("state_key") == ""
+            for e in knock_room_state
+        )
+        if not create_event_present:
+            logger.warning(
+                "knock_room_state for room %s is missing m.room.create event",
+                event.room_id,
+            )
+
+        event.unsigned["knock_room_state"] = [
+            strip_event_dict(e)
+            for e in knock_room_state
+            if isinstance(e, dict) and e.get("type")
+        ]
 
         context = EventContext.for_outlier(self._storage_controllers)
         stream_id = await self._federation_event_handler.persist_events_and_notify(

synapse/storage/databases/main/events_worker.py

@@ -1151,6 +1151,42 @@ async def get_stripped_room_state_from_event_context(
 
         return [strip_event(e) for e in state_to_include.values()]
 
+    async def get_room_state_pdus_from_event_context(
+        self,
+        context: EventContext,
+        state_keys_to_include: StateFilter,
+        membership_user_id: str | None = None,
+    ) -> list[JsonDict]:
+        """
+        Retrieve room state events as full PDUs, for use in federation
+        invite_room_state and knock_room_state (MSC4311).
+
+        Args:
+            context: The event context to retrieve state of the room from.
+            state_keys_to_include: The state events to include, for each event type.
+            membership_user_id: An optional user ID to include the membership state
+                events of.
+
+        Returns:
+            A list of full PDU dicts representing the room state.
+        """
+        if membership_user_id:
+            types = chain(
+                state_keys_to_include.to_types(),
+                [(EventTypes.Member, membership_user_id)],
+            )
+            state_filter = StateFilter.from_types(types)
+        else:
+            state_filter = state_keys_to_include
+        selected_state_ids = await context.get_current_state_ids(state_filter)
+
+        assert selected_state_ids is not None
+
+        selected_state_ids = state_filter.filter_state(selected_state_ids)
+        state_to_include = await self.get_events(selected_state_ids.values())
+
+        return [e.get_pdu_json() for e in state_to_include.values()]
+
     def _maybe_start_fetch_thread(self) -> None:
         """Starts an event fetch thread if we are not yet at the maximum number."""
         with self._event_fetch_lock:

tests/federation/test_federation_server.py

@@ -43,6 +43,7 @@
 from synapse.util.clock import Clock
 
 from tests import unittest
+from tests.server import FakeChannel
 from tests.unittest import override_config
 
 logger = logging.getLogger(__name__)
@@ -714,6 +715,132 @@ def test_send_join_contributes_to_room_join_rate_limit_and_is_limited(self) -> N
     #   is probably sufficient to reassure that the bucket is updated.
 
 
+class MSC4311FederationInviteTestCase(unittest.FederatingHomeserverTestCase):
+    """MSC4311: Tests for invite_room_state validation and stripping over federation."""
+
+    servlets = [
+        admin.register_servlets,
+        room.register_servlets,
+        login.register_servlets,
+    ]
+
+    def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None:
+        super().prepare(reactor, clock, hs)
+        self.local_user = self.register_user("user", "pass")
+        self.remote_room_id = f"!room:{self.OTHER_SERVER_NAME}"
+        self.remote_sender = f"@creator:{self.OTHER_SERVER_NAME}"
+
+    def _make_invite_request(
+        self,
+        invite_room_state: list,
+        room_version: str = RoomVersions.V10.identifier,
+    ) -> FakeChannel:
+        rv = KNOWN_ROOM_VERSIONS[room_version]
+        room_create_event = make_event_from_dict(
+            self.add_hashes_and_signatures_from_other_server(
+                {
+                    "room_id": self.remote_room_id,
+                    "sender": self.remote_sender,
+                    "depth": 1,
+                    "origin_server_ts": 1,
+                    "type": EventTypes.Create,
+                    "state_key": "",
+                    "content": {
+                        "creator": self.remote_sender,
+                        "room_version": room_version,
+                    },
+                    "auth_events": [],
+                    "prev_events": [],
+                },
+                rv,
+            ),
+            rv,
+        )
+        invite_event = make_event_from_dict(
+            self.add_hashes_and_signatures_from_other_server(
+                {
+                    "room_id": self.remote_room_id,
+                    "sender": self.remote_sender,
+                    "depth": 2,
+                    "origin_server_ts": 2,
+                    "type": EventTypes.Member,
+                    "state_key": self.local_user,
+                    "content": {"membership": Membership.INVITE},
+                    "auth_events": [room_create_event.event_id],
+                    "prev_events": [room_create_event.event_id],
+                },
+                rv,
+            ),
+            rv,
+        )
+        return self.make_signed_federation_request(
+            "PUT",
+            f"/_matrix/federation/v2/invite/{self.remote_room_id}/{invite_event.event_id}",
+            content={
+                "event": invite_event.get_dict(),
+                "invite_room_state": invite_room_state,
+                "room_version": room_version,
+            },
+        )
+
+    def test_full_pdus_stripped_for_client(self) -> None:
+        """invite_room_state full PDUs are stripped to 4 fields for the C-S API."""
+        rv = KNOWN_ROOM_VERSIONS[RoomVersions.V10.identifier]
+        create_pdu = make_event_from_dict(
+            self.add_hashes_and_signatures_from_other_server(
+                {
+                    "room_id": self.remote_room_id,
+                    "sender": self.remote_sender,
+                    "depth": 1,
+                    "origin_server_ts": 1,
+                    "type": EventTypes.Create,
+                    "state_key": "",
+                    "content": {
+                        "creator": self.remote_sender,
+                        "room_version": RoomVersions.V10.identifier,
+                    },
+                    "auth_events": [],
+                    "prev_events": [],
+                },
+                rv,
+            ),
+            rv,
+        )
+        # A full PDU has signatures, hashes, etc.
+        self.assertIn("signatures", create_pdu.get_pdu_json())
+
+        channel = self._make_invite_request(
+            invite_room_state=[create_pdu.get_pdu_json()]
+        )
+        self.assertEqual(channel.code, 200, channel.json_body)
+
+        # Retrieve the stored invite event and verify invite_room_state is stripped.
+        store = self.hs.get_datastores().main
+        invite_memberships = self.get_success(
+            store.get_invited_rooms_for_local_user(self.local_user)
+        )
+        self.assertEqual(len(invite_memberships), 1)
+        invite_event = self.get_success(store.get_event(invite_memberships[0].event_id))
+        invite_state = invite_event.unsigned.get("invite_room_state", [])
+
+        create_events = [e for e in invite_state if e.get("type") == EventTypes.Create]
+        self.assertEqual(len(create_events), 1)
+        create = create_events[0]
+        # Must be stripped state: only these 4 fields
+        self.assertIn("type", create)
+        self.assertIn("state_key", create)
+        self.assertIn("sender", create)
+        self.assertIn("content", create)
+        self.assertNotIn("signatures", create)
+        self.assertNotIn("hashes", create)
+        self.assertNotIn("auth_events", create)
+
+    def test_missing_create_event_warns_but_accepts(self) -> None:
+        """invite_room_state without m.room.create is accepted with a warning."""
+        channel = self._make_invite_request(invite_room_state=[])
+        self.assertEqual(channel.code, 200, channel.json_body)
+
+
 class StripUnsignedFromEventsTestCase(unittest.TestCase):
     """
     Test to make sure that we handle the raw JSON events from federation carefully and