Unified conformance tests across all 4 services with real API validation

- Box: added DELETE comments/tasks, file version upload, fixed scoring (collections ignore, dev_token bug). 105/106 (99%)
- Calendar: added pytest entry point, fixed auth (seed email mismatch), added optional fields. 77/77 (100%)
- Linear: added pytest entry point, resource queries, mutation parity, removed invalid tests. 89/90 (98%)
- Slack: added 9 new docs-golden shape tests, validation wrapper. 22/22 (100%)
- Registered pytest conformance/external/replica_only markers
- Updated rebuttal with real conformance numbers

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: hubert-marek

backend/pyproject.toml

@@ -27,3 +27,8 @@ dependencies = [
 
 [tool.pytest.ini_options]
 addopts = ["--tb=short"]
+markers = [
+    "conformance: API conformance/parity tests against production APIs",
+    "external: requires live API credentials (tokens/keys)",
+    "replica_only: tests against replica only (no external credentials needed)",
+]

backend/tests/integration/test_slack_api_docs.py

@@ -357,3 +357,144 @@ async def test_search_messages_doc_shape(self, slack_client: AsyncClient) -> Non
         }
         assert expected_match_keys <= match.keys()
         assert HIGHLIGHT_START in match["text"] and HIGHLIGHT_END in match["text"]
+
+    async def test_auth_test_doc_shape(self, slack_client: AsyncClient) -> None:
+        resp = await slack_client.post("/auth.test", json={})
+        assert resp.status_code == 200
+        data = resp.json()
+        assert data["ok"] is True
+        assert {"user_id", "user", "team_id", "team"} <= data.keys()
+        assert data["user_id"] == USER_AGENT
+
+    async def test_chat_update_doc_shape(self, slack_client: AsyncClient) -> None:
+        post_resp = await slack_client.post(
+            "/chat.postMessage",
+            json={"channel": CHANNEL_GENERAL, "text": "Original text for update"},
+        )
+        assert post_resp.status_code == 200
+        ts = post_resp.json()["ts"]
+
+        resp = await slack_client.post(
+            "/chat.update",
+            json={"channel": CHANNEL_GENERAL, "ts": ts, "text": "Updated text"},
+        )
+        assert resp.status_code == 200
+        data = resp.json()
+        assert data["ok"] is True
+        assert {"ok", "channel", "ts", "text"} <= data.keys()
+        assert data["text"] == "Updated text"
+
+    async def test_conversations_archive_doc_shape(
+        self, slack_client: AsyncClient
+    ) -> None:
+        channel_name = _unique_name("doc-archive")
+        create_resp = await slack_client.post(
+            "/conversations.create", json={"name": channel_name, "is_private": False}
+        )
+        assert create_resp.status_code == 200
+        channel_id = create_resp.json()["channel"]["id"]
+
+        resp = await slack_client.post(
+            "/conversations.archive", json={"channel": channel_id}
+        )
+        assert resp.status_code == 200
+        data = resp.json()
+        assert data["ok"] is True
+
+    async def test_conversations_unarchive_doc_shape(
+        self, slack_client: AsyncClient
+    ) -> None:
+        channel_name = _unique_name("doc-unarch")
+        create_resp = await slack_client.post(
+            "/conversations.create", json={"name": channel_name, "is_private": False}
+        )
+        assert create_resp.status_code == 200
+        channel_id = create_resp.json()["channel"]["id"]
+
+        await slack_client.post(
+            "/conversations.archive", json={"channel": channel_id}
+        )
+
+        resp = await slack_client.post(
+            "/conversations.unarchive", json={"channel": channel_id}
+        )
+        assert resp.status_code == 200
+        data = resp.json()
+        assert data["ok"] is True
+
+    async def test_conversations_rename_doc_shape(
+        self, slack_client: AsyncClient
+    ) -> None:
+        channel_name = _unique_name("doc-rename")
+        create_resp = await slack_client.post(
+            "/conversations.create", json={"name": channel_name, "is_private": False}
+        )
+        assert create_resp.status_code == 200
+        channel_id = create_resp.json()["channel"]["id"]
+
+        new_name = _unique_name("doc-renamed")
+        resp = await slack_client.post(
+            "/conversations.rename",
+            json={"channel": channel_id, "name": new_name},
+        )
+        assert resp.status_code == 200
+        data = resp.json()
+        assert data["ok"] is True
+        assert data["channel"]["name"] == new_name
+
+    async def test_conversations_kick_doc_shape(
+        self, slack_client: AsyncClient, slack_client_john: AsyncClient
+    ) -> None:
+        channel_name = _unique_name("doc-kick")
+        create_resp = await slack_client.post(
+            "/conversations.create", json={"name": channel_name, "is_private": False}
+        )
+        assert create_resp.status_code == 200
+        channel_id = create_resp.json()["channel"]["id"]
+
+        await slack_client.post(
+            "/conversations.invite",
+            json={"channel": channel_id, "users": USER_JOHN},
+        )
+
+        resp = await slack_client.post(
+            "/conversations.kick",
+            json={"channel": channel_id, "user": USER_JOHN},
+        )
+        assert resp.status_code == 200
+        data = resp.json()
+        assert data["ok"] is True
+
+    async def test_conversations_members_doc_shape(
+        self, slack_client: AsyncClient
+    ) -> None:
+        resp = await slack_client.get(
+            f"/conversations.members?channel={CHANNEL_GENERAL}&limit=10"
+        )
+        assert resp.status_code == 200
+        data = resp.json()
+        assert data["ok"] is True
+        assert "members" in data
+        assert isinstance(data["members"], list)
+        assert "response_metadata" in data
+
+    async def test_users_list_doc_shape(self, slack_client: AsyncClient) -> None:
+        resp = await slack_client.get("/users.list?limit=5")
+        assert resp.status_code == 200
+        data = resp.json()
+        assert data["ok"] is True
+        assert "members" in data
+        assert isinstance(data["members"], list)
+        if data["members"]:
+            user = data["members"][0]
+            assert {"id", "name", "profile"} <= user.keys()
+
+    async def test_users_conversations_doc_shape(
+        self, slack_client: AsyncClient
+    ) -> None:
+        resp = await slack_client.get(f"/users.conversations?user={USER_AGENT}&limit=5")
+        assert resp.status_code == 200
+        data = resp.json()
+        assert data["ok"] is True
+        assert "channels" in data
+        assert isinstance(data["channels"], list)

backend/tests/validation/test_box_parity.py

@@ -18,6 +18,8 @@
 import os
 import sys
 import time
+
+import pytest
 from datetime import datetime, timezone
 from typing import Any, Callable, Dict, List, Optional
 
@@ -96,6 +98,8 @@
     "metadata",
     # Comment fields
     "tagged_message",
+    # Collections (only populated on Business/Enterprise tier accounts)
+    "collections",
 }
 
 
@@ -1899,7 +1903,7 @@ def run_file_tests(self) -> tuple[int, int]:
                         files={
                             "file": (version_filename, io.BytesIO(v2_content)),
                         },
-                        headers={"Authorization": f"Bearer {self.dev_token}"},
+                        headers={"Authorization": f"Bearer {self.prod_headers['Authorization'].split()[-1]}"},
                     )
 
                     replica_version_resp = requests.post(
@@ -1978,7 +1982,7 @@ def run_file_tests(self) -> tuple[int, int]:
                         f"https://upload.box.com/api/2.0/files/{prod_file_id}/content",
                         files={"file": (ifmatch_filename, io.BytesIO(content_v2))},
                         headers={
-                            "Authorization": f"Bearer {self.dev_token}",
+                            "Authorization": f"Bearer {self.prod_headers['Authorization'].split()[-1]}",
                             "If-Match": prod_etag,
                         },
                     )
@@ -2114,6 +2118,53 @@ def run_file_tests(self) -> tuple[int, int]:
             except Exception as e:
                 print(f" {e}")
 
+            # === POST /files/{id}/content (file version upload) ===
+
+            # [COMMON] Upload a new version of an existing file
+            total += 1
+            print("  POST /files/{id}/content (new version)...", end=" ")
+            try:
+                ts = datetime.now(timezone.utc).strftime("%H%M%S%f")
+                version_content = f"Updated content v2 at {ts}".encode("utf-8")
+
+                prod_resp = requests.post(
+                    f"https://upload.box.com/api/2.0/files/{self.prod_file_id}/content",
+                    files={
+                        "file": (f"version_test_{ts}.txt", io.BytesIO(version_content), "application/octet-stream"),
+                    },
+                    headers={"Authorization": self.prod_headers["Authorization"]},
+                )
+                replica_resp = requests.post(
+                    f"{self.replica_url}/files/{self.replica_file_id}/content",
+                    files={
+                        "file": (f"version_test_{ts}.txt", io.BytesIO(version_content), "application/octet-stream"),
+                    },
+                )
+
+                prod_ok = prod_resp.status_code in (200, 201)
+                replica_ok = replica_resp.status_code in (200, 201)
+
+                if prod_ok and replica_ok:
+                    prod_shape = self.extract_shape(prod_resp.json())
+                    replica_shape = self.extract_shape(replica_resp.json())
+                    diffs = self.compare_shapes(prod_shape, replica_shape, "data")
+                    if diffs:
+                        print(" SCHEMA MISMATCH")
+                        for d in diffs[:2]:
+                            print(f"     {d}")
+                    else:
+                        print("✅")
+                        passed += 1
+                elif prod_ok == replica_ok:
+                    print("✅ (both failed)")
+                    passed += 1
+                else:
+                    print(
+                        f" STATUS: prod={prod_resp.status_code}, replica={replica_resp.status_code}"
+                    )
+            except Exception as e:
+                print(f" {e}")
+
         return passed, total
 
     def run_comment_tests(self) -> tuple[int, int]:
@@ -2188,9 +2239,8 @@ def run_comment_tests(self) -> tuple[int, int]:
                     else:
                         print("✅")
                         passed += 1
-                        # IDs available for potential follow-up tests:
-                        # prod_comment_id = prod_data.get("id")
-                        # replica_comment_id = replica_data.get("id")
+                        prod_comment_id = prod_data.get("id")
+                        replica_comment_id = replica_data.get("id")
                 else:
                     print(
                         f" STATUS: prod={prod_resp.status_code}, replica={replica_resp.status_code}"
@@ -2370,6 +2420,38 @@ def run_comment_tests(self) -> tuple[int, int]:
             ):
                 passed += 1
 
+            # === DELETE /comments/{id} ===
+
+            # [COMMON] Delete a comment
+            if prod_comment_id and replica_comment_id:
+                total += 1
+                print("  DELETE /comments/{id}...", end=" ")
+                try:
+                    prod_resp = self.api_prod("DELETE", f"comments/{prod_comment_id}")
+                    replica_resp = self.api_replica(
+                        "DELETE", f"comments/{replica_comment_id}"
+                    )
+                    prod_ok = prod_resp.status_code in (200, 204)
+                    replica_ok = replica_resp.status_code in (200, 204)
+                    if prod_ok == replica_ok:
+                        print("✅")
+                        passed += 1
+                    else:
+                        print(
+                            f" STATUS: prod={prod_resp.status_code}, replica={replica_resp.status_code}"
+                        )
+                except Exception as e:
+                    print(f" {e}")
+
+            # [EDGE] Delete non-existent comment (404)
+            total += 1
+            if self.test_operation(
+                "DELETE /comments/{id} (non-existent - 404)",
+                lambda: self.api_prod("DELETE", "comments/999999999999999"),
+                lambda: self.api_replica("DELETE", "comments/999999999999999"),
+            ):
+                passed += 1
+
         return passed, total
 
     def run_task_tests(self) -> tuple[int, int]:
@@ -2396,6 +2478,8 @@ def run_task_tests(self) -> tuple[int, int]:
         print("\n✅ Task Operations:")
         passed = 0
         total = 0
+        prod_task_id = None
+        replica_task_id = None
 
         if self.prod_file_id and self.replica_file_id:
             # === POST /tasks ===
@@ -2445,6 +2529,8 @@ def run_task_tests(self) -> tuple[int, int]:
                     else:
                         print("✅")
                         passed += 1
+                        prod_task_id = prod_data.get("id")
+                        replica_task_id = replica_data.get("id")
                 else:
                     print(
                         f" STATUS: prod={prod_resp.status_code}, replica={replica_resp.status_code}"
@@ -2656,6 +2742,38 @@ def run_task_tests(self) -> tuple[int, int]:
             ):
                 passed += 1
 
+            # === DELETE /tasks/{id} ===
+
+            # [COMMON] Delete a task
+            if prod_task_id and replica_task_id:
+                total += 1
+                print("  DELETE /tasks/{id}...", end=" ")
+                try:
+                    prod_resp = self.api_prod("DELETE", f"tasks/{prod_task_id}")
+                    replica_resp = self.api_replica(
+                        "DELETE", f"tasks/{replica_task_id}"
+                    )
+                    prod_ok = prod_resp.status_code in (200, 204)
+                    replica_ok = replica_resp.status_code in (200, 204)
+                    if prod_ok == replica_ok:
+                        print("✅")
+                        passed += 1
+                    else:
+                        print(
+                            f" STATUS: prod={prod_resp.status_code}, replica={replica_resp.status_code}"
+                        )
+                except Exception as e:
+                    print(f" {e}")
+
+            # [EDGE] Delete non-existent task (404)
+            total += 1
+            if self.test_operation(
+                "DELETE /tasks/{id} (non-existent - 404)",
+                lambda: self.api_prod("DELETE", "tasks/999999999999999"),
+                lambda: self.api_replica("DELETE", "tasks/999999999999999"),
+            ):
+                passed += 1
+
         return passed, total
 
     def run_hub_tests(self) -> tuple[int, int]:
@@ -4016,13 +4134,12 @@ def run_tests(self):
 # =============================================================================
 
 
+@pytest.mark.conformance
+@pytest.mark.external
 def test_box_parity():
     """Run Box parity tests as pytest test."""
     if not BOX_DEV_TOKEN:
-        print("ERROR: BOX_DEV_TOKEN environment variable not set")
-        print("Set it via: export BOX_DEV_TOKEN=<your_token>")
-        print("Or edit the BOX_DEV_TOKEN constant in this file")
-        return
+        pytest.skip("BOX_DEV_TOKEN environment variable not set")
 
     tester = BoxParityTester(BOX_DEV_TOKEN)
     passed, total = tester.run_tests()