Merge pull request #9167 from OpenMined/eelco/update-migration-tests

Update migration tests

Author: eelcovdw

notebooks/tutorials/version-upgrades/0-prepare-migration-data.ipynb

@@ -69,9 +69,9 @@
     "    name=\"test_upgradability\",\n",
     "    dev_mode=True,\n",
     "    reset=True,\n",
-    "    local_db=True,\n",
     "    n_consumers=2,\n",
     "    create_producer=True,\n",
+    "    port=\"auto\",\n",
     ")"
    ]
   },
@@ -251,7 +251,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.12.2"
+   "version": "3.10.13"
   }
  },
  "nbformat": 4,

notebooks/tutorials/version-upgrades/1-dump-database-to-file.ipynb

@@ -8,6 +8,7 @@
    "outputs": [],
    "source": [
     "# stdlib\n",
+    "import os\n",
     "from pathlib import Path\n",
     "\n",
     "# syft absolute\n",
@@ -24,9 +25,8 @@
     "server = sy.orchestra.launch(\n",
     "    name=\"test_upgradability\",\n",
     "    dev_mode=True,\n",
-    "    local_db=True,\n",
-    "    n_consumers=2,\n",
-    "    create_producer=True,\n",
+    "    reset=False,\n",
+    "    port=\"auto\",\n",
     "    migrate=False,\n",
     ")\n",
     "\n",
@@ -83,8 +83,11 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "blob_path = Path(\"./migration.blob\")\n",
-    "yaml_path = Path(\"migration.yaml\")\n",
+    "migration_data_dir = Path(os.getenv(\"MIGRATION_DATA_DIR\", \".\"))\n",
+    "migration_data_dir.mkdir(exist_ok=True)\n",
+    "\n",
+    "blob_path = migration_data_dir / \"migration.blob\"\n",
+    "yaml_path = migration_data_dir / \"migration.yaml\"\n",
     "\n",
     "blob_path.unlink(missing_ok=True)\n",
     "yaml_path.unlink(missing_ok=True)"
@@ -141,7 +144,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.12.2"
+   "version": "3.10.13"
   }
  },
  "nbformat": 4,

notebooks/tutorials/version-upgrades/2-migrate-from-file.ipynb

@@ -8,6 +8,7 @@
    "outputs": [],
    "source": [
     "# stdlib\n",
+    "import os\n",
     "from pathlib import Path\n",
     "\n",
     "# syft absolute\n",
@@ -70,16 +71,37 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "blob_path = Path(\"./migration.blob\")\n",
-    "print(f\"Loading migration data from {str(blob_path.resolve())}\")\n",
+    "migration_data_dir = Path(os.getenv(\"MIGRATION_DATA_DIR\", \".\"))\n",
+    "blob_path = migration_data_dir / \"migration.blob\"\n",
+    "yaml_path = migration_data_dir / \"migration.yaml\"\n",
     "\n",
+    "print(f\"Loading migration data from {str(blob_path.resolve())}\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "6",
+   "metadata": {},
+   "outputs": [],
+   "source": [
     "res = client.load_migration_data(blob_path)\n",
-    "assert isinstance(res, sy.SyftSuccess)"
+    "assert isinstance(res, sy.SyftSuccess), res.message"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "7",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "res"
    ]
   },
   {
    "cell_type": "markdown",
-   "id": "6",
+   "id": "8",
    "metadata": {},
    "source": [
     "# Post migration tests"
@@ -88,7 +110,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "7",
+   "id": "9",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -98,7 +120,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "8",
+   "id": "10",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -108,15 +130,15 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "9",
+   "id": "11",
    "metadata": {},
    "outputs": [],
    "source": []
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "10",
+   "id": "12",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -127,7 +149,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "11",
+   "id": "13",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -137,7 +159,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "12",
+   "id": "14",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -147,7 +169,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "13",
+   "id": "15",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -157,7 +179,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "14",
+   "id": "16",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -171,7 +193,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "15",
+   "id": "17",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -182,7 +204,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "16",
+   "id": "18",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -193,7 +215,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "17",
+   "id": "19",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -203,7 +225,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "18",
+   "id": "20",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -213,7 +235,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "19",
+   "id": "21",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -223,7 +245,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "20",
+   "id": "22",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -233,7 +255,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "21",
+   "id": "23",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -243,7 +265,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "22",
+   "id": "24",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -254,7 +276,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "23",
+   "id": "25",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -265,7 +287,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "24",
+   "id": "26",
    "metadata": {},
    "outputs": [],
    "source": []
@@ -287,7 +309,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.12.2"
+   "version": "3.10.13"
   }
  },
  "nbformat": 4,

packages/syft/src/syft/service/migration/migration_service.py

@@ -14,6 +14,7 @@
 from ...store.document_store import StorePartition
 from ...types.blob_storage import BlobStorageEntry
 from ...types.syft_object import SyftObject
+from ...types.syft_object_registry import SyftObjectRegistry
 from ..action.action_object import Action
 from ..action.action_object import ActionObject
 from ..action.action_permissions import ActionObjectPermission
@@ -435,11 +436,15 @@ def _migrate_objects(
         migrated_objects = []
         for klass, objects in migration_objects.items():
             canonical_name = klass.__canonical_name__
+            latest_version = SyftObjectRegistry.get_latest_version(canonical_name)
+
             # Migrate data for objects in document store
-            print(f"Migrating data for: {canonical_name} table.")
+            print(
+                f"Migrating data for: {canonical_name} table to version {latest_version}"
+            )
             for object in objects:
                 try:
-                    migrated_value = object.migrate_to(klass.__version__, context)
+                    migrated_value = object.migrate_to(latest_version, context)
                     migrated_objects.append(migrated_value)
                 except Exception:
                     # stdlib

packages/syft/src/syft/service/migration/object_migration_state.py

@@ -164,7 +164,7 @@ def make_migration_config(self) -> dict[str, Any]:
     def from_file(self, path: str | Path) -> Self | SyftError:
         path = Path(path)
         if not path.exists():
-            return SyftError(f"File {str(path)} does not exist.")
+            return SyftError(message=f"File {str(path)} does not exist.")
 
         with open(path, "rb") as f:
             res: MigrationData = _deserialize(f.read(), from_bytes=True)

packages/syft/src/syft/types/syft_object_registry.py

@@ -38,6 +38,13 @@ def get_versions(cls, canonical_name: str) -> list[int]:
         )
         return list(available_versions.keys())
 
+    @classmethod
+    def get_latest_version(cls, canonical_name: str) -> int:
+        available_versions = cls.get_versions(canonical_name)
+        if not available_versions:
+            return 0
+        return sorted(available_versions, reverse=True)[0]
+
     @classmethod
     def get_identifier_for_type(cls, obj: Any) -> tuple[str, int]:
         """

scripts/latest_pypi_version.py

@@ -0,0 +1,14 @@
+# third party
+import requests
+
+PROJECT_NAME = "syft"
+PYPI_JSON_URL = f"https://pypi.org/pypi/{PROJECT_NAME}/json"
+
+
+def get_latest_pypi_version():
+    response = requests.get(PYPI_JSON_URL)
+    data = response.json()
+    return data["info"]["version"]
+
+
+print(get_latest_pypi_version())