Merge pull request #1928 from weaviate/incremental_backup_support

Add support for incremental backups

Author: dirkkul

.github/workflows/main.yaml

@@ -27,7 +27,8 @@ env:
   WEAVIATE_133: 1.33.10
   WEAVIATE_134: 1.34.5
   WEAVIATE_135: 1.35.0
-  WEAVIATE_136: 1.36.0-rc.0
+  WEAVIATE_136: 1.36.0
+  WEAVIATE_137: 1.37.0-dev-29d5c87.amd64
 
 
 jobs:
@@ -310,6 +311,7 @@ jobs:
           $WEAVIATE_134,
           $WEAVIATE_135,
           $WEAVIATE_136
+          $WEAVIATE_137
         ]
     steps:
       - name: Checkout

ci/docker-compose.yml

@@ -30,6 +30,8 @@ services:
       DISABLE_TELEMETRY: 'true'
       DISABLE_LAZY_LOAD_SHARDS: 'true'
       GRPC_MAX_MESSAGE_SIZE: 100000000  # 100mb
+      OBJECTS_TTL_DELETE_SCHEDULE: "@every 12h" # for objectTTL tests to work
+
   contextionary:
     environment:
       OCCURRENCE_WEIGHT_LINEAR_FACTOR: 0.75

integration/test_backup_v4.py

@@ -742,6 +742,46 @@ def test_overwrite_alias_true(
     assert literature.collection == "Article", "alias must point to the original collection"
 
 
+def test_incremental_backup(client: weaviate.WeaviateClient, request: SubRequest) -> None:
+    """Create and restore incremental backup."""
+    if client._connection._weaviate_version.is_lower_than(1, 37, 0):
+        pytest.skip("List backups is only supported from 1.37.0")
+
+    backup_id = unique_backup_id(request.node.name)
+    base_backup_id = backup_id + "_base"
+
+    # create base backup
+    client.backup.create(
+        backup_id=base_backup_id,
+        backend=BACKEND,
+        include_collections=["Article"],
+        wait_for_completion=True,
+    )
+
+    # create incremental backup
+    client.backup.create(
+        backup_id=backup_id,
+        backend=BACKEND,
+        include_collections=["Article"],
+        wait_for_completion=True,
+        incremental_backup_base_id=base_backup_id,
+    )
+
+    # remove existing class
+    client.collections.delete("Article")
+
+    # restore incremental backup
+    client.backup.restore(
+        backup_id=backup_id,
+        backend=BACKEND,
+        include_collections=["Article"],
+        wait_for_completion=True,
+    )
+
+    # check data exists again
+    assert len(client.collections.use("Article")) == len(ARTICLES_IDS)
+
+
 # This test has been disabled temporarily until the behaviour of this scenario is clarified.
 # It worked in version 1.33.0-rc.1, but broken in version 1.33.0+
 # def test_overwrite_alias_false(

weaviate/backup/async_.pyi

@@ -20,6 +20,7 @@ class _BackupAsync(_BackupExecutor[ConnectionAsync]):
         backend: BackupStorage,
         include_collections: Union[List[str], str, None] = None,
         exclude_collections: Union[List[str], str, None] = None,
+        incremental_base_backup_id: Optional[str] = None,
         wait_for_completion: bool = False,
         config: Optional[BackupConfigCreate] = None,
         backup_location: Optional[BackupLocationType] = None,

weaviate/backup/executor.py

@@ -47,6 +47,7 @@ def create(
         backend: BackupStorage,
         include_collections: Union[List[str], str, None] = None,
         exclude_collections: Union[List[str], str, None] = None,
+        incremental_base_backup_id: Optional[str] = None,
         wait_for_completion: bool = False,
         config: Optional[BackupConfigCreate] = None,
         backup_location: Optional[BackupLocationType] = None,
@@ -60,6 +61,8 @@ def create(
                 collections will be included. Either `include_collections` or `exclude_collections` can be set. By default None.
             exclude_collections: The collection/list of collections to be excluded in the backup.
                 Either `include_collections` or `exclude_collections` can be set. By default None.
+            incremental_base_backup_id: The identifier name of the base backup for an incremental backup. Files that are identical
+                to the base backup will not be included in the incremental backup and restored from the base. By default None.
             wait_for_completion: Whether to wait until the backup is done. By default False.
             config: The configuration of the backup creation. By default None.
             backup_location: The dynamic location of a backup. By default None.
@@ -89,6 +92,7 @@ def create(
             "id": backup_id,
             "include": include_collections,
             "exclude": exclude_collections,
+            "incremental_base_backup_id": incremental_base_backup_id,
         }
 
         if config is not None:

weaviate/backup/sync.pyi

@@ -20,6 +20,7 @@ class _Backup(_BackupExecutor[ConnectionSync]):
         backend: BackupStorage,
         include_collections: Union[List[str], str, None] = None,
         exclude_collections: Union[List[str], str, None] = None,
+        incremental_base_backup_id: Optional[str] = None,
         wait_for_completion: bool = False,
         config: Optional[BackupConfigCreate] = None,
         backup_location: Optional[BackupLocationType] = None,