Merge pull request #152 from weaviate/djanicek/incremental-backups

incremental backups support

Author: DanielJanicek

.claude/skills/operating-weaviate-cli/SKILL.md

@@ -209,13 +209,14 @@ See [references/tenants.md](references/tenants.md).
 ```bash
 weaviate-cli create backup --backend s3 --backup_id my-backup --wait --json
 weaviate-cli create backup --backend s3 --backup_id my-backup --include "Movies,Books" --wait --json
+weaviate-cli create backup --backend s3 --backup_id my-incremental --incremental_base_backup_id my-backup --json
 weaviate-cli get backup --backend s3 --backup_id my-backup --json
 weaviate-cli get backup --backend s3 --backup_id my-backup --restore --json
 weaviate-cli restore backup --backend s3 --backup_id my-backup --wait --json
 weaviate-cli cancel backup --backend s3 --backup_id my-backup --json
 ```
 
-Backends: `s3`, `gcs`, `filesystem`. Options: `--include`, `--exclude`, `--wait`, `--cpu_for_backup N`, `--override-alias`
+Backends: `s3`, `gcs`, `filesystem`. Options: `--include`, `--exclude`, `--wait`, `--cpu_for_backup N`, `--override-alias`, `--incremental_base_backup_id`
 
 See [references/backups.md](references/backups.md).
 

.claude/skills/operating-weaviate-cli/references/backups.md

@@ -7,6 +7,7 @@ Create, inspect, restore, and cancel Weaviate backups.
 weaviate-cli create backup --backend s3 --backup_id my-backup --wait --json
 weaviate-cli create backup --backend s3 --backup_id my-backup --include "Movies,Books" --wait --json
 weaviate-cli create backup --backend gcs --backup_id my-backup --exclude "TempData" --cpu_for_backup 60 --json
+weaviate-cli create backup --backend s3 --backup_id my-incremental-backup --incremental_base_backup_id my-backup --json
 ```
 
 ## Check Backup Status
@@ -40,6 +41,7 @@ weaviate-cli cancel backup --backend s3 --backup_id my-backup --json
 - `--exclude` -- Comma-separated collections to exclude
 - `--wait` -- Wait for completion
 - `--cpu_for_backup` -- CPU percentage for backup (default: 40)
+- `--incremental_base_backup_id` -- Backup ID of a previous backup to create an incremental backup from. Requires Weaviate 1.34+.
 
 **Restore:**
 - `--backend`, `--backup_id` -- Same as create
@@ -59,3 +61,4 @@ weaviate-cli cancel backup --backend s3 --backup_id my-backup --json
 - Without `--wait`, the command returns immediately and you must poll with `get backup`
 - `--cpu_for_backup` controls backup speed vs. resource consumption tradeoff
 - `--include` and `--exclude` are mutually exclusive
+- Incremental backups require Weaviate 1.34+ and can only include collections that were part of the base backup

weaviate_cli/commands/create.py

@@ -406,12 +406,25 @@ def create_tenants_cli(
     default=CreateBackupDefaults.cpu_for_backup,
     help="The percentage of CPU to use for the backup (default: 40). The larger, the faster it will occur, but it will also consume more memory.",
 )
+@click.option(
+    "--incremental_base_backup_id",
+    default=CreateBackupDefaults.incremental_base_backup_id,
+    help="The backup_id of the previous backup to create an incremental backup from.",
+)
 @click.option(
     "--json", "json_output", is_flag=True, default=False, help="Output in JSON format."
 )
 @click.pass_context
 def create_backup_cli(
-    ctx, backend, backup_id, include, exclude, wait, cpu_for_backup, json_output
+    ctx,
+    backend,
+    backup_id,
+    include,
+    exclude,
+    wait,
+    cpu_for_backup,
+    incremental_base_backup_id,
+    json_output,
 ):
     """Create a backup in Weaviate."""
 
@@ -426,6 +439,7 @@ def create_backup_cli(
             exclude=exclude,
             wait=wait,
             cpu_for_backup=cpu_for_backup,
+            incremental_base_backup_id=incremental_base_backup_id,
             json_output=json_output,
         )
     except Exception as e:

weaviate_cli/defaults.py

@@ -103,6 +103,7 @@ class CreateTenantsDefaults:
 class CreateBackupDefaults:
     backend: str = "s3"
     backup_id: str = "test-backup"
+    incremental_base_backup_id: Optional[str] = None
     include: Optional[str] = None
     exclude: Optional[str] = None
     wait: bool = False

weaviate_cli/managers/backup_manager.py

@@ -25,6 +25,9 @@ def create_backup(
         wait: bool = CreateBackupDefaults.wait,
         cpu_for_backup: int = CreateBackupDefaults.cpu_for_backup,
         json_output: bool = False,
+        incremental_base_backup_id: Optional[
+            str
+        ] = CreateBackupDefaults.incremental_base_backup_id,
     ) -> None:
 
         version = semver.Version.parse(self.client.get_meta()["version"])
@@ -44,6 +47,7 @@ def create_backup(
         result = self.client.backup.create(
             backup_id=backup_id,
             backend=backend,
+            incremental_base_backup_id=incremental_base_backup_id,
             include_collections=include.split(",") if include else None,
             exclude_collections=exclude.split(",") if exclude else None,
             wait_for_completion=wait,

weaviate_cli/managers/collection_manager.py

@@ -524,6 +524,7 @@ def create_collection(
             wvc.Property(name="releaseDate", data_type=wvc.DataType.DATE),
             wvc.Property(name="revenue", data_type=wvc.DataType.NUMBER),
             wvc.Property(name="status", data_type=wvc.DataType.TEXT),
+            wvc.Property(name="coverImage", data_type=wvc.DataType.BLOB),
         ]
 
         rds_map = {

weaviate_cli/managers/data_manager.py

@@ -1,3 +1,4 @@
+import base64
 import importlib.resources as resources
 import json
 import math
@@ -205,6 +206,9 @@ def generate_movie_object(is_update: bool = False, seed: Optional[int] = None) -
     # Generate a movie title with proper capitalization
     title = fake.catch_phrase()
 
+    # Generate blob data to simulate an image
+    cover_image = base64.b64encode(fake.binary(10)).decode("utf-8")
+
     # Prefix for update operations
     prefix = "updated-" if is_update else ""
 
@@ -224,6 +228,7 @@ def generate_movie_object(is_update: bool = False, seed: Optional[int] = None) -
         "status": random.choice(STATUSES),
         "spokenLanguages": spoken_languages,
         "productionCountries": production_countries,
+        "coverImage": cover_image,
     }
 
 
@@ -1128,6 +1133,8 @@ def __update_data(
                             obj.properties[property] += 1.0
                         elif isinstance(value, datetime):
                             obj.properties[property] = value + timedelta(days=1)
+                        elif isinstance(value, bytes):
+                            obj.properties[property] = value << 1
 
                     cl_collection.data.update(
                         uuid=obj.uuid,