Add an IntrospectionMode enum

Extra argument for `AddExternalRepositoriesRequest` to determine which
tables to reintrospect when running `sgr cloud add` (by default, reintrospect
tables without a schema).

Author: mildbyte

splitgraph/cloud/__init__.py

@@ -36,6 +36,7 @@
     ExportJobStatus,
     ExternalResponse,
     IngestionJobStatus,
+    IntrospectionMode,
     ListExternalCredentialsResponse,
     MetadataResponse,
     Plugin,
@@ -530,8 +531,14 @@ def ensure_external_credential(
                 raise JSONSchemaValidationError(message="[MASKED]")
             raise
 
-    def bulk_upsert_external(self, repositories: List[AddExternalRepositoryRequest]):
-        request = AddExternalRepositoriesRequest(repositories=repositories)
+    def bulk_upsert_external(
+        self,
+        repositories: List[AddExternalRepositoryRequest],
+        introspection_mode: IntrospectionMode = IntrospectionMode.EMPTY,
+    ):
+        request = AddExternalRepositoriesRequest(
+            repositories=repositories, introspection_mode=introspection_mode
+        )
         self._perform_request(
             "/bulk-add",
             self.access_token,

splitgraph/cloud/models.py

@@ -1,6 +1,7 @@
 """
 Definitions for responses from the cloud GQL/REST APIs
 """
+import enum
 import logging
 from datetime import datetime
 from typing import Any, Dict, List, Optional
@@ -30,6 +31,16 @@ class Plugin(BaseModel):
     supports_sync: bool
 
 
+class IntrospectionMode(str, enum.Enum):
+    """
+    Which tables to (re)introspect when adding an external.
+    """
+
+    NONE = "none"  # Don't reintrospect any tables
+    EMPTY = "empty"  # Introspect tables with an empty schema
+    ALL = "all"  # Reintrospect all tables
+
+
 # GQL response for the catalog metadata
 
 
@@ -289,3 +300,4 @@ def from_external(
 
 class AddExternalRepositoriesRequest(BaseModel):
     repositories: List[AddExternalRepositoryRequest]
+    introspection_mode: IntrospectionMode = IntrospectionMode.EMPTY

splitgraph/commandline/cloud.py

@@ -15,7 +15,7 @@
 import click
 from click import wrap_text
 
-from splitgraph.cloud.models import AddExternalRepositoryRequest
+from splitgraph.cloud.models import AddExternalRepositoryRequest, IntrospectionMode
 from splitgraph.cloud.project.models import Metadata, SplitgraphYAML
 from splitgraph.commandline.common import (
     ImageType,
@@ -613,9 +613,22 @@ def dump_c(remote, readme_dir, repositories_file, limit_repositories):
     is_flag=True,
     help="Only set up the metadata, not the external data source settings",
 )
+@click.option(
+    "--introspection-mode",
+    type=click.Choice(IntrospectionMode),
+    default=IntrospectionMode.EMPTY,
+    help="Whether to reintrospect tables. none: never reintrospect. all: reintrospect all tables. "
+    "empty: only reintrospect tables with an empty schema.",
+)
 @click.argument("limit_repositories", type=str, nargs=-1)
 def load_c(
-    remote, readme_dir, skip_external, initial_private, repositories_file, limit_repositories
+    remote,
+    readme_dir,
+    skip_external,
+    initial_private,
+    repositories_file,
+    limit_repositories,
+    introspection_mode,
 ):
     """
     Load a Splitgraph catalog from a YAML file.
@@ -666,7 +679,9 @@ def load_c(
                     initial_private=initial_private,
                 )
                 external_repositories.append(external_repository)
-        rest_client.bulk_upsert_external(repositories=external_repositories)
+        rest_client.bulk_upsert_external(
+            repositories=external_repositories, introspection_mode=introspection_mode
+        )
         logging.info(f"Uploaded images for {pluralise('repository', len(external_repositories))}")
 
     logging.info("Updating metadata...")

test/splitgraph/commandline/http_fixtures.py

@@ -404,6 +404,7 @@ def cb(request, uri, response_headers):
         data = json.loads(request.body)
 
         assert data["repositories"] is not None
+        assert data["introspection_mode"] == "empty"
         assert data["repositories"] == [
             {
                 "credential_id": "98765432-aaaa-bbbb-a456-000000000000",