db[table].create(..., transform=True) and create-table --transform

Closes #467

Author: simonw

docs/cli-reference.rst

@@ -865,6 +865,7 @@ See :ref:`cli_create_table`.
                                 foreign key
       --ignore                  If table already exists, do nothing
       --replace                 If table already exists, replace it
+      --transform               If table already exists, try to transform the schema
       --load-extension TEXT     Path to SQLite extension, with optional :entrypoint
       -h, --help                Show this message and exit.
 

docs/cli.rst

@@ -1520,6 +1520,8 @@ You can specify foreign key relationships between the tables you are creating us
 
 If a table with the same name already exists, you will get an error. You can choose to silently ignore this error with ``--ignore``, or you can replace the existing table with a new, empty table using ``--replace``.
 
+You can also pass ``--transform`` to transform the existing table to match the new schema. See :ref:`python_api_explicit_create` in the Python library documentation for details of how this option works.
+
 .. _cli_duplicate_table:
 
 Duplicating tables

docs/python-api.rst

@@ -554,6 +554,25 @@ To do nothing if the table already exists, add ``if_not_exists=True``:
         "name": str,
     }, pk="id", if_not_exists=True)
 
+You can also pass ``transform=True`` to have any existing tables :ref:`transformed <python_api_transform>` to match your new table specification. This is a **dangerous operation** as it may drop columns that are no longer listed in your call to ``.create()``, so be careful when running this.
+
+.. code-block:: python
+
+    db["cats"].create({
+        "id": int,
+        "name": str,
+        "weight": float,
+    }, pk="id", transform=True)
+
+The ``transform=True`` option will update the table schema if any of the following have changed:
+
+- The specified columns or their types
+- The specified primary key
+- The order of the columns, defined using ``column_order=``
+- The ``not_null=`` or ``defaults=`` arguments
+
+Changes to ``foreign_keys=`` are not currently detected and applied by ``transform=True``.
+
 .. _python_api_compound_primary_keys:
 
 Compound primary keys

sqlite_utils/cli.py

@@ -1453,9 +1453,24 @@ def create_database(path, enable_wal, init_spatialite, load_extension):
     is_flag=True,
     help="If table already exists, replace it",
 )
+@click.option(
+    "--transform",
+    is_flag=True,
+    help="If table already exists, try to transform the schema",
+)
 @load_extension_option
 def create_table(
-    path, table, columns, pk, not_null, default, fk, ignore, replace, load_extension
+    path,
+    table,
+    columns,
+    pk,
+    not_null,
+    default,
+    fk,
+    ignore,
+    replace,
+    transform,
+    load_extension,
 ):
     """
     Add a table with the specified columns. Columns should be specified using
@@ -1490,14 +1505,21 @@ def create_table(
             return
         elif replace:
             db[table].drop()
+        elif transform:
+            pass
         else:
             raise click.ClickException(
                 'Table "{}" already exists. Use --replace to delete and replace it.'.format(
                     table
                 )
             )
     db[table].create(
-        coltypes, pk=pk, not_null=not_null, defaults=dict(default), foreign_keys=fk
+        coltypes,
+        pk=pk,
+        not_null=not_null,
+        defaults=dict(default),
+        foreign_keys=fk,
+        transform=transform,
     )
 
 

sqlite_utils/db.py

@@ -34,7 +34,6 @@
     Union,
     Optional,
     List,
-    Set,
     Tuple,
 )
 import uuid
@@ -876,6 +875,7 @@ def create_table(
         hash_id_columns: Optional[Iterable[str]] = None,
         extracts: Optional[Union[Dict[str, str], List[str]]] = None,
         if_not_exists: bool = False,
+        transform: bool = False,
     ) -> "Table":
         """
         Create a table with the specified name and the specified ``{column_name: type}`` columns.
@@ -893,7 +893,61 @@ def create_table(
         :param hash_id_columns: List of columns to be used when calculating the hash ID for a row
         :param extracts: List or dictionary of columns to be extracted during inserts, see :ref:`python_api_extracts`
         :param if_not_exists: Use ``CREATE TABLE IF NOT EXISTS``
-        """
+        :param transform: If table already exists transform it to fit the specified schema
+        """
+        # Transform table to match the new definition if table already exists:
+        if transform and self[name].exists():
+            table = cast(Table, self[name])
+            should_transform = False
+            # First add missing columns and figure out columns to drop
+            existing_columns = table.columns_dict
+            missing_columns = dict(
+                (col_name, col_type)
+                for col_name, col_type in columns.items()
+                if col_name not in existing_columns
+            )
+            columns_to_drop = [
+                column for column in existing_columns if column not in columns
+            ]
+            if missing_columns:
+                for col_name, col_type in missing_columns.items():
+                    table.add_column(col_name, col_type)
+            if missing_columns or columns_to_drop or columns != existing_columns:
+                should_transform = True
+            # Do we need to change the column order?
+            if (
+                column_order
+                and list(existing_columns)[: len(column_order)] != column_order
+            ):
+                should_transform = True
+            # Has the primary key changed?
+            current_pks = table.pks
+            desired_pk = None
+            if isinstance(pk, str):
+                desired_pk = [pk]
+            elif pk:
+                desired_pk = list(pk)
+            if desired_pk and current_pks != desired_pk:
+                should_transform = True
+            # Any not-null changes?
+            current_not_null = {c.name for c in table.columns if c.notnull}
+            desired_not_null = set(not_null) if not_null else set()
+            if current_not_null != desired_not_null:
+                should_transform = True
+            # How about defaults?
+            if defaults and defaults != table.default_values:
+                should_transform = True
+            # Only run .transform() if there is something to do
+            if should_transform:
+                table.transform(
+                    types=columns,
+                    drop=columns_to_drop,
+                    column_order=column_order,
+                    not_null=not_null,
+                    defaults=defaults,
+                    pk=pk,
+                )
+            return table
         sql = self.create_table_sql(
             name=name,
             columns=columns,
@@ -908,7 +962,7 @@ def create_table(
             if_not_exists=if_not_exists,
         )
         self.execute(sql)
-        table = self.table(
+        created_table = self.table(
             name,
             pk=pk,
             foreign_keys=foreign_keys,
@@ -918,7 +972,7 @@ def create_table(
             hash_id=hash_id,
             hash_id_columns=hash_id_columns,
         )
-        return cast(Table, table)
+        return cast(Table, created_table)
 
     def create_view(
         self, name: str, sql: str, ignore: bool = False, replace: bool = False
@@ -1487,6 +1541,7 @@ def create(
         hash_id_columns: Optional[Iterable[str]] = None,
         extracts: Optional[Union[Dict[str, str], List[str]]] = None,
         if_not_exists: bool = False,
+        transform: bool = False,
     ) -> "Table":
         """
         Create a table with the specified columns.
@@ -1518,6 +1573,7 @@ def create(
                 hash_id_columns=hash_id_columns,
                 extracts=extracts,
                 if_not_exists=if_not_exists,
+                transform=transform,
             )
         return self
 
@@ -1544,7 +1600,7 @@ def transform(
         rename: Optional[dict] = None,
         drop: Optional[Iterable] = None,
         pk: Optional[Any] = DEFAULT,
-        not_null: Optional[Set[str]] = None,
+        not_null: Optional[Iterable[str]] = None,
         defaults: Optional[Dict[str, Any]] = None,
         drop_foreign_keys: Optional[Iterable] = None,
         column_order: Optional[List[str]] = None,
@@ -1664,10 +1720,12 @@ def transform_sql(
                     create_table_not_null.add(key)
         elif isinstance(not_null, set):
             create_table_not_null.update((rename.get(k) or k) for k in not_null)
-        elif not_null is None:
+        elif not not_null:
             pass
         else:
-            assert False, "not_null must be a dict or a set or None"
+            assert False, "not_null must be a dict or a set or None, it was {}".format(
+                repr(not_null)
+            )
         # defaults=
         create_table_defaults = {
             (rename.get(c.name) or c.name): c.default_value
@@ -2861,7 +2919,7 @@ def insert(
         pk=DEFAULT,
         foreign_keys=DEFAULT,
         column_order: Optional[Union[List[str], Default]] = DEFAULT,
-        not_null: Optional[Union[Set[str], Default]] = DEFAULT,
+        not_null: Optional[Union[Iterable[str], Default]] = DEFAULT,
         defaults: Optional[Union[Dict[str, Any], Default]] = DEFAULT,
         hash_id: Optional[Union[str, Default]] = DEFAULT,
         hash_id_columns: Optional[Union[Iterable[str], Default]] = DEFAULT,
@@ -3141,7 +3199,7 @@ def lookup(
         pk: Optional[str] = "id",
         foreign_keys: Optional[ForeignKeysType] = None,
         column_order: Optional[List[str]] = None,
-        not_null: Optional[Set[str]] = None,
+        not_null: Optional[Iterable[str]] = None,
         defaults: Optional[Dict[str, Any]] = None,
         extracts: Optional[Union[Dict[str, str], List[str]]] = None,
         conversions: Optional[Dict[str, str]] = None,

tests/test_create.py

@@ -1155,3 +1155,79 @@ def test_create_if_no_columns(fresh_db):
     with pytest.raises(AssertionError) as error:
         fresh_db["t"].create({})
     assert error.value.args[0] == "Tables must have at least one column"
+
+
+@pytest.mark.parametrize(
+    "cols,kwargs,expected_schema,should_transform",
+    (
+        # Change nothing
+        (
+            {"id": int, "name": str},
+            {"pk": "id"},
+            "CREATE TABLE [demo] (\n   [id] INTEGER PRIMARY KEY,\n   [name] TEXT\n)",
+            False,
+        ),
+        # Drop name column, remove primary key
+        ({"id": int}, {}, 'CREATE TABLE "demo" (\n   [id] INTEGER\n)', True),
+        # Add a new column
+        (
+            {"id": int, "name": str, "age": int},
+            {"pk": "id"},
+            'CREATE TABLE "demo" (\n   [id] INTEGER PRIMARY KEY,\n   [name] TEXT,\n   [age] INTEGER\n)',
+            True,
+        ),
+        # Change a column type
+        (
+            {"id": int, "name": bytes},
+            {"pk": "id"},
+            'CREATE TABLE "demo" (\n   [id] INTEGER PRIMARY KEY,\n   [name] BLOB\n)',
+            True,
+        ),
+        # Change the primary key
+        (
+            {"id": int, "name": str},
+            {"pk": "name"},
+            'CREATE TABLE "demo" (\n   [id] INTEGER,\n   [name] TEXT PRIMARY KEY\n)',
+            True,
+        ),
+        # Change in column order
+        (
+            {"id": int, "name": str},
+            {"pk": "id", "column_order": ["name"]},
+            'CREATE TABLE "demo" (\n   [name] TEXT,\n   [id] INTEGER PRIMARY KEY\n)',
+            True,
+        ),
+        # Same column order is ignored
+        (
+            {"id": int, "name": str},
+            {"pk": "id", "column_order": ["id", "name"]},
+            "CREATE TABLE [demo] (\n   [id] INTEGER PRIMARY KEY,\n   [name] TEXT\n)",
+            False,
+        ),
+        # Change not null
+        (
+            {"id": int, "name": str},
+            {"pk": "id", "not_null": {"name"}},
+            'CREATE TABLE "demo" (\n   [id] INTEGER PRIMARY KEY,\n   [name] TEXT NOT NULL\n)',
+            True,
+        ),
+        # Change default values
+        (
+            {"id": int, "name": str},
+            {"pk": "id", "defaults": {"id": 0, "name": "Bob"}},
+            "CREATE TABLE \"demo\" (\n   [id] INTEGER PRIMARY KEY DEFAULT 0,\n   [name] TEXT DEFAULT 'Bob'\n)",
+            True,
+        ),
+    ),
+)
+def test_create_transform(fresh_db, cols, kwargs, expected_schema, should_transform):
+    fresh_db.create_table("demo", {"id": int, "name": str}, pk="id")
+    fresh_db["demo"].insert({"id": 1, "name": "Cleo"})
+    traces = []
+    with fresh_db.tracer(lambda sql, parameters: traces.append((sql, parameters))):
+        fresh_db["demo"].create(cols, **kwargs, transform=True)
+    at_least_one_create_table = any(sql.startswith("CREATE TABLE") for sql, _ in traces)
+    assert should_transform == at_least_one_create_table
+    new_schema = fresh_db["demo"].schema
+    assert new_schema == expected_schema, repr(new_schema)
+    assert fresh_db["demo"].count == 1