fix: convert empty strings to NULL when transforming text columns to integer/float (#488)

Author: SAY-5

sqlite_utils/db.py

@@ -2004,14 +2004,23 @@ def transform_sql(
         current_column_pairs = list(self.columns_dict.items())
         new_column_pairs = []
         copy_from_to = {column: column for column, _ in current_column_pairs}
-        for name, type_ in current_column_pairs:
-            type_ = types.get(name) or type_
+        # Columns whose type is being changed from text to integer/float; their
+        # empty-string values should be copied as NULL (issue #488).
+        nullif_empty_columns = set()
+        for name, old_type in current_column_pairs:
+            type_ = types.get(name) or old_type
             if name in drop:
                 del [copy_from_to[name]]
                 continue
             new_name = rename.get(name) or name
             new_column_pairs.append((new_name, type_))
             copy_from_to[name] = new_name
+            if (
+                name in types
+                and COLUMN_TYPE_MAPPING.get(old_type) == "TEXT"
+                and COLUMN_TYPE_MAPPING.get(type_) in ("INTEGER", "REAL", "FLOAT")
+            ):
+                nullif_empty_columns.add(name)
 
         if pk is DEFAULT:
             pks_renamed = tuple(
@@ -2084,10 +2093,16 @@ def transform_sql(
         if "rowid" not in new_cols:
             new_cols.insert(0, "rowid")
             old_cols.insert(0, "rowid")
+        select_exprs = []
+        for col in old_cols:
+            if col in nullif_empty_columns:
+                select_exprs.append("NULLIF({}, '')".format(quote_identifier(col)))
+            else:
+                select_exprs.append(quote_identifier(col))
         copy_sql = "INSERT INTO {} ({new_cols})\n   SELECT {old_cols} FROM {};".format(
             quote_identifier(new_table_name),
             quote_identifier(self.name),
-            old_cols=", ".join(quote_identifier(col) for col in old_cols),
+            old_cols=", ".join(select_exprs),
             new_cols=", ".join(quote_identifier(col) for col in new_cols),
         )
         sqls.append(copy_sql)

tests/test_transform.py

@@ -21,7 +21,7 @@
             {"types": {"age": int}},
             [
                 'CREATE TABLE "dogs_new_suffix" (\n   "id" INTEGER PRIMARY KEY,\n   "name" TEXT,\n   "age" INTEGER\n);',
-                'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "age")\n   SELECT "rowid", "id", "name", "age" FROM "dogs";',
+                'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "age")\n   SELECT "rowid", "id", "name", NULLIF("age", \'\') FROM "dogs";',
                 'DROP TABLE "dogs";',
                 'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";',
             ],
@@ -51,7 +51,7 @@
             {"types": {"age": int}, "rename": {"age": "dog_age"}},
             [
                 'CREATE TABLE "dogs_new_suffix" (\n   "id" INTEGER PRIMARY KEY,\n   "name" TEXT,\n   "dog_age" INTEGER\n);',
-                'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "dog_age")\n   SELECT "rowid", "id", "name", "age" FROM "dogs";',
+                'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "dog_age")\n   SELECT "rowid", "id", "name", NULLIF("age", \'\') FROM "dogs";',
                 'DROP TABLE "dogs";',
                 'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";',
             ],
@@ -144,7 +144,7 @@ def tracer(sql, params):
             {"types": {"age": int}},
             [
                 'CREATE TABLE "dogs_new_suffix" (\n   "id" INTEGER,\n   "name" TEXT,\n   "age" INTEGER\n);',
-                'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "age")\n   SELECT "rowid", "id", "name", "age" FROM "dogs";',
+                'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "age")\n   SELECT "rowid", "id", "name", NULLIF("age", \'\') FROM "dogs";',
                 'DROP TABLE "dogs";',
                 'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";',
             ],
@@ -659,3 +659,32 @@ def test_transform_with_unique_constraint_implicit_index(fresh_db):
         "You must manually drop this index prior to running this transformation and manually recreate the new index after running this transformation."
         in str(excinfo.value)
     )
+
+
+def test_transform_empty_string_to_null_for_numeric_types(fresh_db):
+    # Issue #488: converting a text column to integer/float should turn
+    # empty-string values into NULLs instead of leaving them as ''.
+    rows = fresh_db["rows"]
+    rows.insert_all(
+        [
+            {"id": 1, "weight": "12.5", "count": "3"},
+            {"id": 2, "weight": "", "count": ""},
+            {"id": 3, "weight": "0", "count": "0"},
+        ],
+        pk="id",
+    )
+    rows.transform(types={"weight": float, "count": int})
+    assert rows.columns_dict == {"id": int, "weight": float, "count": int}
+    assert list(rows.rows) == [
+        {"id": 1, "weight": 12.5, "count": 3},
+        {"id": 2, "weight": None, "count": None},
+        {"id": 3, "weight": 0.0, "count": 0},
+    ]
+
+
+def test_transform_does_not_nullify_text_columns(fresh_db):
+    # Empty strings in text columns must be preserved (issue #488).
+    rows = fresh_db["rows"]
+    rows.insert_all([{"id": 1, "name": ""}, {"id": 2, "name": "x"}], pk="id")
+    rows.transform(types={"name": str})
+    assert list(rows.rows) == [{"id": 1, "name": ""}, {"id": 2, "name": "x"}]