fix(schema): widen MusicBrainz Discogs ID columns to BIGINT (#376)

brainztableinator throws continuous PostgreSQL "integer out of range"
errors when processing MusicBrainz releases: discogs_release_id (and its
siblings discogs_artist_id, discogs_label_id, discogs_master_id) were
declared INTEGER (int4, max 2,147,483,647), but the extractor parses
Discogs IDs as i64 and release IDs now exceed 2.1B.

Fix both fresh installs and existing databases:
- Change the four cross-reference columns to BIGINT in the CREATE TABLE
  definitions (fresh installs).
- Add idempotent `ALTER COLUMN ... TYPE BIGINT` migrations, since
  CREATE TABLE IF NOT EXISTS is a no-op on pre-existing tables.

Neo4j (brainzgraphinator) is unaffected — Neo4j integers are 64-bit.

Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: SimplicityGuy

brainztableinator/README.md

@@ -80,7 +80,7 @@ CREATE TABLE musicbrainz.artists (
     begin_area         TEXT,
     end_area           TEXT,
     disambiguation     TEXT,
-    discogs_artist_id  INTEGER,
+    discogs_artist_id  BIGINT,
     aliases            JSONB,
     tags               JSONB,
     data               JSONB,
@@ -102,7 +102,7 @@ CREATE TABLE musicbrainz.labels (
     ended              BOOLEAN DEFAULT FALSE,
     area               TEXT,
     disambiguation     TEXT,
-    discogs_label_id   INTEGER,
+    discogs_label_id   BIGINT,
     data               JSONB,
     created_at         TIMESTAMPTZ DEFAULT NOW(),
     updated_at         TIMESTAMPTZ DEFAULT NOW()
@@ -118,7 +118,7 @@ CREATE TABLE musicbrainz.releases (
     barcode            TEXT,
     status             TEXT,
     release_group_mbid UUID,
-    discogs_release_id INTEGER,
+    discogs_release_id BIGINT,
     data               JSONB,
     created_at         TIMESTAMPTZ DEFAULT NOW(),
     updated_at         TIMESTAMPTZ DEFAULT NOW()
@@ -135,7 +135,7 @@ CREATE TABLE musicbrainz.release_groups (
     secondary_types     JSONB,
     first_release_date  TEXT,
     disambiguation      TEXT,
-    discogs_master_id   INTEGER,
+    discogs_master_id   BIGINT,
     data                JSONB,
     created_at          TIMESTAMPTZ DEFAULT NOW(),
     updated_at          TIMESTAMPTZ DEFAULT NOW()

schema-init/postgres_schema.py

@@ -539,7 +539,7 @@
             begin_area TEXT,
             end_area TEXT,
             disambiguation TEXT,
-            discogs_artist_id INTEGER,
+            discogs_artist_id BIGINT,
             aliases JSONB,
             tags JSONB,
             data JSONB,
@@ -559,7 +559,7 @@
             ended BOOLEAN DEFAULT FALSE,
             area TEXT,
             disambiguation TEXT,
-            discogs_label_id INTEGER,
+            discogs_label_id BIGINT,
             data JSONB,
             created_at TIMESTAMPTZ DEFAULT NOW(),
             updated_at TIMESTAMPTZ DEFAULT NOW()
@@ -573,7 +573,7 @@
             barcode TEXT,
             status TEXT,
             release_group_mbid UUID,
-            discogs_release_id INTEGER,
+            discogs_release_id BIGINT,
             data JSONB,
             created_at TIMESTAMPTZ DEFAULT NOW(),
             updated_at TIMESTAMPTZ DEFAULT NOW()
@@ -588,7 +588,7 @@
             secondary_types JSONB,
             first_release_date TEXT,
             disambiguation TEXT,
-            discogs_master_id INTEGER,
+            discogs_master_id BIGINT,
             data JSONB,
             created_at TIMESTAMPTZ DEFAULT NOW(),
             updated_at TIMESTAMPTZ DEFAULT NOW()
@@ -623,6 +623,27 @@
             UNIQUE (mbid, entity_type, service_name, url)
         )""",
     ),
+    # Widen Discogs cross-reference IDs from INTEGER (int4, max 2,147,483,647)
+    # to BIGINT. Discogs IDs are emitted as i64 by the extractor and release IDs
+    # now exceed 2.1B, causing "integer out of range" on INSERT. CREATE TABLE
+    # IF NOT EXISTS above won't alter pre-existing tables, so widen explicitly.
+    # ALTER COLUMN ... TYPE BIGINT is a no-op when the column is already BIGINT.
+    (
+        "musicbrainz.artists.discogs_artist_id widen to BIGINT",
+        "ALTER TABLE musicbrainz.artists ALTER COLUMN discogs_artist_id TYPE BIGINT",
+    ),
+    (
+        "musicbrainz.labels.discogs_label_id widen to BIGINT",
+        "ALTER TABLE musicbrainz.labels ALTER COLUMN discogs_label_id TYPE BIGINT",
+    ),
+    (
+        "musicbrainz.releases.discogs_release_id widen to BIGINT",
+        "ALTER TABLE musicbrainz.releases ALTER COLUMN discogs_release_id TYPE BIGINT",
+    ),
+    (
+        "musicbrainz.release_groups.discogs_master_id widen to BIGINT",
+        "ALTER TABLE musicbrainz.release_groups ALTER COLUMN discogs_master_id TYPE BIGINT",
+    ),
 ]
 
 

tests/schema-init/test_musicbrainz_schema.py

@@ -14,11 +14,40 @@ def test_musicbrainz_tables_defined():
 
 
 def test_musicbrainz_tables_use_if_not_exists():
+    # ALTER migrations (e.g. widening columns to BIGINT) cannot use IF NOT EXISTS;
+    # only CREATE TABLE statements must be idempotent via IF NOT EXISTS.
     for name, sql in _MUSICBRAINZ_TABLES:
-        if name != "musicbrainz schema":
+        if "CREATE TABLE" in sql:
             assert "IF NOT EXISTS" in sql, f"{name} missing IF NOT EXISTS"
 
 
+def test_musicbrainz_discogs_id_columns_are_bigint():
+    # Discogs IDs are i64 from the extractor and now exceed int4 range (2.1B);
+    # cross-reference columns must be BIGINT to avoid "integer out of range".
+    expected = {
+        "musicbrainz.artists table": "discogs_artist_id BIGINT",
+        "musicbrainz.labels table": "discogs_label_id BIGINT",
+        "musicbrainz.releases table": "discogs_release_id BIGINT",
+        "musicbrainz.release_groups table": "discogs_master_id BIGINT",
+    }
+    found = dict.fromkeys(expected, False)
+    for name, sql in _MUSICBRAINZ_TABLES:
+        if name in expected:
+            assert expected[name] in sql, f"{name} missing {expected[name]}"
+            found[name] = True
+    assert all(found.values()), f"missing tables: {[n for n, ok in found.items() if not ok]}"
+
+
+def test_musicbrainz_has_bigint_widening_migrations():
+    # Existing databases need explicit ALTER COLUMN ... TYPE BIGINT; CREATE TABLE
+    # IF NOT EXISTS is a no-op on tables that already exist as INTEGER.
+    migrations = [sql for name, sql in _MUSICBRAINZ_TABLES if "ALTER COLUMN" in sql]
+    assert any("musicbrainz.artists ALTER COLUMN discogs_artist_id TYPE BIGINT" in m for m in migrations)
+    assert any("musicbrainz.labels ALTER COLUMN discogs_label_id TYPE BIGINT" in m for m in migrations)
+    assert any("musicbrainz.releases ALTER COLUMN discogs_release_id TYPE BIGINT" in m for m in migrations)
+    assert any("musicbrainz.release_groups ALTER COLUMN discogs_master_id TYPE BIGINT" in m for m in migrations)
+
+
 def test_musicbrainz_indexes_defined():
     index_names = [name for name, _ in _MUSICBRAINZ_INDEXES]
     assert "idx_mb_artists_discogs_id" in index_names

tests/schema-init/test_postgres_schema.py

@@ -232,6 +232,10 @@ async def capture(stmt: Any, *_: Any, **__: Any) -> None:
 
         for stmt in captured:
             upper = stmt.upper()
+            # ALTER COLUMN ... TYPE is inherently idempotent (re-applying the same
+            # target type is a no-op), so it needs no IF NOT EXISTS guard.
+            if "ALTER COLUMN" in upper and "TYPE " in upper:
+                continue
             assert "IF NOT EXISTS" in upper, f"Statement is not idempotent: {stmt[:80]}..."
             # A multi-statement blob could still hide an un-guarded DROP that would
             # not be idempotent — any DROP must be guarded with IF EXISTS.