Convert us-zipcodes from GeoJSON to TopoJSON for 76% size reduction

- Replace 384 MB us-zipcodes.json with 91 MB us-zipcodes.topojson
- Enables GitHub compatibility (under 100 MB limit)
- Add topojson-client package for client-side conversion
- Update fetchGeoJson to detect and convert TopoJSON to GeoJSON for ECharts
- Update .gitignore to track .topojson instead of .geojson
- Update process-zipcodes.py to generate both formats
- Successfully downloaded all 50 US states (32,936 total ZIP codes)

Compression results:
- Original GeoJSON with simplification: 384 MB
- TopoJSON with mapshaper: 91 MB (76% reduction)
- Individual state files: 1-26 MB each (not committed)

Author: cgivre

.gitignore

@@ -53,11 +53,11 @@ coverage/
 result-cache/
 
 # Large GeoJSON files (for local development only, not committed to git)
-# Exception: us-zipcodes.json (merged all-states) is committed to git (~105 MB)
+# Exception: us-zipcodes.topojson (merged all-states, 91 MB) is committed to git
 # Individual state files (us-al-zipcodes.json, us-ca-zipcodes.json, etc.) are git-ignored
 # These can be regenerated locally using scripts/process-zipcodes.py
 /exec/java-exec/src/main/resources/webapp/public/geojson/us-*-zipcodes.json
-!/exec/java-exec/src/main/resources/webapp/public/geojson/us-zipcodes.json
+/exec/java-exec/src/main/resources/webapp/public/geojson/us-zipcodes.json
+!/exec/java-exec/src/main/resources/webapp/public/geojson/us-zipcodes.topojson
 /exec/java-exec/src/main/resources/webapp/public/geojson/*.zip
 /exec/java-exec/src/main/resources/webapp/public/geojson/cb_*
-/exec/java-exec/src/main/resources/webapp/public/geojson/*-topo.json

exec/java-exec/src/main/resources/webapp/package-lock.json

@@ -37,7 +37,8 @@
     "react-router-dom": "^6.21.0",
     "regression": "^2.0.1",
     "rehype-raw": "^7.0.0",
-    "sql-formatter": "^15.0.2"
+    "sql-formatter": "^15.0.2",
+    "topojson-client": "^3.1.0"
   },
   "devDependencies": {
     "@testing-library/jest-dom": "^6.9.1",

exec/java-exec/src/main/resources/webapp/package.json

@@ -279,6 +279,27 @@ def write_geojson_file(path, features, minify=True):
             # Pretty-printed for debugging
             json.dump(geojson, f, indent=2)
 
+
+def convert_geojson_to_topojson(geojson_path, topojson_path):
+    """
+    Convert GeoJSON to TopoJSON using mapshaper (5-10x compression).
+    TopoJSON uses delta encoding and shared arcs.
+    """
+    try:
+        # Use mapshaper command-line tool
+        # -o format=topojson specifies output format
+        # -o quantization=1e4 sets coordinate precision
+        subprocess.run([
+            'mapshaper', geojson_path,
+            '-o', 'format=topojson',
+            '-o', 'quantization=1e4',
+            topojson_path
+        ], check=True, capture_output=True, text=True)
+        return True
+    except (FileNotFoundError, subprocess.CalledProcessError) as e:
+        # mapshaper not installed, skip TopoJSON conversion
+        return False
+
 def create_state_files(state_groups):
     """Create individual state ZIP code files"""
     print("Creating state-level ZIP code files...")
@@ -323,6 +344,18 @@ def main():
         # Create merged all-states file (committed to git)
         create_merged_file(all_features)
 
+        # Convert to TopoJSON for better compression
+        print("Converting to TopoJSON format...")
+        merged_geojson = GEOJSON_DIR / 'us-zipcodes.json'
+        merged_topojson = GEOJSON_DIR / 'us-zipcodes.topojson'
+        if convert_geojson_to_topojson(str(merged_geojson), str(merged_topojson)):
+            geojson_size = os.path.getsize(merged_geojson) / 1024 / 1024
+            topojson_size = os.path.getsize(merged_topojson) / 1024 / 1024
+            compression = (1 - topojson_size / geojson_size) * 100
+            print(f"  ✓ {merged_topojson.name}: {topojson_size:.1f} MB ({compression:.1f}% smaller)")
+        else:
+            print("  ⚠ mapshaper not found, skipping TopoJSON conversion")
+
         print("\n✓ Processing complete!\n")
         print(f"Summary:")
         print(f"  Total US ZIP codes: {len(all_features):,}")

exec/java-exec/src/main/resources/webapp/public/geojson/us-zipcodes.topojson

@@ -16,55 +16,89 @@
  * limitations under the License.
  */
 
+import * as topojson from 'topojson-client';
+
 /**
  * Fetch GeoJSON for a map by ID from the public/geojson/ directory.
+ * Supports both .geojson and .topojson formats.
+ * TopoJSON is automatically converted to GeoJSON for ECharts compatibility.
  * The base URL is relative to the page location to work in both dev and prod.
  */
 export async function fetchGeoJson(mapId: string): Promise<object> {
   try {
-    // Use relative path that works regardless of deployment location
-    // In dev: served by Vite from public/
-    // In prod: served from webapp resources
-    const url = `/sqllab/geojson/${mapId}.json`;
+    // Try TopoJSON first (more compact), fall back to GeoJSON
+    let data = await tryFetchMap(mapId, '.topojson');
+    if (!data) {
+      data = await tryFetchMap(mapId, '.json');
+    }
+
+    if (!data || typeof data !== 'object') {
+      throw new Error(`Invalid GeoJSON for map ${mapId}`);
+    }
+
+    // If it's TopoJSON, convert to GeoJSON
+    if ('objects' in data) {
+      console.debug(`[GeoJSON] Converting TopoJSON to GeoJSON for ${mapId}`);
+      return convertTopoJsonToGeoJson(data as topojson.Topology);
+    }
+
+    return data;
+  } catch (error) {
+    console.error(`[GeoJSON] Error loading ${mapId}:`, error);
+    throw error;
+  }
+}
 
+async function tryFetchMap(mapId: string, ext: string): Promise<unknown> {
+  try {
+    const url = `/sqllab/geojson/${mapId}${ext}`;
     console.debug(`[GeoJSON] Fetching from: ${url}`);
 
     const response = await fetch(url);
-
     if (!response.ok) {
-      console.error(`[GeoJSON] 404 for ${url} - trying fallback paths`);
-
       // Try alternative paths
       const altPaths = [
-        `/geojson/${mapId}.json`,
-        `./geojson/${mapId}.json`,
+        `/geojson/${mapId}${ext}`,
+        `./geojson/${mapId}${ext}`,
       ];
 
       for (const altPath of altPaths) {
         try {
           const altResponse = await fetch(altPath);
           if (altResponse.ok) {
             console.log(`[GeoJSON] Found at fallback path: ${altPath}`);
-            const data = await altResponse.json();
-            return data;
+            return altResponse.json();
           }
         } catch {
           // Continue to next fallback
         }
       }
-
-      throw new Error(`HTTP ${response.status} - map file not found at ${url}`);
+      return null;
     }
 
-    const data = await response.json();
+    return response.json();
+  } catch (error) {
+    console.debug(`[GeoJSON] Error trying ${ext}:`, error);
+    return null;
+  }
+}
 
-    if (!data || typeof data !== 'object') {
-      throw new Error(`Invalid GeoJSON for map ${mapId}`);
+function convertTopoJsonToGeoJson(
+  topoData: topojson.Topology
+): Record<string, unknown> {
+  try {
+    // Get the first object from the topology (usually 'features' or a similar key)
+    const objectKey = Object.keys(topoData.objects)[0];
+    if (!objectKey) {
+      throw new Error('TopoJSON has no objects');
     }
 
-    return data;
+    // Convert the topology object to GeoJSON FeatureCollection
+    const geoJson = topojson.feature(topoData, topoData.objects[objectKey]);
+    console.debug(`[GeoJSON] Converted TopoJSON with key "${objectKey}"`);
+    return geoJson;
   } catch (error) {
-    console.error(`[GeoJSON] Error loading ${mapId}:`, error);
-    throw error;
+    console.error(`[GeoJSON] TopoJSON conversion failed:`, error);
+    throw new Error(`Failed to convert TopoJSON: ${error}`);
   }
 }