|
| 1 | +""" |
| 2 | +Export Firestore and Cloud Storage data from running Firebase emulators |
| 3 | +into a portable format suitable for seeding other environments (including production). |
| 4 | +
|
| 5 | +Usage: |
| 6 | + python3 bin/export-emulator-data.py [output-dir] [--skip-storage-prefix PREFIX ...] |
| 7 | +""" |
| 8 | + |
| 9 | +from __future__ import annotations |
| 10 | + |
| 11 | +import argparse |
| 12 | +import json |
| 13 | +import os |
| 14 | +import sys |
| 15 | +from datetime import datetime, timezone |
| 16 | +from pathlib import Path |
| 17 | +from typing import Any |
| 18 | + |
| 19 | +import firebase_admin |
| 20 | +from firebase_admin import credentials, firestore, storage |
| 21 | +from google.cloud.firestore import Client as FirestoreClient |
| 22 | +from google.cloud.firestore_v1.base_document import DocumentSnapshot |
| 23 | +from google.cloud.storage import Bucket |
| 24 | +import google.auth.credentials |
| 25 | + |
| 26 | + |
| 27 | +# --------------------------------------------------------------------------- |
| 28 | +# Emulator credentials (same pattern as load-library-metadata.py) |
| 29 | +# --------------------------------------------------------------------------- |
| 30 | + |
| 31 | +class EmulatorCredentials(credentials.Base): |
| 32 | + """Mock credentials for use with Firebase emulators.""" |
| 33 | + _mock_credential: google.auth.credentials.Credentials |
| 34 | + |
| 35 | + def __init__(self) -> None: |
| 36 | + self._mock_credential = google.auth.credentials.AnonymousCredentials() |
| 37 | + |
| 38 | + def get_credential(self) -> google.auth.credentials.Credentials: |
| 39 | + return self._mock_credential |
| 40 | + |
| 41 | + |
| 42 | +# --------------------------------------------------------------------------- |
| 43 | +# Configuration |
| 44 | +# --------------------------------------------------------------------------- |
| 45 | + |
| 46 | +parser = argparse.ArgumentParser(description="Export Firebase emulator data") |
| 47 | +parser.add_argument("output_dir", nargs="?", |
| 48 | + default="exported-data-" + datetime.now(tz=timezone.utc).strftime("%Y%m%dT%H%M%SZ"), |
| 49 | + help="Output directory (default: timestamped directory)") |
| 50 | +parser.add_argument("--skip-storage-prefix", action="append", default=[], |
| 51 | + metavar="PREFIX", |
| 52 | + help="Skip storage blobs whose path starts with PREFIX (can be repeated)") |
| 53 | +args = parser.parse_args() |
| 54 | + |
| 55 | +OUTPUT_DIR = Path(args.output_dir) |
| 56 | +SKIP_STORAGE_PREFIXES: list[str] = args.skip_storage_prefix |
| 57 | + |
| 58 | +# Ensure emulator env vars are set |
| 59 | +if not os.getenv("FIRESTORE_EMULATOR_HOST"): |
| 60 | + os.environ["FIRESTORE_EMULATOR_HOST"] = "localhost:8080" |
| 61 | + |
| 62 | +storage_host_override = os.getenv("QUARKUS_GOOGLE_CLOUD_STORAGE_HOST_OVERRIDE") |
| 63 | +if storage_host_override: |
| 64 | + os.environ["STORAGE_EMULATOR_HOST"] = storage_host_override |
| 65 | +elif not os.getenv("STORAGE_EMULATOR_HOST"): |
| 66 | + os.environ["STORAGE_EMULATOR_HOST"] = "http://localhost:9199" |
| 67 | + |
| 68 | +STORAGE_BUCKET = os.getenv("GCS_BUCKET_NAME", "demo-bdt-dev.appspot.com") |
| 69 | +PROJECT_ID = os.getenv("QUARKUS_GOOGLE_CLOUD_PROJECT_ID", "demo-bdt-dev") |
| 70 | + |
| 71 | + |
| 72 | +# --------------------------------------------------------------------------- |
| 73 | +# Initialize Firebase |
| 74 | +# --------------------------------------------------------------------------- |
| 75 | + |
| 76 | +cred = EmulatorCredentials() |
| 77 | +firebase_admin.initialize_app(cred, { |
| 78 | + "storageBucket": STORAGE_BUCKET, |
| 79 | + "projectId": PROJECT_ID, |
| 80 | +}) |
| 81 | + |
| 82 | +db: FirestoreClient = firestore.client() |
| 83 | +bucket: Bucket = storage.bucket() |
| 84 | + |
| 85 | + |
| 86 | +# --------------------------------------------------------------------------- |
| 87 | +# Helpers |
| 88 | +# --------------------------------------------------------------------------- |
| 89 | + |
| 90 | +def serialize_value(value: Any) -> Any: |
| 91 | + """Convert Firestore field values into JSON-serializable types.""" |
| 92 | + if value is None: |
| 93 | + return None |
| 94 | + if isinstance(value, datetime): |
| 95 | + return {"_type": "timestamp", "value": value.isoformat()} |
| 96 | + if isinstance(value, dict): |
| 97 | + return {k: serialize_value(v) for k, v in value.items()} |
| 98 | + if isinstance(value, list): |
| 99 | + return [serialize_value(v) for v in value] |
| 100 | + if isinstance(value, (str, int, float, bool)): |
| 101 | + return value |
| 102 | + # Fallback: convert to string |
| 103 | + return str(value) |
| 104 | + |
| 105 | + |
| 106 | +def export_document(doc: DocumentSnapshot, base_path: Path) -> None: |
| 107 | + """Export a single Firestore document and its subcollections.""" |
| 108 | + data = doc.to_dict() |
| 109 | + if data is None: |
| 110 | + return |
| 111 | + |
| 112 | + # Add the document ID into the exported data for reference |
| 113 | + data["_id"] = doc.id |
| 114 | + |
| 115 | + serialized = serialize_value(data) |
| 116 | + |
| 117 | + # Write document JSON |
| 118 | + doc_file = base_path / f"{doc.id}.json" |
| 119 | + doc_file.parent.mkdir(parents=True, exist_ok=True) |
| 120 | + with open(doc_file, "w", encoding="utf-8") as f: |
| 121 | + json.dump(serialized, f, indent=2, ensure_ascii=False) |
| 122 | + |
| 123 | + # Recurse into subcollections |
| 124 | + for subcol_ref in doc.reference.collections(): |
| 125 | + subcol_path = base_path / doc.id / subcol_ref.id |
| 126 | + docs = subcol_ref.stream() |
| 127 | + for sub_doc in docs: |
| 128 | + export_document(sub_doc, subcol_path) |
| 129 | + |
| 130 | + |
| 131 | +def export_firestore(output_dir: Path) -> int: |
| 132 | + """Export all Firestore collections to JSON files. Returns document count.""" |
| 133 | + firestore_dir = output_dir / "firestore" |
| 134 | + doc_count = 0 |
| 135 | + |
| 136 | + # Iterate over all root-level collections |
| 137 | + for collection_ref in db.collections(): |
| 138 | + col_name = collection_ref.id |
| 139 | + col_path = firestore_dir / col_name |
| 140 | + print(f" Exporting collection: {col_name}") |
| 141 | + |
| 142 | + for doc in collection_ref.stream(): |
| 143 | + export_document(doc, col_path) |
| 144 | + doc_count += 1 |
| 145 | + |
| 146 | + return doc_count |
| 147 | + |
| 148 | + |
| 149 | +def export_storage(output_dir: Path, skip_prefixes: list[str]) -> int: |
| 150 | + """Export all Storage blobs to files. Returns file count.""" |
| 151 | + storage_dir = output_dir / "storage" |
| 152 | + file_count = 0 |
| 153 | + |
| 154 | + blobs = bucket.list_blobs() |
| 155 | + for blob in blobs: |
| 156 | + # Skip zero-byte placeholder objects |
| 157 | + if blob.size == 0: |
| 158 | + continue |
| 159 | + |
| 160 | + # Skip blobs matching any of the skip prefixes |
| 161 | + if any(blob.name.startswith(prefix) for prefix in skip_prefixes): |
| 162 | + print(f" Skipping blob: {blob.name} (matched skip prefix)") |
| 163 | + continue |
| 164 | + |
| 165 | + dest = storage_dir / blob.name |
| 166 | + dest.parent.mkdir(parents=True, exist_ok=True) |
| 167 | + |
| 168 | + print(f" Exporting blob: {blob.name} ({blob.size} bytes)") |
| 169 | + blob.download_to_filename(str(dest)) |
| 170 | + file_count += 1 |
| 171 | + |
| 172 | + return file_count |
| 173 | + |
| 174 | + |
| 175 | +# --------------------------------------------------------------------------- |
| 176 | +# Main |
| 177 | +# --------------------------------------------------------------------------- |
| 178 | + |
| 179 | +def main() -> None: |
| 180 | + print(f"Output directory: {OUTPUT_DIR}") |
| 181 | + print() |
| 182 | + |
| 183 | + # Clean previous export |
| 184 | + if OUTPUT_DIR.exists(): |
| 185 | + import shutil |
| 186 | + shutil.rmtree(OUTPUT_DIR) |
| 187 | + |
| 188 | + OUTPUT_DIR.mkdir(parents=True, exist_ok=True) |
| 189 | + |
| 190 | + # Export Firestore |
| 191 | + print("Exporting Firestore data...") |
| 192 | + doc_count = export_firestore(OUTPUT_DIR) |
| 193 | + print(f" -> {doc_count} document(s) exported") |
| 194 | + print() |
| 195 | + |
| 196 | + # Export Storage |
| 197 | + print("Exporting Storage data...") |
| 198 | + file_count = export_storage(OUTPUT_DIR, SKIP_STORAGE_PREFIXES) |
| 199 | + print(f" -> {file_count} file(s) exported") |
| 200 | + print() |
| 201 | + |
| 202 | + # Write a manifest for reference |
| 203 | + manifest = { |
| 204 | + "exportedAt": datetime.now(tz=timezone.utc).isoformat() + "Z", |
| 205 | + "source": "firebase-emulators", |
| 206 | + "projectId": PROJECT_ID, |
| 207 | + "storageBucket": STORAGE_BUCKET, |
| 208 | + "firestoreDocuments": doc_count, |
| 209 | + "storageFiles": file_count, |
| 210 | + } |
| 211 | + manifest_path = OUTPUT_DIR / "manifest.json" |
| 212 | + with open(manifest_path, "w", encoding="utf-8") as f: |
| 213 | + json.dump(manifest, f, indent=2) |
| 214 | + |
| 215 | + print(f"Manifest written to {manifest_path}") |
| 216 | + |
| 217 | + |
| 218 | +if __name__ == "__main__": |
| 219 | + main() |
0 commit comments