Merge pull request #2119 from Websoft9/feature/appstore-publish-migration

fix

Author: zhaojing1987

build/__pycache__/library_publish.cpython-39.pyc

@@ -120,8 +120,23 @@ def v2_full_package_names(channel: str, dataset_version: str) -> tuple[str, str]
     return (f"{V2_FULL_PACKAGE_BASENAME}-{dataset_version}.zip", f"{V2_FULL_PACKAGE_BASENAME}-{channel}.zip")
 
 
-def v2_app_package_names(dataset_version: str) -> tuple[str, str]:
-    return (f"{dataset_version}.zip", "latest.zip")
+def _app_version_from_hash(app_hash: str) -> str:
+    """Derive a stable, content-addressed version identifier from an app's fingerprint hash.
+
+    Using the content hash means the same app content always produces the same
+    versioned filename, so unchanged apps never need to be rebuilt or re-uploaded.
+    """
+    return app_hash[:16]
+
+
+def v2_app_package_names(app_version: str) -> tuple[str, str]:
+    """Return (versioned_zip_name, latest_zip_name) for a single app.
+
+    ``app_version`` should be a per-app content-derived identifier, not the
+    global publish timestamp, so that only actually-changed apps get new
+    versioned packages.
+    """
+    return (f"{app_version}.zip", "latest.zip")
 
 
 def sha256_file(path: Path) -> str:
@@ -181,17 +196,17 @@ def current_app_fingerprint(app_dir: Path) -> str:
     return digest.hexdigest()
 
 
-def build_app_package_entry(app_name: str, dataset_version: str) -> dict:
-    versioned_name, latest_name = v2_app_package_names(dataset_version)
+def build_app_package_entry(app_name: str, app_version: str) -> dict:
+    versioned_name, latest_name = v2_app_package_names(app_version)
     app_base = f"apps/{app_name}"
     return {
         "versioned": f"{app_base}/{versioned_name}",
         "latest": f"{app_base}/{latest_name}",
     }
 
 
-def build_app_checksum_entry(app_name: str, dataset_version: str) -> dict:
-    package_entry = build_app_package_entry(app_name, dataset_version)
+def build_app_checksum_entry(app_name: str, app_version: str) -> dict:
+    package_entry = build_app_package_entry(app_name, app_version)
     return {
         "versioned": f"{package_entry['versioned']}.sha256",
         "latest": f"{package_entry['latest']}.sha256",
@@ -203,6 +218,8 @@ def build_apps_index(dataset_version: str, channel: str, generated_at: str) -> d
     for app_dir in sorted(path for path in APPS_DIR.iterdir() if path.is_dir()):
         variables = load_variables_json(app_dir)
         app_name = app_dir.name
+        app_hash = current_app_fingerprint(app_dir)
+        app_version = _app_version_from_hash(app_hash)
         apps.append(
             {
                 "app": app_name,
@@ -211,9 +228,9 @@ def build_apps_index(dataset_version: str, channel: str, generated_at: str) -> d
                 "release": variables.get("release"),
                 "versions": summarize_versions(variables.get("edition", [])),
                 "path": f"apps/{app_name}",
-                "hash": current_app_fingerprint(app_dir),
-                "package": build_app_package_entry(app_name, dataset_version),
-                "checksum": build_app_checksum_entry(app_name, dataset_version),
+                "hash": app_hash,
+                "package": build_app_package_entry(app_name, app_version),
+                "checksum": build_app_checksum_entry(app_name, app_version),
             }
         )
 
@@ -402,7 +419,7 @@ def validate_catalog_artifacts(output_dir: Path, manifest: dict) -> None:
         raise SystemExit("missing catalog manifest.json")
 
 
-def validate_library_artifacts(output_dir: Path, manifest: dict) -> None:
+def validate_library_artifacts(output_dir: Path, manifest: dict, changed_app_names: set[str] | None = None) -> None:
     for name in (
         manifest["fullPackage"]["versioned"],
         manifest["fullPackage"]["latest"],
@@ -423,7 +440,13 @@ def validate_library_artifacts(output_dir: Path, manifest: dict) -> None:
             raise SystemExit(f"invalid apps delta structure: {key}")
 
     apps_index = json.loads((output_dir / manifest["appsIndex"]).read_text(encoding="utf-8"))
+    # Only validate per-app packages for apps that were actually (re)built
+    # in this publish run.  Unchanged apps keep their existing packages on R2.
+    validate_set = changed_app_names if changed_app_names is not None else {e["app"] for e in apps_index.get("apps", [])}
+
     for entry in apps_index.get("apps", []):
+        if entry.get("app") not in validate_set:
+            continue
         package = entry.get("package") or {}
         checksum = entry.get("checksum") or {}
         for path in (package.get("versioned"), package.get("latest"), checksum.get("versioned"), checksum.get("latest")):
@@ -561,11 +584,39 @@ def build_v2_appstore_artifacts(
 
     shutil.copy2(full_dir / full_versioned_name, full_dir / full_latest_name)
 
+    # ── library – compute index & delta BEFORE per-app zips ──
+    apps_index = build_apps_index(dataset_version, channel, generated_at)
+    apps_delta = build_apps_delta(
+        apps_index=apps_index,
+        from_ref=from_ref,
+        from_version=from_version,
+        to_version=dataset_version,
+        channel=channel,
+        generated_at=generated_at,
+    )
+
+    # Determine which apps actually need (re)built packages.
+    # On the very first publish (from_ref is None) every app is new.
+    if from_ref is None:
+        changed_app_names: set[str] = {entry["app"] for entry in apps_index["apps"]}
+    else:
+        changed_app_names = set(apps_delta.get("addedApps", [])) | set(apps_delta.get("changedApps", []))
+
+    # ── library – per-app packages (only for changed apps) ───
     for app_dir in sorted(path for path in APPS_DIR.iterdir() if path.is_dir()):
         app_name = app_dir.name
+        if app_name not in changed_app_names:
+            continue
+
+        # Look up the app's content-derived version identifier from the index.
+        app_entry = next((e for e in apps_index["apps"] if e["app"] == app_name), None)
+        if app_entry is None:
+            continue
+        app_version = _app_version_from_hash(app_entry["hash"])
+        app_versioned_name, app_latest_name = v2_app_package_names(app_version)
+
         app_output_dir = apps_packages_dir / app_name
         app_output_dir.mkdir(parents=True, exist_ok=True)
-        app_versioned_name, app_latest_name = v2_app_package_names(dataset_version)
 
         with tempfile.TemporaryDirectory() as tmp_dir_name:
             tmp_dir = Path(tmp_dir_name)
@@ -577,15 +628,7 @@ def build_v2_appstore_artifacts(
         write_checksum_file(app_output_dir / app_versioned_name)
         write_checksum_file(app_output_dir / app_latest_name)
 
-    apps_index = build_apps_index(dataset_version, channel, generated_at)
-    apps_delta = build_apps_delta(
-        apps_index=apps_index,
-        from_ref=from_ref,
-        from_version=from_version,
-        to_version=dataset_version,
-        channel=channel,
-        generated_at=generated_at,
-    )
+    # ── library – write index, delta, manifest ───────────────
     write_json(library_dir / apps_index_name, apps_index)
     write_json(library_dir / apps_delta_name, apps_delta)
 
@@ -610,7 +653,7 @@ def build_v2_appstore_artifacts(
     )
     write_json(library_dir / "manifest.json", library_manifest)
     write_checksum_file(library_dir / "manifest.json")
-    validate_library_artifacts(library_dir, library_manifest)
+    validate_library_artifacts(library_dir, library_manifest, changed_app_names)
 
     appstore_manifest = build_appstore_manifest(dataset_version, channel, generated_at)
     write_json(manifests_dir / "appstore-manifest.json", appstore_manifest)

build/library_publish.py

@@ -0,0 +1 @@
+[]