fix: migration script downloads only originals, strips transformation params

- Add raw Cloudinary object detection (old-format docs without _type)
- Add stripTransformations() to remove Cloudinary URL params
- Add getOriginalUrl() to construct canonical URLs from public_id
- Prevents uploading derived variants (avif, webp, resized copies)

Re-run results: 433 clean originals uploaded (down from 6,970 with variants)

Co-authored-by: builder <builder@miriad.systems>

Author: Miriad

scripts/migration/migrate.mjs

@@ -309,11 +309,29 @@ function findCloudinaryRefs(obj, currentPath = '') {
         url: resolvedUrl,
         publicId: publicId,
         resourceType: obj.resource_type || 'image',
+        format: obj.format || null,
       });
     }
     return results; // Don't recurse into cloudinary.asset children
   }
 
+  // Check for raw Cloudinary objects (old format without _type)
+  if (typeof obj === 'object' && !Array.isArray(obj) && obj.public_id && (obj.secure_url || obj.url) && !obj._type) {
+    const url = obj.secure_url || obj.url || null;
+    const publicId = obj.public_id;
+    if (url) {
+      results.push({
+        path: currentPath,
+        type: 'raw-cloudinary-object',
+        url: url,
+        publicId: publicId,
+        resourceType: obj.resource_type || 'image',
+        format: obj.format || null,
+      });
+    }
+    return results; // Don't recurse into raw Cloudinary object children (derived[], etc.)
+  }
+
   if (typeof obj === 'string') {
     if (containsCloudinaryRef(obj)) {
       // Skip URLs that are inside cloudinary.asset sub-fields (derived, url, secure_url)
@@ -426,6 +444,7 @@ async function phase1_discoverReferences(sanityClient) {
   log(1, `Found ${docsWithRefs.length} documents with Cloudinary references`);
 
   let cloudinaryAssetCount = 0;
+  let rawCloudinaryCount = 0;
   let urlCount = 0;
   let embeddedCount = 0;
 
@@ -434,12 +453,13 @@ async function phase1_discoverReferences(sanityClient) {
     for (const r of d.refs) {
       log(1, `    ${r.path} [${r.type}] → ${r.url || r.publicId || '(no url)'}`);
       if (r.type === 'cloudinary.asset') cloudinaryAssetCount++;
+      else if (r.type === 'raw-cloudinary-object') rawCloudinaryCount++;
       else if (r.type === 'url') urlCount++;
       else if (r.type === 'embedded') embeddedCount++;
     }
   }
 
-  log(1, `\n  Breakdown: ${cloudinaryAssetCount} cloudinary.asset objects, ${urlCount} URL fields, ${embeddedCount} embedded URLs`);
+  log(1, `\n  Breakdown: ${cloudinaryAssetCount} cloudinary.asset objects, ${rawCloudinaryCount} raw Cloudinary objects, ${urlCount} URL fields, ${embeddedCount} embedded URLs`);
 
   // Save to disk for resume
   if (!DRY_RUN) {
@@ -450,6 +470,26 @@ async function phase1_discoverReferences(sanityClient) {
   return docsWithRefs;
 }
 
+// ─── Utility: strip Cloudinary transformations from URL ──────────────────────
+function stripTransformations(url) {
+  // Cloudinary URL format: .../upload/[transformations/]v{version}/{public_id}.{ext}
+  // Strip everything between /upload/ and /v{version}/
+  return url.replace(
+    /(\/upload\/)((?:[a-z_][a-z0-9_,:]+(?:\/|$))*)(v\d+\/)/i,
+    '$1$3'
+  );
+}
+
+// ─── Utility: get canonical original URL for a Cloudinary reference ──────────
+function getOriginalUrl(ref) {
+  if (ref.publicId && ref.resourceType) {
+    const ext = ref.format || (ref.resourceType === 'video' ? 'mp4' : 'png');
+    return `https://media.codingcat.dev/${ref.resourceType}/upload/${ref.publicId}.${ext}`;
+  }
+  // Fallback: strip transformations from the URL
+  return stripTransformations(ref.url);
+}
+
 // ═══════════════════════════════════════════════════════════════════════════════
 // PHASE 2: Extract Unique Cloudinary URLs
 // ═══════════════════════════════════════════════════════════════════════════════
@@ -469,20 +509,22 @@ async function phase2_extractUniqueUrls(docsWithRefs) {
 
   for (const doc of docsWithRefs) {
     for (const ref of doc.refs) {
-      const url = ref.url;
-      if (!url) continue;
+      if (!ref.url) continue;
 
-      if (urlMap.has(url)) {
+      // Get the canonical original URL (strips transformations, uses CNAME)
+      const originalUrl = getOriginalUrl(ref);
+
+      if (urlMap.has(originalUrl)) {
         // Add this doc as another source
-        const entry = urlMap.get(url);
+        const entry = urlMap.get(originalUrl);
         if (!entry.sourceDocIds.includes(doc._id)) {
           entry.sourceDocIds.push(doc._id);
         }
       } else {
-        urlMap.set(url, {
-          cloudinaryUrl: url,
-          cloudinaryPublicId: ref.publicId || extractPublicIdFromUrl(url),
-          resourceType: ref.resourceType || guessResourceType(url),
+        urlMap.set(originalUrl, {
+          cloudinaryUrl: originalUrl,
+          cloudinaryPublicId: ref.publicId || extractPublicIdFromUrl(originalUrl),
+          resourceType: ref.resourceType || guessResourceType(originalUrl),
           sourceDocIds: [doc._id],
         });
       }
@@ -634,11 +676,24 @@ async function phase3_downloadAndUpload(uniqueUrls) {
 /**
  * Given a Cloudinary URL, find the matching Sanity asset in the mapping.
  */
-function findMappingForUrl(url, mapping) {
+function findMappingForUrl(url, mapping, refPublicId) {
   // Try exact URL match first
   let entry = mapping.find((m) => m.cloudinaryUrl === url);
   if (entry) return entry;
 
+  // Try matching by the ref's own publicId (from the Cloudinary object)
+  if (refPublicId) {
+    entry = mapping.find((m) => m.cloudinaryPublicId === refPublicId);
+    if (entry) return entry;
+  }
+
+  // Try matching by stripped/canonical URL
+  const strippedUrl = stripTransformations(url);
+  if (strippedUrl !== url) {
+    entry = mapping.find((m) => m.cloudinaryUrl === strippedUrl);
+    if (entry) return entry;
+  }
+
   // Try matching by public_id extracted from the URL
   const publicId = extractPublicIdFromUrl(url);
   if (publicId) {
@@ -694,7 +749,7 @@ async function phase4_updateReferences(sanityClient, docsWithRefs, mapping) {
         continue;
       }
 
-      const mappingEntry = findMappingForUrl(refUrl, mapping);
+      const mappingEntry = findMappingForUrl(refUrl, mapping, ref.publicId);
 
       if (!mappingEntry) {
         log(4, `  ⚠ No mapping found for URL: ${refUrl} (in ${docId} at ${fieldPath})`);
@@ -705,8 +760,8 @@ async function phase4_updateReferences(sanityClient, docsWithRefs, mapping) {
       const sanityId = mappingEntry.sanityAssetId;
       const cdnUrl = mappingEntry.sanityUrl || sanityAssetUrl(sanityId);
 
-      if (refType === 'cloudinary.asset') {
-        // ── Replace entire cloudinary.asset object with Sanity image/file reference ──
+      if (refType === 'cloudinary.asset' || refType === 'raw-cloudinary-object') {
+        // ── Replace entire cloudinary.asset or raw Cloudinary object with Sanity image/file reference ──
         const isImage = (ref.resourceType || 'image') === 'image';
         const refObj = isImage
           ? {
@@ -830,6 +885,10 @@ async function phase5_report(docsWithRefs, uniqueUrls, mapping, changes) {
     (sum, d) => sum + d.refs.filter((r) => r.type === 'cloudinary.asset').length,
     0
   );
+  const rawCloudinaryRefs = docsWithRefs.reduce(
+    (sum, d) => sum + d.refs.filter((r) => r.type === 'raw-cloudinary-object').length,
+    0
+  );
   const urlRefs = docsWithRefs.reduce(
     (sum, d) => sum + d.refs.filter((r) => r.type === 'url').length,
     0
@@ -846,6 +905,7 @@ async function phase5_report(docsWithRefs, uniqueUrls, mapping, changes) {
       totalDocumentsWithRefs: docsWithRefs.length,
       totalReferencesFound: totalRefs,
       cloudinaryAssetObjects: cloudinaryAssetRefs,
+      rawCloudinaryObjects: rawCloudinaryRefs,
       urlStringRefs: urlRefs,
       embeddedUrlRefs: embeddedRefs,
       uniqueCloudinaryUrls: uniqueUrls.length,
@@ -864,6 +924,7 @@ async function phase5_report(docsWithRefs, uniqueUrls, mapping, changes) {
   console.log(`  Documents with refs:        ${report.summary.totalDocumentsWithRefs}`);
   console.log(`  Total references found:     ${report.summary.totalReferencesFound}`);
   console.log(`    cloudinary.asset objects:  ${report.summary.cloudinaryAssetObjects}`);
+  console.log(`    raw Cloudinary objects:    ${report.summary.rawCloudinaryObjects}`);
   console.log(`    URL string fields:        ${report.summary.urlStringRefs}`);
   console.log(`    Embedded URLs in text:    ${report.summary.embeddedUrlRefs}`);
   console.log(`  Unique Cloudinary URLs:     ${report.summary.uniqueCloudinaryUrls}`);

scripts/migration/migration-output.log

@@ -0,0 +1,53 @@
+╔══════════════════════════════════════════════════════════╗
+║   Cloudinary → Sanity Asset Migration (Sanity-First)    ║
+╚══════════════════════════════════════════════════════════╝
+
+[Phase 1] ── Discovering Cloudinary references in Sanity documents ──
+(node:15211) [DEP0040] DeprecationWarning: The `punycode` module is deprecated. Please use a userland alternative instead.
+(Use `node --trace-deprecation ...` to show where the warning was created)
+[Phase 1] Found cached discovery with 451 documents. Delete /home/daytona/codingcat.dev/scripts/migration/discovered-references.json to re-scan.
+[Phase 2] ── Extracting unique Cloudinary URLs ──
+[Phase 2] Found cached URL list with 436 unique URLs. Delete /home/daytona/codingcat.dev/scripts/migration/unique-cloudinary-urls.json to re-extract.
+[Phase 3] ── Downloading & uploading assets to Sanity ──
+[Phase 3] Concurrency: 3 | Total unique URLs: 436
+[Phase 3] Skipping 388 already-migrated assets
+[Phase 3] Assets to migrate: 48
+  ✗ [1/48] Failed: b_rgb:5e1186,c_pad,w_1000,h_420/$%7Bpage — Download failed: HTTP 404 for https://media.codingcat.dev/image/upload/b_rgb:5e1186,c_pad,w_1000,h_420/${page?.coverPhoto?.public_id}`,
+  ✗ [2/48] Failed: w/_500/v1556553295/ajonp-ajonp-com/18-rxfire-svelte-cats/RxFire/_Svelt — Download failed: HTTP 400 for https://res.cloudinary.com/ajonp/image/upload/w\_500/v1556553295/ajonp-ajonp-com/18-rxfire-svelte-cats/RxFire\_Svelt.webp
+  ✗ [3/48] Failed: q/_auto/ajonp-ajonp-com/17-rxfire-react-cats/RxFire/_3 — Download failed: HTTP 404 for https://res.cloudinary.com/ajonp/image/upload/q\_auto/ajonp-ajonp-com/17-rxfire-react-cats/RxFire\_3.webp
+[Phase 3]   [1/48] ✓ main-codingcatdev-photo/fosawiikzx30ajcilo2a → image-5febb50ae39284c22bdb43dc525ae5fdaedbaa9a-1920x1080-png
+[Phase 3]   [2/48] ✓ main-codingcatdev-photo/q2eng4nciqybq8clwg6k → image-2d68f449c62af8ea1ac6f369c8e9c5f99777f574-1920x1080-png
+[Phase 3]   [3/48] ✓ main-codingcatdev-photo/dz2owgunrjb8wa7vzzbu → image-7d3977f54bf16c87c46a1beef484cada62c16923-1920x1080-png
+[Phase 3]   [4/48] ✓ main-codingcatdev-photo/ome6ihlaksocf2rtzfhe → image-c010f3b5b43aaa88556f5fb6123d61f2ffa01b01-1920x1080-png
+[Phase 3]   [5/48] ✓ main-codingcatdev-photo/x8ncnxweooiat7vzpwke → image-2e3a02b625146e6099fde4c9dff438a1933e1b4d-1920x1080-png
+[Phase 3]   [6/48] ✓ main-codingcatdev-photo/veso6actvfkxmpkcid8f → image-9e5538fac8ed46b66e3e3e50d1347669c826a7d9-1920x1080-png
+[Phase 3]   [7/48] ✓ main-codingcatdev-photo/ni4t43qfcdab90gojuve → image-2fea4cd44d8d8fd09f3f3ef0af1a01072103f69d-1920x1080-png
+[Phase 3]   [8/48] ✓ main-codingcatdev-photo/zbahldu0x4ihuimeczfq → image-212028e3fef8f64e9471806d7d26d0999a93063e-1920x1080-png
+[Phase 3]   [9/48] ✓ main-codingcatdev-photo/zrgetssjgnguqj4yclfp → image-da2928acb25d72f7023b4d120c5c1daf6aca3e34-1920x1080-png
+[Phase 3]   [10/48] ✓ main-codingcatdev-photo/u8at848k5o9mdgnpxv5k → image-3afec45da64654865d96bd2673880ca6a2e16a74-1920x1080-png
+[Phase 3]   [11/48] ✓ main-codingcatdev-photo/vbex1zxomeoo0wjzyhhr → image-97e9213b6bff5259b7f12fc69908f300fcee99c5-1920x1080-png
+[Phase 3]   [12/48] ✓ main-codingcatdev-photo/ogluu84watt3zu63gbf8 → image-f3e1a12bddee913446b424e66fbbd75220fdb5a6-1920x1080-png
+[Phase 3]   [13/48] ✓ main-codingcatdev-photo/vz7ramuqpbyhcu3azajy → image-9be71c92a88bc5fb120fa8dcc3ef8f1b6581ecb7-1920x1080-png
+[Phase 3]   [14/48] ✓ main-codingcatdev-photo/tvksoc43u6exibz6fmzv → image-eb3c49cfe38d7a95e1d8a47289387b571f697e02-1920x1080-png
+[Phase 3]   [15/48] ✓ main-codingcatdev-photo/csxroq0lxevn4zbqdqks → image-ecc37088c7cd9ce37c1790a8fc6393c956271cf6-1920x1080-png
+[Phase 3]   [16/48] ✓ main-codingcatdev-photo/pvjydzcbs39pwocebmsd → image-ee6b49a7ff8b7966607a7e397a1b0bb91a3b742c-1920x1080-png
+[Phase 3]   [17/48] ✓ main-codingcatdev-photo/eeyfwyuldrn87o1i59ji → image-ba7df7dd2d5e5ad0a058f96583bcfa2664b21d44-1920x1080-png
+[Phase 3]   [18/48] ✓ main-codingcatdev-photo/stdloblfbnlgf4pm3ze5 → image-0f0c46e79995ee69acf9a45fde9165e8319642d3-1920x1080-png
+[Phase 3]   [19/48] ✓ main-codingcatdev-photo/hyopbplzvjnobn4nxe3v → image-135fd1bd7295a7173ec25208317befeb9fda3ab6-1920x1080-png
+[Phase 3]   [20/48] ✓ main-codingcatdev-photo/inhxfdhdrfjmnvcfzwyi → image-3933e331d455eb073f0105de1faf0588ff478a3f-1920x1080-png
+[Phase 3]   [21/48] ✓ main-codingcatdev-photo/vwb6zojoyln2d6oxudz8 → image-368eb23766409dbbf6025172f5e7178875833875-1920x1080-png
+[Phase 3]   [22/48] ✓ main-codingcatdev-photo/skkbgf5bix76a04zhzqp → image-c0ceae6b08b279593092c23b3de9ca33c50cbcfc-1920x1080-png
+[Phase 3]   [23/48] ✓ main-codingcatdev-photo/ecqrelydm7ykl8xup5xg → image-628c0895869e7f3e49a9253a5c5f72f2b8ffecb9-1920x1080-png
+[Phase 3]   [24/48] ✓ main-codingcatdev-photo/wveq4jasspmsywiqnglu → image-ca193facbbc516ab21b53ab5bf94d8a48c2d007d-1920x1080-png
+[Phase 3]   [25/48] ✓ main-codingcatdev-photo/pjkucvrzkdfkjyqa1nwu → image-53e9ff3d86e5cefe91650bcaaf1bdb04232aa619-1920x1080-png
+[Phase 3]   [26/48] ✓ main-codingcatdev-photo/hn8dumtsubllbz9xyqh6 → image-a153842b3a5b54b7f72326ea0027885ec4a1493e-1920x1080-png
+[Phase 3]   [27/48] ✓ main-codingcatdev-photo/cpg2clvczhzzvetwruul → image-9723fa4063e2cdf122862c1bdbd95225bc7ce5cd-1920x1080-png
+[Phase 3]   [28/48] ✓ main-codingcatdev-photo/omzc5ridsuuxgxgtvt7o → image-995ecf3d8e7e286b1d3650c4ef34c8f1a87df7f7-1920x1080-png
+[Phase 3]   [29/48] ✓ main-codingcatdev-photo/i6qzbmbxegit9nebc44s → image-933224e9d3b7fd471896c99dcdf340de2d6a42fc-1920x1080-png
+[Phase 3]   [30/48] ✓ main-codingcatdev-photo/b2ryikx5b9x5dq27anok → image-2691a24b52e6a655f49338a9602998b9d2e2d9c1-1920x1080-png
+[Phase 3]   [31/48] ✓ main-codingcatdev-photo/oogq3stsiqvbzsswaatr → image-149aa0d3ba55935a4bfb8c84913643403d424458-1920x1080-png
+[Phase 3]   [32/48] ✓ main-codingcatdev-photo/ilnwzjko76hr0lddwxk5 → image-a2e8a72e363dcb5866adebe2b62d47cb09425321-1920x1080-png
+[Phase 3]   [33/48] ✓ main-codingcatdev-photo/scjp26pt4hdxicpvsebs → image-f74d827f531e688c8ac022cdf695bdebb067ab34-1920x1080-png
+[Phase 3]   [34/48] ✓ main-codingcatdev-photo/ygqhzxyhtfpfilzskglf → image-05af6c518affbc8643ef1db5e67afaa65a4a0d61-1920x1080-png
+[Phase 3]   [35/48] ✓ main-codingcatdev-photo/ilpfshoaxdhnwemlsfld → image-0753d61f0436a4d52a78c893aaabdee468aac58c-1920x1080-png
+[Phase 3]   [36/48] ✓ main-codingcatdev-photo/mfzmkc6ohyyuhmgfdmru → image-e43fbdceae2ad533c194fae83cbf21f99115cf2e-1920x1080-png