feat: progressive loading for org packages (#1953)

Co-authored-by: Philippe Serhal <philippe.serhal@gmail.com>
Co-authored-by: Willow (GHOST) <git@willow.sh>
Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>

Author: 5 people

app/composables/npm/useAlgoliaSearch.ts

@@ -217,11 +217,9 @@ export function useAlgoliaSearch() {
     }
   }
 
-  /** Fetch metadata for specific packages by exact name using Algolia's getObjects API. */
-  async function getPackagesByName(packageNames: string[]): Promise<NpmSearchResponse> {
-    if (packageNames.length === 0) {
-      return { isStale: false, objects: [], total: 0, time: new Date().toISOString() }
-    }
+  /** Fetch metadata for a single batch of packages (max 1000) by exact name. */
+  async function getPackagesByNameSlice(names: string[]): Promise<NpmSearchResult[]> {
+    if (names.length === 0) return []
 
     const response = await $fetch<{ results: (AlgoliaHit | null)[] }>(
       `https://${algolia.appId}-dsn.algolia.net/1/indexes/*/objects`,
@@ -232,7 +230,7 @@ export function useAlgoliaSearch() {
           'x-algolia-application-id': algolia.appId,
         },
         body: {
-          requests: packageNames.map(name => ({
+          requests: names.map(name => ({
             indexName,
             objectID: name,
             attributesToRetrieve: ATTRIBUTES_TO_RETRIEVE,
@@ -241,11 +239,41 @@ export function useAlgoliaSearch() {
       },
     )
 
-    const hits = response.results.filter((r): r is AlgoliaHit => r !== null && 'name' in r)
+    return response.results
+      .filter((r): r is AlgoliaHit => r !== null && 'name' in r)
+      .map(hitToSearchResult)
+  }
+
+  /** Fetch metadata for specific packages by exact name using Algolia's getObjects API. */
+  async function getPackagesByName(packageNames: string[]): Promise<NpmSearchResponse> {
+    if (packageNames.length === 0) {
+      return { isStale: false, objects: [], total: 0, time: new Date().toISOString() }
+    }
+
+    // Algolia getObjects has a limit of 1000 objects per request, so batch if needed
+    const BATCH_SIZE = 1000
+    const batches: string[][] = []
+    for (let i = 0; i < packageNames.length; i += BATCH_SIZE) {
+      batches.push(packageNames.slice(i, i + BATCH_SIZE))
+    }
+
+    // Fetch batches with concurrency limit to avoid overwhelming the API
+    const CONCURRENCY = 3
+    const allObjects: NpmSearchResult[] = []
+    for (let i = 0; i < batches.length; i += CONCURRENCY) {
+      const chunk = batches.slice(i, i + CONCURRENCY)
+      const results = await Promise.all(chunk.map(batch => getPackagesByNameSlice(batch)))
+      for (const result of results) {
+        for (const pkg of result) {
+          allObjects.push(pkg)
+        }
+      }
+    }
+
     return {
       isStale: false,
-      objects: hits.map(hitToSearchResult),
-      total: hits.length,
+      objects: allObjects,
+      total: allObjects.length,
       time: new Date().toISOString(),
     }
   }

app/composables/npm/useOrgPackages.ts

@@ -1,8 +1,12 @@
+import type { NpmSearchResponse, NpmSearchResult, PackageMetaResponse } from '#shared/types'
+import { emptySearchResponse, metaToSearchResult } from './search-utils'
+import { mapWithConcurrency } from '#shared/utils/async'
+
 /**
  * Fetch all packages for an npm organization.
  *
  * 1. Gets the authoritative package list from the npm registry (single request)
- * 2. Fetches metadata from Algolia by exact name (single request)
+ * 2. Fetches metadata from Algolia by exact name (batched, max 1000 per request)
  * 3. Falls back to lightweight server-side package-meta lookups
  */
 export function useOrgPackages(orgName: MaybeRefOrGetter<string>) {
@@ -32,7 +36,6 @@ export function useOrgPackages(orgName: MaybeRefOrGetter<string>) {
         )
         packageNames = packages
       } catch (err) {
-        // Check if this is a 404 (org not found)
         if (err && typeof err === 'object' && 'statusCode' in err && err.statusCode === 404) {
           const error = createError({
             statusCode: 404,
@@ -44,15 +47,14 @@ export function useOrgPackages(orgName: MaybeRefOrGetter<string>) {
           }
           throw error
         }
-        // For other errors (network, etc.), return empty array to be safe
         packageNames = []
       }
 
       if (packageNames.length === 0) {
         return emptySearchResponse()
       }
 
-      // Fetch metadata + downloads from Algolia (single request via getObjects)
+      // Fetch metadata from Algolia (batched in chunks of 1000, parallel)
       if (searchProviderValue.value === 'algolia') {
         try {
           const response = await getPackagesByName(packageNames)
@@ -64,6 +66,9 @@ export function useOrgPackages(orgName: MaybeRefOrGetter<string>) {
         }
       }
 
+      // Staleness guard
+      if (toValue(orgName) !== org) return emptySearchResponse()
+
       // npm fallback: fetch lightweight metadata via server proxy
       const metaResults = await mapWithConcurrency(
         packageNames,

i18n/locales/en.json

@@ -935,7 +935,7 @@
       "failed_to_load": "Failed to load organization packages",
       "no_match": "No packages match \"{query}\"",
       "not_found": "Organization not found",
-      "not_found_message": "The organization \"{'@'}{name}\" does not exist on npm"
+      "not_found_message": "The organization {'@'}{name} does not exist on npm"
     }
   },
   "user": {