S3 list folders before files (#367)

Author: platypii

package.json

@@ -59,32 +59,32 @@
     "hyparquet": "1.24.0",
     "hyparquet-compressors": "1.1.1",
     "icebird": "0.3.1",
-    "squirreling": "0.7.8"
+    "squirreling": "0.7.9"
   },
   "devDependencies": {
-    "@storybook/react-vite": "10.1.11",
-    "@testing-library/react": "16.3.1",
-    "@types/node": "25.0.9",
-    "@types/react": "19.2.8",
+    "@storybook/react-vite": "10.2.0",
+    "@testing-library/react": "16.3.2",
+    "@types/node": "25.0.10",
+    "@types/react": "19.2.9",
     "@types/react-dom": "19.2.3",
     "@vitejs/plugin-react": "5.1.2",
-    "@vitest/coverage-v8": "4.0.17",
+    "@vitest/coverage-v8": "4.0.18",
     "eslint": "9.39.2",
     "eslint-plugin-react": "7.37.5",
     "eslint-plugin-react-hooks": "7.0.1",
     "eslint-plugin-react-refresh": "0.4.26",
-    "eslint-plugin-storybook": "10.1.11",
-    "globals": "17.0.0",
+    "eslint-plugin-storybook": "10.2.0",
+    "globals": "17.1.0",
     "jsdom": "27.4.0",
     "nodemon": "3.1.11",
     "npm-run-all": "4.1.5",
     "react": "19.2.3",
     "react-dom": "19.2.3",
-    "storybook": "10.1.11",
+    "storybook": "10.2.0",
     "typescript": "5.9.3",
-    "typescript-eslint": "8.53.0",
+    "typescript-eslint": "8.53.1",
     "vite": "7.3.1",
-    "vitest": "4.0.17"
+    "vitest": "4.0.18"
   },
   "peerDependencies": {
     "react": "^18.3.1 || ^19",

src/lib/sources/httpSource.ts

@@ -18,6 +18,23 @@ async function s3list(bucket: string, prefix: string): Promise<S3ListItem[]> {
   const text = await result.text()
   const results: S3ListItem[] = []
 
+  // Parse CommonPrefixes (virtual directories)
+  const prefixRegex = /<CommonPrefixes>(.*?)<\/CommonPrefixes>/gs
+  const prefixMatches = text.match(prefixRegex) ?? []
+
+  for (const match of prefixMatches) {
+    const prefixMatch = /<Prefix>(.*?)<\/Prefix>/.exec(match)
+    if (!prefixMatch) continue
+
+    const key = prefixMatch[1]
+    results.push({
+      key,
+      lastModified: new Date().toISOString(), // No lastModified for CommonPrefixes
+      size: 0,
+      isCommonPrefix: true,
+    })
+  }
+
   // Parse regular objects (files and explicit directories)
   const contentsRegex = /<Contents>(.*?)<\/Contents>/gs
   const contentsMatches = text.match(contentsRegex) ?? []
@@ -38,23 +55,6 @@ async function s3list(bucket: string, prefix: string): Promise<S3ListItem[]> {
     results.push({ key, lastModified, size, eTag })
   }
 
-  // Parse CommonPrefixes (virtual directories)
-  const prefixRegex = /<CommonPrefixes>(.*?)<\/CommonPrefixes>/gs
-  const prefixMatches = text.match(prefixRegex) ?? []
-
-  for (const match of prefixMatches) {
-    const prefixMatch = /<Prefix>(.*?)<\/Prefix>/.exec(match)
-    if (!prefixMatch) continue
-
-    const key = prefixMatch[1]
-    results.push({
-      key,
-      lastModified: new Date().toISOString(), // No lastModified for CommonPrefixes
-      size: 0,
-      isCommonPrefix: true,
-    })
-  }
-
   return results
 }
 
@@ -101,7 +101,8 @@ export function getHttpSource(sourceId: string, options?: {requestInit?: Request
       prefix,
       listFiles: () => s3list(bucket, prefix).then(items =>
         items
-          .filter(item => item.key !== undefined)
+          // skip s3 directory placeholder
+          .filter(item => item.key !== undefined && item.key !== prefix)
           .map(item => {
             if (!item.key) {
               throw new Error('Key is undefined')