-
Notifications
You must be signed in to change notification settings - Fork 1
Expand file tree
/
Copy pathhttpSource.ts
More file actions
137 lines (119 loc) · 4.13 KB
/
httpSource.ts
File metadata and controls
137 lines (119 loc) · 4.13 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
import type { DirSource, FileMetadata, FileSource, SourcePart } from './types.js'
import { getFileName } from './utils.js'
interface S3ListItem {
key?: string
lastModified?: string
size?: number
eTag?: string
isCommonPrefix?: boolean
}
async function s3list(bucket: string, prefix: string): Promise<S3ListItem[]> {
const url = `https://${bucket}.s3.amazonaws.com/?list-type=2&prefix=${prefix}&delimiter=/`
const result = await fetch(url)
if (!result.ok) {
throw new Error(`${result.status} ${result.statusText}`)
}
const text = await result.text()
const results: S3ListItem[] = []
// Parse CommonPrefixes (virtual directories)
const prefixRegex = /<CommonPrefixes>(.*?)<\/CommonPrefixes>/gs
const prefixMatches = text.match(prefixRegex) ?? []
for (const match of prefixMatches) {
const prefixMatch = /<Prefix>(.*?)<\/Prefix>/.exec(match)
if (!prefixMatch) continue
const key = prefixMatch[1]
results.push({
key,
lastModified: new Date().toISOString(), // No lastModified for CommonPrefixes
size: 0,
isCommonPrefix: true,
})
}
// Parse regular objects (files and explicit directories)
const contentsRegex = /<Contents>(.*?)<\/Contents>/gs
const contentsMatches = text.match(contentsRegex) ?? []
for (const match of contentsMatches) {
const keyMatch = /<Key>(.*?)<\/Key>/.exec(match)
const lastModifiedMatch = /<LastModified>(.*?)<\/LastModified>/.exec(match)
const sizeMatch = /<Size>(.*?)<\/Size>/.exec(match)
const eTagMatch = /<ETag>"(.*?)"<\/ETag>/.exec(match) ?? /<ETag>"(.*?)"<\/ETag>/.exec(match)
if (!keyMatch || !lastModifiedMatch) continue
const key = keyMatch[1]
const lastModified = lastModifiedMatch[1]
const size = sizeMatch ? parseInt(sizeMatch[1] ?? '', 10) : undefined
const eTag = eTagMatch ? eTagMatch[1] : undefined
results.push({ key, lastModified, size, eTag })
}
return results
}
function getSourceParts(sourceId: string): SourcePart[] {
const [protocol, rest] = sourceId.split('://', 2)
const parts = rest
? [`${protocol}://${rest.split('/', 1)[0]}`, ...rest.split('/').slice(1)]
: sourceId.split('/')
const sourceParts = [
...parts.map((part, depth) => {
const slashSuffix = depth === parts.length - 1 ? '' : '/'
return {
text: part + slashSuffix,
sourceId: parts.slice(0, depth + 1).join('/') + slashSuffix,
}
}),
]
if (sourceParts[sourceParts.length - 1]?.text === '') {
sourceParts.pop()
}
return sourceParts
}
export function getHttpSource(sourceId: string, options?: {requestInit?: RequestInit}): FileSource | DirSource | undefined {
if (!URL.canParse(sourceId)) {
return undefined
}
const sourceParts = getSourceParts(sourceId)
if (sourceId.endsWith('/')) {
const url = new URL(sourceId)
const bucket = url.hostname.split('.')[0]
const prefix = url.pathname.slice(1)
if (!bucket) {
return undefined
}
return {
kind: 'directory',
sourceId,
sourceParts,
prefix,
listFiles: () => s3list(bucket, prefix).then(items =>
items
// skip s3 directory placeholder
.filter(item => item.key !== undefined && item.key !== prefix)
.map(item => {
if (!item.key) {
throw new Error('Key is undefined')
}
const isDirectory = item.key.endsWith('/')
const itemSourceId = `https://${bucket}.s3.amazonaws.com/${item.key}`
// eslint-disable-next-line @typescript-eslint/prefer-nullish-coalescing
let name = item.key.split('/').pop() || item.key
if (name && isDirectory) {
name = name.replace(prefix, '')
}
return {
name,
fileSize: item.size,
lastModified: item.lastModified,
sourceId: itemSourceId,
kind: isDirectory ? 'directory' : 'file',
} as FileMetadata
})
),
} as DirSource
}
return {
kind: 'file',
sourceId,
sourceParts,
fileName: getFileName(sourceId),
resolveUrl: sourceId,
requestInit: options?.requestInit,
}
}