Skip to content

Commit 71f2df3

Browse files
crowlKatscrowlKats
committed
feat: differential sync
1 parent db34c82 commit 71f2df3

1 file changed

Lines changed: 168 additions & 88 deletions

File tree

‎publish.ts‎

Lines changed: 168 additions & 88 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
11
import { TarStream, type TarStreamDir, type TarStreamFile } from "@std/tar";
22
import { compile as gitignoreCompile } from "@cfa/gitignore-parser";
3-
import { walk } from "@std/fs";
3+
import { walk, type WalkEntry } from "@std/fs";
44
import { ProgressBar } from "@std/cli/unstable-progress-bar";
5+
import { Spinner } from "@std/cli/unstable-spinner";
56
import { join, relative, resolve } from "@std/path";
67
import { green, yellow } from "@std/fmt/colors";
78
import { type Config, writeConfig } from "./config.ts";
@@ -10,6 +11,13 @@ import { error } from "./util.ts";
1011

1112
const SEPARATOR_PATTERN = Deno.build.os === "windows" ? "\\\\" : "/";
1213

14+
type Chunk =
15+
& { chunk: WalkEntry; relativePath: string }
16+
& ({ hash?: undefined; data?: undefined } | {
17+
hash: string;
18+
data: Uint8Array;
19+
});
20+
1321
export async function publish(
1422
deployUrl: string,
1523
rootPath: string,
@@ -38,10 +46,12 @@ export async function publish(
3846

3947
console.log(`Publishing '${resolve(rootPath)}'`);
4048

41-
const stream = ReadableStream.from(walk(rootPath, { skip: excludes }))
49+
const stream: ReadableStream<Chunk> = ReadableStream.from(
50+
walk(rootPath, { skip: excludes }),
51+
)
4252
.pipeThrough(
4353
new TransformStream({
44-
transform(chunk, controller) {
54+
async transform(chunk, controller) {
4555
const path = relative(rootPath, chunk.path);
4656
const relativePath = join(
4757
"source",
@@ -51,111 +61,181 @@ export async function publish(
5161
return;
5262
}
5363

54-
controller.enqueue({ chunk, relativePath });
64+
if (!chunk.isDirectory) {
65+
const data = await Deno.readFile(chunk.path);
66+
67+
const hashBuffer = await crypto.subtle.digest("SHA-256", data!);
68+
const hashArray = Array.from(new Uint8Array(hashBuffer));
69+
const hash = hashArray.map((b) => b.toString(16).padStart(2, "0"))
70+
.join("");
71+
72+
controller.enqueue({
73+
chunk,
74+
relativePath,
75+
data,
76+
hash,
77+
});
78+
} else {
79+
controller.enqueue({
80+
chunk,
81+
relativePath,
82+
});
83+
}
5584
},
5685
}),
5786
);
5887

5988
const [counter, body] = stream.tee();
6089

90+
const manifest: Record<string, string> = {};
6191
let total = 0;
62-
for await (const { chunk } of counter) {
92+
93+
const hashesSpinner = new Spinner({
94+
message: "Generating hashes...",
95+
});
96+
hashesSpinner.start();
97+
for await (const { chunk, hash, relativePath } of counter) {
6398
if (!chunk.isDirectory) {
6499
total++;
100+
const parts = relativePath.split("/");
101+
parts.shift();
102+
manifest[parts.join("/")] = hash!;
65103
}
66104
}
105+
hashesSpinner.stop();
106+
console.log(`${green("✔")} Generated hashes`);
67107

68-
const progress = new ProgressBar({
69-
max: total,
70-
emptyChar: " ",
71-
fillChar: green("█"),
72-
formatter(formatter) {
73-
const minutes = (formatter.time / 1000 / 60 | 0).toString().padStart(
74-
2,
75-
"0",
76-
);
77-
const seconds = (formatter.time / 1000 % 60 | 0).toString().padStart(
78-
2,
79-
"0",
80-
);
81-
82-
const length = formatter.max.toString().length;
83-
return `[${yellow(minutes)}:${
84-
yellow(seconds)
85-
}] ${formatter.progressBar} ${
86-
yellow(formatter.value.toString().padStart(length, " "))
87-
}/${yellow(formatter.max.toString())} files uploaded.`;
88-
},
89-
});
90-
91-
const tarball = body
92-
.pipeThrough(
93-
new TransformStream({
94-
async transform({ chunk, relativePath }, controller) {
95-
if (chunk.isDirectory) {
96-
controller.enqueue(
97-
{
98-
type: "directory",
99-
path: relativePath,
100-
} satisfies TarStreamDir,
101-
);
102-
} else {
103-
const [stat, file] = await Promise.all([
104-
Deno.stat(chunk.path),
105-
Deno.open(chunk.path),
106-
]);
107-
108-
controller.enqueue(
109-
{
110-
type: "file",
111-
path: relativePath,
112-
size: stat.size,
113-
readable: file.readable.pipeThrough(
114-
new TransformStream({
115-
flush() {
116-
progress.value += 1;
117-
},
118-
}),
119-
),
120-
} satisfies TarStreamFile,
121-
);
122-
}
123-
},
124-
}),
125-
)
126-
.pipeThrough(new TarStream())
127-
.pipeThrough(new CompressionStream("gzip"));
128-
129-
const resp = await authedFetch(deployUrl, "/api/trigger_tarball_build", {
108+
const initiatedBuildRes = await authedFetch(deployUrl, "api/initiate_cli_build", {
130109
method: "POST",
131110
headers: {
132-
"x-meta": JSON.stringify({
133-
org,
134-
app,
135-
production: prod,
136-
}),
111+
"content-type": "application/json",
137112
},
138-
body: tarball,
113+
body: JSON.stringify({
114+
org,
115+
app,
116+
production: prod,
117+
manifest,
118+
}),
139119
});
140120

141-
const resBody = await resp.json();
121+
const { revisionId }: { revisionId: string; } = await initiatedBuildRes.json();
122+
123+
let missingHashes: string[];
124+
125+
const s = Date.now();
126+
while (true) {
127+
await new Promise(resolve => setTimeout(resolve, 1000));
128+
const maybeHashesRes = await authedFetch(deployUrl, `api/diffsync/${org}/${app}/${revisionId}`, {});
129+
if (maybeHashesRes.status !== 202) {
130+
if (maybeHashesRes.ok) {
131+
missingHashes = await maybeHashesRes.json();
132+
break;
133+
} else {
134+
const err = await maybeHashesRes.json();
135+
error(`Failed getting file hashes: ${err.message}`, maybeHashesRes);
136+
}
137+
}
142138

143-
await progress.stop();
139+
if ((Date.now() - s) >= 30 * 1000) {
140+
error(`Failed getting file hashes`, maybeHashesRes);
141+
}
142+
}
144143

145-
console.log();
144+
if (missingHashes.length > 0) {
145+
const skippedFilesCount = total - missingHashes.length;
146146

147-
if (!resp.ok) {
148-
error(resBody.message, resp);
149-
} else {
150-
console.log("Successfully uploaded your application!");
151-
console.log(
152-
`You can view your application overview here:\n ${deployUrl}/${org}/${app}`,
153-
);
154-
console.log(
155-
`You can view the revision here:\n ${deployUrl}/${org}/${app}/builds/${resBody.revisionId}`,
156-
);
157-
// TODO: print out the preview url
147+
if (skippedFilesCount > 0) {
148+
console.log(`Found ${skippedFilesCount} already uploaded files, which will be skipped from uploading`);
149+
}
150+
151+
const progress = new ProgressBar({
152+
max: missingHashes.length,
153+
emptyChar: " ",
154+
fillChar: green("█"),
155+
formatter(formatter) {
156+
const minutes = (formatter.time / 1000 / 60 | 0).toString().padStart(
157+
2,
158+
"0",
159+
);
160+
const seconds = (formatter.time / 1000 % 60 | 0).toString().padStart(
161+
2,
162+
"0",
163+
);
164+
165+
const length = formatter.max.toString().length;
166+
return `[${yellow(minutes)}:${
167+
yellow(seconds)
168+
}] ${formatter.progressBar} ${
169+
yellow(formatter.value.toString().padStart(length, " "))
170+
}/${yellow(formatter.max.toString())} files uploaded.`;
171+
},
172+
});
173+
174+
const tarball = body
175+
.pipeThrough(
176+
new TransformStream({
177+
async transform({ chunk, relativePath, data, hash }, controller) {
178+
if (chunk.isDirectory) {
179+
controller.enqueue(
180+
{
181+
type: "directory",
182+
path: relativePath,
183+
} satisfies TarStreamDir,
184+
);
185+
} else if (missingHashes.includes(hash!)) {
186+
const stat = await Deno.stat(chunk.path);
187+
188+
progress.value += 1;
189+
190+
controller.enqueue(
191+
{
192+
type: "file",
193+
path: relativePath,
194+
size: stat.size,
195+
readable: ReadableStream.from([data!]),
196+
} satisfies TarStreamFile,
197+
);
198+
}
199+
},
200+
}),
201+
)
202+
.pipeThrough(new TarStream())
203+
.pipeThrough(new CompressionStream("gzip"));
204+
205+
const resp = await authedFetch(deployUrl, `api/diffsync/${org}/${app}/${revisionId}`, {
206+
method: "POST",
207+
headers: {
208+
"x-meta": JSON.stringify({
209+
org,
210+
app,
211+
production: prod,
212+
}),
213+
},
214+
body: tarball,
215+
});
216+
217+
await progress.stop();
218+
219+
console.log();
220+
221+
if (!resp.ok) {
222+
const resBody = await resp.json();
223+
error(resBody.message, resp);
224+
}
158225

159-
await writeConfig(configContent, rootPath, org, app);
226+
console.log("Successfully uploaded your application!");
227+
} else {
228+
console.log("No files were changed.");
160229
}
230+
231+
console.log(
232+
`You can view your application overview here:\n ${deployUrl}/${org}/${app}`,
233+
);
234+
console.log(
235+
`You can view the revision here:\n ${deployUrl}/${org}/${app}/builds/${revisionId}`,
236+
);
237+
// TODO: print out the preview url
238+
239+
await writeConfig(configContent, rootPath, org, app);
240+
161241
}

0 commit comments

Comments
 (0)