From 15a2747984a2cd8a8724c61ca82c4e6d8ccbef70 Mon Sep 17 00:00:00 2001 From: Mohammad Bagher Abiyat Date: Sun, 11 May 2025 22:43:09 +0330 Subject: [PATCH 01/22] init --- packages/app/server/routes/rm/stale.post.ts | 30 +++++++++++++-------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/packages/app/server/routes/rm/stale.post.ts b/packages/app/server/routes/rm/stale.post.ts index 5ce3afe9..37de681e 100644 --- a/packages/app/server/routes/rm/stale.post.ts +++ b/packages/app/server/routes/rm/stale.post.ts @@ -34,7 +34,7 @@ async function iterateAndDelete(event: H3Event, opts: R2ListOptions) { const downloadedAtBucket = useDownloadedAtBucket(event); const today = Date.parse(new Date().toString()); - const removed: string[] = []; + const removed: { key: string; uploaded: Date; downloadedAt: Date }[] = []; while (truncated) { // TODO: Avoid using context.cloudflare and migrate to unstorage, but it does not have truncated for now const next = await binding.list({ @@ -45,11 +45,15 @@ async function iterateAndDelete(event: H3Event, opts: R2ListOptions) { const uploaded = Date.parse(object.uploaded.toString()); // remove the object anyway if it's 6 months old already if ((today - uploaded) / (1000 * 3600 * 24 * 30 * 6) >= 1) { - removed.push(object.key); - event.context.cloudflare.context.waitUntil(binding.delete(object.key)); - event.context.cloudflare.context.waitUntil( - downloadedAtBucket.removeItem(object.key), - ); + removed.push({ + key: object.key, + uploaded: new Date(object.uploaded), + downloadedAt: new Date((await downloadedAtBucket.getItem(object.key))!), + }); + // event.context.cloudflare.context.waitUntil(binding.delete(object.key)); + // event.context.cloudflare.context.waitUntil( + // downloadedAtBucket.removeItem(object.key), + // ); } const downloadedAt = (await downloadedAtBucket.getItem(object.key))!; // if it has not been downloaded in the last month and it's at least 1 month old @@ -57,11 +61,15 @@ async function iterateAndDelete(event: H3Event, opts: R2ListOptions) { !((today - downloadedAt) / (1000 * 3600 * 24 * 30) < 1) && (today - uploaded) / (1000 * 3600 * 24 * 30) >= 1 ) { - removed.push(object.key); - event.context.cloudflare.context.waitUntil(binding.delete(object.key)); - event.context.cloudflare.context.waitUntil( - downloadedAtBucket.removeItem(object.key), - ); + removed.push({ + key: object.key, + uploaded: new Date(object.uploaded), + downloadedAt: new Date(downloadedAt), + }); + // event.context.cloudflare.context.waitUntil(binding.delete(object.key)); + // event.context.cloudflare.context.waitUntil( + // downloadedAtBucket.removeItem(object.key), + // ); } } From 82b72b1d7b7a8c328a4831fe49f21aae0fd37e26 Mon Sep 17 00:00:00 2001 From: Mohammad Bagher Abiyat Date: Sun, 11 May 2025 23:09:16 +0330 Subject: [PATCH 02/22] remove auth --- packages/app/server/routes/rm/stale.post.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/app/server/routes/rm/stale.post.ts b/packages/app/server/routes/rm/stale.post.ts index 37de681e..420f34a2 100644 --- a/packages/app/server/routes/rm/stale.post.ts +++ b/packages/app/server/routes/rm/stale.post.ts @@ -3,11 +3,11 @@ import type { H3Event } from "h3"; export default eventHandler(async (event) => { const rmStaleKeyHeader = getHeader(event, "sb-rm-stale-key"); const { rmStaleKey } = useRuntimeConfig(event); - if (rmStaleKeyHeader !== rmStaleKey) { - throw createError({ - status: 403, - }); - } + // if (rmStaleKeyHeader !== rmStaleKey) { + // throw createError({ + // status: 403, + // }); + // } return { ok: true, removed: [ From f8de5219f1dcef1774f1d02993e303132b98bfd8 Mon Sep 17 00:00:00 2001 From: Mohammad Bagher Abiyat Date: Sun, 11 May 2025 23:30:45 +0330 Subject: [PATCH 03/22] use the holy streams --- packages/app/server/routes/rm/stale.post.ts | 44 ++++++++++----------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/packages/app/server/routes/rm/stale.post.ts b/packages/app/server/routes/rm/stale.post.ts index 420f34a2..e4a10e12 100644 --- a/packages/app/server/routes/rm/stale.post.ts +++ b/packages/app/server/routes/rm/stale.post.ts @@ -8,24 +8,25 @@ export default eventHandler(async (event) => { // status: 403, // }); // } - return { - ok: true, - removed: [ - ...(await Promise.all([ - iterateAndDelete(event, { - prefix: usePackagesBucket.base, - limit: 100, - }), - iterateAndDelete(event, { - prefix: useTemplatesBucket.base, - limit: 100, - }), - ]).then((results) => results.flat())), - ], - }; + const { readable, writable } = new TransformStream() + + await iterateAndDelete(event, writable, { + prefix: usePackagesBucket.base, + limit: 100, + }) + + await iterateAndDelete(event, writable, { + prefix: useTemplatesBucket.base, + limit: 100, + }) + + writable.close() + + return readable }); -async function iterateAndDelete(event: H3Event, opts: R2ListOptions) { +async function iterateAndDelete(event: H3Event, writable: WritableStream, opts: R2ListOptions) { + const writer = writable.getWriter() const binding = useBinding(event); let truncated = true; @@ -34,7 +35,6 @@ async function iterateAndDelete(event: H3Event, opts: R2ListOptions) { const downloadedAtBucket = useDownloadedAtBucket(event); const today = Date.parse(new Date().toString()); - const removed: { key: string; uploaded: Date; downloadedAt: Date }[] = []; while (truncated) { // TODO: Avoid using context.cloudflare and migrate to unstorage, but it does not have truncated for now const next = await binding.list({ @@ -45,11 +45,11 @@ async function iterateAndDelete(event: H3Event, opts: R2ListOptions) { const uploaded = Date.parse(object.uploaded.toString()); // remove the object anyway if it's 6 months old already if ((today - uploaded) / (1000 * 3600 * 24 * 30 * 6) >= 1) { - removed.push({ + writer.write(JSON.stringify({ key: object.key, uploaded: new Date(object.uploaded), downloadedAt: new Date((await downloadedAtBucket.getItem(object.key))!), - }); + }) + "\n") // event.context.cloudflare.context.waitUntil(binding.delete(object.key)); // event.context.cloudflare.context.waitUntil( // downloadedAtBucket.removeItem(object.key), @@ -61,11 +61,11 @@ async function iterateAndDelete(event: H3Event, opts: R2ListOptions) { !((today - downloadedAt) / (1000 * 3600 * 24 * 30) < 1) && (today - uploaded) / (1000 * 3600 * 24 * 30) >= 1 ) { - removed.push({ + writer.write(JSON.stringify({ key: object.key, uploaded: new Date(object.uploaded), downloadedAt: new Date(downloadedAt), - }); + }) + "\n") // event.context.cloudflare.context.waitUntil(binding.delete(object.key)); // event.context.cloudflare.context.waitUntil( // downloadedAtBucket.removeItem(object.key), @@ -78,5 +78,5 @@ async function iterateAndDelete(event: H3Event, opts: R2ListOptions) { cursor = next.cursor; } } - return removed; + writer.releaseLock() } From d724babe5fb7f6bbcc1bc86399265098b9163895 Mon Sep 17 00:00:00 2001 From: Mohammad Bagher Abiyat Date: Sun, 11 May 2025 23:38:33 +0330 Subject: [PATCH 04/22] add abort --- packages/app/server/routes/rm/stale.post.ts | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/packages/app/server/routes/rm/stale.post.ts b/packages/app/server/routes/rm/stale.post.ts index e4a10e12..722b65fb 100644 --- a/packages/app/server/routes/rm/stale.post.ts +++ b/packages/app/server/routes/rm/stale.post.ts @@ -2,6 +2,7 @@ import type { H3Event } from "h3"; export default eventHandler(async (event) => { const rmStaleKeyHeader = getHeader(event, "sb-rm-stale-key"); + const signal = toWebRequest(event).signal; const { rmStaleKey } = useRuntimeConfig(event); // if (rmStaleKeyHeader !== rmStaleKey) { // throw createError({ @@ -10,12 +11,12 @@ export default eventHandler(async (event) => { // } const { readable, writable } = new TransformStream() - await iterateAndDelete(event, writable, { + await iterateAndDelete(event, writable, signal, { prefix: usePackagesBucket.base, limit: 100, }) - await iterateAndDelete(event, writable, { + await iterateAndDelete(event, writable, signal, { prefix: useTemplatesBucket.base, limit: 100, }) @@ -25,7 +26,7 @@ export default eventHandler(async (event) => { return readable }); -async function iterateAndDelete(event: H3Event, writable: WritableStream, opts: R2ListOptions) { +async function iterateAndDelete(event: H3Event, writable: WritableStream, signal: AbortSignal, opts: R2ListOptions) { const writer = writable.getWriter() const binding = useBinding(event); @@ -35,13 +36,16 @@ async function iterateAndDelete(event: H3Event, writable: WritableStream, opts: const downloadedAtBucket = useDownloadedAtBucket(event); const today = Date.parse(new Date().toString()); - while (truncated) { + while (truncated && !signal.aborted) { // TODO: Avoid using context.cloudflare and migrate to unstorage, but it does not have truncated for now const next = await binding.list({ ...opts, cursor, }); for (const object of next.objects) { + if (signal.aborted) { + break; + } const uploaded = Date.parse(object.uploaded.toString()); // remove the object anyway if it's 6 months old already if ((today - uploaded) / (1000 * 3600 * 24 * 30 * 6) >= 1) { From c077347783198611c93898f722147acb624f2266 Mon Sep 17 00:00:00 2001 From: Mohammad Bagher Abiyat Date: Sun, 11 May 2025 23:48:30 +0330 Subject: [PATCH 05/22] i was doing it wrong baby --- packages/app/server/routes/rm/stale.post.ts | 22 +++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/packages/app/server/routes/rm/stale.post.ts b/packages/app/server/routes/rm/stale.post.ts index 722b65fb..4eb714be 100644 --- a/packages/app/server/routes/rm/stale.post.ts +++ b/packages/app/server/routes/rm/stale.post.ts @@ -11,17 +11,19 @@ export default eventHandler(async (event) => { // } const { readable, writable } = new TransformStream() - await iterateAndDelete(event, writable, signal, { - prefix: usePackagesBucket.base, - limit: 100, - }) + event.context.cloudflare.context.waitUntil(Promise.all([ + iterateAndDelete(event, writable, signal, { + prefix: usePackagesBucket.base, + limit: 100, + }), - await iterateAndDelete(event, writable, signal, { - prefix: useTemplatesBucket.base, - limit: 100, - }) - - writable.close() + iterateAndDelete(event, writable, signal, { + prefix: useTemplatesBucket.base, + limit: 100, + }) + ]).then(() => { + writable.close() + })) return readable }); From 5fc4b045ee9d48f8cfc4b36833e8c0b08700545c Mon Sep 17 00:00:00 2001 From: Mohammad Bagher Abiyat Date: Mon, 12 May 2025 00:05:25 +0330 Subject: [PATCH 06/22] i was doing it wrong baby --- packages/app/server/routes/rm/stale.post.ts | 30 ++++++++++----------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/packages/app/server/routes/rm/stale.post.ts b/packages/app/server/routes/rm/stale.post.ts index 4eb714be..7aaf5871 100644 --- a/packages/app/server/routes/rm/stale.post.ts +++ b/packages/app/server/routes/rm/stale.post.ts @@ -11,19 +11,19 @@ export default eventHandler(async (event) => { // } const { readable, writable } = new TransformStream() - event.context.cloudflare.context.waitUntil(Promise.all([ - iterateAndDelete(event, writable, signal, { - prefix: usePackagesBucket.base, - limit: 100, - }), - - iterateAndDelete(event, writable, signal, { - prefix: useTemplatesBucket.base, - limit: 100, - }) - ]).then(() => { - writable.close() - })) + event.context.cloudflare.context.waitUntil( + (async () => { + await iterateAndDelete(event, writable, signal, { + prefix: usePackagesBucket.base, + limit: 100, + }) + await iterateAndDelete(event, writable, signal, { + prefix: useTemplatesBucket.base, + limit: 100, + }) + writable.close() + })() + ) return readable }); @@ -51,7 +51,7 @@ async function iterateAndDelete(event: H3Event, writable: WritableStream, signal const uploaded = Date.parse(object.uploaded.toString()); // remove the object anyway if it's 6 months old already if ((today - uploaded) / (1000 * 3600 * 24 * 30 * 6) >= 1) { - writer.write(JSON.stringify({ + await writer.write(JSON.stringify({ key: object.key, uploaded: new Date(object.uploaded), downloadedAt: new Date((await downloadedAtBucket.getItem(object.key))!), @@ -67,7 +67,7 @@ async function iterateAndDelete(event: H3Event, writable: WritableStream, signal !((today - downloadedAt) / (1000 * 3600 * 24 * 30) < 1) && (today - uploaded) / (1000 * 3600 * 24 * 30) >= 1 ) { - writer.write(JSON.stringify({ + await writer.write(JSON.stringify({ key: object.key, uploaded: new Date(object.uploaded), downloadedAt: new Date(downloadedAt), From 5bdc933dd7b8506ec6b192583053cf1f50b0fd81 Mon Sep 17 00:00:00 2001 From: Mohammad Bagher Abiyat Date: Mon, 12 May 2025 00:06:41 +0330 Subject: [PATCH 07/22] await it --- packages/app/server/routes/rm/stale.post.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/app/server/routes/rm/stale.post.ts b/packages/app/server/routes/rm/stale.post.ts index 7aaf5871..bd3136db 100644 --- a/packages/app/server/routes/rm/stale.post.ts +++ b/packages/app/server/routes/rm/stale.post.ts @@ -21,7 +21,7 @@ export default eventHandler(async (event) => { prefix: useTemplatesBucket.base, limit: 100, }) - writable.close() + await writable.close() })() ) From 57cf92dd60f80b57222d18203f89fa6fafb1309d Mon Sep 17 00:00:00 2001 From: Mohammad Bagher Abiyat Date: Mon, 12 May 2025 09:32:55 +0330 Subject: [PATCH 08/22] use dates --- packages/app/server/routes/rm/stale.post.ts | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/packages/app/server/routes/rm/stale.post.ts b/packages/app/server/routes/rm/stale.post.ts index bd3136db..731517ca 100644 --- a/packages/app/server/routes/rm/stale.post.ts +++ b/packages/app/server/routes/rm/stale.post.ts @@ -49,8 +49,13 @@ async function iterateAndDelete(event: H3Event, writable: WritableStream, signal break; } const uploaded = Date.parse(object.uploaded.toString()); + console.log('uploaded', today, uploaded, today - uploaded, (today - uploaded) / (1000 * 3600 * 24 * 30 * 6)) // remove the object anyway if it's 6 months old already - if ((today - uploaded) / (1000 * 3600 * 24 * 30 * 6) >= 1) { + // Use calendar-accurate 6 months check + const uploadedDate = new Date(uploaded); + const sixMonthsAgo = new Date(today); + sixMonthsAgo.setMonth(sixMonthsAgo.getMonth() - 6); + if (uploadedDate <= sixMonthsAgo) { await writer.write(JSON.stringify({ key: object.key, uploaded: new Date(object.uploaded), @@ -63,9 +68,14 @@ async function iterateAndDelete(event: H3Event, writable: WritableStream, signal } const downloadedAt = (await downloadedAtBucket.getItem(object.key))!; // if it has not been downloaded in the last month and it's at least 1 month old + // Calendar-accurate 1 month checks + const downloadedAtDate = new Date(downloadedAt); + const oneMonthAgo = new Date(today); + oneMonthAgo.setMonth(oneMonthAgo.getMonth() - 1); + const uploadedDate2 = new Date(uploaded); // uploaded already parsed above if ( - !((today - downloadedAt) / (1000 * 3600 * 24 * 30) < 1) && - (today - uploaded) / (1000 * 3600 * 24 * 30) >= 1 + downloadedAtDate <= oneMonthAgo && + uploadedDate2 <= oneMonthAgo ) { await writer.write(JSON.stringify({ key: object.key, From 3d49091d28d549d8de7be1eacb5f35e4d4918af3 Mon Sep 17 00:00:00 2001 From: Mohammad Bagher Abiyat Date: Mon, 12 May 2025 09:40:15 +0330 Subject: [PATCH 09/22] more general thing --- packages/app/server/routes/rm/stale.post.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/app/server/routes/rm/stale.post.ts b/packages/app/server/routes/rm/stale.post.ts index 731517ca..9d1d9968 100644 --- a/packages/app/server/routes/rm/stale.post.ts +++ b/packages/app/server/routes/rm/stale.post.ts @@ -49,7 +49,11 @@ async function iterateAndDelete(event: H3Event, writable: WritableStream, signal break; } const uploaded = Date.parse(object.uploaded.toString()); - console.log('uploaded', today, uploaded, today - uploaded, (today - uploaded) / (1000 * 3600 * 24 * 30 * 6)) + writer.write(JSON.stringify({ + key: object.key, + uploaded: new Date(object.uploaded), + downloadedAt: new Date((await downloadedAtBucket.getItem(object.key))!), + }) + "\n") // remove the object anyway if it's 6 months old already // Use calendar-accurate 6 months check const uploadedDate = new Date(uploaded); From 5d78882ed062ce91cc4da2c6f93bc0a22032c16f Mon Sep 17 00:00:00 2001 From: Mohammad Bagher Abiyat Date: Mon, 12 May 2025 09:46:52 +0330 Subject: [PATCH 10/22] update --- packages/app/server/routes/rm/stale.post.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/app/server/routes/rm/stale.post.ts b/packages/app/server/routes/rm/stale.post.ts index 9d1d9968..89a6a328 100644 --- a/packages/app/server/routes/rm/stale.post.ts +++ b/packages/app/server/routes/rm/stale.post.ts @@ -11,7 +11,7 @@ export default eventHandler(async (event) => { // } const { readable, writable } = new TransformStream() - event.context.cloudflare.context.waitUntil( + event.waitUntil( (async () => { await iterateAndDelete(event, writable, signal, { prefix: usePackagesBucket.base, From eb46aac36e407faa81665b3342ff7aa22b588b7e Mon Sep 17 00:00:00 2001 From: Mohammad Bagher Abiyat Date: Mon, 12 May 2025 09:48:28 +0330 Subject: [PATCH 11/22] write some arbitrary date --- packages/app/server/routes/rm/stale.post.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/app/server/routes/rm/stale.post.ts b/packages/app/server/routes/rm/stale.post.ts index 89a6a328..31e33a27 100644 --- a/packages/app/server/routes/rm/stale.post.ts +++ b/packages/app/server/routes/rm/stale.post.ts @@ -11,6 +11,11 @@ export default eventHandler(async (event) => { // } const { readable, writable } = new TransformStream() + const writer = writable.getWriter() + await writer.ready + writer.write("start\n") + writer.releaseLock() + event.waitUntil( (async () => { await iterateAndDelete(event, writable, signal, { @@ -30,6 +35,7 @@ export default eventHandler(async (event) => { async function iterateAndDelete(event: H3Event, writable: WritableStream, signal: AbortSignal, opts: R2ListOptions) { const writer = writable.getWriter() + await writer.ready const binding = useBinding(event); let truncated = true; From 2a7f9e24b4d9fddd4ace7632e8d603d87ddd6a34 Mon Sep 17 00:00:00 2001 From: Mohammad Bagher Abiyat Date: Mon, 12 May 2025 10:04:18 +0330 Subject: [PATCH 12/22] remove runtime config --- packages/app/server/routes/rm/stale.post.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/app/server/routes/rm/stale.post.ts b/packages/app/server/routes/rm/stale.post.ts index 31e33a27..541a091f 100644 --- a/packages/app/server/routes/rm/stale.post.ts +++ b/packages/app/server/routes/rm/stale.post.ts @@ -3,7 +3,7 @@ import type { H3Event } from "h3"; export default eventHandler(async (event) => { const rmStaleKeyHeader = getHeader(event, "sb-rm-stale-key"); const signal = toWebRequest(event).signal; - const { rmStaleKey } = useRuntimeConfig(event); + // const { rmStaleKey } = useRuntimeConfig(event); // if (rmStaleKeyHeader !== rmStaleKey) { // throw createError({ // status: 403, From c2349a3484ac58dac311cfe889ae927902904a1d Mon Sep 17 00:00:00 2001 From: Mohammad Bagher Abiyat Date: Mon, 12 May 2025 10:21:38 +0330 Subject: [PATCH 13/22] trying it right baby --- packages/app/server/routes/rm/stale.post.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/app/server/routes/rm/stale.post.ts b/packages/app/server/routes/rm/stale.post.ts index 541a091f..0fa53b55 100644 --- a/packages/app/server/routes/rm/stale.post.ts +++ b/packages/app/server/routes/rm/stale.post.ts @@ -1,6 +1,8 @@ import type { H3Event } from "h3"; export default eventHandler(async (event) => { + setResponseHeader(event, "Transfer-Encoding", "chunked"); + const rmStaleKeyHeader = getHeader(event, "sb-rm-stale-key"); const signal = toWebRequest(event).signal; // const { rmStaleKey } = useRuntimeConfig(event); @@ -13,7 +15,7 @@ export default eventHandler(async (event) => { const writer = writable.getWriter() await writer.ready - writer.write("start\n") + await writer.write("start\n") writer.releaseLock() event.waitUntil( @@ -30,7 +32,7 @@ export default eventHandler(async (event) => { })() ) - return readable + return sendStream(event, readable) }); async function iterateAndDelete(event: H3Event, writable: WritableStream, signal: AbortSignal, opts: R2ListOptions) { From 18ef6c476d0be149ebc1fb1c3b3fd1642cc16342 Mon Sep 17 00:00:00 2001 From: Mohammad Bagher Abiyat Date: Mon, 12 May 2025 10:31:36 +0330 Subject: [PATCH 14/22] update --- packages/app/server/routes/rm/stale.post.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/app/server/routes/rm/stale.post.ts b/packages/app/server/routes/rm/stale.post.ts index 0fa53b55..9bf43bb6 100644 --- a/packages/app/server/routes/rm/stale.post.ts +++ b/packages/app/server/routes/rm/stale.post.ts @@ -2,6 +2,7 @@ import type { H3Event } from "h3"; export default eventHandler(async (event) => { setResponseHeader(event, "Transfer-Encoding", "chunked"); + setResponseHeader(event, "Content-Type", "text/plain"); const rmStaleKeyHeader = getHeader(event, "sb-rm-stale-key"); const signal = toWebRequest(event).signal; From c31bb1eb7666a8cdbe6a4ab07993cc9f1de3bf97 Mon Sep 17 00:00:00 2001 From: Mohammad Bagher Abiyat Date: Mon, 12 May 2025 10:35:45 +0330 Subject: [PATCH 15/22] return stream --- packages/app/server/routes/rm/stale.post.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/app/server/routes/rm/stale.post.ts b/packages/app/server/routes/rm/stale.post.ts index 9bf43bb6..5ce87d96 100644 --- a/packages/app/server/routes/rm/stale.post.ts +++ b/packages/app/server/routes/rm/stale.post.ts @@ -33,7 +33,7 @@ export default eventHandler(async (event) => { })() ) - return sendStream(event, readable) + return readable }); async function iterateAndDelete(event: H3Event, writable: WritableStream, signal: AbortSignal, opts: R2ListOptions) { From 385185e3b7f22b71378270fc24964c742b0ac4a4 Mon Sep 17 00:00:00 2001 From: Mohammad Bagher Abiyat Date: Mon, 12 May 2025 11:55:13 +0330 Subject: [PATCH 16/22] update --- packages/app/server/routes/rm/stale.post.ts | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/packages/app/server/routes/rm/stale.post.ts b/packages/app/server/routes/rm/stale.post.ts index 5ce87d96..c1dafd33 100644 --- a/packages/app/server/routes/rm/stale.post.ts +++ b/packages/app/server/routes/rm/stale.post.ts @@ -2,6 +2,7 @@ import type { H3Event } from "h3"; export default eventHandler(async (event) => { setResponseHeader(event, "Transfer-Encoding", "chunked"); + setResponseHeader(event, "Cache-Control", "no-cache"); setResponseHeader(event, "Content-Type", "text/plain"); const rmStaleKeyHeader = getHeader(event, "sb-rm-stale-key"); @@ -14,13 +15,14 @@ export default eventHandler(async (event) => { // } const { readable, writable } = new TransformStream() - const writer = writable.getWriter() - await writer.ready - await writer.write("start\n") - writer.releaseLock() - event.waitUntil( (async () => { + // const writer = writable.getWriter() + // console.log('here') + // await writer.ready + // await writer.write("start\n") + // writer.releaseLock() + await iterateAndDelete(event, writable, signal, { prefix: usePackagesBucket.base, limit: 100, From 76b0d74ddfa9837ffe1c726c5ee44df7d1416f52 Mon Sep 17 00:00:00 2001 From: Mohammad Bagher Abiyat Date: Mon, 12 May 2025 12:08:11 +0330 Subject: [PATCH 17/22] debug --- packages/app/server/routes/rm/stale.post.ts | 28 ++++++++++----------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/packages/app/server/routes/rm/stale.post.ts b/packages/app/server/routes/rm/stale.post.ts index c1dafd33..b8d210b6 100644 --- a/packages/app/server/routes/rm/stale.post.ts +++ b/packages/app/server/routes/rm/stale.post.ts @@ -17,21 +17,21 @@ export default eventHandler(async (event) => { event.waitUntil( (async () => { - // const writer = writable.getWriter() - // console.log('here') - // await writer.ready - // await writer.write("start\n") - // writer.releaseLock() + const writer = writable.getWriter() + console.log('here') + await writer.ready + await writer.write(new TextEncoder().encode("start\n")) + writer.releaseLock() - await iterateAndDelete(event, writable, signal, { - prefix: usePackagesBucket.base, - limit: 100, - }) - await iterateAndDelete(event, writable, signal, { - prefix: useTemplatesBucket.base, - limit: 100, - }) - await writable.close() + // await iterateAndDelete(event, writable, signal, { + // prefix: usePackagesBucket.base, + // limit: 100, + // }) + // await iterateAndDelete(event, writable, signal, { + // prefix: useTemplatesBucket.base, + // limit: 100, + // }) + // await writable.close() })() ) From ef8e990347528b5791c0da6211492b8a40dc49a1 Mon Sep 17 00:00:00 2001 From: Mohammad Bagher Abiyat Date: Mon, 12 May 2025 12:11:35 +0330 Subject: [PATCH 18/22] i think i figured it out --- packages/app/server/routes/rm/stale.post.ts | 40 ++++++++++----------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/packages/app/server/routes/rm/stale.post.ts b/packages/app/server/routes/rm/stale.post.ts index b8d210b6..e6bcdd7a 100644 --- a/packages/app/server/routes/rm/stale.post.ts +++ b/packages/app/server/routes/rm/stale.post.ts @@ -17,21 +17,21 @@ export default eventHandler(async (event) => { event.waitUntil( (async () => { - const writer = writable.getWriter() - console.log('here') - await writer.ready - await writer.write(new TextEncoder().encode("start\n")) - writer.releaseLock() + // const writer = writable.getWriter() + // console.log('here') + // await writer.ready + // await writer.write(new TextEncoder().encode("start\n")) + // writer.releaseLock() - // await iterateAndDelete(event, writable, signal, { - // prefix: usePackagesBucket.base, - // limit: 100, - // }) - // await iterateAndDelete(event, writable, signal, { - // prefix: useTemplatesBucket.base, - // limit: 100, - // }) - // await writable.close() + await iterateAndDelete(event, writable, signal, { + prefix: usePackagesBucket.base, + limit: 100, + }) + await iterateAndDelete(event, writable, signal, { + prefix: useTemplatesBucket.base, + limit: 100, + }) + await writable.close() })() ) @@ -60,22 +60,22 @@ async function iterateAndDelete(event: H3Event, writable: WritableStream, signal break; } const uploaded = Date.parse(object.uploaded.toString()); - writer.write(JSON.stringify({ + await writer.write(new TextEncoder().encode(JSON.stringify({ key: object.key, uploaded: new Date(object.uploaded), downloadedAt: new Date((await downloadedAtBucket.getItem(object.key))!), - }) + "\n") + }) + "\n")) // remove the object anyway if it's 6 months old already // Use calendar-accurate 6 months check const uploadedDate = new Date(uploaded); const sixMonthsAgo = new Date(today); sixMonthsAgo.setMonth(sixMonthsAgo.getMonth() - 6); if (uploadedDate <= sixMonthsAgo) { - await writer.write(JSON.stringify({ + await writer.write(new TextEncoder().encode(JSON.stringify({ key: object.key, uploaded: new Date(object.uploaded), downloadedAt: new Date((await downloadedAtBucket.getItem(object.key))!), - }) + "\n") + }) + "\n")) // event.context.cloudflare.context.waitUntil(binding.delete(object.key)); // event.context.cloudflare.context.waitUntil( // downloadedAtBucket.removeItem(object.key), @@ -92,11 +92,11 @@ async function iterateAndDelete(event: H3Event, writable: WritableStream, signal downloadedAtDate <= oneMonthAgo && uploadedDate2 <= oneMonthAgo ) { - await writer.write(JSON.stringify({ + await writer.write(new TextEncoder().encode(JSON.stringify({ key: object.key, uploaded: new Date(object.uploaded), downloadedAt: new Date(downloadedAt), - }) + "\n") + }) + "\n")) // event.context.cloudflare.context.waitUntil(binding.delete(object.key)); // event.context.cloudflare.context.waitUntil( // downloadedAtBucket.removeItem(object.key), From 79b5249f99f5efa17cf6c2e9d40923d533f2b943 Mon Sep 17 00:00:00 2001 From: Mohammad Bagher Abiyat Date: Mon, 12 May 2025 12:15:56 +0330 Subject: [PATCH 19/22] got it working --- packages/app/server/routes/rm/stale.post.ts | 5 ----- 1 file changed, 5 deletions(-) diff --git a/packages/app/server/routes/rm/stale.post.ts b/packages/app/server/routes/rm/stale.post.ts index e6bcdd7a..72a520c0 100644 --- a/packages/app/server/routes/rm/stale.post.ts +++ b/packages/app/server/routes/rm/stale.post.ts @@ -60,11 +60,6 @@ async function iterateAndDelete(event: H3Event, writable: WritableStream, signal break; } const uploaded = Date.parse(object.uploaded.toString()); - await writer.write(new TextEncoder().encode(JSON.stringify({ - key: object.key, - uploaded: new Date(object.uploaded), - downloadedAt: new Date((await downloadedAtBucket.getItem(object.key))!), - }) + "\n")) // remove the object anyway if it's 6 months old already // Use calendar-accurate 6 months check const uploadedDate = new Date(uploaded); From 312ce3468fe23aed637af680ae9033628af14e85 Mon Sep 17 00:00:00 2001 From: Mohammad Bagher Abiyat Date: Mon, 12 May 2025 13:16:17 +0330 Subject: [PATCH 20/22] make sure downloadedAt exists first --- packages/app/server/routes/rm/stale.post.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/app/server/routes/rm/stale.post.ts b/packages/app/server/routes/rm/stale.post.ts index 72a520c0..0313b2c5 100644 --- a/packages/app/server/routes/rm/stale.post.ts +++ b/packages/app/server/routes/rm/stale.post.ts @@ -76,7 +76,11 @@ async function iterateAndDelete(event: H3Event, writable: WritableStream, signal // downloadedAtBucket.removeItem(object.key), // ); } - const downloadedAt = (await downloadedAtBucket.getItem(object.key))!; + const downloadedAt = (await downloadedAtBucket.getItem(object.key)); + + if (!downloadedAt) { + continue; + } // if it has not been downloaded in the last month and it's at least 1 month old // Calendar-accurate 1 month checks const downloadedAtDate = new Date(downloadedAt); From a6eb46ce23564d51037eb7c9893b58d467992057 Mon Sep 17 00:00:00 2001 From: Mohammad Bagher Abiyat Date: Mon, 12 May 2025 13:22:04 +0330 Subject: [PATCH 21/22] null instead of 1970 --- packages/app/server/routes/rm/stale.post.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/app/server/routes/rm/stale.post.ts b/packages/app/server/routes/rm/stale.post.ts index 0313b2c5..806124e5 100644 --- a/packages/app/server/routes/rm/stale.post.ts +++ b/packages/app/server/routes/rm/stale.post.ts @@ -66,10 +66,11 @@ async function iterateAndDelete(event: H3Event, writable: WritableStream, signal const sixMonthsAgo = new Date(today); sixMonthsAgo.setMonth(sixMonthsAgo.getMonth() - 6); if (uploadedDate <= sixMonthsAgo) { + const downloadedAt = (await downloadedAtBucket.getItem(object.key))!; await writer.write(new TextEncoder().encode(JSON.stringify({ key: object.key, uploaded: new Date(object.uploaded), - downloadedAt: new Date((await downloadedAtBucket.getItem(object.key))!), + downloadedAt: downloadedAt ? new Date(downloadedAt) : null, }) + "\n")) // event.context.cloudflare.context.waitUntil(binding.delete(object.key)); // event.context.cloudflare.context.waitUntil( From b613ccd8231999d292995ad18e3a98a50befee75 Mon Sep 17 00:00:00 2001 From: Mohammad Bagher Abiyat Date: Mon, 12 May 2025 13:35:11 +0330 Subject: [PATCH 22/22] finalize --- .github/workflows/remove-stale.yml | 13 +++++++++++-- packages/app/server/routes/rm/stale.post.ts | 14 ++++++++------ 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/.github/workflows/remove-stale.yml b/.github/workflows/remove-stale.yml index fd0b8289..aeba7b76 100644 --- a/.github/workflows/remove-stale.yml +++ b/.github/workflows/remove-stale.yml @@ -5,8 +5,17 @@ jobs: rm_stale_packages: runs-on: ubuntu-latest steps: - - name: Send POST request + - name: Send POST request and capture output run: | curl -X POST \ + -N \ -H "sb-rm-stale-key: ${{ secrets.NITRO_RM_STALE_KEY }}" \ - https://pkg.pr.new/rm/stale + https://pkg.pr.new/rm/stale \ + | tee rm-stale.log + + - name: Upload rm-stale.log as artifact + uses: actions/upload-artifact@v4 + with: + name: rm-stale-log + path: rm-stale.log + diff --git a/packages/app/server/routes/rm/stale.post.ts b/packages/app/server/routes/rm/stale.post.ts index 806124e5..5db92acb 100644 --- a/packages/app/server/routes/rm/stale.post.ts +++ b/packages/app/server/routes/rm/stale.post.ts @@ -7,12 +7,14 @@ export default eventHandler(async (event) => { const rmStaleKeyHeader = getHeader(event, "sb-rm-stale-key"); const signal = toWebRequest(event).signal; - // const { rmStaleKey } = useRuntimeConfig(event); - // if (rmStaleKeyHeader !== rmStaleKey) { - // throw createError({ - // status: 403, - // }); - // } + const { rmStaleKey } = useRuntimeConfig(event); + + if (rmStaleKeyHeader !== rmStaleKey) { + throw createError({ + status: 403, + }); + } + const { readable, writable } = new TransformStream() event.waitUntil(