diff --git a/.github/workflows/remove-stale.yml b/.github/workflows/remove-stale.yml index fd0b8289..aeba7b76 100644 --- a/.github/workflows/remove-stale.yml +++ b/.github/workflows/remove-stale.yml @@ -5,8 +5,17 @@ jobs: rm_stale_packages: runs-on: ubuntu-latest steps: - - name: Send POST request + - name: Send POST request and capture output run: | curl -X POST \ + -N \ -H "sb-rm-stale-key: ${{ secrets.NITRO_RM_STALE_KEY }}" \ - https://pkg.pr.new/rm/stale + https://pkg.pr.new/rm/stale \ + | tee rm-stale.log + + - name: Upload rm-stale.log as artifact + uses: actions/upload-artifact@v4 + with: + name: rm-stale-log + path: rm-stale.log + diff --git a/packages/app/server/routes/rm/stale.post.ts b/packages/app/server/routes/rm/stale.post.ts index 5ce3afe9..5db92acb 100644 --- a/packages/app/server/routes/rm/stale.post.ts +++ b/packages/app/server/routes/rm/stale.post.ts @@ -1,31 +1,48 @@ import type { H3Event } from "h3"; export default eventHandler(async (event) => { + setResponseHeader(event, "Transfer-Encoding", "chunked"); + setResponseHeader(event, "Cache-Control", "no-cache"); + setResponseHeader(event, "Content-Type", "text/plain"); + const rmStaleKeyHeader = getHeader(event, "sb-rm-stale-key"); + const signal = toWebRequest(event).signal; const { rmStaleKey } = useRuntimeConfig(event); + if (rmStaleKeyHeader !== rmStaleKey) { throw createError({ status: 403, }); } - return { - ok: true, - removed: [ - ...(await Promise.all([ - iterateAndDelete(event, { - prefix: usePackagesBucket.base, - limit: 100, - }), - iterateAndDelete(event, { - prefix: useTemplatesBucket.base, - limit: 100, - }), - ]).then((results) => results.flat())), - ], - }; + + const { readable, writable } = new TransformStream() + + event.waitUntil( + (async () => { + // const writer = writable.getWriter() + // console.log('here') + // await writer.ready + // await writer.write(new TextEncoder().encode("start\n")) + // writer.releaseLock() + + await iterateAndDelete(event, writable, signal, { + prefix: usePackagesBucket.base, + limit: 100, + }) + await iterateAndDelete(event, writable, signal, { + prefix: useTemplatesBucket.base, + limit: 100, + }) + await writable.close() + })() + ) + + return readable }); -async function iterateAndDelete(event: H3Event, opts: R2ListOptions) { +async function iterateAndDelete(event: H3Event, writable: WritableStream, signal: AbortSignal, opts: R2ListOptions) { + const writer = writable.getWriter() + await writer.ready const binding = useBinding(event); let truncated = true; @@ -34,34 +51,58 @@ async function iterateAndDelete(event: H3Event, opts: R2ListOptions) { const downloadedAtBucket = useDownloadedAtBucket(event); const today = Date.parse(new Date().toString()); - const removed: string[] = []; - while (truncated) { + while (truncated && !signal.aborted) { // TODO: Avoid using context.cloudflare and migrate to unstorage, but it does not have truncated for now const next = await binding.list({ ...opts, cursor, }); for (const object of next.objects) { + if (signal.aborted) { + break; + } const uploaded = Date.parse(object.uploaded.toString()); // remove the object anyway if it's 6 months old already - if ((today - uploaded) / (1000 * 3600 * 24 * 30 * 6) >= 1) { - removed.push(object.key); - event.context.cloudflare.context.waitUntil(binding.delete(object.key)); - event.context.cloudflare.context.waitUntil( - downloadedAtBucket.removeItem(object.key), - ); + // Use calendar-accurate 6 months check + const uploadedDate = new Date(uploaded); + const sixMonthsAgo = new Date(today); + sixMonthsAgo.setMonth(sixMonthsAgo.getMonth() - 6); + if (uploadedDate <= sixMonthsAgo) { + const downloadedAt = (await downloadedAtBucket.getItem(object.key))!; + await writer.write(new TextEncoder().encode(JSON.stringify({ + key: object.key, + uploaded: new Date(object.uploaded), + downloadedAt: downloadedAt ? new Date(downloadedAt) : null, + }) + "\n")) + // event.context.cloudflare.context.waitUntil(binding.delete(object.key)); + // event.context.cloudflare.context.waitUntil( + // downloadedAtBucket.removeItem(object.key), + // ); + } + const downloadedAt = (await downloadedAtBucket.getItem(object.key)); + + if (!downloadedAt) { + continue; } - const downloadedAt = (await downloadedAtBucket.getItem(object.key))!; // if it has not been downloaded in the last month and it's at least 1 month old + // Calendar-accurate 1 month checks + const downloadedAtDate = new Date(downloadedAt); + const oneMonthAgo = new Date(today); + oneMonthAgo.setMonth(oneMonthAgo.getMonth() - 1); + const uploadedDate2 = new Date(uploaded); // uploaded already parsed above if ( - !((today - downloadedAt) / (1000 * 3600 * 24 * 30) < 1) && - (today - uploaded) / (1000 * 3600 * 24 * 30) >= 1 + downloadedAtDate <= oneMonthAgo && + uploadedDate2 <= oneMonthAgo ) { - removed.push(object.key); - event.context.cloudflare.context.waitUntil(binding.delete(object.key)); - event.context.cloudflare.context.waitUntil( - downloadedAtBucket.removeItem(object.key), - ); + await writer.write(new TextEncoder().encode(JSON.stringify({ + key: object.key, + uploaded: new Date(object.uploaded), + downloadedAt: new Date(downloadedAt), + }) + "\n")) + // event.context.cloudflare.context.waitUntil(binding.delete(object.key)); + // event.context.cloudflare.context.waitUntil( + // downloadedAtBucket.removeItem(object.key), + // ); } } @@ -70,5 +111,5 @@ async function iterateAndDelete(event: H3Event, opts: R2ListOptions) { cursor = next.cursor; } } - return removed; + writer.releaseLock() }