From 07974ac391d87400fa42e194f4e4ffc82881ca16 Mon Sep 17 00:00:00 2001 From: slugb0t Date: Wed, 1 Jul 2026 14:46:18 -0700 Subject: [PATCH 1/2] wip: :construction: Zenodo timeout fix --- .../[owner]/[repo]/release/zenodo.vue | 109 +++++++++- .../[repo]/release/zenodo/index.post.ts | 11 + ui/server/services/archival/zenodo.ts | 189 ++++++++---------- 3 files changed, 200 insertions(+), 109 deletions(-) diff --git a/ui/pages/dashboard/[owner]/[repo]/release/zenodo.vue b/ui/pages/dashboard/[owner]/[repo]/release/zenodo.vue index d3e050d5..9720ba65 100644 --- a/ui/pages/dashboard/[owner]/[repo]/release/zenodo.vue +++ b/ui/pages/dashboard/[owner]/[repo]/release/zenodo.vue @@ -596,6 +596,62 @@ const navigateToDashboard = async () => { await navigateTo(`/dashboard/${owner}/${repo}/`); }; +const reconcilePublishStatus = async () => { + // The SSE stream ended without a terminal event - the proxy/browser likely + // dropped the long-lived connection. The backend may still be finishing, so + // poll the authoritative DB status a few times before deciding the outcome. + const maxAttempts = 5; + const delayMs = 3000; + + for (let attempt = 0; attempt < maxAttempts; attempt++) { + try { + const status = await $fetch<{ + zenodoDoi: string; + zenodoWorkflowStatus: string; + }>(`/api/${owner}/${repo}/release/zenodo/status`, { + credentials: "include", + }); + + if (status.zenodoWorkflowStatus === "published") { + // Release actually succeeded + zenodoPublishStatus.value = "published"; + if (status.zenodoDoi) { + zenodoPublishDOI.value = status.zenodoDoi; + } + for (const step of publishSteps.value) { + if (step.status !== "error") { + step.status = "completed"; + } + } + resetForm(); + return; + } + + if (status.zenodoWorkflowStatus === "error") { + zenodoPublishStatus.value = "error"; + zenodoPublishErrorMessage.value = + "The release failed on the server. Please review your settings and try again."; + const inProgress = publishSteps.value.find( + (s) => s.status === "in_progress", + ); + if (inProgress) { + inProgress.status = "error"; + } + return; + } + } catch { + // retry on the next attempt + } + + if (attempt < maxAttempts - 1) { + await new Promise((resolve) => setTimeout(resolve, delayMs)); + } + } + + zenodoPublishStatus.value = "pending"; + zenodoPublishErrorMessage.value = ""; +}; + const startZenodoPublishProcess = async (shouldPublish: boolean = false) => { if (shouldPublish) { zenodoPublishSpinner.value = true; @@ -620,6 +676,10 @@ const startZenodoPublishProcess = async (shouldPublish: boolean = false) => { zenodoPreflightError.value = null; + // Track whether the SSE stream delivered a terminal (complete/error) event. + let streamingStarted = false; + let receivedTerminal = false; + try { const response = await fetch(`/api/${owner}/${repo}/release/zenodo`, { body, @@ -692,6 +752,7 @@ const startZenodoPublishProcess = async (shouldPublish: boolean = false) => { showZenodoPublishProgressModal.value = true; zenodoPublishStatus.value = "inProgress"; resetPublishSteps(); + streamingStarted = true; const reader = response.body!.getReader(); const decoder = new TextDecoder(); @@ -710,6 +771,9 @@ const startZenodoPublishProcess = async (shouldPublish: boolean = false) => { try { const event = JSON.parse(line.slice(6)); handleSSEEvent(event); + if (event.step === "complete" || event.step === "error") { + receivedTerminal = true; + } } catch { // ignore malformed lines } @@ -718,14 +782,19 @@ const startZenodoPublishProcess = async (shouldPublish: boolean = false) => { } } catch (error) { console.error("Failed to start Zenodo publish process:", error); - push.error({ - title: "Failed to start Zenodo publish process", - message: "Please try again later", - }); - zenodoPublishStatus.value = "error"; + if (!streamingStarted) { + push.error({ + title: "Failed to start Zenodo publish process", + message: "Please try again later", + }); + zenodoPublishStatus.value = "error"; + } } finally { zenodoPublishSpinner.value = false; zenodoDraftSpinner.value = false; + if (shouldPublish && streamingStarted && !receivedTerminal) { + await reconcilePublishStatus(); + } fetchZenodoBadge(); } }; @@ -1848,6 +1917,12 @@ onBeforeUnmount(() => { class="h-6 w-6 text-blue-600" /> + +

{{ @@ -1857,7 +1932,9 @@ onBeforeUnmount(() => { ? "Zenodo publish error" : zenodoPublishStatus === "published" ? "Zenodo publish success" - : "Loading..." + : zenodoPublishStatus === "pending" + ? "Zenodo publish still processing" + : "Loading..." }}

@@ -2069,6 +2146,20 @@ onBeforeUnmount(() => { + + +

+ Your release is still finishing on the server. Large releases can take + a few minutes to upload and archive. You can safely leave this page — + check your repository dashboard shortly; the release may still + complete successfully. +

+ +

Please wait while we get the status of your workflow.

@@ -2113,6 +2204,12 @@ onBeforeUnmount(() => { Return to Dashboard + + + + Return to Dashboard + + diff --git a/ui/server/api/[owner]/[repo]/release/zenodo/index.post.ts b/ui/server/api/[owner]/[repo]/release/zenodo/index.post.ts index 2c17dfb1..1c039e58 100644 --- a/ui/server/api/[owner]/[repo]/release/zenodo/index.post.ts +++ b/ui/server/api/[owner]/[repo]/release/zenodo/index.post.ts @@ -245,6 +245,15 @@ export default defineEventHandler(async (event) => { } }; + // Keep the connection alive during long steps. Without traffic, an idle reverse proxy / load + // balancer / browser can silently drop the stream, making a successful + // release look failed to the client. + const heartbeat = setInterval(() => { + if (clientConnected && !res.writableEnded) { + res.write(": ping\n\n"); + } + }, 15000); + /** * Forwards a publication progress event to the SSE stream. * @param progress - Progress event from the Zenodo publication workflow. @@ -294,6 +303,8 @@ export default defineEventHandler(async (event) => { status: "error", step: "error", }); + } finally { + clearInterval(heartbeat); } if (!res.writableEnded) { diff --git a/ui/server/services/archival/zenodo.ts b/ui/server/services/archival/zenodo.ts index b8f31048..3b854b62 100644 --- a/ui/server/services/archival/zenodo.ts +++ b/ui/server/services/archival/zenodo.ts @@ -19,6 +19,7 @@ import type { ArchivalTokenValidation, ExistingDeposition, ProgressCallback, + ProgressStep, PublicationResult, } from "./interface"; import { GitHubRepositoryProvider } from "~/server/services/providers/github"; @@ -45,6 +46,12 @@ function zenodoEndpoint(): string { return process.env.ZENODO_ENDPOINT ?? ""; } +// Request timeouts so a stalled network call can never hang the publication +// workflow indefinitely. Metadata/API calls are quick; file up/downloads and +// the repo zipball can be large, so they get a much longer budget. +const ZENODO_API_TIMEOUT_MS = 60_000; // metadata / publish / deposition / API calls +const ZENODO_UPLOAD_TIMEOUT_MS = 300_000; // file up/downloads & repo zipball + // ===== Error helper ================================================== /** @@ -91,6 +98,7 @@ async function refreshZenodoToken( body: body.toString(), headers: { "Content-Type": "application/x-www-form-urlencoded" }, method: "POST", + signal: AbortSignal.timeout(ZENODO_API_TIMEOUT_MS), }); if (!res.ok) { @@ -153,6 +161,7 @@ export async function validateZenodoToken( const res = await fetch(`${zenodoApiEndpoint()}/deposit/depositions`, { headers: { Authorization: `Bearer ${tokenRecord.token}` }, + signal: AbortSignal.timeout(ZENODO_API_TIMEOUT_MS), }); if (!res.ok) { @@ -213,6 +222,7 @@ async function createNewZenodoDeposition(token: string): Promise { "Content-Type": "application/json", }, method: "POST", + signal: AbortSignal.timeout(ZENODO_API_TIMEOUT_MS), }); if (!res.ok) { @@ -244,7 +254,10 @@ async function fetchExistingZenodoDeposition( ): Promise { const latestRes = await fetch( `${zenodoApiEndpoint()}/records/${depositionId}/versions/latest`, - { headers: { Authorization: `Bearer ${token}` } }, + { + headers: { Authorization: `Bearer ${token}` }, + signal: AbortSignal.timeout(ZENODO_API_TIMEOUT_MS), + }, ); if (latestRes.status === 404) { @@ -256,7 +269,10 @@ async function fetchExistingZenodoDeposition( }); const draftRes = await fetch( `${zenodoApiEndpoint()}/deposit/depositions/${depositionId}`, - { headers: { Authorization: `Bearer ${token}` } }, + { + headers: { Authorization: `Bearer ${token}` }, + signal: AbortSignal.timeout(ZENODO_API_TIMEOUT_MS), + }, ); if (!draftRes.ok) { @@ -307,6 +323,7 @@ async function createNewVersionOfDeposition( const res = await fetch(url, { headers: { Authorization: `Bearer ${token}` }, method: "POST", + signal: AbortSignal.timeout(ZENODO_API_TIMEOUT_MS), }); if (!res.ok) { @@ -347,7 +364,10 @@ async function deleteFileFromZenodo( ): Promise { const res = await fetch( `${zenodoApiEndpoint()}/records/${depositionId}/draft/files/${encodeURIComponent(fileName)}?access_token=${token}`, - { method: "DELETE" }, + { + method: "DELETE", + signal: AbortSignal.timeout(ZENODO_API_TIMEOUT_MS), + }, ); if (!res.ok) { @@ -498,6 +518,7 @@ async function updateDepositionMetadata( "Content-Type": "application/json", }, method: "PUT", + signal: AbortSignal.timeout(ZENODO_API_TIMEOUT_MS), }); if (!res.ok) { @@ -555,6 +576,7 @@ async function uploadFileToZenodoBucket( "Content-Type": "application/octet-stream", }, method: "PUT", + signal: AbortSignal.timeout(ZENODO_UPLOAD_TIMEOUT_MS), }); if (!res.ok) { @@ -595,6 +617,7 @@ async function publishZenodoDeposition( "Content-Type": "application/json", }, method: "POST", + signal: AbortSignal.timeout(ZENODO_API_TIMEOUT_MS), }); if (!res.ok) { @@ -758,6 +781,7 @@ async function downloadRepositoryZip( "X-GitHub-Api-Version": "2022-11-28", }, redirect: "follow", + signal: AbortSignal.timeout(ZENODO_UPLOAD_TIMEOUT_MS), }, ); @@ -808,6 +832,7 @@ async function fetchReleaseAssets( Authorization: `Bearer ${userToken}`, "X-GitHub-Api-Version": "2022-11-28", }, + signal: AbortSignal.timeout(ZENODO_API_TIMEOUT_MS), }, ); @@ -859,6 +884,7 @@ async function downloadReleaseAsset( "X-GitHub-Api-Version": "2022-11-28", }, redirect: "follow", + signal: AbortSignal.timeout(ZENODO_UPLOAD_TIMEOUT_MS), }); if (!res.ok) { @@ -903,6 +929,7 @@ async function publishGitHubRelease( Authorization: `Bearer ${userToken}`, "X-GitHub-Api-Version": "2022-11-28", }, + signal: AbortSignal.timeout(ZENODO_API_TIMEOUT_MS), }); const defaultBranch = repoRes.ok @@ -935,6 +962,7 @@ async function publishGitHubRelease( "X-GitHub-Api-Version": "2022-11-28", }, method: "PATCH", + signal: AbortSignal.timeout(ZENODO_API_TIMEOUT_MS), }, ); @@ -1032,38 +1060,72 @@ export async function beginZenodoPublication( const token = tokenRecord.token; - // == Step 1: deposition =================================== - await onProgress?.({ - message: "Preparing Zenodo deposition…", - status: "in_progress", - step: "deposition", - }); + // Mark the deposition as actively publishing. This lets a dropped SSE stream + // be reconciled against the DB: "publishing" means the workflow is still + // running, distinct from a "draft" (saved but not started) or a terminal + // "published"/"error" state. The row was already upserted by the POST handler. + await prisma.zenodoDeposition + .update({ + data: { status: "publishing" }, + where: { repository_id: repositoryId }, + }) + .catch(() => undefined); - let deposition: any; - try { - deposition = await getWorkingDeposition(mode, depositionId, token); - } catch (err: any) { + /** + * Records a workflow step failure: logs it, marks the deposition row as + * errored (so a dropped SSE stream reconciles to a real failure instead of a + * false "still running"), emits an SSE error event, and returns the failed + * result. Used by every step's catch so failures are persisted uniformly. + * @param step - The workflow step that failed. + * @param err - The thrown error. + * @returns A failed PublicationResult carrying the error message. + */ + const failPublication = async ( + step: ProgressStep, + err: any, + ): Promise => { logwatch.error({ - action: "zenodo.publish.deposition", + action: `zenodo.publish.${step}`, error: err.stack ?? err.message, message: err.message, owner, repo, userId, }); + await prisma.zenodoDeposition + .update({ + data: { status: "error" }, + where: { repository_id: repositoryId }, + }) + .catch(() => undefined); await onProgress?.({ message: err.message, status: "error", - step: "deposition", + step, }); return { error: err.message, success: false }; + }; + + // == Step 1: deposition =================================== + await onProgress?.({ + message: "Preparing Zenodo deposition…", + status: "in_progress", + step: "deposition", + }); + + let deposition: any; + try { + deposition = await getWorkingDeposition(mode, depositionId, token); + } catch (err: any) { + return await failPublication("deposition", err); } const newDepositionId: number = deposition.record_id; const bucketUrl: string = deposition.links.bucket; const doi: string = deposition.metadata.prereserve_doi.doi; - // Upsert DB record (status: draft) + // Upsert DB record. Keep status "publishing" (set above) so the workflow + // stays reconcilable as in-progress through the long upload steps below. const existingDep = await prisma.zenodoDeposition.findFirst({ where: { repository_id: repositoryId }, }); @@ -1073,7 +1135,7 @@ export async function beginZenodoPublication( data: { github_release_id: parseInt(release) || null, github_tag_name: tag, - status: "draft", + status: "publishing", user_id: userId, zenodo_id: newDepositionId, zenodo_metadata: metadata, @@ -1087,7 +1149,7 @@ export async function beginZenodoPublication( github_release_id: parseInt(release) || null, github_tag_name: tag, repository_id: repositoryId, - status: "draft", + status: "publishing", user_id: userId, zenodo_id: newDepositionId, zenodo_metadata: metadata, @@ -1171,20 +1233,7 @@ export async function beginZenodoPublication( ); } } catch (err: any) { - logwatch.error({ - action: "zenodo.publish.metadata", - error: err.stack ?? err.message, - message: err.message, - owner, - repo, - userId, - }); - await onProgress?.({ - message: err.message, - status: "error", - step: "metadata", - }); - return { error: err.message, success: false }; + return await failPublication("metadata", err); } await onProgress?.({ @@ -1201,28 +1250,10 @@ export async function beginZenodoPublication( }); try { - const zenodoMeta = buildZenodoMetadata( - codemeta, - metadata, - doi, - licenseId, - ); + const zenodoMeta = buildZenodoMetadata(codemeta, metadata, doi, licenseId); await updateDepositionMetadata(newDepositionId, token, zenodoMeta); } catch (err: any) { - logwatch.error({ - action: "zenodo.publish.upload_metadata", - error: err.stack ?? err.message, - message: err.message, - owner, - repo, - userId, - }); - await onProgress?.({ - message: err.message, - status: "error", - step: "upload_metadata", - }); - return { error: err.message, success: false }; + return await failPublication("upload_metadata", err); } await onProgress?.({ @@ -1249,20 +1280,7 @@ export async function beginZenodoPublication( licenseId, ); } catch (err: any) { - logwatch.error({ - action: "zenodo.publish.update_repo", - error: err.stack ?? err.message, - message: err.message, - owner, - repo, - userId, - }); - await onProgress?.({ - message: err.message, - status: "error", - step: "update_repo", - }); - return { error: err.message, success: false }; + return await failPublication("update_repo", err); } await onProgress?.({ @@ -1297,20 +1315,7 @@ export async function beginZenodoPublication( const zipFilename = `${repo}-${tag}.zip`; await uploadFileToZenodoBucket(bucketUrl, token, zipFilename, zipBytes); } catch (err: any) { - logwatch.error({ - action: "zenodo.publish.upload_files", - error: err.stack ?? err.message, - message: err.message, - owner, - repo, - userId, - }); - await onProgress?.({ - message: err.message, - status: "error", - step: "upload_files", - }); - return { error: err.message, success: false }; + return await failPublication("upload_files", err); } await onProgress?.({ @@ -1362,29 +1367,7 @@ export async function beginZenodoPublication( }); } } catch (err: any) { - logwatch.error({ - action: "zenodo.publish.publish", - error: err.stack ?? err.message, - message: err.message, - owner, - repo, - userId, - }); - - // Mark DB as error - await prisma.zenodoDeposition - .update({ - data: { status: "error" }, - where: { repository_id: repositoryId }, - }) - .catch(() => undefined); - - await onProgress?.({ - message: err.message, - status: "error", - step: "publish", - }); - return { error: err.message, success: false }; + return await failPublication("publish", err); } await onProgress?.({ From 6fbb1a4f3f5922bb68c2fd45140fb89f09cd68a3 Mon Sep 17 00:00:00 2001 From: slugb0t Date: Fri, 10 Jul 2026 16:39:53 -0700 Subject: [PATCH 2/2] refactor: :recycle: Extend kamal proxy's response timeout for Zenodo releases --- ui/config/deploy.yml | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/ui/config/deploy.yml b/ui/config/deploy.yml index faa7e320..9a56c133 100644 --- a/ui/config/deploy.yml +++ b/ui/config/deploy.yml @@ -9,7 +9,7 @@ image: <%= ENV["KAMAL_APP_NAME_UI"] %> # Deploy to these servers. servers: web: - hosts: + hosts: - <%= ENV["KAMAL_SERVER_IP"] %> # job: # hosts: @@ -19,12 +19,21 @@ servers: # Enable SSL auto certification via Let's Encrypt and allow for multiple apps on a single web server. # Remove this section when using multiple web servers and ensure you terminate SSL at your load balancer. # -# Note: If using Cloudflare, set encryption mode in SSL/TLS setting to "Full" to enable CF-to-app encryption. -proxy: +# Note: If using Cloudflare, set encryption mode in SSL/TLS setting to "Full" to enable CF-to-app encryption. +proxy: ssl: true host: <%= ENV["KAMAL_APP_DOMAIN"] %> # Proxy connects to your container on port 80 by default. app_port: 3000 + # The Zenodo release streams progress over a long-lived SSE connection that + # can run for several minutes on releases with many or large assets. Without + # these two settings kamal-proxy (a) buffers the whole response - which breaks + # incremental progress events and nullifies the 15s heartbeat - and (b) cuts + # the request off at its 30s default `response_timeout`, so a release that + # completes on the server shows up as a failure in the browser. + response_timeout: 600 + buffering: + responses: false # Credentials for your image host. registry: