Log viewer improvements (#24)

* Log viewer improvements

* Close EventSource after receiving all logs in places where no new logs will ever be received

Author: FluxCapacitor2

backend/src/db/repo/app.ts

@@ -3,6 +3,7 @@ import { randomBytes } from "node:crypto";
 import {
   DeploymentSource,
   type DeploymentStatus,
+  type LogType,
   type PermissionLevel,
   type WebhookEvent,
 } from "../../generated/prisma/enums.ts";
@@ -16,6 +17,7 @@ import type {
   AppCreate,
   Deployment,
   DeploymentConfig,
+  Log,
 } from "../models.ts";
 import { DeploymentRepo } from "./deployment.ts";
 
@@ -215,6 +217,30 @@ export class AppRepo {
     return app.config.deployment;
   }
 
+  async getLogs(
+    appId: number,
+    deploymentId: number | null,
+    cursor: number,
+    type: LogType,
+    limit: number,
+  ): Promise<Log[]> {
+    // Fetch them in reverse order so that we can take only the 500 most recent lines
+    return (
+      await this.client.log.findMany({
+        where: {
+          id: { gt: cursor },
+          type: type,
+          deployment: {
+            appId,
+            ...(deploymentId !== null ? { id: deploymentId } : {}),
+          },
+        },
+        orderBy: [{ timestamp: "desc" }, { index: "desc" }],
+        take: limit,
+      })
+    ).reverse();
+  }
+
   async getDeploymentConfig(appId: number): Promise<DeploymentConfig> {
     const app = await this.client.app.findUnique({
       where: { id: appId },

backend/src/db/repo/deployment.ts

@@ -2,7 +2,6 @@ import { randomBytes } from "node:crypto";
 import type {
   AppType,
   DeploymentStatus,
-  LogType,
   PermissionLevel,
 } from "../../generated/prisma/enums.ts";
 import type {
@@ -311,26 +310,6 @@ export class DeploymentRepo {
     return count === 1;
   }
 
-  async getLogs(
-    deploymentId: number,
-    cursor: number,
-    type: LogType,
-    limit: number,
-  ): Promise<Log[]> {
-    // Fetch them in reverse order so that we can take only the 500 most recent lines
-    return (
-      await this.client.log.findMany({
-        where: {
-          id: { gt: cursor },
-          deploymentId: deploymentId,
-          type: type,
-        },
-        orderBy: [{ timestamp: "desc" }, { index: "desc" }],
-        take: limit,
-      })
-    ).reverse();
-  }
-
   async insertLogs(logs: Omit<Log, "id">[]) {
     await this.client.log.createMany({
       data: logs,
@@ -341,14 +320,35 @@ export class DeploymentRepo {
       deploymentIds.add(log.deploymentId);
     }
 
-    await Promise.all(
-      [...deploymentIds].map((deploymentId) => {
+    const deploymentIdArray = [...deploymentIds];
+
+    const deploymentMessages = Promise.all(
+      deploymentIdArray.map((deploymentId) => {
         if (typeof deploymentId !== "number") {
           return Promise.resolve();
         }
         return this.publish(`deployment_${deploymentId}_logs`, "");
       }),
     );
+
+    const appMessages = this.client.app
+      .findMany({
+        where: { deployments: { some: { id: { in: deploymentIdArray } } } },
+        select: { id: true },
+      })
+      .then((ids) => [...new Set(ids.map((it) => it.id))])
+      .then((appIds) =>
+        Promise.all(
+          [...new Set(appIds)].map((appId) => {
+            if (typeof appId !== "number") {
+              return Promise.resolve();
+            }
+            return this.publish(`app_${appId}_logs`, "");
+          }),
+        ),
+      );
+
+    await Promise.all([deploymentMessages, appMessages]);
   }
 
   async unlinkRepositoryFromAllDeployments(repoId: number) {

backend/src/handlers/getAppLogs.ts

@@ -40,10 +40,9 @@ export const getAppLogsHandler: HandlerMap["getAppLogs"] = async (
         // Just start at the beginning
       }
     }
-
     await getAppLogs(
       ctx.request.params.appId,
-      ctx.request.params.deploymentId,
+      ctx.request.query.deploymentId ?? null,
       req.user.id,
       ctx.request.query.type,
       lastLogId,

backend/src/service/getAppLogs.ts

@@ -28,7 +28,7 @@ const k8sConcurrentViewers = meter.createUpDownCounter(
 
 export async function getAppLogs(
   appId: number,
-  deploymentId: number,
+  deploymentId: number | null,
   userId: number,
   type: LogType,
   lastLogId: number,
@@ -44,7 +44,7 @@ export async function getAppLogs(
   }
 
   // Pull logs from Postgres and send them to the client as they come in
-  if (typeof deploymentId !== "number") {
+  if (typeof deploymentId !== "number" && deploymentId !== null) {
     // Extra sanity check due to potential SQL injection below in `subscribe`; should never happen because of openapi-backend's request validation and additional sanitization in `subscribe()`
     throw new Error("deploymentId must be a number.");
   }
@@ -56,7 +56,8 @@ export async function getAppLogs(
 
   if (collectLogs || type === "BUILD") {
     const fetchNewLogs = async () => {
-      const newLogs = await db.deployment.getLogs(
+      const newLogs = await db.app.getLogs(
+        app.id,
         deploymentId,
         lastLogId,
         type,
@@ -80,9 +81,14 @@ export async function getAppLogs(
       );
     };
 
+    const channel =
+      deploymentId === null
+        ? `app_${appId}_logs`
+        : `deployment_${deploymentId}_logs`;
+
     // When new logs come in, send them to the client
     const unsubscribe = await db.subscribe(
-      `deployment_${deploymentId}_logs`,
+      channel,
       () =>
         void fetchNewLogs().catch((err) =>
           logger.error(err, "Failed to fetch new logs"),
@@ -106,6 +112,11 @@ export async function getAppLogs(
       );
     }
 
+    if (!deploymentId) {
+      const recentDeployment = await db.app.getMostRecentDeployment(appId);
+      deploymentId = recentDeployment.id;
+    }
+
     const { CoreV1Api: core, Log: log } = await getClientsForRequest(
       userId,
       app.projectId,

frontend/src/components/Logs.tsx

@@ -2,26 +2,28 @@ import type { components, paths } from "@/generated/openapi";
 import { useEventSource } from "@/hooks/useEventSource";
 import clsx from "clsx";
 import { AlertTriangle, FileClock, Loader, SatelliteDish } from "lucide-react";
-import { useLayoutEffect, useRef, useState } from "react";
+import { useEffect, useLayoutEffect, useRef, useState } from "react";
 import { Button } from "./ui/button";
 
 type Deployment =
   paths["/app/{appId}/deployments/{deploymentId}"]["get"]["responses"]["200"]["content"]["application/json"];
 
 type LogType = components["schemas"]["LogLine"]["type"];
 
-export const Logs = ({
-  deployment,
-  type,
-}: {
-  deployment: Pick<
+type LogsProps = {
+  deployment?: Pick<
     Deployment,
     "status" | "id" | "appId" | "updatedAt" | "podStatus"
   >;
   type: LogType;
-}) => {
+  appId?: number;
+  follow: boolean;
+};
+
+export const Logs = ({ deployment, type, appId, follow }: LogsProps) => {
   const [logs, setLogs] = useState<components["schemas"]["LogLine"][]>([]);
   const [noLogs, setNoLogs] = useState(false); // Set to true when we know there are no logs for this deployment
+  const [done, setDone] = useState(false); // Set to true when all past logs have been sent and the viewer is up-to-date
 
   const logsBody = useRef<HTMLDivElement | null>(null);
   const lastScroll = useRef({ scrollTop: 0, hasScrolledUp: false });
@@ -39,13 +41,17 @@ export const Logs = ({
     }
   }, [logs]);
 
-  const { connecting, connected } = useEventSource(
-    new URL(
-      `${window.location.protocol}//${window.location.host}/api/app/${deployment.appId}/deployments/${deployment.id}/logs?type=${type}`,
-    ),
+  const url =
+    deployment === undefined
+      ? `${window.location.protocol}//${window.location.host}/api/app/${appId}/logs?type=${type}`
+      : `${window.location.protocol}//${window.location.host}/api/app/${appId}/logs?type=${type}&deploymentId=${deployment.id}`;
+
+  const { connecting, connected, close, reconnect } = useEventSource(
+    new URL(url),
     ["log", "pastLogsSent"],
     (eventName, event) => {
       if (eventName === "pastLogsSent") {
+        setDone(true);
         if (logs.length === 0) {
           setNoLogs(true);
         }
@@ -69,25 +75,39 @@ export const Logs = ({
     },
   );
 
+  useEffect(() => {
+    if (!follow && done && connected) {
+      close();
+    }
+
+    if (follow && !connected && !connecting) {
+      reconnect();
+    }
+  }, [done, follow, connected, connecting, close, reconnect]);
+
   return (
     <>
-      {connecting ? (
-        <p className="flex items-center gap-2 font-mono text-sm">
-          <Loader className="animate-spin" /> Connecting...
-        </p>
-      ) : !connected ? (
-        <p className="mb-2 flex items-center gap-2 font-mono text-sm text-amber-600">
-          <AlertTriangle /> Disconnected. New logs will not appear until the
-          connection is re-established.
-        </p>
-      ) : (
-        <p className="mb-2 flex items-center gap-2 font-mono text-sm text-blue-500">
-          <div className="relative h-5 w-4">
-            <div className="absolute top-1/2 left-1/2 size-2 -translate-1/2 animate-pulse rounded-full bg-blue-500" />
-            <div className="absolute top-1/2 left-1/2 size-2 -translate-1/2 animate-ping rounded-full bg-blue-500" />
-          </div>
-          Receiving logs in realtime
-        </p>
+      {follow && (
+        <>
+          {connecting ? (
+            <p className="flex items-center gap-2 font-mono text-sm">
+              <Loader className="animate-spin" /> Connecting...
+            </p>
+          ) : !connected ? (
+            <p className="mb-2 flex items-center gap-2 font-mono text-sm text-amber-600">
+              <AlertTriangle /> Disconnected. New logs will not appear until the
+              connection is re-established.
+            </p>
+          ) : (
+            <p className="mb-2 flex items-center gap-2 font-mono text-sm text-blue-500">
+              <div className="relative h-5 w-4">
+                <div className="absolute top-1/2 left-1/2 size-2 -translate-1/2 animate-pulse rounded-full bg-blue-500" />
+                <div className="absolute top-1/2 left-1/2 size-2 -translate-1/2 animate-ping rounded-full bg-blue-500" />
+              </div>
+              Receiving logs in realtime
+            </p>
+          )}
+        </>
       )}
 
       <div
@@ -120,7 +140,8 @@ export const Logs = ({
                     {/* "w-0" above forces this column to take up as little horizontal space as possible */}
                     {new Date(log.time).toLocaleString()}
                   </td>
-                  {(deployment.podStatus?.total ?? 1) > 1 && (
+                  {(deployment === undefined ||
+                    (deployment.podStatus?.total ?? 1) > 1) && (
                     <td className="px-2">
                       <span className="opacity-70">{log.pod}</span>
                     </td>
@@ -138,7 +159,8 @@ export const Logs = ({
               <SatelliteDish /> Logs Unavailable
             </p>
             <p className="ml-8 opacity-50">
-              {["PENDING", "BUILDING"].includes(deployment.status)
+              {deployment !== undefined &&
+              ["PENDING", "BUILDING"].includes(deployment.status)
                 ? "Waiting for the build to start."
                 : null}
             </p>
@@ -149,9 +171,11 @@ export const Logs = ({
               <FileClock /> No Logs Found
             </p>
             <p className="ml-8 opacity-50">
-              {["PENDING", "BUILDING", "DEPLOYING"].includes(deployment.status)
+              {deployment !== undefined &&
+              ["PENDING", "BUILDING", "DEPLOYING"].includes(deployment.status)
                 ? "Waiting for your app to be deployed."
-                : ["COMPLETE", "STOPPED"].includes(deployment.status) &&
+                : deployment !== undefined &&
+                    ["COMPLETE", "STOPPED"].includes(deployment.status) &&
                     type === "BUILD"
                   ? "Build completed with no log output."
                   : "Logs from your app will appear here."}

frontend/src/hooks/useEventSource.ts

@@ -1,4 +1,4 @@
-import { useEffect, useRef, useState } from "react";
+import { useEffect, useEffectEvent, useRef, useState } from "react";
 
 export const useEventSource = <T extends string>(
   url: URL,
@@ -7,6 +7,8 @@ export const useEventSource = <T extends string>(
 ) => {
   const [hasConnected, setHasConnected] = useState(false); // Whether a connection has been established before; true after the first connection is successfully opened
   const [connected, setConnected] = useState(false);
+  const [reconnectCounter, setReconnectCounter] = useState(0);
+  const source = useRef<EventSource | null>(null);
   const shouldOpen = useRef(true);
 
   const setupEventSource = (
@@ -25,9 +27,11 @@ export const useEventSource = <T extends string>(
       setTimeout(() => {
         if (!shouldOpen.current) return; // The component has unmounted; we shouldn't try to reconnect anymore
         reconnectCallback(setupEventSource(reconnectCallback));
-      }, 3000);
+      }, 500);
     };
 
+    source.current = eventSource;
+
     for (const eventName of eventNames) {
       eventSource.addEventListener(eventName, (event: MessageEvent) =>
         onMessage(eventName, event),
@@ -37,18 +41,31 @@ export const useEventSource = <T extends string>(
     return eventSource;
   };
 
+  const setup = useEffectEvent(setupEventSource);
+  const urlString = url.toString(); // Equal URLs don't have Object.is() equality, so the useEffect would be triggered on every render if we didn't convert this into a string first.
+
   useEffect(() => {
     shouldOpen.current = true;
     let eventSource: EventSource;
-    eventSource = setupEventSource((newEventSource) => {
+    eventSource = setup((newEventSource) => {
       eventSource = newEventSource;
     });
 
     return () => {
       shouldOpen.current = false;
       eventSource.close();
     };
-  }, [url.toString()]);
+  }, [urlString, reconnectCounter]);
 
-  return { connected, connecting: !connected && !hasConnected };
+  return {
+    connected,
+    connecting: !connected && !hasConnected,
+    close: () => {
+      source.current?.close();
+      setConnected(false);
+    },
+    reconnect: () => {
+      setReconnectCounter((current) => current + 1);
+    },
+  };
 };