Skip to content

Commit 04c8c5e

Browse files
committed
surface last error message
1 parent 3b214c3 commit 04c8c5e

8 files changed

Lines changed: 175 additions & 16 deletions

File tree

backend/src/ee/services/audit-log-stream/audit-log-stream-schemas.ts

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
import { z } from "zod";
2+
13
import { AuditLogStreamsSchema } from "@app/db/schemas";
24

35
export const BaseProviderSchema = AuditLogStreamsSchema.omit({
@@ -11,4 +13,7 @@ export const BaseProviderSchema = AuditLogStreamsSchema.omit({
1113
encryptedHeadersKeyEncoding: true,
1214
encryptedHeadersTag: true,
1315
url: true
16+
}).extend({
17+
lastErrorMessage: z.string().nullish(),
18+
lastErrorTimestamp: z.date().nullish()
1419
});

backend/src/ee/services/audit-log-stream/audit-log-stream-service.test.ts

Lines changed: 27 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,9 @@ const STREAM_ID = "stream-1";
99
const ORG_ID = "org-1";
1010
const PROVIDER = "datadog";
1111

12+
const FAILURE_MESSAGE = "upstream failure";
1213
const failingProviderStreamLog = vi.fn(async () => {
13-
throw new Error("upstream failure");
14+
throw new Error(FAILURE_MESSAGE);
1415
});
1516

1617
vi.mock("@app/lib/config/env", () => ({
@@ -88,7 +89,11 @@ const createService = () => {
8889
const notificationService = {
8990
createUserNotifications: vi.fn(async () => undefined)
9091
};
91-
const smtpService = { sendMail: vi.fn(async () => undefined) };
92+
const smtpService = {
93+
sendMail: vi.fn<
94+
(payload: { substitutions: { lastErrorMessage: string; lastErrorTimestamp: string } }) => Promise<void>
95+
>(async () => undefined)
96+
};
9297
const orgDAL = {
9398
findOrgMembersByRole: vi.fn(async () => [
9499
{ status: "accepted", user: { id: "user-1", email: "admin@example.com" } }
@@ -193,4 +198,24 @@ describe("auditLogStreamServiceFactory failure tracking", () => {
193198
expect(keyStore.setItemWithExpiryNX).not.toHaveBeenCalled();
194199
expect(notificationService.createUserNotifications).not.toHaveBeenCalled();
195200
});
201+
202+
test("stores the last error message and timestamp in the cooldown payload", async () => {
203+
const { service, store, smtpService } = createService();
204+
205+
await driveFailures(service, FAILURE_THRESHOLD);
206+
207+
const payload = store[KeyStorePrefixes.AuditLogStreamAlertSent(STREAM_ID)];
208+
expect(payload).toBeDefined();
209+
const parsed = JSON.parse(payload as string) as { message: string; timestamp: string };
210+
expect(parsed.message).toContain(FAILURE_MESSAGE);
211+
expect(typeof parsed.timestamp).toBe("string");
212+
expect(Number.isNaN(new Date(parsed.timestamp).getTime())).toBe(false);
213+
214+
expect(smtpService.sendMail).toHaveBeenCalledTimes(1);
215+
const mailCall = smtpService.sendMail.mock.calls[0]?.[0] as {
216+
substitutions: { lastErrorMessage: string; lastErrorTimestamp: string };
217+
};
218+
expect(mailCall.substitutions.lastErrorMessage).toContain(FAILURE_MESSAGE);
219+
expect(mailCall.substitutions.lastErrorTimestamp).toBe(parsed.timestamp);
220+
});
196221
});

backend/src/ee/services/audit-log-stream/audit-log-stream-service.ts

Lines changed: 58 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -110,6 +110,24 @@ export const auditLogStreamServiceFactory = ({
110110
]);
111111
};
112112

113+
const readLastErrorFromCooldown = async (streamId: string) => {
114+
const raw = await keyStore.getItem(KeyStorePrefixes.AuditLogStreamAlertSent(streamId));
115+
if (!raw) return { lastErrorMessage: null, lastErrorTimestamp: null };
116+
117+
try {
118+
const parsed = JSON.parse(raw) as { message?: unknown; timestamp?: unknown };
119+
const message = typeof parsed.message === "string" ? parsed.message : null;
120+
const timestampStr = typeof parsed.timestamp === "string" ? parsed.timestamp : null;
121+
const timestamp = timestampStr ? new Date(timestampStr) : null;
122+
return {
123+
lastErrorMessage: message,
124+
lastErrorTimestamp: timestamp && !Number.isNaN(timestamp.getTime()) ? timestamp : null
125+
};
126+
} catch {
127+
return { lastErrorMessage: null, lastErrorTimestamp: null };
128+
}
129+
};
130+
113131
const updateById = async (
114132
{ logStreamId, provider, credentials }: TUpdateAuditLogStreamDTO,
115133
actor: OrgServiceActor
@@ -239,7 +257,9 @@ export const auditLogStreamServiceFactory = ({
239257
});
240258
}
241259

242-
return decryptLogStream(logStream, kmsService);
260+
const decrypted = await decryptLogStream(logStream, kmsService);
261+
const lastError = await readLastErrorFromCooldown(logStream.id);
262+
return { ...decrypted, ...lastError };
243263
};
244264

245265
const list = async (actor: OrgServiceActor) => {
@@ -256,10 +276,22 @@ export const auditLogStreamServiceFactory = ({
256276

257277
const logStreams = await auditLogStreamDAL.find({ orgId: actor.orgId });
258278

259-
return Promise.all(logStreams.map((stream) => decryptLogStream(stream, kmsService)));
279+
return Promise.all(
280+
logStreams.map(async (stream) => {
281+
const decrypted = await decryptLogStream(stream, kmsService);
282+
const lastError = await readLastErrorFromCooldown(stream.id);
283+
return { ...decrypted, ...lastError };
284+
})
285+
);
260286
};
261287

262-
const notifyStreamFailure = async (orgId: string, streamId: string, provider: string, failureCount: number) => {
288+
const notifyStreamFailure = async (
289+
orgId: string,
290+
streamId: string,
291+
provider: string,
292+
failureCount: number,
293+
lastError: { message: string; timestamp: string }
294+
) => {
263295
const appCfg = getConfig();
264296
const orgAdmins = await orgDAL.findOrgMembersByRole(orgId, OrgMembershipRole.Admin);
265297
const activeAdmins = orgAdmins.filter((admin) => admin.status !== OrgMembershipStatus.Invited);
@@ -295,15 +327,22 @@ export const auditLogStreamServiceFactory = ({
295327
provider,
296328
windowFailureCount: failureCount,
297329
windowMinutes: FAILURE_WINDOW_MINUTES,
298-
streamUrl
330+
streamUrl,
331+
lastErrorMessage: lastError.message,
332+
lastErrorTimestamp: lastError.timestamp
299333
}
300334
})
301335
.catch((err) =>
302336
logger.error(err, `Failed to send audit log stream failure email [streamId=${streamId}] [orgId=${orgId}]`)
303337
);
304338
};
305339

306-
const evaluateErrorSlidingWindow = async (orgId: string, streamId: string, provider: string) => {
340+
const evaluateErrorSlidingWindow = async (
341+
orgId: string,
342+
streamId: string,
343+
provider: string,
344+
errorMessage: string
345+
) => {
307346
try {
308347
const alertKey = KeyStorePrefixes.AuditLogStreamAlertSent(streamId);
309348

@@ -318,11 +357,19 @@ export const auditLogStreamServiceFactory = ({
318357

319358
if (failureCount < FAILURE_THRESHOLD) return;
320359

360+
// Capture the last error message and timestamp inside the cooldown payload so list reads and
361+
// the failure email both surface the most recent root cause to admins.
362+
const lastError = { message: errorMessage, timestamp: new Date().toISOString() };
363+
321364
// NX lock ensures only one worker fires the alert within the cooldown window.
322-
const acquired = await keyStore.setItemWithExpiryNX(alertKey, FAILURE_ALERT_COOLDOWN_SECONDS, "1");
365+
const acquired = await keyStore.setItemWithExpiryNX(
366+
alertKey,
367+
FAILURE_ALERT_COOLDOWN_SECONDS,
368+
JSON.stringify(lastError)
369+
);
323370
if (!acquired) return;
324371

325-
await notifyStreamFailure(orgId, streamId, provider, failureCount).catch(async (notifyErr) => {
372+
await notifyStreamFailure(orgId, streamId, provider, failureCount, lastError).catch(async (notifyErr) => {
326373
logger.error(
327374
notifyErr,
328375
`Failed to send audit log stream failure notification [streamId=${streamId}] [orgId=${orgId}]`
@@ -354,18 +401,21 @@ export const auditLogStreamServiceFactory = ({
354401
try {
355402
await factory.streamLog({ credentials, auditLog });
356403
} catch (error) {
404+
let errorMessage: string;
357405
if (isAxiosError(error)) {
406+
errorMessage = `${error?.message ?? "Request failed"}${JSON.stringify(error?.response?.data) ?? ""}`;
358407
logger.error(
359408
`audit-log-queue: Failed to stream audit log due to request error [auditLogId=${auditLog.id}] [event=${auditLog.eventType}] [provider=${provider}] [orgId=${orgId}] [projectId=${auditLog.projectId}] [message=${error?.message}] [response=${JSON.stringify(error?.response?.data)}]`
360409
);
361410
} else {
411+
errorMessage = (error as Error)?.message ?? "Unknown error";
362412
logger.error(
363413
error,
364414
`audit-log-queue: Failed to stream audit log [auditLogId=${auditLog.id}] [event=${auditLog.eventType}] [provider=${provider}] [orgId=${orgId}] [projectId=${auditLog.projectId}]: ${(error as Error)?.message}`
365415
);
366416
}
367417

368-
await evaluateErrorSlidingWindow(orgId, id, provider);
418+
await evaluateErrorSlidingWindow(orgId, id, provider, errorMessage);
369419
}
370420
})
371421
);

backend/src/services/smtp/emails/AuditLogStreamFailedTemplate.tsx

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,14 +9,18 @@ interface AuditLogStreamFailedTemplateProps extends Omit<BaseEmailWrapperProps,
99
windowFailureCount: number;
1010
windowMinutes: number;
1111
streamUrl: string;
12+
lastErrorMessage?: string;
13+
lastErrorTimestamp?: string;
1214
}
1315

1416
export const AuditLogStreamFailedTemplate = ({
1517
provider,
1618
windowFailureCount,
1719
windowMinutes,
1820
streamUrl,
19-
siteUrl
21+
siteUrl,
22+
lastErrorMessage,
23+
lastErrorTimestamp
2024
}: AuditLogStreamFailedTemplateProps) => {
2125
return (
2226
<BaseEmailWrapper
@@ -32,6 +36,18 @@ export const AuditLogStreamFailedTemplate = ({
3236
<Text className="text-[14px] mt-[4px]">{provider}</Text>
3337
<strong>Consecutive failures</strong>
3438
<Text className="text-[14px] text-red-600 mt-[4px]">{windowFailureCount}</Text>
39+
{lastErrorTimestamp && (
40+
<>
41+
<strong>Last error at</strong>
42+
<Text className="text-[14px] mt-[4px]">{lastErrorTimestamp}</Text>
43+
</>
44+
)}
45+
{lastErrorMessage && (
46+
<>
47+
<strong>Last error message</strong>
48+
<Text className="text-[14px] text-red-600 mt-[4px] break-all">{lastErrorMessage}</Text>
49+
</>
50+
)}
3551
</Section>
3652
<Text className="text-[14px]">
3753
Your stream has logged {windowFailureCount} failures in a row, with no more than {windowMinutes} minutes between
@@ -52,5 +68,7 @@ AuditLogStreamFailedTemplate.PreviewProps = {
5268
windowFailureCount: 12,
5369
windowMinutes: 5,
5470
streamUrl: "https://app.infisical.com/organizations/example-org/settings?selectedTab=tag-audit-log-streams",
55-
siteUrl: "https://app.infisical.com"
71+
siteUrl: "https://app.infisical.com",
72+
lastErrorMessage: "Request failed with status code 401 — {\"errors\":[\"Invalid API key\"]}",
73+
lastErrorTimestamp: "2026-05-19T14:32:11.000Z"
5674
} as AuditLogStreamFailedTemplateProps;

frontend/src/hooks/api/auditLogStreams/types/providers/root-provider.ts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,4 +3,6 @@ export type TRootProviderLogStream = {
33
orgId: string;
44
createdAt: string;
55
updatedAt: string;
6+
lastErrorMessage?: string | null;
7+
lastErrorTimestamp?: string | null;
68
};

frontend/src/layouts/OrganizationLayout/components/NetworkHealthBanner/NetworkHealthBanner.tsx

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -183,8 +183,8 @@ export const NetworkHealthBanner = () => {
183183
const ids = visibleStreamNotifications.map((n) => n.id);
184184
items.push({
185185
id: "audit-log-stream-failed",
186-
message: "Error streaming audit logs.",
187-
linkLabel: "View log stream",
186+
message: "Unable to stream audit logs.",
187+
linkLabel: "Click to view stream configuration.",
188188
linkTo: visibleStreamNotifications[0].link ?? undefined,
189189
onDismiss: () => {
190190
ids.forEach((id) => updateNotification({ notificationId: id, isRead: true }));

frontend/src/pages/organization/SettingsPage/components/AuditLogStreamTab/components/AuditLogStreamRow.tsx

Lines changed: 24 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import { faAsterisk, faEllipsisV, faTrash } from "@fortawesome/free-solid-svg-icons";
22
import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
3+
import { AlertTriangleIcon } from "lucide-react";
34
import { twMerge } from "tailwind-merge";
45

56
import { OrgPermissionCan } from "@app/components/permissions";
@@ -13,23 +14,28 @@ import {
1314
Tooltip,
1415
Tr
1516
} from "@app/components/v2";
17+
import { Tooltip as TooltipV3, TooltipContent, TooltipTrigger } from "@app/components/v3";
1618
import { OrgPermissionSubjects } from "@app/context";
1719
import { OrgPermissionActions } from "@app/context/OrgPermissionContext/types";
1820
import { AUDIT_LOG_STREAM_PROVIDER_MAP, getProviderUrl } from "@app/helpers/auditLogStreams";
1921
import { TAuditLogStream } from "@app/hooks/api/types";
2022

23+
import { LastErrorSection } from "./LastErrorSection";
24+
2125
type Props = {
2226
logStream: TAuditLogStream;
2327
onDelete: (logStream: TAuditLogStream) => void;
2428
onEditCredentials: (logStream: TAuditLogStream) => void;
2529
};
2630

2731
export const AuditLogStreamRow = ({ logStream, onDelete, onEditCredentials }: Props) => {
28-
const { id, provider } = logStream;
32+
const { id, provider, lastErrorMessage, lastErrorTimestamp } = logStream;
2933

3034
const providerDetails = AUDIT_LOG_STREAM_PROVIDER_MAP[provider];
3135
const url = getProviderUrl(logStream);
3236

37+
const hasLastError = Boolean(lastErrorMessage || lastErrorTimestamp);
38+
3339
return (
3440
<Tr
3541
className={twMerge("group h-12 transition-colors duration-100 hover:bg-mineshaft-700")}
@@ -57,8 +63,24 @@ export const AuditLogStreamRow = ({ logStream, onDelete, onEditCredentials }: Pr
5763
</div>
5864
</Td>
5965
<Td className="max-w-0 min-w-32!">
60-
<div className="flex w-full items-center">
66+
<div className="flex w-full items-center gap-2">
6167
<p className="truncate">{url}</p>
68+
{hasLastError && (
69+
<TooltipV3>
70+
<TooltipTrigger>
71+
<AlertTriangleIcon
72+
className="size-4 shrink-0 text-yellow-500"
73+
aria-label="Stream is failing"
74+
/>
75+
</TooltipTrigger>
76+
<TooltipContent className="max-w-96 min-w-52 px-3">
77+
<LastErrorSection
78+
lastErrorMessage={lastErrorMessage ?? null}
79+
lastErrorTimestamp={lastErrorTimestamp ?? null}
80+
/>
81+
</TooltipContent>
82+
</TooltipV3>
83+
)}
6284
</div>
6385
</Td>
6486
<Td>
Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
import { format } from "date-fns";
2+
import { ClockIcon, MessageSquareWarningIcon } from "lucide-react";
3+
4+
type Props = {
5+
lastErrorMessage: string | null;
6+
lastErrorTimestamp: string | null;
7+
};
8+
9+
export const LastErrorSection = ({ lastErrorMessage, lastErrorTimestamp }: Props) => (
10+
<div className="py-1">
11+
<div className="mb-2 flex items-center gap-2 border-b border-foreground/25 pb-1">
12+
<div className="font-medium">Last Error</div>
13+
</div>
14+
{lastErrorMessage && (
15+
<div className="mb-2 flex items-start gap-2 text-sm">
16+
<div className="flex items-center justify-center rounded-sm bg-container/50 p-2">
17+
<MessageSquareWarningIcon className="size-5" />
18+
</div>
19+
<div className="flex flex-col">
20+
<div className="text-xs font-medium text-label">Message</div>
21+
<div className="text-sm break-words">{lastErrorMessage}</div>
22+
</div>
23+
</div>
24+
)}
25+
{lastErrorTimestamp && (
26+
<div className="flex items-center gap-2 text-sm">
27+
<div className="flex items-center justify-center rounded-sm bg-container/50 p-2">
28+
<ClockIcon className="size-5" />
29+
</div>
30+
<div className="flex flex-col">
31+
<div className="text-xs font-medium text-label">Time</div>
32+
<div className="text-sm">{format(lastErrorTimestamp, "PPpp")}</div>
33+
</div>
34+
</div>
35+
)}
36+
</div>
37+
);

0 commit comments

Comments
 (0)