chore: fine tune slack notifications for temporal interceptors

Signed-off-by: Uroš Marolt <uros@marolt.me>

Author: themarolt

services/archetypes/worker/src/activities/activityInterceptor.ts

@@ -29,7 +29,7 @@ export class ActivityMonitoringInterceptor implements ActivityInboundCallsInterc
       runId: this.ctx.info.workflowExecution.runId,
     })
 
-    if (this.ctx.info.attempt > 10) {
+    if (this.ctx.info.attempt % 50 === 0) {
       const message = `Activity \`${this.ctx.info.activityType}\` with id \`${this.ctx.info.activityId}\` was retried ${this.ctx.info.attempt} times!\n\n*Workflow:* ${this.ctx.info.workflowType}\n*Workflow ID:* ${this.ctx.info.workflowExecution.workflowId}\n*Task Queue:* ${this.ctx.info.taskQueue}`
 
       // Fire and forget - don't await to avoid blocking the activity

services/archetypes/worker/src/activities/index.ts

@@ -20,13 +20,8 @@ async function telemetryIncrement(
   telemetry.increment(name, value, tags)
 }
 
-async function slackNotify(message: string, persona?: SlackPersona) {
-  await sendSlackNotificationAsync(
-    SlackChannel.ALERTS,
-    persona ?? SlackPersona.ERROR_REPORTER,
-    'Temporal Alert',
-    message,
-  )
+async function slackNotify(message: string, persona: SlackPersona) {
+  await sendSlackNotificationAsync(SlackChannel.ALERTS, persona, 'Temporal Alert', message)
   log.info('Slack notification sent from Temporal activity')
 }
 

services/archetypes/worker/src/interceptors.ts

@@ -1,17 +1,48 @@
 import {
+  ActivityFailure,
+  ApplicationFailure,
   Next,
   WorkflowExecuteInput,
   WorkflowInboundCallsInterceptor,
   proxyActivities,
   workflowInfo,
 } from '@temporalio/workflow'
 
+import { SlackPersona } from '@crowd/slack'
+
 import * as activities from './activities'
 
 const activity = proxyActivities<typeof activities>({
   startToCloseTimeout: '10 seconds',
 })
 
+/**
+ * Extract detailed error information when an activity reaches retry limit
+ */
+function getActivityRetryLimitDetails(err: ActivityFailure): string {
+  let details = `*Activity:* \`${err.activityType}\`\n`
+  details += `*Activity ID:* \`${err.activityId || 'N/A'}\`\n`
+  details += `*Retry State:* ${err.retryState}\n\n`
+
+  // Get the root cause error message and type
+  if (err.cause) {
+    details += `*Error:* ${err.cause.message}\n`
+
+    // If it's an ApplicationFailure, get the type (e.g., AxiosError)
+    if (err.cause instanceof ApplicationFailure && err.cause.type) {
+      details += `*Error Type:* ${err.cause.type}\n`
+    }
+
+    // Add stack trace (first 10 lines for context)
+    if (err.cause.stack) {
+      const stackLines = err.cause.stack.split('\n').slice(0, 10)
+      details += `\n*Stack Trace (first 10 lines):*\n\`\`\`\n${stackLines.join('\n')}\n\`\`\``
+    }
+  }
+
+  return details
+}
+
 export class WorkflowMonitoringInterceptor implements WorkflowInboundCallsInterceptor {
   async execute(
     input: WorkflowExecuteInput,
@@ -35,9 +66,19 @@ export class WorkflowMonitoringInterceptor implements WorkflowInboundCallsInterc
     } catch (err) {
       if (err.message !== 'Workflow continued as new') {
         await activity.telemetryIncrement('temporal.workflow_execution_error', 1, tags)
-        await activity.slackNotify(
-          `Workflow ${info.workflowType} with id ${info.workflowId} failed with error: ${err.message}!`,
-        )
+
+        // Only send detailed notification if it's an activity that reached retry limit
+        if (err instanceof ActivityFailure && err.retryState === 'MAXIMUM_ATTEMPTS_REACHED') {
+          const errorDetails = getActivityRetryLimitDetails(err)
+          const message = `*Workflow Failed: Activity Retry Limit Reached*\n\n*Workflow:* \`${info.workflowType}\`\n*Workflow ID:* \`${info.workflowId}\`\n*Run ID:* \`${info.runId}\`\n\n${errorDetails}`
+
+          await activity.slackNotify(message, SlackPersona.ERROR_REPORTER)
+        } else {
+          // For other errors, send a simpler notification
+          const message = `*Workflow Failed*\n\n*Workflow:* \`${info.workflowType}\`\n*Workflow ID:* \`${info.workflowId}\`\n*Run ID:* \`${info.runId}\`\n*Error:* ${err.message}`
+
+          await activity.slackNotify(message, SlackPersona.ERROR_REPORTER)
+        }
       }
 
       throw err