Skip to content

Commit 6a91c29

Browse files
committed
chore: streamline watchdog
1 parent 5df3910 commit 6a91c29

6 files changed

Lines changed: 54 additions & 38 deletions

File tree

api/event.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,9 @@ import (
88
"github.com/google/uuid"
99
)
1010

11+
// A dummy event that lives only within the application (never in the event queue)
12+
const EventWatchdog = "notification.watchdog"
13+
1114
// List of all sync events in the `event_queue` table.
1215
//
1316
// These events are generated by the database in response to updates on some of the tables.

go.mod

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,6 @@ require (
7878
go.opentelemetry.io/otel/trace v1.34.0
7979
golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f
8080
google.golang.org/genai v0.5.0
81-
gorm.io/hints v1.1.2
8281
k8s.io/api v0.31.1
8382
)
8483

@@ -369,7 +368,7 @@ require (
369368
sigs.k8s.io/yaml v1.4.0
370369
)
371370

372-
replace github.com/flanksource/duty => ../duty
371+
// replace github.com/flanksource/duty => ../duty
373372

374373
// replace github.com/flanksource/gomplate/v3 => ../gomplat3
375374

go.sum

Lines changed: 2 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -287,6 +287,8 @@ github.com/flanksource/artifacts v1.0.14 h1:Vv70bccsae0MwGaf/uSPp34J5V1/PyKfct9z
287287
github.com/flanksource/artifacts v1.0.14/go.mod h1:qHVCnQu5k50aWNJ5UhpcAKEl7pAzqUrFFKGSm147G70=
288288
github.com/flanksource/commons v1.36.1 h1:SDppXOYO6vk06d12SPOYVZJX+8gV72FmK1l9ar5fkjc=
289289
github.com/flanksource/commons v1.36.1/go.mod h1:nsYCn6JOhENXNLR5pz4zNco70DQV+bGObQ8NbOrUeuQ=
290+
github.com/flanksource/duty v1.0.916 h1:u5fpdDhXmkyhzIeb31zo5ci6ME/rh/khYuJ+JsTtdwA=
291+
github.com/flanksource/duty v1.0.916/go.mod h1:UqK5GanKtGSXKAJyfyUjEqZ2YB+Dq6l0qFZHUyT0XaU=
290292
github.com/flanksource/gomplate/v3 v3.24.56 h1:ndscvg7mnuDigBfyOsbCzXZCLBB9s/4v6zL0x7d27Jw=
291293
github.com/flanksource/gomplate/v3 v3.24.56/go.mod h1:hGEObNtnOQs8rNUX8sM8aJTAhnt4ehjyOw1MvDhl6AU=
292294
github.com/flanksource/is-healthy v1.0.73 h1:V6dpeQf1iKb/vxSuFRBI6FaGSQf2gevP909VU0JppIo=
@@ -681,10 +683,6 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE
681683
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
682684
github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc=
683685
github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
684-
github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
685-
github.com/mattn/go-sqlite3 v1.14.16/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
686-
github.com/mattn/go-sqlite3 v1.14.17 h1:mCRHCLDUBXgpKAqIKsaAaAsrAlbkeomtRFKXh2L6YIM=
687-
github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
688686
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
689687
github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
690688
github.com/microsoft/go-mssqldb v1.6.0 h1:mM3gYdVwEPFrlg/Dvr2DNVEgYFG7L42l+dGc67NNNpc=
@@ -1458,18 +1456,13 @@ gorm.io/driver/mysql v1.5.7 h1:MndhOPYOfEp2rHKgkZIhJ16eVUIRf2HmzgoPmh7FCWo=
14581456
gorm.io/driver/mysql v1.5.7/go.mod h1:sEtPWMiqiN1N1cMXoXmBbd8C6/l+TESwriotuRRpkDM=
14591457
gorm.io/driver/postgres v1.5.9 h1:DkegyItji119OlcaLjqN11kHoUgZ/j13E0jkJZgD6A8=
14601458
gorm.io/driver/postgres v1.5.9/go.mod h1:DX3GReXH+3FPWGrrgffdvCk3DQ1dwDPdmbenSkweRGI=
1461-
gorm.io/driver/sqlite v1.5.0 h1:zKYbzRCpBrT1bNijRnxLDJWPjVfImGEn0lSnUY5gZ+c=
1462-
gorm.io/driver/sqlite v1.5.0/go.mod h1:kDMDfntV9u/vuMmz8APHtHF0b4nyBB7sfCieC6G8k8I=
14631459
gorm.io/driver/sqlserver v1.5.3 h1:rjupPS4PVw+rjJkfvr8jn2lJ8BMhT4UW5FwuJY0P3Z0=
14641460
gorm.io/driver/sqlserver v1.5.3/go.mod h1:B+CZ0/7oFJ6tAlefsKoyxdgDCXJKSgwS2bMOQZT0I00=
1465-
gorm.io/gorm v1.24.7-0.20230306060331-85eaf9eeda11/go.mod h1:L4uxeKpfBml98NYqVqwAdmV1a2nBtAec/cf3fpucW/k=
14661461
gorm.io/gorm v1.25.0/go.mod h1:L4uxeKpfBml98NYqVqwAdmV1a2nBtAec/cf3fpucW/k=
14671462
gorm.io/gorm v1.25.7-0.20240204074919-46816ad31dde/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8=
14681463
gorm.io/gorm v1.25.7/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8=
14691464
gorm.io/gorm v1.25.12 h1:I0u8i2hWQItBq1WfE0o2+WuL9+8L21K9e2HHSTE/0f8=
14701465
gorm.io/gorm v1.25.12/go.mod h1:xh7N7RHfYlNc5EmcI/El95gXusucDrQnHXe0+CgWcLQ=
1471-
gorm.io/hints v1.1.2 h1:b5j0kwk5p4+3BtDtYqqfY+ATSxjj+6ptPgVveuynn9o=
1472-
gorm.io/hints v1.1.2/go.mod h1:/ARdpUHAtyEMCh5NNi3tI7FsGh+Cj/MIUlvNxCNCFWg=
14731466
gorm.io/plugin/dbresolver v1.5.3 h1:wFwINGZZmttuu9h7XpvbDHd8Lf9bb8GNzp/NpAMV2wU=
14741467
gorm.io/plugin/dbresolver v1.5.3/go.mod h1:TSrVhaUg2DZAWP3PrHlDlITEJmNOkL0tFTjvTEsQ4XE=
14751468
gorm.io/plugin/prometheus v0.1.0 h1:kDQwAfCUsT9D6jDUpIp7pnc7bCJu/6voM8I/BmFjxUQ=

notification/events.go

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -659,6 +659,17 @@ func isHealthReportable(events []string, previousHealth, currentHealth models.He
659659
func GetEnvForEvent(ctx context.Context, event models.Event) (*celVariables, error) {
660660
var env celVariables
661661

662+
if event.Name == api.EventWatchdog {
663+
notificationID := event.Properties["id"]
664+
665+
stats, err := query.GetNotificationStats(ctx, notificationID)
666+
if err != nil {
667+
return nil, fmt.Errorf("failed to get notification statistics: %w", err)
668+
}
669+
670+
env.Summary = stats[0]
671+
}
672+
662673
if strings.HasPrefix(event.Name, "check.") {
663674
checkID := event.Properties["id"]
664675
lastRuntime := event.Properties["last_runtime"]

notification/job.go

Lines changed: 36 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -559,15 +559,6 @@ func WatchdogNotificationJob(ctx context.Context, notificationID string, interva
559559
// SendWatchdogNotification sends a watchdog notification containing statistics
560560
// for the specified notification
561561
func SendWatchdogNotification(ctx context.Context, notificationID string) error {
562-
stats, err := query.GetNotificationStats(ctx, notificationID)
563-
if err != nil {
564-
return fmt.Errorf("failed to get notification statistics: %w", err)
565-
}
566-
567-
if len(stats) == 0 {
568-
return nil
569-
}
570-
571562
notificationUUID, err := uuid.Parse(notificationID)
572563
if err != nil {
573564
return fmt.Errorf("failed to parse notification ID: %w", err)
@@ -578,26 +569,46 @@ func SendWatchdogNotification(ctx context.Context, notificationID string) error
578569
return fmt.Errorf("failed to get notification: %w", err)
579570
}
580571

581-
celEnv := &celVariables{
582-
Summary: stats[0],
583-
}
572+
if notification.PlaybookID == nil {
573+
// NOTE: Watchdog notifications aren't sent to playbook recievers.
584574

585-
payload := NotificationEventPayload{
586-
EventName: "notification.watchdog",
587-
EventCreatedAt: time.Now(),
588-
PersonID: notification.PersonID,
589-
TeamID: notification.TeamID,
590-
}
575+
// Manually craft a payload (unlike other payloads that are generated from events)
576+
// This allows us to bypass the event queue and process synchronously
577+
payload := NotificationEventPayload{
578+
EventName: api.EventWatchdog,
579+
EventCreatedAt: time.Now(),
580+
ID: notificationUUID,
581+
PersonID: notification.PersonID,
582+
NotificationID: notificationUUID,
583+
TeamID: notification.TeamID,
584+
Properties: fmt.Appendf(nil, `{"id": "%s"}`, notificationUUID.String()),
585+
}
586+
if len(notification.CustomNotifications) > 0 {
587+
payload.CustomService = &notification.CustomNotifications[0]
588+
}
591589

592-
if len(notification.CustomNotifications) > 0 {
593-
payload.CustomService = &notification.CustomNotifications[0]
590+
if err := sendNotification(ctx, payload); err != nil {
591+
return fmt.Errorf("failed to send watchdog notification to primary recipient: %w", err)
592+
}
594593
}
595594

596-
nCtx := NewContext(ctx, notificationUUID)
597-
nCtx.WithSource(payload.EventName, payload.ID)
598-
if err := PrepareAndSendEventNotification(nCtx, payload, celEnv); err != nil {
599-
return fmt.Errorf("failed to send watchdog notification: %w", err)
595+
if notification.FallbackPlaybookID == nil {
596+
// Also, send to fallback recipient
597+
payload := NotificationEventPayload{
598+
EventName: api.EventWatchdog,
599+
EventCreatedAt: time.Now(),
600+
ID: notificationUUID,
601+
NotificationID: notificationUUID,
602+
PersonID: notification.FallbackPersonID,
603+
TeamID: notification.FallbackTeamID,
604+
CustomService: notification.FallbackCustomNotification,
605+
Properties: fmt.Appendf(nil, `{"id": "%s"}`, notificationUUID.String()),
606+
}
607+
608+
if err := sendNotification(ctx, payload); err != nil {
609+
return fmt.Errorf("failed to send watchdog notification to fallback recipient: %w", err)
610+
}
600611
}
601612

602-
return nCtx.EndLog()
613+
return nil
603614
}

notification/send.go

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -259,8 +259,7 @@ func SendNotification(ctx *Context, connectionName, shoutrrrURL string, celEnv m
259259
// based on the given event.
260260
func DefaultTitleAndBody(event string) (title string, body string) {
261261
switch event {
262-
case "notification.watchdog":
263-
// A dummy event that lives only within the application (never in the event queue)
262+
case api.EventWatchdog:
264263
title = `{{ if ne channel "slack"}}Notification {{.summary.namespace}}/{{.summary.name}} Summary{{end}}`
265264
content, _ := templates.ReadFile(fmt.Sprintf("templates/%s", event))
266265
body = string(content)

0 commit comments

Comments
 (0)