From 56b7fb550c32f5df267e3e59fcbe032f96d3f500 Mon Sep 17 00:00:00 2001 From: Tommy Reilly Date: Tue, 12 Aug 2025 12:40:48 -0400 Subject: [PATCH] Revamp oom logging --- go.mod | 2 +- go.sum | 4 ++-- interpreter/oomwatcher/log.go | 36 ++++++++++++++++++++++++++++ interpreter/oomwatcher/oomwatcher.go | 16 ++++++------- 4 files changed, 46 insertions(+), 12 deletions(-) create mode 100644 interpreter/oomwatcher/log.go diff --git a/go.mod b/go.mod index b06db7a67..c027a20b5 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/klauspost/compress v1.18.0 github.com/mdlayher/kobject v0.0.0-20200520190114-19ca17470d7d github.com/minio/sha256-simd v1.0.1 - github.com/parca-dev/oomprof v0.1.5-0.20250811212952-b45abb82a5fd + github.com/parca-dev/oomprof v0.1.5-0.20250812163638-9a36e2d4a690 github.com/peterbourgon/ff/v3 v3.4.0 github.com/sirupsen/logrus v1.9.3 github.com/stretchr/testify v1.10.0 diff --git a/go.sum b/go.sum index 33090ec35..b7c0ae4c4 100644 --- a/go.sum +++ b/go.sum @@ -169,8 +169,8 @@ github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQ github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= github.com/opencontainers/runtime-spec v1.2.0 h1:z97+pHb3uELt/yiAWD691HNHQIF07bE7dzrbT927iTk= github.com/opencontainers/runtime-spec v1.2.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/parca-dev/oomprof v0.1.5-0.20250811212952-b45abb82a5fd h1:uHo+bScocZbyqOnH7FeynMvaK/Exk+QQGFn5dfI4TRU= -github.com/parca-dev/oomprof v0.1.5-0.20250811212952-b45abb82a5fd/go.mod h1:+vw0+rZYq/rTVouF6G+/+Uol758EOkUINi8/8RXPvX0= +github.com/parca-dev/oomprof v0.1.5-0.20250812163638-9a36e2d4a690 h1:LKaTgZjbkz7wxHYU5VMKZ655Cp3IfG9a4ttPWRtflvU= +github.com/parca-dev/oomprof v0.1.5-0.20250812163638-9a36e2d4a690/go.mod h1:+vw0+rZYq/rTVouF6G+/+Uol758EOkUINi8/8RXPvX0= github.com/peterbourgon/ff/v3 v3.4.0 h1:QBvM/rizZM1cB0p0lGMdmR7HxZeI/ZrBWB4DqLkMUBc= github.com/peterbourgon/ff/v3 v3.4.0/go.mod h1:zjJVUhx+twciwfDl0zBcFzl4dW8axCRyXE/eKY9RztQ= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= diff --git a/interpreter/oomwatcher/log.go b/interpreter/oomwatcher/log.go new file mode 100644 index 000000000..c569ad363 --- /dev/null +++ b/interpreter/oomwatcher/log.go @@ -0,0 +1,36 @@ +// Copyright 2024 The Parca Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + +package oomwatcher // import "go.opentelemetry.io/ebpf-profiler/interpreter/oomwatcher" + +import ( + "os" + + "github.com/sirupsen/logrus" +) + +// Set this to true when debug env var is set. +var development bool + +func init() { + _, dbgEnv := os.LookupEnv("OOM_DEBUG") + development = dbgEnv +} + +// logf logs debugging as higher level so they stick out w/o +// enabling debug firehose if LUA_DEBUG env var is set. +func logf(format string, args ...interface{}) { + if development { + logrus.Infof(format, args...) + } else { + logrus.Debugf(format, args...) + } +} diff --git a/interpreter/oomwatcher/oomwatcher.go b/interpreter/oomwatcher/oomwatcher.go index 9dbdaf01e..728b25f75 100644 --- a/interpreter/oomwatcher/oomwatcher.go +++ b/interpreter/oomwatcher/oomwatcher.go @@ -7,8 +7,6 @@ import ( "fmt" "strings" - log "github.com/sirupsen/logrus" - "github.com/parca-dev/oomprof/oomprof" "go.opentelemetry.io/ebpf-profiler/interpreter" "go.opentelemetry.io/ebpf-profiler/libpf" @@ -35,20 +33,20 @@ func Loader(_ interpreter.EbpfHandler, info *interpreter.LoaderInfo) (interprete if err != nil { // oomprof not initialized, OOM prof must be initialized early by the otel container // project so it can be wired up to a TraceReporter instance. - log.Debugf("OOM watcher: oomprof not initialized, disabling OOM watching: %v", err) + logf("oom: oomprof not initialized, disabling OOM watching: %v", err) return nil, nil } file, err := info.GetELF() if err != nil { - log.Debugf("OOM watcher: elf err, disabling OOM watching: %v", err) + logf("oom: elf err, disabling OOM watching: %v", err) return nil, err } // Check if this is a Go binary if !file.IsGolang() { if strings.Contains(info.FileName(), "oomprof") { - log.Debugf("OOM watcher: not go, disabling OOM watching: %s", info.FileName()) + logf("oom: not go, disabling OOM watching: %s", info.FileName()) } return nil, nil } @@ -66,12 +64,12 @@ func Loader(_ interpreter.EbpfHandler, info *interpreter.LoaderInfo) (interprete } if sym != nil { - log.Debugf("OOM watcher: found mbuckets symbol (%v) in %s", sym, info.FileName()) + logf("oom: found mbuckets symbol (%v) in %s", sym, info.FileName()) return &oomWatcherData{ state: state, }, nil } - log.Debugf("OOM watcher: no mbuckets symbol found %s", info.FileName()) + logf("oom: no mbuckets symbol found %s", info.FileName()) return nil, nil } @@ -86,7 +84,7 @@ func (d *oomWatcherData) Attach(ebpf interpreter.EbpfHandler, pid libpf.PID, // Only watch PIDs for Go processes with mbucket symbol if d.state != nil { - log.Debugf("OOM watcher: watching PID %d", pid) + logf("oom: watching PID %d", pid) if err := d.state.WatchPid(uint32(pid)); err != nil { if err == oomprof.ErrSelfWatch { return nil, nil @@ -100,7 +98,7 @@ func (d *oomWatcherData) Attach(ebpf interpreter.EbpfHandler, pid libpf.PID, // Detach stops watching the process. func (i *oomWatcherInstance) Detach(_ interpreter.EbpfHandler, pid libpf.PID) error { - log.Debugf("OOM watcher: stopping watch for PID %d", pid) + logf("oom: stopping watch for PID %d", pid) i.data.state.UnwatchPid(uint32(pid)) return nil }