Merge pull request #6197 from oasisprotocol/kostko/feature/log-manager

go/runtime: Add log manager

Author: kostko

.changelog/6197.feature.md

@@ -0,0 +1,4 @@
+go/runtime: Add log manager
+
+Components can access logs of the bundles they deployed via the API as long
+as they have the proper `log_view` permissions.

docs/oasis-node/metrics.md

@@ -74,10 +74,10 @@ oasis_p2p_topics | Gauge | Number of supported P2P topics. |  | [p2p](https://gi
 oasis_registry_entities | Gauge | Number of registry entities. |  | [registry](https://github.com/oasisprotocol/oasis-core/tree/master/go/registry/metrics.go)
 oasis_registry_nodes | Gauge | Number of registry nodes. |  | [registry](https://github.com/oasisprotocol/oasis-core/tree/master/go/registry/metrics.go)
 oasis_registry_runtimes | Gauge | Number of registry runtimes. |  | [registry](https://github.com/oasisprotocol/oasis-core/tree/master/go/registry/metrics.go)
-oasis_rhp_failures | Counter | Number of failed Runtime Host calls. | call | [runtime/host/protocol](https://github.com/oasisprotocol/oasis-core/tree/master/go/runtime/host/protocol/connection.go)
-oasis_rhp_latency | Summary | Runtime Host call latency (seconds). | call | [runtime/host/protocol](https://github.com/oasisprotocol/oasis-core/tree/master/go/runtime/host/protocol/connection.go)
-oasis_rhp_successes | Counter | Number of successful Runtime Host calls. | call | [runtime/host/protocol](https://github.com/oasisprotocol/oasis-core/tree/master/go/runtime/host/protocol/connection.go)
-oasis_rhp_timeouts | Counter | Number of timed out Runtime Host calls. |  | [runtime/host/protocol](https://github.com/oasisprotocol/oasis-core/tree/master/go/runtime/host/protocol/connection.go)
+oasis_rhp_failures | Counter | Number of failed Runtime Host calls. | call | [runtime/host/protocol](https://github.com/oasisprotocol/oasis-core/tree/master/go/runtime/host/protocol/metrics.go)
+oasis_rhp_latency | Summary | Runtime Host call latency (seconds). | call | [runtime/host/protocol](https://github.com/oasisprotocol/oasis-core/tree/master/go/runtime/host/protocol/metrics.go)
+oasis_rhp_successes | Counter | Number of successful Runtime Host calls. | call | [runtime/host/protocol](https://github.com/oasisprotocol/oasis-core/tree/master/go/runtime/host/protocol/metrics.go)
+oasis_rhp_timeouts | Counter | Number of timed out Runtime Host calls. |  | [runtime/host/protocol](https://github.com/oasisprotocol/oasis-core/tree/master/go/runtime/host/protocol/metrics.go)
 oasis_roothash_block_interval | Summary | Time between roothash blocks (seconds). | runtime | [roothash](https://github.com/oasisprotocol/oasis-core/tree/master/go/roothash/metrics.go)
 oasis_storage_failures | Counter | Number of storage failures. | call | [storage/api](https://github.com/oasisprotocol/oasis-core/tree/master/go/storage/api/metrics.go)
 oasis_storage_latency | Summary | Storage call latency (seconds). | call | [storage/api](https://github.com/oasisprotocol/oasis-core/tree/master/go/storage/api/metrics.go)

go/.golangci.yml

@@ -1,7 +1,7 @@
 linters-settings:
   goconst:
     ignore-tests: true
-    ignore-strings: 'unix:.*'
+    ignore-strings: "unix:.*"
   goimports:
     # Put local imports after 3rd-party packages.
     local-prefixes: github.com/oasisprotocol/oasis-core
@@ -53,6 +53,7 @@ linters-settings:
           - github.com/tidwall/btree
           - github.com/tyler-smith/go-bip39
           - github.com/mdlayher/vsock
+          - github.com/nxadm/tail
 
 linters:
   disable-all: true

go/common/logging/filter.go

@@ -0,0 +1,42 @@
+package logging
+
+import (
+	"github.com/go-kit/log"
+)
+
+// filterLogger is a logger wrapper that filters out specific keys.
+type filterLogger struct {
+	logger        log.Logger
+	excludeFields map[any]struct{}
+}
+
+func (l *filterLogger) Log(keyvals ...any) error {
+	if len(keyvals) == 0 {
+		return nil
+	}
+
+	filteredKvs := make([]any, 0, len(keyvals))
+	for i := 0; i < len(keyvals); i += 2 {
+		if _, ok := l.excludeFields[keyvals[i]]; ok {
+			continue
+		}
+
+		filteredKvs = append(filteredKvs, keyvals[i])
+		if i+1 < len(keyvals) {
+			filteredKvs = append(filteredKvs, keyvals[i+1])
+		}
+	}
+
+	return l.logger.Log(filteredKvs...)
+}
+
+// NewFilterLogger creates a logger wrapper that filters out specific keys.
+func NewFilterLogger(base *Logger, excludeFields map[any]struct{}) *Logger {
+	return &Logger{
+		logger: &filterLogger{
+			logger:        base.logger,
+			excludeFields: excludeFields,
+		},
+		level: base.level,
+	}
+}

go/common/logging/filter_test.go

@@ -0,0 +1,22 @@
+package logging
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestFilterLogger(t *testing.T) {
+	require := require.New(t)
+
+	var buffer bytes.Buffer
+	logger := NewJSONLogger(&buffer)
+	logger = NewFilterLogger(logger, map[any]struct{}{
+		"foo": {},
+	})
+	logger.Info("this is a test", "bar", 42, "foo", 17)
+
+	const expectedOutput = `{"bar":42,"level":"info","msg":"this is a test"}` + "\n"
+	require.Equal(expectedOutput, buffer.String())
+}

go/common/logging/json.go

@@ -0,0 +1,14 @@
+package logging
+
+import (
+	"io"
+
+	"github.com/go-kit/log"
+)
+
+// NewJSONLogger creates a new logger which logs JSON-serialized logs directly to the given writer.
+func NewJSONLogger(w io.Writer) *Logger {
+	return &Logger{
+		logger: log.NewJSONLogger(w),
+	}
+}

go/common/logging/multi.go

@@ -0,0 +1,44 @@
+package logging
+
+import (
+	"errors"
+
+	"github.com/go-kit/log"
+)
+
+type multiLogger struct {
+	loggers []log.Logger
+}
+
+func (l *multiLogger) Log(keyvals ...any) error {
+	var mergedErr error
+	for _, logger := range l.loggers {
+		err := logger.Log(keyvals...)
+		mergedErr = errors.Join(mergedErr, err)
+	}
+	return mergedErr
+}
+
+// NewMultiLogger creates a new multi logger which logs to extra logging backends in addition to
+// the base one.
+func NewMultiLogger(base *Logger, extra ...*Logger) *Logger {
+	loggers := make([]log.Logger, 0, len(extra)+1)
+	loggers = append(loggers, base.logger)
+	for _, l := range extra {
+		loggers = append(loggers, l.logger)
+	}
+
+	var logger log.Logger
+	logger = &multiLogger{
+		loggers: loggers,
+	}
+	if base.module != "" {
+		logger = log.WithPrefix(logger, "module", base.module)
+	}
+
+	return &Logger{
+		logger: logger,
+		level:  base.level,
+		module: base.module,
+	}
+}

go/common/logging/multi_test.go

@@ -0,0 +1,36 @@
+package logging
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestMultiLogger(t *testing.T) {
+	require := require.New(t)
+
+	var (
+		bufferA bytes.Buffer
+		bufferB bytes.Buffer
+	)
+	loggerA := NewJSONLogger(&bufferA)
+	loggerB := NewJSONLogger(&bufferB)
+	multi := NewMultiLogger(loggerA, loggerB)
+	multi.Info("this is a test", "foo", 3)
+
+	const expectedOutput1 = `{"foo":3,"level":"info","msg":"this is a test"}` + "\n"
+	require.Equal(expectedOutput1, bufferA.String())
+	require.Equal(expectedOutput1, bufferB.String())
+
+	var bufferC bytes.Buffer
+	loggerC := NewJSONLogger(&bufferC).With("module", "test")
+	loggerC.module = "test"
+	multi = NewMultiLogger(loggerC, loggerB)
+	bufferB.Reset()
+	multi.Info("this is another test", "foo", 42)
+
+	const expectedOutput2 = `{"foo":42,"level":"info","module":"test","msg":"this is another test"}` + "\n"
+	require.Equal(expectedOutput2, bufferB.String())
+	require.Equal(expectedOutput2, bufferC.String())
+}

go/go.mod

@@ -37,6 +37,7 @@ require (
 	github.com/libp2p/go-libp2p-pubsub v0.13.0
 	github.com/mdlayher/vsock v1.2.1
 	github.com/multiformats/go-multiaddr v0.14.0
+	github.com/nxadm/tail v1.4.11
 	github.com/oasisprotocol/curve25519-voi v0.0.0-20230904125328-1f23a7beb09a
 	github.com/oasisprotocol/deoxysii v0.0.0-20220228165953-2091330c22b7
 	github.com/olekukonko/tablewriter v0.0.5

go/runtime/config/config.go

@@ -159,6 +159,9 @@ type Config struct {
 
 	// TDX is configuration specific to Intel TDX.
 	TDX TdxConfig `yaml:"tdx,omitempty"`
+
+	// Log is the runtime log config.
+	Log LogConfig `yaml:"log,omitempty"`
 }
 
 // GetComponent returns the configuration for the given component
@@ -206,6 +209,20 @@ type TdxConfig struct {
 	CidCount uint32 `yaml:"cid_count,omitempty"`
 }
 
+// LogConfig is the runtime log configuration.
+type LogConfig struct {
+	// MaxLogSize is the maximum log size in bytes.
+	MaxLogSize int `yaml:"max_log_size,omitempty"`
+}
+
+// Validate validates the log configuration for correctness.
+func (l *LogConfig) Validate() error {
+	if l.MaxLogSize < 1024 {
+		return fmt.Errorf("maximum log size must be at least 1024 bytes")
+	}
+	return nil
+}
+
 // RuntimeConfig is the runtime configuration.
 type RuntimeConfig struct {
 	// ID is the runtime identifier.
@@ -327,6 +344,9 @@ const (
 
 	// PermissionVolumeRemove is the permission that grants the component rights to remove volumes.
 	PermissionVolumeRemove ComponentPermission = "volume_remove"
+
+	// PermissionLogView is the permission that grants the component rights to view logs.
+	PermissionLogView ComponentPermission = "log_view"
 )
 
 // NetworkingConfig is the networking configuration.
@@ -414,6 +434,10 @@ func (c *Config) Validate() error {
 		return fmt.Errorf("cannot specify more than 128 instances for load balancing")
 	}
 
+	if err := c.Log.Validate(); err != nil {
+		return err
+	}
+
 	for _, rt := range c.Runtimes {
 		if err := rt.Validate(); err != nil {
 			return err
@@ -522,5 +546,8 @@ func DefaultConfig() Config {
 			CidStart: 0xA5150000,
 			CidCount: 1024,
 		},
+		Log: LogConfig{
+			MaxLogSize: 1024 * 1024,
+		},
 	}
 }