Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 11 additions & 5 deletions cmd/overlaybd-snapshotter/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -48,10 +48,12 @@ var pconfig *overlaybd.BootConfig
var commitID string = "unknown"

func requestIDInterceptor(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) {
requestID := mylog.GenerateRequestID()
requestID := tracing.GetRequestID(ctx)
if requestID == "" {
requestID = mylog.GenerateRequestID()
}

ctx = mylog.WithRequestID(ctx, requestID)

// Add to containerd's logger context
ctx = log.WithLogger(ctx, log.G(ctx).WithField("req_id", requestID))

return handler(ctx, req)
Expand Down Expand Up @@ -136,8 +138,12 @@ func main() {
// Initialize random seed for request ID generation
rand.Seed(time.Now().UnixNano())

srv := grpc.NewServer(grpc.UnaryInterceptor(requestIDInterceptor))
snapshotsapi.RegisterSnapshotsServer(srv, tracing.WithTracing(snapshotservice.FromSnapshotter(sn)))
// Use simplified tracing with request ID interceptor
srv := grpc.NewServer(
tracing.WithServerTracing(),
grpc.UnaryInterceptor(requestIDInterceptor),
)
snapshotsapi.RegisterSnapshotsServer(srv, snapshotservice.FromSnapshotter(sn))

address := strings.TrimSpace(pconfig.Address)

Expand Down
19 changes: 11 additions & 8 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -25,18 +25,20 @@ require (
github.com/sirupsen/logrus v1.9.3
github.com/spf13/cobra v1.8.1
github.com/urfave/cli/v2 v2.27.6
go.opentelemetry.io/otel v1.35.0
go.opentelemetry.io/otel v1.37.0
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.35.0
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.35.0
go.opentelemetry.io/otel/sdk v1.35.0
go.opentelemetry.io/otel/trace v1.35.0
go.opentelemetry.io/otel/sdk v1.37.0
go.opentelemetry.io/otel/trace v1.37.0
golang.org/x/sync v0.15.0
golang.org/x/sys v0.33.0
google.golang.org/grpc v1.72.2
google.golang.org/grpc v1.73.0
gotest.tools/gotestsum v1.12.3
oras.land/oras-go/v2 v2.5.0
)

require go.opentelemetry.io/otel/sdk/metric v1.37.0 // indirect

require (
filippo.io/edwards25519 v1.1.0 // indirect
github.com/Microsoft/go-winio v0.6.2 // indirect
Expand Down Expand Up @@ -65,7 +67,7 @@ require (
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/fsnotify/fsnotify v1.9.0 // indirect
github.com/fxamacker/cbor/v2 v2.7.0 // indirect
github.com/go-logr/logr v1.4.2 // indirect
github.com/go-logr/logr v1.4.3 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/godbus/dbus/v5 v5.1.0 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
Expand Down Expand Up @@ -105,17 +107,18 @@ require (
go.etcd.io/bbolt v1.4.0 // indirect
go.opencensus.io v0.24.0 // indirect
go.opentelemetry.io/auto/sdk v1.1.0 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.62.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 // indirect
go.opentelemetry.io/otel/metric v1.35.0 // indirect
go.opentelemetry.io/otel/metric v1.37.0 // indirect
go.opentelemetry.io/proto/otlp v1.5.0 // indirect
golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f // indirect
golang.org/x/mod v0.25.0 // indirect
golang.org/x/net v0.41.0 // indirect
golang.org/x/term v0.32.0 // indirect
golang.org/x/text v0.26.0 // indirect
golang.org/x/tools v0.34.0 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20250218202821-56aae31c358a // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20250218202821-56aae31c358a // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20250324211829-b45e905df463 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822 // indirect
google.golang.org/protobuf v1.36.6 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
Expand Down
38 changes: 20 additions & 18 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -87,8 +87,8 @@ github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8
github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E=
github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ=
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
github.com/go-quicktest/qt v1.101.0 h1:O1K29Txy5P2OK0dGo59b7b0LR6wKfIhttaAhHUyn7eI=
Expand Down Expand Up @@ -277,22 +277,24 @@ go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA=
go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.62.0 h1:rbRJ8BBoVMsQShESYZ0FkvcITu8X8QNwJogcLUmDNNw=
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.62.0/go.mod h1:ru6KHrNtNHxM4nD/vd6QrLVWgKhxPYgblq4VAtNawTQ=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 h1:sbiXRNDSWJOTobXh5HyQKjq6wUC5tNybqjIqDpAY4CU=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0/go.mod h1:69uWxva0WgAA/4bu2Yy70SLDBwZXuQ6PbBpbsa5iZrQ=
go.opentelemetry.io/otel v1.35.0 h1:xKWKPxrxB6OtMCbmMY021CqC45J+3Onta9MqjhnusiQ=
go.opentelemetry.io/otel v1.35.0/go.mod h1:UEqy8Zp11hpkUrL73gSlELM0DupHoiq72dR+Zqel/+Y=
go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ=
go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.35.0 h1:1fTNlAIJZGWLP5FVu0fikVry1IsiUnXjf7QFvoNN3Xw=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.35.0/go.mod h1:zjPK58DtkqQFn+YUMbx0M2XV3QgKU0gS9LeGohREyK4=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.35.0 h1:m639+BofXTvcY1q8CGs4ItwQarYtJPOWmVobfM1HpVI=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.35.0/go.mod h1:LjReUci/F4BUyv+y4dwnq3h/26iNOeC3wAIqgvTIZVo=
go.opentelemetry.io/otel/metric v1.35.0 h1:0znxYu2SNyuMSQT4Y9WDWej0VpcsxkuklLa4/siN90M=
go.opentelemetry.io/otel/metric v1.35.0/go.mod h1:nKVFgxBZ2fReX6IlyW28MgZojkoAkJGaE8CpgeAU3oE=
go.opentelemetry.io/otel/sdk v1.35.0 h1:iPctf8iprVySXSKJffSS79eOjl9pvxV9ZqOWT0QejKY=
go.opentelemetry.io/otel/sdk v1.35.0/go.mod h1:+ga1bZliga3DxJ3CQGg3updiaAJoNECOgJREo9KHGQg=
go.opentelemetry.io/otel/sdk/metric v1.35.0 h1:1RriWBmCKgkeHEhM7a2uMjMUfP7MsOF5JpUCaEqEI9o=
go.opentelemetry.io/otel/sdk/metric v1.35.0/go.mod h1:is6XYCUMpcKi+ZsOvfluY5YstFnhW0BidkR+gL+qN+w=
go.opentelemetry.io/otel/trace v1.35.0 h1:dPpEfJu1sDIqruz7BHFG3c7528f6ddfSWfFDVt/xgMs=
go.opentelemetry.io/otel/trace v1.35.0/go.mod h1:WUk7DtFp1Aw2MkvqGdwiXYDZZNvA/1J8o6xRXLrIkyc=
go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE=
go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E=
go.opentelemetry.io/otel/sdk v1.37.0 h1:ItB0QUqnjesGRvNcmAcU0LyvkVyGJ2xftD29bWdDvKI=
go.opentelemetry.io/otel/sdk v1.37.0/go.mod h1:VredYzxUvuo2q3WRcDnKDjbdvmO0sCzOvVAiY+yUkAg=
go.opentelemetry.io/otel/sdk/metric v1.37.0 h1:90lI228XrB9jCMuSdA0673aubgRobVZFhbjxHHspCPc=
go.opentelemetry.io/otel/sdk/metric v1.37.0/go.mod h1:cNen4ZWfiD37l5NhS+Keb5RXVWZWpRE+9WyVCpbo5ps=
go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4=
go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0=
go.opentelemetry.io/proto/otlp v1.5.0 h1:xJvq7gMzB31/d406fB8U5CBdyQGw4P399D1aQWU/3i4=
go.opentelemetry.io/proto/otlp v1.5.0/go.mod h1:keN8WnHxOy8PG0rQZjJJ5A2ebUoafqWp0eVQ4yIXvJ4=
go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs=
Expand Down Expand Up @@ -368,17 +370,17 @@ google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
google.golang.org/genproto/googleapis/api v0.0.0-20250218202821-56aae31c358a h1:nwKuGPlUAt+aR+pcrkfFRrTU1BVrSmYyYMxYbUIVHr0=
google.golang.org/genproto/googleapis/api v0.0.0-20250218202821-56aae31c358a/go.mod h1:3kWAYMk1I75K4vykHtKt2ycnOgpA6974V7bREqbsenU=
google.golang.org/genproto/googleapis/rpc v0.0.0-20250218202821-56aae31c358a h1:51aaUVRocpvUOSQKM6Q7VuoaktNIaMCLuhZB6DKksq4=
google.golang.org/genproto/googleapis/rpc v0.0.0-20250218202821-56aae31c358a/go.mod h1:uRxBH1mhmO8PGhU89cMcHaXKZqO+OfakD8QQO0oYwlQ=
google.golang.org/genproto/googleapis/api v0.0.0-20250324211829-b45e905df463 h1:hE3bRWtU6uceqlh4fhrSnUyjKHMKB9KrTLLG+bc0ddM=
google.golang.org/genproto/googleapis/api v0.0.0-20250324211829-b45e905df463/go.mod h1:U90ffi8eUL9MwPcrJylN5+Mk2v3vuPDptd5yyNUiRR8=
google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822 h1:fc6jSaCT0vBduLYZHYrBBNY4dsWuvgyff9noRNDdBeE=
google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
google.golang.org/grpc v1.72.2 h1:TdbGzwb82ty4OusHWepvFWGLgIbNo1/SUynEN0ssqv8=
google.golang.org/grpc v1.72.2/go.mod h1:wH5Aktxcg25y1I3w7H69nHfXdOG3UiadoBtjh3izSDM=
google.golang.org/grpc v1.73.0 h1:VIWSmpI2MegBtTuFt5/JWy2oXxtjJ/e89Z70ImfD2ok=
google.golang.org/grpc v1.73.0/go.mod h1:50sbHOUqWoCQGI8V2HQLJM0B+LMlIUjNSZmow7EVBQc=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
Expand Down
191 changes: 191 additions & 0 deletions pkg/tracing/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,191 @@
# gRPC Tracing Integration

This package provides OpenTelemetry tracing integration for gRPC calls in the accelerated-container-image project.

## Features

- **Automatic trace propagation**: Trace contexts are automatically propagated between client and server calls
- **Comprehensive tracing**: Both client and server interceptors capture request/response metadata
- **OpenTelemetry integration**: Uses standard OpenTelemetry libraries for compatibility
- **Error handling**: Automatically records errors and sets appropriate span status

## Usage

### Server Setup

The main snapshotter service already includes tracing interceptors:

```go
// Chain interceptors: tracing first, then request ID
interceptors := grpc.ChainUnaryInterceptor(
tracing.UnaryServerInterceptor(),
requestIDInterceptor,
)
srv := grpc.NewServer(interceptors)
```

Or use the convenience function:

```go
srv := grpc.NewServer(tracing.WithServerTracing())
```

### Client Setup

For gRPC clients, add the tracing interceptor:

```go
conn, err := grpc.DialContext(ctx, target,
grpc.WithUnaryInterceptor(tracing.UnaryClientInterceptor()),
// other options...
)
```

Or use the convenience function:

```go
conn, err := grpc.DialContext(ctx, target,
tracing.WithClientTracing(),
// other options...
)
```

### Trace Propagation

Trace IDs automatically propagate from:
1. **Incoming requests** → Server-side processing
2. **Server-side processing** → Outgoing client calls
3. **Client calls** → Downstream services

Example flow:
```
Client Request [trace-id: abc123]
gRPC Server Interceptor [trace-id: abc123]
Snapshotter Service [trace-id: abc123]
Outgoing Client Call [trace-id: abc123]
```

## Trace Attributes

The interceptors automatically add these attributes to spans:

### Common Attributes
- `rpc.system`: "grpc"
- `rpc.service`: Service name (e.g., "containerd.services.snapshots.v1.Snapshots")
- `rpc.method`: Method name (e.g., "Prepare")
- `rpc.grpc.status_code`: gRPC status code
- `rpc.grpc.duration_ms`: Request duration in milliseconds

### Error Handling
- Span status set to ERROR for failed requests
- Error details recorded in span
- gRPC status code captured

## Configuration

Set these environment variables for tracing:

```bash
# Service name for traces
export OTEL_SERVICE_NAME="accelerated-container-image"

# OTLP endpoint (e.g., Jaeger collector)
export OTEL_EXPORTER_OTLP_ENDPOINT="http://localhost:4317"

# Environment tag
export ENVIRONMENT="production"
```

## Baggage API Usage

The implementation includes full support for OpenTelemetry Baggage, allowing you to propagate key-value pairs across service boundaries.

### Setting Baggage Values

```go
import "github.com/containerd/accelerated-container-image/pkg/tracing"

// Set individual values
ctx = tracing.SetRequestID(ctx, "req-12345")
ctx = tracing.SetUserID(ctx, "user-67890")
ctx = tracing.SetOperation(ctx, "prepare-snapshot")

// Set image information
ctx = tracing.SetImageInfo(ctx, "alpine", "3.18")

// Set multiple values at once
baggageMap := map[string]string{
"environment": "production",
"namespace": "default",
"container.id": "container-123",
}
ctx = tracing.SetBaggageFromMap(ctx, baggageMap)
```

### Getting Baggage Values

```go
// Get individual values
requestID := tracing.GetRequestID(ctx)
userID := tracing.GetUserID(ctx)
operation := tracing.GetOperation(ctx)

// Get image information
imageName, imageTag := tracing.GetImageInfo(ctx)

// Get all baggage as a map
allBaggage := tracing.GetBaggageAsMap(ctx)
```

### Automatic Baggage Propagation

Baggage values are automatically:
1. **Extracted** from incoming gRPC metadata
2. **Injected** into outgoing gRPC metadata
3. **Added as span attributes** with `baggage.` prefix
4. **Propagated** through the entire request chain

### Common Baggage Keys

The package defines standard baggage keys:
- `request.id` - Request identifier
- `user.id` - User identifier
- `session.id` - Session identifier
- `operation` - Operation name
- `image.name` - Container image name
- `image.tag` - Container image tag
- `snapshot.key` - Snapshot key
- `container.id` - Container identifier
- `namespace` - Kubernetes namespace
- `environment` - Environment (prod, staging, etc.)

## Integration with Request ID

The request ID interceptor now uses baggage for propagation:

```go
// Check if request ID exists in baggage (from upstream)
requestID := tracing.GetRequestID(ctx)
if requestID == "" {
// Generate new ID if not present
requestID = generateRequestID()
ctx = tracing.SetRequestID(ctx, requestID)
}
```

This ensures request IDs propagate across service boundaries while maintaining backward compatibility.

## Integration with Existing Code

The tracing is integrated with the existing snapshotter service:

1. **Initialization**: `tracing.InitTracer()` sets up OpenTelemetry with baggage propagation
2. **Server interceptor**: Added to gRPC server chain, extracts baggage from metadata
3. **Service wrapper**: `tracing.WithTracing()` wraps the snapshotter service
4. **Client interceptor**: Added to any outgoing gRPC clients, injects baggage into metadata
5. **Baggage utilities**: Helper functions for common baggage operations

This provides comprehensive tracing coverage at both the transport (gRPC) and application (snapshotter operations) levels, with full context propagation via baggage.
35 changes: 35 additions & 0 deletions pkg/tracing/baggage.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
/*
Copyright The Accelerated Container Image Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package tracing

import (
"context"

"go.opentelemetry.io/otel/baggage"
)

// SetRequestID sets the request ID in baggage
func SetRequestID(ctx context.Context, requestID string) context.Context {
member, _ := baggage.NewMember("request.id", requestID)
bag, _ := baggage.FromContext(ctx).SetMember(member)
return baggage.ContextWithBaggage(ctx, bag)
}

// GetRequestID gets the request ID from baggage
func GetRequestID(ctx context.Context) string {
return baggage.FromContext(ctx).Member("request.id").Value()
}
Loading
Loading