Add OTEL tracing

README.md

@@ -98,7 +98,49 @@ docker run overlaybd-convertor -r registry.hub.docker.com/library/redis -i 6.2.1
 
 * See how to use TurboOCIv1 at [TurboOCIv1](docs/TURBO_OCI.md).
 
-* Welcome to contribute! [CONTRIBUTING](docs/CONTRIBUTING.md)
+* See how to use OpenTelemetry tracing at [TRACING](docs/TRACING.md).
+
+## Testing
+
+The project uses Go's standard testing framework. To run the tests:
+
+```bash
+# Run all tests
+go test ./...
+
+# Run tests for a specific package
+go test ./pkg/tracing/...
+
+# Run tests with verbose output
+go test -v ./...
+
+# Run tests and show code coverage
+go test -cover ./...
+
+# Run tests and generate coverage report
+go test -coverprofile=coverage.out ./...
+go tool cover -html=coverage.out  # View coverage in browser
+```
+
+### Test Requirements
+
+Some tests require specific setup:
+
+- **Tracing Tests**: No external setup needed. Tests use in-memory tracing.
+- **Integration Tests**: Uses in-memory gRPC server, no external setup needed.
+- **Snapshotter Tests**: Requires root privileges for some tests. Run with `sudo` if needed.
+
+### Writing Tests
+
+When contributing new code:
+
+1. Add unit tests for new packages in `*_test.go` files
+2. Add integration tests for new features
+3. Follow existing test patterns in the codebase
+4. Use table-driven tests where appropriate
+5. Ensure tests are deterministic and don't depend on external services
+
+For more details on contributing, see [CONTRIBUTING](docs/CONTRIBUTING.md).
 
 ## Release Version Support
 

cmd/convertor/main.go

@@ -25,6 +25,7 @@ import (
 
 	"github.com/containerd/accelerated-container-image/cmd/convertor/builder"
 	"github.com/containerd/accelerated-container-image/cmd/convertor/database"
+	"github.com/containerd/accelerated-container-image/pkg/tracing"
 	"github.com/containerd/containerd/v2/core/remotes"
 	_ "github.com/go-sql-driver/mysql"
 	"github.com/sirupsen/logrus"
@@ -359,11 +360,28 @@ func init() {
 }
 
 func main() {
+	ctx := context.Background()
+
+	// Initialize OpenTelemetry
+	shutdown, err := tracing.InitTracer(ctx)
+	if err != nil {
+		logrus.Errorf("Failed to initialize tracer: %v", err)
+		os.Exit(1)
+	}
+	defer func() {
+		if err := shutdown(ctx); err != nil {
+			logrus.Errorf("Failed to shutdown tracer: %v", err)
+		}
+	}()
+
 	sigChan := make(chan os.Signal, 1)
 	signal.Notify(sigChan, os.Interrupt)
 
 	go func() {
 		<-sigChan
+		if err := shutdown(context.Background()); err != nil {
+			logrus.Errorf("Failed to shutdown tracer: %v", err)
+		}
 		os.Exit(0)
 	}()
 

cmd/overlaybd-snapshotter/main.go

@@ -31,6 +31,7 @@ import (
 	mylog "github.com/containerd/accelerated-container-image/internal/log"
 	"github.com/containerd/accelerated-container-image/pkg/metrics"
 	overlaybd "github.com/containerd/accelerated-container-image/pkg/snapshot"
+	"github.com/containerd/accelerated-container-image/pkg/tracing"
 
 	snapshotsapi "github.com/containerd/containerd/api/services/snapshots/v1"
 	"github.com/containerd/containerd/v2/contrib/snapshotservice"
@@ -72,6 +73,20 @@ func parseConfig(fpath string) error {
 
 // TODO: use github.com/urfave/cli/v2
 func main() {
+	ctx := context.Background()
+
+	// Initialize OpenTelemetry
+	shutdown, err := tracing.InitTracer(ctx)
+	if err != nil {
+		logrus.Errorf("Failed to initialize tracer: %v", err)
+		os.Exit(1)
+	}
+	defer func() {
+		if err := shutdown(ctx); err != nil {
+			logrus.Errorf("Failed to shutdown tracer: %v", err)
+		}
+	}()
+
 	pconfig = overlaybd.DefaultBootConfig()
 	fnConfig := defaultConfigPath
 	if len(os.Args) == 2 {
@@ -122,7 +137,7 @@ func main() {
 	rand.Seed(time.Now().UnixNano())
 
 	srv := grpc.NewServer(grpc.UnaryInterceptor(requestIDInterceptor))
-	snapshotsapi.RegisterSnapshotsServer(srv, snapshotservice.FromSnapshotter(sn))
+	snapshotsapi.RegisterSnapshotsServer(srv, tracing.WithTracing(snapshotservice.FromSnapshotter(sn)))
 
 	address := strings.TrimSpace(pconfig.Address)
 

docs/TRACING.md

@@ -0,0 +1,80 @@
+# OpenTelemetry Tracing Support
+
+The accelerated-container-image project includes OpenTelemetry (OTEL) tracing support to help monitor and debug image conversion operations. This document describes how to use and configure tracing in the project.
+
+## Overview
+
+Tracing is implemented using OpenTelemetry, which provides detailed insights into the image conversion process. Key operations that are traced include:
+
+- Overall image conversion process
+- Layer conversion operations
+- Individual layer application and processing
+- Remote layer operations (when using remote storage)
+
+## Configuration
+
+Tracing can be configured using standard OpenTelemetry environment variables:
+
+- `OTEL_SERVICE_NAME`: Sets the service name for traces (default: "accelerated-container-image")
+- `OTEL_EXPORTER_OTLP_ENDPOINT`: The endpoint where traces should be sent (e.g., "http://localhost:4317")
+- `OTEL_EXPORTER_OTLP_PROTOCOL`: The protocol to use (default: "grpc")
+- `ENVIRONMENT`: The environment name to be included in traces (e.g., "production", "staging")
+
+## Key Spans and Attributes
+
+The following spans are created during image conversion:
+
+### Convert Operation
+- Name: `Convert`
+- Attributes:
+  - `fsType`: The filesystem type being used
+  - `layerCount`: Number of layers in the source image
+
+### Layer Conversion
+- Name: `convertLayers`
+- Attributes:
+  - `fsType`: The filesystem type being used
+  - `layerCount`: Number of layers to convert
+
+### Layer Application
+- Name: `applyOCIV1LayerInObd`
+- Attributes:
+  - `layerDigest`: The digest of the layer being applied
+  - `layerSize`: Size of the layer in bytes
+  - `parentID`: ID of the parent snapshot
+
+## Example Usage
+
+1. Start your OpenTelemetry collector:
+   ```bash
+   docker run -d --name otel-collector \
+     -p 4317:4317 \
+     -p 4318:4318 \
+     otel/opentelemetry-collector-contrib
+   ```
+
+2. Set the required environment variables:
+   ```bash
+   export OTEL_SERVICE_NAME="accelerated-container-image"
+   export OTEL_EXPORTER_OTLP_ENDPOINT="http://localhost:4317"
+   export ENVIRONMENT="development"
+   ```
+
+3. Run the image conversion as normal. Traces will be automatically collected and sent to your configured endpoint.
+
+## Viewing Traces
+
+Traces can be viewed in any OpenTelemetry-compatible tracing backend, such as:
+- Jaeger
+- Zipkin
+- Grafana Tempo
+- Cloud provider tracing services (e.g., AWS X-Ray, Google Cloud Trace)
+
+## Troubleshooting
+
+If you're not seeing traces:
+
+1. Verify your OpenTelemetry collector is running and accessible
+2. Check that the `OTEL_EXPORTER_OTLP_ENDPOINT` is correctly configured
+3. Enable debug logging with the `--verbose` flag to see more detailed output
+4. Check your collector's logs for any connection or configuration issues

go.mod

@@ -36,6 +36,7 @@ require (
 	github.com/Microsoft/go-winio v0.6.2 // indirect
 	github.com/Microsoft/hcsshim v0.13.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cenkalti/backoff/v4 v4.3.0 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/cilium/ebpf v0.16.0 // indirect
 	github.com/containerd/cgroups/v3 v3.0.5 // indirect
@@ -61,6 +62,8 @@ require (
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/google/go-cmp v0.7.0 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.1 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/intel/goresctrl v0.8.0 // indirect
 	github.com/klauspost/compress v1.18.0 // indirect
@@ -92,12 +95,17 @@ require (
 	go.opentelemetry.io/auto/sdk v1.1.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 // indirect
 	go.opentelemetry.io/otel v1.35.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.35.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.35.0 // indirect
 	go.opentelemetry.io/otel/metric v1.35.0 // indirect
+	go.opentelemetry.io/otel/sdk v1.35.0 // indirect
 	go.opentelemetry.io/otel/trace v1.35.0 // indirect
+	go.opentelemetry.io/proto/otlp v1.5.0 // indirect
 	golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f // indirect
 	golang.org/x/mod v0.24.0 // indirect
 	golang.org/x/net v0.38.0 // indirect
 	golang.org/x/text v0.23.0 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20250218202821-56aae31c358a // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20250218202821-56aae31c358a // indirect
 	google.golang.org/protobuf v1.36.6 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect

go.sum

@@ -12,6 +12,8 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM=
 github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ=
+github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8=
+github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
 github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
@@ -127,6 +129,8 @@ github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+
 github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.1 h1:e9Rjr40Z98/clHv5Yg79Is0NtosR5LXRvdr7o/6NwbA=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.1/go.mod h1:tIxuGz/9mpox++sgp9fJjHO0+q1X9/UOWd798aAm22M=
 github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
@@ -264,6 +268,10 @@ go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 h1:sbiXRND
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0/go.mod h1:69uWxva0WgAA/4bu2Yy70SLDBwZXuQ6PbBpbsa5iZrQ=
 go.opentelemetry.io/otel v1.35.0 h1:xKWKPxrxB6OtMCbmMY021CqC45J+3Onta9MqjhnusiQ=
 go.opentelemetry.io/otel v1.35.0/go.mod h1:UEqy8Zp11hpkUrL73gSlELM0DupHoiq72dR+Zqel/+Y=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.35.0 h1:1fTNlAIJZGWLP5FVu0fikVry1IsiUnXjf7QFvoNN3Xw=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.35.0/go.mod h1:zjPK58DtkqQFn+YUMbx0M2XV3QgKU0gS9LeGohREyK4=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.35.0 h1:m639+BofXTvcY1q8CGs4ItwQarYtJPOWmVobfM1HpVI=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.35.0/go.mod h1:LjReUci/F4BUyv+y4dwnq3h/26iNOeC3wAIqgvTIZVo=
 go.opentelemetry.io/otel/metric v1.35.0 h1:0znxYu2SNyuMSQT4Y9WDWej0VpcsxkuklLa4/siN90M=
 go.opentelemetry.io/otel/metric v1.35.0/go.mod h1:nKVFgxBZ2fReX6IlyW28MgZojkoAkJGaE8CpgeAU3oE=
 go.opentelemetry.io/otel/sdk v1.35.0 h1:iPctf8iprVySXSKJffSS79eOjl9pvxV9ZqOWT0QejKY=
@@ -272,6 +280,8 @@ go.opentelemetry.io/otel/sdk/metric v1.35.0 h1:1RriWBmCKgkeHEhM7a2uMjMUfP7MsOF5J
 go.opentelemetry.io/otel/sdk/metric v1.35.0/go.mod h1:is6XYCUMpcKi+ZsOvfluY5YstFnhW0BidkR+gL+qN+w=
 go.opentelemetry.io/otel/trace v1.35.0 h1:dPpEfJu1sDIqruz7BHFG3c7528f6ddfSWfFDVt/xgMs=
 go.opentelemetry.io/otel/trace v1.35.0/go.mod h1:WUk7DtFp1Aw2MkvqGdwiXYDZZNvA/1J8o6xRXLrIkyc=
+go.opentelemetry.io/proto/otlp v1.5.0 h1:xJvq7gMzB31/d406fB8U5CBdyQGw4P399D1aQWU/3i4=
+go.opentelemetry.io/proto/otlp v1.5.0/go.mod h1:keN8WnHxOy8PG0rQZjJJ5A2ebUoafqWp0eVQ4yIXvJ4=
 go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs=
 go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8=
 go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA=
@@ -341,6 +351,9 @@ google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
 google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de h1:F6qOa9AZTYJXOUEr4jDysRDLrm4PHePlge4v4TGAlxY=
+google.golang.org/genproto/googleapis/api v0.0.0-20250218202821-56aae31c358a h1:nwKuGPlUAt+aR+pcrkfFRrTU1BVrSmYyYMxYbUIVHr0=
+google.golang.org/genproto/googleapis/api v0.0.0-20250218202821-56aae31c358a/go.mod h1:3kWAYMk1I75K4vykHtKt2ycnOgpA6974V7bREqbsenU=
 google.golang.org/genproto/googleapis/rpc v0.0.0-20250218202821-56aae31c358a h1:51aaUVRocpvUOSQKM6Q7VuoaktNIaMCLuhZB6DKksq4=
 google.golang.org/genproto/googleapis/rpc v0.0.0-20250218202821-56aae31c358a/go.mod h1:uRxBH1mhmO8PGhU89cMcHaXKZqO+OfakD8QQO0oYwlQ=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=