diff --git a/pkg/snapshot/overlay.go b/pkg/snapshot/overlay.go index e7c2afd6..7670020d 100644 --- a/pkg/snapshot/overlay.go +++ b/pkg/snapshot/overlay.go @@ -31,6 +31,9 @@ import ( "github.com/containerd/accelerated-container-image/pkg/label" "github.com/containerd/accelerated-container-image/pkg/snapshot/diskquota" + "github.com/containerd/accelerated-container-image/pkg/tracing" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" mylog "github.com/containerd/accelerated-container-image/internal/log" "github.com/containerd/accelerated-container-image/pkg/metrics" @@ -448,7 +451,6 @@ func (o *snapshotter) isPrepareRootfs(info snapshots.Info) bool { } func (o *snapshotter) createMountPoint(ctx context.Context, kind snapshots.Kind, key string, parent string, opts ...snapshots.Opt) (_ []mount.Mount, retErr error) { - ctx, t, err := o.ms.TransactionContext(ctx, true) if err != nil { return nil, err @@ -550,6 +552,7 @@ func (o *snapshotter) createMountPoint(ctx context.Context, kind snapshots.Kind, if err != nil { return nil, err } + trace.SpanFromContext(ctx).AddEvent("identifyStorageType", trace.WithAttributes(attribute.Int("stype", int(stype)))) // Download blob downloadBlob := info.Labels[label.DownloadRemoteBlob] @@ -994,6 +997,12 @@ func (o *snapshotter) Commit(ctx context.Context, name, key string, opts ...snap } func (o *snapshotter) commit(ctx context.Context, name, key string, opts ...snapshots.Opt) (string, snapshots.Info, error) { + ctx, span := tracing.GetDefaultTracer().Start(ctx, "snapshotter.commit", trace.WithAttributes( + attribute.String("snapshot_name", name), + attribute.String("key", key), + )) + defer span.End() + id, _, _, err := storage.GetInfo(ctx, key) if err != nil { return "", snapshots.Info{}, err @@ -1380,6 +1389,12 @@ func (o *snapshotter) getDiskQuotaSize(info *snapshots.Info) string { } func (o *snapshotter) createSnapshot(ctx context.Context, kind snapshots.Kind, key, parent string, opts []snapshots.Opt) (_ string, _ snapshots.Info, err error) { + ctx, span := tracing.GetDefaultTracer().Start(ctx, "snapshotter.createSnapshot", trace.WithAttributes( + attribute.String("kind", kind.String()), + attribute.String("parent", parent), + )) + defer span.End() + var td, path string defer func() { if err != nil { diff --git a/pkg/snapshot/storage.go b/pkg/snapshot/storage.go index 6e9f5f2d..4b56921c 100644 --- a/pkg/snapshot/storage.go +++ b/pkg/snapshot/storage.go @@ -33,6 +33,8 @@ import ( "time" sn "github.com/containerd/accelerated-container-image/pkg/types" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/propagation" "github.com/containerd/accelerated-container-image/pkg/label" "github.com/containerd/accelerated-container-image/pkg/utils" @@ -503,9 +505,12 @@ func (o *snapshotter) constructOverlayBDSpec(ctx context.Context, key string, wr return errors.Wrapf(err, "failed to identify storage of snapshot %s", key) } + var carrier = propagation.MapCarrier{} + otel.GetTextMapPropagator().Inject(ctx, carrier) configJSON := sn.OverlayBDBSConfig{ - Lowers: []sn.OverlayBDBSConfigLower{}, - ResultFile: o.overlaybdInitDebuglogPath(id), + Lowers: []sn.OverlayBDBSConfigLower{}, + ResultFile: o.overlaybdInitDebuglogPath(id), + TraceContext: carrier, } // load the parent's config and reuse the lowerdir diff --git a/pkg/tracing/snapshotter.go b/pkg/tracing/snapshotter.go index 1ea48510..062dde90 100644 --- a/pkg/tracing/snapshotter.go +++ b/pkg/tracing/snapshotter.go @@ -21,13 +21,10 @@ import ( snapshotsapi "github.com/containerd/containerd/api/services/snapshots/v1" ptypes "github.com/containerd/containerd/v2/pkg/protobuf/types" - "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" ) -const tracerName = "accelerated-container-image/snapshotter" - // TracingSnapshotter wraps a snapshotter service with OpenTelemetry tracing type TracingSnapshotter struct { server snapshotsapi.SnapshotsServer @@ -45,7 +42,7 @@ func WithTracing(server snapshotsapi.SnapshotsServer) snapshotsapi.SnapshotsServ } func (s *TracingSnapshotter) Prepare(ctx context.Context, pr *snapshotsapi.PrepareSnapshotRequest) (*snapshotsapi.PrepareSnapshotResponse, error) { - ctx, span := otel.GetTracerProvider().Tracer(tracerName).Start(ctx, "snapshotter.Prepare", trace.WithAttributes( + ctx, span := GetDefaultTracer().Start(ctx, "snapshotter.Prepare", trace.WithAttributes( attribute.String("key", pr.Key), attribute.String("parent", pr.Parent), )) @@ -59,7 +56,7 @@ func (s *TracingSnapshotter) Prepare(ctx context.Context, pr *snapshotsapi.Prepa } func (s *TracingSnapshotter) View(ctx context.Context, pr *snapshotsapi.ViewSnapshotRequest) (*snapshotsapi.ViewSnapshotResponse, error) { - ctx, span := otel.GetTracerProvider().Tracer(tracerName).Start(ctx, "snapshotter.View", trace.WithAttributes( + ctx, span := GetDefaultTracer().Start(ctx, "snapshotter.View", trace.WithAttributes( attribute.String("key", pr.Key), attribute.String("parent", pr.Parent), )) @@ -73,7 +70,7 @@ func (s *TracingSnapshotter) View(ctx context.Context, pr *snapshotsapi.ViewSnap } func (s *TracingSnapshotter) Mounts(ctx context.Context, mr *snapshotsapi.MountsRequest) (*snapshotsapi.MountsResponse, error) { - ctx, span := otel.GetTracerProvider().Tracer(tracerName).Start(ctx, "snapshotter.Mounts", trace.WithAttributes( + ctx, span := GetDefaultTracer().Start(ctx, "snapshotter.Mounts", trace.WithAttributes( attribute.String("key", mr.Key), )) defer span.End() @@ -86,7 +83,7 @@ func (s *TracingSnapshotter) Mounts(ctx context.Context, mr *snapshotsapi.Mounts } func (s *TracingSnapshotter) Commit(ctx context.Context, cr *snapshotsapi.CommitSnapshotRequest) (*ptypes.Empty, error) { - ctx, span := otel.GetTracerProvider().Tracer(tracerName).Start(ctx, "snapshotter.Commit", trace.WithAttributes( + ctx, span := GetDefaultTracer().Start(ctx, "snapshotter.Commit", trace.WithAttributes( attribute.String("name", cr.Name), attribute.String("key", cr.Key), )) @@ -100,7 +97,7 @@ func (s *TracingSnapshotter) Commit(ctx context.Context, cr *snapshotsapi.Commit } func (s *TracingSnapshotter) Remove(ctx context.Context, rr *snapshotsapi.RemoveSnapshotRequest) (*ptypes.Empty, error) { - ctx, span := otel.GetTracerProvider().Tracer(tracerName).Start(ctx, "snapshotter.Remove", trace.WithAttributes( + ctx, span := GetDefaultTracer().Start(ctx, "snapshotter.Remove", trace.WithAttributes( attribute.String("key", rr.Key), )) defer span.End() @@ -113,20 +110,12 @@ func (s *TracingSnapshotter) Remove(ctx context.Context, rr *snapshotsapi.Remove } func (s *TracingSnapshotter) Stat(ctx context.Context, sr *snapshotsapi.StatSnapshotRequest) (*snapshotsapi.StatSnapshotResponse, error) { - ctx, span := otel.GetTracerProvider().Tracer(tracerName).Start(ctx, "snapshotter.Stat", trace.WithAttributes( - attribute.String("key", sr.Key), - )) - defer span.End() - resp, err := s.server.Stat(ctx, sr) - if err != nil { - span.RecordError(err) - } return resp, err } func (s *TracingSnapshotter) Update(ctx context.Context, sr *snapshotsapi.UpdateSnapshotRequest) (*snapshotsapi.UpdateSnapshotResponse, error) { - ctx, span := otel.GetTracerProvider().Tracer(tracerName).Start(ctx, "snapshotter.Update", trace.WithAttributes( + ctx, span := GetDefaultTracer().Start(ctx, "snapshotter.Update", trace.WithAttributes( attribute.String("name", sr.Info.Name), )) defer span.End() @@ -139,7 +128,7 @@ func (s *TracingSnapshotter) Update(ctx context.Context, sr *snapshotsapi.Update } func (s *TracingSnapshotter) List(sr *snapshotsapi.ListSnapshotsRequest, ss snapshotsapi.Snapshots_ListServer) error { - ctx, span := otel.GetTracerProvider().Tracer(tracerName).Start(ss.Context(), "snapshotter.List") + ctx, span := GetDefaultTracer().Start(ss.Context(), "snapshotter.List") defer span.End() err := s.server.List(sr, &tracingListServer{ @@ -162,26 +151,12 @@ func (t *tracingListServer) Context() context.Context { } func (s *TracingSnapshotter) Usage(ctx context.Context, ur *snapshotsapi.UsageRequest) (*snapshotsapi.UsageResponse, error) { - ctx, span := otel.GetTracerProvider().Tracer(tracerName).Start(ctx, "snapshotter.Usage", trace.WithAttributes( - attribute.String("key", ur.Key), - )) - defer span.End() - resp, err := s.server.Usage(ctx, ur) - if err != nil { - span.RecordError(err) - } - if resp != nil { - span.SetAttributes( - attribute.Int64("inodes", resp.Inodes), - attribute.Int64("size", resp.Size), - ) - } return resp, err } func (s *TracingSnapshotter) Cleanup(ctx context.Context, cr *snapshotsapi.CleanupRequest) (*ptypes.Empty, error) { - ctx, span := otel.GetTracerProvider().Tracer(tracerName).Start(ctx, "snapshotter.Cleanup") + ctx, span := GetDefaultTracer().Start(ctx, "snapshotter.Cleanup") defer span.End() resp, err := s.server.Cleanup(ctx, cr) diff --git a/pkg/tracing/tracing.go b/pkg/tracing/tracing.go index 7f2813f0..294999f8 100644 --- a/pkg/tracing/tracing.go +++ b/pkg/tracing/tracing.go @@ -13,8 +13,11 @@ import ( "go.opentelemetry.io/otel/sdk/resource" sdktrace "go.opentelemetry.io/otel/sdk/trace" semconv "go.opentelemetry.io/otel/semconv/v1.24.0" + "go.opentelemetry.io/otel/trace" ) +const tracerName = "accelerated-container-image/snapshotter" + var ( serviceName = os.Getenv("OTEL_SERVICE_NAME") ) @@ -62,3 +65,7 @@ func InitTracer(ctx context.Context) (func(context.Context) error, error) { // Return shutdown function return tp.Shutdown, nil } + +func GetDefaultTracer() trace.Tracer { + return otel.GetTracerProvider().Tracer(tracerName) +} diff --git a/pkg/types/types.go b/pkg/types/types.go index 52cadb19..cd5f75cd 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -24,6 +24,7 @@ type OverlayBDBSConfig struct { ResultFile string `json:"resultFile"` AccelerationLayer bool `json:"accelerationLayer,omitempty"` RecordTracePath string `json:"recordTracePath,omitempty"` + TraceContext map[string]string `json:"traceContext,omitempty"` } // OverlayBDBSConfigLower diff --git a/pkg/utils/cmd.go b/pkg/utils/cmd.go index a08965e0..2e864850 100644 --- a/pkg/utils/cmd.go +++ b/pkg/utils/cmd.go @@ -26,9 +26,12 @@ import ( "path/filepath" "strings" + "github.com/containerd/accelerated-container-image/pkg/tracing" sn "github.com/containerd/accelerated-container-image/pkg/types" "github.com/containerd/log" "github.com/pkg/errors" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) const ( @@ -83,6 +86,12 @@ var defaultServiceTemplate = ` ` func Create(ctx context.Context, dir string, opts ...string) error { + ctx, span := tracing.GetDefaultTracer().Start(ctx, "util.cmd.Create", trace.WithAttributes( + attribute.String("dir", dir), + attribute.StringSlice("opts", opts), + )) + defer span.End() + dataPath := path.Join(dir, dataFile) indexPath := path.Join(dir, idxFile) os.RemoveAll(dataPath) @@ -91,12 +100,19 @@ func Create(ctx context.Context, dir string, opts ...string) error { log.G(ctx).Debugf("%s %s", obdBinCreate, strings.Join(args, " ")) out, err := exec.CommandContext(ctx, obdBinCreate, args...).CombinedOutput() if err != nil { + span.RecordError(err) return errors.Wrapf(err, "failed to overlaybd-create: %s", out) } return nil } func Seal(ctx context.Context, dir, toDir string, opts ...string) error { + ctx, span := tracing.GetDefaultTracer().Start(ctx, "util.cmd.Seal", trace.WithAttributes( + attribute.String("dir", dir), + attribute.StringSlice("opts", opts), + )) + defer span.End() + args := append([]string{ "--seal", path.Join(dir, dataFile), @@ -105,9 +121,11 @@ func Seal(ctx context.Context, dir, toDir string, opts ...string) error { log.G(ctx).Debugf("%s %s", obdBinCommit, strings.Join(args, " ")) out, err := exec.CommandContext(ctx, obdBinCommit, args...).CombinedOutput() if err != nil { + span.RecordError(err) return errors.Wrapf(err, "failed to seal writable overlaybd: %s", out) } if err := os.Rename(path.Join(dir, dataFile), path.Join(toDir, sealedFile)); err != nil { + span.RecordError(err) return errors.Wrapf(err, "failed to rename sealed overlaybd file") } os.RemoveAll(path.Join(dir, idxFile)) @@ -115,6 +133,14 @@ func Seal(ctx context.Context, dir, toDir string, opts ...string) error { } func Commit(ctx context.Context, dir, toDir string, sealed bool, opts ...string) error { + ctx, span := tracing.GetDefaultTracer().Start(ctx, "util.cmd.Commit", trace.WithAttributes( + attribute.String("dir", dir), + attribute.String("toDir", toDir), + attribute.Bool("sealed", sealed), + attribute.StringSlice("opts", opts), + )) + defer span.End() + var args []string if sealed { args = append([]string{ @@ -132,15 +158,25 @@ func Commit(ctx context.Context, dir, toDir string, sealed bool, opts ...string) log.G(ctx).Debugf("%s %s", obdBinCommit, strings.Join(args, " ")) out, err := exec.CommandContext(ctx, obdBinCommit, args...).CombinedOutput() if err != nil { + span.RecordError(err) return errors.Wrapf(err, "failed to overlaybd-commit: %s", out) } if sealed { - return os.Rename(path.Join(toDir, commitTempFile), path.Join(toDir, commitFile)) + err := os.Rename(path.Join(toDir, commitTempFile), path.Join(toDir, commitFile)) + if err != nil { + span.RecordError(err) + } + return err } return nil } func ApplyOverlaybd(ctx context.Context, dir string, opts ...string) error { + ctx, span := tracing.GetDefaultTracer().Start(ctx, "util.cmd.ApplyOverlaybd", trace.WithAttributes( + attribute.String("dir", dir), + attribute.StringSlice("opts", opts), + )) + defer span.End() args := append([]string{ path.Join(dir, "layer.tar"), @@ -148,12 +184,18 @@ func ApplyOverlaybd(ctx context.Context, dir string, opts ...string) error { log.G(ctx).Debugf("%s %s", obdBinApply, strings.Join(args, " ")) out, err := exec.CommandContext(ctx, obdBinApply, args...).CombinedOutput() if err != nil { + span.RecordError(err) return errors.Wrapf(err, "failed to overlaybd-apply[native]: %s", out) } return nil } func ApplyTurboOCI(ctx context.Context, dir, gzipMetaFile string, opts ...string) error { + ctx, span := tracing.GetDefaultTracer().Start(ctx, "util.cmd.ApplyTurboOCI", trace.WithAttributes( + attribute.String("dir", dir), + attribute.StringSlice("opts", opts), + )) + defer span.End() args := append([]string{ path.Join(dir, "layer.tar"), @@ -162,20 +204,28 @@ func ApplyTurboOCI(ctx context.Context, dir, gzipMetaFile string, opts ...string log.G(ctx).Debugf("%s %s", obdBinApply, strings.Join(args, " ")) out, err := exec.CommandContext(ctx, obdBinTurboOCIApply, args...).CombinedOutput() if err != nil { + span.RecordError(err) return errors.Wrapf(err, "failed to overlaybd-apply[turboOCI]: %s", out) } return nil } func GenerateTarMeta(ctx context.Context, srcTarFile string, dstTarMeta string) error { + ctx, span := tracing.GetDefaultTracer().Start(ctx, "util.cmd.Create", trace.WithAttributes( + attribute.String("srcTarFile", srcTarFile), + attribute.String("dstTarMeta", dstTarMeta), + )) + defer span.End() if _, err := os.Stat(srcTarFile); os.IsNotExist(err) { return nil } else if err != nil { + span.RecordError(err) return fmt.Errorf("error stating tar file: %w", err) } log.G(ctx).Infof("generate layer meta for %s", srcTarFile) if err := exec.Command(obdBinTurboOCIApply, srcTarFile, dstTarMeta, "--export").Run(); err != nil { + span.RecordError(err) return fmt.Errorf("failed to convert tar file to overlaybd device: %w", err) } return nil @@ -183,11 +233,18 @@ func GenerateTarMeta(ctx context.Context, srcTarFile string, dstTarMeta string) // ConvertLayer produce a turbooci layer, target is path of ext4.fs.meta func ConvertLayer(ctx context.Context, opt *ConvertOption, fs_type string) error { + ctx, span := tracing.GetDefaultTracer().Start(ctx, "util.cmd.Create", trace.WithAttributes( + attribute.String("opt.Workdir", opt.Workdir), + attribute.String("fs_type", fs_type), + )) + defer span.End() + if opt.Workdir == "" { opt.Workdir = "tmp_conv" } if err := os.MkdirAll(opt.Workdir, 0755); err != nil { + span.RecordError(err) return fmt.Errorf("failed to create work dir: %w", err) } @@ -203,10 +260,12 @@ func ConvertLayer(ctx context.Context, opt *ConvertOption, fs_type string) error args = append(args, "--mkfs") } if out, err := exec.CommandContext(ctx, obdBinCreate, args...).CombinedOutput(); err != nil { + span.RecordError(err) return fmt.Errorf("failed to overlaybd-create: %w, output: %s", err, out) } file, err := os.Create(pathFakeTarget) if err != nil { + span.RecordError(err) return fmt.Errorf("failed to create fake target: %w", err) } file.Close() @@ -220,13 +279,16 @@ func ConvertLayer(ctx context.Context, opt *ConvertOption, fs_type string) error if err := os.WriteFile(pathService, []byte(fmt.Sprintf(defaultServiceTemplate, filepath.Join(opt.Workdir, "cache"))), 0644, ); err != nil { + span.RecordError(err) return fmt.Errorf("failed to write service.json: %w", err) } configBytes, err := json.Marshal(opt.Config) if err != nil { + span.RecordError(err) return fmt.Errorf("failed to marshal overlaybd config: %w", err) } if err := os.WriteFile(pathConfig, configBytes, 0644); err != nil { + span.RecordError(err) return fmt.Errorf("failed to write overlaybd config: %w", err) } args = []string{ @@ -242,6 +304,7 @@ func ConvertLayer(ctx context.Context, opt *ConvertOption, fs_type string) error if out, err := exec.CommandContext(ctx, obdBinTurboOCIApply, args..., ).CombinedOutput(); err != nil { + span.RecordError(err) return fmt.Errorf("failed to turboOCI-apply: %w, output: %s", err, out) } @@ -252,6 +315,7 @@ func ConvertLayer(ctx context.Context, opt *ConvertOption, fs_type string) error opt.Ext4FSMetaPath, "-z", "--turboOCI", ).CombinedOutput(); err != nil { + span.RecordError(err) return fmt.Errorf("failed to overlaybd-commit: %w, output: %s", err, out) } return nil