chore(metrics): dual-export OCR2 transmit metrics via OpenTelemetry

Record confirmed/reverted/fatal counters and unconfirmed gauge via
beholder alongside Prometheus; pass context into RecordOutcome (DF-21989).

Author: cl-efornaciari

pkg/ocr2transmit/metrics.go

@@ -1,11 +1,17 @@
 package ocr2transmit
 
 import (
+	"context"
 	"math/big"
+	"sync"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/metric"
+
+	"github.com/smartcontractkit/chainlink-common/pkg/beholder"
 )
 
 var (
@@ -27,8 +33,64 @@ var (
 	}, []string{"chain_id", "from_address"})
 )
 
+type ocr2TransmitOtelInstruments struct {
+	confirmed   metric.Int64Counter
+	reverted    metric.Int64Counter
+	fatal       metric.Int64Counter
+	unconfirmed metric.Int64Gauge
+}
+
+var (
+	ocr2TransmitOtelInst *ocr2TransmitOtelInstruments
+	ocr2TransmitOtelOnce sync.Once
+)
+
+func loadOCR2TransmitOtel() *ocr2TransmitOtelInstruments {
+	ocr2TransmitOtelOnce.Do(func() {
+		m := beholder.GetMeter()
+		confirmed, err := m.Int64Counter("ocr2_transmit_tx_confirmed_total")
+		if err != nil {
+			return
+		}
+		reverted, err := m.Int64Counter("ocr2_transmit_tx_reverted_total")
+		if err != nil {
+			return
+		}
+		fatal, err := m.Int64Counter("ocr2_transmit_tx_fatal_total")
+		if err != nil {
+			return
+		}
+		unconfirmed, err := m.Int64Gauge("ocr2_transmit_unconfirmed_tx_count")
+		if err != nil {
+			return
+		}
+		ocr2TransmitOtelInst = &ocr2TransmitOtelInstruments{
+			confirmed:   confirmed,
+			reverted:    reverted,
+			fatal:       fatal,
+			unconfirmed: unconfirmed,
+		}
+	})
+	return ocr2TransmitOtelInst
+}
+
+func transmitAttrs(chainID *big.Int, contract, from string) metric.MeasurementOption {
+	return metric.WithAttributes(
+		attribute.String("chain_id", chainID.String()),
+		attribute.String("contract_address", contract),
+		attribute.String("from_address", from),
+	)
+}
+
+func unconfirmedAttrs(chainID *big.Int, from string) metric.MeasurementOption {
+	return metric.WithAttributes(
+		attribute.String("chain_id", chainID.String()),
+		attribute.String("from_address", from),
+	)
+}
+
 // RecordOutcome increments confirmed / reverted / fatal counters when calldata matches OCR2 transmit.
-func RecordOutcome(chainID *big.Int, from, to common.Address, encodedPayload []byte, fwdrDest *common.Address, outcome string) {
+func RecordOutcome(ctx context.Context, chainID *big.Int, from, to common.Address, encodedPayload []byte, fwdrDest *common.Address, outcome string) {
 	if chainID == nil || !IsTransmitCalldata(encodedPayload) {
 		return
 	}
@@ -41,13 +103,29 @@ func RecordOutcome(chainID *big.Int, from, to common.Address, encodedPayload []b
 		promTransmitReverted.WithLabelValues(labels...).Inc()
 	case "fatal":
 		promTransmitFatal.WithLabelValues(labels...).Inc()
+	default:
+		return
+	}
+	if ot := loadOCR2TransmitOtel(); ot != nil {
+		opts := transmitAttrs(chainID, contract, from.Hex())
+		switch outcome {
+		case "confirmed":
+			ot.confirmed.Add(ctx, 1, opts)
+		case "reverted":
+			ot.reverted.Add(ctx, 1, opts)
+		case "fatal":
+			ot.fatal.Add(ctx, 1, opts)
+		}
 	}
 }
 
 // SetUnconfirmedGauge sets the gauge for OCR2-shaped unconfirmed txs for TXM v2 (optional / phase 3).
-func SetUnconfirmedGauge(chainID *big.Int, from common.Address, n int) {
+func SetUnconfirmedGauge(ctx context.Context, chainID *big.Int, from common.Address, n int) {
 	if chainID == nil {
 		return
 	}
 	promTransmitUnconfirmed.WithLabelValues(chainID.String(), from.Hex()).Set(float64(n))
+	if ot := loadOCR2TransmitOtel(); ot != nil {
+		ot.unconfirmed.Record(ctx, int64(n), unconfirmedAttrs(chainID, from.Hex()))
+	}
 }

pkg/txm/txm.go

@@ -448,7 +448,7 @@ func (t *Txm) extractMetrics(ctx context.Context, txs []*types.Transaction) []ui
 		if meta, err := tx.GetMeta(); err == nil && meta != nil && meta.FwdrDestAddress != nil {
 			fwdr = meta.FwdrDestAddress
 		}
-		ocr2transmit.RecordOutcome(t.chainID, tx.FromAddress, tx.ToAddress, tx.Data, fwdr, "confirmed")
+		ocr2transmit.RecordOutcome(ctx, t.chainID, tx.FromAddress, tx.ToAddress, tx.Data, fwdr, "confirmed")
 	}
 	return confirmedTxIDs
 }

pkg/txmgr/finalizer.go

@@ -504,10 +504,10 @@ func (f *evmFinalizer) validateReceipt(ctx context.Context, receipt *types.Recei
 		}
 		// This might increment more than once e.g. in case of re-orgs going back and forth we might re-fetch the same receipt
 		f.metrics.IncrementNumRevertedTxs(ctx)
-		recordOCR2TransmitOutcomeV1(f.chainID, &attempt.Tx, "reverted")
+		recordOCR2TransmitOutcomeV1(ctx, f.chainID, &attempt.Tx, "reverted")
 	} else {
 		f.metrics.IncrementNumSuccessfulTxs(ctx)
-		recordOCR2TransmitOutcomeV1(f.chainID, &attempt.Tx, "confirmed")
+		recordOCR2TransmitOutcomeV1(ctx, f.chainID, &attempt.Tx, "confirmed")
 	}
 
 	// This is only recording forwarded tx that were mined and have a status.
@@ -603,7 +603,7 @@ func (f *evmFinalizer) ProcessOldTxsWithoutReceipts(ctx context.Context, oldTxID
 		if err = f.txStore.UpdateTxFatalErrorAndDeleteAttempts(ctx, oldTx); err != nil {
 			errorList = append(errorList, fmt.Errorf("failed to mark tx with ID %d as fatal: %w", oldTx.ID, err))
 		} else {
-			recordOCR2TransmitOutcomeV1(f.chainID, oldTx, "fatal")
+			recordOCR2TransmitOutcomeV1(ctx, f.chainID, oldTx, "fatal")
 		}
 	}
 	if len(errorList) > 0 {

pkg/txmgr/ocr2_transmit_metrics.go

@@ -1,14 +1,15 @@
 package txmgr
 
 import (
+	"context"
 	"math/big"
 
 	"github.com/ethereum/go-ethereum/common"
 
 	"github.com/smartcontractkit/chainlink-evm/pkg/ocr2transmit"
 )
 
-func recordOCR2TransmitOutcomeV1(chainID *big.Int, tx *Tx, outcome string) {
+func recordOCR2TransmitOutcomeV1(ctx context.Context, chainID *big.Int, tx *Tx, outcome string) {
 	if tx == nil {
 		return
 	}
@@ -20,5 +21,5 @@ func recordOCR2TransmitOutcomeV1(chainID *big.Int, tx *Tx, outcome string) {
 	if cid == nil {
 		cid = tx.ChainID
 	}
-	ocr2transmit.RecordOutcome(cid, tx.FromAddress, tx.ToAddress, tx.EncodedPayload, fwdr, outcome)
+	ocr2transmit.RecordOutcome(ctx, cid, tx.FromAddress, tx.ToAddress, tx.EncodedPayload, fwdr, outcome)
 }