diff --git a/cmd/constants/marketmaps/markets.go b/cmd/constants/marketmaps/markets.go index 9dd5b2f74..b4783c9a4 100644 --- a/cmd/constants/marketmaps/markets.go +++ b/cmd/constants/marketmaps/markets.go @@ -9818,6 +9818,35 @@ var ( } }` + // PythMarketMap is used to initialize the Pyth market map. This only includes + // the markets that are supported by Pyth. + PythMarketMap mmtypes.MarketMap + + // PythMarketMapJSON is the JSON representation of the Pyth MarketMap that can be used + // to initialize for a genesis state or used by the sidecar as a static market map. + PythMarketMapJSON = ` +{ + "markets": { + "WTI/USD": { + "ticker": { + "currency_pair": { + "Base": "WTI", + "Quote": "USD" + }, + "decimals": 8, + "min_provider_count": 1, + "enabled": true + }, + "provider_configs": [ + { + "name": "pyth_api", + "off_chain_ticker": "2694" + } + ] + } + } + }` + // ForexMarketMap is used to initialize the forex market map. This only includes // forex markets quoted in usdt. ForexMarketMap mmtypes.MarketMap @@ -9919,6 +9948,7 @@ func init() { unmarshalValidate("Polymarket", PolymarketMarketMapJSON, &PolymarketMarketMap), unmarshalValidate("Forex", ForexMarketMapJSON, &ForexMarketMap), unmarshalValidate("Stork", StorkMarketMapJSON, &StorkMarketMap), + unmarshalValidate("Pyth", PythMarketMapJSON, &PythMarketMap), ) if err != nil { panic(err) diff --git a/cmd/constants/providers.go b/cmd/constants/providers.go index c209b0e7c..e49f07ab1 100644 --- a/cmd/constants/providers.go +++ b/cmd/constants/providers.go @@ -16,6 +16,7 @@ import ( krakenapi "github.com/dydxprotocol/slinky/providers/apis/kraken" "github.com/dydxprotocol/slinky/providers/apis/marketmap" "github.com/dydxprotocol/slinky/providers/apis/polymarket" + "github.com/dydxprotocol/slinky/providers/apis/pyth" "github.com/dydxprotocol/slinky/providers/apis/stork" "github.com/dydxprotocol/slinky/providers/volatile" binancews "github.com/dydxprotocol/slinky/providers/websockets/binance" @@ -163,6 +164,13 @@ var ( Type: types.ConfigType, }, + // Pyth provider + { + Name: pyth.Name, + API: pyth.DefaultAPIConfig, + Type: types.ConfigType, + }, + // Stork provider { Name: stork.Name, diff --git a/providers/apis/pyth/api_handler.go b/providers/apis/pyth/api_handler.go new file mode 100644 index 000000000..49d0e0566 --- /dev/null +++ b/providers/apis/pyth/api_handler.go @@ -0,0 +1,182 @@ +package pyth + +import ( + "encoding/json" + "fmt" + "io" + "math/big" + "net/http" + "strconv" + "strings" + "time" + + providertypes "github.com/dydxprotocol/slinky/providers/types" + + "github.com/dydxprotocol/slinky/oracle/config" + "github.com/dydxprotocol/slinky/oracle/types" +) + +var _ types.PriceAPIDataHandler = (*APIHandler)(nil) + +// APIHandler implements the PriceAPIDataHandler interface for Pyth. +type APIHandler struct { + api config.APIConfig + cache types.ProviderTickers +} + +// NewAPIHandler returns a new Pyth PriceAPIDataHandler. +func NewAPIHandler( + api config.APIConfig, +) (types.PriceAPIDataHandler, error) { + if api.Name != Name { + return nil, fmt.Errorf("expected api config name %s, got %s", Name, api.Name) + } + + if !api.Enabled { + return nil, fmt.Errorf("api config for %s is not enabled", Name) + } + + if err := api.ValidateBasic(); err != nil { + return nil, fmt.Errorf("invalid api config for %s: %w", Name, err) + } + + return &APIHandler{ + api: api, + cache: types.NewProviderTickers(), + }, nil +} + +// CreateURL returns the URL used to fetch prices from the Pyth oracle service. +// Feed IDs are passed as a comma-separated "asset" query parameter, with +// "&provider=pyth" appended. +func (h *APIHandler) CreateURL( + tickers []types.ProviderTicker, +) (string, error) { + if len(tickers) == 0 { + return "", fmt.Errorf("no tickers provided") + } + + ids := make([]string, len(tickers)) + for i, ticker := range tickers { + ids[i] = ticker.GetOffChainTicker() + h.cache.Add(ticker) + } + + return fmt.Sprintf( + "%s?asset=%s&provider=pyth", + h.api.Endpoints[0].URL, + strings.Join(ids, ","), + ), nil +} + +// ParseResponse parses a batch Pyth API response ({"data": [...]}), verifies +// each entry's Pyth Solana ed25519 signature, and returns the parsed prices. +// +// If the signed payload contains both price mantissa and exponent, the price is +// computed directly from signed data (mantissa * 10^exponent). If the payload +// only contains price (no exponent), the signature and feed ID are still +// verified, but the JSON price field is used as a fallback. +func (h *APIHandler) ParseResponse( + tickers []types.ProviderTicker, + resp *http.Response, +) types.PriceResponse { + body, err := io.ReadAll(resp.Body) + if err != nil { + return types.NewPriceResponseWithErr( + tickers, + providertypes.NewErrorWithCode( + fmt.Errorf("failed to read pyth response body: %w", err), + providertypes.ErrorFailedToDecode, + ), + ) + } + + var batch BatchPriceResponse + if err := json.Unmarshal(body, &batch); err != nil { + return types.NewPriceResponseWithErr( + tickers, + providertypes.NewErrorWithCode( + fmt.Errorf("failed to decode pyth response: %w", err), + providertypes.ErrorFailedToDecode, + ), + ) + } + + byMarket := make(map[string]PriceResponse, len(batch.Data)) + for _, item := range batch.Data { + byMarket[item.Market] = item + } + + var ( + resolved = make(types.ResolvedPrices) + unresolved = make(types.UnResolvedPrices) + ) + + for _, ticker := range tickers { + offChain := ticker.GetOffChainTicker() + + item, ok := byMarket[offChain] + if !ok { + unresolved[ticker] = providertypes.UnresolvedResult{ + ErrorWithCode: providertypes.NewErrorWithCode( + fmt.Errorf("no pyth response for feed %s", offChain), + providertypes.ErrorNoResponse, + ), + } + continue + } + + feedID, err := strconv.ParseUint(offChain, 10, 32) + if err != nil { + unresolved[ticker] = providertypes.UnresolvedResult{ + ErrorWithCode: providertypes.NewErrorWithCode( + fmt.Errorf("invalid feed ID %q: %w", offChain, err), + providertypes.ErrorInvalidResponse, + ), + } + continue + } + + feed, err := VerifyAndExtractFeed(item.PythSolanaPayload, uint32(feedID)) + if err != nil { + unresolved[ticker] = providertypes.UnresolvedResult{ + ErrorWithCode: providertypes.NewErrorWithCode( + fmt.Errorf("pyth payload verification failed for feed %s: %w", offChain, err), + providertypes.ErrorInvalidResponse, + ), + } + continue + } + + var price *big.Float + if feed.HasPrice && feed.HasExponent { + price, err = feed.ComputePrice() + if err != nil { + unresolved[ticker] = providertypes.UnresolvedResult{ + ErrorWithCode: providertypes.NewErrorWithCode( + fmt.Errorf("failed to compute price from signed payload for feed %s: %w", offChain, err), + providertypes.ErrorFailedToParsePrice, + ), + } + continue + } + } else { + // Exponent not in the signed payload; fall back to JSON price. + // Signature and feed ID have already been verified above. + price, ok = new(big.Float).SetString(item.Price) + if !ok { + unresolved[ticker] = providertypes.UnresolvedResult{ + ErrorWithCode: providertypes.NewErrorWithCode( + fmt.Errorf("failed to parse JSON price %q for feed %s", item.Price, offChain), + providertypes.ErrorFailedToParsePrice, + ), + } + continue + } + } + + resolved[ticker] = types.NewPriceResult(price, time.Now().UTC()) + } + + return types.NewPriceResponse(resolved, unresolved) +} diff --git a/providers/apis/pyth/api_handler_test.go b/providers/apis/pyth/api_handler_test.go new file mode 100644 index 000000000..27f4ed651 --- /dev/null +++ b/providers/apis/pyth/api_handler_test.go @@ -0,0 +1,619 @@ +package pyth_test + +import ( + "crypto/ed25519" + "encoding/base64" + "encoding/binary" + "fmt" + "math" + "math/big" + "net/http" + "strings" + "testing" + "time" + + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/require" + + "github.com/dydxprotocol/slinky/oracle/types" + "github.com/dydxprotocol/slinky/providers/apis/pyth" + "github.com/dydxprotocol/slinky/providers/base/testutils" + providertypes "github.com/dydxprotocol/slinky/providers/types" +) + +var ( + feed2694 = types.DefaultProviderTicker{ + OffChainTicker: "2694", + } + feed3032 = types.DefaultProviderTicker{ + OffChainTicker: "3032", + } +) + +// buildInnerPayload constructs a Pyth Lazer inner payload (LE) containing a +// single feed with Price and Exponent properties. +func buildInnerPayload(feedID uint32, priceMantissa int64, exponent int16) []byte { + buf := make([]byte, 31) + off := 0 + + binary.LittleEndian.PutUint32(buf[off:], pyth.PayloadFormatMagic) + off += 4 + binary.LittleEndian.PutUint64(buf[off:], uint64(time.Now().UnixMicro())) //nolint:gosec // test only, always positive + off += 8 + buf[off] = 3 // channel_id: FIXED_RATE_200 + off++ + buf[off] = 1 // num_feeds + off++ + + binary.LittleEndian.PutUint32(buf[off:], feedID) + off += 4 + buf[off] = 2 // num_properties (price + exponent) + off++ + + buf[off] = 0 // propPrice tag + off++ + binary.LittleEndian.PutUint64(buf[off:], uint64(priceMantissa)) //nolint:gosec // reinterpret signed as unsigned bits + off += 8 + + buf[off] = 4 // propExponent tag + off++ + binary.LittleEndian.PutUint16(buf[off:], uint16(exponent)) //nolint:gosec // reinterpret signed as unsigned bits + + return buf +} + +// buildInnerPayloadPriceOnly constructs a Pyth Lazer inner payload with only +// Price and FeedUpdateTimestamp (no Exponent), matching real production payloads. +func buildInnerPayloadPriceOnly(feedID uint32, priceMantissa int64) []byte { + // magic(4) + timestamp(8) + channel(1) + numFeeds(1) + // + feedID(4) + numProps(1) + // + propPrice tag(1) + i64(8) + // + propFeedUpdateTimestamp tag(1) + present(1) + u64(8) + buf := make([]byte, 38) + off := 0 + + binary.LittleEndian.PutUint32(buf[off:], pyth.PayloadFormatMagic) + off += 4 + ts := uint64(time.Now().UnixMicro()) //nolint:gosec // test only, always positive + binary.LittleEndian.PutUint64(buf[off:], ts) + off += 8 + buf[off] = 3 // channel_id + off++ + buf[off] = 1 // num_feeds + off++ + + binary.LittleEndian.PutUint32(buf[off:], feedID) + off += 4 + buf[off] = 2 // num_properties (price + feedUpdateTimestamp) + off++ + + buf[off] = 0 // propPrice tag + off++ + binary.LittleEndian.PutUint64(buf[off:], uint64(priceMantissa)) //nolint:gosec // reinterpret signed as unsigned bits + off += 8 + + buf[off] = 12 // propFeedUpdateTimestamp tag + off++ + buf[off] = 1 // present + off++ + binary.LittleEndian.PutUint64(buf[off:], ts) + + return buf +} + +// buildSolanaPayload constructs a Pyth Lazer Solana envelope: +// +// magic(4) || signature(64) || pubkey(32) || msgLen(2) || msg +func buildSolanaPayload(t *testing.T, privKey ed25519.PrivateKey, msg []byte) string { + t.Helper() + sig := ed25519.Sign(privKey, msg) + + require.LessOrEqual(t, len(msg), math.MaxUint16) + buf := make([]byte, 4+64+32+2+len(msg)) + binary.LittleEndian.PutUint32(buf[0:4], pyth.SolanaFormatMagic) + copy(buf[4:68], sig) + copy(buf[68:100], privKey.Public().(ed25519.PublicKey)) + binary.LittleEndian.PutUint16(buf[100:102], uint16(len(msg))) //nolint:gosec // bounded above + copy(buf[102:], msg) + + return base64.StdEncoding.EncodeToString(buf) +} + +// signedItemJSON returns a PriceResponse JSON entry with a valid signed Pyth +// Solana payload containing the given feed ID, price mantissa, and exponent. +func signedItemJSON( + t *testing.T, + privKey ed25519.PrivateKey, + feedID uint32, + priceMantissa int64, + exponent int16, +) string { + t.Helper() + innerPayload := buildInnerPayload(feedID, priceMantissa, exponent) + solPayload := buildSolanaPayload(t, privKey, innerPayload) + return fmt.Sprintf( + `{"market":"%d","price":"0","timestampMs":1774625883288,"pythSolanaPayload":%q}`, + feedID, solPayload, + ) +} + +// signedItemJSONPriceOnly returns a PriceResponse JSON entry with a signed +// payload that has no Exponent property (only Price + FeedUpdateTimestamp), +// plus a JSON price field for fallback. +func signedItemJSONPriceOnly( + t *testing.T, + privKey ed25519.PrivateKey, + feedID uint32, + priceMantissa int64, + jsonPrice string, +) string { + t.Helper() + innerPayload := buildInnerPayloadPriceOnly(feedID, priceMantissa) + solPayload := buildSolanaPayload(t, privKey, innerPayload) + return fmt.Sprintf( + `{"market":"%d","price":%q,"timestampMs":1774625883288,"pythSolanaPayload":%q}`, + feedID, jsonPrice, solPayload, + ) +} + +// signedBatchJSON builds a full {"data":[...]} response with price+exponent payloads. +func signedBatchJSON(t *testing.T, items ...struct { + feedID uint32 + priceMantissa int64 + exponent int16 +}, +) (string, ed25519.PublicKey) { + t.Helper() + pub, priv, err := ed25519.GenerateKey(nil) + require.NoError(t, err) + t.Setenv(pyth.PythPubKeyEnv, solana.PublicKeyFromBytes(pub).String()) + + parts := make([]string, len(items)) + for i, item := range items { + parts[i] = signedItemJSON(t, priv, item.feedID, item.priceMantissa, item.exponent) + } + return `{"data":[` + strings.Join(parts, ",") + `]}`, pub +} + +// badSigBatchJSON returns a response where the envelope pubkey differs from PYTH_PUB_KEY. +func badSigBatchJSON(t *testing.T) string { + t.Helper() + pub1, _, err := ed25519.GenerateKey(nil) + require.NoError(t, err) + _, priv2, err := ed25519.GenerateKey(nil) + require.NoError(t, err) + t.Setenv(pyth.PythPubKeyEnv, solana.PublicKeyFromBytes(pub1).String()) + + innerPayload := buildInnerPayload(2694, 9726473, -8) + solPayload := buildSolanaPayload(t, priv2, innerPayload) + return fmt.Sprintf( + `{"data":[{"market":"2694","price":"0","timestampMs":1774625883288,"pythSolanaPayload":%q}]}`, + solPayload, + ) +} + +func TestCreateURL(t *testing.T) { + testCases := []struct { + name string + cps []types.ProviderTicker + url string + expectedErr bool + }{ + { + name: "empty", + cps: []types.ProviderTicker{}, + url: "", + expectedErr: true, + }, + { + name: "valid single", + cps: []types.ProviderTicker{feed2694}, + url: fmt.Sprintf("%s?asset=%s&provider=pyth", pyth.URL, "2694"), + }, + { + name: "valid multiple", + cps: []types.ProviderTicker{feed2694, feed3032}, + url: fmt.Sprintf("%s?asset=%s&provider=pyth", pyth.URL, "2694,3032"), + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + h, err := pyth.NewAPIHandler(pyth.DefaultAPIConfig) + require.NoError(t, err) + + url, err := h.CreateURL(tc.cps) + if tc.expectedErr { + require.Error(t, err) + } else { + require.NoError(t, err) + require.Equal(t, tc.url, url) + } + }) + } +} + +func TestParseResponse(t *testing.T) { + type item struct { + feedID uint32 + priceMantissa int64 + exponent int16 + } + + testCases := []struct { + name string + cps []types.ProviderTicker + response func(t *testing.T) *http.Response + expected types.PriceResponse + }{ + { + name: "valid single feed - mantissa 9726473 exp -8 = 0.09726473", + cps: []types.ProviderTicker{feed2694}, + response: func(t *testing.T) *http.Response { + t.Helper() + body, _ := signedBatchJSON(t, item{2694, 9726473, -8}) + return testutils.CreateResponseFromJSON(body) + }, + expected: types.NewPriceResponse( + types.ResolvedPrices{ + feed2694: {Value: big.NewFloat(0.09726473)}, + }, + types.UnResolvedPrices{}, + ), + }, + { + name: "multiple feeds resolved", + cps: []types.ProviderTicker{feed2694, feed3032}, + response: func(t *testing.T) *http.Response { + t.Helper() + body, _ := signedBatchJSON(t, + item{2694, 9726473, -8}, + item{3032, 345678000000, -8}, + ) + return testutils.CreateResponseFromJSON(body) + }, + expected: types.NewPriceResponse( + types.ResolvedPrices{ + feed2694: {Value: big.NewFloat(0.09726473)}, + feed3032: {Value: big.NewFloat(3456.78)}, + }, + types.UnResolvedPrices{}, + ), + }, + { + name: "no exponent in payload - falls back to JSON price", + cps: []types.ProviderTicker{feed2694}, + response: func(t *testing.T) *http.Response { + t.Helper() + pub, priv, err := ed25519.GenerateKey(nil) + require.NoError(t, err) + t.Setenv(pyth.PythPubKeyEnv, solana.PublicKeyFromBytes(pub).String()) + + jsonBody := `{"data":[` + + signedItemJSONPriceOnly(t, priv, 2694, 10438000, "0.10438000") + + `]}` + return testutils.CreateResponseFromJSON(jsonBody) + }, + expected: types.NewPriceResponse( + types.ResolvedPrices{ + feed2694: {Value: big.NewFloat(0.10438)}, + }, + types.UnResolvedPrices{}, + ), + }, + { + name: "bad json response", + cps: []types.ProviderTicker{feed2694}, + response: func(_ *testing.T) *http.Response { + return testutils.CreateResponseFromJSON(`not valid json`) + }, + expected: types.NewPriceResponse( + types.ResolvedPrices{}, + types.UnResolvedPrices{ + feed2694: providertypes.UnresolvedResult{ + ErrorWithCode: providertypes.NewErrorWithCode( + fmt.Errorf("decode error"), providertypes.ErrorAPIGeneral, + ), + }, + }, + ), + }, + { + name: "signature verification fails - wrong pubkey", + cps: []types.ProviderTicker{feed2694}, + response: func(t *testing.T) *http.Response { + t.Helper() + return testutils.CreateResponseFromJSON(badSigBatchJSON(t)) + }, + expected: types.NewPriceResponse( + types.ResolvedPrices{}, + types.UnResolvedPrices{ + feed2694: providertypes.UnresolvedResult{ + ErrorWithCode: providertypes.NewErrorWithCode( + fmt.Errorf("sig mismatch"), providertypes.ErrorAPIGeneral, + ), + }, + }, + ), + }, + { + name: "feed not in batch response", + cps: []types.ProviderTicker{feed2694, feed3032}, + response: func(t *testing.T) *http.Response { + t.Helper() + body, _ := signedBatchJSON(t, item{2694, 9726473, -8}) + return testutils.CreateResponseFromJSON(body) + }, + expected: types.NewPriceResponse( + types.ResolvedPrices{ + feed2694: {Value: big.NewFloat(0.09726473)}, + }, + types.UnResolvedPrices{ + feed3032: providertypes.UnresolvedResult{ + ErrorWithCode: providertypes.NewErrorWithCode( + fmt.Errorf("no response"), providertypes.ErrorAPIGeneral, + ), + }, + }, + ), + }, + { + name: "empty batch data", + cps: []types.ProviderTicker{feed2694}, + response: func(_ *testing.T) *http.Response { + return testutils.CreateResponseFromJSON(`{"data":[]}`) + }, + expected: types.NewPriceResponse( + types.ResolvedPrices{}, + types.UnResolvedPrices{ + feed2694: providertypes.UnresolvedResult{ + ErrorWithCode: providertypes.NewErrorWithCode( + fmt.Errorf("no response"), providertypes.ErrorAPIGeneral, + ), + }, + }, + ), + }, + { + name: "zero price mantissa with exponent produces unresolved", + cps: []types.ProviderTicker{feed2694}, + response: func(t *testing.T) *http.Response { + t.Helper() + body, _ := signedBatchJSON(t, item{2694, 0, -8}) + return testutils.CreateResponseFromJSON(body) + }, + expected: types.NewPriceResponse( + types.ResolvedPrices{}, + types.UnResolvedPrices{ + feed2694: providertypes.UnresolvedResult{ + ErrorWithCode: providertypes.NewErrorWithCode( + fmt.Errorf("zero/absent"), providertypes.ErrorAPIGeneral, + ), + }, + }, + ), + }, + { + name: "positive exponent - mantissa 5 exp 2 = 500", + cps: []types.ProviderTicker{feed2694}, + response: func(t *testing.T) *http.Response { + t.Helper() + body, _ := signedBatchJSON(t, item{2694, 5, 2}) + return testutils.CreateResponseFromJSON(body) + }, + expected: types.NewPriceResponse( + types.ResolvedPrices{ + feed2694: {Value: big.NewFloat(500)}, + }, + types.UnResolvedPrices{}, + ), + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + h, err := pyth.NewAPIHandler(pyth.DefaultAPIConfig) + require.NoError(t, err) + + _, err = h.CreateURL(tc.cps) + require.NoError(t, err) + + now := time.Now() + resp := h.ParseResponse(tc.cps, tc.response(t)) + + require.Len(t, resp.Resolved, len(tc.expected.Resolved)) + require.Len(t, resp.UnResolved, len(tc.expected.UnResolved)) + + for cp, result := range tc.expected.Resolved { + require.Contains(t, resp.Resolved, cp) + r := resp.Resolved[cp] + require.Equal(t, result.Value.SetPrec(18), r.Value.SetPrec(18)) + require.True(t, r.Timestamp.After(now)) + } + + for cp := range tc.expected.UnResolved { + require.Contains(t, resp.UnResolved, cp) + require.Error(t, resp.UnResolved[cp]) + } + }) + } +} + +func TestVerifyAndExtractFeed(t *testing.T) { + t.Run("valid single feed with exponent", func(t *testing.T) { + pub, priv, err := ed25519.GenerateKey(nil) + require.NoError(t, err) + t.Setenv(pyth.PythPubKeyEnv, solana.PublicKeyFromBytes(pub).String()) + + inner := buildInnerPayload(2694, 9726473, -8) + payload := buildSolanaPayload(t, priv, inner) + + feed, err := pyth.VerifyAndExtractFeed(payload, 2694) + require.NoError(t, err) + require.True(t, feed.HasPrice) + require.True(t, feed.HasExponent) + require.Equal(t, int64(9726473), feed.PriceMantissa) + require.Equal(t, int16(-8), feed.Exponent) + + price, err := feed.ComputePrice() + require.NoError(t, err) + expected := big.NewFloat(0.09726473) + require.Equal(t, expected.SetPrec(18), price.SetPrec(18)) + }) + + t.Run("payload without exponent (price + feedUpdateTimestamp only)", func(t *testing.T) { + pub, priv, err := ed25519.GenerateKey(nil) + require.NoError(t, err) + t.Setenv(pyth.PythPubKeyEnv, solana.PublicKeyFromBytes(pub).String()) + + inner := buildInnerPayloadPriceOnly(2694, 10438000) + payload := buildSolanaPayload(t, priv, inner) + + feed, err := pyth.VerifyAndExtractFeed(payload, 2694) + require.NoError(t, err) + require.True(t, feed.HasPrice) + require.False(t, feed.HasExponent) + require.True(t, feed.HasTimestamp) + require.Equal(t, int64(10438000), feed.PriceMantissa) + require.Equal(t, uint32(2694), feed.FeedID) + }) + + t.Run("real production payload", func(t *testing.T) { + // Actual payload from production Pyth Lazer for feed 2694 (WTI/USD). + // Pubkey (base58): AbGSbqM2M5FeyPNqwiMkGBCqPb63HhR7RK9dBjxJ4mF1 + prodPayload := "uQEagpolDOQCDv+Fd6/CBw7HBUGPxzSY+41jnaBoGCA+eX+fqgdMmWAVMOBTmO23hWBiV0H7xIPReVnaAE9lVt4B0AeA78H0gMVhWvP7Zz1CKH6ZPan7w1BrbkHfoylQggwubCYAddPHk4AM8BtUTgYAAwGGCgAAAgBwRZ8AAAAAAAwBgAzwG1ROBgA=" + + // The pubkey embedded in the envelope (hex): + // 80efc1f480c5615af3fb673d42287e993da9fbc3506b6e41dfa32950820c2e6c + pubKeyBytes, err := base64.StdEncoding.DecodeString(prodPayload) + require.NoError(t, err) + embeddedPubKey := pubKeyBytes[68:100] + t.Setenv(pyth.PythPubKeyEnv, solana.PublicKeyFromBytes(embeddedPubKey).String()) + + feed, err := pyth.VerifyAndExtractFeed(prodPayload, 2694) + require.NoError(t, err) + require.Equal(t, uint32(2694), feed.FeedID) + require.True(t, feed.HasPrice) + require.Equal(t, int64(10438000), feed.PriceMantissa) + require.False(t, feed.HasExponent) + require.True(t, feed.HasTimestamp) + require.Equal(t, uint64(1774973013200000), feed.TimestampUs) + }) + + t.Run("feed not found in payload", func(t *testing.T) { + pub, priv, err := ed25519.GenerateKey(nil) + require.NoError(t, err) + t.Setenv(pyth.PythPubKeyEnv, solana.PublicKeyFromBytes(pub).String()) + + inner := buildInnerPayload(2694, 9726473, -8) + payload := buildSolanaPayload(t, priv, inner) + + _, err = pyth.VerifyAndExtractFeed(payload, 9999) + require.Error(t, err) + require.Contains(t, err.Error(), "feed 9999 not found") + }) + + t.Run("wrong public key", func(t *testing.T) { + pub1, _, err := ed25519.GenerateKey(nil) + require.NoError(t, err) + _, priv2, err := ed25519.GenerateKey(nil) + require.NoError(t, err) + t.Setenv(pyth.PythPubKeyEnv, solana.PublicKeyFromBytes(pub1).String()) + + inner := buildInnerPayload(2694, 9726473, -8) + payload := buildSolanaPayload(t, priv2, inner) + + _, err = pyth.VerifyAndExtractFeed(payload, 2694) + require.Error(t, err) + require.Contains(t, err.Error(), "public key mismatch") + }) + + t.Run("tampered payload", func(t *testing.T) { + pub, priv, err := ed25519.GenerateKey(nil) + require.NoError(t, err) + t.Setenv(pyth.PythPubKeyEnv, solana.PublicKeyFromBytes(pub).String()) + + original := buildInnerPayload(2694, 9726473, -8) + sig := ed25519.Sign(priv, original) + + tampered := buildInnerPayload(2694, 99999999, -8) + require.LessOrEqual(t, len(tampered), math.MaxUint16) + buf := make([]byte, 4+64+32+2+len(tampered)) + binary.LittleEndian.PutUint32(buf[0:4], pyth.SolanaFormatMagic) + copy(buf[4:68], sig) + copy(buf[68:100], pub) + binary.LittleEndian.PutUint16(buf[100:102], uint16(len(tampered))) //nolint:gosec // bounded above + copy(buf[102:], tampered) + + payload := base64.StdEncoding.EncodeToString(buf) + _, err = pyth.VerifyAndExtractFeed(payload, 2694) + require.Error(t, err) + require.Contains(t, err.Error(), "ed25519 signature verification failed") + }) + + t.Run("env var not set", func(t *testing.T) { + t.Setenv(pyth.PythPubKeyEnv, "") + _, err := pyth.VerifyAndExtractFeed("dGVzdA==", 2694) + require.Error(t, err) + require.Contains(t, err.Error(), "PYTH_PUB_KEY") + }) + + t.Run("invalid base64", func(t *testing.T) { + t.Setenv(pyth.PythPubKeyEnv, "11111111111111111111111111111111") + _, err := pyth.VerifyAndExtractFeed("!!!not-base64!!!", 2694) + require.Error(t, err) + require.Contains(t, err.Error(), "base64") + }) + + t.Run("payload too short", func(t *testing.T) { + t.Setenv(pyth.PythPubKeyEnv, "11111111111111111111111111111111") + short := base64.StdEncoding.EncodeToString([]byte("tooshort")) + _, err := pyth.VerifyAndExtractFeed(short, 2694) + require.Error(t, err) + require.Contains(t, err.Error(), "too short") + }) + + t.Run("wrong envelope magic", func(t *testing.T) { + pub, _, err := ed25519.GenerateKey(nil) + require.NoError(t, err) + t.Setenv(pyth.PythPubKeyEnv, solana.PublicKeyFromBytes(pub).String()) + + buf := make([]byte, 102) + binary.LittleEndian.PutUint32(buf[0:4], 0xDEADBEEF) + payload := base64.StdEncoding.EncodeToString(buf) + _, err = pyth.VerifyAndExtractFeed(payload, 2694) + require.Error(t, err) + require.Contains(t, err.Error(), "invalid envelope magic") + }) + + t.Run("negative price mantissa", func(t *testing.T) { + pub, priv, err := ed25519.GenerateKey(nil) + require.NoError(t, err) + t.Setenv(pyth.PythPubKeyEnv, solana.PublicKeyFromBytes(pub).String()) + + inner := buildInnerPayload(2694, -5000000, -5) + payload := buildSolanaPayload(t, priv, inner) + + feed, err := pyth.VerifyAndExtractFeed(payload, 2694) + require.NoError(t, err) + price, err := feed.ComputePrice() + require.NoError(t, err) + expected := big.NewFloat(-50.0) + require.Equal(t, expected.SetPrec(18), price.SetPrec(18)) + }) + + t.Run("zero price mantissa", func(t *testing.T) { + pub, priv, err := ed25519.GenerateKey(nil) + require.NoError(t, err) + t.Setenv(pyth.PythPubKeyEnv, solana.PublicKeyFromBytes(pub).String()) + + inner := buildInnerPayload(2694, 0, -8) + payload := buildSolanaPayload(t, priv, inner) + + feed, err := pyth.VerifyAndExtractFeed(payload, 2694) + require.NoError(t, err) + _, err = feed.ComputePrice() + require.Error(t, err) + require.Contains(t, err.Error(), "zero/absent") + }) +} diff --git a/providers/apis/pyth/utils.go b/providers/apis/pyth/utils.go new file mode 100644 index 000000000..8af290fe4 --- /dev/null +++ b/providers/apis/pyth/utils.go @@ -0,0 +1,316 @@ +package pyth + +import ( + "bytes" + "crypto/ed25519" + "encoding/base64" + "encoding/binary" + "fmt" + "math/big" + "os" + "time" + + "github.com/gagliardetto/solana-go" + + "github.com/dydxprotocol/slinky/oracle/config" +) + +const ( + PythPubKeyEnv = "PYTH_PUB_KEY" +) + +const ( + Name = "pyth_api" + + URL = "http://localhost:8444/prices" + + // SolanaFormatMagic is the Pyth Lazer Solana envelope magic (LE u32). + SolanaFormatMagic uint32 = 0x821a01b9 + + // solanaEnvelopeMinLen is magic(4) + sig(64) + pubkey(32) + msgLen(2). + solanaEnvelopeMinLen = 102 + + // PayloadFormatMagic is the first 4 bytes (LE) of the inner signed payload. + // Decimal 2479346549. + PayloadFormatMagic uint32 = 0x93C7D375 +) + +// PriceFeedProperty discriminant values from the Pyth Lazer protocol. +const ( + propPrice uint8 = 0 + propBestBidPrice uint8 = 1 + propBestAskPrice uint8 = 2 + propPublisherCount uint8 = 3 + propExponent uint8 = 4 + propConfidence uint8 = 5 + propFundingRate uint8 = 6 + propFundingTimestamp uint8 = 7 + propFundingRateInterval uint8 = 8 + propMarketSession uint8 = 9 + propEmaPrice uint8 = 10 + propEmaConfidence uint8 = 11 + propFeedUpdateTimestamp uint8 = 12 +) + +var DefaultAPIConfig = config.APIConfig{ + Name: Name, + Atomic: false, + Enabled: true, + Timeout: 3000 * time.Millisecond, + Interval: 3000 * time.Millisecond, + ReconnectTimeout: 2000 * time.Millisecond, + MaxQueries: 1, + Endpoints: []config.Endpoint{{URL: URL}}, +} + +// BatchPriceResponse is the top-level response containing prices for multiple feeds. +type BatchPriceResponse struct { + Data []PriceResponse `json:"data"` +} + +// PriceResponse is a single feed entry within the batch response. +type PriceResponse struct { + Market string `json:"market"` + Price string `json:"price"` + TimestampMs int64 `json:"timestampMs"` + PythSolanaPayload string `json:"pythSolanaPayload"` +} + +// ParsedFeedPrice holds price data extracted from a signed Pyth Lazer payload. +type ParsedFeedPrice struct { + FeedID uint32 + PriceMantissa int64 + HasPrice bool + Exponent int16 + HasExponent bool + TimestampUs uint64 + HasTimestamp bool +} + +// ComputePrice returns mantissa * 10^exponent as a *big.Float. +func (p *ParsedFeedPrice) ComputePrice() (*big.Float, error) { + if !p.HasPrice { + return nil, fmt.Errorf("no price property in signed payload for feed %d", p.FeedID) + } + if !p.HasExponent { + return nil, fmt.Errorf("no exponent property in signed payload for feed %d", p.FeedID) + } + if p.PriceMantissa == 0 { + return nil, fmt.Errorf("price is zero/absent in signed payload for feed %d", p.FeedID) + } + + mantissa := new(big.Float).SetInt64(p.PriceMantissa) + exp := int(p.Exponent) + factor := new(big.Float).SetInt(new(big.Int).Exp(big.NewInt(10), big.NewInt(int64(abs(exp))), nil)) + + if exp >= 0 { + return mantissa.Mul(mantissa, factor), nil + } + return mantissa.Quo(mantissa, factor), nil +} + +func abs(x int) int { + if x < 0 { + return -x + } + return x +} + +// VerifyAndExtractFeed decodes a base64 Pyth Lazer Solana payload, verifies +// the ed25519 signature against the trusted PYTH_PUB_KEY, parses the signed +// message, and returns the ParsedFeedPrice for the requested feedID. +// +// The caller should use ComputePrice() if both price and exponent are present. +// If the subscription doesn't include exponent, the caller may fall back to +// the JSON price while still benefiting from signature + feed ID verification. +// +// Solana envelope layout: +// +// [0..4) magic – LE u32, must be 0x821a01b9 +// [4..68) signature – 64-byte ed25519 signature +// [68..100) pubkey – 32-byte ed25519 public key +// [100..102) msgLen – LE u16 +// [102..102+msgLen) msg – signed payload bytes +// +// Inner payload layout (LE): +// +// [0..4) PAYLOAD_FORMAT_MAGIC (LE u32 = 0x93C7D375) +// [4..12) timestamp_us (LE u64) +// [12] channel_id (u8) +// [13] num_feeds (u8) +// Per feed: +// feed_id (LE u32), num_properties (u8), +// then tag(u8)+value pairs per property. +func VerifyAndExtractFeed(payloadBase64 string, feedID uint32) (*ParsedFeedPrice, error) { + expectedKeyStr := os.Getenv(PythPubKeyEnv) + if expectedKeyStr == "" { + return nil, fmt.Errorf("%s environment variable is not set", PythPubKeyEnv) + } + + expectedKey, err := solana.PublicKeyFromBase58(expectedKeyStr) + if err != nil { + return nil, fmt.Errorf("invalid %s value: %w", PythPubKeyEnv, err) + } + + data, err := base64.StdEncoding.DecodeString(payloadBase64) + if err != nil { + return nil, fmt.Errorf("failed to decode base64 payload: %w", err) + } + + if len(data) < solanaEnvelopeMinLen { + return nil, fmt.Errorf("payload too short: %d bytes, minimum %d", len(data), solanaEnvelopeMinLen) + } + + magic := binary.LittleEndian.Uint32(data[0:4]) + if magic != SolanaFormatMagic { + return nil, fmt.Errorf("invalid envelope magic: got 0x%08x, want 0x%08x", magic, SolanaFormatMagic) + } + + sig := data[4:68] + pubKey := data[68:100] + msgSize := int(binary.LittleEndian.Uint16(data[100:102])) + + if len(data) < solanaEnvelopeMinLen+msgSize { + return nil, fmt.Errorf("payload truncated: need %d bytes, have %d", solanaEnvelopeMinLen+msgSize, len(data)) + } + + msg := data[102 : 102+msgSize] + + if !bytes.Equal(pubKey, expectedKey[:]) { + return nil, fmt.Errorf("public key mismatch: got %s, expected %s", + solana.PublicKeyFromBytes(pubKey).String(), expectedKey.String()) + } + + if !ed25519.Verify(ed25519.PublicKey(pubKey), msg, sig) { + return nil, fmt.Errorf("ed25519 signature verification failed") + } + + feeds, err := parsePythPayload(msg) + if err != nil { + return nil, fmt.Errorf("failed to parse signed payload: %w", err) + } + + for i := range feeds { + if feeds[i].FeedID == feedID { + return &feeds[i], nil + } + } + + return nil, fmt.Errorf("feed %d not found in signed payload (have %d feeds)", feedID, len(feeds)) +} + +// parsePythPayload parses the inner LE-encoded Pyth Lazer payload message, +// extracting feed IDs and their properties. +func parsePythPayload(msg []byte) ([]ParsedFeedPrice, error) { + if len(msg) < 14 { + return nil, fmt.Errorf("payload too short: %d bytes", len(msg)) + } + + payloadMagic := binary.LittleEndian.Uint32(msg[0:4]) + if payloadMagic != PayloadFormatMagic { + return nil, fmt.Errorf("invalid payload magic: got 0x%08x, want 0x%08x", payloadMagic, PayloadFormatMagic) + } + + // timestamp_us at [4..12), channel_id at [12], num_feeds at [13] + numFeeds := msg[13] + off := 14 + + feeds := make([]ParsedFeedPrice, 0, numFeeds) + for i := 0; i < int(numFeeds); i++ { + if off+5 > len(msg) { + return nil, fmt.Errorf("payload truncated reading feed %d header", i) + } + feedID := binary.LittleEndian.Uint32(msg[off : off+4]) + numProps := msg[off+4] + off += 5 + + feed := ParsedFeedPrice{FeedID: feedID} + for j := 0; j < int(numProps); j++ { + if off >= len(msg) { + return nil, fmt.Errorf("payload truncated reading property %d of feed %d", j, i) + } + propTag := msg[off] + off++ + + var err error + off, err = parseProperty(msg, off, propTag, &feed) + if err != nil { + return nil, fmt.Errorf("feed %d property %d (tag %d): %w", i, j, propTag, err) + } + } + feeds = append(feeds, feed) + } + + return feeds, nil +} + +// parseProperty reads a single property value from msg at the given offset, +// populates feed fields as appropriate, and returns the new offset. +func parseProperty(msg []byte, off int, tag uint8, feed *ParsedFeedPrice) (int, error) { + switch tag { + case propPrice, propBestBidPrice, propBestAskPrice, propConfidence, + propEmaPrice, propEmaConfidence: + if off+8 > len(msg) { + return 0, fmt.Errorf("need 8 bytes for i64 property, have %d", len(msg)-off) + } + val := int64(binary.LittleEndian.Uint64(msg[off : off+8])) //nolint:gosec // reinterpreting unsigned bits as signed per Pyth wire format + if tag == propPrice { + feed.PriceMantissa = val + feed.HasPrice = true + } + return off + 8, nil + + case propPublisherCount: + if off+2 > len(msg) { + return 0, fmt.Errorf("need 2 bytes for u16 property, have %d", len(msg)-off) + } + return off + 2, nil + + case propExponent: + if off+2 > len(msg) { + return 0, fmt.Errorf("need 2 bytes for i16 property, have %d", len(msg)-off) + } + feed.Exponent = int16(binary.LittleEndian.Uint16(msg[off : off+2])) //nolint:gosec // reinterpreting unsigned bits as signed per Pyth wire format + feed.HasExponent = true + return off + 2, nil + + case propMarketSession: + if off+2 > len(msg) { + return 0, fmt.Errorf("need 2 bytes for i16 property, have %d", len(msg)-off) + } + return off + 2, nil + + case propFundingRate: + // u8 present flag + optional i64 + if off >= len(msg) { + return 0, fmt.Errorf("need 1 byte for presence flag, have %d", len(msg)-off) + } + if msg[off] != 0 { + if off+9 > len(msg) { + return 0, fmt.Errorf("need 9 bytes for optional i64, have %d", len(msg)-off) + } + return off + 9, nil + } + return off + 1, nil + + case propFundingTimestamp, propFundingRateInterval, propFeedUpdateTimestamp: + // u8 present flag + optional u64 + if off >= len(msg) { + return 0, fmt.Errorf("need 1 byte for presence flag, have %d", len(msg)-off) + } + if msg[off] != 0 { + if off+9 > len(msg) { + return 0, fmt.Errorf("need 9 bytes for optional u64, have %d", len(msg)-off) + } + if tag == propFeedUpdateTimestamp { + feed.TimestampUs = binary.LittleEndian.Uint64(msg[off+1 : off+9]) + feed.HasTimestamp = true + } + return off + 9, nil + } + return off + 1, nil + + default: + return 0, fmt.Errorf("unknown property tag %d", tag) + } +} diff --git a/providers/apis/stork/api_handler.go b/providers/apis/stork/api_handler.go index fa30b629f..57892cf3e 100644 --- a/providers/apis/stork/api_handler.go +++ b/providers/apis/stork/api_handler.go @@ -60,7 +60,7 @@ func (h *APIHandler) CreateURL( h.cache.Add(ticker) } - return fmt.Sprintf("%s?asset=%s", h.api.Endpoints[0].URL, strings.Join(ids, ",")), nil + return fmt.Sprintf("%s?asset=%s&provider=stork", h.api.Endpoints[0].URL, strings.Join(ids, ",")), nil } // scaleFactor is 10^18 as a big.Float, used to divide Stork's scaled price values. diff --git a/providers/factories/oracle/api.go b/providers/factories/oracle/api.go index 592c7244c..670849155 100644 --- a/providers/factories/oracle/api.go +++ b/providers/factories/oracle/api.go @@ -21,6 +21,7 @@ import ( "github.com/dydxprotocol/slinky/providers/apis/geckoterminal" "github.com/dydxprotocol/slinky/providers/apis/kraken" "github.com/dydxprotocol/slinky/providers/apis/polymarket" + "github.com/dydxprotocol/slinky/providers/apis/pyth" "github.com/dydxprotocol/slinky/providers/apis/stork" apihandlers "github.com/dydxprotocol/slinky/providers/base/api/handlers" "github.com/dydxprotocol/slinky/providers/base/api/metrics" @@ -99,6 +100,8 @@ func APIQueryHandlerFactory( apiPriceFetcher, err = osmosis.NewAPIPriceFetcher(logger, cfg.API, metrics) case providerName == polymarket.Name: apiDataHandler, err = polymarket.NewAPIHandler(cfg.API) + case providerName == pyth.Name: + apiDataHandler, err = pyth.NewAPIHandler(cfg.API) case providerName == stork.Name: apiDataHandler, err = stork.NewAPIHandler(cfg.API) default: diff --git a/scripts/genesis.go b/scripts/genesis.go index d19a5bca9..387aa9ef4 100644 --- a/scripts/genesis.go +++ b/scripts/genesis.go @@ -25,6 +25,7 @@ var ( usePolymarket = flag.Bool("use-polymarket", false, "use polymarket markets") useForex = flag.Bool("use-forex", false, "use forex markets") useStork = flag.Bool("use-stork", false, "use stork markets") + usePyth = flag.Bool("use-pyth", false, "use pyth markets") tempFile = flag.String("temp-file", "markets.json", "temporary file to store the market map") ) @@ -111,6 +112,11 @@ func main() { marketMap = mergeMarketMaps(marketMap, marketmaps.StorkMarketMap) } + if *usePyth { + fmt.Fprintf(flag.CommandLine.Output(), "Using pyth markets\n") + marketMap = mergeMarketMaps(marketMap, marketmaps.PythMarketMap) + } + if err := marketMap.ValidateBasic(); err != nil { fmt.Fprintf(flag.CommandLine.Output(), "failed to validate market map: %s\n", err) panic(err) diff --git a/scripts/genesis.sh b/scripts/genesis.sh index da0f64471..9c32e5b0f 100644 --- a/scripts/genesis.sh +++ b/scripts/genesis.sh @@ -4,7 +4,8 @@ set -eux go run $SCRIPT_DIR/genesis.go --use-core=$USE_CORE_MARKETS --use-raydium=$USE_RAYDIUM_MARKETS \ --use-uniswapv3-base=$USE_UNISWAPV3_BASE_MARKETS --use-coingecko=$USE_COINGECKO_MARKETS \ --use-polymarket=$USE_POLYMARKET_MARKETS --use-coinmarketcap=$USE_COINMARKETCAP_MARKETS \ - --use-osmosis=$USE_OSMOSIS_MARKETS --use-stork=$USE_STORK_MARKETS --temp-file=markets.json + --use-osmosis=$USE_OSMOSIS_MARKETS --use-stork=$USE_STORK_MARKETS --use-pyth=$USE_PYTH_MARKETS \ + --temp-file=markets.json MARKETS=$(cat markets.json) echo "MARKETS content: $MARKETS"