feat(net): to/from ipld bindnode types, more cbor protoc improvements

Author: mvdan

go.mod

@@ -4,6 +4,7 @@ go 1.16
 
 require (
 	github.com/google/uuid v1.3.0
+	github.com/google/go-cmp v0.5.6
 	github.com/hannahhoward/cbor-gen-for v0.0.0-20200817222906-ea96cece81f1
 	github.com/hannahhoward/go-pubsub v0.0.0-20200423002714-8d62886cc36e
 	github.com/ipfs/go-block-format v0.0.3
@@ -26,7 +27,7 @@ require (
 	github.com/ipfs/go-peertaskqueue v0.7.1
 	github.com/ipfs/go-unixfs v0.2.4
 	github.com/ipld/go-codec-dagpb v1.3.0
-	github.com/ipld/go-ipld-prime v0.14.4-0.20220110161855-fc09d6b768e9
+	github.com/ipld/go-ipld-prime v0.14.4
 	github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c
 	github.com/libp2p/go-buffer-pool v0.0.2
 	github.com/libp2p/go-libp2p v0.16.0
@@ -45,5 +46,3 @@ require (
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1
 	google.golang.org/protobuf v1.27.1
 )
-
-replace github.com/ipld/go-ipld-prime => ../../ipld/go-ipld-prime

go.sum

@@ -457,8 +457,8 @@ github.com/ipld/go-codec-dagpb v1.3.0 h1:czTcaoAuNNyIYWs6Qe01DJ+sEX7B+1Z0LcXjSat
 github.com/ipld/go-codec-dagpb v1.3.0/go.mod h1:ga4JTU3abYApDC3pZ00BC2RSvC3qfBb9MSJkMLSwnhA=
 github.com/ipld/go-ipld-prime v0.9.1-0.20210324083106-dc342a9917db/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8=
 github.com/ipld/go-ipld-prime v0.11.0/go.mod h1:+WIAkokurHmZ/KwzDOMUuoeJgaRQktHtEaLglS3ZeV8=
-github.com/ipld/go-ipld-prime v0.14.4-0.20220110161855-fc09d6b768e9 h1:fqQSvdPznhyE5jZoCwbvykHbK+QukxJcS1SFWbj9ig0=
-github.com/ipld/go-ipld-prime v0.14.4-0.20220110161855-fc09d6b768e9/go.mod h1:QcE4Y9n/ZZr8Ijg5bGPT0GqYWgZ1704nH0RDcQtgTP0=
+github.com/ipld/go-ipld-prime v0.14.4 h1:bqhmume8+nbNsX4/+J6eohktfZHAI8GKrF3rQ0xgOyc=
+github.com/ipld/go-ipld-prime v0.14.4/go.mod h1:QcE4Y9n/ZZr8Ijg5bGPT0GqYWgZ1704nH0RDcQtgTP0=
 github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA=
 github.com/jackpal/go-nat-pmp v1.0.1/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc=
 github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus=

message/bench_test.go

@@ -1,13 +1,15 @@
-package message_test
+package message
 
 import (
 	"bytes"
 	"math/rand"
+	"reflect"
 	"testing"
 
+	"github.com/google/go-cmp/cmp"
 	blocks "github.com/ipfs/go-block-format"
 	"github.com/ipfs/go-graphsync"
-	"github.com/ipfs/go-graphsync/message"
+	"github.com/ipfs/go-graphsync/message/ipldbind"
 	"github.com/ipfs/go-graphsync/testutil"
 	"github.com/ipld/go-ipld-prime/codec/dagcbor"
 	"github.com/ipld/go-ipld-prime/node/basicnode"
@@ -21,16 +23,17 @@ func BenchmarkMessageEncodingRoundtrip(b *testing.B) {
 	ssb := builder.NewSelectorSpecBuilder(basicnode.Prototype.Any)
 	selector := ssb.Matcher().Node()
 	extensionName := graphsync.ExtensionName("graphsync/awesome")
-	extension := message.NamedExtension{
+	extension := graphsync.ExtensionData{
 		Name: extensionName,
-		Data: basicnode.NewBytes(testutil.RandomBytes(100)),
+		Data: testutil.RandomBytes(100),
 	}
 	id := graphsync.NewRequestID()
 	priority := graphsync.Priority(rand.Int31())
 	status := graphsync.RequestAcknowledged
 
-	builder := message.NewBuilder()
-	builder.AddRequest(message.NewRequest(id, root, selector, priority, extension))
+	builder := NewBuilder()
+	builder.AddRequest(NewRequest(id, root, selector, priority, extension))
+	builder.AddRequest(NewRequest(id, root, selector, priority))
 	builder.AddResponseCode(id, status)
 	builder.AddExtensionData(id, extension)
 	builder.AddBlock(blocks.NewBlock([]byte("W")))
@@ -48,12 +51,17 @@ func BenchmarkMessageEncodingRoundtrip(b *testing.B) {
 			for pb.Next() {
 				buf.Reset()
 
-				err := message.NewMessageHandler().ToNet(gsm, buf)
+				err := NewMessageHandler().ToNet(gsm, buf)
 				require.NoError(b, err)
 
-				gsm2, err := message.NewMessageHandler().FromNet(buf)
+				gsm2, err := NewMessageHandler().FromNet(buf)
 				require.NoError(b, err)
-				require.Equal(b, gsm, gsm2)
+
+				// Note that require.Equal doesn't seem to handle maps well.
+				// It says they are non-equal simply because their order isn't deterministic.
+				if diff := cmp.Diff(gsm, gsm2, cmp.Exporter(func(reflect.Type) bool { return true })); diff != "" {
+					b.Fatal(diff)
+				}
 			}
 		})
 	})
@@ -65,16 +73,24 @@ func BenchmarkMessageEncodingRoundtrip(b *testing.B) {
 			for pb.Next() {
 				buf.Reset()
 
-				node := bindnode.Wrap(&gsm, message.Prototype.Message.Type())
-				err := dagcbor.Encode(node.Representation(), buf)
+				ipldGSM, err := gsm.ToIPLD()
+				require.NoError(b, err)
+				node := bindnode.Wrap(ipldGSM, ipldbind.Prototype.Message.Type())
+				err = dagcbor.Encode(node.Representation(), buf)
 				require.NoError(b, err)
 
-				builder := message.Prototype.Message.Representation().NewBuilder()
+				builder := ipldbind.Prototype.Message.Representation().NewBuilder()
 				err = dagcbor.Decode(builder, buf)
 				require.NoError(b, err)
 				node2 := builder.Build()
-				gsm2 := *bindnode.Unwrap(node2).(*message.GraphSyncMessage)
-				require.Equal(b, gsm, gsm2)
+				ipldGSM2 := bindnode.Unwrap(node2).(*ipldbind.GraphSyncMessage)
+				gsm2, err := messageFromIPLD(ipldGSM2)
+				require.NoError(b, err)
+
+				// same as above.
+				if diff := cmp.Diff(gsm, gsm2, cmp.Exporter(func(reflect.Type) bool { return true })); diff != "" {
+					b.Fatal(diff)
+				}
 			}
 		})
 	})

message/builder.go

@@ -2,10 +2,9 @@ package message
 
 import (
 	blocks "github.com/ipfs/go-block-format"
-	cid "github.com/ipfs/go-cid"
+	"github.com/ipfs/go-cid"
 	"github.com/ipld/go-ipld-prime"
 	cidlink "github.com/ipld/go-ipld-prime/linking/cid"
-	"github.com/ipld/go-ipld-prime/node/basicnode"
 
 	"github.com/ipfs/go-graphsync"
 	"github.com/ipfs/go-graphsync/metadata"
@@ -19,7 +18,7 @@ type Builder struct {
 	blkSize            uint64
 	completedResponses map[graphsync.RequestID]graphsync.ResponseStatusCode
 	outgoingResponses  map[graphsync.RequestID]metadata.Metadata
-	extensions         map[graphsync.RequestID][]NamedExtension
+	extensions         map[graphsync.RequestID][]graphsync.ExtensionData
 	requests           map[graphsync.RequestID]GraphSyncRequest
 }
 
@@ -30,13 +29,13 @@ func NewBuilder() *Builder {
 		outgoingBlocks:     make(map[cid.Cid]blocks.Block),
 		completedResponses: make(map[graphsync.RequestID]graphsync.ResponseStatusCode),
 		outgoingResponses:  make(map[graphsync.RequestID]metadata.Metadata),
-		extensions:         make(map[graphsync.RequestID][]NamedExtension),
+		extensions:         make(map[graphsync.RequestID][]graphsync.ExtensionData),
 	}
 }
 
 // AddRequest registers a new request to be added to the message.
 func (b *Builder) AddRequest(request GraphSyncRequest) {
-	b.requests[request.ID] = request
+	b.requests[request.ID()] = request
 }
 
 // AddBlock adds the given block to the message.
@@ -46,7 +45,7 @@ func (b *Builder) AddBlock(block blocks.Block) {
 }
 
 // AddExtensionData adds the given extension data to to the message
-func (b *Builder) AddExtensionData(requestID graphsync.RequestID, extension NamedExtension) {
+func (b *Builder) AddExtensionData(requestID graphsync.RequestID, extension graphsync.ExtensionData) {
 	b.extensions[requestID] = append(b.extensions[requestID], extension)
 	// make sure this extension goes out in next response even if no links are sent
 	_, ok := b.outgoingResponses[requestID]
@@ -110,30 +109,21 @@ func (b *Builder) ScrubResponses(requestIDs []graphsync.RequestID) uint64 {
 
 // Build assembles and encodes message data from the added requests, links, and blocks.
 func (b *Builder) Build() (GraphSyncMessage, error) {
-	requests := make([]GraphSyncRequest, 0, len(b.requests))
-	for _, request := range b.requests {
-		requests = append(requests, request)
-	}
-	responses := make([]GraphSyncResponse, 0, len(b.outgoingResponses))
+	responses := make(map[graphsync.RequestID]GraphSyncResponse, len(b.outgoingResponses))
 	for requestID, linkMap := range b.outgoingResponses {
 		mdRaw, err := metadata.EncodeMetadata(linkMap)
 		if err != nil {
 			return GraphSyncMessage{}, err
 		}
-		b.extensions[requestID] = append(b.extensions[requestID], NamedExtension{
+		b.extensions[requestID] = append(b.extensions[requestID], graphsync.ExtensionData{
 			Name: graphsync.ExtensionMetadata,
-			Data: basicnode.NewBytes(mdRaw), // TODO: likely wrong
+			Data: mdRaw,
 		})
 		status, isComplete := b.completedResponses[requestID]
-		responses = append(responses, NewResponse(requestID, responseCode(status, isComplete), b.extensions[requestID]...))
-	}
-	blocks := make([]GraphSyncBlock, 0, len(b.outgoingBlocks))
-	for _, block := range b.outgoingBlocks {
-		blocks = append(blocks, FromBlockFormat(block))
+		responses[requestID] = NewResponse(requestID, responseCode(status, isComplete), b.extensions[requestID]...)
 	}
-	// TODO: sort requests, responses, and blocks? map order is randomized
 	return GraphSyncMessage{
-		requests, responses, blocks,
+		b.requests, responses, b.outgoingBlocks,
 	}, nil
 }