Use GraphQL to retrieve products and variants

REST endpoint is deprecated. New method is A LOT slower (~4x)

Author: sshaw

exportformat/csv.go

@@ -4,9 +4,8 @@ import (
 	"encoding/csv"
 	"fmt"
 	"os"
-	"strconv"
 
-	shopify "github.com/bold-commerce/go-shopify/v3"
+	"github.com/screenstaring/shopify_id_export/gql"
 )
 
 type CSV struct {
@@ -40,20 +39,21 @@ func NewCSV(shop string) (*CSV, error) {
 	return c, nil
 }
 
-func (c *CSV) Dump(product shopify.Product) error {
+func (c *CSV) Dump(product gql.Product) error {
 	var err error
 
 	if !c.headerWritten {
 		c.out.Write(c.header)
 		c.headerWritten = true
 	}
 
-	for _, variant := range product.Variants {
+	for _, edge := range product.Variants.Edges {
+		variant := edge.Node
 		row := []string{
-			strconv.FormatInt(variant.ProductID, 10),
+			product.ID,
 			product.Title,
 			product.ProductType,
-			strconv.FormatInt(variant.ID, 10),
+			variant.ID,
 			variant.Title,
 			variant.Sku,
 			variant.Barcode,

exportformat/json.go

@@ -4,15 +4,14 @@ import (
 	"encoding/json"
 	"fmt"
 	"os"
-	"strconv"
 
-	shopify "github.com/bold-commerce/go-shopify/v3"
+	"github.com/screenstaring/shopify_id_export/gql"
 )
 
 type JSON struct {
 	out      *os.File
 	root     string
-	products []shopify.Product
+	products []gql.Product
 }
 
 var JSONRootProperties = []string{
@@ -34,22 +33,22 @@ func isValidJSONRootProperty(name string) bool {
 	return false
 }
 
-func productMap(product shopify.Product) map[string]interface{} {
+func productMap(product gql.Product) map[string]interface{} {
 	record := make(map[string]interface{})
 
-	record["product_id"] = strconv.FormatInt(product.ID, 10)
+	record["product_id"] = product.ID
 	record["handle"] = product.Handle
 	record["product_title"] = product.Title
 	record["product_type"] = product.ProductType
 
 	return record
 }
 
-func variantMap(variant shopify.Variant) map[string]string {
+func variantMap(variant gql.Variant) map[string]string {
 	record := make(map[string]string)
 
 	record["barcode"] = variant.Barcode
-	record["variant_id"] = strconv.FormatInt(variant.ID, 10)
+	record["variant_id"] = variant.ID
 	record["variant_title"] = variant.Title
 	record["sku"] = variant.Sku
 
@@ -68,7 +67,8 @@ func (j *JSON) formatWithVariantRoot() map[string]interface{} {
 	output := make(map[string]interface{})
 
 	for _, product := range j.products {
-		for _, variant := range product.Variants {
+		for _, vEdge := range product.Variants.Edges {
+			variant := vEdge.Node
 			record := variantMap(variant)
 			key := record[j.root]
 			if len(key) == 0 {
@@ -103,7 +103,8 @@ func (j *JSON) formatWithProduct() interface{} {
 		record := productMap(product)
 
 		var variants []map[string]string
-		for _, variant := range product.Variants {
+		for _, vEdge := range product.Variants.Edges {
+			variant := vEdge.Node
 			variants = append(variants, variantMap(variant))
 		}
 
@@ -121,7 +122,8 @@ func (j *JSON) formatWithProductRoot() map[string]interface{} {
 		record := productMap(product)
 
 		var variants []map[string]string
-		for _, variant := range product.Variants {
+		for _, vEdge := range product.Variants.Edges {
+			variant := vEdge.Node
 			variants = append(variants, variantMap(variant))
 		}
 
@@ -156,7 +158,7 @@ func NewJSON(shop string, jsonRoot string) (*JSON, error) {
 	return j, nil
 }
 
-func (j *JSON) Dump(product shopify.Product) error {
+func (j *JSON) Dump(product gql.Product) error {
 	j.products = append(j.products, product)
 
 	return nil

go.mod

@@ -2,4 +2,9 @@ module github.com/screenstaring/shopify_id_export
 
 go 1.16
 
-require github.com/bold-commerce/go-shopify/v3 v3.17.0 // indirect
+require (
+	github.com/bold-commerce/go-shopify/v3 v3.17.0 // indirect
+	github.com/bold-commerce/go-shopify/v4 v4.6.0 // indirect
+	github.com/google/go-querystring v1.1.0 // indirect
+	github.com/shopspring/decimal v1.4.0 // indirect
+)

go.sum

@@ -1,9 +1,17 @@
 github.com/bold-commerce/go-shopify/v3 v3.17.0 h1:1qZenleSsJMVFh5hu6R2z2NfmWP5xG0P8MawePr46K0=
 github.com/bold-commerce/go-shopify/v3 v3.17.0/go.mod h1:qOrEfYoy5RRO/PAq4vGyHW03NZmt2iX/fPGuaZwemtI=
+github.com/bold-commerce/go-shopify/v4 v4.6.0 h1:TqCNix3Qk2pwMDkrN1ZXJ91c3TMiIbtIkAPyx94EUZA=
+github.com/bold-commerce/go-shopify/v4 v4.6.0/go.mod h1:Sjg+C2CLNhYeCbwB6EedKgj2AHEBOP3ng+Eu1IfMf1U=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
 github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
+github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
+github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
 github.com/jarcoal/httpmock v1.3.0/go.mod h1:3yb8rc4BI7TCBhFY8ng0gjuLKJNquuDNiPaZjnENuYg=
 github.com/maxatome/go-testdeep v1.12.0/go.mod h1:lPZc/HAcJMP92l7yI6TRz1aZN5URwUBUAfUNvrclaNM=
 github.com/shopspring/decimal v0.0.0-20200105231215-408a2507e114 h1:Pm6R878vxWWWR+Sa3ppsLce/Zq+JNTs6aVvRu13jv9A=
 github.com/shopspring/decimal v0.0.0-20200105231215-408a2507e114/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
+github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k=
+github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

gql/client.go

@@ -0,0 +1,85 @@
+package gql
+
+import (
+	"context"
+	shopify "github.com/bold-commerce/go-shopify/v4"
+)
+
+const productsQuery = `
+  query findProducts($after: String) {
+    products(first:250 after:$after) {
+      pageInfo {
+	hasNextPage
+	endCursor
+      }
+      edges {
+	node {
+          legacyResourceId
+          handle
+          productType
+	  title
+	  variants(first:100) {
+	    edges {
+	      node {
+                barcode
+		legacyResourceId
+                sku
+		title
+	      }
+	    }
+	  }
+	}
+      }
+    }
+  }
+`
+
+type productResponse struct {
+	Products Products `json:"products"`
+}
+
+type Products struct {
+	Edges []ProductEdge `json:"edges"`
+	PageInfo PageInfo `json:"pageInfo"`
+}
+
+type ProductEdge struct {
+	Node Product `json:"node"`
+}
+
+type PageInfo struct {
+	HasNextPage bool `json:"hasNextPage"`
+	Cursor string `json:"endCursor"`
+}
+
+type Product struct {
+	Handle   string   `json:"handle"`
+	ID       string   `json:"legacyResourceId"`
+	Title    string   `json:"title"`
+	ProductType string  `json:"productType"`
+	Variants Variants `json:"variants"`
+}
+
+type Variants struct {
+	Edges []VariantEdge `json:"edges"`
+}
+
+type VariantEdge struct {
+	Node Variant `json:"node"`
+}
+
+type Variant struct {
+	Barcode string `json:"barcode"`
+	ID    string `json:"legacyResourceId"`
+	ProductID string `json:"productId"`
+	Sku string `json:"sku"`
+	Title string `json:"title"`
+}
+
+func FindProducts(client *shopify.Client, options map[string]interface{}) (Products, error) {
+	response := productResponse{}
+
+	err := client.GraphQL.Query(context.Background(), productsQuery, options, &response)
+
+	return response.Products, err
+}

main.go

@@ -8,11 +8,12 @@ import (
 	"strings"
 	"time"
 
-	shopify "github.com/bold-commerce/go-shopify/v3"
+	shopify "github.com/bold-commerce/go-shopify/v4"
 	"github.com/screenstaring/shopify_id_export/exportformat"
+	"github.com/screenstaring/shopify_id_export/gql"
 )
 
-const version = "v0.0.7"
+const version = "v0.1.0"
 const shopifyFields = "id,title,product_type,handle,variants"
 
 const usage = `shopify_id_export [hjv] [-k key] [-p password] [-r root-property] [-t token] shop
@@ -35,7 +36,7 @@ Valid properties for the --json-root option are: %s
 `
 
 type dumper interface {
-	Dump(shopify.Product) error
+	Dump(gql.Product) error
 	Close() error
 }
 
@@ -45,34 +46,32 @@ func exitFailure(error string, code int) {
 }
 
 func dumpProducts(client *shopify.Client, dumper dumper, pageSize int) error {
-	listOptions := shopify.ListOptions{
-		Fields: shopifyFields,
-		Limit: pageSize,
+	listOptions := map[string]interface{}{
+		"after": nil,
 	}
 
 	for {
-		products, pages, err := client.Product.ListWithPagination(
-			shopify.ProductListOptions{
-				ListOptions: listOptions,
-			},
-		)
-
+		products, err := gql.FindProducts(client, listOptions)
 		if err != nil {
 			return fmt.Errorf("Failed retrieve products: "+err.Error())
 		}
 
-		for _, product := range products {
+		page := products.PageInfo
+
+		for _, edge := range(products.Edges) {
+			product := edge.Node
+
 			err = dumper.Dump(product)
 			if err != nil {
 				return fmt.Errorf("Failed saving products: "+err.Error())
 			}
 		}
 
-		if pages.NextPageOptions == nil {
+		if !page.HasNextPage {
 			break
 		}
 
-		listOptions.PageInfo = pages.NextPageOptions.PageInfo
+		listOptions["after"] = page.Cursor
 	}
 
 	return nil
@@ -86,7 +85,12 @@ func main() {
 	var pageSize int
 	var client *shopify.Client
 
-	options := []shopify.Option{shopify.WithRetry(5)}
+	options := []shopify.Option{
+		shopify.WithRetry(5),
+		// Bug forces us to specify version
+		// https://github.com/bold-commerce/go-shopify/issues/314
+		shopify.WithVersion("2024-10"),
+	}
 
 	flag.Usage = func() {
 		exitFailure(fmt.Sprintf(usage, strings.Join(exportformat.JSONRootProperties, ", ")), 2)
@@ -141,6 +145,7 @@ func main() {
 	if timeout > -1 {
 		options = append(
 			options,
+			// FIXME: update for GQL
 			shopify.WithHTTPClient(
 				&http.Client{
 					Timeout: time.Second * time.Duration(timeout),
@@ -153,7 +158,10 @@ func main() {
 		options = append(options, shopify.WithLogger(&shopify.LeveledLogger{Level: shopify.LevelDebug}))
 	}
 
-	client = shopify.NewClient(app, argv[0], token, options...)
+	client, err = shopify.NewClient(app, argv[0], token, options...)
+	if err != nil {
+		exitFailure(err.Error(), 2)
+	}
 
 	err = dumpProducts(client, dumper, pageSize)
 	dumpErr := dumper.Close()