Skip to content

Complement crawled data #27

@tiagostutz

Description

@tiagostutz

The result of the crawler made by @douglasferlini in #5 is a JSON array with the overall information of the dresses os La Fiancee. Now, we need now to get the details of the dresses and generate a more detailed JSON array.

To do so, you will read the attached JSON and will make a request for the product API providing the urlPartin the following format:

const productID = <urlPart_from_json_array>
await fetch("https://www.lafiancee.com.br/_api/wix-ecommerce-storefront-web/api", {
    "credentials": "include",
    "headers": {
        "Accept": "*/*",
        "Authorization": "brUTfgwc9eaqQ4m_KjbIkjnR-MRt9rGfCLGikGEPiRU.eyJpbnN0YW5jZUlkIjoiMWI0OTQ1ODItZDg5Zi00MmY2LTg0YzAtNTAxOGE3NzI1Y2MyIiwiYXBwRGVmSWQiOiIxMzgwYjcwMy1jZTgxLWZmMDUtZjExNS0zOTU3MWQ5NGRmY2QiLCJtZXRhU2l0ZUlkIjoiN2RlM2ExNjgtNDEyNC00NDljLTg4ZDYtZmViNjkzYWY3NzRjIiwic2lnbkRhdGUiOiIyMDIwLTA5LTIzVDEyOjI3OjE4LjUyOVoiLCJ2ZW5kb3JQcm9kdWN0SWQiOiJQcmVtaXVtMSIsImRlbW9Nb2RlIjpmYWxzZSwiYWlkIjoiOWE0ZjJjNDAtMTIzNC00ZGM3LTg3OWEtMjIzZDMxMzI0N2E1IiwiYmlUb2tlbiI6IjY2YWFlNGVhLTk5YmItMDY2YS0wYzE2LWFlYWUzNGRkMmI4ZSIsInNpdGVPd25lcklkIjoiZmI0Y2Y2ODQtODZkZS00N2E0LWE2NjUtZjE4ZDcxYzA3YzUxIn0",
        "Content-Type": "application/json; charset=utf-8",
    },
    "body": `{"query":"query getProductBySlug($externalId: String!, $slug: String!, $withPricePerUnit: Boolean!, $withCountryCodes: Boolean!) {
          appSettings(externalId: $externalId) {
            widgetSettings
      }
      catalog {
            product(slug: $slug, onlyVisible: true) {
                id
                description
                isVisible
                sku
                ribbon
                price
                comparePrice
                discountedPrice
                formattedPrice
                formattedComparePrice
                formattedDiscountedPrice
                pricePerUnit @include(if: $withPricePerUnit)
                formattedPricePerUnit @include(if: $withPricePerUnit)
                pricePerUnitData @include(if: $withPricePerUnit) {
                baseQuantity
                baseMeasurementUnit
          }
          seoTitle
          seoDescription
          createVersion
          digitalProductFileItems {
                fileId
                fileType
                fileName
          }
          productItems {
                price
                comparePrice
                formattedPrice
                formattedComparePrice
                pricePerUnit @include(if: $withPricePerUnit)
                formattedPricePerUnit @include(if: $withPricePerUnit)
                optionsSelections
                isVisible
                inventory {
                status
                quantity
            }
            sku
            weight
            surcharge
            subscriptionPlans {
                list {
                id
                price
                formattedPrice
                pricePerUnit @include(if: $withPricePerUnit)
                formattedPricePerUnit @include(if: $withPricePerUnit)
              }
            }
          }
          name
          isTrackingInventory
          inventory {
            status
            quantity
          }
          isVisible
          isManageProductItems
          isInStock
          media {
            id
            url
            fullUrl
            altText
            thumbnailFullUrl: fullUrl(width: 50, height: 50)
            mediaType
            videoType
            videoFiles {
                url
                width
                height
                format
                quality
            }
            width
            height
            index
            title
          }
          customTextFields {
            title
            isMandatory
            inputLimit
          }
          nextOptionsSelectionId
          options {
            title
            optionType
            selections {
                id
                value
                description
                linkedMediaItems {
                    altText
                    url
                    fullUrl
                    thumbnailFullUrl: fullUrl(width: 50, height: 50)
                    mediaType
                    width
                    height
                    index
                    title
                    videoFiles {
                        url
                        width
                        height
                        format
                        quality
                    }
                }
            }
          }
          productType
          urlPart
          additionalInfo {
                id
            title
            description
            index
          }
          subscriptionPlans {
                list(onlyVisible: true) {
                  id
              name
              tagline
              frequency
              duration
              price
              formattedPrice
              pricePerUnit @include(if: $withPricePerUnit)
              formattedPricePerUnit @include(if: $withPricePerUnit)
            }
            oneTimePurchase {
                  index
            }
          }
          discount {
                mode
            value
          }
          currency
          weight
          seoJson
        }
      }
      localeData(language: "en") @include(if: $withCountryCodes) {
            countries {
              key
          shortKey
        }
      }
    }","variables":{"slug":productID,"externalId":"","withPricePerUnit":true,"withCountryCodes":false},"source":"WixStoresWebClient","operationName":"getProductBySlug"}`,
    "method": "POST",
});

This request will return a JSON that has the product options with title and selections which will be the features.

With this enhanced JSON Array we can build the database to serve this data.

Metadata

Metadata

Assignees

Type

No type
No fields configured for issues without a type.

Projects

No projects

Milestone

No milestone

Relationships

None yet

Development

No branches or pull requests

Issue actions