#564 - Reference resolution of custom object key-value-document references (and set of) as an attribute of nested type. (#563)

Author: ahmetoz

docs/RELEASE_NOTES.md

@@ -35,10 +35,11 @@
 [Javadoc](https://commercetools.github.io/commercetools-sync-java/v/2.2.0/) | 
 [Jar](https://bintray.com/commercetools/maven/commercetools-sync-java/2.2.0)
 
-- 🎉 **New Features** (1)
+- 🎉 **New Features** (2)
+    - **Product Sync** - Added support for resolving `key-value-document` (custom object) references on attributes of type `Reference`, `Set` of `Reference`, `NestedType` or `Set` of `NestedType`. [#564](https://github.com/commercetools/commercetools-sync-java/issues/564)     
     - Introduced new concept for the validation of `the drafts in batches` for each `Sync` instance, exposed with 
     `BaseBatchValidator` implementations (i.e. ProductBatchValidator, CategoryBatchValidator). [#233](https://github.com/commercetools/commercetools-sync-java/issues/233)
-
+    
 - ✨ **Enhancements** (2)    
     - **Category Sync** - Passed category keys in batch to `cacheKeysToIds` method of `CategoryService` to avoid fetching all categories for every batch. 
 [#235](https://github.com/commercetools/commercetools-sync-java/issues/235)

docs/usage/PRODUCT_SYNC.md

@@ -46,7 +46,12 @@ order for the sync to resolve the actual ids of those references, those `key`s h
      > Note: Some references in the product like `state`, `customerGroup` of prices, and variant attributes with type `reference` do not support the `ResourceIdentifier` yet, 
       for those references you need to provide the `key` value on the `id` field of the reference. This means that calling `getId()` on the
       reference would return its `key`. 
-
+    
+    - For resolving `key-value-document` (custom object) references on attributes of type `Reference`, `Set` of `Reference`, `NestedType` or `Set` of `NestedType`, The `id` field of the reference in the attribute draft should be defined in the correct format. 
+    The correct format must have a vertical bar `|` character between the values of the container and key.
+    For example, if the custom object has a container value `container` and key value `key`, the `id` field should be `container|key"`,  
+    also, the key and container value should match the pattern `[-_~.a-zA-Z0-9]+`.
+     
 4. Create a `sphereClient` [as described here](IMPORTANT_USAGE_TIPS.md#sphereclient-creation).
 
 5. After the `sphereClient` is set up, a `ProductSyncOptions` should be built as follows: 
@@ -198,7 +203,7 @@ attributes.
     | `“category”`  | ✅ |
     | `“channel”`  | ❌ |
     | `“customer”`  | ❌ |
-    | `“key-value-document”`  | ❌ |
+    | `“key-value-document”`  | ✅ |
     | `“order”`  | ❌ |
     | `“product”` | ✅ |
     | `“product-type”` | ✅ |

src/integration-test/java/com/commercetools/sync/integration/externalsource/products/ProductSyncWithNestedReferencedCustomObjectsIT.java

@@ -140,17 +140,20 @@ void fetchMatchingCustomObjects_WithDifferentExistingCombinationOfKeysAndContain
                 OLD_CUSTOM_OBJECT_CONTAINER + "_2", OLD_CUSTOM_OBJECT_VALUE);
 
         final Set<CustomObjectCompositeIdentifier> customObjectCompositeIdentifiers = new HashSet<>();
+        customObjectCompositeIdentifiers.add(CustomObjectCompositeIdentifier.of(
+                OLD_CUSTOM_OBJECT_KEY + "_1", OLD_CUSTOM_OBJECT_CONTAINER + "_1"));
         customObjectCompositeIdentifiers.add(CustomObjectCompositeIdentifier.of(
                 OLD_CUSTOM_OBJECT_KEY + "_1", OLD_CUSTOM_OBJECT_CONTAINER + "_2"));
         customObjectCompositeIdentifiers.add(CustomObjectCompositeIdentifier.of(
                 OLD_CUSTOM_OBJECT_KEY + "_2", OLD_CUSTOM_OBJECT_CONTAINER + "_1"));
 
+
         final Set<CustomObject<JsonNode>> matchingCustomObjects = customObjectService
                 .fetchMatchingCustomObjects(customObjectCompositeIdentifiers)
                 .toCompletableFuture()
                 .join();
 
-        assertThat(matchingCustomObjects).size().isEqualTo(2);
+        assertThat(matchingCustomObjects).size().isEqualTo(3);
         assertThat(errorCallBackExceptions).isEmpty();
         assertThat(errorCallBackMessages).isEmpty();
 

src/integration-test/java/com/commercetools/sync/integration/services/impl/CustomObjectServiceImplIT.java

@@ -9,8 +9,10 @@
 import java.util.ArrayList;
 import java.util.List;
 import java.util.function.Supplier;
+import java.util.regex.Pattern;
 
 public final class SyncUtils {
+    private static final String UUID_REGEX = "[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}";
 
     /**
      * Given a list of elements and a {@code batchSize}, this method distributes the elements into batches with the
@@ -83,6 +85,17 @@ public static <T extends WithKey> ResourceIdentifier<T> getResourceIdentifierWit
         return null;
     }
 
+    /**
+     * Given an id as {@link String}, this method checks whether if it is in UUID format or not.
+     *
+     * @param id to check if it is in UUID format.
+     * @return true if it is in UUID format, otherwise false.
+     */
+    public static boolean isUuid(@Nonnull final String id) {
+        final Pattern regexPattern = Pattern.compile(UUID_REGEX);
+        return regexPattern.matcher(id).matches();
+    }
+
     private SyncUtils() {
     }
 }

src/main/java/com/commercetools/sync/commons/utils/SyncUtils.java

@@ -18,6 +18,9 @@ public final class CustomObjectCompositeIdentifier {
 
     private final String key;
     private final String container;
+    private static final String WRONG_FORMAT_ERROR_MESSAGE = "The custom object identifier value: \"%s\" does not have "
+        + "the correct format. The correct format must have a vertical bar \"|\" character "
+        + "between the container and key.";
 
     private CustomObjectCompositeIdentifier(@Nonnull final String key,
                                             @Nonnull final String container) {
@@ -68,7 +71,24 @@ public static CustomObjectCompositeIdentifier of(@Nonnull final CustomObject cus
      */
     @Nonnull
     public static CustomObjectCompositeIdentifier of(@Nonnull final String key, @Nonnull final String container) {
-        return new CustomObjectCompositeIdentifier(key, container);
+        return CustomObjectCompositeIdentifier.of(
+            CustomObjectDraft.ofUnversionedUpsert(container, key, null));
+    }
+
+    /**
+     * Given a {@link String} with a format "container|key", creates a {@link CustomObjectCompositeIdentifier} using
+     * the following fields from the supplied identifier as string.
+     *
+     * @param identifierAsString  string format of the identifier to build composite Id.
+     * @return a composite id comprised of the fields of the supplied {@code key} and {@code container}.
+     */
+    @Nonnull
+    public static CustomObjectCompositeIdentifier of(@Nonnull final String identifierAsString) {
+        final String[] containerAndKey = identifierAsString.split("\\|");
+        if (containerAndKey.length == 2) {
+            return of(containerAndKey[1], containerAndKey[0]);
+        }
+        throw new IllegalArgumentException(format(WRONG_FORMAT_ERROR_MESSAGE, identifierAsString));
     }
 
     public String getKey() {
@@ -79,11 +99,6 @@ public String getContainer() {
         return this.container;
     }
 
-    @Override
-    public String toString() {
-        return format("{key='%s', container='%s'}", key, container);
-    }
-
     @Override
     public boolean equals(final Object obj) {
         if (this == obj) {
@@ -100,4 +115,9 @@ public boolean equals(final Object obj) {
     public int hashCode() {
         return Objects.hash(key, container);
     }
+
+    @Override
+    public String toString() {
+        return format("%s|%s", container, key);
+    }
 }

src/main/java/com/commercetools/sync/customobjects/helpers/CustomObjectCompositeIdentifier.java

@@ -4,11 +4,13 @@
 import com.commercetools.sync.commons.BaseSync;
 import com.commercetools.sync.commons.exceptions.SyncException;
 import com.commercetools.sync.commons.models.WaitingToBeResolved;
+import com.commercetools.sync.customobjects.CustomObjectSyncOptionsBuilder;
 import com.commercetools.sync.products.helpers.ProductBatchValidator;
 import com.commercetools.sync.products.helpers.ProductReferenceResolver;
 import com.commercetools.sync.products.helpers.ProductSyncStatistics;
 import com.commercetools.sync.services.CategoryService;
 import com.commercetools.sync.services.ChannelService;
+import com.commercetools.sync.services.CustomObjectService;
 import com.commercetools.sync.services.CustomerGroupService;
 import com.commercetools.sync.services.ProductService;
 import com.commercetools.sync.services.ProductTypeService;
@@ -18,6 +20,7 @@
 import com.commercetools.sync.services.UnresolvedReferencesService;
 import com.commercetools.sync.services.impl.CategoryServiceImpl;
 import com.commercetools.sync.services.impl.ChannelServiceImpl;
+import com.commercetools.sync.services.impl.CustomObjectServiceImpl;
 import com.commercetools.sync.services.impl.CustomerGroupServiceImpl;
 import com.commercetools.sync.services.impl.ProductServiceImpl;
 import com.commercetools.sync.services.impl.ProductTypeServiceImpl;
@@ -93,21 +96,23 @@ public ProductSync(@Nonnull final ProductSyncOptions productSyncOptions) {
             new CustomerGroupServiceImpl(productSyncOptions),
             new TaxCategoryServiceImpl(TaxCategorySyncOptionsBuilder.of(productSyncOptions.getCtpClient()).build()),
             new StateServiceImpl(StateSyncOptionsBuilder.of(productSyncOptions.getCtpClient()).build()),
-            new UnresolvedReferencesServiceImpl(productSyncOptions));
+            new UnresolvedReferencesServiceImpl(productSyncOptions),
+            new CustomObjectServiceImpl(CustomObjectSyncOptionsBuilder.of(productSyncOptions.getCtpClient()).build()));
     }
 
     ProductSync(@Nonnull final ProductSyncOptions productSyncOptions, @Nonnull final ProductService productService,
                 @Nonnull final ProductTypeService productTypeService, @Nonnull final CategoryService categoryService,
                 @Nonnull final TypeService typeService, @Nonnull final ChannelService channelService,
                 @Nonnull final CustomerGroupService customerGroupService,
                 @Nonnull final TaxCategoryService taxCategoryService, @Nonnull final StateService stateService,
-                @Nonnull final UnresolvedReferencesService unresolvedReferencesService) {
+                @Nonnull final UnresolvedReferencesService unresolvedReferencesService,
+                @Nonnull final CustomObjectService customObjectService) {
         super(new ProductSyncStatistics(), productSyncOptions);
         this.productService = productService;
         this.productTypeService = productTypeService;
         this.productReferenceResolver = new ProductReferenceResolver(getSyncOptions(), productTypeService,
             categoryService, typeService, channelService, customerGroupService, taxCategoryService, stateService,
-            productService);
+            productService, customObjectService);
         this.unresolvedReferencesService = unresolvedReferencesService;
         this.batchValidator = new ProductBatchValidator(getSyncOptions(), getStatistics());
     }

src/main/java/com/commercetools/sync/products/ProductSync.java

@@ -1,9 +1,12 @@
 package com.commercetools.sync.products.helpers;
 
 import com.commercetools.sync.commons.helpers.BaseBatchValidator;
+import com.commercetools.sync.commons.utils.SyncUtils;
+import com.commercetools.sync.customobjects.helpers.CustomObjectCompositeIdentifier;
 import com.commercetools.sync.products.ProductSyncOptions;
 import com.fasterxml.jackson.databind.JsonNode;
 import io.sphere.sdk.categories.Category;
+import io.sphere.sdk.customobjects.CustomObject;
 import io.sphere.sdk.products.Product;
 import io.sphere.sdk.products.ProductDraft;
 import io.sphere.sdk.products.ProductVariantDraft;
@@ -15,6 +18,7 @@
 import javax.annotation.Nullable;
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Objects;
@@ -27,6 +31,7 @@
 import static java.lang.String.format;
 import static java.util.Collections.emptySet;
 import static java.util.Objects.requireNonNull;
+import static java.util.stream.Collectors.toSet;
 import static org.apache.commons.lang3.StringUtils.isBlank;
 
 public class ProductBatchValidator
@@ -181,6 +186,9 @@ private void collectReferencedKeysInAttributes(
 
         referencedKeys.productTypeKeys.addAll(
             getReferencedKeysWithReferenceTypeId(variantDraft, ProductType.referenceTypeId()));
+
+        referencedKeys.customObjectCompositeIdentifiers.addAll(
+            getReferencedKeysWithReferenceTypeId(variantDraft, CustomObject.referenceTypeId()));
     }
 
     @Nonnull
@@ -274,6 +282,7 @@ public static class ReferencedKeys {
         private final Set<String> typeKeys = new HashSet<>();
         private final Set<String> channelKeys = new HashSet<>();
         private final Set<String> customerGroupKeys = new HashSet<>();
+        private final Set<String> customObjectCompositeIdentifiers = new HashSet<>();
 
         public Set<String> getProductKeys() {
             return productKeys;
@@ -306,5 +315,36 @@ public Set<String> getChannelKeys() {
         public Set<String> getCustomerGroupKeys() {
             return customerGroupKeys;
         }
+
+        /**
+         * Applies mapping of {@link Set}&lt;{@link String}&gt; identifiers
+         * (collected from reference id fields of product `key-value-document` references) to {@link Set}&lt;
+         * {@link CustomObjectCompositeIdentifier}&gt; to be used for caching purposes.
+         *
+         * <p>Note: Invalid identifiers and uuid formatted identifiers will be filtered out.
+         * Validation handling will be part of the {@link VariantReferenceResolver}.
+         *
+         * @return a result set with valid identifiers mapped to {@link CustomObjectCompositeIdentifier}.
+         */
+        public Set<CustomObjectCompositeIdentifier> getCustomObjectCompositeIdentifiers() {
+            if (!customObjectCompositeIdentifiers.isEmpty()) {
+                return customObjectCompositeIdentifiers
+                    .stream()
+                    .map((identifierAsString) -> {
+                        if (SyncUtils.isUuid(identifierAsString)) {
+                            return null;
+                        }
+                        try {
+                            return CustomObjectCompositeIdentifier.of(identifierAsString);
+                        } catch (IllegalArgumentException ignored) {
+                            return null;
+                        }
+                    })
+                    .filter(Objects::nonNull)
+                    .collect(toSet());
+            }
+
+            return Collections.emptySet();
+        }
     }
 }

src/main/java/com/commercetools/sync/products/helpers/ProductBatchValidator.java

@@ -2,9 +2,11 @@
 
 import com.commercetools.sync.commons.exceptions.ReferenceResolutionException;
 import com.commercetools.sync.commons.helpers.BaseReferenceResolver;
+import com.commercetools.sync.customobjects.helpers.CustomObjectCompositeIdentifier;
 import com.commercetools.sync.products.ProductSyncOptions;
 import com.commercetools.sync.services.CategoryService;
 import com.commercetools.sync.services.ChannelService;
+import com.commercetools.sync.services.CustomObjectService;
 import com.commercetools.sync.services.CustomerGroupService;
 import com.commercetools.sync.services.ProductService;
 import com.commercetools.sync.services.ProductTypeService;
@@ -54,6 +56,7 @@ public final class ProductReferenceResolver extends BaseReferenceResolver<Produc
     private final ChannelService channelService;
     private final ProductService productService;
     private final CustomerGroupService customerGroupService;
+    private final CustomObjectService customObjectService;
 
     public static final String FAILED_TO_RESOLVE_REFERENCE = "Failed to resolve '%s' resource identifier on "
         + "ProductDraft with key:'%s'. Reason: %s";
@@ -80,6 +83,7 @@ public final class ProductReferenceResolver extends BaseReferenceResolver<Produc
      * @param stateService         the service to fetch product states for reference resolution.
      * @param productService       the service to fetch products for product reference resolution on reference
      *                             attributes.
+     * @param customObjectService  the service to fetch custom objects for reference resolution.
      */
     public ProductReferenceResolver(@Nonnull final ProductSyncOptions productSyncOptions,
                                     @Nonnull final ProductTypeService productTypeService,
@@ -89,7 +93,8 @@ public ProductReferenceResolver(@Nonnull final ProductSyncOptions productSyncOpt
                                     @Nonnull final CustomerGroupService customerGroupService,
                                     @Nonnull final TaxCategoryService taxCategoryService,
                                     @Nonnull final StateService stateService,
-                                    @Nonnull final ProductService productService) {
+                                    @Nonnull final ProductService productService,
+                                    @Nonnull final CustomObjectService customObjectService) {
         super(productSyncOptions);
         this.productTypeService = productTypeService;
         this.categoryService = categoryService;
@@ -99,9 +104,10 @@ public ProductReferenceResolver(@Nonnull final ProductSyncOptions productSyncOpt
         this.channelService = channelService;
         this.productService = productService;
         this.customerGroupService = customerGroupService;
+        this.customObjectService = customObjectService;
         this.variantReferenceResolver =
             new VariantReferenceResolver(productSyncOptions, typeService, channelService, customerGroupService,
-                productService, productTypeService, categoryService);
+                productService, productTypeService, categoryService, customObjectService);
     }
 
     /**
@@ -422,6 +428,12 @@ public CompletableFuture<Map<String, String>> populateKeyToIdCachesForReferenced
             futures.add(customerGroupService.cacheKeysToIds(customerGroupKeys));
         }
 
+        final Set<CustomObjectCompositeIdentifier> customObjectCompositeIdentifiers =
+            referencedKeys.getCustomObjectCompositeIdentifiers();
+        if (!customObjectCompositeIdentifiers.isEmpty()) {
+            futures.add(customObjectService.cacheKeysToIds(customObjectCompositeIdentifiers));
+        }
+
         return collectionOfFuturesToFutureOfCollection(futures, toList())
             .thenApply(maps -> productKeys.isEmpty() ? Collections.emptyMap() : maps.get(0));
     }