Added error check for duplicate reference ids

Author: ardila

nucleus/dataset.py

@@ -1,3 +1,4 @@
+from collections import Counter
 from typing import List, Dict, Any, Optional
 
 from nucleus.utils import format_dataset_item_response
@@ -158,6 +159,19 @@ def append(
             'ignored_items': int,
         }
         """
+        ref_ids = []
+        for dataset_item in dataset_items:
+            if dataset_item.reference_id is not None:
+                ref_ids.append(dataset_item.reference_id)
+        if len(ref_ids) != len(set(ref_ids)):
+            duplicates = {
+                f"{key}": f"Count: {value}"
+                for key, value in Counter(ref_ids).items()
+            }
+            raise ValueError(
+                "Duplicate reference ids found among dataset_items: %s"
+                % duplicates
+            )
         return self._client.populate_dataset(
             self.id,
             dataset_items,

tests/test_dataset.py

@@ -127,3 +127,19 @@ def test_dataset_list_autotags(CLIENT, dataset):
     # List of Autotags should be empty
     autotag_response = CLIENT.list_autotags(dataset.id)
     assert autotag_response == []
+
+
+def test_raises_error_for_duplicate():
+    fake_dataset = Dataset("fake", NucleusClient("fake"))
+    with pytest.raises(ValueError) as error:
+        fake_dataset.append(
+            [
+                DatasetItem("fake", "duplicate"),
+                DatasetItem("fake", "duplicate"),
+            ]
+        )
+    assert (
+        str(error.value)
+        == "Duplicate reference ids found among dataset_items:"
+        " {'duplicate': 'Count: 2'}"
+    )