tqdm when deduplicating

Author: thiswillbeyourgithub

WDoc/utils/batch_file_loader.py

@@ -413,7 +413,7 @@ def deterministic_sorter(doc_dict: DocDict) -> int:
         dupes = set()
         deduped = {}
         [dupes.add(ch) for ch in content_hash if content_hash.count(ch) > 1]
-        for idoc, doc in enumerate(docs):
+        for idoc, doc in enumerate(tqdm(docs, desc="Deduplicating", unit="doc")):
             ch = doc.metadata["content_hash"]
             if not dupes:
                 break