handle strings size is bigger than 16mb (kevoreilly#2635)

Author: doomedraven

dev_utils/mongo_hooks.py

@@ -1,7 +1,10 @@
 import itertools
 import logging
+from contextlib import suppress
 
 from pymongo import UpdateOne, errors
+from pymongo.errors import InvalidDocument, BulkWriteError
+import bson
 
 from dev_utils.mongodb import (
     mongo_bulk_write,
@@ -61,13 +64,12 @@ def normalize_file(file_dict, task_id):
     )
     new_dict = {}
     for fld in static_fields:
-        try:
+        with suppress(KeyError):
             new_dict[fld] = file_dict.pop(fld)
-        except KeyError:
-            pass
 
     new_dict["_id"] = key
     file_dict[FILE_REF_KEY] = key
+
     return UpdateOne({"_id": key}, {"$set": new_dict, "$addToSet": {TASK_IDS_KEY: task_id}}, upsert=True, hint=[("_id", 1)])
 
 
@@ -87,8 +89,32 @@ def normalize_files(report):
     try:
         if requests:
             mongo_bulk_write(FILES_COLL, requests, ordered=False)
-    except errors.OperationFailure as exc:
-        log.error("Mongo hook 'normalize_files' failed with code %d: %s", exc.code, exc)
+    except (errors.OperationFailure, InvalidDocument, BulkWriteError) as exc:
+        log.warning("Mongo hook 'normalize_files' failed: %s. Attempting to sanitize strings and retry.", exc)
+        for req in requests:
+            # req._doc is the update document: {"$set": new_dict, ...}
+            # Accessing private attribute _doc to modify in place for retry
+            try:
+                if hasattr(req, "_doc") and "$set" in req._doc and "strings" in req._doc["$set"]:
+                    strings_val = req._doc["$set"]["strings"]
+                    # Check if strings field alone is too large (buffer safe 15MB)
+                    if strings_val and len(bson.encode({"strings": strings_val})) > 15 * 1024 * 1024:
+                        log.warning("Truncating oversized strings field for retry.")
+                        if isinstance(strings_val, list):
+                            req._doc["$set"]["strings"] = strings_val[:1000]
+                        else:
+                            req._doc["$set"]["strings"] = []
+                        # If still too large, clear it
+                        if len(bson.encode({"strings": req._doc["$set"]["strings"]})) > 15 * 1024 * 1024:
+                            req._doc["$set"]["strings"] = []
+            except Exception as e:
+                log.error("Failed to sanitize request during retry: %s", e)
+
+        # Retry the bulk write
+        try:
+            mongo_bulk_write(FILES_COLL, requests, ordered=False)
+        except Exception as retry_exc:
+            log.error("Retry of 'normalize_files' failed: %s", retry_exc)
 
     return report