Merge pull request #9790 from ThomasWaldmann/support-msgpack121-1.4

support msgpack 1.2.1

Author: ThomasWaldmann

docs/changes.rst

@@ -473,7 +473,7 @@ Note: Many of the fixed issues listed below relate to rather rare or theoretical
 
 Other changes:
 
-- msgpack: allow 1.2.0
+- msgpack: also allow 1.2.0 and 1.2.1
 - use F_FULLFSYNC on macOS for SyncFile data durability, #9383
 - mount: improve error msg when uid/gid cannot be resolved, #9574
 - docs:

pyproject.toml

@@ -35,7 +35,7 @@ dependencies = [
     # Please note:
     # Using any other msgpack version is not supported by Borg development and
     # any feedback related to issues caused by this will be ignored.
-    "msgpack >=1.0.3, <=1.2.0",
+    "msgpack >=1.0.3, <=1.2.1",
     "packaging",
 ]
 

src/borg/archive.py

@@ -1035,22 +1035,35 @@ def chunk_decref(id, stats, part=False):
                 data = self.key.decrypt(items_id, data)
                 unpacker.feed(data)
                 chunk_decref(items_id, stats)
-                try:
-                    for item in unpacker:
+                while True:
+                    try:
+                        item = next(unpacker)
+                    except StopIteration:
+                        # no more (complete) items in the buffer, feed the next chunk
+                        break
+                    except msgpack.UnpackException:
+                        # items metadata corrupted. the unpacker can't be reused after an
+                        # unpacking failure, so create a fresh one and skip the rest of this chunk.
+                        if forced == 0:
+                            raise
+                        error = True
+                        unpacker = msgpack.Unpacker(use_list=False)
+                        break
+                    try:
                         item = Item(internal_dict=item)
                         if 'chunks' in item:
                             part = not self.consider_part_files and 'part' in item
                             for chunk_id, size, csize in item.chunks:
                                 chunk_decref(chunk_id, stats, part=part)
-                except (TypeError, ValueError):
-                    # if items metadata spans multiple chunks and one chunk got dropped somehow,
-                    # it could be that unpacker yields bad types
-                    if forced == 0:
-                        raise
-                    error = True
+                    except (TypeError, ValueError):
+                        # if items metadata spans multiple chunks and one chunk got dropped somehow,
+                        # it could be that unpacker yields bad types
+                        if forced == 0:
+                            raise
+                        error = True
             if progress:
                 pi.finish()
-        except (msgpack.UnpackException, Repository.ObjectNotFound):
+        except Repository.ObjectNotFound:
             # items metadata corrupted
             if forced == 0:
                 raise

src/borg/helpers/msgpack.py

@@ -145,7 +145,7 @@ def is_supported_msgpack():
     version_check = os.environ.get('BORG_MSGPACK_VERSION_CHECK', 'yes').strip().lower()
 
     return version_check == 'no' or (
-        (1, 0, 3) <= msgpack.version[:3] <= (1, 2, 0) and
+        (1, 0, 3) <= msgpack.version[:3] <= (1, 2, 1) and
         msgpack.version not in []
     )