Merge pull request #31 from thegushi/fix/body-line-iterator-decode

Fix body_line_iterator(decode=True) returning no lines in Python 3.11

Author: jaredmauch

Mailman/Bouncers/AOL.py

@@ -29,7 +29,10 @@ def process(msg):
         return
     addrs = []
     found = False
-    for line in msg.get_payload(decode=True).splitlines():
+    payload = msg.get_payload(decode=True)
+    if isinstance(payload, bytes):
+        payload = payload.decode('us-ascii', errors='replace')
+    for line in payload.splitlines():
         if scre.search(line):
             found = True
             continue

Mailman/Bouncers/SimpleMatch.py

@@ -17,10 +17,30 @@
 
 """Recognizes simple heuristically delimited bounces."""
 
+import io
 import re
 import email.iterators
 
 
+def _body_line_iterator(msg):
+    """Iterate body lines with Content-Transfer-Encoding properly decoded.
+
+    Python 3's body_line_iterator only yields str payloads; get_payload(
+    decode=True) returns bytes (CTE-decoded). This helper bridges the gap so
+    quoted-printable and base64 bodies are decoded before pattern matching.
+    """
+    for subpart in email.iterators.typed_subpart_iterator(msg):
+        payload = subpart.get_payload(decode=True)
+        if not isinstance(payload, bytes):
+            continue
+        charset = subpart.get_content_charset('us-ascii') or 'us-ascii'
+        try:
+            text = payload.decode(charset, errors='replace')
+        except LookupError:
+            text = payload.decode('us-ascii', errors='replace')
+        yield from io.StringIO(text)
+
+
 
 def _c(pattern):
     return re.compile(pattern, re.IGNORECASE)
@@ -224,7 +244,7 @@ def process(msg, patterns=None):
     # we process the message multiple times anyway.
     for scre, ecre, acre in patterns:
         state = 0
-        for line in email.iterators.body_line_iterator(msg, decode=True):
+        for line in _body_line_iterator(msg):
             if state == 0:
                 if scre.search(line):
                     state = 1

Mailman/Bouncers/SimpleWarning.py

@@ -21,7 +21,7 @@
 import email.iterators
 
 from Mailman.Bouncers.BouncerAPI import Stop
-from Mailman.Bouncers.SimpleMatch import _c
+from Mailman.Bouncers.SimpleMatch import _c, _body_line_iterator
 
 
 
@@ -75,7 +75,7 @@ def process(msg):
     addrs = {}
     for scre, ecre, acre in patterns:
         state = 0
-        for line in email.iterators.body_line_iterator(msg, decode=True):
+        for line in _body_line_iterator(msg):
             if state == 0:
                 if scre.search(line):
                     state = 1

Mailman/Handlers/Tagger.py

@@ -26,6 +26,7 @@
 from email.header import decode_header
 
 from Mailman import Utils
+from Mailman.Bouncers.SimpleMatch import _body_line_iterator
 from Mailman.Logging.Syslog import syslog
 from Mailman.Handlers.CookHeaders import change_header
 
@@ -97,7 +98,7 @@ def scanbody(msg, numlines=None):
     # the first numlines of body text.
     lines = []
     lineno = 0
-    reader = list(email.iterators.body_line_iterator(msg, decode=True))
+    reader = list(_body_line_iterator(msg))
     while numlines is None or lineno < numlines:
         try:
             line = reader.pop(0)