gh-146311: Reject non-canonical padding bits in base32, 64, & 85 decoding

Doc/library/base64.rst

@@ -76,8 +76,8 @@ POST request.
       Added the *padded* and *wrapcol* parameters.
 
 
-.. function:: b64decode(s, altchars=None, validate=False, *, padded=True)
-              b64decode(s, altchars=None, validate=True, *, ignorechars, padded=True)
+.. function:: b64decode(s, altchars=None, validate=False, *, padded=True, canonical=False)
+              b64decode(s, altchars=None, validate=True, *, ignorechars, padded=True, canonical=False)
 
    Decode the Base64 encoded :term:`bytes-like object` or ASCII string
    *s* and return the decoded :class:`bytes`.
@@ -112,10 +112,13 @@ POST request.
    If *validate* is true, these non-alphabet characters in the input
    result in a :exc:`binascii.Error`.
 
+   If *canonical* is true, non-zero padding bits are rejected.
+   See :func:`binascii.a2b_base64` for details.
+
    For more information about the strict base64 check, see :func:`binascii.a2b_base64`
 
    .. versionchanged:: 3.15
-      Added the *ignorechars* and *padded* parameters.
+      Added the *ignorechars*, *padded*, and *canonical* parameters.
 
    .. deprecated:: 3.15
       Accepting the ``+`` and ``/`` characters with an alternative alphabet
@@ -179,7 +182,7 @@ POST request.
       Added the *padded* and *wrapcol* parameters.
 
 
-.. function:: b32decode(s, casefold=False, map01=None, *, padded=True, ignorechars=b'')
+.. function:: b32decode(s, casefold=False, map01=None, *, padded=True, ignorechars=b'', canonical=False)
 
    Decode the Base32 encoded :term:`bytes-like object` or ASCII string *s* and
    return the decoded :class:`bytes`.
@@ -203,12 +206,15 @@ POST request.
    *ignorechars* should be a :term:`bytes-like object` containing characters
    to ignore from the input.
 
+   If *canonical* is true, non-zero padding bits are rejected.
+   See :func:`binascii.a2b_base32` for details.
+
    A :exc:`binascii.Error` is raised if *s* is
    incorrectly padded or if there are non-alphabet characters present in the
    input.
 
    .. versionchanged:: next
-      Added the *ignorechars* and *padded* parameters.
+      Added the *ignorechars*, *padded*, and *canonical* parameters.
 
 
 .. function:: b32hexencode(s, *, padded=True, wrapcol=0)
@@ -222,7 +228,7 @@ POST request.
       Added the *padded* and *wrapcol* parameters.
 
 
-.. function:: b32hexdecode(s, casefold=False, *, padded=True, ignorechars=b'')
+.. function:: b32hexdecode(s, casefold=False, *, padded=True, ignorechars=b'', canonical=False)
 
    Similar to :func:`b32decode` but uses the Extended Hex Alphabet, as defined in
    :rfc:`4648`.
@@ -235,7 +241,7 @@ POST request.
    .. versionadded:: 3.10
 
    .. versionchanged:: next
-      Added the *ignorechars* and *padded* parameters.
+      Added the *ignorechars*, *padded*, and *canonical* parameters.
 
 
 .. function:: b16encode(s, *, wrapcol=0)
@@ -315,7 +321,7 @@ Refer to the documentation of the individual functions for more information.
    .. versionadded:: 3.4
 
 
-.. function:: a85decode(b, *, foldspaces=False, adobe=False, ignorechars=b' \t\n\r\v')
+.. function:: a85decode(b, *, foldspaces=False, adobe=False, ignorechars=b' \t\n\r\v', canonical=False)
 
    Decode the Ascii85 encoded :term:`bytes-like object` or ASCII string *b* and
    return the decoded :class:`bytes`.
@@ -332,8 +338,14 @@ Refer to the documentation of the individual functions for more information.
    This should only contain whitespace characters, and by
    default contains all whitespace characters in ASCII.
 
+   If *canonical* is true, non-canonical encodings are rejected.
+   See :func:`binascii.a2b_ascii85` for details.
+
    .. versionadded:: 3.4
 
+   .. versionchanged:: next
+      Added the *canonical* parameter.
+
 
 .. function:: b85encode(b, pad=False, *, wrapcol=0)
 
@@ -353,7 +365,7 @@ Refer to the documentation of the individual functions for more information.
       Added the *wrapcol* parameter.
 
 
-.. function:: b85decode(b, *, ignorechars=b'')
+.. function:: b85decode(b, *, ignorechars=b'', canonical=False)
 
    Decode the base85-encoded :term:`bytes-like object` or ASCII string *b* and
    return the decoded :class:`bytes`.  Padding is implicitly removed, if
@@ -362,10 +374,13 @@ Refer to the documentation of the individual functions for more information.
    *ignorechars* should be a :term:`bytes-like object` containing characters
    to ignore from the input.
 
+   If *canonical* is true, non-canonical encodings are rejected.
+   See :func:`binascii.a2b_base85` for details.
+
    .. versionadded:: 3.4
 
    .. versionchanged:: next
-      Added the *ignorechars* parameter.
+      Added the *ignorechars* and *canonical* parameters.
 
 
 .. function:: z85encode(s, pad=False, *, wrapcol=0)
@@ -390,7 +405,7 @@ Refer to the documentation of the individual functions for more information.
       Added the *wrapcol* parameter.
 
 
-.. function:: z85decode(s, *, ignorechars=b'')
+.. function:: z85decode(s, *, ignorechars=b'', canonical=False)
 
    Decode the Z85-encoded :term:`bytes-like object` or ASCII string *s* and
    return the decoded :class:`bytes`.  See `Z85  specification
@@ -399,10 +414,13 @@ Refer to the documentation of the individual functions for more information.
    *ignorechars* should be a :term:`bytes-like object` containing characters
    to ignore from the input.
 
+   If *canonical* is true, non-canonical encodings are rejected.
+   See :func:`binascii.a2b_base85` for details.
+
    .. versionadded:: 3.13
 
    .. versionchanged:: next
-      Added the *ignorechars* parameter.
+      Added the *ignorechars* and *canonical* parameters.
 
 
 .. _base64-legacy:

Doc/library/binascii.rst

@@ -48,8 +48,8 @@ The :mod:`!binascii` module defines the following functions:
       Added the *backtick* parameter.
 
 
-.. function:: a2b_base64(string, /, *, padded=True, alphabet=BASE64_ALPHABET, strict_mode=False)
-              a2b_base64(string, /, *, ignorechars, padded=True, alphabet=BASE64_ALPHABET, strict_mode=True)
+.. function:: a2b_base64(string, /, *, padded=True, alphabet=BASE64_ALPHABET, strict_mode=False, canonical=False)
+              a2b_base64(string, /, *, ignorechars, padded=True, alphabet=BASE64_ALPHABET, strict_mode=True, canonical=False)
 
    Convert a block of base64 data back to binary and return the binary data. More
    than one line may be passed at a time.
@@ -80,11 +80,15 @@ The :mod:`!binascii` module defines the following functions:
    * Contains no excess data after padding (including excess padding, newlines, etc.).
    * Does not start with a padding.
 
+   If *canonical* is true, non-zero padding bits in the last group are rejected
+   with :exc:`binascii.Error`, enforcing canonical encoding as defined in
+   :rfc:`4648` section 3.5.  This check is independent of *strict_mode*.
+
    .. versionchanged:: 3.11
       Added the *strict_mode* parameter.
 
    .. versionchanged:: 3.15
-      Added the *alphabet*, *ignorechars* and *padded* parameters.
+      Added the *alphabet*, *ignorechars*, *padded*, and *canonical* parameters.
 
 
 .. function:: b2a_base64(data, *, padded=True, alphabet=BASE64_ALPHABET, wrapcol=0, newline=True)
@@ -110,7 +114,7 @@ The :mod:`!binascii` module defines the following functions:
       Added the *alphabet*, *padded* and *wrapcol* parameters.
 
 
-.. function:: a2b_ascii85(string, /, *, foldspaces=False, adobe=False, ignorechars=b'')
+.. function:: a2b_ascii85(string, /, *, foldspaces=False, adobe=False, ignorechars=b'', canonical=False)
 
    Convert Ascii85 data back to binary and return the binary data.
 
@@ -132,6 +136,11 @@ The :mod:`!binascii` module defines the following functions:
    to ignore from the input.
    This should only contain whitespace characters.
 
+   If *canonical* is true, non-canonical encodings in the final group are
+   rejected with :exc:`binascii.Error`.  This includes single-character
+   final groups (which no conforming encoder produces) and final groups whose
+   padding digits are not what the encoder would produce.
+
    Invalid Ascii85 data will raise :exc:`binascii.Error`.
 
    .. versionadded:: 3.15
@@ -160,7 +169,7 @@ The :mod:`!binascii` module defines the following functions:
    .. versionadded:: 3.15
 
 
-.. function:: a2b_base85(string, /, *, alphabet=BASE85_ALPHABET, ignorechars=b'')
+.. function:: a2b_base85(string, /, *, alphabet=BASE85_ALPHABET, ignorechars=b'', canonical=False)
 
    Convert Base85 data back to binary and return the binary data.
    More than one line may be passed at a time.
@@ -176,6 +185,11 @@ The :mod:`!binascii` module defines the following functions:
    *ignorechars* should be a :term:`bytes-like object` containing characters
    to ignore from the input.
 
+   If *canonical* is true, non-canonical encodings in the final group are
+   rejected with :exc:`binascii.Error`.  This includes single-character
+   final groups (which no conforming encoder produces) and final groups whose
+   padding digits are not what the encoder would produce.
+
    Invalid Base85 data will raise :exc:`binascii.Error`.
 
    .. versionadded:: 3.15
@@ -199,7 +213,7 @@ The :mod:`!binascii` module defines the following functions:
    .. versionadded:: 3.15
 
 
-.. function:: a2b_base32(string, /, *, padded=True, alphabet=BASE32_ALPHABET, ignorechars=b'')
+.. function:: a2b_base32(string, /, *, padded=True, alphabet=BASE32_ALPHABET, ignorechars=b'', canonical=False)
 
    Convert base32 data back to binary and return the binary data.
 
@@ -228,6 +242,10 @@ The :mod:`!binascii` module defines the following functions:
    presented before the end of the encoded data and the excess pad characters
    will be ignored.
 
+   If *canonical* is true, non-zero padding bits in the last group are rejected
+   with :exc:`binascii.Error`, enforcing canonical encoding as defined in
+   :rfc:`4648` section 3.5.
+
    Invalid base32 data will raise :exc:`binascii.Error`.
 
    .. versionadded:: next

Include/internal/pycore_global_strings.h

@@ -359,6 +359,7 @@ struct _Py_global_strings {
         STRUCT_FOR_ID(callable)
         STRUCT_FOR_ID(callback)
         STRUCT_FOR_ID(cancel)
+        STRUCT_FOR_ID(canonical)
         STRUCT_FOR_ID(capath)
         STRUCT_FOR_ID(capitals)
         STRUCT_FOR_ID(category)

Lib/base64.py

@@ -68,7 +68,7 @@ def b64encode(s, altchars=None, *, padded=True, wrapcol=0):
 
 
 def b64decode(s, altchars=None, validate=_NOT_SPECIFIED,
-              *, padded=True, ignorechars=_NOT_SPECIFIED):
+              *, padded=True, ignorechars=_NOT_SPECIFIED, canonical=False):
     """Decode the Base64 encoded bytes-like object or ASCII string s.
 
     Optional altchars must be a bytes-like object or ASCII string of length 2
@@ -110,11 +110,13 @@ def b64decode(s, altchars=None, validate=_NOT_SPECIFIED,
             alphabet = binascii.BASE64_ALPHABET[:-2] + altchars
             return binascii.a2b_base64(s, strict_mode=validate,
                                        alphabet=alphabet,
-                                       padded=padded, ignorechars=ignorechars)
+                                       padded=padded, ignorechars=ignorechars,
+                                       canonical=canonical)
     if ignorechars is _NOT_SPECIFIED:
         ignorechars = b''
     result = binascii.a2b_base64(s, strict_mode=validate,
-                                 padded=padded, ignorechars=ignorechars)
+                                 padded=padded, ignorechars=ignorechars,
+                                 canonical=canonical)
     if badchar is not None:
         import warnings
         if validate:
@@ -230,7 +232,8 @@ def b32encode(s, *, padded=True, wrapcol=0):
     return binascii.b2a_base32(s, padded=padded, wrapcol=wrapcol)
 b32encode.__doc__ = _B32_ENCODE_DOCSTRING.format(encoding='base32')
 
-def b32decode(s, casefold=False, map01=None, *, padded=True, ignorechars=b''):
+def b32decode(s, casefold=False, map01=None, *, padded=True, ignorechars=b'',
+              canonical=False):
     s = _bytes_from_decode_data(s)
     # Handle section 2.4 zero and one mapping.  The flag map01 will be either
     # False, or the character to map the digit 1 (one) to.  It should be
@@ -241,7 +244,8 @@ def b32decode(s, casefold=False, map01=None, *, padded=True, ignorechars=b''):
         s = s.translate(bytes.maketrans(b'01', b'O' + map01))
     if casefold:
         s = s.upper()
-    return binascii.a2b_base32(s, padded=padded, ignorechars=ignorechars)
+    return binascii.a2b_base32(s, padded=padded, ignorechars=ignorechars,
+                               canonical=canonical)
 b32decode.__doc__ = _B32_DECODE_DOCSTRING.format(encoding='base32',
                                         extra_args=_B32_DECODE_MAP01_DOCSTRING)
 
@@ -250,13 +254,15 @@ def b32hexencode(s, *, padded=True, wrapcol=0):
                                alphabet=binascii.BASE32HEX_ALPHABET)
 b32hexencode.__doc__ = _B32_ENCODE_DOCSTRING.format(encoding='base32hex')
 
-def b32hexdecode(s, casefold=False, *, padded=True, ignorechars=b''):
+def b32hexdecode(s, casefold=False, *, padded=True, ignorechars=b'',
+                 canonical=False):
     s = _bytes_from_decode_data(s)
     # base32hex does not have the 01 mapping
     if casefold:
         s = s.upper()
     return binascii.a2b_base32(s, alphabet=binascii.BASE32HEX_ALPHABET,
-                               padded=padded, ignorechars=ignorechars)
+                               padded=padded, ignorechars=ignorechars,
+                               canonical=canonical)
 b32hexdecode.__doc__ = _B32_DECODE_DOCSTRING.format(encoding='base32hex',
                                                     extra_args='')
 
@@ -324,7 +330,8 @@ def a85encode(b, *, foldspaces=False, wrapcol=0, pad=False, adobe=False):
     return binascii.b2a_ascii85(b, foldspaces=foldspaces,
                                 adobe=adobe, wrapcol=wrapcol, pad=pad)
 
-def a85decode(b, *, foldspaces=False, adobe=False, ignorechars=b' \t\n\r\v'):
+def a85decode(b, *, foldspaces=False, adobe=False, ignorechars=b' \t\n\r\v',
+              canonical=False):
     """Decode the Ascii85 encoded bytes-like object or ASCII string b.
 
     foldspaces is a flag that specifies whether the 'y' short sequence should be
@@ -338,10 +345,13 @@ def a85decode(b, *, foldspaces=False, adobe=False, ignorechars=b' \t\n\r\v'):
     input. This should only contain whitespace characters, and by default
     contains all whitespace characters in ASCII.
 
+    If canonical is true, non-canonical encodings are rejected.
+
     The result is returned as a bytes object.
     """
     return binascii.a2b_ascii85(b, foldspaces=foldspaces,
-                                adobe=adobe, ignorechars=ignorechars)
+                                adobe=adobe, ignorechars=ignorechars,
+                                canonical=canonical)
 
 def b85encode(b, pad=False, *, wrapcol=0):
     """Encode bytes-like object b in base85 format and return a bytes object.
@@ -354,12 +364,15 @@ def b85encode(b, pad=False, *, wrapcol=0):
     """
     return binascii.b2a_base85(b, wrapcol=wrapcol, pad=pad)
 
-def b85decode(b, *, ignorechars=b''):
+def b85decode(b, *, ignorechars=b'', canonical=False):
     """Decode the base85-encoded bytes-like object or ASCII string b
 
+    If canonical is true, non-canonical encodings are rejected.
+
     The result is returned as a bytes object.
     """
-    return binascii.a2b_base85(b, ignorechars=ignorechars)
+    return binascii.a2b_base85(b, ignorechars=ignorechars,
+                               canonical=canonical)
 
 def z85encode(s, pad=False, *, wrapcol=0):
     """Encode bytes-like object b in z85 format and return a bytes object.
@@ -373,12 +386,15 @@ def z85encode(s, pad=False, *, wrapcol=0):
     return binascii.b2a_base85(s, wrapcol=wrapcol, pad=pad,
                                alphabet=binascii.Z85_ALPHABET)
 
-def z85decode(s, *, ignorechars=b''):
+def z85decode(s, *, ignorechars=b'', canonical=False):
     """Decode the z85-encoded bytes-like object or ASCII string b
 
+    If canonical is true, non-canonical encodings are rejected.
+
     The result is returned as a bytes object.
     """
-    return binascii.a2b_base85(s, alphabet=binascii.Z85_ALPHABET, ignorechars=ignorechars)
+    return binascii.a2b_base85(s, alphabet=binascii.Z85_ALPHABET,
+                               ignorechars=ignorechars, canonical=canonical)
 
 # Legacy interface.  This code could be cleaned up since I don't believe
 # binascii has any line length limitations.  It just doesn't seem worth it