Add Ascii85 and base85 support to binascii

Add Ascii85 and base85 encoders and decoders to `binascii` and four new
functions, `binascii.a2b_ascii85()`, `a2b_base85()`, `b2a_ascii85()`,
and `b2a_base85()`.

These replace the existing implementations in `base64.a85encode()`,
`b85encode()`, `a85decode()`, and `b85encode()`. Performance is greatly
improved, and memory usage is now constant instead of linear.

No API or documentation changes are necessary with respect to
`base64.a85encode()`, `b85encode()`, etc., and all existing unit tests
for those functions continue to pass without modification.

Note that attempting to decode Ascii85 or base85 data of length 1 mod 5
(after accounting for Ascii85 quirks) now produces an error, as no
encoder would emit such data. This should be the only significant
externally visible difference compared to the old implementation.

Resolves: gh-101178

Author: kangtastic

Doc/library/binascii.rst

@@ -77,6 +77,73 @@ The :mod:`binascii` module defines the following functions:
       Added the *newline* parameter.
 
 
+.. function:: a2b_ascii85(string, /, *, fold_spaces=False, wrap=False, ignore=b"")
+
+   Convert Ascii85 data back to binary and return the binary data.
+
+   Valid Ascii85 data contains characters from the Ascii85 alphabet in groups
+   of five (except for the final group, which may have from two to five
+   characters). Each group encodes 32 bits of binary data in the range from
+   ``0`` to ``2 ** 32 - 1``, inclusive. The special character ``z`` is
+   accepted as a short form of the group ``!!!!!``, which encodes four
+   consecutive null bytes.
+
+   If *fold_spaces* is true, the special character ``y`` is also accepted as a
+   short form of the group ``+<VdL``, which encodes four consecutive spaces.
+   Note that neither short form is permitted if it occurs in the middle of
+   another group.
+
+   If *wrap* is true, the input begins with ``<~`` and ends with ``~>``, as in
+   the Adobe Ascii85 format.
+
+   *ignore* is an optional bytes-like object that specifies characters to
+   ignore in the input.
+
+   Invalid Ascii85 data will raise :exc:`binascii.Error`.
+
+
+.. function:: b2a_ascii85(data, /, *, fold_spaces=False, wrap=False, width=0, pad=False)
+
+   Convert binary data to a formatted sequence of ASCII characters in Ascii85
+   coding. The return value is the converted data.
+
+   If *fold_spaces* is true, four consecutive spaces are encoded as the
+   special character ``y`` instead of the sequence ``+<VdL``.
+
+   If *wrap* is true, the output begins with ``<~`` and ends with ``~>``, as
+   in the Adobe Ascii85 format.
+
+   If *width* is provided and greater than 0, the output is split into lines
+   of no more than the specified width separated by the ASCII newline
+   character.
+
+   If *pad* is true, the input is padded to a multiple of 4 before encoding.
+
+
+.. function:: a2b_base85(string, /, *, strict_mode=False)
+
+   Convert base85 data back to binary and return the binary data.
+   More than one line may be passed at a time.
+
+   If *strict_mode* is true, only valid base85 data will be converted.
+   Invalid base85 data will raise :exc:`binascii.Error`.
+
+   Valid base85 data contains characters from the base85 alphabet in groups
+   of five (except for the final group, which may have from two to five
+   characters). Each group encodes 32 bits of binary data in the range from
+   ``0`` to ``2 ** 32 - 1``, inclusive.
+
+
+.. function:: b2a_base85(data, /, *, pad=False, newline=True)
+
+   Convert binary data to a line of ASCII characters in base85 coding.
+   The return value is the converted line.
+
+   If *pad* is true, the input is padded to a multiple of 4 before encoding.
+
+   If *newline* is true, a newline char is appended to the result.
+
+
 .. function:: a2b_qp(data, header=False)
 
    Convert a block of quoted-printable data back to binary and return the binary

Lib/base64.py

@@ -295,36 +295,6 @@ def b16decode(s, casefold=False):
 #
 # Ascii85 encoding/decoding
 #
-
-_a85chars = None
-_a85chars2 = None
-_A85START = b"<~"
-_A85END = b"~>"
-
-def _85encode(b, chars, chars2, pad=False, foldnuls=False, foldspaces=False):
-    # Helper function for a85encode and b85encode
-    if not isinstance(b, bytes_types):
-        b = memoryview(b).tobytes()
-
-    padding = (-len(b)) % 4
-    if padding:
-        b = b + b'\0' * padding
-    words = struct.Struct('!%dI' % (len(b) // 4)).unpack(b)
-
-    chunks = [b'z' if foldnuls and not word else
-              b'y' if foldspaces and word == 0x20202020 else
-              (chars2[word // 614125] +
-               chars2[word // 85 % 7225] +
-               chars[word % 85])
-              for word in words]
-
-    if padding and not pad:
-        if chunks[-1] == b'z':
-            chunks[-1] = chars[0] * 5
-        chunks[-1] = chunks[-1][:-padding]
-
-    return b''.join(chunks)
-
 def a85encode(b, *, foldspaces=False, wrapcol=0, pad=False, adobe=False):
     """Encode bytes-like object b using Ascii85 and return a bytes object.
 
@@ -342,29 +312,8 @@ def a85encode(b, *, foldspaces=False, wrapcol=0, pad=False, adobe=False):
     adobe controls whether the encoded byte sequence is framed with <~ and ~>,
     which is used by the Adobe implementation.
     """
-    global _a85chars, _a85chars2
-    # Delay the initialization of tables to not waste memory
-    # if the function is never called
-    if _a85chars2 is None:
-        _a85chars = [bytes((i,)) for i in range(33, 118)]
-        _a85chars2 = [(a + b) for a in _a85chars for b in _a85chars]
-
-    result = _85encode(b, _a85chars, _a85chars2, pad, True, foldspaces)
-
-    if adobe:
-        result = _A85START + result
-    if wrapcol:
-        wrapcol = max(2 if adobe else 1, wrapcol)
-        chunks = [result[i: i + wrapcol]
-                  for i in range(0, len(result), wrapcol)]
-        if adobe:
-            if len(chunks[-1]) + 2 > wrapcol:
-                chunks.append(b'')
-        result = b'\n'.join(chunks)
-    if adobe:
-        result += _A85END
-
-    return result
+    return binascii.b2a_ascii85(b, fold_spaces=foldspaces,
+                                wrap=adobe, width=wrapcol, pad=pad)
 
 def a85decode(b, *, foldspaces=False, adobe=False, ignorechars=b' \t\n\r\v'):
     """Decode the Ascii85 encoded bytes-like object or ASCII string b.
@@ -383,121 +332,24 @@ def a85decode(b, *, foldspaces=False, adobe=False, ignorechars=b' \t\n\r\v'):
     The result is returned as a bytes object.
     """
     b = _bytes_from_decode_data(b)
-    if adobe:
-        if not b.endswith(_A85END):
-            raise ValueError(
-                "Ascii85 encoded byte sequences must end "
-                "with {!r}".format(_A85END)
-                )
-        if b.startswith(_A85START):
-            b = b[2:-2]  # Strip off start/end markers
-        else:
-            b = b[:-2]
-    #
-    # We have to go through this stepwise, so as to ignore spaces and handle
-    # special short sequences
-    #
-    packI = struct.Struct('!I').pack
-    decoded = []
-    decoded_append = decoded.append
-    curr = []
-    curr_append = curr.append
-    curr_clear = curr.clear
-    for x in b + b'u' * 4:
-        if b'!'[0] <= x <= b'u'[0]:
-            curr_append(x)
-            if len(curr) == 5:
-                acc = 0
-                for x in curr:
-                    acc = 85 * acc + (x - 33)
-                try:
-                    decoded_append(packI(acc))
-                except struct.error:
-                    raise ValueError('Ascii85 overflow') from None
-                curr_clear()
-        elif x == b'z'[0]:
-            if curr:
-                raise ValueError('z inside Ascii85 5-tuple')
-            decoded_append(b'\0\0\0\0')
-        elif foldspaces and x == b'y'[0]:
-            if curr:
-                raise ValueError('y inside Ascii85 5-tuple')
-            decoded_append(b'\x20\x20\x20\x20')
-        elif x in ignorechars:
-            # Skip whitespace
-            continue
-        else:
-            raise ValueError('Non-Ascii85 digit found: %c' % x)
-
-    result = b''.join(decoded)
-    padding = 4 - len(curr)
-    if padding:
-        # Throw away the extra padding
-        result = result[:-padding]
-    return result
-
-# The following code is originally taken (with permission) from Mercurial
-
-_b85alphabet = (b"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
-                b"abcdefghijklmnopqrstuvwxyz!#$%&()*+-;<=>?@^_`{|}~")
-_b85chars = None
-_b85chars2 = None
-_b85dec = None
+    return binascii.a2b_ascii85(b, fold_spaces=foldspaces,
+                                wrap=adobe, ignore=ignorechars)
 
 def b85encode(b, pad=False):
     """Encode bytes-like object b in base85 format and return a bytes object.
 
     If pad is true, the input is padded with b'\\0' so its length is a multiple of
     4 bytes before encoding.
     """
-    global _b85chars, _b85chars2
-    # Delay the initialization of tables to not waste memory
-    # if the function is never called
-    if _b85chars2 is None:
-        _b85chars = [bytes((i,)) for i in _b85alphabet]
-        _b85chars2 = [(a + b) for a in _b85chars for b in _b85chars]
-    return _85encode(b, _b85chars, _b85chars2, pad)
+    return binascii.b2a_base85(b, pad=pad, newline=False)
 
 def b85decode(b):
     """Decode the base85-encoded bytes-like object or ASCII string b
 
     The result is returned as a bytes object.
     """
-    global _b85dec
-    # Delay the initialization of tables to not waste memory
-    # if the function is never called
-    if _b85dec is None:
-        _b85dec = [None] * 256
-        for i, c in enumerate(_b85alphabet):
-            _b85dec[c] = i
-
     b = _bytes_from_decode_data(b)
-    padding = (-len(b)) % 5
-    b = b + b'~' * padding
-    out = []
-    packI = struct.Struct('!I').pack
-    for i in range(0, len(b), 5):
-        chunk = b[i:i + 5]
-        acc = 0
-        try:
-            for c in chunk:
-                acc = acc * 85 + _b85dec[c]
-        except TypeError:
-            for j, c in enumerate(chunk):
-                if _b85dec[c] is None:
-                    raise ValueError('bad base85 character at position %d'
-                                    % (i + j)) from None
-            raise
-        try:
-            out.append(packI(acc))
-        except struct.error:
-            raise ValueError('base85 overflow in hunk starting at byte %d'
-                             % i) from None
-
-    result = b''.join(out)
-    if padding:
-        result = result[:-padding]
-    return result
+    return binascii.a2b_base85(b, strict_mode=True)
 
 # Legacy interface.  This code could be cleaned up since I don't believe
 # binascii has any line length limitations.  It just doesn't seem worth it