Skip to content

Commit 8483948

Browse files
mayeutmayeut
committed
decoding
1 parent 682bda4 commit 8483948

3 files changed

Lines changed: 58 additions & 92 deletions

File tree

‎README.rst‎

Lines changed: 42 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -87,66 +87,67 @@ Benchmark
8787

8888
.. begin benchmark
8989
90-
Running Python 3.15.0a8, Apple clang version 21.0.0 (clang-2100.0.123.102), macOS 26.3.1, Apple M1 Max
90+
Running Python 3.15.0b3, Apple clang version 21.0.0 (clang-2100.1.1.101), macOS 26.5.2, Apple M1 Max
9191

9292
.. code::
9393
9494
pybase64 1.5.0 (C extension active - NEON)
9595
bench: altchars=None, validate=True, padded=True
96-
pybase64.encodebytes: 5129 MB/s (135,696 bytes -> 183,309 bytes)
97-
pybase64.b64encode: 16339 MB/s (135,696 bytes -> 180,928 bytes)
98-
pybase64.b64decode: 8821 MB/s (180,928 bytes -> 135,696 bytes)
99-
base64.encodebytes: 2339 MB/s (135,696 bytes -> 183,309 bytes)
100-
base64.b64encode: 2642 MB/s (135,696 bytes -> 180,928 bytes)
101-
base64.b64decode: 2613 MB/s (180,928 bytes -> 135,696 bytes)
96+
pybase64.encodebytes: 6518 MB/s (135,696 bytes -> 183,309 bytes)
97+
pybase64.b64encode: 17518 MB/s (135,696 bytes -> 180,928 bytes)
98+
pybase64.b64decode: 9016 MB/s (180,928 bytes -> 135,696 bytes)
99+
base64.encodebytes: 2372 MB/s (135,696 bytes -> 183,309 bytes)
100+
base64.b64encode: 2662 MB/s (135,696 bytes -> 180,928 bytes)
101+
base64.b64decode: 2635 MB/s (180,928 bytes -> 135,696 bytes)
102102
bench: altchars=None, validate=True, padded=False
103-
pybase64.b64encode: 16369 MB/s (135,696 bytes -> 180,928 bytes)
104-
pybase64.b64decode: 2480 MB/s (180,928 bytes -> 135,696 bytes)
105-
base64.b64encode: 2573 MB/s (135,696 bytes -> 180,928 bytes)
106-
base64.b64decode: 2583 MB/s (180,928 bytes -> 135,696 bytes)
103+
pybase64.b64encode: 17528 MB/s (135,696 bytes -> 180,928 bytes)
104+
pybase64.b64decode: 8984 MB/s (180,928 bytes -> 135,696 bytes)
105+
base64.b64encode: 2639 MB/s (135,696 bytes -> 180,928 bytes)
106+
base64.b64decode: 2646 MB/s (180,928 bytes -> 135,696 bytes)
107107
bench: altchars=None, ignorechars=b'', padded=False
108-
pybase64.b64decode: 2478 MB/s (180,928 bytes -> 135,696 bytes)
109-
base64.b64decode: 2610 MB/s (180,928 bytes -> 135,696 bytes)
108+
pybase64.b64decode: 8975 MB/s (180,928 bytes -> 135,696 bytes)
109+
base64.b64decode: 2619 MB/s (180,928 bytes -> 135,696 bytes)
110110
bench: altchars=None, ignorechars=b'\n', padded=True
111-
pybase64.b64decode: 2338 MB/s (183,308 bytes -> 135,696 bytes)
112-
base64.b64decode: 2083 MB/s (183,308 bytes -> 135,696 bytes)
111+
pybase64.b64decode: 2357 MB/s (183,308 bytes -> 135,696 bytes)
112+
base64.b64decode: 2073 MB/s (183,308 bytes -> 135,696 bytes)
113113
bench: altchars=None, ignorechars=b'\n', padded=False
114-
pybase64.b64decode: 2337 MB/s (183,308 bytes -> 135,696 bytes)
115-
base64.b64decode: 2013 MB/s (183,308 bytes -> 135,696 bytes)
114+
pybase64.b64decode: 2357 MB/s (183,308 bytes -> 135,696 bytes)
115+
base64.b64decode: 1932 MB/s (183,308 bytes -> 135,696 bytes)
116116
bench: altchars=None, validate=False, padded=True
117-
pybase64.b64decode: 2356 MB/s (183,308 bytes -> 135,696 bytes)
118-
base64.b64decode: 2156 MB/s (183,308 bytes -> 135,696 bytes)
117+
pybase64.b64decode: 2360 MB/s (183,308 bytes -> 135,696 bytes)
118+
base64.b64decode: 2021 MB/s (183,308 bytes -> 135,696 bytes)
119119
bench: altchars=None, validate=False, padded=False
120-
pybase64.b64decode: 2332 MB/s (183,308 bytes -> 135,696 bytes)
121-
base64.b64decode: 2079 MB/s (183,308 bytes -> 135,696 bytes)
120+
pybase64.b64decode: 2359 MB/s (183,308 bytes -> 135,696 bytes)
121+
base64.b64decode: 1976 MB/s (183,308 bytes -> 135,696 bytes)
122122
bench: altchars=b'-_', validate=True, padded=True
123-
pybase64.b64encode: 10328 MB/s (135,696 bytes -> 180,928 bytes)
124-
pybase64.b64decode: 5861 MB/s (180,928 bytes -> 135,696 bytes)
125-
base64.b64encode: 2617 MB/s (135,696 bytes -> 180,928 bytes)
126-
base64.b64decode: 1183 MB/s (180,928 bytes -> 135,696 bytes)
123+
pybase64.b64encode: 10733 MB/s (135,696 bytes -> 180,928 bytes)
124+
pybase64.b64decode: 5968 MB/s (180,928 bytes -> 135,696 bytes)
125+
base64.b64encode: 2653 MB/s (135,696 bytes -> 180,928 bytes)
126+
base64.b64decode: 1194 MB/s (180,928 bytes -> 135,696 bytes)
127127
bench: altchars=b'-_', validate=True, padded=False
128-
pybase64.b64encode: 10344 MB/s (135,696 bytes -> 180,928 bytes)
129-
pybase64.b64decode: 2163 MB/s (180,928 bytes -> 135,696 bytes)
130-
base64.b64encode: 2625 MB/s (135,696 bytes -> 180,928 bytes)
131-
base64.b64decode: 1164 MB/s (180,928 bytes -> 135,696 bytes)
128+
pybase64.b64encode: 10714 MB/s (135,696 bytes -> 180,928 bytes)
129+
pybase64.b64decode: 5941 MB/s (180,928 bytes -> 135,696 bytes)
130+
base64.b64encode: 2618 MB/s (135,696 bytes -> 180,928 bytes)
131+
base64.b64decode: 1161 MB/s (180,928 bytes -> 135,696 bytes)
132132
bench: altchars=b'-_', ignorechars=b'', padded=True
133-
pybase64.b64decode: 5815 MB/s (180,928 bytes -> 135,696 bytes)
134-
base64.b64decode: 2598 MB/s (180,928 bytes -> 135,696 bytes)
133+
pybase64.b64decode: 5839 MB/s (180,928 bytes -> 135,696 bytes)
134+
base64.b64decode: 2590 MB/s (180,928 bytes -> 135,696 bytes)
135135
bench: altchars=b'-_', ignorechars=b'', padded=False
136-
pybase64.b64decode: 2146 MB/s (180,928 bytes -> 135,696 bytes)
137-
base64.b64decode: 2586 MB/s (180,928 bytes -> 135,696 bytes)
136+
pybase64.b64decode: 5807 MB/s (180,928 bytes -> 135,696 bytes)
137+
base64.b64decode: 2588 MB/s (180,928 bytes -> 135,696 bytes)
138138
bench: altchars=b'-_', ignorechars=b'\n', padded=True
139-
pybase64.b64decode: 2042 MB/s (183,308 bytes -> 135,696 bytes)
140-
base64.b64decode: 2056 MB/s (183,308 bytes -> 135,696 bytes)
139+
pybase64.b64decode: 2044 MB/s (183,308 bytes -> 135,696 bytes)
140+
base64.b64decode: 2044 MB/s (183,308 bytes -> 135,696 bytes)
141141
bench: altchars=b'-_', ignorechars=b'\n', padded=False
142-
pybase64.b64decode: 2034 MB/s (183,308 bytes -> 135,696 bytes)
143-
base64.b64decode: 1952 MB/s (183,308 bytes -> 135,696 bytes)
142+
pybase64.b64decode: 1978 MB/s (183,308 bytes -> 135,696 bytes)
143+
base64.b64decode: 1872 MB/s (183,308 bytes -> 135,696 bytes)
144144
bench: altchars=b'-_', validate=False, padded=True
145-
pybase64.b64decode: 2021 MB/s (183,308 bytes -> 135,696 bytes)
146-
base64.b64decode: 1063 MB/s (183,308 bytes -> 135,696 bytes)
145+
pybase64.b64decode: 2013 MB/s (183,308 bytes -> 135,696 bytes)
146+
base64.b64decode: 1014 MB/s (183,308 bytes -> 135,696 bytes)
147147
bench: altchars=b'-_', validate=False, padded=False
148-
pybase64.b64decode: 2014 MB/s (183,308 bytes -> 135,696 bytes)
149-
base64.b64decode: 1042 MB/s (183,308 bytes -> 135,696 bytes)
148+
pybase64.b64decode: 2025 MB/s (183,308 bytes -> 135,696 bytes)
149+
base64.b64decode: 1000 MB/s (183,308 bytes -> 135,696 bytes)
150+
150151
151152
.. end benchmark
152153

‎src/pybase64/_pybase64.c‎

Lines changed: 10 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -966,7 +966,7 @@ static PyObject* pybase64_decode_impl(PyObject* self, PyObject* args, PyObject *
966966
if (validation) {
967967
translate_fn = &translate;
968968
}
969-
if ((ignorechars_buffer.len == 0) && padded) {
969+
if (ignorechars_buffer.len == 0) {
970970
PyBuffer_Release(&ignorechars_buffer);
971971
Py_DECREF(ignorechars_object);
972972
ignorechars_object = NULL;
@@ -1146,9 +1146,11 @@ static PyObject* pybase64_decode_impl(PyObject* self, PyObject* args, PyObject *
11461146
/* not interacting with Python objects from here, release the GIL */
11471147
Py_BEGIN_ALLOW_THREADS
11481148

1149+
int const b64_flags = (canonical ? BASE64_CANONICAL : 0) |
1150+
(padded ? 0 : BASE64_NO_PADDING);
11491151
struct base64_state b64_state;
11501152

1151-
base64_stream_decode_init(&b64_state, 0);
1153+
base64_stream_decode_init(&b64_state, b64_flags);
11521154

11531155
while (len > src_slice) {
11541156
size_t dst_len = dst_slice;
@@ -1167,8 +1169,8 @@ static PyObject* pybase64_decode_impl(PyObject* self, PyObject* args, PyObject *
11671169
if (result > 0) {
11681170
translate_fn(src, cache, len, alphabet, &has_bad_char);
11691171
result = base64_stream_decode(&b64_state, cache, len, dst, &out_len);
1170-
if (b64_state.bytes != 0) {
1171-
result = 0;
1172+
if (result > 0) {
1173+
result = base64_stream_decode_final(&b64_state);
11721174
}
11731175
}
11741176

@@ -1179,26 +1181,6 @@ static PyObject* pybase64_decode_impl(PyObject* self, PyObject* args, PyObject *
11791181
PyErr_SetString(state->binAsciiError, "Non-base64 digit found");
11801182
goto EXCEPT;
11811183
}
1182-
if (canonical && (len > 0)) {
1183-
int fail = 0;
1184-
uint8_t const* pSrc = (uint8_t const*)cache;
1185-
assert(len >= 4);
1186-
if (pSrc[len - 1] == '=') {
1187-
uint8_t c = pSrc[len - 2];
1188-
uint8_t carry;
1189-
int shift = 6;
1190-
if (c == '=') {
1191-
c = pSrc[len - 3];
1192-
shift = 4;
1193-
}
1194-
carry = base64_table_dec_8bit[c] << shift;
1195-
fail = carry != 0U;
1196-
}
1197-
if (fail) {
1198-
PyErr_SetString(state->binAsciiError, "Non-zero padding bits");
1199-
goto EXCEPT;
1200-
}
1201-
}
12021184
out_len += (dst - (char*)dest);
12031185
}
12041186
else {
@@ -1207,7 +1189,10 @@ static PyObject* pybase64_decode_impl(PyObject* self, PyObject* args, PyObject *
12071189
/* not interacting with Python objects from here, release the GIL */
12081190
Py_BEGIN_ALLOW_THREADS
12091191

1210-
result = base64_decode(source, source_len, dest, &out_len, 0);
1192+
int const b64_flags = (canonical ? BASE64_CANONICAL : 0) |
1193+
(padded ? 0 : BASE64_NO_PADDING);
1194+
1195+
result = base64_decode(source, source_len, dest, &out_len, b64_flags);
12111196

12121197
/* restore the GIL */
12131198
Py_END_ALLOW_THREADS
@@ -1216,26 +1201,6 @@ static PyObject* pybase64_decode_impl(PyObject* self, PyObject* args, PyObject *
12161201
PyErr_SetString(state->binAsciiError, "Non-base64 digit found");
12171202
goto EXCEPT;
12181203
}
1219-
if (canonical && (source_len > 0)) {
1220-
int fail = 0;
1221-
uint8_t const* pSrc = source;
1222-
assert(source_len >= 4);
1223-
if (pSrc[source_len - 1] == '=') {
1224-
uint8_t c = pSrc[source_len - 2];
1225-
uint8_t carry;
1226-
int shift = 6;
1227-
if (c == '=') {
1228-
c = pSrc[source_len - 3];
1229-
shift = 4;
1230-
}
1231-
carry = base64_table_dec_8bit[c] << shift;
1232-
fail = carry != 0U;
1233-
}
1234-
if (fail) {
1235-
PyErr_SetString(state->binAsciiError, "Non-zero padding bits");
1236-
goto EXCEPT;
1237-
}
1238-
}
12391204
}
12401205
if (return_bytearray) {
12411206
PyByteArray_Resize(out_object, (Py_ssize_t)out_len);

‎tests/test_pybase64.py‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1006,9 +1006,9 @@ def test_canonical_invalid(
10061006
utils.unused_args(simd)
10071007
assert dfn(vector, altchars=altchars, validate=True) == expected
10081008
assert dfn(vector, altchars=altchars) == expected
1009-
with pytest.raises(BinAsciiError, match=re.escape("Non-zero padding bits")):
1009+
with pytest.raises(BinAsciiError, match=r"Non-zero padding bits|Non-base64 digit found"):
10101010
dfn(vector, altchars=altchars, validate=True, canonical=True)
1011-
with pytest.raises(BinAsciiError, match=re.escape("Non-zero padding bits")):
1011+
with pytest.raises(BinAsciiError, match=r"Non-zero padding bits|Non-base64 digit found"):
10121012
dfn(vector, altchars=altchars, canonical=True)
10131013

10141014

@@ -1026,17 +1026,17 @@ def test_canonical_check_all(
10261026
vector = b"A" + bytes([byte]) + b"=="
10271027
expected = dfn(vector, altchars=altchars, validate=True)
10281028
assert dfn(vector, altchars=altchars) == expected
1029-
with pytest.raises(BinAsciiError, match=re.escape("Non-zero padding bits")):
1029+
with pytest.raises(BinAsciiError, match=r"Non-zero padding bits|Non-base64 digit found"):
10301030
dfn(vector, altchars=altchars, validate=True, canonical=True)
1031-
with pytest.raises(BinAsciiError, match=re.escape("Non-zero padding bits")):
1031+
with pytest.raises(BinAsciiError, match=r"Non-zero padding bits|Non-base64 digit found"):
10321032
dfn(vector, altchars=altchars, canonical=True)
10331033
for byte in altchars + b"1235679BCDFGHJKLNOPRSTVWXZabdefhijlmnpqrtuvxyz":
10341034
vector = b"AA" + bytes([byte]) + b"="
10351035
expected = dfn(vector, altchars=altchars, validate=True)
10361036
assert dfn(vector, altchars=altchars) == expected
1037-
with pytest.raises(BinAsciiError, match=re.escape("Non-zero padding bits")):
1037+
with pytest.raises(BinAsciiError, match=r"Non-zero padding bits|Non-base64 digit found"):
10381038
dfn(vector, altchars=altchars, validate=True, canonical=True)
1039-
with pytest.raises(BinAsciiError, match=re.escape("Non-zero padding bits")):
1039+
with pytest.raises(BinAsciiError, match=r"Non-zero padding bits|Non-base64 digit found"):
10401040
dfn(vector, altchars=altchars, canonical=True)
10411041
# zero padding bits
10421042
for byte in b"AQgw":

0 commit comments

Comments
 (0)