Skip to content

Commit 59d86a4

Browse files
mayeutmayeut
committed
decoding
1 parent 7ba5ec2 commit 59d86a4

3 files changed

Lines changed: 58 additions & 92 deletions

File tree

‎README.rst‎

Lines changed: 42 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -87,66 +87,67 @@ Benchmark
8787

8888
.. begin benchmark
8989
90-
Running Python 3.15.0a8, Apple clang version 21.0.0 (clang-2100.0.123.102), macOS 26.3.1, Apple M1 Max
90+
Running Python 3.15.0b3, Apple clang version 21.0.0 (clang-2100.1.1.101), macOS 26.5.2, Apple M1 Max
9191

9292
.. code::
9393
9494
pybase64 1.5.0 (C extension active - NEON)
9595
bench: altchars=None, validate=True, padded=True
96-
pybase64.encodebytes: 5129 MB/s (135,696 bytes -> 183,309 bytes)
97-
pybase64.b64encode: 16339 MB/s (135,696 bytes -> 180,928 bytes)
98-
pybase64.b64decode: 8821 MB/s (180,928 bytes -> 135,696 bytes)
99-
base64.encodebytes: 2339 MB/s (135,696 bytes -> 183,309 bytes)
100-
base64.b64encode: 2642 MB/s (135,696 bytes -> 180,928 bytes)
101-
base64.b64decode: 2613 MB/s (180,928 bytes -> 135,696 bytes)
96+
pybase64.encodebytes: 6518 MB/s (135,696 bytes -> 183,309 bytes)
97+
pybase64.b64encode: 17518 MB/s (135,696 bytes -> 180,928 bytes)
98+
pybase64.b64decode: 9016 MB/s (180,928 bytes -> 135,696 bytes)
99+
base64.encodebytes: 2372 MB/s (135,696 bytes -> 183,309 bytes)
100+
base64.b64encode: 2662 MB/s (135,696 bytes -> 180,928 bytes)
101+
base64.b64decode: 2635 MB/s (180,928 bytes -> 135,696 bytes)
102102
bench: altchars=None, validate=True, padded=False
103-
pybase64.b64encode: 16369 MB/s (135,696 bytes -> 180,928 bytes)
104-
pybase64.b64decode: 2480 MB/s (180,928 bytes -> 135,696 bytes)
105-
base64.b64encode: 2573 MB/s (135,696 bytes -> 180,928 bytes)
106-
base64.b64decode: 2583 MB/s (180,928 bytes -> 135,696 bytes)
103+
pybase64.b64encode: 17528 MB/s (135,696 bytes -> 180,928 bytes)
104+
pybase64.b64decode: 8984 MB/s (180,928 bytes -> 135,696 bytes)
105+
base64.b64encode: 2639 MB/s (135,696 bytes -> 180,928 bytes)
106+
base64.b64decode: 2646 MB/s (180,928 bytes -> 135,696 bytes)
107107
bench: altchars=None, ignorechars=b'', padded=False
108-
pybase64.b64decode: 2478 MB/s (180,928 bytes -> 135,696 bytes)
109-
base64.b64decode: 2610 MB/s (180,928 bytes -> 135,696 bytes)
108+
pybase64.b64decode: 8975 MB/s (180,928 bytes -> 135,696 bytes)
109+
base64.b64decode: 2619 MB/s (180,928 bytes -> 135,696 bytes)
110110
bench: altchars=None, ignorechars=b'\n', padded=True
111-
pybase64.b64decode: 2338 MB/s (183,308 bytes -> 135,696 bytes)
112-
base64.b64decode: 2083 MB/s (183,308 bytes -> 135,696 bytes)
111+
pybase64.b64decode: 2357 MB/s (183,308 bytes -> 135,696 bytes)
112+
base64.b64decode: 2073 MB/s (183,308 bytes -> 135,696 bytes)
113113
bench: altchars=None, ignorechars=b'\n', padded=False
114-
pybase64.b64decode: 2337 MB/s (183,308 bytes -> 135,696 bytes)
115-
base64.b64decode: 2013 MB/s (183,308 bytes -> 135,696 bytes)
114+
pybase64.b64decode: 2357 MB/s (183,308 bytes -> 135,696 bytes)
115+
base64.b64decode: 1932 MB/s (183,308 bytes -> 135,696 bytes)
116116
bench: altchars=None, validate=False, padded=True
117-
pybase64.b64decode: 2356 MB/s (183,308 bytes -> 135,696 bytes)
118-
base64.b64decode: 2156 MB/s (183,308 bytes -> 135,696 bytes)
117+
pybase64.b64decode: 2360 MB/s (183,308 bytes -> 135,696 bytes)
118+
base64.b64decode: 2021 MB/s (183,308 bytes -> 135,696 bytes)
119119
bench: altchars=None, validate=False, padded=False
120-
pybase64.b64decode: 2332 MB/s (183,308 bytes -> 135,696 bytes)
121-
base64.b64decode: 2079 MB/s (183,308 bytes -> 135,696 bytes)
120+
pybase64.b64decode: 2359 MB/s (183,308 bytes -> 135,696 bytes)
121+
base64.b64decode: 1976 MB/s (183,308 bytes -> 135,696 bytes)
122122
bench: altchars=b'-_', validate=True, padded=True
123-
pybase64.b64encode: 10328 MB/s (135,696 bytes -> 180,928 bytes)
124-
pybase64.b64decode: 5861 MB/s (180,928 bytes -> 135,696 bytes)
125-
base64.b64encode: 2617 MB/s (135,696 bytes -> 180,928 bytes)
126-
base64.b64decode: 1183 MB/s (180,928 bytes -> 135,696 bytes)
123+
pybase64.b64encode: 10733 MB/s (135,696 bytes -> 180,928 bytes)
124+
pybase64.b64decode: 5968 MB/s (180,928 bytes -> 135,696 bytes)
125+
base64.b64encode: 2653 MB/s (135,696 bytes -> 180,928 bytes)
126+
base64.b64decode: 1194 MB/s (180,928 bytes -> 135,696 bytes)
127127
bench: altchars=b'-_', validate=True, padded=False
128-
pybase64.b64encode: 10344 MB/s (135,696 bytes -> 180,928 bytes)
129-
pybase64.b64decode: 2163 MB/s (180,928 bytes -> 135,696 bytes)
130-
base64.b64encode: 2625 MB/s (135,696 bytes -> 180,928 bytes)
131-
base64.b64decode: 1164 MB/s (180,928 bytes -> 135,696 bytes)
128+
pybase64.b64encode: 10714 MB/s (135,696 bytes -> 180,928 bytes)
129+
pybase64.b64decode: 5941 MB/s (180,928 bytes -> 135,696 bytes)
130+
base64.b64encode: 2618 MB/s (135,696 bytes -> 180,928 bytes)
131+
base64.b64decode: 1161 MB/s (180,928 bytes -> 135,696 bytes)
132132
bench: altchars=b'-_', ignorechars=b'', padded=True
133-
pybase64.b64decode: 5815 MB/s (180,928 bytes -> 135,696 bytes)
134-
base64.b64decode: 2598 MB/s (180,928 bytes -> 135,696 bytes)
133+
pybase64.b64decode: 5839 MB/s (180,928 bytes -> 135,696 bytes)
134+
base64.b64decode: 2590 MB/s (180,928 bytes -> 135,696 bytes)
135135
bench: altchars=b'-_', ignorechars=b'', padded=False
136-
pybase64.b64decode: 2146 MB/s (180,928 bytes -> 135,696 bytes)
137-
base64.b64decode: 2586 MB/s (180,928 bytes -> 135,696 bytes)
136+
pybase64.b64decode: 5807 MB/s (180,928 bytes -> 135,696 bytes)
137+
base64.b64decode: 2588 MB/s (180,928 bytes -> 135,696 bytes)
138138
bench: altchars=b'-_', ignorechars=b'\n', padded=True
139-
pybase64.b64decode: 2042 MB/s (183,308 bytes -> 135,696 bytes)
140-
base64.b64decode: 2056 MB/s (183,308 bytes -> 135,696 bytes)
139+
pybase64.b64decode: 2044 MB/s (183,308 bytes -> 135,696 bytes)
140+
base64.b64decode: 2044 MB/s (183,308 bytes -> 135,696 bytes)
141141
bench: altchars=b'-_', ignorechars=b'\n', padded=False
142-
pybase64.b64decode: 2034 MB/s (183,308 bytes -> 135,696 bytes)
143-
base64.b64decode: 1952 MB/s (183,308 bytes -> 135,696 bytes)
142+
pybase64.b64decode: 1978 MB/s (183,308 bytes -> 135,696 bytes)
143+
base64.b64decode: 1872 MB/s (183,308 bytes -> 135,696 bytes)
144144
bench: altchars=b'-_', validate=False, padded=True
145-
pybase64.b64decode: 2021 MB/s (183,308 bytes -> 135,696 bytes)
146-
base64.b64decode: 1063 MB/s (183,308 bytes -> 135,696 bytes)
145+
pybase64.b64decode: 2013 MB/s (183,308 bytes -> 135,696 bytes)
146+
base64.b64decode: 1014 MB/s (183,308 bytes -> 135,696 bytes)
147147
bench: altchars=b'-_', validate=False, padded=False
148-
pybase64.b64decode: 2014 MB/s (183,308 bytes -> 135,696 bytes)
149-
base64.b64decode: 1042 MB/s (183,308 bytes -> 135,696 bytes)
148+
pybase64.b64decode: 2025 MB/s (183,308 bytes -> 135,696 bytes)
149+
base64.b64decode: 1000 MB/s (183,308 bytes -> 135,696 bytes)
150+
150151
151152
.. end benchmark
152153

‎src/pybase64/_pybase64.c‎

Lines changed: 10 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -968,7 +968,7 @@ static PyObject* pybase64_decode_impl(PyObject* self, PyObject* args, PyObject *
968968
if (validation) {
969969
translate_fn = &translate;
970970
}
971-
if ((ignorechars_buffer.len == 0) && padded) {
971+
if (ignorechars_buffer.len == 0) {
972972
PyBuffer_Release(&ignorechars_buffer);
973973
Py_DECREF(ignorechars_object);
974974
ignorechars_object = NULL;
@@ -1148,9 +1148,11 @@ static PyObject* pybase64_decode_impl(PyObject* self, PyObject* args, PyObject *
11481148
/* not interacting with Python objects from here, release the GIL */
11491149
Py_BEGIN_ALLOW_THREADS
11501150

1151+
int const b64_flags = (canonical ? BASE64_CANONICAL : 0) |
1152+
(padded ? 0 : BASE64_NO_PADDING);
11511153
struct base64_state b64_state;
11521154

1153-
base64_stream_decode_init(&b64_state, 0);
1155+
base64_stream_decode_init(&b64_state, b64_flags);
11541156

11551157
while (len > src_slice) {
11561158
size_t dst_len = dst_slice;
@@ -1169,8 +1171,8 @@ static PyObject* pybase64_decode_impl(PyObject* self, PyObject* args, PyObject *
11691171
if (result > 0) {
11701172
translate_fn(src, cache, len, alphabet, &has_bad_char);
11711173
result = base64_stream_decode(&b64_state, cache, len, dst, &out_len);
1172-
if (b64_state.bytes != 0) {
1173-
result = 0;
1174+
if (result > 0) {
1175+
result = base64_stream_decode_final(&b64_state);
11741176
}
11751177
}
11761178

@@ -1181,26 +1183,6 @@ static PyObject* pybase64_decode_impl(PyObject* self, PyObject* args, PyObject *
11811183
PyErr_SetString(state->binAsciiError, "Non-base64 digit found");
11821184
goto EXCEPT;
11831185
}
1184-
if (canonical && (len > 0)) {
1185-
int fail = 0;
1186-
uint8_t const* pSrc = (uint8_t const*)cache;
1187-
assert(len >= 4);
1188-
if (pSrc[len - 1] == '=') {
1189-
uint8_t c = pSrc[len - 2];
1190-
uint8_t carry;
1191-
int shift = 6;
1192-
if (c == '=') {
1193-
c = pSrc[len - 3];
1194-
shift = 4;
1195-
}
1196-
carry = base64_table_dec_8bit[c] << shift;
1197-
fail = carry != 0U;
1198-
}
1199-
if (fail) {
1200-
PyErr_SetString(state->binAsciiError, "Non-zero padding bits");
1201-
goto EXCEPT;
1202-
}
1203-
}
12041186
out_len += (dst - (char*)dest);
12051187
}
12061188
else {
@@ -1209,7 +1191,10 @@ static PyObject* pybase64_decode_impl(PyObject* self, PyObject* args, PyObject *
12091191
/* not interacting with Python objects from here, release the GIL */
12101192
Py_BEGIN_ALLOW_THREADS
12111193

1212-
result = base64_decode(source, source_len, dest, &out_len, 0);
1194+
int const b64_flags = (canonical ? BASE64_CANONICAL : 0) |
1195+
(padded ? 0 : BASE64_NO_PADDING);
1196+
1197+
result = base64_decode(source, source_len, dest, &out_len, b64_flags);
12131198

12141199
/* restore the GIL */
12151200
Py_END_ALLOW_THREADS
@@ -1218,26 +1203,6 @@ static PyObject* pybase64_decode_impl(PyObject* self, PyObject* args, PyObject *
12181203
PyErr_SetString(state->binAsciiError, "Non-base64 digit found");
12191204
goto EXCEPT;
12201205
}
1221-
if (canonical && (source_len > 0)) {
1222-
int fail = 0;
1223-
uint8_t const* pSrc = source;
1224-
assert(source_len >= 4);
1225-
if (pSrc[source_len - 1] == '=') {
1226-
uint8_t c = pSrc[source_len - 2];
1227-
uint8_t carry;
1228-
int shift = 6;
1229-
if (c == '=') {
1230-
c = pSrc[source_len - 3];
1231-
shift = 4;
1232-
}
1233-
carry = base64_table_dec_8bit[c] << shift;
1234-
fail = carry != 0U;
1235-
}
1236-
if (fail) {
1237-
PyErr_SetString(state->binAsciiError, "Non-zero padding bits");
1238-
goto EXCEPT;
1239-
}
1240-
}
12411206
}
12421207
if (return_bytearray) {
12431208
PyByteArray_Resize(out_object, (Py_ssize_t)out_len);

‎tests/test_pybase64.py‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1006,9 +1006,9 @@ def test_canonical_invalid(
10061006
utils.unused_args(simd)
10071007
assert dfn(vector, altchars=altchars, validate=True) == expected
10081008
assert dfn(vector, altchars=altchars) == expected
1009-
with pytest.raises(BinAsciiError, match=re.escape("Non-zero padding bits")):
1009+
with pytest.raises(BinAsciiError, match=r"Non-zero padding bits|Non-base64 digit found"):
10101010
dfn(vector, altchars=altchars, validate=True, canonical=True)
1011-
with pytest.raises(BinAsciiError, match=re.escape("Non-zero padding bits")):
1011+
with pytest.raises(BinAsciiError, match=r"Non-zero padding bits|Non-base64 digit found"):
10121012
dfn(vector, altchars=altchars, canonical=True)
10131013

10141014

@@ -1026,17 +1026,17 @@ def test_canonical_check_all(
10261026
vector = b"A" + bytes([byte]) + b"=="
10271027
expected = dfn(vector, altchars=altchars, validate=True)
10281028
assert dfn(vector, altchars=altchars) == expected
1029-
with pytest.raises(BinAsciiError, match=re.escape("Non-zero padding bits")):
1029+
with pytest.raises(BinAsciiError, match=r"Non-zero padding bits|Non-base64 digit found"):
10301030
dfn(vector, altchars=altchars, validate=True, canonical=True)
1031-
with pytest.raises(BinAsciiError, match=re.escape("Non-zero padding bits")):
1031+
with pytest.raises(BinAsciiError, match=r"Non-zero padding bits|Non-base64 digit found"):
10321032
dfn(vector, altchars=altchars, canonical=True)
10331033
for byte in altchars + b"1235679BCDFGHJKLNOPRSTVWXZabdefhijlmnpqrtuvxyz":
10341034
vector = b"AA" + bytes([byte]) + b"="
10351035
expected = dfn(vector, altchars=altchars, validate=True)
10361036
assert dfn(vector, altchars=altchars) == expected
1037-
with pytest.raises(BinAsciiError, match=re.escape("Non-zero padding bits")):
1037+
with pytest.raises(BinAsciiError, match=r"Non-zero padding bits|Non-base64 digit found"):
10381038
dfn(vector, altchars=altchars, validate=True, canonical=True)
1039-
with pytest.raises(BinAsciiError, match=re.escape("Non-zero padding bits")):
1039+
with pytest.raises(BinAsciiError, match=r"Non-zero padding bits|Non-base64 digit found"):
10401040
dfn(vector, altchars=altchars, canonical=True)
10411041
# zero padding bits
10421042
for byte in b"AQgw":

0 commit comments

Comments
 (0)