Skip to content

Commit 5c0ea2e

Browse files
timridtimrid
authored
Merge pull request #33 from timrid/feature/update-to-construct-v2.10.70
Update to construct==2.10.70
2 parents ef933cc + a9f4b44 commit 5c0ea2e

4 files changed

Lines changed: 203 additions & 23 deletions

File tree

‎construct-stubs/core.pyi‎

Lines changed: 65 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,9 @@ from construct.lib import (
1717
ListType,
1818
RebufferedBytesIO,
1919
)
20+
from cryptography.hazmat.primitives.ciphers import Cipher
21+
from cryptography.hazmat.primitives.ciphers.aead import AESCCM, AESGCM, ChaCha20Poly1305
22+
from cryptography.hazmat.primitives.ciphers.modes import Mode
2023
from typing_extensions import Buffer
2124

2225
# unfortunately, there are a few duplications with "typing", e.g. Union and Optional, which is why the t. prefix must be used everywhere
@@ -67,6 +70,7 @@ class RawCopyError(ConstructError): ...
6770
class RotationError(ConstructError): ...
6871
class ChecksumError(ConstructError): ...
6972
class CancelParsing(ConstructError): ...
73+
class CipherError(ConstructError): ...
7074

7175
# ===============================================================================
7276
# used internally
@@ -86,6 +90,17 @@ def stream_size(stream: StreamType) -> int: ...
8690
def stream_iseof(stream: StreamType) -> bool: ...
8791
def evaluate(param: ConstantOrContextLambda2[T], context: Context) -> T: ...
8892

93+
class BytesIOWithOffsets(io.BytesIO):
94+
@staticmethod
95+
def from_reading(
96+
stream: StreamType, length: int, path: PathType
97+
) -> BytesIOWithOffsets: ...
98+
def __init__(
99+
self, contents: bytes, parent_stream: StreamType, offset: int
100+
) -> None: ...
101+
def tell(self) -> int: ...
102+
def seek(self, offset: int, whence: int = ...) -> int: ...
103+
89104
# ===============================================================================
90105
# abstract constructs
91106
# ===============================================================================
@@ -135,12 +150,19 @@ class Construct(t.Generic[ParsedType, BuildTypes]):
135150
) -> Renamed[ParsedType, BuildTypes]: ...
136151
def __add__(self, other: Construct[t.Any, t.Any]) -> Struct: ...
137152
def __rshift__(self, other: Construct[t.Any, t.Any]) -> Sequence: ...
138-
def __getitem__(
139-
self, count: t.Union[int, t.Callable[[Context], int]]
140-
) -> Array[ParsedType, BuildTypes,]: ...
141-
def _parse(self, stream: StreamType, context: Context, path: PathType) -> ParsedType: ...
142-
def _parsereport(self, stream: StreamType, context: Context, path: PathType) -> ParsedType: ...
143-
def _build(self, obj: BuildTypes, stream: StreamType, context: Context, path: PathType) -> int: ...
153+
def __getitem__(self, count: t.Union[int, t.Callable[[Context], int]]) -> Array[
154+
ParsedType,
155+
BuildTypes,
156+
]: ...
157+
def _parse(
158+
self, stream: StreamType, context: Context, path: PathType
159+
) -> ParsedType: ...
160+
def _parsereport(
161+
self, stream: StreamType, context: Context, path: PathType
162+
) -> ParsedType: ...
163+
def _build(
164+
self, obj: BuildTypes, stream: StreamType, context: Context, path: PathType
165+
) -> int: ...
144166
def _sizeof(self, context: Context, path: PathType) -> int: ...
145167

146168
@t.type_check_only
@@ -234,15 +256,11 @@ class Bytes(Construct[bytes, t.Union[bytes, bytearray, int]]):
234256

235257
GreedyBytes: Construct[bytes, t.Union[bytes, bytearray]]
236258

237-
def Bitwise(
238-
subcon: Construct[SubconParsedType, SubconBuildTypes]
239-
) -> t.Union[
259+
def Bitwise(subcon: Construct[SubconParsedType, SubconBuildTypes]) -> t.Union[
240260
Transformed[SubconParsedType, SubconBuildTypes],
241261
Restreamed[SubconParsedType, SubconBuildTypes],
242262
]: ...
243-
def Bytewise(
244-
subcon: Construct[SubconParsedType, SubconBuildTypes]
245-
) -> t.Union[
263+
def Bytewise(subcon: Construct[SubconParsedType, SubconBuildTypes]) -> t.Union[
246264
Transformed[SubconParsedType, SubconBuildTypes],
247265
Restreamed[SubconParsedType, SubconBuildTypes],
248266
]: ...
@@ -880,6 +898,16 @@ class Peek(
880898
subcon: Construct[SubconParsedType, SubconBuildTypes],
881899
) -> None: ...
882900

901+
class OffsettedEnd(
902+
Subconstruct[SubconParsedType, SubconBuildTypes, SubconParsedType, SubconBuildTypes]
903+
):
904+
endoffset: ConstantOrContextLambda[int]
905+
def __init__(
906+
self,
907+
endoffset: ConstantOrContextLambda[int],
908+
subcon: Construct[SubconParsedType, SubconBuildTypes],
909+
) -> None: ...
910+
883911
class Seek(Construct[int, None]):
884912
at: ConstantOrContextLambda[int]
885913
if sys.version_info >= (3, 8):
@@ -924,9 +952,7 @@ class RawCopy(
924952
def ByteSwapped(
925953
subcon: Construct[SubconParsedType, SubconBuildTypes]
926954
) -> Transformed[SubconParsedType, SubconBuildTypes]: ...
927-
def BitsSwapped(
928-
subcon: Construct[SubconParsedType, SubconBuildTypes]
929-
) -> t.Union[
955+
def BitsSwapped(subcon: Construct[SubconParsedType, SubconBuildTypes]) -> t.Union[
930956
Transformed[SubconParsedType, SubconBuildTypes],
931957
Restreamed[SubconParsedType, SubconBuildTypes],
932958
]: ...
@@ -946,7 +972,10 @@ class Prefixed(
946972
def PrefixedArray(
947973
countfield: Construct[int, int],
948974
subcon: Construct[SubconParsedType, SubconBuildTypes],
949-
) -> Array[SubconParsedType, SubconBuildTypes,]: ...
975+
) -> Array[
976+
SubconParsedType,
977+
SubconBuildTypes,
978+
]: ...
950979

951980
class FixedSized(
952981
Subconstruct[SubconParsedType, SubconBuildTypes, SubconParsedType, SubconBuildTypes]
@@ -1095,6 +1124,26 @@ class Rebuffered(
10951124
tailcutoff: t.Optional[int] = ...,
10961125
) -> None: ...
10971126

1127+
class EncryptedSym(Tunnel[SubconParsedType, SubconBuildTypes]):
1128+
cipher: ConstantOrContextLambda2[Cipher[Mode]]
1129+
def __init__(
1130+
self,
1131+
subcon: Construct[SubconParsedType, SubconBuildTypes],
1132+
cipher: ConstantOrContextLambda2[Cipher[Mode]],
1133+
) -> None: ...
1134+
1135+
class EncryptedSymAead(Tunnel[SubconParsedType, SubconBuildTypes]):
1136+
cipher: ConstantOrContextLambda2[t.Union[AESGCM, AESCCM, ChaCha20Poly1305]]
1137+
nonce: ConstantOrContextLambda2[bytes]
1138+
associated_data: ConstantOrContextLambda2[bytes]
1139+
def __init__(
1140+
self,
1141+
subcon: Construct[SubconParsedType, SubconBuildTypes],
1142+
cipher: ConstantOrContextLambda2[t.Union[AESGCM, AESCCM, ChaCha20Poly1305]],
1143+
nonce: ConstantOrContextLambda2[bytes],
1144+
associated_data: ConstantOrContextLambda2[bytes] = ...,
1145+
) -> None: ...
1146+
10981147
# ===============================================================================
10991148
# lazy equivalents
11001149
# ===============================================================================

‎requirements.txt‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
construct==2.10.68
1+
construct==2.10.70
22
pytest>=6.2.0
33
numpy
44
arrow
@@ -7,4 +7,5 @@ cloudpickle
77
lz4
88
black
99
isort
10-
mypy
10+
mypy
11+
cryptography

‎setup.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@
2222
author="Tim Riddermann",
2323
python_requires=">=3.7",
2424
install_requires=[
25-
"construct==2.10.68",
25+
"construct==2.10.70",
2626
"typing_extensions>=4.6.0"
2727
],
2828
keywords=[

‎tests/test_core.py‎

Lines changed: 134 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -151,17 +151,29 @@ def test_formatfield_bool_issue_901() -> None:
151151
assert d.sizeof() == 1
152152

153153
def test_bytesinteger() -> None:
154+
d = BytesInteger(0)
155+
assert raises(d.parse, b"") == IntegerError
156+
assert raises(d.build, 0) == IntegerError
154157
d = BytesInteger(4, signed=True, swapped=False)
155158
common(d, b"\x01\x02\x03\x04", 0x01020304, 4)
156159
common(d, b"\xff\xff\xff\xff", -1, 4)
157160
d = BytesInteger(4, signed=False, swapped=this.swapped)
158161
common(d, b"\x01\x02\x03\x04", 0x01020304, 4, swapped=False)
159162
common(d, b"\x04\x03\x02\x01", 0x01020304, 4, swapped=True)
163+
assert raises(BytesInteger(-1).parse, b"") == IntegerError
164+
assert raises(BytesInteger(-1).build, 0) == IntegerError
165+
assert raises(BytesInteger(8).build, None) == IntegerError
166+
assert raises(BytesInteger(8, signed=False).build, -1) == IntegerError
167+
assert raises(BytesInteger(8, True).build, -2**64) == IntegerError
168+
assert raises(BytesInteger(8, True).build, 2**64) == IntegerError
169+
assert raises(BytesInteger(8, False).build, -2**64) == IntegerError
170+
assert raises(BytesInteger(8, False).build, 2**64) == IntegerError
160171
assert raises(BytesInteger(this.missing).sizeof) == SizeofError
161-
assert raises(BytesInteger(4, signed=False).build, -1) == IntegerError
162-
common(BytesInteger(0), b"", 0, 0)
163172

164173
def test_bitsinteger() -> None:
174+
d = BitsInteger(0)
175+
assert raises(d.parse, b"") == IntegerError
176+
assert raises(d.build, 0) == IntegerError
165177
d = BitsInteger(8)
166178
common(d, b"\x01\x01\x01\x01\x01\x01\x01\x01", 255, 8)
167179
d = BitsInteger(8, signed=True)
@@ -171,9 +183,17 @@ def test_bitsinteger() -> None:
171183
d = BitsInteger(16, swapped=this.swapped)
172184
common(d, b"\x01\x01\x01\x01\x01\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00", 0xff00, 16, swapped=False)
173185
common(d, b"\x00\x00\x00\x00\x00\x00\x00\x00\x01\x01\x01\x01\x01\x01\x01\x01", 0xff00, 16, swapped=True)
174-
assert raises(BitsInteger(this.missing).sizeof) == SizeofError
186+
assert raises(BitsInteger(-1).parse, b"") == IntegerError
187+
assert raises(BitsInteger(-1).build, 0) == IntegerError
188+
assert raises(BitsInteger(5, swapped=True).parse, bytes(5)) == IntegerError
189+
assert raises(BitsInteger(5, swapped=True).build, 0) == IntegerError
190+
assert raises(BitsInteger(8).build, None) == IntegerError
175191
assert raises(BitsInteger(8, signed=False).build, -1) == IntegerError
176-
common(BitsInteger(0), b"", 0, 0)
192+
assert raises(BitsInteger(8, True).build, -2**64) == IntegerError
193+
assert raises(BitsInteger(8, True).build, 2**64) == IntegerError
194+
assert raises(BitsInteger(8, False).build, -2**64) == IntegerError
195+
assert raises(BitsInteger(8, False).build, 2**64) == IntegerError
196+
assert raises(BitsInteger(this.missing).sizeof) == SizeofError
177197

178198
def test_varint() -> None:
179199
d = VarInt
@@ -926,6 +946,17 @@ def test_peek() -> None:
926946
assert d4.build(Container(a=0x01, b=0x0102)) == b""
927947
assert d4.sizeof() == 0
928948

949+
def test_offsettedend() -> None:
950+
d1 = Struct(
951+
"header" / Bytes(2),
952+
"data" / OffsettedEnd(-2, GreedyBytes),
953+
"footer" / Bytes(2),
954+
)
955+
common(d1, b"\x01\x02\x03\x04\x05\x06\x07", Container(header=b'\x01\x02', data=b'\x03\x04\x05', footer=b'\x06\x07'))
956+
957+
d2 = OffsettedEnd(0, Byte)
958+
assert raises(d2.sizeof) == SizeofError
959+
929960
def test_seek() -> None:
930961
d = Seek(5)
931962
assert d.parse(b"") == 5
@@ -1334,6 +1365,105 @@ def test_compressed_prefixed() -> None:
13341365
assert st.parse(st.build(Container(one=zeros,two=zeros))) == Container(one=zeros,two=zeros)
13351366
assert raises(d.sizeof) == SizeofError
13361367

1368+
@pytest.mark.xfail(ONWINDOWS and PYPY, reason="no wheel for 'cryptography' is currently available for pypy on windows")
1369+
def test_encryptedsym() -> None:
1370+
from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
1371+
key128 = b"\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
1372+
key256 = b"\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
1373+
iv = b"\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
1374+
nonce = iv
1375+
1376+
# AES 128/256 bit - ECB
1377+
d = EncryptedSym(GreedyBytes, lambda ctx: Cipher(algorithms.AES(ctx.key), modes.ECB()))
1378+
common(d, b"\xf4\x0f\x54\xb7\x6a\x7a\xf1\xdb\x92\x73\x14\xde\x2f\xa0\x3e\x2d", b'Secret Message..', key=key128, iv=iv)
1379+
common(d, b"\x82\x6b\x01\x82\x90\x02\xa1\x9e\x35\x0a\xe2\xc3\xee\x1a\x42\xf5", b'Secret Message..', key=key256, iv=iv)
1380+
1381+
# AES 128/256 bit - CBC
1382+
d = EncryptedSym(GreedyBytes, lambda ctx: Cipher(algorithms.AES(ctx.key), modes.CBC(ctx.iv)))
1383+
common(d, b"\xba\x79\xc2\x62\x22\x08\x29\xb9\xfb\xd3\x90\xc4\x04\xb7\x55\x87", b'Secret Message..', key=key128, iv=iv)
1384+
common(d, b"\x60\xc2\x45\x0d\x7e\x41\xd4\xf8\x85\xd4\x8a\x64\xd1\x45\x49\xe3", b'Secret Message..', key=key256, iv=iv)
1385+
1386+
# AES 128/256 bit - CTR
1387+
d = EncryptedSym(GreedyBytes, lambda ctx: Cipher(algorithms.AES(ctx.key), modes.CTR(ctx.nonce)))
1388+
common(d, b"\x80\x78\xb6\x0c\x07\xf5\x0c\x90\xce\xa2\xbf\xcb\x5b\x22\xb9\xb5", b'Secret Message..', key=key128, nonce=nonce)
1389+
common(d, b"\x6a\xae\x7b\x86\x1a\xa6\xe0\x6a\x49\x02\x02\x1b\xf2\x3c\xd8\x0d", b'Secret Message..', key=key256, nonce=nonce)
1390+
1391+
assert raises(EncryptedSym(GreedyBytes, "AES").build, b"") == CipherError # type: ignore
1392+
assert raises(EncryptedSym(GreedyBytes, "AES").parse, b"") == CipherError # type: ignore
1393+
1394+
@pytest.mark.xfail(ONWINDOWS and PYPY, reason="no wheel for 'cryptography' is currently available for pypy on windows")
1395+
def test_encryptedsym_cbc_example() -> None:
1396+
from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
1397+
d = Struct(
1398+
"iv" / Default(Bytes(16), os.urandom(16)),
1399+
"enc_data" / EncryptedSym(
1400+
Aligned(16,
1401+
Struct(
1402+
"width" / Int16ul,
1403+
"height" / Int16ul
1404+
)
1405+
),
1406+
lambda ctx: Cipher(algorithms.AES(ctx._.key), modes.CBC(ctx.iv))
1407+
)
1408+
)
1409+
key128 = b"\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
1410+
byts = d.build({"enc_data": {"width": 5, "height": 4}}, key=key128)
1411+
obj = d.parse(byts, key=key128)
1412+
assert obj.enc_data == Container(width=5, height=4)
1413+
1414+
@pytest.mark.xfail(ONWINDOWS and PYPY, reason="no wheel for 'cryptography' is currently available for pypy on windows")
1415+
def test_encryptedsymaead() -> None:
1416+
from cryptography.hazmat.primitives.ciphers import aead
1417+
key128 = b"\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
1418+
key256 = b"\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
1419+
nonce = b"\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
1420+
1421+
# AES 128/256 bit - GCM
1422+
d = Struct(
1423+
"associated_data" / Bytes(21),
1424+
"data" / EncryptedSymAead(
1425+
GreedyBytes,
1426+
lambda ctx: aead.AESGCM(ctx._.key),
1427+
this._.nonce,
1428+
this.associated_data
1429+
)
1430+
)
1431+
common(
1432+
d,
1433+
b"This is authenticated\xb6\xd3\x64\x0c\x7a\x31\xaa\x16\xa3\x58\xec\x17\x39\x99\x2e\xf8\x4e\x41\x17\x76\x3f\xd1\x06\x47\x04\x9f\x42\x1c\xf4\xa9\xfd\x99\x9c\xe9",
1434+
Container(associated_data=b"This is authenticated", data=b"The secret message"),
1435+
key=key128,
1436+
nonce=nonce
1437+
)
1438+
common(
1439+
d,
1440+
b"This is authenticated\xde\xb4\x41\x79\xc8\x7f\xea\x8d\x0e\x41\xf6\x44\x2f\x93\x21\xe6\x37\xd1\xd3\x29\xa4\x97\xc3\xb5\xf4\x81\x72\xa1\x7f\x3b\x9b\x53\x24\xe4",
1441+
Container(associated_data=b"This is authenticated", data=b"The secret message"),
1442+
key=key256,
1443+
nonce=nonce
1444+
)
1445+
assert raises(EncryptedSymAead(GreedyBytes, "AESGCM", bytes(16)).build, b"") == CipherError # type: ignore
1446+
assert raises(EncryptedSymAead(GreedyBytes, "AESGCM", bytes(16)).parse, b"") == CipherError # type: ignore
1447+
1448+
@pytest.mark.xfail(ONWINDOWS and PYPY, reason="no wheel for 'cryptography' is currently available for pypy on windows")
1449+
def test_encryptedsymaead_gcm_example() -> None:
1450+
from cryptography.hazmat.primitives.ciphers import aead
1451+
d = Struct(
1452+
"nonce" / Default(Bytes(16), os.urandom(16)),
1453+
"associated_data" / Bytes(21),
1454+
"enc_data" / EncryptedSymAead(
1455+
GreedyBytes,
1456+
lambda ctx: aead.AESGCM(ctx._.key),
1457+
this.nonce,
1458+
this.associated_data
1459+
)
1460+
)
1461+
key128 = b"\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
1462+
byts = d.build({"associated_data": b"This is authenticated", "enc_data": b"The secret message"}, key=key128)
1463+
obj = d.parse(byts, key=key128)
1464+
assert obj.enc_data == b"The secret message"
1465+
assert obj.associated_data == b"This is authenticated"
1466+
13371467
def test_rebuffered() -> None:
13381468
data = b"0" * 1000
13391469
assert Rebuffered(Array(1000,Byte)).parse_stream(io.BytesIO(data)) == [48]*1000

0 commit comments

Comments
 (0)