Skip to content

Commit 9bcb6e3

Browse files
RogerPodacterRogerPodacter
authored
Merge pull request #132 from ethscriptions-protocol/byte_pack
Add BytePackLib for efficient content storage
2 parents 51cc758 + 08d416c commit 9bcb6e3

6 files changed

Lines changed: 421 additions & 14 deletions

File tree

‎contracts/src/Ethscriptions.sol‎

Lines changed: 24 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ pragma solidity 0.8.24;
44
import "./ERC721EthscriptionsSequentialEnumerableUpgradeable.sol";
55
import {LibString} from "solady/utils/LibString.sol";
66
import "./libraries/SSTORE2Unlimited.sol";
7+
import "./libraries/BytePackLib.sol";
78
import "./libraries/EthscriptionsRendererLib.sol";
89
import "./EthscriptionsProver.sol";
910
import "./libraries/Predeploys.sol";
@@ -75,8 +76,8 @@ contract Ethscriptions is ERC721EthscriptionsSequentialEnumerableUpgradeable {
7576
/// @dev Ethscription ID (L1 tx hash) => Ethscription data
7677
mapping(bytes32 => Ethscription) internal ethscriptions;
7778

78-
/// @dev Content SHA => SSTORE2 pointer (single address, no array)
79-
mapping(bytes32 => address) public contentPointerBySha;
79+
/// @dev Content SHA => packed content (for <32 bytes) or SSTORE2 pointer (for >=32 bytes)
80+
mapping(bytes32 => bytes32) public contentStorageBySha;
8081

8182
/// @dev Content URI hash => first ethscription tx hash that used it (for protocol uniqueness check)
8283
/// @dev bytes32(0) means unused, non-zero means the content URI has been used
@@ -373,7 +374,16 @@ contract Ethscriptions is ERC721EthscriptionsSequentialEnumerableUpgradeable {
373374
/// @notice Get content for an ethscription
374375
function getEthscriptionContent(bytes32 ethscriptionId) public view returns (bytes memory) {
375376
Ethscription storage ethscription = _getEthscriptionOrRevert(ethscriptionId);
376-
address pointer = contentPointerBySha[ethscription.contentSha];
377+
bytes32 ref = contentStorageBySha[ethscription.contentSha];
378+
379+
// Check if it's inline content using BytePackLib
380+
if (BytePackLib.isPacked(ref)) {
381+
return BytePackLib.unpack(ref);
382+
}
383+
384+
// It's a pointer to SSTORE2 contract
385+
address pointer = address(uint160(uint256(ref)));
386+
377387
return SSTORE2Unlimited.read(pointer);
378388
}
379389

@@ -496,14 +506,21 @@ contract Ethscriptions is ERC721EthscriptionsSequentialEnumerableUpgradeable {
496506
contentSha = sha256(content);
497507

498508
// Check if content already exists
499-
address existingPointer = contentPointerBySha[contentSha];
500-
if (existingPointer != address(0)) {
509+
bytes32 existing = contentStorageBySha[contentSha];
510+
if (existing != bytes32(0)) {
501511
// Content already stored, just return the SHA
502512
return contentSha;
503513
}
504514

505-
// Content doesn't exist, store it using SSTORE2Unlimited (handles any size)
506-
contentPointerBySha[contentSha] = SSTORE2Unlimited.write(content);
515+
// Store based on size
516+
if (content.length <= 31) {
517+
// Pack small content directly into bytes32 (0-31 bytes)
518+
contentStorageBySha[contentSha] = BytePackLib.packCalldata(content);
519+
} else {
520+
// Deploy via SSTORE2 for larger content (32+ bytes)
521+
address pointer = SSTORE2Unlimited.write(content);
522+
contentStorageBySha[contentSha] = bytes32(uint256(uint160(pointer)));
523+
}
507524

508525
return contentSha;
509526
}

‎contracts/src/libraries/BytePackLib.sol‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
// SPDX-License-Identifier: MIT
2+
pragma solidity ^0.8.4;
3+
4+
/// @title BytePackLib
5+
/// @notice Library for packing small byte arrays (0-31 bytes) into a single bytes32 slot
6+
/// @dev Uses a tag byte (length + 1) to distinguish packed data from regular addresses/data
7+
library BytePackLib {
8+
error ContentTooLarge(uint256 size);
9+
error NotPackedData();
10+
11+
/// @notice Pack bytes calldata up to 31 bytes into a bytes32
12+
/// @dev Calldata version for gas optimization when called with external data
13+
/// @param data The data to pack (must be <= 31 bytes)
14+
/// @return packed The packed bytes32 value
15+
function packCalldata(bytes calldata data) internal pure returns (bytes32 packed) {
16+
uint256 len = data.length;
17+
if (len >= 32) revert ContentTooLarge(len);
18+
19+
assembly {
20+
// Pack: tag byte (len+1) | first 31 bytes of data
21+
packed := or(
22+
shl(248, add(len, 1)), // Tag in first byte
23+
shr(8, calldataload(data.offset)) // Data in remaining 31 bytes
24+
)
25+
}
26+
}
27+
28+
/// @notice Unpack a bytes32 value into bytes
29+
/// @dev Extracts the data based on the tag byte (length + 1)
30+
/// @param packed The packed bytes32 value
31+
/// @return data The unpacked bytes data
32+
function unpack(bytes32 packed) internal pure returns (bytes memory data) {
33+
uint256 tag = uint8(uint256(packed >> 248));
34+
if (tag == 0 || tag > 32) revert NotPackedData();
35+
36+
uint256 len = tag - 1;
37+
data = new bytes(len);
38+
39+
if (len > 0) {
40+
assembly {
41+
// Store the data (shift left by 8 to remove tag byte)
42+
mstore(add(data, 0x20), shl(8, packed))
43+
// Note: No need to zero memory after the data since new bytes() already zeroes it
44+
// and we're only writing up to 31 bytes into a 32-byte word
45+
}
46+
}
47+
}
48+
49+
/// @notice Check if a bytes32 value is packed data
50+
/// @dev Returns true if the first byte indicates packed data (tag between 1-32)
51+
/// @param value The bytes32 value to check
52+
/// @return True if the value is packed data, false otherwise
53+
function isPacked(bytes32 value) internal pure returns (bool) {
54+
// Packed data has a tag byte between 1-32 in the first byte
55+
uint256 tag = uint8(uint256(value >> 248));
56+
return tag > 0 && tag <= 32;
57+
}
58+
59+
/// @notice Get the length of packed data without unpacking
60+
/// @dev Returns the length stored in the tag byte
61+
/// @param packed The packed bytes32 value
62+
/// @return The length of the packed data (0-31)
63+
function packedLength(bytes32 packed) internal pure returns (uint256) {
64+
uint256 tag = uint8(uint256(packed >> 248));
65+
if (tag == 0 || tag > 32) revert NotPackedData();
66+
return tag - 1;
67+
}
68+
}

‎contracts/test/BytePackLib.t.sol‎

Lines changed: 203 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,203 @@
1+
// SPDX-License-Identifier: MIT
2+
pragma solidity ^0.8.24;
3+
4+
import "forge-std/Test.sol";
5+
import "forge-std/console2.sol";
6+
import "../src/libraries/BytePackLib.sol";
7+
8+
/// @title TestWrapper
9+
/// @notice Wrapper contract to expose internal library functions for testing
10+
contract TestWrapper {
11+
function packCalldata(bytes calldata data) external pure returns (bytes32) {
12+
return BytePackLib.packCalldata(data);
13+
}
14+
15+
function unpack(bytes32 packed) external pure returns (bytes memory) {
16+
return BytePackLib.unpack(packed);
17+
}
18+
19+
function isPacked(bytes32 value) external pure returns (bool) {
20+
return BytePackLib.isPacked(value);
21+
}
22+
23+
function packedLength(bytes32 packed) external pure returns (uint256) {
24+
return BytePackLib.packedLength(packed);
25+
}
26+
}
27+
28+
contract BytePackLibTest is Test {
29+
TestWrapper wrapper;
30+
31+
function setUp() public {
32+
wrapper = new TestWrapper();
33+
}
34+
35+
/// @notice Test packing and unpacking for all valid sizes (0-31 bytes)
36+
function test_AllValidSizes() public {
37+
for (uint256 i = 0; i <= 31; i++) {
38+
// Test with incrementing pattern
39+
bytes memory data = new bytes(i);
40+
for (uint256 j = 0; j < i; j++) {
41+
data[j] = bytes1(uint8(j % 256));
42+
}
43+
this.helperPackUnpack(data);
44+
45+
// Also test with all zeros
46+
bytes memory zeros = new bytes(i);
47+
this.helperPackUnpack(zeros);
48+
49+
// Also test with all 0xFF
50+
bytes memory ones = new bytes(i);
51+
for (uint256 j = 0; j < i; j++) {
52+
ones[j] = bytes1(0xFF);
53+
}
54+
this.helperPackUnpack(ones);
55+
}
56+
}
57+
58+
function helperPackUnpack(bytes calldata data) external view {
59+
require(data.length < 32, "Data must be less than 32 bytes");
60+
61+
bytes32 packed = wrapper.packCalldata(data);
62+
63+
// Verify it's marked as packed
64+
assertTrue(wrapper.isPacked(packed), "Should be marked as packed");
65+
66+
// Verify the length is correct
67+
assertEq(wrapper.packedLength(packed), data.length, "Length should match");
68+
69+
// Verify unpacking gives back the original data
70+
bytes memory unpacked = wrapper.unpack(packed);
71+
assertEq(unpacked, data, "Unpacked data should match original");
72+
73+
// Verify the tag byte is correct (length + 1)
74+
uint8 tag = uint8(uint256(packed >> 248));
75+
assertEq(tag, data.length + 1, "Tag should be length + 1");
76+
}
77+
78+
/// @notice Test that packing 32+ bytes reverts
79+
function test_PackingTooLarge_Reverts() public {
80+
for (uint256 size = 32; size <= 40; size++) {
81+
bytes memory data = new bytes(size);
82+
83+
vm.expectRevert(abi.encodeWithSelector(BytePackLib.ContentTooLarge.selector, size));
84+
this.helperPackLarge(data);
85+
}
86+
}
87+
88+
function helperPackLarge(bytes calldata data) external view {
89+
wrapper.packCalldata(data);
90+
}
91+
92+
/// @notice Test that unpacking non-packed data reverts
93+
function test_UnpackNonPacked_Reverts() public {
94+
// Test with zero bytes32 (tag = 0)
95+
bytes32 zero = bytes32(0);
96+
assertFalse(wrapper.isPacked(zero), "Zero should not be packed");
97+
vm.expectRevert(BytePackLib.NotPackedData.selector);
98+
wrapper.unpack(zero);
99+
100+
// Test with an address-like value (no tag byte)
101+
bytes32 addressLike = bytes32(uint256(uint160(address(0x1234567890123456789012345678901234567890))));
102+
assertFalse(wrapper.isPacked(addressLike), "Address should not be packed");
103+
vm.expectRevert(BytePackLib.NotPackedData.selector);
104+
wrapper.unpack(addressLike);
105+
106+
// Test with tag byte > 32 (e.g., 0x21 = 33)
107+
bytes32 invalidTag = bytes32(uint256(0x21) << 248);
108+
assertFalse(wrapper.isPacked(invalidTag), "Tag > 32 should not be packed");
109+
vm.expectRevert(BytePackLib.NotPackedData.selector);
110+
wrapper.unpack(invalidTag);
111+
112+
// Test with tag byte = 255 (maximum uint8)
113+
bytes32 maxTag = bytes32(uint256(0xFF) << 248);
114+
assertFalse(wrapper.isPacked(maxTag), "Tag = 255 should not be packed");
115+
vm.expectRevert(BytePackLib.NotPackedData.selector);
116+
wrapper.unpack(maxTag);
117+
118+
// Test getting length of non-packed data
119+
vm.expectRevert(BytePackLib.NotPackedData.selector);
120+
wrapper.packedLength(zero);
121+
122+
vm.expectRevert(BytePackLib.NotPackedData.selector);
123+
wrapper.packedLength(addressLike);
124+
125+
vm.expectRevert(BytePackLib.NotPackedData.selector);
126+
wrapper.packedLength(invalidTag);
127+
128+
vm.expectRevert(BytePackLib.NotPackedData.selector);
129+
wrapper.packedLength(maxTag);
130+
}
131+
132+
/// @notice Test isPacked detection
133+
function test_IsPacked_Detection() public {
134+
// Pack some data and verify detection
135+
bytes memory testData = hex"74657374"; // "test"
136+
bytes32 packed = wrapper.packCalldata(testData);
137+
assertTrue(wrapper.isPacked(packed), "Packed data should be detected");
138+
139+
// Regular addresses should not be detected as packed
140+
address addr = address(0x1234567890123456789012345678901234567890);
141+
bytes32 addrBytes = bytes32(uint256(uint160(addr)));
142+
assertFalse(wrapper.isPacked(addrBytes), "Address should not be detected as packed");
143+
144+
// Zero should not be detected as packed
145+
assertFalse(wrapper.isPacked(bytes32(0)), "Zero should not be detected as packed");
146+
147+
// Random data without tag byte (first byte is 0x00) should not be detected as packed
148+
bytes32 randomData = bytes32(uint256(0x00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff));
149+
assertFalse(wrapper.isPacked(randomData), "Random data should not be detected as packed");
150+
}
151+
152+
/// @notice Test the exact packed format
153+
function test_PackedFormat() public {
154+
// Test empty bytes
155+
bytes memory empty = "";
156+
bytes32 packedEmpty = wrapper.packCalldata(empty);
157+
assertEq(uint256(packedEmpty), uint256(bytes32(bytes1(0x01))), "Empty should pack to 0x01 followed by zeros");
158+
159+
// Test single byte 'A' (0x41)
160+
bytes memory single = hex"41";
161+
bytes32 packedSingle = wrapper.packCalldata(single);
162+
// Should be: tag=0x02, data=0x41, rest zeros
163+
assertEq(uint8(uint256(packedSingle >> 248)), 0x02, "Tag for 1 byte should be 2");
164+
assertEq(uint8(uint256(packedSingle >> 240)), 0x41, "Data should be 0x41");
165+
166+
// Test "ABC" (0x414243)
167+
bytes memory abc = hex"414243";
168+
bytes32 packedAbc = wrapper.packCalldata(abc);
169+
// Should be: tag=0x04, data=0x414243, rest zeros
170+
assertEq(uint8(uint256(packedAbc >> 248)), 0x04, "Tag for 3 bytes should be 4");
171+
assertEq(uint8(uint256(packedAbc >> 240)), 0x41, "First byte should be 0x41");
172+
assertEq(uint8(uint256(packedAbc >> 232)), 0x42, "Second byte should be 0x42");
173+
assertEq(uint8(uint256(packedAbc >> 224)), 0x43, "Third byte should be 0x43");
174+
}
175+
176+
/// @notice Test edge cases
177+
function test_EdgeCases() public {
178+
// Test 31 bytes (maximum packable)
179+
bytes memory max = new bytes(31);
180+
for (uint i = 0; i < 31; i++) {
181+
max[i] = bytes1(uint8(i));
182+
}
183+
184+
bytes32 packed = wrapper.packCalldata(max);
185+
assertTrue(wrapper.isPacked(packed), "31 bytes should be packed");
186+
assertEq(wrapper.packedLength(packed), 31, "Length should be 31");
187+
188+
bytes memory unpacked = wrapper.unpack(packed);
189+
assertEq(unpacked, max, "31 bytes should unpack correctly");
190+
191+
// Verify tag is 32 (31 + 1) - the maximum valid tag
192+
assertEq(uint8(uint256(packed >> 248)), 32, "Tag for 31 bytes should be 32");
193+
194+
// Manually create a bytes32 with tag = 32 (boundary case) and verify it's valid
195+
bytes32 dataBytes;
196+
assembly {
197+
dataBytes := mload(add(max, 0x20))
198+
}
199+
bytes32 boundaryTag = bytes32(uint256(32) << 248) | (dataBytes >> 8);
200+
assertTrue(wrapper.isPacked(boundaryTag), "Tag = 32 should be valid packed data");
201+
assertEq(wrapper.packedLength(boundaryTag), 31, "Tag = 32 means 31 bytes of data");
202+
}
203+
}

‎contracts/test/EthscriptionsWithTestFunctions.sol‎

Lines changed: 21 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ pragma solidity 0.8.24;
33

44
import "../src/Ethscriptions.sol";
55
import "../src/libraries/SSTORE2Unlimited.sol";
6+
import "../src/libraries/BytePackLib.sol";
67

78
/// @title EthscriptionsWithTestFunctions
89
/// @notice Test contract that extends Ethscriptions with additional functions for testing
@@ -14,18 +15,34 @@ contract EthscriptionsWithTestFunctions is Ethscriptions {
1415
/// @dev Test-only function to check if content exists
1516
function hasContent(bytes32 ethscriptionId) external view returns (bool) {
1617
Ethscription storage ethscription = _getEthscriptionOrRevert(ethscriptionId);
17-
return contentPointerBySha[ethscription.contentSha] != address(0);
18+
return contentStorageBySha[ethscription.contentSha] != bytes32(0);
1819
}
1920

20-
/// @notice Get the content pointer for an ethscription
21+
/// @notice Get the content storage value for an ethscription
22+
/// @dev Test-only function to inspect storage (either packed bytes or SSTORE2 address)
23+
function getContentStorage(bytes32 ethscriptionId) external view returns (bytes32) {
24+
Ethscription storage ethscription = _getEthscriptionOrRevert(ethscriptionId);
25+
return contentStorageBySha[ethscription.contentSha];
26+
}
27+
28+
/// @notice Get the content pointer for an ethscription (only for SSTORE2 stored content)
2129
/// @dev Test-only function to inspect SSTORE2 address
2230
function getContentPointer(bytes32 ethscriptionId) external view returns (address) {
2331
Ethscription storage ethscription = _getEthscriptionOrRevert(ethscriptionId);
24-
return contentPointerBySha[ethscription.contentSha];
32+
bytes32 stored = contentStorageBySha[ethscription.contentSha];
33+
34+
// Check if it's inline content using BytePackLib
35+
if (BytePackLib.isPacked(stored)) {
36+
// It's packed inline content, not a pointer
37+
return address(0);
38+
}
39+
40+
// It's a pointer to SSTORE2 contract
41+
return address(uint160(uint256(stored)));
2542
}
2643

2744
/// @notice Read content directly
28-
/// @dev Test-only function to read content from SSTORE2
45+
/// @dev Test-only function to read content
2946
/// @param ethscriptionId The ethscription ID (L1 tx hash)
3047
/// @return The content data
3148
function readContent(bytes32 ethscriptionId) external view returns (bytes memory) {

0 commit comments

Comments
 (0)