Skip to content

Commit a4fad94

Browse files
glennhartmannglennhartmann
authored
Add pattern for BTRFS Send Streams. (#495)
* Add pattern for BTRFS Send Streams. * Support BTRFS Send Stream protocol version 2.
1 parent 03f36b8 commit a4fad94

4 files changed

Lines changed: 266 additions & 0 deletions

File tree

‎README.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@ Everything will immediately show up in ImHex's Content Store and gets bundled wi
5353
| BIN | | [`patterns/selinux.hexpat`](patterns/selinux.pat) | SE Linux modules |
5454
| BINKA | | [`patterns/binka.hexpat`](patterns/binka.pat) | RAD Game Tools Bink Audio (BINKA) files |
5555
| BSON | `application/bson` | [`patterns/bson.hexpat`](patterns/bson.hexpat) | BSON (Binary JSON) format |
56+
| BTRFS Send Stream | | [`patterns/btrfs_send_stream.hexpat`](patterns/btrfs_send_stream.hexpat) | BTRFS Send Stream format |
5657
| bplist | `application/x-bplist` | [`patterns/bplist.hexpat`](patterns/bplist.hexpat) | Apple's binary property list format (bplist) |
5758
| BSP | | [`patterns/bsp_goldsrc.hexpat`](patterns/bsp_goldsrc.hexpat) | GoldSrc engine maps format (used in Half-Life 1) |
5859
| BZIP3 | | [`patterns/bzip3.hexpat`](patterns/bzip3.hexpat) | Parses BZip3 compression (file format) by Kamila Szewczyk |

‎patterns/btrfs_send_stream.hexpat‎

Lines changed: 265 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,265 @@
1+
#pragma author Glenn Hartmann
2+
#pragma description BTRFS Send Stream format - see https://btrfs.readthedocs.io/en/latest/dev/dev-send-stream.html. Currently supports versions 1 and 2.
3+
4+
#pragma magic [ 62 74 72 66 73 2D 73 74 72 65 61 6D ] @ 0x0
5+
6+
#pragma endian little
7+
8+
import std.hash;
9+
import std.io;
10+
import std.mem;
11+
import std.sys;
12+
import type.guid;
13+
import type.magic;
14+
import type.time;
15+
16+
// See https://btrfs.readthedocs.io/en/latest/dev/dev-send-stream.html#stream-version-1.
17+
enum CommandType : u16 {
18+
// VERSION 1
19+
BTRFS_SEND_C_UNSPEC = 0,
20+
BTRFS_SEND_C_SUBVOL = 1,
21+
BTRFS_SEND_C_SNAPSHOT = 2,
22+
BTRFS_SEND_C_MKFILE = 3,
23+
BTRFS_SEND_C_MKDIR = 4,
24+
BTRFS_SEND_C_MKNOD = 5,
25+
BTRFS_SEND_C_MKFIFO = 6,
26+
BTRFS_SEND_C_MKSOCK = 7,
27+
BTRFS_SEND_C_SYMLINK = 8,
28+
BTRFS_SEND_C_RENAME = 9,
29+
BTRFS_SEND_C_LINK = 10,
30+
BTRFS_SEND_C_UNLINK = 11,
31+
BTRFS_SEND_C_RMDIR = 12,
32+
BTRFS_SEND_C_SET_XATTR = 13,
33+
BTRFS_SEND_C_REMOVE_XATTR = 14,
34+
BTRFS_SEND_C_WRITE = 15,
35+
BTRFS_SEND_C_CLONE = 16,
36+
BTRFS_SEND_C_TRUNCATE = 17,
37+
BTRFS_SEND_C_CHMOD = 18,
38+
BTRFS_SEND_C_CHOWN = 19,
39+
BTRFS_SEND_C_UTIMES = 20,
40+
BTRFS_SEND_C_END = 21,
41+
BTRFS_SEND_C_UPDATE_EXTENT = 22,
42+
43+
// VERSION 2
44+
BTRFS_SEND_C_FALLOCATE = 23,
45+
BTRFS_SEND_C_FILEATTR = 24,
46+
BTRFS_SEND_C_ENCODED_WRITE = 25,
47+
};
48+
49+
// See https://btrfs.readthedocs.io/en/latest/dev/dev-send-stream.html#attributes-tlv-types.
50+
enum AttributeType : u16 {
51+
// VERSION 1
52+
BTRFS_SEND_A_UNSPEC = 0,
53+
BTRFS_SEND_A_UUID = 1,
54+
BTRFS_SEND_A_CTRANSID = 2,
55+
BTRFS_SEND_A_INO = 3,
56+
BTRFS_SEND_A_SIZE = 4,
57+
BTRFS_SEND_A_MODE = 5,
58+
BTRFS_SEND_A_UID = 6,
59+
BTRFS_SEND_A_GID = 7,
60+
BTRFS_SEND_A_RDEV = 8,
61+
BTRFS_SEND_A_CTIME = 9,
62+
BTRFS_SEND_A_MTIME = 10,
63+
BTRFS_SEND_A_ATIME = 11,
64+
BTRFS_SEND_A_OTIME = 12,
65+
BTRFS_SEND_A_XATTR_NAME = 13,
66+
BTRFS_SEND_A_XATTR_DATA = 14,
67+
BTRFS_SEND_A_PATH = 15,
68+
BTRFS_SEND_A_PATH_TO = 16,
69+
BTRFS_SEND_A_PATH_LINK = 17,
70+
BTRFS_SEND_A_FILE_OFFSET = 18,
71+
BTRFS_SEND_A_DATA = 19,
72+
BTRFS_SEND_A_CLONE_UUID = 20,
73+
BTRFS_SEND_A_CLONE_CTRANSID = 21,
74+
BTRFS_SEND_A_CLONE_PATH = 22,
75+
BTRFS_SEND_A_CLONE_OFFSET = 23,
76+
BTRFS_SEND_A_CLONE_LEN = 24,
77+
78+
// VERSION 2
79+
BTRFS_SEND_A_FALLOCATE_MODE = 25,
80+
BTRFS_SEND_A_FILEATTR = 26,
81+
BTRFS_SEND_A_UNENCODED_FILE_LEN = 27,
82+
BTRFS_SEND_A_UNENCODED_LEN = 28,
83+
BTRFS_SEND_A_UNENCODED_OFFSET = 29,
84+
BTRFS_SEND_A_COMPRESSION = 30,
85+
BTRFS_SEND_A_ENCRYPTION = 31,
86+
};
87+
88+
const u8 num_v1_attribute_types = 25;
89+
const AttributeType v1_attribute_types[num_v1_attribute_types] = {
90+
AttributeType::BTRFS_SEND_A_UNSPEC,
91+
AttributeType::BTRFS_SEND_A_UUID,
92+
AttributeType::BTRFS_SEND_A_CTRANSID,
93+
AttributeType::BTRFS_SEND_A_INO,
94+
AttributeType::BTRFS_SEND_A_SIZE,
95+
AttributeType::BTRFS_SEND_A_MODE,
96+
AttributeType::BTRFS_SEND_A_UID,
97+
AttributeType::BTRFS_SEND_A_GID,
98+
AttributeType::BTRFS_SEND_A_RDEV,
99+
AttributeType::BTRFS_SEND_A_CTIME,
100+
AttributeType::BTRFS_SEND_A_MTIME,
101+
AttributeType::BTRFS_SEND_A_ATIME,
102+
AttributeType::BTRFS_SEND_A_OTIME,
103+
AttributeType::BTRFS_SEND_A_XATTR_NAME,
104+
AttributeType::BTRFS_SEND_A_XATTR_DATA,
105+
AttributeType::BTRFS_SEND_A_PATH,
106+
AttributeType::BTRFS_SEND_A_PATH_TO,
107+
AttributeType::BTRFS_SEND_A_PATH_LINK,
108+
AttributeType::BTRFS_SEND_A_FILE_OFFSET,
109+
AttributeType::BTRFS_SEND_A_DATA,
110+
AttributeType::BTRFS_SEND_A_CLONE_UUID,
111+
AttributeType::BTRFS_SEND_A_CLONE_CTRANSID,
112+
AttributeType::BTRFS_SEND_A_CLONE_PATH,
113+
AttributeType::BTRFS_SEND_A_CLONE_OFFSET,
114+
AttributeType::BTRFS_SEND_A_CLONE_LEN
115+
};
116+
117+
const u8 num_v2_attribute_types = 7;
118+
const AttributeType v2_attribute_types[num_v2_attribute_types] = {
119+
AttributeType::BTRFS_SEND_A_FALLOCATE_MODE,
120+
AttributeType::BTRFS_SEND_A_FILEATTR,
121+
AttributeType::BTRFS_SEND_A_UNENCODED_FILE_LEN,
122+
AttributeType::BTRFS_SEND_A_UNENCODED_LEN,
123+
AttributeType::BTRFS_SEND_A_UNENCODED_OFFSET,
124+
AttributeType::BTRFS_SEND_A_COMPRESSION,
125+
AttributeType::BTRFS_SEND_A_ENCRYPTION
126+
};
127+
128+
fn is_in(auto arr, u8 len, AttributeType val) {
129+
for (u8 i = 0, i < len, i += 1) {
130+
if (arr[i] == val) {
131+
return true;
132+
}
133+
}
134+
return false;
135+
};
136+
137+
fn is_v1_attribute_type(AttributeType val) {
138+
return is_in(v1_attribute_types, num_v1_attribute_types, val);
139+
};
140+
141+
fn is_v2_attribute_type(AttributeType val) {
142+
return is_in(v2_attribute_types, num_v2_attribute_types, val);
143+
};
144+
145+
struct TimeSpec {
146+
u64 seconds;
147+
u32 nanoseconds;
148+
} [[format("format_time_spec")]];
149+
150+
// Formats the "seconds" part of the TimeSpec as a time64_t.
151+
fn format_time_spec(TimeSpec ts) {
152+
return type::impl::format_time_t(ts.seconds);
153+
};
154+
155+
struct Attribute<auto Version, auto EndAddr> {
156+
AttributeType type;
157+
158+
if (Version == 2 && type == AttributeType::BTRFS_SEND_A_DATA) {
159+
u64 length = EndAddr - $ [[export]];
160+
} else {
161+
u16 length;
162+
}
163+
164+
u64 pre_data_pos = $;
165+
166+
if (is_v1_attribute_type(type)) {
167+
match (type) {
168+
(AttributeType::BTRFS_SEND_A_UNSPEC): std::error("got unspecified attribute type");
169+
(AttributeType::BTRFS_SEND_A_UUID): type::GUID uuid;
170+
(AttributeType::BTRFS_SEND_A_CLONE_UUID): type::GUID clone_uuid;
171+
(AttributeType::BTRFS_SEND_A_CTRANSID): u64 ctransid;
172+
(AttributeType::BTRFS_SEND_A_INO): u64 ino;
173+
(AttributeType::BTRFS_SEND_A_SIZE): u64 size;
174+
(AttributeType::BTRFS_SEND_A_MODE): u64 mode;
175+
(AttributeType::BTRFS_SEND_A_UID): u64 uid;
176+
(AttributeType::BTRFS_SEND_A_GID): u64 gid;
177+
(AttributeType::BTRFS_SEND_A_RDEV): u64 rdev;
178+
(AttributeType::BTRFS_SEND_A_CTIME): TimeSpec ctime;
179+
(AttributeType::BTRFS_SEND_A_MTIME): TimeSpec mtime;
180+
(AttributeType::BTRFS_SEND_A_ATIME): TimeSpec atime;
181+
(AttributeType::BTRFS_SEND_A_OTIME): TimeSpec otime;
182+
(AttributeType::BTRFS_SEND_A_XATTR_NAME): char xattr_name[length];
183+
(AttributeType::BTRFS_SEND_A_XATTR_DATA): u8 xattr_data[length];
184+
(AttributeType::BTRFS_SEND_A_PATH): char path[length];
185+
(AttributeType::BTRFS_SEND_A_PATH_TO): char path_to[length];
186+
(AttributeType::BTRFS_SEND_A_PATH_LINK): char path_link[length];
187+
(AttributeType::BTRFS_SEND_A_FILE_OFFSET): u64 file_offset;
188+
(AttributeType::BTRFS_SEND_A_DATA): u8 data[length];
189+
(AttributeType::BTRFS_SEND_A_CLONE_CTRANSID): u64 clone_ctransid;
190+
(AttributeType::BTRFS_SEND_A_CLONE_PATH): char clone_path[length];
191+
(AttributeType::BTRFS_SEND_A_CLONE_OFFSET): u64 clone_offset;
192+
(AttributeType::BTRFS_SEND_A_CLONE_LEN): u64 clone_len;
193+
(_): std::error(std::format("unknown v1 attribute type: {} (this is a bug in the pattern)", type));
194+
}
195+
} else if (Version == 2 && is_v2_attribute_type(type)) {
196+
match (type) {
197+
(AttributeType::BTRFS_SEND_A_FALLOCATE_MODE): u32 fallocate_mode;
198+
(AttributeType::BTRFS_SEND_A_FILEATTR): u64 fileattr;
199+
(AttributeType::BTRFS_SEND_A_UNENCODED_FILE_LEN): u64 unencoded_file_len;
200+
(AttributeType::BTRFS_SEND_A_UNENCODED_LEN): u64 unencoded_len;
201+
(AttributeType::BTRFS_SEND_A_UNENCODED_OFFSET): u64 unencoded_offset;
202+
(AttributeType::BTRFS_SEND_A_COMPRESSION): u32 compression;
203+
(AttributeType::BTRFS_SEND_A_ENCRYPTION): u32 encryption;
204+
(_): std::error(std::format("unknown v2 attribute type: {} (this is a bug in the pattern)", type));
205+
}
206+
} else {
207+
std::error(std::format("unknown attribute type: {}", type));
208+
}
209+
210+
std::assert($ - pre_data_pos == length,
211+
std::format("bad attribute length: expected {}, got {}", length, $ - pre_data_pos));
212+
};
213+
214+
// Length-defined array of |Attribute|s. As the name suggests, |ByteLength| is the array length in
215+
// bytes, not in elements.
216+
struct Attributes<auto Version, auto ByteLength> {
217+
u64 end_addr = addressof(this) + ByteLength;
218+
Attribute<Version, end_addr> attributes[while(!std::mem::reached(end_addr))] [[inline]];
219+
};
220+
221+
// The actual command structure. This is intended to be embedded inside a |Command| and
222+
// [[inline]]d, so the user will never see the type name.
223+
struct CommandInternal<auto Version> {
224+
u32 attributes_byte_length;
225+
CommandType type;
226+
u32 checksum; // CRC32C with initial seed 0
227+
Attributes<Version, attributes_byte_length> attributes;
228+
};
229+
230+
// Wrapper structure around a CommandInternal to verify the checksum.
231+
struct Command<auto Version> {
232+
CommandInternal<Version> command [[inline]];
233+
std::mem::Section mySection = std::mem::create_section("checksum buffer");
234+
std::mem::set_section_size(mySection, sizeof(command));
235+
std::mem::copy_value_to_section(command, mySection, 0x0);
236+
237+
// The checksum needs to be computed on the entire |command|, but with its |checksum| member
238+
// zeroed out, so we have to copy the data into a Section and modify it.
239+
CommandInternal<Version> section_command @ 0x0 in mySection;
240+
std::assert(sizeof(command) == sizeof(section_command),
241+
std::format("|section_command| is the wrong size: expected {}, got {}",
242+
sizeof(command), sizeof(section_command)));
243+
section_command.checksum = 0;
244+
245+
// For some reason, running the crc32 on |section_command| directly always gets the wrong value.
246+
u8 data[sizeof(command)] @ 0x0 in mySection;
247+
u32 crc32c = std::hash::crc32(data, 0 /* init */, 0x1EDC6F41 /* poly */,
248+
0 /* xorout */, true /* reflect_in */, true /* reflect_out */);
249+
std::assert(crc32c == command.checksum,
250+
std::format("bad command checksum: expected {}, got {}", command.checksum, crc32c));
251+
252+
std::mem::delete_section(mySection);
253+
};
254+
255+
struct SendStream {
256+
type::Magic<"btrfs-stream"> magic;
257+
padding[1];
258+
u32 version;
259+
std::assert(version >= 1 && version <= 2,
260+
std::format("Only versions 1 and 2 are currently supported (got version {})", version));
261+
Command<version> commands[while(!std::mem::eof())];
262+
};
263+
264+
SendStream send_stream @ 0x0;
265+
std::assert(std::mem::eof(), "Parsing did not consume whole file.");

‎tests/patterns/test_data/btrfs_send_stream.hexpat/v1.bin‎

1.94 KB
Binary file not shown.

‎tests/patterns/test_data/btrfs_send_stream.hexpat/v2.bin‎

1.08 KB
Binary file not shown.

0 commit comments

Comments
 (0)