feat: add MsgPack+LZ4 output format for tracegrind

Add --output-format=csv|msgpack option. MsgPack format uses LZ4 block
compression achieving ~12x compression vs CSV.

New files:
- tg_msgpack.c/h: MsgPack encoder (write-only)
- tg_lz4.c/h: LZ4 compression wrapper with VG_() adaptations
- lz4.c/h: Vendored LZ4 library (BSD-2-Clause)
- docs/tracegrind-msgpack-format.md: Format specification

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

Author: art049

tracegrind/Makefile.am

@@ -11,7 +11,11 @@ pkginclude_HEADERS = tracegrind.h
 noinst_HEADERS = \
 	costs.h \
 	events.h \
-	global.h
+	global.h \
+	lz4.c \
+	lz4.h \
+	tg_lz4.h \
+	tg_msgpack.h
 
 #----------------------------------------------------------------------------
 # tracegrind-<platform>
@@ -36,7 +40,9 @@ TRACEGRIND_SOURCES_COMMON = \
 	jumps.c \
 	main.c \
 	sim.c \
-	threads.c
+	threads.c \
+	tg_lz4.c \
+	tg_msgpack.c
 
 # We sneakily include "cg_branchpred.c" and "cg_arch.c" from cachegrind
 TRACEGRIND_CFLAGS_COMMON = -I$(top_srcdir)/cachegrind

tracegrind/callstack.c

@@ -250,10 +250,10 @@ void TG_(push_call_stack)(BBCC* from, UInt jmp, BBCC* to, Addr sp, Bool skip)
 
     TG_(current_call_stack).sp++;
 
-    /* Emit CSV trace sample on function entry */
+    /* Emit trace sample on function entry */
     if (!skip && TG_(current_state).collect) {
 	fn_node* to_fn = to->cxt->fn[0];
-	TG_(trace_emit_sample)(TG_(current_tid), "ENTER", to_fn);
+	TG_(trace_emit_sample)(TG_(current_tid), True, to_fn);
     }
 
     /* To allow for above assertion we set context of next frame to 0 */
@@ -359,9 +359,9 @@ void TG_(pop_call_stack)(void)
 	}
 	TG_(stat).ret_counter++;
 
-	/* Emit CSV trace sample on function exit */
+	/* Emit trace sample on function exit */
 	if (TG_(current_state).collect) {
-	    TG_(trace_emit_sample)(TG_(current_tid), "EXIT", to_fn);
+	    TG_(trace_emit_sample)(TG_(current_tid), False, to_fn);
 	}
 
 	/* restore context */

tracegrind/clo.c

@@ -518,6 +518,11 @@ Bool TG_(process_cmd_line_option)(const HChar* arg)
 
    else if VG_STR_CLO(arg, "--tracegrind-out-file", TG_(clo).out_format) {}
 
+   else if VG_XACT_CLO(arg, "--output-format=csv",
+                       TG_(clo).output_format, output_format_csv) {}
+   else if VG_XACT_CLO(arg, "--output-format=msgpack",
+                       TG_(clo).output_format, output_format_msgpack) {}
+
    else if VG_BOOL_CLO(arg, "--mangle-names", TG_(clo).mangle_names) {}
 
    else if VG_BOOL_CLO(arg, "--skip-direct-rec",
@@ -573,6 +578,7 @@ void TG_(print_usage)(void)
    VG_(printf)(
 "\n   dump creation options:\n"
 "    --tracegrind-out-file=<f>  Output file name [tracegrind.out.%%p]\n"
+"    --output-format=csv|msgpack  Output format [csv]\n"
 "    --dump-line=no|yes        Dump source lines of costs? [yes]\n"
 "    --dump-instr=no|yes       Dump instruction address of costs? [no]\n"
 "    --compress-strings=no|yes Compress strings in profile dump? [yes]\n"
@@ -697,4 +703,6 @@ void TG_(set_clo_defaults)(void)
   TG_(clo).verbose = 0;
   TG_(clo).verbose_start = 0;
 #endif
+
+  TG_(clo).output_format = output_format_csv;
 }

tracegrind/docs/tracegrind-msgpack-format.md

@@ -0,0 +1,122 @@
+# Tracegrind MsgPack+LZ4 Output Format
+
+## Overview
+
+Tracegrind's `--output-format=msgpack` produces a binary trace file combining MsgPack serialization with LZ4 block compression. Files use the `.msgpack.lz4` extension.
+
+## File Structure
+
+```
+┌─────────────────────────────────┐
+│       File Header (8 bytes)     │
+├─────────────────────────────────┤
+│       Schema Chunk              │
+├─────────────────────────────────┤
+│       Data Chunk 1..N           │
+├─────────────────────────────────┤
+│       End Marker (8 bytes)      │
+└─────────────────────────────────┘
+```
+
+## File Header
+
+| Offset | Size | Field   | Description |
+|--------|------|---------|-------------|
+| 0      | 4    | magic   | ASCII `TGMP` (0x54 0x47 0x4D 0x50) |
+| 4      | 4    | version | Format version, uint32 LE (currently 1) |
+
+## Chunk Format
+
+Each chunk (schema and data) has the same header:
+
+| Offset | Size | Field             | Description |
+|--------|------|-------------------|-------------|
+| 0      | 4    | uncompressed_size | Size after decompression, uint32 LE |
+| 4      | 4    | compressed_size   | Size of LZ4 block, uint32 LE |
+| 8      | N    | data              | LZ4 block-compressed MsgPack data |
+
+## Schema Chunk
+
+The first chunk contains a MsgPack map:
+
+```json
+{
+    "version": 1,
+    "format": "tracegrind-msgpack",
+    "columns": ["seq", "tid", "event", "fn", "obj", "file", "line", "Ir", ...]
+}
+```
+
+### Fixed Columns
+
+| Index | Name  | Type   | Description |
+|-------|-------|--------|-------------|
+| 0     | seq   | uint64 | Sequence number |
+| 1     | tid   | int32  | Thread ID |
+| 2     | event | int    | 0 = ENTER, 1 = EXIT |
+| 3     | fn    | string | Function name |
+| 4     | obj   | string | Shared object path |
+| 5     | file  | string | Source file path |
+| 6     | line  | int32  | Line number (0 if unknown) |
+
+### Event Columns (index 7+)
+
+Event counters as delta values: `Ir`, `Dr`, `Dw`, `I1mr`, `D1mr`, `D1mw`, `ILmr`, `DLmr`, `DLmw`, `Bc`, `Bcm`, `Bi`, `Bim`. Which columns are present depends on Tracegrind options.
+
+## Data Chunks
+
+Each data chunk contains concatenated MsgPack arrays (one per row):
+
+```
+[seq, tid, event, fn, obj, file, line, delta_Ir, ...]
+```
+
+The reference implementation writes 4096 rows per chunk.
+
+## End Marker
+
+8 zero bytes (uncompressed_size = 0, compressed_size = 0).
+
+## Example: Reading in Python
+
+```python
+import struct, lz4.block, msgpack
+
+def read_tracegrind(filepath):
+    with open(filepath, 'rb') as f:
+        assert f.read(4) == b'TGMP'
+        version = struct.unpack('<I', f.read(4))[0]
+
+        # Read schema chunk
+        usize, csize = struct.unpack('<II', f.read(8))
+        schema = msgpack.unpackb(
+            lz4.block.decompress(f.read(csize), uncompressed_size=usize))
+        columns = [c.decode() if isinstance(c, bytes) else c
+                   for c in schema[b'columns']]
+
+        # Read data chunks
+        rows = []
+        while True:
+            usize, csize = struct.unpack('<II', f.read(8))
+            if usize == 0 and csize == 0:
+                break
+            chunk = lz4.block.decompress(f.read(csize), uncompressed_size=usize)
+            unpacker = msgpack.Unpacker(raw=False)
+            unpacker.feed(chunk)
+            for row in unpacker:
+                rows.append(dict(zip(columns, row)))
+
+        return columns, rows
+```
+
+## References
+
+- [MsgPack Specification](https://github.com/msgpack/msgpack/blob/master/spec.md)
+- [LZ4 Block Format](https://github.com/lz4/lz4/blob/dev/doc/lz4_Block_format.md)
+
+## Reference Implementation
+
+- `tracegrind/tg_msgpack.c/h` - MsgPack encoder
+- `tracegrind/tg_lz4.c/h` - LZ4 compression wrapper
+- `tracegrind/lz4.c/h` - Vendored LZ4 library
+- `tracegrind/dump.c` - Trace output integration