.

Author: petrelharp

c/examples/json_struct_metadata.c

@@ -8,6 +8,7 @@
 #define JSON_STRUCT_CODEC_HEADER_SIZE 21
 
 const uint8_t json_struct_codec_magic[4] = { 'J', 'B', 'L', 'B' };
+const uint8_t json_struct_codec_version = 1;
 
 // little-endian read of a uint64_t from an address
 static uint64_t
@@ -38,8 +39,9 @@ set_u64_le(uint8_t *dest, uint64_t value)
     dest[7] = (uint8_t) ((value >> 56) & 0xFF);
 }
 
-// extract the json and binary payloads from the `json+struct` codec data buffer passed
-// in note that the output pointers reference memory inside the buffer passed in
+// Extract the json and binary payloads from the `json+struct` codec data buffer.
+// Note that the output pointers `json` and `binary` reference memory
+// inside the `metadata` buffer passed in.
 void
 json_struct_codec_get_components(uint8_t *metadata, tsk_size_t metadata_length,
     uint8_t **json, tsk_size_t *json_length, uint8_t **binary, tsk_size_t *binary_length)
@@ -54,7 +56,7 @@ json_struct_codec_get_components(uint8_t *metadata, tsk_size_t metadata_length,
         errx(EXIT_FAILURE, "bad magic bytes.");
 
     uint8_t version = metadata[4];
-    if (version != 1)
+    if (version != json_struct_codec_version)
         errx(EXIT_FAILURE, "bad version number.");
 
     uint64_t json_length_u64 = load_u64_le(metadata + 5);
@@ -89,15 +91,14 @@ json_struct_codec_get_components(uint8_t *metadata, tsk_size_t metadata_length,
     *binary_length = (tsk_size_t) binary_length_u64;
 }
 
-// malloc and return a data buffer for the `json+struct` codec that contains the given
-// components
+// malloc and return a data buffer for the `json+struct` codec
+// that contains the given components
 void
 json_struct_codec_create_buffer(const uint8_t *json, tsk_size_t json_length,
     const uint8_t *binary, tsk_size_t binary_length, uint8_t **buffer,
     tsk_size_t *buffer_length)
 {
-    // first figure out the total length of the codec's data and allocate the buffer for
-    // it
+    // figure out the total length of the codec's data and allocate the buffer for it
     tsk_size_t header_length = JSON_STRUCT_CODEC_HEADER_SIZE;
     tsk_size_t padding_length = (8 - ((header_length + json_length) & 0x07)) % 8;
     tsk_size_t total_length
@@ -108,7 +109,7 @@ json_struct_codec_create_buffer(const uint8_t *json, tsk_size_t json_length,
 
     // then set up the bytes for the codec header
     memcpy(bytes, json_struct_codec_magic, 4);
-    bytes[4] = 1;
+    bytes[4] = json_struct_codec_version;
     set_u64_le(bytes + 5, (uint64_t) json_length);
     set_u64_le(bytes + 13, (uint64_t) binary_length);
 
@@ -146,13 +147,13 @@ main(int argc, char **argv)
 
     // print the recovered data to demonstrate that the round-trip worked
     // note that the JSON data is not NULL-terminated unless you put a NULL there!
-    printf("JSON payload: %.*s\n", (int) decoded_json_length, decoded_json);
+    printf("JSON: %.*s\n", (int) decoded_json_length, decoded_json);
 
-    printf("Binary payload:");
-    for (tsk_size_t binary_index = 0; binary_index < decoded_binary_length;
-        binary_index++)
-        printf(" %#04x", decoded_binary[binary_index]);
+    printf("Binary data:");
+    for (tsk_size_t j = 0; j < decoded_binary_length; j++)
+        printf(" %#04x", decoded_binary[j]);
     printf("\n");
 
+    free(metadata);
     return EXIT_SUCCESS;
 }

docs/c-api.rst

@@ -959,13 +959,37 @@ parse metadata using an external JSON library, and for
 struct-encoded metadata the values can be directly unpacked.
 Examples of both can be found in 
 `the SLiM code <https://messerlab.github.com/slim/>`_.
+(In Python, tskit automatically decodes both JSON and binary
+metadata and provides it as Python-data-typed metadata,
+just as for other codecs.)
 
 The :ref:`"json+struct" <sec_metadata_codecs_jsonstruct>`_
 metadata codec is a little less straightforward to use,
 so we provide here an example of how to write to it
-and read from it in C.
+and read from it in C. See :ref:`sec_metadata_codes_jsonstruct` 
+for details of how the metadata is encoded.
+
+The structure of this example is as follows:
+
+1. Values specific to the metadata's header (e.g., the magic bytes `JBLB`).
+2. Functions that encode/decode `uint_64t`, used to store the lengths
+    of the two components in the header.
+3. A method to "read" the metadata: really, to get pointers to the
+    json and struct components.
+4. A method to write the metadata, again just given pointers to
+    and lengths of the two components.
+5. The program itself just round-trips a very simple chunk of metadata,
+    consisting of the JSON "`{"a": 1}`" and some binary `uint_8t` bytes ("`1234`").
 
 .. literalinclude:: ../c/examples/json_struct_metadata.c
     :language: c
 
+Much of the complexity of the code is careful error checking of the lengths.
 
+Here ``json_struct_codec_get_components`` takes a pointer to binary metadata
+and returns pointers to *within that memory*.
+A different approach might have copied the two portions of the metadata
+into two buffers (to then be decoded, for instance).
+However, that would double the memory footprint,
+and since this codec is intended for large metadata,
+we did not use that approach in this example.