Move C code for json+struct codec to an example in the docs

Author: petrelharp

c/examples/Makefile

@@ -23,7 +23,8 @@ TSKIT_SOURCE=../tskit/*.c ../subprojects/kastore/kastore.c
 targets = api_structure error_handling \
 	haploid_wright_fisher streaming \
 	tree_iteration tree_traversal \
-	take_ownership
+	take_ownership \
+	json_struct_metadata
 
 all: $(targets)
 

c/examples/json_struct_metadata.c

@@ -0,0 +1,159 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <err.h>
+#include <string.h>
+#include <tskit.h>
+
+// these are properties of the ``json+struct`` codec, documented in tskit
+#define JSON_STRUCT_HEADER_SIZE 21
+
+const uint8_t json_struct_codec_magic[4] = { 'J', 'B', 'L', 'B' };
+const uint8_t json_struct_codec_version = 1;
+
+// little-endian read of a uint64_t from an address
+static uint64_t
+load_u64_le(const uint8_t *p)
+{
+    uint64_t value = (uint64_t) p[0];
+    value |= (uint64_t) p[1] << 8;
+    value |= (uint64_t) p[2] << 16;
+    value |= (uint64_t) p[3] << 24;
+    value |= (uint64_t) p[4] << 32;
+    value |= (uint64_t) p[5] << 40;
+    value |= (uint64_t) p[6] << 48;
+    value |= (uint64_t) p[7] << 56;
+    return value;
+}
+
+// little-endian write of a uint64_t to an address
+static void
+set_u64_le(uint8_t *dest, uint64_t value)
+{
+    dest[0] = (uint8_t) (value & 0xFF);
+    dest[1] = (uint8_t) ((value >> 8) & 0xFF);
+    dest[2] = (uint8_t) ((value >> 16) & 0xFF);
+    dest[3] = (uint8_t) ((value >> 24) & 0xFF);
+    dest[4] = (uint8_t) ((value >> 32) & 0xFF);
+    dest[5] = (uint8_t) ((value >> 40) & 0xFF);
+    dest[6] = (uint8_t) ((value >> 48) & 0xFF);
+    dest[7] = (uint8_t) ((value >> 56) & 0xFF);
+}
+
+// Extract the json and binary payloads from the `json+struct` codec data buffer.
+// Note that the output pointers `json` and `binary` reference memory
+// inside the `metadata` buffer passed in.
+void
+json_struct_codec_get_components(uint8_t *metadata, tsk_size_t metadata_length,
+    uint8_t **json, tsk_size_t *json_length, uint8_t **binary, tsk_size_t *binary_length)
+{
+    // check the structure of the codec header and the sizes it specifies
+    if (metadata == NULL || json == NULL || json_length == NULL || binary == NULL
+        || binary_length == NULL)
+        errx(EXIT_FAILURE, "bad parameter value.");
+    if (metadata_length < JSON_STRUCT_HEADER_SIZE)
+        errx(EXIT_FAILURE, "metadata truncated.");
+    if (memcmp(metadata, json_struct_codec_magic, sizeof(json_struct_codec_magic)) != 0)
+        errx(EXIT_FAILURE, "bad magic bytes.");
+
+    uint8_t version = metadata[4];
+    if (version != json_struct_codec_version)
+        errx(EXIT_FAILURE, "bad version number.");
+
+    uint64_t json_length_u64 = load_u64_le(metadata + 5);
+    uint64_t binary_length_u64 = load_u64_le(metadata + 13);
+    if (json_length_u64 > UINT64_MAX - (uint64_t) JSON_STRUCT_HEADER_SIZE)
+        errx(EXIT_FAILURE, "invalid length.");
+
+    // determine the number of padding bytes and do more safety checks
+    uint64_t length = (uint64_t) JSON_STRUCT_HEADER_SIZE + json_length_u64;
+    uint64_t padding_length = (8 - (length & 0x07)) % 8;
+    if (padding_length > UINT64_MAX - length)
+        errx(EXIT_FAILURE, "invalid length.");
+
+    length += padding_length;
+    if (binary_length_u64 > UINT64_MAX - length)
+        errx(EXIT_FAILURE, "invalid length.");
+
+    length += binary_length_u64;
+    if ((uint64_t) metadata_length != length)
+        errx(EXIT_FAILURE, "unexpected size.");
+
+    uint8_t *padding_start = metadata + JSON_STRUCT_HEADER_SIZE + json_length_u64;
+    for (uint64_t j = 0; j < padding_length; ++j)
+        if (*(padding_start + j) != 0)
+            errx(EXIT_FAILURE, "padding bytes are nonzero.");
+
+    // the structure of the codec data seems valid; return components
+    *json = metadata + JSON_STRUCT_HEADER_SIZE;
+    *json_length = (tsk_size_t) json_length_u64;
+
+    *binary = metadata + JSON_STRUCT_HEADER_SIZE + json_length_u64 + padding_length;
+    *binary_length = (tsk_size_t) binary_length_u64;
+}
+
+// malloc and return a data buffer for the `json+struct` codec
+// that contains the given components
+void
+json_struct_codec_create_buffer(const uint8_t *json, tsk_size_t json_length,
+    const uint8_t *binary, tsk_size_t binary_length, uint8_t **buffer,
+    tsk_size_t *buffer_length)
+{
+    // figure out the total length of the codec's data and allocate the buffer for it
+    tsk_size_t header_length = JSON_STRUCT_HEADER_SIZE;
+    tsk_size_t padding_length = (8 - ((header_length + json_length) & 0x07)) % 8;
+    tsk_size_t total_length
+        = header_length + json_length + padding_length + binary_length;
+    uint8_t *bytes = malloc(total_length);
+    if (!bytes)
+        errx(EXIT_FAILURE, "memory for buffer could not be allocated.");
+
+    // then set up the bytes for the codec header
+    memcpy(bytes, json_struct_codec_magic, 4);
+    bytes[4] = json_struct_codec_version;
+    set_u64_le(bytes + 5, (uint64_t) json_length);
+    set_u64_le(bytes + 13, (uint64_t) binary_length);
+
+    // copy in the JSON and binary data, separated by the padding bytes; the goal of the
+    // padding bytes is to ensure that the binary data is 8-byte-aligned relative to the
+    // start of the buffer
+    memcpy(bytes + header_length, json, json_length);
+    memset(bytes + header_length + json_length, 0, padding_length);
+    memcpy(bytes + header_length + json_length + padding_length, binary, binary_length);
+
+    // return the buffer and its length; the caller takes ownership of the buffer
+    *buffer = bytes;
+    *buffer_length = total_length;
+}
+
+int
+main(int argc, char **argv)
+{
+    // we start with JSON and binary payloads that we encode into a new buffer
+    // note that the JSON payload does not have to end with a trailing NULL
+    const char json_payload[] = { '{', '"', 'a', '"', ':', '1', '}' };
+    const uint8_t binary_payload[] = { 0x01, 0x02, 0x03, 0x04 };
+    uint8_t *metadata;
+    tsk_size_t metadata_length;
+
+    json_struct_codec_create_buffer((const uint8_t *) json_payload, sizeof(json_payload),
+        binary_payload, sizeof(binary_payload), &metadata, &metadata_length);
+
+    // then we decode that buffer to recover the json and binary data
+    uint8_t *decoded_json, *decoded_binary;
+    tsk_size_t decoded_json_length, decoded_binary_length;
+
+    json_struct_codec_get_components(metadata, metadata_length, &decoded_json,
+        &decoded_json_length, &decoded_binary, &decoded_binary_length);
+
+    // print the recovered data to demonstrate that the round-trip worked
+    // note that the JSON data is not NULL-terminated unless you put a NULL there!
+    printf("JSON: %.*s\n", (int) decoded_json_length, decoded_json);
+
+    printf("Binary data:");
+    for (tsk_size_t j = 0; j < decoded_binary_length; j++)
+        printf(" %#04x", decoded_binary[j]);
+    printf("\n");
+
+    free(metadata);
+    return EXIT_SUCCESS;
+}

c/meson.build

@@ -125,6 +125,9 @@ if not meson.is_subproject()
       executable('multichrom_wright_fisher_singlethreaded',
           sources: ['examples/multichrom_wright_fisher_singlethreaded.c'], 
           link_with: [tskit_lib], dependencies: lib_deps)
+      executable('json_struct_metadata',
+          sources: ['examples/json_struct_metadata.c'], 
+          link_with: [tskit_lib], dependencies: lib_deps)
 
       thread_dep = dependency('threads')
       executable('multichrom_wright_fisher',

docs/c-api.rst

@@ -949,3 +949,60 @@ nodes need to be retained, and use
 .. literalinclude:: ../c/examples/multichrom_wright_fisher.c
     :language: c
 
+----------------------------
+Reading and writing metadata
+----------------------------
+
+The C API does not provide any functionality for manipulating
+the contents of metadata. For JSON metadata it is easy to
+parse metadata using an external JSON library, and for
+struct-encoded metadata the values can be directly unpacked.
+Examples of both can be found in 
+`the SLiM code <https://messerlab.github.com/slim/>`_.
+
+The :ref:`"json+struct" <sec_metadata_codecs_jsonstruct>`
+metadata codec is a little less straightforward to use,
+so we provide here an example of how to write to it
+and read from it in C. See :ref:`sec_metadata_codecs_jsonstruct` 
+for details of how the metadata is encoded.
+(In Python, tskit automatically decodes both JSON and binary
+metadata and provides it as Python-data-typed metadata,
+just as for other codecs.)
+
+The structure of this example is as follows:
+
+1. Values specific to the metadata's header (e.g., the magic bytes `JBLB`).
+2. Functions that encode/decode `uint64_t`, used to store the lengths
+    of the two components in the header.
+3. A method to "read" the metadata: really, to get pointers to the
+    json and struct components.
+4. A method to "write" the metadata, again just given pointers to
+    and lengths of the two components.
+5. The program itself just round-trips a very simple chunk of metadata,
+    consisting of the JSON "`{"a": 1}`" and some binary `uint8_t` bytes ("`1234`").
+
+.. literalinclude:: ../c/examples/json_struct_metadata.c
+    :language: c
+
+Much of the complexity of the code is careful error checking of the lengths.
+
+Here ``json_struct_codec_get_components`` takes a pointer to binary metadata
+and returns pointers to *within that memory*.
+A different approach might have copied the two portions of the metadata
+into two buffers (to then be decoded, for instance).
+However, that would double the memory footprint,
+and since this codec is intended for large metadata,
+we did not use that approach in this example.
+
+Along the same lines, it is worth noting that this example does make a copy of
+the JSON and binary data when writing, in ``json_struct_codec_create_buffer()``,
+which doubles the memory footprint at that point, and adds the
+overhead of copying the data.  A more efficient approach would be to calculate
+the buffer length needed for the codec’s data, allocate the buffer with that
+length, and then generate the necessary JSON and binary metadata directly into
+that buffer.  This would require the metadata-generating code to be more
+closely entwined with the code for handling the json+struct codec header and
+padding bytes, and so we have chosen not to adopt that approach here, for
+pedagogical purposes; but if your use of this codec will involve large
+metadata, such an approach is recommended.
+

docs/development.md

@@ -832,7 +832,7 @@ To generate and view coverage reports for the C tests locally:
 Compile with coverage enabled:
    ```bash
    cd c
-   meson build -D b_coverage=true
+   meson setup build -D b_coverage=true
    ninja -C build
    ```
 
@@ -853,7 +853,7 @@ Lines prefixed with `#####` were never executed, lines with numbers show executi
 `lcov` can be used to create browsable HTML coverage reports:
   ```bash
   sudo apt-get install lcov  # if needed
-  lcov --capture --directory build-gcc --output-file coverage.info
+  lcov --capture --directory build --output-file coverage.info
   genhtml coverage.info --output-directory coverage_html
   firefox coverage_html/index.html
   ```