Add compression context cache

BUILD.bazel

@@ -203,6 +203,7 @@ cc_library(
         "src/lib/OpenEXRCore/internal_ht_common.h",
         "src/lib/OpenEXRCore/internal_huf.c",
         "src/lib/OpenEXRCore/internal_huf.h",
+        "src/lib/OpenEXRCore/internal_legacy_structs.h",
         "src/lib/OpenEXRCore/internal_memory.h",
         "src/lib/OpenEXRCore/internal_opaque.h",
         "src/lib/OpenEXRCore/internal_piz.c",

src/lib/OpenEXR/ImfDeepScanLineInputFile.cpp

@@ -84,7 +84,7 @@ struct ScanLineProcess
     bool                  first = true;
     bool                  counts_only = false;
     exr_chunk_info_t      cinfo;
-    exr_decode_pipeline_t decoder;
+    exr_decode_pipeline_t decoder = EXR_DECODE_PIPELINE_INITIALIZER;
 
     ScanLineProcess*      next;
 };

src/lib/OpenEXR/ImfDeepTiledInputFile.cpp

@@ -69,7 +69,7 @@ struct TileProcess
     bool                  first = true;
     bool                  counts_only = false;
     exr_chunk_info_t      cinfo;
-    exr_decode_pipeline_t decoder;
+    exr_decode_pipeline_t decoder = EXR_DECODE_PIPELINE_INITIALIZER;
 
     TileProcess*          next;
 };

src/lib/OpenEXR/ImfScanLineInputFile.cpp

@@ -65,7 +65,7 @@ struct ScanLineProcess
     exr_result_t          last_decode_err = EXR_ERR_UNKNOWN;
     bool                  first = true;
     exr_chunk_info_t      cinfo;
-    exr_decode_pipeline_t decoder;
+    exr_decode_pipeline_t decoder = EXR_DECODE_PIPELINE_INITIALIZER;
 
     // requirement to use process group
     ScanLineProcess* next;

src/lib/OpenEXR/ImfTiledInputFile.cpp

@@ -60,7 +60,7 @@ struct TileProcess
 
     bool                  first = true;
     exr_chunk_info_t      cinfo;
-    exr_decode_pipeline_t decoder;
+    exr_decode_pipeline_t decoder = EXR_DECODE_PIPELINE_INITIALIZER;
 
     TileProcess*          next;
 };

src/lib/OpenEXRCore/compression.c

@@ -13,6 +13,7 @@
 #include "internal_coding.h"
 #include "internal_file.h"
 #include "internal_huf.h"
+#include "internal_legacy_structs.h"
 
 #include "OpenEXRConfigInternal.h"
 
@@ -141,42 +142,115 @@ exr_compress_buffer (
 
 /**************************************/
 
-exr_result_t
-exr_uncompress_buffer (
-    exr_const_context_t ctxt,
-    const void*         in,
-    size_t              in_bytes,
-    void*               out,
-    size_t              out_bytes_avail,
-    size_t*             actual_out)
+static struct libdeflate_decompressor*
+create_deflate_context (exr_const_context_t ctxt)
 {
-    struct libdeflate_decompressor* decomp;
-    enum libdeflate_result          res;
-    size_t                          actual_in_bytes;
 #ifdef EXR_USE_CONFIG_DEFLATE_STRUCT
     struct libdeflate_options opt = {
         .sizeof_options = sizeof (struct libdeflate_options),
         .malloc_func    = ctxt ? ctxt->alloc_fn : internal_exr_alloc,
         .free_func      = ctxt ? ctxt->free_fn : internal_exr_free};
 #endif
 
-//    if (in_bytes == out_bytes_avail)
-//    {
-//        if (actual_out) *actual_out = in_bytes;
-//        if (in != out)
-//            memcpy(out, in, in_bytes);
-//
-//        return EXR_ERR_SUCCESS;
-//    }
-
 #ifdef EXR_USE_CONFIG_DEFLATE_STRUCT
-    decomp = libdeflate_alloc_decompressor_ex (&opt);
+    return libdeflate_alloc_decompressor_ex (&opt);
 #else
     libdeflate_set_memory_allocator (
         ctxt ? ctxt->alloc_fn : internal_exr_alloc,
         ctxt ? ctxt->free_fn : internal_exr_free);
-    decomp = libdeflate_alloc_decompressor ();
+    return libdeflate_alloc_decompressor ();
 #endif
+}
+
+static inline exr_result_t
+translate_deflate_result (
+    enum libdeflate_result res,
+    size_t actual_in_bytes,
+    size_t in_bytes)
+{
+    if (res == LIBDEFLATE_SUCCESS)
+    {
+        if (in_bytes == actual_in_bytes) return EXR_ERR_SUCCESS;
+        /* it's an error to not consume the full buffer, right? */
+    }
+    else if (res == LIBDEFLATE_INSUFFICIENT_SPACE)
+    {
+        return EXR_ERR_OUT_OF_MEMORY;
+    }
+    else if (res == LIBDEFLATE_SHORT_OUTPUT)
+    {
+        /* Decompression succeeded; *actual_out is the byte count. This is
+         * not an error when out_bytes_avail exceeds the true uncompressed
+         * size (e.g. PXR24/ZIP use padded scratch buffers). Callers that
+         * need an exact payload size must compare *actual_out (see e.g.
+         * undo_pxr24_impl). */
+        return EXR_ERR_SUCCESS;
+    }
+    return EXR_ERR_CORRUPT_CHUNK;
+}
+
+/**************************************/
+
+static void internal_free_zip_context (exr_decode_pipeline_t* decode)
+{
+    struct libdeflate_decompressor* decomp = decode->compression_context;
+    if (decomp)
+        libdeflate_free_decompressor (decomp);
+}
+
+exr_result_t internal_exr_decode_uncompress_buffer (
+    exr_decode_pipeline_t* decode,
+    const void*            in,
+    size_t                 in_bytes,
+    void*                  out,
+    size_t                 out_bytes_avail,
+    size_t*                actual_out)
+{
+    if (decode->pipe_size >= sizeof (exr_decode_pipeline_v2_t))
+    {
+        struct libdeflate_decompressor* decomp;
+        enum libdeflate_result          res;
+        size_t                          actual_in_bytes;
+
+        /* we can cache the context... */
+        if (decode->compression_context == NULL)
+        {
+            decode->compression_context = create_deflate_context (decode->context);
+            decode->free_compression_context = &internal_free_zip_context;
+        }
+        decomp = decode->compression_context;
+
+        res = libdeflate_zlib_decompress_ex (
+            decomp,
+            in,
+            in_bytes,
+            out,
+            out_bytes_avail,
+            &actual_in_bytes,
+            actual_out);
+
+        return translate_deflate_result (res, actual_in_bytes, in_bytes);
+    }
+    return exr_uncompress_buffer (decode->context, in, in_bytes,
+                                  out, out_bytes_avail, actual_out);
+}
+
+/**************************************/
+
+exr_result_t
+exr_uncompress_buffer (
+    exr_const_context_t ctxt,
+    const void*         in,
+    size_t              in_bytes,
+    void*               out,
+    size_t              out_bytes_avail,
+    size_t*             actual_out)
+{
+    struct libdeflate_decompressor* decomp;
+    enum libdeflate_result          res;
+    size_t                          actual_in_bytes;
+
+    decomp = create_deflate_context (ctxt);
     if (decomp)
     {
         res = libdeflate_zlib_decompress_ex (
@@ -190,25 +264,7 @@ exr_uncompress_buffer (
 
         libdeflate_free_decompressor (decomp);
 
-        if (res == LIBDEFLATE_SUCCESS)
-        {
-            if (in_bytes == actual_in_bytes) return EXR_ERR_SUCCESS;
-            /* it's an error to not consume the full buffer, right? */
-        }
-        else if (res == LIBDEFLATE_INSUFFICIENT_SPACE)
-        {
-            return EXR_ERR_OUT_OF_MEMORY;
-        }
-        else if (res == LIBDEFLATE_SHORT_OUTPUT)
-        {
-            /* Decompression succeeded; *actual_out is the byte count. This is
-             * not an error when out_bytes_avail exceeds the true uncompressed
-             * size (e.g. PXR24/ZIP use padded scratch buffers). Callers that
-             * need an exact payload size must compare *actual_out (see e.g.
-             * undo_pxr24_impl). */
-            return EXR_ERR_SUCCESS;
-        }
-        return EXR_ERR_CORRUPT_CHUNK;
+        return translate_deflate_result (res, actual_in_bytes, in_bytes);
     }
     return EXR_ERR_OUT_OF_MEMORY;
 }

src/lib/OpenEXRCore/decoding.c

@@ -11,6 +11,8 @@
 #include "internal_decompress.h"
 #include "internal_structs.h"
 #include "internal_xdr.h"
+#include "internal_xdr.h"
+#include "internal_legacy_structs.h"
 
 #include <stdio.h>
 #include <string.h>
@@ -284,8 +286,8 @@ exr_decoding_initialize (
     const exr_chunk_info_t* cinfo,
     exr_decode_pipeline_t*  decode)
 {
+    size_t                pipe_ver;
     exr_result_t          rv;
-    exr_decode_pipeline_t nil = {0};
     exr_const_priv_part_t part;
 
     if (!ctxt) return EXR_ERR_MISSING_CONTEXT_ARG;
@@ -296,9 +298,23 @@ exr_decoding_initialize (
     if (part_index < 0 || part_index >= ctxt->num_parts)
         return EXR_ERR_ARGUMENT_OUT_OF_RANGE;
 
-    part = ctxt->parts[part_index];
+    if (decode->pipe_size != sizeof (exr_decode_pipeline_v2_t))
+    {
+        /* There was no check for the pipe_size during the v1 struct
+         * versions of the library. If we add more versions of the
+         * struct later, update this condition to check for the new
+         * versions but just let the old assumption that the
+         * size is correct keep running
+         */
+        decode->pipe_size = sizeof (exr_decode_pipeline_v1_t);
+    }
+
+    pipe_ver = decode->pipe_size;
+    part     = ctxt->parts[part_index];
 
-    *decode = nil;
+    // ensure everything is 0 / null...
+    memset (decode, 0, pipe_ver);
+    decode->pipe_size = pipe_ver;
 
     if (part->storage_mode == EXR_STORAGE_DEEP_SCANLINE ||
         part->storage_mode == EXR_STORAGE_DEEP_TILED)
@@ -675,11 +691,12 @@ exr_decoding_run (
 exr_result_t
 exr_decoding_destroy (exr_const_context_t ctxt, exr_decode_pipeline_t* decode)
 {
+    size_t pipe_ver;
+
     if (!ctxt) return EXR_ERR_MISSING_CONTEXT_ARG;
 
     if (decode)
     {
-        exr_decode_pipeline_t nil = {0};
         if (decode->channels != decode->_quick_chan_store)
             ctxt->free_fn (decode->channels);
 
@@ -722,7 +739,21 @@ exr_decoding_destroy (exr_const_context_t ctxt, exr_decode_pipeline_t* decode)
             EXR_TRANSCODE_BUFFER_PACKED_SAMPLES,
             &(decode->packed_sample_count_table),
             &(decode->packed_sample_count_alloc_size));
-        *decode = nil;
+
+        if (decode->pipe_size > sizeof (exr_decode_pipeline_v1_t))
+        {
+            /* safe to check the compression context member and free routine */
+            if (decode->compression_context && decode->free_compression_context)
+            {
+                decode->free_compression_context (decode);
+            }
+        }
+
+        /* zilch out everything except the version field */
+        pipe_ver = decode->pipe_size;
+
+        memset (decode, 0, pipe_ver);
+        decode->pipe_size = pipe_ver;
     }
     return EXR_ERR_SUCCESS;
 }

src/lib/OpenEXRCore/internal_decompress.h

@@ -24,6 +24,18 @@ exr_result_t internal_exr_undo_rle (
     void*                  uncompressed_data,
     uint64_t               uncompressed_size);
 
+/*
+ * rather than just straight using uncompress buffer
+ * uses a cached zip context in the decode struct...
+ */
+exr_result_t internal_exr_decode_uncompress_buffer (
+    exr_decode_pipeline_t* decode,
+    const void*            in,
+    size_t                 in_bytes,
+    void*                  out,
+    size_t                 out_bytes_avail,
+    size_t*                actual_out);
+
 exr_result_t internal_exr_undo_zip (
     exr_decode_pipeline_t* decode,
     const void*            compressed_data,

src/lib/OpenEXRCore/internal_ht.cpp

@@ -6,6 +6,7 @@
 #include <limits>
 #include <string>
 #include <fstream>
+#include <memory>
 
 #include <openjph/ojph_arch.h>
 #include <openjph/ojph_file.h>
@@ -17,6 +18,7 @@
 #include "openexr_decode.h"
 #include "openexr_encode.h"
 #include "internal_ht_common.h"
+#include "internal_legacy_structs.h"
 
 /**
  * OpenJPH output file that is backed by a fixed-size memory buffer
@@ -157,6 +159,20 @@ class staticmem_outfile : public ojph::outfile_base
     ojph::ui8 *cur_ptr;
   };
 
+struct ht_context_cache
+{
+    std::vector<CodestreamChannelInfo> cs_to_file_ch;
+    ojph::codestream cs;
+};
+
+static void destroy_ht_decompress_context (exr_decode_pipeline_t* decode)
+{
+    ht_context_cache* ctxt = static_cast<ht_context_cache*> (decode->compression_context);
+    delete ctxt;
+    decode->compression_context = NULL;
+    decode->free_compression_context = NULL;
+}
+
 static exr_result_t
 ht_undo_impl (
     exr_decode_pipeline_t* decode,
@@ -167,10 +183,29 @@ ht_undo_impl (
 {
     exr_result_t rv = EXR_ERR_SUCCESS;
 
-    std::vector<CodestreamChannelInfo> cs_to_file_ch (decode->channel_count);
+    std::unique_ptr<ht_context_cache> legacy_support;
 
-    /* read the channel map */
+    ht_context_cache* ctxt;
+    if (decode->pipe_size >= sizeof (exr_decode_pipeline_v2_t))
+    {
+        ctxt = static_cast<ht_context_cache*> (decode->compression_context);
+        if ( ! ctxt )
+        {
+            ctxt = new ht_context_cache;
+            decode->compression_context = ctxt;
+            decode->free_compression_context = &destroy_ht_decompress_context;
+        }
+    }
+    else
+    {
+        legacy_support = std::make_unique<ht_context_cache>();
+        ctxt = legacy_support.get();
+    }
+
+    std::vector<CodestreamChannelInfo> &cs_to_file_ch = ctxt->cs_to_file_ch;
+    cs_to_file_ch.resize(decode->channel_count);
 
+    /* read the channel map */
     size_t header_sz;
     try
     {
@@ -214,7 +249,8 @@ ht_undo_impl (
         reinterpret_cast<const ojph::ui8*> (compressed_data) + header_sz,
         codestream_sz);
 
-    ojph::codestream cs;
+    ojph::codestream &cs = ctxt->cs;
+    cs.restart ();
     cs.read_headers (&infile);
 
     ojph::param_siz siz = cs.access_siz ();