exrmetrics print disk size of part (#2424)

* Adding feature which allows the disk sizes of induvidual parts in a multipart EXR to be printed as one of the exrmetrics stats. The current method only works when no output file is specified because we use the write position of the dummy output stream to derive bytes written per part. Added flag to enable part size printout.

Signed-off-by: Aries Moczar <arcmantis@protulae.com>

* Improved help description. Appended to website/bin/exrmetrics.rst

Signed-off-by: Aries Moczar <arcmantis@protulae.com>

* Improved help message to be more clear. Modified parse() to report error when user attempts to use --part-disk-size and -o or vice versa in command. Added test in test_exrmetrics.py to verify json ouput for --part-disk-size.

Signed-off-by: Aries Moczar <arcmantis@protulae.com>

---------

Signed-off-by: Aries Moczar <arcmantis@protulae.com>
Co-authored-by: Cary Phillips <cary@ilm.com>

Author: Arcmantis18

src/bin/exrmetrics/exrmetrics.cpp

@@ -764,11 +764,21 @@ writeFile (
     MultiPartOutputFile& out,
     vector<partData>&    parts,
     fileMetrics&         metrics,
-    bool                 logPerformance)
+    bool                 logPerformance,
+    OStream*             os = nullptr)
 {
     //
     // Call initialization functions, Read image from source.
     //
+
+    uint64_t lastPosition  = 0;
+    uint64_t currentPosition = 0;
+
+    if (os)
+    {
+        lastPosition = os->tellp();
+    }
+
     for (size_t p = 0; p < parts.size (); ++p)
     {
         string type = out.header (p).type ();
@@ -780,6 +790,7 @@ writeFile (
                 outpart,
                 parts[p].readBuf.scanlineBuf,
                 logPerformance ? &metrics.stats[p].writePerf : nullptr);
+            if (os) currentPosition = os->tellp();
         }
         else if (type == TILEDIMAGE)
         {
@@ -788,6 +799,7 @@ writeFile (
                 outpart,
                 parts[p].readBuf.tiledBuf,
                 logPerformance ? &metrics.stats[p].writePerf : nullptr);
+            if (os) currentPosition = os->tellp();
         }
         else if (type == DEEPSCANLINE)
         {
@@ -796,6 +808,7 @@ writeFile (
                 outpart,
                 parts[p].readBuf.deepbuf,
                 logPerformance ? &metrics.stats[p].writePerf : nullptr);
+            if (os) currentPosition = os->tellp();
         }
         else if (type == DEEPTILE)
         {
@@ -804,6 +817,13 @@ writeFile (
                 outpart,
                 parts[p].readBuf.deepbuf,
                 logPerformance ? &metrics.stats[p].writePerf : nullptr);
+            if (os) currentPosition = os->tellp();
+        }
+
+        if (os)
+        {
+            metrics.stats[p].sizeOnDisk = currentPosition - lastPosition;
+            lastPosition = currentPosition;
         }
     }
 }
@@ -1117,13 +1137,14 @@ exrmetrics (
             {
                 out = new MultiPartOutputFile (
                     outFileName, outHeaders.data (), outHeaders.size ());
+                writeFile (*out, parts, metrics, true);
             }
             else
             {
                 out = new MultiPartOutputFile (
                     ostream, outHeaders.data (), outHeaders.size ());
+                writeFile (*out, parts, metrics, true,&ostream);
             }
-            writeFile (*out, parts, metrics, true);
             delete out;
 
             if (reread)

src/bin/exrmetrics/exrmetrics.h

@@ -55,6 +55,7 @@ struct partStats
         countRereadPerf; // for deep only, time rereading the per-pixel sample count
     std::vector<double>
         rereadPerf; // for deep, times reading the sample count, otherwise times reading the entire data
+    uint64_t sizeOnDisk = 0; // record compressed size of part on disk.
 
     partSizeData sizeData;
 };

src/bin/exrmetrics/main.cpp

@@ -85,6 +85,8 @@ usageMessage (ostream& stream, const char* program_name, bool verbose = false)
                " --time list                  comma separated list of operations to report timing for.\n"
                "                              operations can be any of read,write,reread (use --time none for no timing)\n"
                " --no-size                    don't output size data\n"
+               " --part-disk-size             Output the on-disk size of the data portion of each part in a multipart EXR file\n"
+               "                              (including chunk headers). File output (-o) is not supported when using this option\n"
                " --json                       print output as JSON dictionary (Default mode)\n"
                " --csv                        print output in csv mode. If passes>1, show median timing\n"
                "                              default is JSON mode\n"
@@ -116,6 +118,7 @@ struct options
     int                      passes  = 1;
     int                      timing  = TIME_READ | TIME_REREAD | TIME_WRITE;
     bool                     outputSizeData = true;
+    bool                     outputPartSizeOnDisk = false;
     bool                     verbose        = false;
     bool                     csv            = false;
     std::vector<PixelMode>   pixelModes;
@@ -196,6 +199,7 @@ printPartStats (
     const partStats& data,
     const string     indent,
     int              timing,
+    bool             partSize,
     bool             raw,
     bool             stats)
 {
@@ -236,6 +240,15 @@ printPartStats (
         out << indent << "\"re-read time\": ";
         printTiming (data.rereadPerf, out, raw, stats);
     }
+
+    if (partSize)
+    {
+        if (output) { out << ",\n"; }
+        output = true;
+        out << indent << "\"size on disk\": "
+            << data.sizeOnDisk
+            << "\n";
+    }
 }
 
 void
@@ -244,6 +257,7 @@ jsonStats (
     list<runData>& data,
     bool           outputSizeData,
     int            timing,
+    bool           partSize,
     bool           raw,
     bool           stats)
 {
@@ -370,7 +384,7 @@ jsonStats (
         {
             out << ",\n";
             printPartStats (
-                out, run.metrics.totalStats, "      ", timing, raw, stats);
+                out, run.metrics.totalStats, "      ", timing,false, raw, stats);
         }
         if (timing && run.metrics.stats.size () > 1)
         {
@@ -388,6 +402,7 @@ jsonStats (
                     run.metrics.stats[part],
                     "          ",
                     timing,
+                    partSize,
                     raw,
                     stats);
                 out << "\n        }";
@@ -546,6 +561,11 @@ main (int argc, char** argv)
         return 1;
     }
 
+    // we are only able to compute part disk size if no output file is specified,
+    // causing memory output stream to be used and allowing us to measure data output.
+
+    bool showPartSizeOnDisk = opts.outputPartSizeOnDisk && !opts.outFile;
+
     if (opts.timing || opts.outputSizeData)
     {
 
@@ -556,7 +576,7 @@ main (int argc, char** argv)
         else
         {
             jsonStats (
-                cout, data, opts.outputSizeData, opts.timing, true, true);
+                cout, data, opts.outputSizeData, opts.timing,showPartSizeOnDisk, true, true);
         }
     }
 
@@ -778,6 +798,12 @@ options::parse (int argc, char* argv[])
                 cerr << "-o output filename can only be specified once\n";
                 return 1;
             }
+            if (outputPartSizeOnDisk)
+            {
+                cerr << "--part-disk-size cannot be used with file output (-o)\n";
+                return 1;
+            }
+
             outFile = argv[i + 1];
             i += 2;
         }
@@ -870,6 +896,17 @@ options::parse (int argc, char* argv[])
             outputSizeData = false;
             i += 1;
         }
+        else if (!strcmp (argv[i], "--part-disk-size"))
+        {
+            if (outFile)
+            {
+                cerr << "--part-disk-size cannot be used with file output (-o)\n";
+                return 1;
+            }
+
+            outputPartSizeOnDisk = true;
+            i += 1;
+        }
         else if (!strcmp (argv[i], "-i"))
         {
             if (i > argc - 2)

src/test/bin/test_exrmetrics.py

@@ -72,4 +72,9 @@ def cleanup():
 data = json.loads(result.stdout)
 assert(len(data)==1),"\n Unexpected list size in JSON object"
 
+# test --part-disk-size json output
+result = do_run ([exrmetrics, test_images["multipart"], "--part-disk-size"])
+data = json.loads(result.stdout)
+assert(len(data)==1),"\n Unexpected list size in JSON object"
+
 print("success")

website/bin/exrmetrics.rst

@@ -81,6 +81,11 @@ Options:
 
    Don't output size data.
 
+.. describe:: --part-disk-size
+
+   Output the on-disk size of the data portion of each part in a multipart EXR file (including chunk headers). 
+   Useful for determining compression impact on part data. File output (``-o``) is not supported when using this option.
+
 .. describe:: --json
 
    Print output as JSON dictionary (default).