Codec-Compare version 0.3.1 (#20)

Add setting for displaying 10th and 90th percentiles as error bars.
Only works for absolute mode for now.

Author: y-guyon

CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changelog
 
+## v0.3.1
+
+- Add setting for displaying 10th and 90th percentiles as error bars.
+
 ## v0.3.0
 
 - Add absolute metrics mode. Default mode remains relative ratios.

package.json

@@ -1,6 +1,6 @@
 {
   "name": "codec_compare",
-  "version": "0.3.0",
+  "version": "0.3.1",
   "description": "Codec performance comparison tool",
   "publisher": "Google LLC",
   "author": "Yannis Guyon",

readme_preview.png

@@ -208,7 +208,7 @@ export class CodecCompare extends LitElement {
           </p>
 
           <p id="credits">
-            Codec Compare beta version 0.3.0<br>
+            Codec Compare version 0.3.1<br>
             <a href="https://github.com/webmproject/codec-compare">
               Sources on GitHub
             </a>

src/codec_compare.ts

@@ -17,7 +17,7 @@
  * See https://en.wikipedia.org/wiki/Geometric_mean.
  */
 export class GeometricMean {
-  /** Add a value to be part of the geometric mean. */
+  /** Adds a value to be part of the geometric mean. */
   add(value: number) {
     this.product *= value;
     ++this.numValues;

src/geometric_mean.ts

@@ -16,6 +16,7 @@ import {getFinalValue} from './constant';
 import {areFieldsComparable, Batch, Field, FieldId} from './entry';
 import {GeometricMean} from './geometric_mean';
 import {Match} from './matcher';
+import {Quantile} from './quantile';
 
 /** References two fields from two selected batches to compare data points. */
 export class FieldMetric {
@@ -32,19 +33,36 @@ export class FieldMetric {
 export class FieldMetricStats {
   // Absolute values
   absoluteArithmeticMean = 0;
+  absoluteArithmeticLowQuantile = 0;
+  absoluteArithmeticHighQuantile = 0;
 
   // Relative ratios
   geometricMean = 1;
   minRatio = 1;
   maxRatio = 1;
   relativeArithmeticMean = 1;
+  relativeArithmeticLowQuantile = 1;
+  relativeArithmeticHighQuantile = 1;
 
   getAbsoluteMean() {
     return this.absoluteArithmeticMean;
   }
+  getAbsoluteLowQuantile() {
+    return this.absoluteArithmeticLowQuantile;
+  }
+  getAbsoluteHighQuantile() {
+    return this.absoluteArithmeticHighQuantile;
+  }
+
   getRelativeMean(geometric: boolean) {
     return geometric ? this.geometricMean : this.relativeArithmeticMean;
   }
+  getRelativeLowQuantile() {
+    return this.relativeArithmeticLowQuantile;
+  }
+  getRelativeHighQuantile() {
+    return this.relativeArithmeticHighQuantile;
+  }
 }
 
 /** Aggregated stats for a unique source media. */
@@ -187,6 +205,8 @@ export function computeStats(
 
     const fieldStats = new FieldMetricStats();
     let numDataPoints = 0;
+    const leftQuantile = new Quantile();
+    const rightQuantile = new Quantile();
     const geometricMean = new GeometricMean();
     let leftSum = 0;
     let rightSum = 0;
@@ -200,6 +220,8 @@ export function computeStats(
       //       it is safer to surface that in the final user interface than
       //       silently skipping that data point here.
 
+      leftQuantile.add(leftValue);
+      rightQuantile.add(rightValue);
       if (numDataPoints === 0) {
         fieldStats.minRatio = ratio;
         fieldStats.maxRatio = ratio;
@@ -214,8 +236,15 @@ export function computeStats(
     }
     if (numDataPoints > 0) {
       fieldStats.absoluteArithmeticMean = leftSum / numDataPoints;
+      // TODO(yguyon): Do not hardcode quantiles but use State fields.
+      fieldStats.absoluteArithmeticLowQuantile = leftQuantile.get(0.1);
+      fieldStats.absoluteArithmeticHighQuantile = leftQuantile.get(0.9);
       fieldStats.geometricMean = geometricMean.get();
       fieldStats.relativeArithmeticMean = getRatio(leftSum, rightSum);
+      fieldStats.relativeArithmeticLowQuantile =
+          getRatio(leftQuantile.get(0.1), rightQuantile.get(0.1));
+      fieldStats.relativeArithmeticHighQuantile =
+          getRatio(leftQuantile.get(0.9), rightQuantile.get(0.9));
     }
     stats.push(fieldStats);
   }

src/metric.ts

@@ -226,6 +226,10 @@ export class PlotUi {
     for (const group of this.state.groups) {
       const xMeans: number[] = [];
       const yMeans: number[] = [];
+      const xLowErrors: number[] = [];
+      const xHighErrors: number[] = [];
+      const yLowErrors: number[] = [];
+      const yHighErrors: number[] = [];
       const meanTexts: string[] = [];
       const colors: string[] = [];
       const batchIndices: number[] = [];
@@ -250,14 +254,37 @@ export class PlotUi {
             this.state.metrics.findIndex((metric) => metric === xMetric);
         const yMetricIndex =
             this.state.metrics.findIndex((metric) => metric === yMetric);
-        if (showRelativeRatios) {
-          xMeans.push(batchSelection.stats[xMetricIndex].getRelativeMean(
-              useGeometricMean));
-          yMeans.push(batchSelection.stats[yMetricIndex].getRelativeMean(
-              useGeometricMean));
-        } else {
-          xMeans.push(batchSelection.stats[xMetricIndex].getAbsoluteMean());
-          yMeans.push(batchSelection.stats[yMetricIndex].getAbsoluteMean());
+        const xMean = showRelativeRatios ?
+            batchSelection.stats[xMetricIndex].getRelativeMean(
+                useGeometricMean) :
+            batchSelection.stats[xMetricIndex].getAbsoluteMean();
+        const yMean = showRelativeRatios ?
+            batchSelection.stats[yMetricIndex].getRelativeMean(
+                useGeometricMean) :
+            batchSelection.stats[yMetricIndex].getAbsoluteMean();
+        xMeans.push(xMean);
+        yMeans.push(yMean);
+        if (!useGeometricMean) {
+          if (this.state.horizontalQuantile === 0.1) {
+            const xLowError = showRelativeRatios ?
+                batchSelection.stats[xMetricIndex].getRelativeLowQuantile() :
+                batchSelection.stats[xMetricIndex].getAbsoluteLowQuantile();
+            const xHighError = showRelativeRatios ?
+                batchSelection.stats[xMetricIndex].getRelativeHighQuantile() :
+                batchSelection.stats[xMetricIndex].getAbsoluteHighQuantile();
+            xLowErrors.push(xMean - xLowError);
+            xHighErrors.push(xHighError - xMean);
+          }
+          if (this.state.verticalQuantile === 0.1) {
+            const yLowError = showRelativeRatios ?
+                batchSelection.stats[yMetricIndex].getRelativeLowQuantile() :
+                batchSelection.stats[yMetricIndex].getAbsoluteLowQuantile();
+            const yHighError = showRelativeRatios ?
+                batchSelection.stats[yMetricIndex].getRelativeHighQuantile() :
+                batchSelection.stats[yMetricIndex].getAbsoluteHighQuantile();
+            yLowErrors.push(yMean - yLowError);
+            yHighErrors.push(yHighError - yMean);
+          }
         }
         meanTexts.push(batch.name);
         colors.push(batch.color);
@@ -269,7 +296,7 @@ export class PlotUi {
         let hovertemplate = '%{text} vs ' + referenceBatch.name + '<br>';
         hovertemplate += xAxis + ': %{x:.2f}<br>';
         hovertemplate += yAxis + ': %{y:.2f}';
-        const geomeans: PlotlyData = {
+        let geomeans: PlotlyData = {
           x: xMeans,
           y: yMeans,
           text: meanTexts,
@@ -284,6 +311,25 @@ export class PlotUi {
           isAggregated: true,
           batchIndices,
         };
+        // TODO(yguyon): Fix quantiles then enable them with showRelativeRatios.
+        if (!showRelativeRatios && !useGeometricMean &&
+            this.state.horizontalQuantile === 0.1) {
+          geomeans.error_x = {
+            type: 'data',
+            symmetric: false,
+            array: xLowErrors,
+            arrayminus: xHighErrors,
+          };
+        }
+        if (!showRelativeRatios && !useGeometricMean &&
+            this.state.verticalQuantile === 0.1) {
+          geomeans.error_y = {
+            type: 'data',
+            symmetric: false,
+            array: yLowErrors,
+            arrayminus: yHighErrors,
+          };
+        }
         this.plotlyData.push(geomeans);
       }
     }

src/plot_ui.ts

@@ -0,0 +1,40 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/** Accumulates values and computes the given quantile. */
+export class Quantile {
+  /**
+   * Adds a value to be part of the points on which the quantile is computed.
+   */
+  add(value: number) {
+    this.values.push(value);
+    this.valuesAreSorted = false;
+  }
+
+  /** Computes and returns the quantile of all the added values so far. */
+  get(quantile: number) {
+    if (this.values.length === 0) {
+      return 0;
+    }
+    if (!this.valuesAreSorted) {
+      this.values.sort((a: number, b: number) => a - b);
+      this.valuesAreSorted = true;
+    }
+    // Naive quantile computation for now.
+    return this.values[Math.floor((this.values.length - 1) * quantile)];
+  }
+
+  private values = new Array<number>();
+  private valuesAreSorted = true;
+}

src/quantile.ts

@@ -118,6 +118,48 @@ export class SettingsUi extends LitElement {
             Geometric mean
           </span>
         </div>
+        <div class="settingGroup">
+          <span title="Do not show horizontal error bars">
+            Hide X error bars
+          </span>
+          <mwc-switch id="settingUseHorizontalErrorBars" ?selected=${
+    !this.state.showRelativeRatios && !this.state.useGeometricMean &&
+        this.state.horizontalQuantile === 0.1}
+            ?disabled=${
+        this.state.showRelativeRatios ||
+        this.state.useGeometricMean}
+            @click=${() => {
+      this.state.horizontalQuantile =
+          this.state.horizontalQuantile === 0.1 ? 0.5 : 0.1;
+      dispatch(EventType.MATCHER_OR_METRIC_CHANGED);
+      dispatch(EventType.SETTINGS_CHANGED);
+    }}>
+          </mwc-switch>
+          <span title="Display the 10th and 90th percentiles as horizontal error bars">
+            Show X error bars
+          </span>
+        </div>
+        <div class="settingGroup">
+          <span title="Do not show vertical error bars">
+            Hide Y error bars
+          </span>
+          <mwc-switch id="settingUseVerticalErrorBars" ?selected=${
+    !this.state.showRelativeRatios && !this.state.useGeometricMean &&
+        this.state.verticalQuantile === 0.1}
+            ?disabled=${
+        this.state.showRelativeRatios ||
+        this.state.useGeometricMean}
+            @click=${() => {
+      this.state.verticalQuantile =
+          this.state.verticalQuantile === 0.1 ? 0.5 : 0.1;
+      dispatch(EventType.MATCHER_OR_METRIC_CHANGED);
+      dispatch(EventType.SETTINGS_CHANGED);
+    }}>
+          </mwc-switch>
+          <span title="Display the 10th and 90th percentiles as vertical error bars">
+            Show Y error bars
+          </span>
+        </div>
         <div class="settingGroup">
           <span title="Only display the first rows of the filtered or matched data point tables">
             Show some rows

src/settings_ui.ts

@@ -54,8 +54,10 @@ export class State {
   metrics: FieldMetric[] = [];
   plotMetricVertical: FieldMetric|undefined = undefined;    // among metrics
   plotMetricHorizontal: FieldMetric|undefined = undefined;  // among metrics
-  verticalLogScale = true;     // Use a linear scale if false.
-  horizontalLogScale = false;  // Use a linear scale if false.
+  verticalLogScale = true;     // Use a linear scale on the Y axis if false.
+  horizontalLogScale = false;  // Use a linear scale on the X axis if false.
+  verticalQuantile = 0.5;      // Display Y error bars if in [0:0.5[.
+  horizontalQuantile = 0.5;    // Display X error bars if in [0:0.5[.
 
   /**
    * If true, each absolute data point or relative match is shown on the plot