Skip to content

Commit 5b1a32f

Browse files
y-guyony-guyon
authored
Codec-Compare version 0.3.0 (#18)
Add absolute metrics mode. Default mode remains relative ratios.
1 parent 55f2a75 commit 5b1a32f

17 files changed

Lines changed: 337 additions & 131 deletions

‎CHANGELOG.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,9 @@
11
# Changelog
22

3+
## v0.3.0
4+
5+
- Add absolute metrics mode. Default mode remains relative ratios.
6+
37
## v0.2.6
48

59
- Fix cross-batch source asset filtering by tag.

‎package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "codec_compare",
3-
"version": "0.2.6",
3+
"version": "0.3.0",
44
"description": "Codec performance comparison tool",
55
"publisher": "Google LLC",
66
"author": "Yannis Guyon",

‎readme_preview.png‎

-35 Bytes
Loading

‎src/batch_merger.ts‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -110,28 +110,33 @@ function mergeBatchesWithSameCodec(batches: BatchSelection[]): BatchSelection|
110110
const mergedStats = new FieldMetricStats();
111111
mergedStats.minRatio = batches[0].stats[stat].minRatio;
112112
mergedStats.maxRatio = batches[0].stats[stat].maxRatio;
113-
mergedStats.arithmeticMean = 0;
113+
mergedStats.absoluteArithmeticMean = 0;
114+
mergedStats.relativeArithmeticMean = 0;
114115

115116
const geometricMean = new GeometricMean();
116117
let weightSum = 0;
117118
for (const batch of batches) {
118119
// Weigh each batch by its match count.
119120
const weight = batch.matchedDataPoints.rows.length;
120121
// TODO: Check the validity of the aggregation methods.
122+
mergedStats.absoluteArithmeticMean +=
123+
batch.stats[stat].absoluteArithmeticMean * weight;
121124
for (let p = 0; p < batch.matchedDataPoints.rows.length; ++p) {
122125
geometricMean.add(batch.stats[stat].geometricMean);
123126
}
124127
mergedStats.minRatio =
125128
Math.min(mergedStats.minRatio, batch.stats[stat].minRatio);
126129
mergedStats.maxRatio =
127130
Math.max(mergedStats.maxRatio, batch.stats[stat].maxRatio);
128-
mergedStats.arithmeticMean += batch.stats[stat].arithmeticMean * weight;
131+
mergedStats.relativeArithmeticMean +=
132+
batch.stats[stat].relativeArithmeticMean * weight;
129133
weightSum += weight;
130134
}
131135

132136
if (weightSum === 0) return undefined;
137+
mergedStats.absoluteArithmeticMean /= weightSum;
133138
mergedStats.geometricMean = geometricMean.get();
134-
mergedStats.arithmeticMean /= weightSum;
139+
mergedStats.relativeArithmeticMean /= weightSum;
135140

136141
mergedBatchSelection.stats.push(mergedStats);
137142
}

‎src/batch_selections_ui.ts‎

Lines changed: 31 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ import {css, html, LitElement} from 'lit';
1818
import {customElement, property} from 'lit/decorators.js';
1919

2020
import {BatchSelection} from './batch_selection';
21-
import {Field} from './entry';
21+
import {Field, fieldUnit} from './entry';
2222
import {dispatch, EventType, listen} from './events';
2323
import {FieldFilter} from './filter';
2424
import {FieldMetric, FieldMetricStats} from './metric';
@@ -47,7 +47,8 @@ export class BatchSelectionsUi extends LitElement {
4747
const referenceBatch =
4848
this.state.batchSelections[this.state.referenceBatchSelectionIndex]
4949
.batch;
50-
if (batchSelectionIndex === this.state.referenceBatchSelectionIndex) {
50+
if (this.state.showRelativeRatios &&
51+
batchSelectionIndex === this.state.referenceBatchSelectionIndex) {
5152
const title = referenceBatch.name + ' is used as reference';
5253
return html`
5354
<td class="stat" title="${title}">
@@ -65,14 +66,34 @@ export class BatchSelectionsUi extends LitElement {
6566
`;
6667
}
6768

68-
const mean = stats.getMean(this.state.useGeometricMean);
69-
const title = field.displayName + ' of ' + batch.name + ' is ' +
70-
String(mean.toFixed(4)) + '× ' + referenceBatch.name;
71-
return html`
72-
<td class="stat" title="${title}">
73-
${getRelativePercent(mean)}
74-
</td>
75-
`;
69+
if (this.state.showRelativeRatios) {
70+
const mean = stats.getRelativeMean(this.state.useGeometricMean);
71+
const title = `${field.displayName} of ${batch.name} is ${
72+
mean.toFixed(4)}× ${referenceBatch.name}`;
73+
return html`
74+
<td class="stat" title="${title}">
75+
${getRelativePercent(mean)}
76+
</td>`;
77+
} else {
78+
let mean = stats.getAbsoluteMean();
79+
let multiple = '';
80+
const unit = fieldUnit(field.id);
81+
const title = `average ${field.displayName} of ${batch.name} is ${
82+
mean} ${unit}`;
83+
84+
if (mean > 1000000 && unit === 'B') {
85+
mean /= 1000000;
86+
multiple = 'M';
87+
} else if (mean > 1000 && unit === 'B') {
88+
mean /= 1000;
89+
multiple = 'k';
90+
}
91+
const meanStr = mean < 1 ? mean.toFixed(4) :
92+
mean < 100 ? mean.toFixed(2) :
93+
mean.toFixed(0);
94+
return html`
95+
<td class="stat" title="${title}">${meanStr}${multiple}${unit}</td>`;
96+
}
7697
}
7798

7899
private filterChipText(field: Field, fieldFilter: FieldFilter) {

‎src/codec_compare.ts‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -208,7 +208,7 @@ export class CodecCompare extends LitElement {
208208
</p>
209209
210210
<p id="credits">
211-
Codec Compare beta version 0.2.6<br>
211+
Codec Compare beta version 0.3.0<br>
212212
<a href="https://github.com/webmproject/codec-compare">
213213
Sources on GitHub
214214
</a>
@@ -392,7 +392,8 @@ export class CodecCompare extends LitElement {
392392
box-shadow: 4px 0 8px rgba(0, 0, 0, 0.2);
393393
transition: width 0.1s;
394394
}
395-
#leftBar:hover {
395+
#leftBar:hover,
396+
#leftBar:has(settings-ui:focus-within) {
396397
width: 500px;
397398
}
398399
#leftBarContent {

‎src/entry.ts‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -150,6 +150,30 @@ function fieldPrettyName(id: FieldId, name: string): string {
150150
return name.replaceAll('_', ' ');
151151
}
152152

153+
export function fieldUnit(id: FieldId): string {
154+
if (DISTORTION_METRIC_FIELD_IDS.includes(id)) {
155+
// Display distortion metrics as uppercase.
156+
return 'dB';
157+
}
158+
switch (id) {
159+
case FieldId.WIDTH:
160+
case FieldId.HEIGHT:
161+
return 'px';
162+
case FieldId.ENCODED_SIZE:
163+
return 'B';
164+
case FieldId.ENCODED_BITS_PER_PIXEL:
165+
return 'bpp';
166+
case FieldId.MEGAPIXELS:
167+
return 'MP';
168+
case FieldId.ENCODING_DURATION:
169+
case FieldId.DECODING_DURATION:
170+
case FieldId.RAW_DECODING_DURATION:
171+
return 's';
172+
default:
173+
return '';
174+
}
175+
}
176+
153177
/** Data column in a Batch. For example: source_image, encoding_time etc. */
154178
export class Field {
155179
id: FieldId;

‎src/matcher.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -422,8 +422,11 @@ export function getDataPoints(
422422
}
423423

424424
if (bestMatch !== undefined) {
425+
// Do not match a data point more than once for a given batch pair.
426+
// TODO: Add a setting for matching a data point multiple times.
425427
bucket[bestIndexInBucket] = bucket[bucket.length - 1];
426428
bucket.pop();
429+
427430
matches.push(bestMatch);
428431
}
429432
}

‎src/matches_table_ui.ts‎

Lines changed: 98 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ import {customElement, property} from 'lit/decorators.js';
1717
import {styleMap} from 'lit/directives/style-map.js';
1818

1919
import {BatchSelection} from './batch_selection';
20-
import {areFieldsComparable, Batch, Field, FieldId} from './entry';
20+
import {areFieldsComparable, Batch, Field, FieldId, fieldUnit} from './entry';
2121
import {dispatch, EventType} from './events';
2222
import {FieldMatcher, Match} from './matcher';
2323
import {FieldMetric, getRatio} from './metric';
@@ -38,8 +38,8 @@ export class MatchesTableUi extends LitElement {
3838

3939
private renderFieldHeader(batch: Batch, fieldIndex: number, rowspan: number) {
4040
return html`
41-
<th rowspan=${rowspan} title="${batch.fields[fieldIndex].description}"
42-
class="headerRow">
41+
<th colspan=2 rowspan=${rowspan}
42+
title="${batch.fields[fieldIndex].description}" class="headerRow">
4343
${batch.fields[fieldIndex].displayName}
4444
</th>`;
4545
}
@@ -53,34 +53,36 @@ export class MatchesTableUi extends LitElement {
5353
selection.rows[selectionRowIndex][selectionFieldIndex];
5454
const referenceValue =
5555
reference.rows[referenceRowIndex][referenceFieldIndex];
56-
if (selectionField.id === FieldId.EFFORT ||
57-
selectionField.id === FieldId.QUALITY) {
58-
const referenceStyle = {'color': reference.color};
59-
const selectionStyle = {'color': selection.color};
60-
return html`
61-
<td class="encodingSettingCell">
62-
<span style=${styleMap(referenceStyle)}>${referenceValue}</span>
63-
<span style=${styleMap(selectionStyle)}>${selectionValue}</span>
64-
</td>`;
65-
}
66-
6756
const referenceField = reference.fields[referenceFieldIndex];
6857
const isNumber = selectionField.isNumber && referenceField.isNumber;
6958
const cssClass = isNumber ? 'numberCell' : '';
7059

71-
if (isNumber && selectionField.id !== FieldId.WIDTH &&
72-
selectionField.id !== FieldId.HEIGHT &&
73-
selectionField.id !== FieldId.FRAME_COUNT &&
74-
selectionField.id !== FieldId.MEGAPIXELS) {
75-
const ratio =
76-
getRatio(selectionValue as number, referenceValue as number);
77-
return html`<td class="${cssClass}">${getRelativePercent(ratio)}</td>`;
78-
}
79-
if (selectionValue === referenceValue) {
80-
return html`<td class="${cssClass}">${selectionValue}</td>`;
60+
if (selectionField.id !== FieldId.EFFORT &&
61+
selectionField.id !== FieldId.QUALITY) {
62+
if (this.state.showRelativeRatios && isNumber &&
63+
selectionField.id !== FieldId.WIDTH &&
64+
selectionField.id !== FieldId.HEIGHT &&
65+
selectionField.id !== FieldId.FRAME_COUNT &&
66+
selectionField.id !== FieldId.MEGAPIXELS) {
67+
const ratio =
68+
getRatio(selectionValue as number, referenceValue as number);
69+
return html`<td colspan=2 class="${cssClass}">${
70+
getRelativePercent(ratio)}</td>`;
71+
}
72+
if (selectionValue === referenceValue) {
73+
return html`<td colspan=2 class="${cssClass}">${selectionValue}</td>`;
74+
}
8175
}
76+
const referenceStyle = {'color': reference.color};
77+
const selectionStyle = {'color': selection.color};
78+
// TODO: Align all floating point numbers in same column at decimal point.
8279
return html`
83-
<td class="${cssClass}">${selectionValue} ≠ ${referenceValue}</td>`;
80+
<td style=${styleMap(referenceStyle)} class="${cssClass}">
81+
${referenceValue}
82+
</td>
83+
<td style=${styleMap(selectionStyle)} class="${cssClass}">
84+
${selectionValue}
85+
</td>`;
8486
}
8587

8688
// Header rows (two needed because some cells use a rowspan)
@@ -89,8 +91,8 @@ export class MatchesTableUi extends LitElement {
8991
reference: Batch, matchers: FieldMatcher[], metrics: FieldMetric[],
9092
referenceSharedFieldIndices: number[]) {
9193
return html`<tr>
92-
<th colspan=${matchers.length} class="headerRow">Matchers</th>
93-
<th colspan=${metrics.length} class="headerRow">Metrics</th>
94+
<th colspan=${matchers.length * 2} class="headerRow">Matchers</th>
95+
<th colspan=${metrics.length * 2} class="headerRow">Metrics</th>
9496
${referenceSharedFieldIndices.map((fieldIndex) => {
9597
return this.renderFieldHeader(reference, fieldIndex, /*rowspan=*/ 2);
9698
})}
@@ -112,6 +114,66 @@ export class MatchesTableUi extends LitElement {
112114
</tr>`;
113115
}
114116

117+
private renderAbsoluteMeanRows(
118+
selection: Batch, numColumns: number, matchers: FieldMatcher[],
119+
metricIndices: number[], selectionSharedFieldIndices: number[]) {
120+
const selectionStyle = {'color': selection.color};
121+
return html`
122+
<tr>
123+
<th colspan=${numColumns * 2}>Arithmetic means</th>
124+
</tr>
125+
<tr>
126+
<td colspan=${matchers.length * 2} class="missing"></td>
127+
${metricIndices.map((metricIndex) => {
128+
const mean = this.batchSelection.stats[metricIndex].getAbsoluteMean();
129+
const metric = this.state.metrics[metricIndex];
130+
const fieldIndex = metric.fieldIndices[this.batchSelection.batch.index];
131+
const field = this.batchSelection.batch.fields[fieldIndex];
132+
return html`
133+
<td colspan=2 class="numberCell" style=${styleMap(selectionStyle)}>
134+
${mean}${fieldUnit(field.id)}
135+
</td>`;
136+
})}
137+
<td colspan=${selectionSharedFieldIndices.length * 2} class="missing">
138+
</td>
139+
</tr>`;
140+
}
141+
142+
private renderRelativeMeanRows(
143+
numColumns: number, matchers: FieldMatcher[], metricIndices: number[],
144+
selectionSharedFieldIndices: number[]) {
145+
return html`
146+
<tr>
147+
<th colspan=${numColumns * 2}>Arithmetic means</th>
148+
</tr>
149+
<tr>
150+
<td colspan=${matchers.length * 2} class="missing"></td>
151+
${metricIndices.map((metricIndex) => {
152+
const mean = this.batchSelection.stats[metricIndex].getRelativeMean(
153+
/*geometric=*/ false);
154+
return html`
155+
<td colspan=2 class="numberCell">${getRelativePercent(mean)}</td>`;
156+
})}
157+
<td colspan=${selectionSharedFieldIndices.length * 2} class="missing">
158+
</td>
159+
</tr>
160+
161+
<tr>
162+
<th colspan=${numColumns * 2}>Geometric means</th>
163+
</tr>
164+
<tr>
165+
<td colspan=${matchers.length * 2} class="missing"></td>
166+
${metricIndices.map((metricIndex) => {
167+
const mean = this.batchSelection.stats[metricIndex].getRelativeMean(
168+
/*geometric=*/ true);
169+
return html`
170+
<td colspan=2 class="numberCell">${getRelativePercent(mean)}</td>`;
171+
})}
172+
<td colspan=${selectionSharedFieldIndices.length * 2} class="missing">
173+
</td>
174+
</tr>`;
175+
}
176+
115177
override render() {
116178
if (!this.batchSelection) return html``;
117179
const reference =
@@ -215,7 +277,7 @@ export class MatchesTableUi extends LitElement {
215277
truncatedRows = html`
216278
<tr>
217279
<td @click=${onDisplayHiddenRow}
218-
colspan=${numColumns} class="hiddenRow">
280+
colspan=${numColumns * 2} class="hiddenRow">
219281
${rows.length - 100} hidden rows. Click to expand.
220282
</td>
221283
</tr>`;
@@ -230,34 +292,14 @@ export class MatchesTableUi extends LitElement {
230292
${this.renderSecondHeaderRow(reference, matchers, metrics)}
231293
${rows.map(renderRow)}
232294
${truncatedRows}
233-
234-
<tr>
235-
<th colspan=${numColumns}>Arithmetic means</th>
236-
</tr>
237-
<tr>
238-
<td colspan=${matchers.length} class="missing"></td>
239-
${metricIndices.map((metricIndex) => {
240-
const mean =
241-
this.batchSelection.stats[metricIndex].getMean(/*geometric=*/ false);
242-
return html`<td class="numberCell">${getRelativePercent(mean)}</td>`;
243-
})}
244-
<td colspan=${selectionSharedFieldIndices.length} class="missing">
245-
</td>
246-
</tr>
247-
248-
<tr>
249-
<th colspan=${numColumns}>Geometric means</th>
250-
</tr>
251-
<tr>
252-
<td colspan=${matchers.length} class="missing"></td>
253-
${metricIndices.map((metricIndex) => {
254-
const mean =
255-
this.batchSelection.stats[metricIndex].getMean(/*geometric=*/ true);
256-
return html`<td class="numberCell">${getRelativePercent(mean)}</td>`;
257-
})}
258-
<td colspan=${selectionSharedFieldIndices.length} class="missing">
259-
</td>
260-
</tr>
295+
${
296+
this.state.showRelativeRatios ?
297+
this.renderRelativeMeanRows(
298+
numColumns, matchers, metricIndices,
299+
selectionSharedFieldIndices) :
300+
this.renderAbsoluteMeanRows(
301+
selection, numColumns, matchers, metricIndices,
302+
selectionSharedFieldIndices)}
261303
</table>`;
262304
}
263305

@@ -299,9 +341,6 @@ export class MatchesTableUi extends LitElement {
299341
.numberCell {
300342
text-align: right;
301343
}
302-
.encodingSettingCell {
303-
text-align: center;
304-
}
305344
306345
tr {
307346
background: var(--mdc-theme-background);

0 commit comments

Comments
 (0)