Skip to content

Commit bfb28f6

Browse files
committed
Generate BSF CSVs both with and without /third_party tests
Add an --exclude-third-party flag to browser-specific-failures.js. build.sh now produces test results for: {stable, experimental} × {with /third_party, without /third_party}. This will let wpt.fyi expose a UI toggle to switch between different BSF chart views. wpt.fyi will be updated in a separate change to default to the without-third-party.csv.
1 parent 0f777f2 commit bfb28f6

2 files changed

Lines changed: 45 additions & 1 deletion

File tree

browser-specific-failures.js

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,41 @@ flags.defineString('output', null,
2121
'{stable, experimental}-browser-specific-failures.csv');
2222
flags.defineBoolean('experimental', false,
2323
'Calculate metrics for experimental runs.');
24+
flags.defineBoolean('exclude-third-party', false,
25+
'Exclude /third_party tests from BSF scoring.');
2426
flags.parse();
2527

28+
// WPT result paths to exclude from browser-specific-failure scoring.
29+
// /third_party/ introduces tests as individual tests
30+
// instead of using the subtest WPT convention. These test failtures
31+
// misrepresent the overal interoperability of the web by causing the
32+
// single browser failure graph to be completely dominated by a feature
33+
// not yet shipped by all browsers.
34+
const EXCLUDED_PATHS = ['/third_party'];
35+
36+
// Returns a copy of |tree| with any subtrees whose path appears in
37+
// |excludedPaths| removed. Subtrees not on a path to an excluded entry are
38+
// shared by reference (no deep copy), so the operation is cheap.
39+
function pruneExcludedPaths(tree, excludedPaths, currentPath = '') {
40+
const relevant = excludedPaths.filter(p =>
41+
p.startsWith(currentPath + '/'));
42+
if (relevant.length === 0) {
43+
return tree;
44+
}
45+
46+
const newTrees = {};
47+
for (const [name, child] of Object.entries(tree.trees)) {
48+
const childPath = `${currentPath}/${name}`;
49+
if (relevant.includes(childPath)) {
50+
continue;
51+
}
52+
newTrees[name] = relevant.some(p => p.startsWith(childPath + '/')) ?
53+
pruneExcludedPaths(child, relevant, childPath) :
54+
child;
55+
}
56+
return {...tree, trees: newTrees};
57+
}
58+
2659

2760
async function main() {
2861
// Sort the products so that output files are consistent.
@@ -39,6 +72,7 @@ async function main() {
3972
const from = moment(flags.get('from'));
4073
const to = moment(flags.get('to'));
4174
const experimental = flags.get('experimental');
75+
const excludeThirdParty = flags.get('exclude-third-party');
4276
const alignedRuns = await lib.runs.fetchAlignedRunsFromServer(
4377
products, from, to, experimental);
4478

@@ -78,6 +112,9 @@ async function main() {
78112
throw new Error('Run JSON contains "tree" field; code needs changed.');
79113
}
80114
run.tree = await lib.results.getGitTree(repo, run);
115+
if (excludeThirdParty) {
116+
run.tree = pruneExcludedPaths(run.tree, EXCLUDED_PATHS);
117+
}
81118
}
82119
}
83120
after = Date.now();

build.sh

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,14 +27,21 @@ update_bsf_csv() {
2727
if [[ $1 == *"experimental"* ]]; then
2828
EXPERIMENTAL_FLAG="--experimental"
2929
fi
30+
local EXCLUDE_THIRD_PARTY_FLAG=""
31+
if [[ $1 == *"without-third-party"* ]]; then
32+
EXCLUDE_THIRD_PARTY_FLAG="--exclude-third-party"
33+
fi
3034

3135
node browser-specific-failures.js \
32-
${EXPERIMENTAL_FLAG} --from=${FROM_DATE} --to=${TO_DATE} \
36+
${EXPERIMENTAL_FLAG} ${EXCLUDE_THIRD_PARTY_FLAG} \
37+
--from=${FROM_DATE} --to=${TO_DATE} \
3338
--output=${OUTPUT}
3439
}
3540

3641
update_bsf_csv out/data/stable-browser-specific-failures.csv
3742
update_bsf_csv out/data/experimental-browser-specific-failures.csv
43+
update_bsf_csv out/data/stable-browser-specific-failures-without-third-party.csv
44+
update_bsf_csv out/data/experimental-browser-specific-failures-without-third-party.csv
3845

3946
update_interop_year() {
4047
local YEAR="${1}"

0 commit comments

Comments
 (0)