Generate BSF CSVs both with and without /third_party tests

build.sh now produces test results for:
{stable, experimental} × {with /third_party, without /third_party}.

This will let wpt.fyi expose a UI toggle to switch between different BSF chart views.
wpt.fyi will be updated in a separate change to default to the without-third-party.csv.

/third_party/ introduces Temporal tests as individual tests instead of using the subtest WPT convention. These test failures misrepresent the overall interoperability of the web by causing the single browser failure graph to be completely dominated by a feature not yet shipped by all browsers.

Author: yulunwu2

browser-specific-failures.js

@@ -21,8 +21,41 @@ flags.defineString('output', null,
     '{stable, experimental}-browser-specific-failures.csv');
 flags.defineBoolean('experimental', false,
     'Calculate metrics for experimental runs.');
+flags.defineBoolean('exclude-third-party', false,
+    'Exclude /third_party tests from BSF scoring.');
 flags.parse();
 
+// WPT result paths to exclude from browser-specific-failure scoring.
+// /third_party/ introduces tests as individual tests
+// instead of using the subtest WPT convention. These test failtures
+// misrepresent the overal interoperability of the web by causing the
+// single browser failure graph to be completely dominated by a feature
+// not yet shipped by all browsers.
+const EXCLUDED_PATHS = ['/third_party'];
+
+// Returns a copy of |tree| with any subtrees whose path appears in
+// |excludedPaths| removed. Subtrees not on a path to an excluded entry are
+// shared by reference (no deep copy), so the operation is cheap.
+function pruneExcludedPaths(tree, excludedPaths, currentPath = '') {
+  const relevant = excludedPaths.filter(p =>
+    p.startsWith(currentPath + '/'));
+  if (relevant.length === 0) {
+    return tree;
+  }
+
+  const newTrees = {};
+  for (const [name, child] of Object.entries(tree.trees)) {
+    const childPath = `${currentPath}/${name}`;
+    if (relevant.includes(childPath)) {
+      continue;
+    }
+    newTrees[name] = relevant.some(p => p.startsWith(childPath + '/')) ?
+        pruneExcludedPaths(child, relevant, childPath) :
+        child;
+  }
+  return {...tree, trees: newTrees};
+}
+
 
 async function main() {
   // Sort the products so that output files are consistent.
@@ -39,6 +72,7 @@ async function main() {
   const from = moment(flags.get('from'));
   const to = moment(flags.get('to'));
   const experimental = flags.get('experimental');
+  const excludeThirdParty = flags.get('exclude-third-party');
   const alignedRuns = await lib.runs.fetchAlignedRunsFromServer(
       products, from, to, experimental);
 
@@ -78,6 +112,9 @@ async function main() {
         throw new Error('Run JSON contains "tree" field; code needs changed.');
       }
       run.tree = await lib.results.getGitTree(repo, run);
+      if (excludeThirdParty) {
+        run.tree = pruneExcludedPaths(run.tree, EXCLUDED_PATHS);
+      }
     }
   }
   after = Date.now();

build.sh

@@ -27,14 +27,21 @@ update_bsf_csv() {
   if [[ $1 == *"experimental"* ]]; then
     EXPERIMENTAL_FLAG="--experimental"
   fi
+  local EXCLUDE_THIRD_PARTY_FLAG=""
+  if [[ $1 == *"without-third-party"* ]]; then
+    EXCLUDE_THIRD_PARTY_FLAG="--exclude-third-party"
+  fi
 
   node browser-specific-failures.js \
-    ${EXPERIMENTAL_FLAG} --from=${FROM_DATE} --to=${TO_DATE} \
+    ${EXPERIMENTAL_FLAG} ${EXCLUDE_THIRD_PARTY_FLAG} \
+    --from=${FROM_DATE} --to=${TO_DATE} \
     --output=${OUTPUT}
 }
 
 update_bsf_csv out/data/stable-browser-specific-failures.csv
 update_bsf_csv out/data/experimental-browser-specific-failures.csv
+update_bsf_csv out/data/stable-browser-specific-failures-without-third-party.csv
+update_bsf_csv out/data/experimental-browser-specific-failures-without-third-party.csv
 
 update_interop_year() {
   local YEAR="${1}"