Generate BSF CSVs both with and without /third_party tests

Add an --exclude-third-party flag to browser-specific-failures.js.
build.sh now produces test results for:
{stable, experimental} × {with /third_party, without /third_party}.

This will let wpt.fyi expose a UI toggle to switch between different BSF chart views.
wpt.fyi will be updated in a separate change to default to the without-third-party.csv.

Author: yulunwu2

browser-specific-failures.js

@@ -21,8 +21,41 @@ flags.defineString('output', null,
     '{stable, experimental}-browser-specific-failures.csv');
 flags.defineBoolean('experimental', false,
     'Calculate metrics for experimental runs.');
+flags.defineBoolean('exclude-third-party', false,
+    'Exclude /third_party tests from BSF scoring.');
 flags.parse();
 
+// WPT result paths to exclude from browser-specific-failure scoring.
+// /third_party/ introduces tests as individual tests
+// instead of using the subtest WPT convention. These test failtures
+// misrepresent the overal interoperability of the web by causing the
+// single browser failure graph to be completely dominated by a feature
+// not yet shipped by all browsers.
+const EXCLUDED_PATHS = ['/third_party'];
+
+// Returns a copy of |tree| with any subtrees whose path appears in
+// |excludedPaths| removed. Subtrees not on a path to an excluded entry are
+// shared by reference (no deep copy), so the operation is cheap.
+function pruneExcludedPaths(tree, excludedPaths, currentPath = '') {
+  const relevant = excludedPaths.filter(p =>
+    p.startsWith(currentPath + '/'));
+  if (relevant.length === 0) {
+    return tree;
+  }
+
+  const newTrees = {};
+  for (const [name, child] of Object.entries(tree.trees)) {
+    const childPath = `${currentPath}/${name}`;
+    if (relevant.includes(childPath)) {
+      continue;
+    }
+    newTrees[name] = relevant.some(p => p.startsWith(childPath + '/')) ?
+        pruneExcludedPaths(child, relevant, childPath) :
+        child;
+  }
+  return {...tree, trees: newTrees};
+}
+
 
 async function main() {
   // Sort the products so that output files are consistent.
@@ -39,6 +72,7 @@ async function main() {
   const from = moment(flags.get('from'));
   const to = moment(flags.get('to'));
   const experimental = flags.get('experimental');
+  const excludeThirdParty = flags.get('exclude-third-party');
   const alignedRuns = await lib.runs.fetchAlignedRunsFromServer(
       products, from, to, experimental);
 
@@ -78,6 +112,9 @@ async function main() {
         throw new Error('Run JSON contains "tree" field; code needs changed.');
       }
       run.tree = await lib.results.getGitTree(repo, run);
+      if (excludeThirdParty) {
+        run.tree = pruneExcludedPaths(run.tree, EXCLUDED_PATHS);
+      }
     }
   }
   after = Date.now();

build.sh

@@ -27,14 +27,21 @@ update_bsf_csv() {
   if [[ $1 == *"experimental"* ]]; then
     EXPERIMENTAL_FLAG="--experimental"
   fi
+  local EXCLUDE_THIRD_PARTY_FLAG=""
+  if [[ $1 == *"without-third-party"* ]]; then
+    EXCLUDE_THIRD_PARTY_FLAG="--exclude-third-party"
+  fi
 
   node browser-specific-failures.js \
-    ${EXPERIMENTAL_FLAG} --from=${FROM_DATE} --to=${TO_DATE} \
+    ${EXPERIMENTAL_FLAG} ${EXCLUDE_THIRD_PARTY_FLAG} \
+    --from=${FROM_DATE} --to=${TO_DATE} \
     --output=${OUTPUT}
 }
 
 update_bsf_csv out/data/stable-browser-specific-failures.csv
 update_bsf_csv out/data/experimental-browser-specific-failures.csv
+update_bsf_csv out/data/stable-browser-specific-failures-without-third-party.csv
+update_bsf_csv out/data/experimental-browser-specific-failures-without-third-party.csv
 
 update_interop_year() {
   local YEAR="${1}"