Generate BSF CSVs both with and without /third_party tests (#275)

Adds an --include-third-party flag to browser-specific-failures.js
(off by default), and extends build.sh to produce four CSVs per run:
the existing {stable,experimental}-browser-specific-failures.csv files
will be updated to exclude /third_party while the new -with-third-party.csv
variants will include them.

This will let wpt.fyi expose a UI toggle to switch between different BSF chart views.
wpt.fyi will be updated in a separate change to include the third_party toggle.

/third_party/ introduces Temporal tests as individual tests instead of using the
subtest WPT convention. These test failures misrepresent the overall interoperability
of the web by causing the single browser failure graph to be completely dominated
by a feature not yet shipped by all browsers.

Author: yulunwu2

browser-specific-failures.js

@@ -21,6 +21,8 @@ flags.defineString('output', null,
     '{stable, experimental}-browser-specific-failures.csv');
 flags.defineBoolean('experimental', false,
     'Calculate metrics for experimental runs.');
+flags.defineBoolean('include-third-party', false,
+    'Include /third_party tests in BSF scoring.');
 flags.parse();
 
 
@@ -39,6 +41,7 @@ async function main() {
   const from = moment(flags.get('from'));
   const to = moment(flags.get('to'));
   const experimental = flags.get('experimental');
+  const includeThirdParty = flags.get('include-third-party');
   const alignedRuns = await lib.runs.fetchAlignedRunsFromServer(
       products, from, to, experimental);
 
@@ -78,6 +81,10 @@ async function main() {
         throw new Error('Run JSON contains "tree" field; code needs changed.');
       }
       run.tree = await lib.results.getGitTree(repo, run);
+      if (!includeThirdParty) {
+        run.tree = lib.browserSpecific.pruneExcludedPaths(
+            run.tree, ['/third_party']);
+      }
     }
   }
   after = Date.now();

build.sh

@@ -27,14 +27,21 @@ update_bsf_csv() {
   if [[ $1 == *"experimental"* ]]; then
     EXPERIMENTAL_FLAG="--experimental"
   fi
+  local INCLUDE_THIRD_PARTY_FLAG=""
+  if [[ $1 == *"with-third-party"* ]]; then
+    INCLUDE_THIRD_PARTY_FLAG="--include-third-party"
+  fi
 
   node browser-specific-failures.js \
-    ${EXPERIMENTAL_FLAG} --from=${FROM_DATE} --to=${TO_DATE} \
+    ${EXPERIMENTAL_FLAG} ${INCLUDE_THIRD_PARTY_FLAG} \
+    --from=${FROM_DATE} --to=${TO_DATE} \
     --output=${OUTPUT}
 }
 
 update_bsf_csv out/data/stable-browser-specific-failures.csv
 update_bsf_csv out/data/experimental-browser-specific-failures.csv
+update_bsf_csv out/data/stable-browser-specific-failures-with-third-party.csv
+update_bsf_csv out/data/experimental-browser-specific-failures-with-third-party.csv
 
 update_interop_year() {
   local YEAR="${1}"

lib/browser-specific.js

@@ -386,4 +386,27 @@ function scoreBrowserSpecificFailures(runs, expectedBrowsers) {
   return new Map(scores.map((score, i) => [runs[i].browser_name, score]));
 }
 
-module.exports = {scoreBrowserSpecificFailures};
+// Returns a copy of |tree| with any subtrees whose path appears in
+// |excludedPaths| removed. Subtrees not on a path to an excluded entry are
+// shared by reference (no deep copy), so the operation is cheap.
+function pruneExcludedPaths(tree, excludedPaths, currentPath = '') {
+  const relevant = excludedPaths.filter(p =>
+    p.startsWith(currentPath + '/'));
+  if (relevant.length === 0) {
+    return tree;
+  }
+
+  const newTrees = {};
+  for (const [name, child] of Object.entries(tree.trees)) {
+    const childPath = `${currentPath}/${name}`;
+    if (relevant.includes(childPath)) {
+      continue;
+    }
+    newTrees[name] = relevant.some(p => p.startsWith(childPath + '/')) ?
+        pruneExcludedPaths(child, relevant, childPath) :
+        child;
+  }
+  return {...tree, trees: newTrees};
+}
+
+module.exports = {scoreBrowserSpecificFailures, pruneExcludedPaths};

test/browser-specific.js

@@ -547,4 +547,78 @@ describe('browser-specific.js', () => {
       assert.deepEqual(scores, new Map([['chrome', 0.5], ['firefox', 0.5], ['safari', 0.0]]));
     });
   });
+
+  describe('pruneExcludedPaths', () => {
+    function collectTestPaths(tree, currentPath = '') {
+      const paths = [];
+      for (const name of Object.keys(tree.tests || {})) {
+        paths.push(`${currentPath}/${name}`);
+      }
+      for (const [name, child] of Object.entries(tree.trees || {})) {
+        paths.push(...collectTestPaths(child, `${currentPath}/${name}`));
+      }
+      return paths.sort();
+    }
+
+    it('removes a top-level excluded subtree', () => {
+      const tree = new TreeBuilder()
+          .addTest('third_party/test.html', 'PASS')
+          .addTest('foo/test.html', 'PASS')
+          .build();
+
+      const pruned = browserSpecific.pruneExcludedPaths(tree, ['/third_party']);
+
+      assert.deepEqual(collectTestPaths(pruned), ['/foo/test.html']);
+    });
+
+    it('does not remove a same-named subtree under a different parent', () => {
+      // /foo/third_party/ is NOT on the excluded list and must be retained.
+      const tree = new TreeBuilder()
+          .addTest('third_party/a.html', 'PASS')
+          .addTest('foo/third_party/b.html', 'PASS')
+          .addTest('bar/baz.html', 'PASS')
+          .build();
+
+      const pruned = browserSpecific.pruneExcludedPaths(tree, ['/third_party']);
+
+      assert.deepEqual(collectTestPaths(pruned), [
+        '/bar/baz.html',
+        '/foo/third_party/b.html',
+      ]);
+    });
+
+    it('does not remove a sibling that shares a name prefix', () => {
+      // /third_party_extra is a sibling of /third_party and must be retained;
+      // the exclusion is by exact path segment, not string prefix.
+      const tree = new TreeBuilder()
+          .addTest('third_party/a.html', 'PASS')
+          .addTest('third_party_extra/b.html', 'PASS')
+          .build();
+
+      const pruned = browserSpecific.pruneExcludedPaths(tree, ['/third_party']);
+
+      assert.deepEqual(collectTestPaths(pruned), ['/third_party_extra/b.html']);
+    });
+
+    it('preserves all tests when no excluded path matches', () => {
+      const tree = new TreeBuilder()
+          .addTest('foo/a.html', 'PASS')
+          .addTest('bar/b.html', 'PASS')
+          .build();
+
+      const pruned = browserSpecific.pruneExcludedPaths(tree, ['/third_party']);
+
+      assert.deepEqual(collectTestPaths(pruned), ['/bar/b.html', '/foo/a.html']);
+    });
+
+    it('handles an empty exclusion list', () => {
+      const tree = new TreeBuilder()
+          .addTest('third_party/test.html', 'PASS')
+          .build();
+
+      const pruned = browserSpecific.pruneExcludedPaths(tree, []);
+
+      assert.strictEqual(pruned, tree);
+    });
+  });
 });