Reduce bugs.webkit.org API calls

index.js

@@ -127,17 +127,29 @@ async function pullRequestPoller() {
     logger.info('Checking for changes to WebKit-exported pull requests');
     try {
         const pull_requests = await github.get("/repos/:owner/:repo/pulls", {});
-        pull_requests.forEach(async function(pull_request) {
-            if (!webkit.related(pull_request.title)) {
-                return;
-            }
+
+        const bugToPRs = {};
+        pull_requests.forEach(function(pull_request) {
+            if (!webkit.related(pull_request.title)) return;
+            const bugId = webkit.bugIdFromTitle(pull_request.title);
+            if (bugId === null) return;
+            if (!bugToPRs[bugId]) bugToPRs[bugId] = [];
+            bugToPRs[bugId].push(pull_request);
+        });
+
+        const everFixedBugs = await webkit.everFixed(Object.keys(bugToPRs).map(Number));
+        const everFixedPRs = Object.keys(bugToPRs)
+            .filter(bugId => everFixedBugs.has(Number(bugId)))
+            .flatMap(bugId => bugToPRs[bugId].map(pull_request => ({pull_request, webkitBug: everFixedBugs.get(Number(bugId))})));
+
+        everFixedPRs.forEach(async function({pull_request, webkitBug}) {
+            const prNumber = pull_request.number;
             const metadata = await get_metadata(
-                pull_request.number,
+                prNumber,
                 pull_request.user.login,
                 pull_request.title,
-                pull_request.body);
-
-            const n = pull_request.number;
+                pull_request.body,
+                webkitBug);
 
             logger.info({webkitExport: {
                 issue: metadata.issue,
@@ -147,16 +159,16 @@ async function pullRequestPoller() {
                 isMergeable: metadata.isMergeable,
                 reviewedDownstream: metadata.reviewedDownstream,
                 flags: metadata.webkit.flags || {},
-            }}, `#${n}: Labelling and approving WebKit issue, if necessary`);
+            }}, `#${prNumber}: Labelling and approving WebKit issue, if necessary`);
 
-            return labelModel.post(n, metadata.labels, flags.get('dry-run')).then(
-                 funkLogMsg(n, "Added missing LABELS if any."),
-                 funkLogErr(n, "Something went wrong while adding missing LABELS.")
+            return labelModel.post(prNumber, metadata.labels, flags.get('dry-run')).then(
+                 funkLogMsg(prNumber, "Added missing LABELS if any."),
+                 funkLogErr(prNumber, "Something went wrong while adding missing LABELS.")
             ).then(function() {
-                return comment(n, metadata, flags.get('dry-run'));
+                return comment(prNumber, metadata, flags.get('dry-run'));
             }).then(
-                funkLogMsg(n, "Added missing REVIEWERS if any."),
-                funkLogErr(n, "Something went wrong while adding missing REVIEWERS.")
+                funkLogMsg(prNumber, "Added missing REVIEWERS if any."),
+                funkLogErr(prNumber, "Something went wrong while adding missing REVIEWERS.")
             );
         });
     } catch (e) {

lib/bugs-webkit.js

@@ -1,7 +1,8 @@
 "use strict";
 var token = process.env.WEBKIT_BUGZILLA_TOKEN;
 const fetch = require('node-fetch'),
-      logger = require('./logger');
+      logger = require('./logger'),
+      throttle = require('./throttle');
 
 function errFrom(response, body) {
     body = JSON.parse(body);
@@ -18,9 +19,10 @@ function getHeaders() {
     };
 }
 
-async function get(url, options) {
-    url = replaceURL(baseURL(url), options);
-    var headers = getHeaders();
+const MIN_DELAY_MS = 1000;
+
+async function fetchOnce(url) {
+    const headers = getHeaders();
     logger.debug("GET", url);
     const response = await fetch(url, { headers: headers });
     if (response.status < 200 || response.status >= 300) {
@@ -30,6 +32,13 @@ async function get(url, options) {
     return response.json();
 }
 
+const throttledFetch = throttle.serialize(fetchOnce, MIN_DELAY_MS);
+
+async function get(url, options) {
+    url = replaceURL(baseURL(url), options);
+    return throttledFetch(url);
+}
+
 function baseURL(url) {
     var base = 'https://bugs.webkit.org';
     url = url || '';
@@ -48,3 +57,4 @@ function setToken(t) {
 
 exports.get = get;
 exports.setToken = setToken;
+exports._throttledFetch = throttledFetch;

lib/metadata/index.js

@@ -30,18 +30,19 @@ function inferDownstreamReview(metadata, content) {
         return "Firefox";
     }
 
-    // WebKit-exported PRs are only considered approved once they have a r+ on
-    // the WebKit side. Note that we treat approval as a one-way process; if we
-    // see r+ once, the PR will stay approved even if the r+ is later removed.
+    // WebKit-exported PRs are only considered approved once they've been
+    // committed (resolved as FIXED) on the WebKit side. Note that we treat
+    // approval as a one-way process; if we see a FIXED resolution once, the PR
+    // will stay approved even if the resolution is later changed.
     //
     // https://hackmd.io/b4ViVtDsSk6Qg8fAycexVA#2-Getting-review-r-on-the-WebKit-patch-approves-the-export-PR
-    if (metadata.isWebKitVerified && metadata.webkit.flags.reviewed) {
+    if (metadata.isWebKitVerified && metadata.webkit.flags.inCommit) {
         return "WebKit";
     }
     return null;
 }
 
-module.exports = async function getMetadata(number, author, title, content) {
+module.exports = async function getMetadata(number, author, title, content, webkitBug) {
     var metadata = {
         issue: number,
         title: title,
@@ -111,24 +112,23 @@ module.exports = async function getMetadata(number, author, title, content) {
 
     metadata.isWebKitVerified = false;
     if (webkit.related(metadata.title)) {
-        var titleWebkitIssueMatch = metadata.title.match(/.*http(?:s?):\/\/bugs.webkit.org\/show_bug.cgi\?id=(\d+).*/i);
-        if (!titleWebkitIssueMatch || titleWebkitIssueMatch.length != 2) {
+        const titleWebkitIssue = webkit.bugIdFromTitle(metadata.title);
+        if (titleWebkitIssue === null) {
             // The chance of anyone using the phrase 'WebKit export of' in
             // their pull request without it being a WebKit export is low
             // enough that we just throw here, to make sure we notice errors.
             throw new Error(`Unable to extract WebKit bug id from PR title: ${metadata.title}`);
         }
 
-        if (titleWebkitIssueMatch.length == 2) {
-            var webkitIssue = titleWebkitIssueMatch[1];
-            metadata.webkit = {};
-            metadata.webkit.issue = webkitIssue;
-            metadata.isWebKitVerified = await webkit.verified(webkitIssue, metadata.issue);
+        const webkitIssue = String(titleWebkitIssue);
+        metadata.webkit = {};
+        metadata.webkit.issue = webkitIssue;
+        const bug = webkitBug || await webkit.fetchBug(webkitIssue);
+        metadata.isWebKitVerified = await webkit.verified(bug, metadata.issue);
 
-            if (metadata.isWebKitVerified) {
-                metadata.labels.push("webkit-export");
-                metadata.webkit.flags = await webkit.flags(metadata.webkit.issue);
-            }
+        if (metadata.isWebKitVerified) {
+            metadata.labels.push("webkit-export");
+            metadata.webkit.flags = await webkit.flags(metadata.webkit.issue);
         }
     }
 

lib/metadata/webkit.js

@@ -5,44 +5,35 @@ exports.related = function(title) {
     return title.indexOf("WebKit export of") > -1;
 };
 
-exports.verified = function(webkitIssue, wptPullRequestNumber) {
-        return bugsWebkit.get("/rest/bug/:id", { id: webkitIssue }).then(function (body) {
-            if (body && body.bugs && body.bugs.length > 0 && body.bugs[0].see_also) {
-                return body.bugs[0].see_also.some(function(url) {
-                    return url == "https://github.com/web-platform-tests/wpt/pull/" + wptPullRequestNumber;
-                });
-            }
-            return false;
-        });
+exports.bugIdFromTitle = function(title) {
+    const match = title.match(/.*http(?:s?):\/\/bugs.webkit.org\/show_bug.cgi\?id=(\d+).*/i);
+    if (match) {
+        return parseInt(match[1], 10);
+    }
+    return null;
+};
+
+exports.fetchBug = async function(issue) {
+    const body = await bugsWebkit.get("/rest/bug/:id", { id: issue });
+    if (!body || !body.bugs || body.bugs.length === 0) return null;
+    return body.bugs[0];
+};
+
+exports.verified = function(bug, wptPullRequestNumber) {
+    return !!(bug && bug.see_also && bug.see_also.some(function(url) {
+        return url == "https://github.com/web-platform-tests/wpt/pull/" + wptPullRequestNumber;
+    }));
 };
 
 exports.flags = function(issue) {
         var flags = {
-            reviewed: false,
             inCommit: false
         };
         return bugsWebkit.get("/rest/bug/:id/history", { id: issue }).then(function (body) {
             body.bugs[0].history.forEach(function(history_item) {
                 history_item.changes.forEach(function(change) {
-                    if (change.field_name == "flagtypes.name") {
-                        if (change.removed.includes("review+")) {
-                            flags.reviewed = false;
-                        }
-                        if (change.removed.includes("commit-queue+")) {
-                            flags.reviewed = false;
-                            flags.inCommit = false;
-                        }
-                        if (change.added.includes("review+")) {
-                            flags.reviewed = true;
-                        }
-                        if (change.added.includes("commit-queue+")) {
-                            flags.reviewed = true;
-                            flags.inCommit = true;
-                        }
-                    }
                     if (change.field_name == "resolution") {
                         if (change.added.includes("FIXED") && ["ews-feeder@webkit.org", "commit-queue@webkit.org"].includes(history_item.who)) {
-                            flags.reviewed = true;
                             flags.inCommit = true;
                         }
                     }
@@ -51,3 +42,19 @@ exports.flags = function(issue) {
             return flags;
         });
 };
+
+// Query bugs whose resolution has, at any point, transitioned to FIXED.
+// Returns a map of {bugId: bugObject}.
+exports.everFixed = async function(bugIds) {
+    if (bugIds.length === 0) {
+        return new Map();
+    }
+    const params = new URLSearchParams({
+        id: bugIds.join(","),
+        f1: "resolution",
+        o1: "changedto",
+        v1: "FIXED"
+    });
+    const body = await bugsWebkit.get("/rest/bug?" + params);
+    return new Map((body.bugs || []).map(b => [b.id, b]));
+};

lib/throttle.js

@@ -0,0 +1,28 @@
+"use strict";
+
+/* global performance */
+
+// Errors propagate to the caller but don't block subsequent calls.
+function serialize(fn, minDelayMs) {
+    let chain = Promise.resolve();
+    let lastEnd = -Infinity;
+    const wrapper = function(...args) {
+        const next = chain.then(async () => {
+            const wait = lastEnd + wrapper.minDelayMs - performance.now();
+            if (wait > 0) {
+                await new Promise(resolve => setTimeout(resolve, wait));
+            }
+            try {
+                return await fn(...args);
+            } finally {
+                lastEnd = performance.now();
+            }
+        });
+        chain = next.catch(() => {});
+        return next;
+    };
+    wrapper.minDelayMs = minDelayMs;
+    return wrapper;
+}
+
+exports.serialize = serialize;

test/fixtures/bugs.webkit.org-443/178124074071118460

@@ -0,0 +1,21 @@
+GET /rest/bug?id=104805%2C252078&f1=resolution&o1=changedto&v1=FIXED
+accept: */*
+accept-encoding: gzip,deflate
+
+HTTP/1.1 200 OK
+server: Apple
+date: Fri, 12 Jun 2026 05:05:40 GMT
+content-type: application/json; charset=UTF-8
+transfer-encoding: chunked
+connection: keep-alive
+access-control-allow-headers: origin, content-type, accept, x-requested-with
+access-control-allow-origin: *
+x-content-type-options: nosniff
+x-frame-options: SAMEORIGIN
+x-xss-protection: 1; mode=block
+set-cookie: Bugzilla_login_request_cookie=uITYyO5Fep; path=/; secure; HttpOnly
+set-cookie: github_secret=sJAbsDdNSbVWQFaO; path=/; HttpOnly;Domain=;secure;httponly
+etag: ZQH1WDrKWHuSm3WyN5yydw
+strict-transport-security: max-age=31536000; includeSubDomains
+
+{"bugs":[{"platform":"All","target_milestone":"---","flags":[],"dupe_of":null,"creator":"menard@kde.org","alias":[],"assigned_to":"mrobinson@webkit.org","is_open":false,"op_sys":"All","cc":["changseok@webkit.org","clopez@igalia.com","eoconnor@apple.com","ericwilligers@chromium.org","esprehn+autocc@chromium.org","ews-watchlist@webkit.org","glenn@skynav.com","gsnedders@apple.com","gyuyoung.kim@webkit.org","joepeck@webkit.org","kondapallykalyan@gmail.com","macpherson@chromium.org","mmaxfield@apple.com","mrobinson@webkit.org","ntim@apple.com","obrufau@igalia.com","pdr@google.com","simon.fraser@apple.com","syoichi@outlook.com","webkit-bug-importer@group.apple.com","webkit@chrisrebert.com","youennf@gmail.com"],"priority":"P2","last_change_time":"2022-11-13T10:55:39Z","depends_on":[104802],"classification":"Unclassified","is_creator_accessible":true,"cc_detail":[{"real_name":"ChangSeok Oh","name":"changseok@webkit.org","id":24683,"email":"changseok@webkit.org"},{"email":"clopez@igalia.com","id":21488,"real_name":"Carlos Alberto Lopez Perez","name":"clopez@igalia.com"},{"email":"eoconnor@apple.com","id":14621,"real_name":"Theresa O'Connor","name":"eoconnor@apple.com"},{"real_name":"Eric Willigers","name":"ericwilligers@chromium.org","email":"ericwilligers@chromium.org","id":27410},{"name":"esprehn+autocc@chromium.org","real_name":"","email":"esprehn+autocc@chromium.org","id":20208},{"id":26490,"email":"ews-watchlist@webkit.org","name":"ews-watchlist@webkit.org","real_name":"EWS Watchlist"},{"email":"glenn@skynav.com","id":14926,"real_name":"Glenn Adams","name":"glenn@skynav.com"},{"name":"gsnedders@apple.com","real_name":"Sam Sneddon [:gsnedders]","email":"gsnedders@apple.com","id":5800},{"email":"gyuyoung.kim@webkit.org","id":14715,"real_name":"Gyuyoung Kim","name":"gyuyoung.kim@webkit.org"},{"email":"joepeck@webkit.org","id":9338,"real_name":"Joseph Pecoraro","name":"joepeck@webkit.org"},{"id":19362,"email":"kondapallykalyan@gmail.com","real_name":"kalyan","name":"kondapallykalyan@gmail.com"},{"name":"macpherson@chromium.org","real_name":"Luke Macpherson","email":"macpherson@chromium.org","id":13452},{"name":"mmaxfield@apple.com","real_name":"Myles C. Maxfield","id":21219,"email":"mmaxfield@apple.com"},{"real_name":"Martin Robinson","name":"mrobinson@webkit.org","email":"mrobinson@webkit.org","id":10387},{"name":"ntim@apple.com","real_name":"Tim Nguyen (:ntim)","id":34428,"email":"ntim@apple.com"},{"id":28830,"email":"obrufau@igalia.com","name":"obrufau@igalia.com","real_name":"Oriol Brufau"},{"real_name":"Philip Rogers","name":"pdr@google.com","id":14877,"email":"pdr@google.com"},{"name":"simon.fraser@apple.com","real_name":"Simon Fraser (smfr)","email":"simon.fraser@apple.com","id":7095},{"real_name":"Syoichi Tsuyuhara","name":"syoichi@outlook.com","email":"syoichi@outlook.com","id":17468},{"real_name":"Radar WebKit Bug Importer","name":"webkit-bug-importer@group.apple.com","email":"webkit-bug-importer@group.apple.com","id":15060},{"real_name":"Chris Rebert","name":"webkit@chrisrebert.com","email":"webkit@chrisrebert.com","id":19103},{"id":19846,"email":"youennf@gmail.com","name":"youennf@gmail.com","real_name":"youenn fablet"}],"summary":"Do not allow unitless values for CSS unprefixed perspective property","blocks":[93136,240341],"keywords":["InRadar","WebExposed","WPTImpact"],"url":"","is_confirmed":true,"see_also":["https://github.com/web-platform-tests/wpt/pull/31522"],"whiteboard":"","version":"WebKit Nightly Build","severity":"Normal","creation_time":"2012-12-12T13:50:06Z","assigned_to_detail":{"email":"mrobinson@webkit.org","id":10387,"real_name":"Martin Robinson","name":"mrobinson@webkit.org"},"is_cc_accessible":true,"status":"RESOLVED","resolution":"FIXED","deadline":null,"creator_detail":{"email":"menard@kde.org","id":12077,"real_name":"Alexis Menard (darktears)","name":"menard@kde.org"},"product":"WebKit","groups":[],"id":104805,"qa_contact":"","component":"CSS"},{"depends_on":[],"classification":"Unclassified","priority":"P2","last_change_time":"2023-03-07T23:42:56Z","cc_detail":[{"email":"webkit-bug-importer@group.apple.com","id":15060,"real_name":"Radar WebKit Bug Importer","name":"webkit-bug-importer@group.apple.com"}],"is_creator_accessible":true,"dupe_of":null,"flags":[],"target_milestone":"---","platform":"Unspecified","alias":[],"creator":"ntim@apple.com","assigned_to":"ntim@apple.com","op_sys":"Unspecified","cc":["webkit-bug-importer@group.apple.com"],"is_open":false,"creation_time":"2023-02-10T22:20:13Z","deadline":null,"resolution":"FIXED","status":"RESOLVED","assigned_to_detail":{"email":"ntim@apple.com","id":34428,"real_name":"Tim Nguyen (:ntim)","name":"ntim@apple.com"},"is_cc_accessible":true,"id":252078,"groups":[],"creator_detail":{"name":"ntim@apple.com","real_name":"Tim Nguyen (:ntim)","email":"ntim@apple.com","id":34428},"product":"WebKit","component":"DOM","qa_contact":"","summary":"[popover] Implement popovertarget & popovertargetaction attributes","keywords":["InRadar"],"url":"","blocks":[250171],"whiteboard":"","is_confirmed":true,"see_also":[],"severity":"Normal","version":"WebKit Nightly Build"}]}

test/get-metadata.js

@@ -1,15 +1,23 @@
 'use strict';
 var assert = require('chai').assert,
     sinon = require('sinon'),
-    getMetadata = require('../lib/metadata');
+    { disableThrottle } = require('./setup'),
+    getMetadata = require('../lib/metadata'),
+    bugsWebkit = require('../lib/bugs-webkit');
 
 suite('getMetadata', function() {
+    disableThrottle(bugsWebkit._throttledFetch);
+
     let sandbox;
     setup(() => {
         sandbox = sinon.createSandbox();
         sandbox.replace(Math, 'random', () => 0.5);
     });
 
+    teardown(() => {
+        sandbox.restore();
+    });
+
     test('retrieval and formatting of metadata', function() {
         var expected = {
             issue: 11698,
@@ -425,7 +433,7 @@ suite('getMetadata', function() {
             reviewersExcludingAuthor: [],
             reviews: [],
             reviewers: [ 'jgraham', 'wolenetz' ],
-            webkit: { flags: { inCommit: true, reviewed: true }, issue: '201401' },
+            webkit: { flags: { inCommit: true }, issue: '201401' },
             isWebKitVerified: true,
             isMergeable: true,
             reviewedDownstream: 'WebKit',
@@ -456,8 +464,4 @@ suite('getMetadata', function() {
                 assert.sameMembers(expected, actual.labels);
             });
     });
-
-    teardown(() => {
-        sandbox.restore();
-    });
 });

test/setup.js

@@ -1,7 +1,46 @@
 'use strict';
 var path = require('path');
 
-var replay = require('replay');
+const bunyan = require('bunyan'),
+      logger = require('../lib/logger'),
+      replay = require('replay'),
+      sinon = require('sinon');
+
+// Redirect all log output into a ring buffer so tests run quietly.
+// On failure, the buffered records are written to stderr for debugging.
+const ringBuffer = new bunyan.RingBuffer({ limit: 100 });
+const originalStreams = logger.streams.slice();
+logger.streams.length = 0;
+logger.addStream({ type: 'raw', stream: ringBuffer, level: 'trace' });
+
+function disableThrottle(throttledFetch) {
+    let sandbox;
+    suiteSetup(function() {
+        sandbox = sinon.createSandbox();
+        sandbox.stub(throttledFetch, 'minDelayMs').value(0);
+    });
+    suiteTeardown(function() {
+        sandbox.restore();
+    });
+}
+
+module.exports = { ringBuffer, disableThrottle };
+
+teardown(function() {
+    if (this.currentTest && this.currentTest.state === 'failed') {
+        ringBuffer.records.forEach(function(r) {
+            process.stderr.write(JSON.stringify(r) + '\n');
+        });
+    }
+    ringBuffer.records = [];
+});
+
+suiteTeardown(function() {
+    logger.streams.length = 0;
+    originalStreams.forEach(function(s) {
+        logger.addStream(s);
+    });
+});
 
 replay.fixtures = path.join(__dirname, 'fixtures');