chore: clotton dev alias for isolated testing (SITES-47125)

package.json

@@ -22,7 +22,7 @@
     "build": "hedy -v --test-bundle",
     "deploy": "hedy -v --deploy --aws-deploy-bucket=spacecat-prod-deploy --pkgVersion=latest --aws-reserved-concurrency=100",
     "deploy-stage": "hedy -v --deploy --aws-deploy-bucket=spacecat-stage-deploy --pkgVersion=latest --aws-reserved-concurrency=10",
-    "deploy-dev": "hedy -v --deploy --pkgVersion=ci$CI_BUILD_NUM -l latest --aws-deploy-bucket=spacecat-dev-deploy --cleanup-ci=24h --aws-reserved-concurrency=10",
+    "deploy-dev": "hedy -v --deploy --pkgVersion=ci$CI_BUILD_NUM -l clotton --aws-deploy-bucket=spacecat-dev-deploy --cleanup-ci=24h --aws-reserved-concurrency=10",
     "deploy-secrets": "hedy --aws-update-secrets --params-file=secrets/secrets.env",
     "prepare": "husky",
     "local-build": "sam build",

src/preflight/links-checks.js

@@ -117,6 +117,33 @@ function isBrokenStatus(status) {
   return status === 404 || status === 410 || status >= 500;
 }
 
+/**
+ * Returns true only for a DNS-resolution failure — the one network-level error that
+ * definitively indicates a broken link.
+ *
+ * A DNS-resolution failure (`ENOTFOUND` — getaddrinfo cannot resolve the hostname) means the
+ * domain does not exist, so the link is unambiguously broken (SITES-40919: nonexistent domains
+ * and intranet hosts that fail DNS must surface as status 0).
+ *
+ * Every other thrown error — connection reset, HTTP/2 stream errors, TLS failures, connection
+ * refused, request timeouts — means the host resolves and is reachable but did not return a
+ * usable HTTP response. That is indistinguishable from a valid page that blocks crawlers at the
+ * connection level (e.g. ups.com resets the HTTP/2 stream with NGHTTP2_INTERNAL_ERROR), so these
+ * must NOT be reported as broken (SITES-47125).
+ *
+ * Node's DNS resolver reliably sets `err.code === 'ENOTFOUND'`, which is the primary signal. The
+ * message check is a narrow fallback for wrappers that strip `.code`: it only fires when no code
+ * is present and matches the canonical `getaddrinfo ENOTFOUND` format — not any message that
+ * merely contains the substring, which would re-introduce false positives (PR #2727 review).
+ *
+ * @param {Error} err
+ * @returns {boolean}
+ */
+function isDnsResolutionFailure(err) {
+  return err.code === 'ENOTFOUND'
+    || (err.code == null && /getaddrinfo ENOTFOUND/.test(err.message));
+}
+
 /**
  * Helper function to check if a link is broken
  * @param {string} href - The URL to check
@@ -203,16 +230,24 @@ async function checkLinkStatus(href, pageUrl, context, options = {
 
     return null;
   } catch (finalErr) {
-    // Network-level failure (DNS, timeout, unsupported protocol) — the link is
-    // unreachable, which is a broken-link condition. status: 0 is the sentinel
-    // for "no HTTP response received".
-    log.info(`[preflight-audit] ${linkType} link ${href} unreachable (${finalErr.message}) — reporting as broken`);
-    return {
-      urlTo: href,
-      href: pageUrl,
-      status: 0,
-      ...toElementTargets(selectors),
-    };
+    // Only a DNS-resolution failure (ENOTFOUND) is a definitive broken-link condition: the
+    // domain does not exist. status: 0 is the sentinel for "no HTTP response received".
+    if (isDnsResolutionFailure(finalErr)) {
+      log.info(`[preflight-audit] ${linkType} link ${href} unreachable — DNS does not resolve (${finalErr.message}) — reporting as broken`);
+      return {
+        urlTo: href,
+        href: pageUrl,
+        status: 0,
+        ...toElementTargets(selectors),
+      };
+    }
+
+    // Any other network-level failure (connection reset, HTTP/2 stream error, TLS, timeout)
+    // means the host is reachable but did not return a usable response — indistinguishable
+    // from a valid page that blocks bots at the connection level. Do NOT report as broken
+    // (SITES-47125: ups.com and similar bot-protected sites were false-flagged as Status 0).
+    log.info(`[preflight-audit] ${linkType} link ${href} probe inconclusive (${finalErr.message}) — not reporting as broken`);
+    return null;
   }
 }
 

test/audits/preflight.test.js

@@ -106,13 +106,14 @@ describe('Preflight Audit', () => {
       ]);
     });
 
-    it('handles fetch errors', async () => {
+    it('reports a DNS-resolution failure (ENOTFOUND) as a broken internal link', async () => {
       const urls = ['https://main--example--page.aem.page/page1'];
+      const dnsError = () => Object.assign(new Error('getaddrinfo ENOTFOUND main--example--page.aem.page'), { code: 'ENOTFOUND' });
       nock('https://main--example--page.aem.page')
         .head('/fail')
-        .replyWithError('network fail')
+        .replyWithError(dnsError())
         .get('/fail')
-        .replyWithError('network fail');
+        .replyWithError(dnsError());
 
       const scrapedObjects = [{
         data: {
@@ -127,6 +128,26 @@ describe('Preflight Audit', () => {
       expect(context.log.info).to.have.been.calledWithMatch(/internal link https:\/\/main--example--page\.aem\.page\/fail unreachable/);
     });
 
+    it('does NOT report a connection-level (non-DNS) failure as a broken internal link (SITES-47125)', async () => {
+      const urls = ['https://main--example--page.aem.page/page1'];
+      nock('https://main--example--page.aem.page')
+        .head('/blocked')
+        .replyWithError('socket hang up')
+        .get('/blocked')
+        .replyWithError('socket hang up');
+
+      const scrapedObjects = [{
+        data: {
+          scrapeResult: { rawBody: '<a href="/blocked">blocked</a>' },
+          finalUrl: urls[0],
+        },
+      }];
+
+      const result = await runLinksChecks(urls, scrapedObjects, context);
+      expect(result.auditResult.brokenInternalLinks).to.have.lengthOf(0);
+      expect(context.log.info).to.have.been.calledWithMatch(/internal link https:\/\/main--example--page\.aem\.page\/blocked probe inconclusive/);
+    });
+
     it('handles HEAD failure with GET fallback success', async () => {
       const urls = ['https://main--example--page.aem.page/page1'];
       nock('https://main--example--page.aem.page')
@@ -335,13 +356,14 @@ describe('Preflight Audit', () => {
       ]);
     });
 
-    it('handles external link fetch errors', async () => {
+    it('reports a DNS-resolution failure (ENOTFOUND) as a broken external link', async () => {
       const urls = ['https://main--example--page.aem.page/page1'];
+      const dnsError = () => Object.assign(new Error('getaddrinfo ENOTFOUND external-site.com'), { code: 'ENOTFOUND' });
       nock('https://external-site.com')
         .head('/fail')
-        .replyWithError('network fail')
+        .replyWithError(dnsError())
         .get('/fail')
-        .replyWithError('network fail');
+        .replyWithError(dnsError());
 
       const scrapedObjects = [{
         data: {
@@ -357,6 +379,28 @@ describe('Preflight Audit', () => {
       expect(context.log.info).to.have.been.calledWithMatch(/external link https:\/\/external-site\.com\/fail unreachable/);
     });
 
+    it('does NOT report a bot-blocking connection-level failure as a broken external link (SITES-47125)', async () => {
+      // ups.com pattern: DNS resolves, but the server resets the HTTP/2 stream to block bots.
+      const urls = ['https://main--example--page.aem.page/page1'];
+      const http2Error = () => Object.assign(new Error('Stream closed with error code NGHTTP2_INTERNAL_ERROR'), { code: 'ERR_HTTP2_STREAM_ERROR' });
+      nock('https://external-site.com')
+        .head('/blocked')
+        .replyWithError(http2Error())
+        .get('/blocked')
+        .replyWithError(http2Error());
+
+      const scrapedObjects = [{
+        data: {
+          scrapeResult: { rawBody: '<a href="https://external-site.com/blocked">external blocked</a>' },
+          finalUrl: urls[0],
+        },
+      }];
+
+      const result = await runLinksChecks(urls, scrapedObjects, context);
+      expect(result.auditResult.brokenExternalLinks).to.have.lengthOf(0);
+      expect(context.log.info).to.have.been.calledWithMatch(/external link https:\/\/external-site\.com\/blocked probe inconclusive/);
+    });
+
     it('handles external HEAD failure with GET fallback success', async () => {
       const urls = ['https://main--example--page.aem.page/page1'];
       nock('https://external-site.com')

test/preflight/links-checks.test.js

@@ -301,13 +301,19 @@ describe('preflight/links-checks - runLinksChecks', () => {
     expect(result.auditResult.brokenExternalLinks).to.have.lengthOf(0);
   });
 
-  it('flags as broken (status 0) when both HEAD and GET throw network errors', async () => {
-    fetchStub.onFirstCall().rejects(new Error('network error'));
-    fetchStub.onSecondCall().rejects(new Error('still failing'));
+  // SITES-47125: only DNS-resolution failures (ENOTFOUND) are definitively broken.
+  // Other network-level failures (connection reset, HTTP/2 stream errors, TLS, timeout)
+  // mean the host exists but won't talk to our bot — these are indistinguishable from a
+  // valid page that blocks crawlers, so they must NOT be reported as broken.
+
+  it('flags as broken (status 0) when both HEAD and GET fail with a DNS-resolution error (ENOTFOUND)', async () => {
+    const dnsError = Object.assign(new Error('getaddrinfo ENOTFOUND www.brokenlinkbrokenlink.za'), { code: 'ENOTFOUND' });
+    fetchStub.onFirstCall().rejects(dnsError);
+    fetchStub.onSecondCall().rejects(dnsError);
 
     const result = await runLinksChecks(
       [pageUrl],
-      makeScrapedObjects('<a href="https://other.com/page">link</a>'),
+      makeScrapedObjects('<a href="https://www.brokenlinkbrokenlink.za/page">link</a>'),
       context,
     );
 
@@ -317,7 +323,60 @@ describe('preflight/links-checks - runLinksChecks', () => {
     expect(context.log.error).to.not.have.been.called;
   });
 
-  it('flags internal link as broken (status 0) when both HEAD and GET throw network errors', async () => {
+  it('does NOT flag as broken when a connection-level error blocks the bot (HTTP/2 reset — SITES-47125)', async () => {
+    // ups.com resolves fine but resets the HTTP/2 stream to block bots. Valid in a browser.
+    const http2Error = Object.assign(
+      new Error('Stream closed with error code NGHTTP2_INTERNAL_ERROR'),
+      { code: 'ERR_HTTP2_STREAM_ERROR' },
+    );
+    fetchStub.onFirstCall().rejects(http2Error);
+    fetchStub.onSecondCall().rejects(http2Error);
+
+    const result = await runLinksChecks(
+      [pageUrl],
+      makeScrapedObjects('<a href="https://www.ups.com/ppwa/doWork?loc=en_US">link</a>'),
+      context,
+    );
+
+    expect(result.auditResult.brokenExternalLinks).to.have.lengthOf(0);
+    expect(context.log.error).to.not.have.been.called;
+  });
+
+  it('does NOT flag as broken when both HEAD and GET fail with a generic (non-DNS) network error', async () => {
+    fetchStub.onFirstCall().rejects(new Error('network error'));
+    fetchStub.onSecondCall().rejects(new Error('still failing'));
+
+    const result = await runLinksChecks(
+      [pageUrl],
+      makeScrapedObjects('<a href="https://other.com/page">link</a>'),
+      context,
+    );
+
+    expect(result.auditResult.brokenExternalLinks).to.have.lengthOf(0);
+    expect(context.log.error).to.not.have.been.called;
+  });
+
+  it('does NOT flag as broken when a non-DNS error message merely contains the substring "ENOTFOUND"', async () => {
+    // The message-fallback must only match the canonical Node format ("getaddrinfo ENOTFOUND"),
+    // not any message that happens to contain the substring — otherwise a wrapper error like this
+    // would re-introduce the false positives this fix removes (MysticatBot review on PR #2727).
+    const wrappedError = new Error('Retry after ENOTFOUND was cached'); // no .code, non-canonical
+    fetchStub.onFirstCall().rejects(wrappedError);
+    fetchStub.onSecondCall().rejects(wrappedError);
+
+    const result = await runLinksChecks(
+      [pageUrl],
+      makeScrapedObjects('<a href="https://other.com/page">link</a>'),
+      context,
+    );
+
+    expect(result.auditResult.brokenExternalLinks).to.have.lengthOf(0);
+    expect(context.log.error).to.not.have.been.called;
+  });
+
+  // Message-fallback branch: error carries the canonical "getaddrinfo ENOTFOUND" message but no
+  // .code (e.g. a fetch wrapper that strips the code). Still a definitive DNS failure → broken.
+  it('flags internal link as broken (status 0) on a DNS-resolution error message without a code', async () => {
     fetchStub.onFirstCall().rejects(new Error('getaddrinfo ENOTFOUND internal.example.com'));
     fetchStub.onSecondCall().rejects(new Error('getaddrinfo ENOTFOUND internal.example.com'));