feat: add urllib4 fetch (#5424)

Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>

Author: killagu

.github/workflows/gh-pages.yml

@@ -25,7 +25,7 @@ jobs:
       run: npm i -g npminstall && npminstall
 
     - name: Build Documents
-      run: npm run site:build && cp vercel.json ./site/dist/vercel.json
+      run: npm run site:build
 
     - name: Deploy Documents
       uses: peaceiris/actions-gh-pages@v3

.github/workflows/nodejs-3.x.yml

@@ -5,6 +5,8 @@ on:
     branches: [ master ]
   pull_request:
     branches: [ master ]
+  merge_group:
+    branches: [ master ]
 
 jobs:
   Job:

index.d.ts

@@ -23,6 +23,10 @@ import {
   RequestOptions,
   HttpClientResponse as HttpClientResponseNext,
 } from 'urllib-next';
+import {
+  FetchFactory,
+  fetch,
+} from 'urllib4';
 import {
   EggCoreBase,
   FileLoaderOption,
@@ -589,6 +593,12 @@ declare module 'egg' {
      */
     httpclient: EggHttpClient;
 
+    /**
+     * node fetch
+     */
+    FetchFactory: FetchFactory;
+    fetch: typeof fetch,
+
     /**
      * Logger for Application, wrapping app.coreLogger with context infomation
      *

lib/core/fetch_factory.js

@@ -0,0 +1,37 @@
+const debug = require('util').debuglog('egg:lib:core:fetch_factory');
+
+const mainNodejsVersion = parseInt(process.versions.node.split('.')[0]);
+let FetchFactory;
+let safeFetch;
+let ssrfFetchFactory;
+
+if (mainNodejsVersion >= 20) {
+  // urllib@4 only works on Node.js >= 20
+  try {
+    const urllib4 = require('urllib4');
+    FetchFactory = urllib4.FetchFactory;
+    debug('urllib4 enable');
+
+    safeFetch = function safeFetch(url, init) {
+      if (!ssrfFetchFactory) {
+        const ssrfConfig = this.config.security.ssrf;
+        const clientOptions = {};
+        if (ssrfConfig?.checkAddress) {
+          clientOptions.checkAddress = ssrfConfig.checkAddress;
+        } else {
+          this.logger.warn('[egg-security] please configure `config.security.ssrf` first');
+        }
+        ssrfFetchFactory = new FetchFactory();
+        ssrfFetchFactory.setClientOptions(clientOptions);
+      }
+      return ssrfFetchFactory.fetch(url, init);
+    };
+  } catch (err) {
+    debug('require urllib4 error: %s', err);
+  }
+}
+
+module.exports = {
+  FetchFactory,
+  safeFetch,
+};

lib/core/httpclient_next.js

@@ -5,8 +5,8 @@ const SSRF_HTTPCLIENT = Symbol('SSRF_HTTPCLIENT');
 
 const mainNodejsVersion = parseInt(process.versions.node.split('.')[0]);
 let HttpClient;
-if (mainNodejsVersion >= 18) {
-  // urllib@4 only works on Node.js >= 18
+if (mainNodejsVersion >= 20) {
+  // urllib@4 only works on Node.js >= 20
   try {
     HttpClient = require('urllib4').HttpClient;
     debug('urllib4 enable');

lib/egg.js

@@ -14,6 +14,7 @@ const Messenger = require('./core/messenger');
 const DNSCacheHttpClient = require('./core/dnscache_httpclient');
 const HttpClient = require('./core/httpclient');
 const HttpClientNext = require('./core/httpclient_next');
+const { FetchFactory, safeFetch } = require('./core/fetch_factory');
 const createLoggers = require('./core/logger');
 const Singleton = require('./core/singleton');
 const utils = require('./core/utils');
@@ -51,6 +52,12 @@ class EggApplication extends EggCore {
     this.ContextHttpClient = ContextHttpClient;
     this.HttpClient = HttpClient;
     this.HttpClientNext = HttpClientNext;
+    this.FetchFactory = FetchFactory;
+    if (FetchFactory) {
+      this.FetchFactory.setClientOptions();
+      this.fetch = FetchFactory.fetch;
+      this.safeFetch = safeFetch.bind(this);
+    }
 
     this.loader.loadConfig();
 

site/config/config.ts

@@ -39,6 +39,10 @@ export default defineConfig({
         title: 'Plugins',
         path: 'https://github.com/search?q=topic%3Aegg-plugin&type=Repositories',
       },
+      {
+        title: 'v3.x',
+        path: 'https://v3.eggjs.org',
+      },
       {
         title: 'v4.x',
         path: 'https://eggjs.org',
@@ -50,13 +54,17 @@ export default defineConfig({
         title: 'GitHub',
         path: 'https://github.com/eggjs/egg',
       },
+      {
+        title: '发布日志',
+        path: 'https://github.com/eggjs/egg/releases',
+      },
       {
         title: '插件列表',
         path: 'https://github.com/search?q=topic%3Aegg-plugin&type=Repositories',
       },
       {
-        title: '发布日志',
-        path: 'https://github.com/eggjs/egg/releases',
+        title: 'v3.x',
+        path: 'https://v3.eggjs.org',
       },
       {
         title: 'v4.x',

test/lib/core/fetch_factory.test.js

@@ -0,0 +1,45 @@
+const assert = require('node:assert');
+const mm = require('egg-mock');
+const { FetchFactory, safeFetch } = require('../../../lib/core/fetch_factory');
+const utils = require('../../utils');
+
+describe('test/lib/core/fetch_factory.test.js', () => {
+  if (!FetchFactory) return;
+  let url;
+
+  before(async () => {
+    url = await utils.startLocalServer();
+  });
+
+  afterEach(mm.restore);
+
+  it('should fetch ok', async () => {
+    const { status } = await FetchFactory.fetch(url);
+    assert(status === 200);
+  });
+
+  it('should support safeFetch', async () => {
+    let ip;
+    let family;
+    let host;
+    const fetch = safeFetch.bind({
+      config: {
+        security: {
+          ssrf: {
+            checkAddress(aIp, aFamily, aHost) {
+              ip = aIp;
+              family = aFamily;
+              host = aHost;
+              return true;
+            },
+          },
+        },
+      },
+      logger: console,
+    });
+    await fetch(url);
+    assert(ip);
+    assert(family);
+    assert(host);
+  });
+});

test/lib/core/httpclient.test.js

@@ -340,7 +340,7 @@ describe('test/lib/core/httpclient.test.js', () => {
         stream.end('hello h2!');
         // console.log(headers);
         const mainNodejsVersion = parseInt(process.versions.node.split('.')[0]);
-        if (mainNodejsVersion >= 18) {
+        if (mainNodejsVersion >= 20) {
           assert.match(headers['user-agent'], /node\-urllib\/4\.\d+\.\d+/);
         } else {
           assert.match(headers['user-agent'], /node\-urllib\/3\.\d+\.\d+/);
@@ -380,7 +380,7 @@ describe('test/lib/core/httpclient.test.js', () => {
 
     it('should request http2 success', async () => {
       for (let i = 0; i < 10; i++) {
-        const result = await app.httpclient.curl('https://registry.npmmirror.com', {
+        const result = await app.httpclient.curl('https://r.cnpmjs.org', {
           dataType: 'json',
           timeout: 5000,
         });