WIP attempt to use webpack 5 asset modules

Author: slorber

packages/docusaurus-mdx-loader/src/remark/transformImage/index.js

@@ -13,17 +13,13 @@ const escapeHtml = require('escape-html');
 const {getFileLoaderUtils} = require('@docusaurus/core/lib/webpack/utils');
 const {posixPath, toMessageRelativeFilePath} = require('@docusaurus/utils');
 
-const {
-  loaders: {inlineMarkdownImageFileLoader},
-} = getFileLoaderUtils();
+const {assetQuery} = getFileLoaderUtils();
 
 const createJSX = (node, pathUrl) => {
   const jsxNode = node;
   jsxNode.type = 'jsx';
   jsxNode.value = `<img ${node.alt ? `alt={"${escapeHtml(node.alt)}"} ` : ''}${
-    node.url
-      ? `src={require("${inlineMarkdownImageFileLoader}${pathUrl}").default}`
-      : ''
+    node.url ? `src={require("${pathUrl}?${assetQuery}").default}` : ''
   }${node.title ? ` title="${escapeHtml(node.title)}"` : ''} />`;
 
   if (jsxNode.url) {

packages/docusaurus-mdx-loader/src/remark/transformLinks/index.js

@@ -15,9 +15,7 @@ const escapeHtml = require('escape-html');
 const {toValue} = require('../utils');
 const {getFileLoaderUtils} = require('@docusaurus/core/lib/webpack/utils');
 
-const {
-  loaders: {inlineMarkdownLinkFileLoader},
-} = getFileLoaderUtils();
+const {assetQuery} = getFileLoaderUtils();
 
 async function ensureAssetFileExist(fileSystemAssetPath, sourceFilePath) {
   const assetExists = await fs.pathExists(fileSystemAssetPath);
@@ -43,7 +41,7 @@ function toAssetRequireNode({node, filePath, requireAssetPath}) {
     ? relativeRequireAssetPath
     : `./${relativeRequireAssetPath}`;
 
-  const href = `require('${inlineMarkdownLinkFileLoader}${relativeRequireAssetPath}').default`;
+  const href = `require('${relativeRequireAssetPath}?${assetQuery}').default`;
   const children = (node.children || []).map((n) => toValue(n)).join('');
   const title = node.title ? `title="${escapeHtml(node.title)}"` : '';
 

packages/docusaurus-plugin-ideal-image/src/index.ts

@@ -31,7 +31,11 @@ export default function (
         module: {
           rules: [
             {
-              test: /\.(png|jpe?g|gif)$/i,
+              test: /\.(png|jpe?g)$/i,
+              type: 'javascript/auto',
+              generator: {
+                emit: !isServer,
+              },
               use: [
                 require.resolve('@docusaurus/lqip-loader'),
                 {

packages/docusaurus/src/commands/build.ts

@@ -24,6 +24,7 @@ import {
   applyConfigurePostCss,
   applyConfigureWebpack,
   compile,
+  getFileLoaderUtils,
 } from '../webpack/utils';
 import CleanWebpackPlugin from '../webpack/plugins/CleanWebpackPlugin';
 import {loadI18n} from '../server/i18n';
@@ -197,6 +198,11 @@ async function buildLocale({
     }
   });
 
+  // Add the very high-priority rules triggered by using a resourceQuery like ?asset
+  const {prependAssetQueryRules} = getFileLoaderUtils();
+  clientConfig = prependAssetQueryRules(clientConfig);
+  serverConfig = prependAssetQueryRules(serverConfig);
+
   // Make sure generated client-manifest is cleaned first so we don't reuse
   // the one from previous builds.
   if (await fs.pathExists(clientManifestPath)) {

packages/docusaurus/src/commands/start.ts

@@ -29,6 +29,7 @@ import {
   applyConfigureWebpack,
   applyConfigurePostCss,
   getHttpsConfig,
+  getFileLoaderUtils,
 } from '../webpack/utils';
 import {getCLIOptionHost, getCLIOptionPort} from './commandUtils';
 import {getTranslationsLocaleDirPath} from '../server/translations/translations';
@@ -157,6 +158,10 @@ export default async function start(
     }
   });
 
+  // Add the very high-priority rules triggered by using a resourceQuery like ?asset
+  const {prependAssetQueryRules} = getFileLoaderUtils();
+  config = prependAssetQueryRules(config);
+
   // https://webpack.js.org/configuration/dev-server
   const devServerConfig: WebpackDevServer.Configuration = {
     ...{

packages/docusaurus/src/webpack/base.ts

@@ -104,6 +104,7 @@ export function createBaseConfig(
       chunkFilename: isProd
         ? 'assets/js/[name].[contenthash:8].js'
         : '[name].js',
+      assetModuleFilename: 'assets/[hash][ext][query]',
       publicPath: baseUrl,
     },
     // Don't throw warning when asset created is over 250kb
@@ -187,7 +188,7 @@ export function createBaseConfig(
         fileLoaderUtils.rules.fonts(),
         fileLoaderUtils.rules.media(),
         fileLoaderUtils.rules.svg(),
-        fileLoaderUtils.rules.otherAssets(),
+        fileLoaderUtils.rules.files(),
         {
           test: /\.(j|t)sx?$/,
           exclude: excludeJS,

packages/docusaurus/src/webpack/utils.ts

@@ -284,9 +284,11 @@ export function compile(config: Configuration[]): Promise<void> {
   });
 }
 
-type AssetFolder = 'images' | 'files' | 'fonts' | 'medias';
+type AssetFolder = 'images' | 'files' | 'fonts' | 'medias' | 'svgs' | 'other';
 
 type FileLoaderUtils = {
+  assetQuery: string;
+  prependAssetQueryRules: (configuration: Configuration) => Configuration;
   loaders: {
     file: (options: {folder: AssetFolder}) => RuleSetRule;
     url: (options: {folder: AssetFolder}) => RuleSetRule;
@@ -298,34 +300,66 @@ type FileLoaderUtils = {
     fonts: () => RuleSetRule;
     media: () => RuleSetRule;
     svg: () => RuleSetRule;
-    otherAssets: () => RuleSetRule;
+    files: () => RuleSetRule;
   };
 };
 
 // Inspired by https://github.com/gatsbyjs/gatsby/blob/8e6e021014da310b9cc7d02e58c9b3efe938c665/packages/gatsby/src/utils/webpack-utils.ts#L447
 export function getFileLoaderUtils(): FileLoaderUtils {
-  // files/images < 10kb will be inlined as base64 strings directly in the html
-  const urlLoaderLimit = 10000;
+  // Asset queries are used to force the usage of the file as an asset
+  // In some case we want to opt-out o
+  // - converting an image to an ideal-image
+  // - converting an SVG to a React component
+  // - other cases
+  const assetQuery = 'asset';
+  const assetQueryRegex = /asset/;
+
+  // Threshold for datauri/file (previously set on url-loader)
+  // files/images < 10kb will be inlined as base64 strings directly in the JS bundle
+  // See https://webpack.js.org/guides/asset-modules/#general-asset-type
+  const dataUrlMaxSize = 10000;
 
   // defines the path/pattern of the assets handled by webpack
-  const fileLoaderFileName = (folder: AssetFolder) =>
+  const generatedFileName = (folder: AssetFolder) =>
     `${OUTPUT_STATIC_ASSETS_DIR_NAME}/${folder}/[name]-[hash].[ext]`;
 
+  function fileNameGenerator(folder: AssetFolder) {
+    return {
+      filename: generatedFileName(folder),
+    };
+  }
+
+  function baseAssetRule(folder: AssetFolder): RuleSetRule {
+    return {
+      parser: {
+        dataUrlCondition: {
+          // Threshold for datauri/file (previously set on url-loader)
+          // files/images < 10kb will be inlined as base64 strings directly in the JS bundle
+          // See https://webpack.js.org/guides/asset-modules/#general-asset-type
+          maxSize: dataUrlMaxSize,
+        },
+      },
+      generator: fileNameGenerator(folder),
+    };
+  }
+
   const loaders: FileLoaderUtils['loaders'] = {
+    // TODO deprecated
     file: (options: {folder: AssetFolder}) => {
       return {
         loader: require.resolve(`file-loader`),
         options: {
-          name: fileLoaderFileName(options.folder),
+          name: generatedFileName(options.folder),
         },
       };
     },
     url: (options: {folder: AssetFolder}) => {
+      // TODO deprecated
       return {
         loader: require.resolve(`url-loader`),
         options: {
-          limit: urlLoaderLimit,
-          name: fileLoaderFileName(options.folder),
+          limit: dataUrlMaxSize,
+          name: generatedFileName(options.folder),
           fallback: require.resolve(`file-loader`),
         },
       };
@@ -336,85 +370,115 @@ export function getFileLoaderUtils(): FileLoaderUtils {
     // Maybe with the ideal image plugin, all md images should be "ideal"?
     // This is used to force url-loader+file-loader on markdown images
     // https://webpack.js.org/concepts/loaders/#inline
-    inlineMarkdownImageFileLoader: `!url-loader?limit=${urlLoaderLimit}&name=${fileLoaderFileName(
+    inlineMarkdownImageFileLoader: `!url-loader?limit=${dataUrlMaxSize}&name=${generatedFileName(
       'images',
     )}&fallback=file-loader!`,
-    inlineMarkdownLinkFileLoader: `!file-loader?name=${fileLoaderFileName(
+    inlineMarkdownLinkFileLoader: `!file-loader?name=${generatedFileName(
       'files',
     )}!`,
   };
 
-  const rules: FileLoaderUtils['rules'] = {
-    /**
-     * Loads image assets, inlines images via a data URI if they are below
-     * the size threshold
-     */
-    images: () => {
-      return {
-        use: [loaders.url({folder: 'images'})],
-        test: /\.(ico|jpg|jpeg|png|gif|webp)(\?.*)?$/,
-      };
-    },
+  function imageAssetRule(): RuleSetRule {
+    return {
+      ...baseAssetRule('images'),
+      test: /\.(ico|jpg|jpeg|png|gif|webp)(\?.*)?$/,
+    };
+  }
 
-    fonts: () => {
-      return {
-        use: [loaders.url({folder: 'fonts'})],
-        test: /\.(woff|woff2|eot|ttf|otf)$/,
-      };
-    },
+  function fontAssetRule(): RuleSetRule {
+    return {
+      ...baseAssetRule('fonts'),
+      test: /\.(woff|woff2|eot|ttf|otf)$/,
+    };
+  }
 
-    /**
-     * Loads audio and video and inlines them via a data URI if they are below
-     * the size threshold
-     */
-    media: () => {
-      return {
-        use: [loaders.url({folder: 'medias'})],
-        test: /\.(mp4|webm|ogv|wav|mp3|m4a|aac|oga|flac)$/,
-      };
-    },
+  function mediaAssetRule(): RuleSetRule {
+    return {
+      ...baseAssetRule('medias'),
+      test: /\.(mp4|webm|ogv|wav|mp3|m4a|aac|oga|flac)$/,
+    };
+  }
 
-    svg: () => {
-      return {
-        test: /\.svg?$/,
-        oneOf: [
-          {
-            use: [
-              {
-                loader: '@svgr/webpack',
-                options: {
-                  prettier: false,
-                  svgo: true,
-                  svgoConfig: {
-                    plugins: [{removeViewBox: false}],
-                  },
-                  titleProp: true,
-                  ref: ![path],
+  function fileAssetRule(): RuleSetRule {
+    return {
+      ...baseAssetRule('files'),
+      test: /\.(pdf|doc|docx|xls|xlsx|zip|rar)$/,
+      type: 'asset/resource',
+    };
+  }
+
+  function svgAssetRule(): RuleSetRule {
+    return {
+      ...baseAssetRule('svgs'),
+      test: /\.svg?$/,
+    };
+  }
+
+  // We convert SVG to React component when required from code only
+  // We don't convert SVG to React components when referenced in CSS
+  function svgComponentOrAssetRule(): RuleSetRule {
+    return {
+      test: /\.svg?$/,
+      oneOf: [
+        {
+          // only convert for those extensions
+          issuer: /\.(ts|tsx|js|jsx|md|mdx)$/,
+          use: [
+            {
+              loader: '@svgr/webpack',
+              options: {
+                prettier: false,
+                svgo: true,
+                svgoConfig: {
+                  plugins: [{removeViewBox: false}],
                 },
+                titleProp: true,
+                ref: ![path],
               },
-            ],
-            // We don't want to use SVGR loader for non-React source code
-            // ie we don't want to use SVGR for CSS files...
-            issuer: {
-              and: [/\.(ts|tsx|js|jsx|md|mdx)$/],
             },
-          },
+          ],
+        },
+        svgAssetRule(),
+      ],
+    };
+  }
+
+  const rules: FileLoaderUtils['rules'] = {
+    images: imageAssetRule,
+    fonts: fontAssetRule,
+    media: mediaAssetRule,
+    svg: svgComponentOrAssetRule,
+    files: fileAssetRule,
+  };
+
+  // Those rules are  triggered conditionally when using ?asset
+  // They must be added at the very beginning of the rules array
+  // Even before the rules prepended by other plugins
+  // This is a replacement for Webpack 4 file/url-loader webpack queries
+  function prependAssetQueryRules(configuration: Configuration): Configuration {
+    return mergeWithCustomize({
+      customizeArray: customizeArray({
+        'module.rules': CustomizeRule.Prepend,
+      }),
+    })(configuration, {
+      module: {
+        rules: [
+          {...imageAssetRule(), resourceQuery: assetQueryRegex},
+          {...fontAssetRule(), resourceQuery: assetQueryRegex},
+          {...mediaAssetRule(), resourceQuery: assetQueryRegex},
+          {...svgAssetRule(), resourceQuery: assetQueryRegex},
+          // Fallback when ?asset is used but the file is unknown
           {
-            use: [loaders.url({folder: 'images'})],
+            type: 'asset/resource',
+            resourceQuery: assetQueryRegex,
+            generator: fileNameGenerator('files'),
           },
         ],
-      };
-    },
-
-    otherAssets: () => {
-      return {
-        use: [loaders.file({folder: 'files'})],
-        test: /\.(pdf|doc|docx|xls|xlsx|zip|rar)$/,
-      };
-    },
-  };
+      },
+    } as Configuration);
+  }
 
-  return {loaders, rules};
+  return {loaders, rules, assetQuery, prependAssetQueryRules};
 }
 
 // Ensure the certificate and key provided are valid and if not