Obey file visibility rules when compressing served files

Author: davidje13

docs/API.md

@@ -558,6 +558,20 @@ _not_ be compressed, as they are assumed to already be compressed as part of the
   - `filter` [`<Function>`] a function which decides which files to attempt to compress. Called with
     the full file path [`<string>`] and mime type [`<string>`], returns [`<boolean>`]. **Default:**
     a function which rejects known image, video, audio, and font mime types.
+  - `subDirectories` [`<boolean>`] | [`<number>`] `true` to compress files in all sub-directories
+    recursively, `false` to only compress files directly inside the base directory. If this is set
+    to a number, it is the depth of sub-directories which can be traversed (`0` is equivalent to
+    `false`). **Default:** `true`.
+  - `allowAllDotfiles` [`<boolean>`] `true` to compress all dotfiles and traverse into directories
+    which begin with a dot. This is not typically desired, as these filenames usually denote hidden
+    or private files. **Default:** `false`.
+  - `allowAllTildefiles` [`<boolean>`] `true` to compress all files beginning with a tilde. This is
+    not typically desired, as these filenames usually denote temporary files created by editors.
+    **Default:** `false`.
+  - `allow` [`<string[]>`][`<string>`] list of files and directories to explicitly allow compression
+    of (which may otherwise be blocked by another rule). **Default:** `['.well-known']`.
+  - `hide` [`<string[]>`][`<string>`] | [`<RegExp[]>`][`<RegExp>`] list of files and directories to
+    avoid compressing. **Default:** `[]`.
 - Returns: [`<Promise>`] Fulfills with [`<Object[]>`][`<Object>`] containing information about the
   compression once all files have been processed.
 

src/bin/compression.mts

@@ -20,6 +20,7 @@ export async function runCompression(servers: ConfigServer[], minCompression: nu
         });
         const filenameFilter = stringPredicate(config.match, true);
         const processed = await compressFilesInDir(mount.dir, config.options, {
+          ...mount.options,
           minCompression,
           filter: (file, mime) => {
             if (['image', 'video', 'audio', 'font'].includes(mime.split('/')[0]!)) {

src/extras/compress/offline.mts

@@ -1,9 +1,10 @@
 import { basename, dirname, extname, join } from 'node:path';
-import { readdir, readFile, writeFile, rm, utimes, stat } from 'node:fs/promises';
-import { Queue } from '../../util/Queue.mts';
+import { readFile, writeFile, rm, utimes, stat } from 'node:fs/promises';
 import { internalMutateName, type FileNegotiationOption } from '../request/Negotiator.mts';
 import { getMime } from '../registries/mime.mts';
 import { internalCompressBuffer, type ContentEncoding } from './encoders.mts';
+import { FileFinderRules, type FileFinderOptions } from '../filesystem/FileFinder.mts';
+import { internalDiscoverFiles } from '../filesystem/staticFileFinder.mts';
 
 export interface CompressionInfo {
   /** the path to the file */
@@ -109,19 +110,15 @@ export async function compressFileOffline(
 export async function compressFilesInDir(
   dir: string,
   encodings: ReadonlyArray<FileNegotiationOption>,
-  options: CompressionOptions = {},
+  options: CompressionOptions &
+    Pick<
+      FileFinderOptions,
+      'subDirectories' | 'allowAllDotfiles' | 'allowAllTildefiles' | 'hide' | 'allow'
+    > = {},
 ): Promise<CompressionInfo[]> {
+  const rules = new FileFinderRules(options);
   const files = new Set<string>();
-  const pathsQueue = new Queue(dir);
-  for (const base of pathsQueue) {
-    for (const f of await readdir(base, { withFileTypes: true, encoding: 'utf-8' })) {
-      if (f.isDirectory()) {
-        pathsQueue.push(join(base, f.name));
-      } else if (f.isFile()) {
-        files.add(join(base, f.name));
-      }
-    }
-  }
+  await internalDiscoverFiles(dir, rules, (path, name) => files.add(join(dir, ...path, name)));
 
   // remove existing compressed files from the set
   for (const file of files) {

src/extras/compress/offline.test.mts

@@ -113,19 +113,24 @@ describe('compressFilesInDir', () => {
   const TEST_DIR = makeTestTempDir('compress-', {
     'compressible.txt': '.'.repeat(1000),
     'incompressible.txt': 'too small',
+    'skipped.txt': '.'.repeat(1000),
     'poor-compression.txt': '.'.repeat(310),
     'already-compressed.txt': 'original',
     'already-compressed.txt.gz': 'compressed',
     'image.png': '.'.repeat(1000), // not compressed even though it could be
     nested: {
       'deep.txt': '.'.repeat(1000),
     },
+    '.nope': {
+      'deep.txt': '.'.repeat(1000),
+    },
   });
 
   it('compresses files which can be reduced', async ({ getTyped }) => {
     const dir = getTyped(TEST_DIR);
     const stats = await compressFilesInDir(dir, [{ value: 'gzip', file: '{file}.gz' }], {
       minCompression: 300,
+      hide: ['skipped.txt'],
     });
     expect(stats).hasLength(6);
 
@@ -155,7 +160,11 @@ describe('compressFilesInDir', () => {
     expect(stats5?.created).equals(1);
     expect(stats5!.bestSize).isLessThan(stats5!.rawSize);
 
+    expect(findStat(join('skipped.txt'))).isUndefined();
+    expect(findStat(join('.nope', 'deep.txt'))).isUndefined();
+
     expect((await readdir(dir)).sort()).equals([
+      '.nope',
       'already-compressed.txt',
       'already-compressed.txt.gz',
       'compressible.txt',
@@ -164,6 +173,7 @@ describe('compressFilesInDir', () => {
       'incompressible.txt',
       'nested',
       'poor-compression.txt',
+      'skipped.txt',
     ]);
     expect((await readdir(join(dir, 'nested'))).sort()).equals(['deep.txt', 'deep.txt.gz']);
   });

src/extras/filesystem/staticFileFinder.mts

@@ -18,29 +18,12 @@ export async function staticFileFinder(
   const rules = new FileFinderRules(options);
   const precomputed = new StaticFileFinder(rules, internalTryReturn);
 
-  const queue = new Queue<string[]>([]);
-  for (const path of queue) {
-    const dirEntries = await readdir(join(root, ...path), {
-      withFileTypes: true,
-      encoding: 'utf-8',
-    });
-    const siblings = new Map(
-      dirEntries.map((v) => [rules._normalise(v.name), join(root, ...path, v.name)]),
-    );
-    for (const file of dirEntries) {
-      if (rules._checkPermitted(file.name)) {
-        if (file.isDirectory()) {
-          const dirPath = [...path, file.name];
-          precomputed._addDir(dirPath);
-          if (path.length < rules._subDirectories) {
-            queue.push(dirPath);
-          }
-        } else if (file.isFile()) {
-          precomputed._addFile(path, file.name, join(root, ...path, file.name), siblings);
-        }
-      }
-    }
-  }
+  await internalDiscoverFiles(
+    root,
+    rules,
+    (path, name, siblings) => precomputed._addFile(path, name, join(root, ...path, name), siblings),
+    (path) => precomputed._addDir(path),
+  );
   return precomputed;
 }
 
@@ -142,6 +125,37 @@ export class StaticFileFinder<T> implements FileFinder {
   }
 }
 
+export async function internalDiscoverFiles(
+  root: string,
+  rules: FileFinderRules,
+  fileCallback: (path: string[], name: string, siblings: Map<string, string>) => void,
+  dirCallback?: (path: string[]) => void,
+) {
+  const queue = new Queue<string[]>([]);
+  for (const path of queue) {
+    const dirEntries = await readdir(join(root, ...path), {
+      withFileTypes: true,
+      encoding: 'utf-8',
+    });
+    const siblings = new Map(
+      dirEntries.map((v) => [rules._normalise(v.name), join(root, ...path, v.name)]),
+    );
+    for (const file of dirEntries) {
+      if (rules._checkPermitted(file.name)) {
+        if (file.isDirectory()) {
+          const dirPath = [...path, file.name];
+          dirCallback?.(dirPath);
+          if (path.length < rules._subDirectories) {
+            queue.push(dirPath);
+          }
+        } else if (file.isFile()) {
+          fileCallback(path, file.name, siblings);
+        }
+      }
+    }
+  }
+}
+
 interface StaticFileInfo<T> {
   data: T | undefined;
   basename: string;