Allow zip contents in all places where files/directories are accepted in the CLI

Author: davidje13

README.md

@@ -90,3 +90,6 @@ and
 Note that this library does not implement rate limiting of any kind, so if you have an endpoint
 which is vulnerable to rapid requests (e.g. a password checking endpoint), you should set up your
 own rate limiting or use a reverse proxy such as NGINX and configure rate limiting there.
+
+The CLI offers several ways of running custom code (such as background tasks and custom handlers),
+so should never be launched with an untrusted configuration or bundle.

docs/API.md

@@ -2540,14 +2540,14 @@ file. Can be combined with [`registerMime`] to register the result.
 
 [`readZip`]: #readzippath
 
-- `path` [`<string>`] path to the zip file to read.
+- `path` [`<string>`] path to the zip archive to read.
 - Returns: [`<Promise>`] Fulfills with [`<ZipDirectory>`].
 
-Reads the structure of a `.zip` file and returns the root directory. This can be used to navigate to
+Reads the structure of a zip archive and returns the root directory. This can be used to navigate to
 and open any file in the zip.
 
 Compressed files in the zip are represented by "virtual" sibling files with `.deflate` suffixes. For
-example, a zip containing:
+example, a zip archive containing:
 
 - `foo/file1.txt` (compressed)
 - `foo/file2.txt` (stored)
@@ -3874,7 +3874,8 @@ connection can be reused _without_ needing to consume the body if `willSendBody`
 [`zipFileFinder`]: #zipfilefinderzipdirectory-options
 
 - `zipDirectory` [`<ZipDirectory>`] the base directory to serve files from. Only content within this
-  directory (or sub-directories) in the zip be served. See [`readZip`] and [`ZipDirectory.find`].
+  directory (or sub-directories) in the zip archive will be served. See [`readZip`] and
+  [`ZipDirectory.find`].
 - `options` [`<Object>`] A set of options controlling how files are matched, and which files are
   visible
   - `subDirectories` [`<boolean>`] | [`<number>`] `true` to allow access to all sub-directories,
@@ -3913,7 +3914,7 @@ Example usage: [Serving zip assets]
 
 - Extends: [`<ZipNode>`]
 
-A representation of a directory in a zip file.
+A representation of a directory in a zip archive.
 
 #### `zipdirectory.allFiles()`
 
@@ -3944,7 +3945,7 @@ Returns the child node at the requested path, or returns `undefined` if the path
 
 - Extends: [`<ZipNode>`]
 
-A representation of a file in a zip file.
+A representation of a file in a zip archive.
 
 #### `zipfile.virtual`
 
@@ -3995,7 +3996,7 @@ Ensure the `FileHandle` is always `close`d when no-longer required.
 
 [`<ZipNode>`]: #zipnode
 
-A representation of a file ([`<ZipFile>`]) or directory ([`<ZipDirectory>`]) in a zip file.
+A representation of a file ([`<ZipFile>`]) or directory ([`<ZipDirectory>`]) in a zip archive.
 
 #### `zipnode.isDirectory`
 
@@ -4309,7 +4310,7 @@ Reference: [`staticJSON`], [`<Router>`]
 
 [Serving zip assets]: #serving-zip-assets
 
-Example of serving static content directly from a `.zip` file:
+Example of serving static content directly from a zip archive:
 
 ```js
 import {
@@ -4335,7 +4336,7 @@ router.use(
 Reference: [`assetServer`], [`negotiateEncoding`], [`Negotiator`], [`readZip`], [`<Router>`],
 [`zipFileFinder`]
 
-The recommended way to generate a zip file for this purpose on MacOS and Unix is:
+The recommended way to generate a zip archive for this purpose on MacOS and Unix is:
 
 ```sh
 zip -9 -X -r -n .gz:.png:.jpg:.jpeg content.zip index.html style.css [...]

docs/CLI.md

@@ -16,6 +16,7 @@ This tool supports:
 - [running multiple servers simultaneously](#run-multiple-servers)
 - [running background tasks](#run-background-tasks)
 - [registering custom Javascript handlers](#custom-handlers)
+- [loading config and serving files directly from zips](#using-zips)
 
 Simple servers can be configured via CLI flags. Complex servers can be configured via JSON.
 
@@ -525,3 +526,51 @@ Notes:
   is possible to add versioning strings to imported paths (e.g. `./my-handler.mjs?v1`) to work
   around this, but doing so will lead to ever-increasing memory usage as old scripts remain in
   memory, and is not suitable for production deployments or long-lived development servers.
+
+### Using Zips
+
+All configuration files, custom handler scripts, and served directories can be specified as paths
+within a zip archive, using the form:
+
+```
+/path/to/file.zip/path/within/zip
+```
+
+If a config file is in a zip archive, any paths it contains will be relative to its location in the
+zip. This can be used to produce single-file "bundles" for simple services. For example, a zip named
+`bundle.zip` containing:
+
+- `config.json`:
+
+  ```json
+  {
+    "servers": [
+      {
+        "port": 8080,
+        "mount": [
+          { "type": "custom", "path": "/custom", "import": "./custom-handler.mjs" },
+          { "type": "files", "path": "/", "dir": "static" }
+        ]
+      }
+    ],
+    "mime": ["file://apache.types"]
+  }
+  ```
+
+- [`apache.types`](#add-mime-types-from-apache-types-file)
+- [`custom-handler.mjs`](#custom-handlers)
+- `static`
+  - `index.html`
+
+Could be loaded with:
+
+```sh
+npx web-listener --config-file ./bundle.zip/config.json
+```
+
+When performing path resolution, the zip is interpreted as a regular directory, so config files
+inside zip archives can still reference files outside the zip by navigating to them using `..`, or
+using absolute paths.
+
+Note that as with all CLI configuration, you should not use untrusted bundles, as they have the
+ability to access files and execute arbitrary code using the permissions of the current user.

package/cli.test.mjs

@@ -168,6 +168,41 @@ describe('cli', () => {
     expect(p.errors).isEmpty();
   });
 
+  it(
+    'accepts compressed configuration files',
+    { timeout: 3000 },
+    async ({ [TEARDOWN]: teardown }) => {
+      const [port] = await findAvailablePorts(1);
+
+      const p = spawnProcess(
+        join(...binDir, 'web-listener'),
+        ['-c', join('cli', 'bundle.zip', 'sample-config.json'), '--port', String(port)],
+        { env: { NO_COLOR: '1' } },
+      );
+      teardown(p.close);
+      if (!(await awaitLine(p.stderr, 'all servers ready'))) {
+        fail('failed to start server');
+      }
+
+      const resFile = await fetch(`http://localhost:${port}/file.txt`);
+      expect(resFile.status).equals(200);
+      expect(resFile.headers.get('content-type')).equals('text/bundled-apache; charset=utf-8');
+      expect(await resFile.text()).equals('Bundled content\n');
+
+      const resFixture = await fetch(`http://localhost:${port}/config.json`);
+      expect(resFixture.status).equals(200);
+      expect(await resFixture.text()).equals('{"env":"bundled"}');
+
+      const resRedirect = await fetch(`http://localhost:${port}/request`, { redirect: 'manual' });
+      expect(resRedirect.status).equals(307);
+      expect(resRedirect.headers.get('location')).equals('/bundled-other');
+
+      const resCustom = await fetch(`http://localhost:${port}/custom`);
+      expect(resCustom.status).equals(200);
+      expect(await resCustom.text()).equals('bundled custom response');
+    },
+  );
+
   const TEARDOWN = beforeEach(({ setParameter }) => {
     const tasks = [];
     setParameter((fn) => tasks.push(fn));

package/cli/bundle.zip

@@ -4,6 +4,7 @@ import type { Readable, Writable } from 'node:stream';
 import { findCause, HTTPError, WebListener, type ListenOptions } from '../index.mts';
 import type { ConfigServer, ConfigServerOptions, ConfigBackgroundTask } from './config/types.mts';
 import { buildRouter } from './routes/buildRouter.mts';
+import { clearZipCache } from './zipCache.mts';
 import type { Logger, AddColour } from './log.mts';
 import { TransientError } from './TransientError.mts';
 
@@ -123,7 +124,10 @@ export class ServerManager {
       if (this._stopping) {
         this._shutdown();
       } else if (doRetry) {
-        this._autoRetry = setTimeout(() => this.set(servers, backgroundTasks, errorHandler), 1000);
+        this._autoRetry = setTimeout(() => {
+          clearZipCache();
+          this.set(servers, backgroundTasks, errorHandler);
+        }, 1000);
       } else if (this._servers.size) {
         this._log(1, 'all servers ready');
       } else {

src/bin/ServerManager.mts

@@ -1,5 +1,5 @@
-import { readFile } from 'node:fs/promises';
 import type { FallbackOptions, FileNegotiation, FileNegotiationOption } from '../../index.mts';
+import { readAnyFile } from '../zipCache.mts';
 import type { Mapper } from './schema.mts';
 import type { Config, ConfigHeaders, ConfigMount } from './types.mts';
 
@@ -161,7 +161,7 @@ export async function loadConfig(
 
   let config: Config;
   if (file) {
-    config = parser(JSON.parse(await readFile(file, 'utf-8')), { file, path: '' });
+    config = parser(JSON.parse(await readAnyFile(file)), { file, path: '' });
   } else if (json) {
     config = parser(JSON.parse(json), { file: '', path: '' });
   } else {

src/bin/config/loader.mts

@@ -114,12 +114,16 @@ See
 below for details on the format.
 When JSON is provided this way, paths in the document are resolved relative to
 the file, not the current working directory.
+This can also be the path to a file inside a .zip archive
+.Po "e.g. "
+.Qq "/path/to/file.zip/config.json"
+.Pc .
 Once running, the configuration can be reloaded by sending SIGHUP, or by
 providing a newline to stdin.
 .
 .It Fl d Ar directory , Fl -dir Ar directory
 The directory to serve files from, relative to the current working directory.
-This can also be the path to a .zip file, or a path within a .zip file
+This can also be the path to a .zip archive, or a path within a .zip archive
 .Po "e.g. "
 .Qq "/path/to/file.zip/path/within/zip"
 .Pc .

src/bin/man1/web-listener.1

@@ -1,14 +1,14 @@
-import { readFile } from 'node:fs/promises';
 import { decompressMime, readMimeTypes, registerMime, resetMime } from '../index.mts';
 import type { ConfigMime } from './config/types.mts';
+import { readAnyFile } from './zipCache.mts';
 
 export async function loadMime(mime: ConfigMime | ConfigMime[]) {
   const newMimes: Map<string, string>[] = [];
   for (const item of Array.isArray(mime) ? mime : [mime]) {
     if (typeof item !== 'string') {
       newMimes.push(new Map(Object.entries(item)));
     } else if (item.startsWith('file://')) {
-      newMimes.push(readMimeTypes(await readFile(item.substring(7), 'utf-8')));
+      newMimes.push(readMimeTypes(await readAnyFile(item.substring(7))));
     } else {
       newMimes.push(decompressMime(item));
     }

src/bin/mime.mts

@@ -1,16 +1,15 @@
 import { stat } from 'node:fs/promises';
-import { join, sep } from 'node:path';
 import {
   dynamicFileFinder,
   negotiateEncoding,
   Negotiator,
-  readZip,
   staticFileFinder,
   zipFileFinder,
   type FileFinder,
   type FileServerOptions,
 } from '../../index.mts';
 import { TransientError } from '../TransientError.mts';
+import { readZipPath } from '../zipCache.mts';
 
 export async function anyFileFinder(path: string, options: FileServerOptions): Promise<FileFinder> {
   const direct = await stat(path).catch(() => null);
@@ -22,35 +21,18 @@ export async function anyFileFinder(path: string, options: FileServerOptions): P
     }
   }
 
-  const parts = path.split(sep);
-  if (parts[parts.length - 1] === '') {
-    parts.pop();
+  const zip = await readZipPath(path, false);
+  if (!zip) {
+    throw new TransientError(`content to serve not found at ${path}`);
   }
-  if (!parts[0]) {
-    parts.shift();
-    if (parts.length > 0) {
-      parts[0] = sep + parts[0];
-    }
+
+  const zipDir = zip.root.find(zip.remaining);
+  if (!zipDir?.isDirectory) {
+    throw new Error(`/${zip.remaining.join('/')} in ${zip.path} is not a directory`);
   }
-  for (let i = parts.length; i > 0; --i) {
-    const filePath = join(...parts.slice(0, i));
-    const stats = await stat(filePath).catch(() => null);
-    if (!stats) {
-      continue;
-    }
-    if (!stats.isFile()) {
-      break;
-    }
-    const zipRoot = await readZip(filePath);
-    const zipDir = zipRoot.find(parts.slice(i));
-    if (!zipDir?.isDirectory) {
-      throw new Error(`${parts.slice(i).join('/')} in ${filePath} is not a directory`);
-    }
-    const adjustedOptions = options;
-    if (!adjustedOptions.negotiator) {
-      adjustedOptions.negotiator = new Negotiator([negotiateEncoding(['deflate'])]);
-    }
-    return zipFileFinder(zipDir, adjustedOptions);
+  const adjustedOptions = options;
+  if (!adjustedOptions.negotiator) {
+    adjustedOptions.negotiator = new Negotiator([negotiateEncoding(['deflate'])]);
   }
-  throw new TransientError(`content to serve not found at ${path}`);
+  return zipFileFinder(zipDir, adjustedOptions);
 }