Make forwarding option a request transform, add path & query transforms

This is a breaking change if you're using thenForwardTo with a
beforeRequest callback (no longer allowed: return a url from
beforeRequest instead), or if you're manually building rules with a
'forwarding' option (use transformRequest.replaceHost instead).

In most cases though, this should work as before, but now supports
transformation of the path & query, match/replace transformation of the
host itself, and simplifies & aligns forwarding with other transform
options generally.

Author: pimterry

src/main.ts

@@ -55,8 +55,10 @@ export type {
     CADefinition,
     ForwardingOptions,
     PassThroughLookupOptions,
-    PassThroughStepConnectionOptions
+    PassThroughStepConnectionOptions,
+    PassThroughInitialTransforms
 } from './rules/passthrough-handling-definitions';
+export type { MatchReplacePairs } from './rules/match-replace';
 
 export type { RequestRuleBuilder } from "./rules/requests/request-rule-builder";
 export type { WebSocketRuleBuilder } from "./rules/websockets/websocket-rule-builder";

src/rules/match-replace.ts

@@ -0,0 +1,37 @@
+/**
+ * An array of match/replace pairs. These will be applied to the initial value
+ * like `input.replace(p1, p2)`, applied in the order provided. The first parameter
+ * can be either a string or RegExp to match, and the second must be a string to
+ * insert. The normal `str.replace` $ placeholders can be used in the second
+ * argument, so that e.g. $1 will insert the 1st matched group.
+ */
+export type MatchReplacePairs = Array<[string | RegExp, string]>;
+
+export function applyMatchReplace(input: string, matchReplace: MatchReplacePairs): string {
+    let result = input;
+    for (const [match, replacement] of matchReplace) {
+        result = result.replace(match, replacement);
+    }
+    return result;
+}
+
+export type SerializedRegex = { regexSource: string, flags: string };
+
+export const serializeRegex = (regex: RegExp): SerializedRegex => ({ regexSource: regex.source, flags: regex.flags });
+export const deserializeRegex = (regex: SerializedRegex) => new RegExp(regex.regexSource, regex.flags);
+
+export type SerializedMatchReplacePairs = Array<[SerializedRegex | string, string]>;
+
+export const serializeMatchReplaceConfiguration = (matchReplace: MatchReplacePairs): SerializedMatchReplacePairs =>
+    matchReplace.map(([match, result]) => [
+        match instanceof RegExp ? serializeRegex(match) : match,
+        result
+    ]);
+
+export const deserializeMatchReplaceConfiguration = (matchReplace: SerializedMatchReplacePairs): MatchReplacePairs =>
+    matchReplace.map(([match, result]) => [
+        typeof match !== 'string' && 'regexSource' in match
+            ? deserializeRegex(match)
+            : match,
+        result
+    ]);

src/rules/passthrough-handling-definitions.ts

@@ -1,4 +1,5 @@
 import { ProxyConfig } from "./proxy-config";
+import { MatchReplacePairs } from "./match-replace";
 
 export interface ForwardingOptions {
     targetHost: string,
@@ -38,13 +39,10 @@ export type CADefinition =
  */
 export interface PassThroughStepConnectionOptions {
     /**
-     * The forwarding configuration for the passthrough rule.
-     * This generally shouldn't be used explicitly unless you're
-     * building rule data by hand. Instead, call `thenPassThrough`
-     * to send data directly or `thenForwardTo` with options to
-     * configure traffic forwarding.
+     * A set of data to automatically transform a request. This includes properties
+     * to support many transformation common use cases.
      */
-    forwarding?: ForwardingOptions,
+    transformRequest?: PassThroughInitialTransforms;
 
     /**
      * A list of hostnames for which server certificate and TLS version errors
@@ -121,4 +119,49 @@ export interface PassThroughStepConnectionOptions {
      * transparently proxy network traffic, errors and all.
      */
     simulateConnectionErrors?: boolean;
+}
+
+/**
+ * This defines the request transforms that we support for all passed through
+ * requests (both HTTP and WebSockets).
+ */
+export interface PassThroughInitialTransforms {
+
+    // Made more specific in subclass overrides
+    setProtocol?: 'http' | 'https' | 'ws' | 'wss';
+
+    /**
+     * Replace the request host with a single fixed value, effectively forwarding
+     * all requests to a different hostname.
+     *
+     * This cannot be combined with matchReplaceHost.
+     *
+     * If updateHostHeader is true, the Host (or :authority for HTTP/2+) header
+     * will be updated automatically to match. If updateHostHeader is a string,
+     * that will be used directly as the header value. If it's false no change
+     * will be made. If not specified this defaults to true.
+     */
+    replaceHost?: { targetHost: string, updateHostHeader?: true | false | string };
+
+    /**
+     * Perform a series of string match & replace operations on the request host.
+     *
+     * This cannot be combined with replaceHost.
+     *
+     * If updateHostHeader is true, the Host (or :authority for HTTP/2+) header
+     * will be updated automatically to match. If updateHostHeader is a string,
+     * that will be used directly as the header value. If it's false no change
+     * will be made. If not specified this defaults to true.
+     */
+    matchReplaceHost?: { replacements: MatchReplacePairs, updateHostHeader?: true | false | string };
+
+    /**
+     * Perform a series of string match & replace operations on the request path.
+     */
+    matchReplacePath?: MatchReplacePairs;
+
+    /**
+     * Perform a series of string match & replace operations on the request query string.
+     */
+    matchReplaceQuery?: MatchReplacePairs;
 }

src/rules/passthrough-handling.ts

@@ -14,7 +14,7 @@ import { CachedDns, dnsLookup, DnsLookupFunction } from '../util/dns';
 import { isMockttpBody, encodeBodyBuffer } from '../util/request-utils';
 import { areFFDHECurvesSupported } from '../util/openssl-compat';
 import { ErrorLike, unreachableCheck } from '@httptoolkit/util';
-import { getHeaderValue } from '../util/header-utils';
+import { findRawHeaderIndex, getHeaderValue } from '../util/header-utils';
 
 import {
     CallbackRequestResult,
@@ -23,8 +23,11 @@ import {
 import { AbortError } from './requests/request-step-impls';
 import {
     CADefinition,
+    PassThroughInitialTransforms,
     PassThroughLookupOptions
 } from './passthrough-handling-definitions';
+import { getDefaultPort } from '../util/url';
+import { applyMatchReplace } from './match-replace';
 
 // TLS settings for proxied connections, intended to avoid TLS fingerprint blocking
 // issues so far as possible, by closely emulating a Firefox Client Hello:
@@ -232,6 +235,91 @@ function deriveUrlLinkedHeader(
     return expectedValue;
 }
 
+export function applyDestinationTransforms(
+    transform: PassThroughInitialTransforms,
+    { isH2Downstream, rawHeaders, port, protocol, hostname, pathname, query }: {
+        isH2Downstream: boolean,
+        rawHeaders: RawHeaders,
+        port: string | null
+        protocol: string | null,
+        hostname: string,
+        pathname: string | null
+        query: string | null
+    },
+) {
+    const {
+        setProtocol,
+        replaceHost,
+        matchReplaceHost,
+        matchReplacePath,
+        matchReplaceQuery,
+    } = transform;
+
+    if (setProtocol) {
+        const wasDefaultPort = port === null || getDefaultPort(protocol || 'http') === parseInt(port, 10);
+        protocol = setProtocol + ':';
+
+        // If we were on the default port, update that accordingly:
+        if (wasDefaultPort) {
+            port = getDefaultPort(protocol).toString();
+        }
+    }
+
+    if (replaceHost) {
+        const { targetHost } = replaceHost;
+        [hostname, port] = targetHost.split(':');
+    }
+
+    if (matchReplaceHost) {
+        const result = applyMatchReplace(port === null ? hostname! : `${hostname}:${port}`, matchReplaceHost.replacements);
+        [hostname, port] = result.split(':');
+    }
+
+    if ((replaceHost?.updateHostHeader ?? matchReplaceHost?.updateHostHeader) !== false) {
+        const updateHostHeader = replaceHost?.updateHostHeader ?? matchReplaceHost?.updateHostHeader;
+        const hostHeaderName = isH2Downstream ? ':authority' : 'host';
+
+        let hostHeaderIndex = findRawHeaderIndex(rawHeaders, hostHeaderName);
+        let hostHeader: [string, string];
+
+        if (hostHeaderIndex === -1) {
+            // Should never happen really, but just in case:
+            hostHeader = [hostHeaderName, hostname!];
+            hostHeaderIndex = rawHeaders.length;
+        } else {
+            // Clone this - we don't want to modify the original headers, as they're used for events
+            hostHeader = _.clone(rawHeaders[hostHeaderIndex]);
+        }
+        rawHeaders[hostHeaderIndex] = hostHeader;
+
+        if (updateHostHeader === undefined || updateHostHeader === true) {
+            // If updateHostHeader is true, or just not specified, match the new target
+            hostHeader[1] = hostname + (port ? `:${port}` : '');
+        } else if (updateHostHeader) {
+            // If it's an explicit custom value, use that directly.
+            hostHeader[1] = updateHostHeader;
+        } // Otherwise: falsey means don't touch it.
+    }
+
+    if (matchReplacePath) {
+        pathname = applyMatchReplace(pathname || '/', matchReplacePath);
+    }
+
+    if (matchReplaceQuery) {
+        query = applyMatchReplace(query || '', matchReplaceQuery);
+    }
+
+    return {
+        reqUrl: new URL(`${protocol}//${hostname}${(port ? `:${port}` : '')}${pathname || '/'}${query || ''}`).toString(),
+        protocol,
+        hostname,
+        port,
+        pathname,
+        query,
+        rawHeaders
+    };
+}
+
 /**
  * Autocorrect the host header only in the case that if you didn't explicitly
  * override it yourself for some reason (e.g. if you're testing bad behaviour).
@@ -421,11 +509,11 @@ export const getDnsLookupFunction = _.memoize((lookupOptions: PassThroughLookupO
 });
 
 export async function getClientRelativeHostname(
-    hostname: string | null,
+    hostname: string,
     remoteIp: string | undefined,
     lookupFn: DnsLookupFunction
 ) {
-    if (!hostname || !remoteIp || isLocalhostAddress(remoteIp)) return hostname;
+    if (!remoteIp || isLocalhostAddress(remoteIp)) return hostname;
 
     // Otherwise, we have a request from a different machine (or Docker container/VM/etc) and we need
     // to make sure that 'localhost' means _that_ machine, not ourselves.
@@ -441,7 +529,7 @@ export async function getClientRelativeHostname(
         // effectively free. We ignore errors to delegate unresolvable etc to request processing later.
         isLocalhostAddress(await dnsLookup(lookupFn, hostname).catch(() => null))
     ) {
-        return normalizeIP(remoteIp) as string | null;
+        return normalizeIP(remoteIp);
 
         // Note that we just redirect - we don't update the host header. From the POV of the target, it's still
         // 'localhost' traffic that should appear identical to normal.

src/rules/requests/request-rule-builder.ts

@@ -1,5 +1,6 @@
 import { merge, isString, isBuffer } from "lodash";
 import { Readable } from "stream";
+import * as url from 'url';
 import { MaybePromise } from '@httptoolkit/util';
 
 import { Headers, CompletedRequest, Method, MockedEndpoint, Trailers } from "../../types";
@@ -379,16 +380,20 @@ export class RequestRuleBuilder extends BaseRuleBuilder {
      * @category Responses
      */
     async thenForwardTo(
-        forwardToLocation: string,
-        options: Omit<PassThroughStepOptions, 'forwarding'> & {
-            forwarding?: Omit<PassThroughStepOptions['forwarding'], 'targetHost'>
-        } = {}
+        target: string,
+        options: PassThroughStepOptions = {}
     ): Promise<MockedEndpoint> {
+        const protocolIndex = target.indexOf('://');
+        let { protocol, host } = protocolIndex !== -1
+            ? { protocol: target.slice(0, protocolIndex), host: target.slice(protocolIndex + 3) }
+            : { host: target, protocol: null};
+
         this.steps.push(new PassThroughStep({
             ...options,
-            forwarding: {
-                ...options.forwarding,
-                targetHost: forwardToLocation
+            transformRequest: {
+                ...options.transformRequest,
+                setProtocol: protocol as 'http' | 'https' | undefined,
+                replaceHost: { targetHost: host }
             }
         }));