Add support for customizing HTTP headers (apple#1196)

This PR adds support for custom HTTP headers, introducing a
`--http-header` CLI flag to accept `key=value` pairs. These headers can
also be specified within the `setting.pkl` file.

Closes apple#633

SPICE: apple/pkl-evolution#24

---------

Co-authored-by: Jen Basch <jbasch94@gmail.com>
Co-authored-by: Islon Scherer <islonscherer@gmail.com>

Author: 3 people

pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/CliBaseOptions.kt

@@ -20,6 +20,7 @@ import java.nio.file.Files
 import java.nio.file.Path
 import java.time.Duration
 import java.util.regex.Pattern
+import org.pkl.core.Pair
 import org.pkl.core.evaluatorSettings.Color
 import org.pkl.core.evaluatorSettings.PklEvaluatorSettings.ExternalReader
 import org.pkl.core.evaluatorSettings.TraceMode
@@ -144,6 +145,9 @@ data class CliBaseOptions(
   /** URL prefixes to rewrite. */
   val httpRewrites: Map<URI, URI>? = null,
 
+  /** HTTP headers to add to the request. */
+  val httpHeaders: List<Pair<Pattern, List<Pair<String, String>>>>? = null,
+
   /** External module reader process specs */
   val externalModuleReaders: Map<String, ExternalReader> = mapOf(),
 

pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/CliCommand.kt

@@ -218,6 +218,10 @@ abstract class CliCommand(protected val cliOptions: CliBaseOptions) {
     cliOptions.httpRewrites ?: evaluatorSettings?.http?.rewrites ?: settings.http?.rewrites()
   }
 
+  private val httpHeaders: List<Pair<Pattern, List<Pair<String, String>>>>? by lazy {
+    cliOptions.httpHeaders ?: project?.evaluatorSettings?.http?.headers ?: settings.http?.headers
+  }
+
   protected val externalModuleReaders: Map<String, PklEvaluatorSettings.ExternalReader> by lazy {
     (evaluatorSettings?.externalModuleReaders ?: emptyMap()) + cliOptions.externalModuleReaders
   }
@@ -277,6 +281,7 @@ abstract class CliCommand(protected val cliOptions: CliBaseOptions) {
         setProxy(proxyAddress, noProxy ?: listOf())
       }
       httpRewrites?.let(::setRewrites)
+      httpHeaders?.let(::setHeaders)
       // Lazy building significantly reduces execution time of commands that do minimal work.
       // However, it means that HTTP client initialization errors won't surface until an HTTP
       // request is made.

pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/commands/BaseOptions.kt

@@ -31,10 +31,12 @@ import java.util.regex.Pattern
 import org.pkl.commons.cli.CliBaseOptions
 import org.pkl.commons.cli.CliException
 import org.pkl.commons.shlex
+import org.pkl.core.Pair as PPair
 import org.pkl.core.evaluatorSettings.Color
 import org.pkl.core.evaluatorSettings.PklEvaluatorSettings.ExternalReader
 import org.pkl.core.evaluatorSettings.TraceMode
 import org.pkl.core.runtime.VmUtils
+import org.pkl.core.util.GlobResolver
 import org.pkl.core.util.IoUtils
 
 @Suppress("MemberVisibilityCanBePrivate")
@@ -285,6 +287,37 @@ class BaseOptions : OptionGroup() {
       .multiple()
       .toMap()
 
+  val httpHeaders: List<PPair<Pattern, List<PPair<String, String>>>> by
+    option(
+        names = arrayOf("--http-headers"),
+        metavar = "<url-pattern>=<header name>:<header value>",
+        help = "HTTP header to add to the request.",
+      )
+      .splitPair()
+      .transformAll { it ->
+        val headersMap = mutableMapOf<String, MutableList<PPair<String, String>>>()
+
+        try {
+          val headerRegex = Regex("""^(.+?):[ \t]*(.+)$""")
+          for ((stringPattern, header) in it) {
+            val (headerName, headerValue) =
+              headerRegex.find(header)?.destructured
+                ?: fail("Header '$header' is not in 'name:value' format.")
+            IoUtils.validateHeaderName(headerName)
+            IoUtils.validateHeaderValue(headerValue)
+            headersMap
+              .computeIfAbsent(stringPattern) { mutableListOf() }
+              .add(PPair(headerName, headerValue))
+          }
+
+          headersMap.entries.map { PPair(GlobResolver.toRegexPattern(it.key), it.value) }
+        } catch (e: IllegalArgumentException) {
+          fail(e.message!!)
+        } catch (e: GlobResolver.InvalidGlobPatternException) {
+          fail(e.message!!)
+        }
+      }
+
   val externalModuleReaders: Map<String, ExternalReader> by
     option(
         names = arrayOf("--external-module-reader"),
@@ -351,6 +384,7 @@ class BaseOptions : OptionGroup() {
       httpProxy = proxy,
       httpNoProxy = noProxy,
       httpRewrites = httpRewrites.ifEmpty { null },
+      httpHeaders = httpHeaders.ifEmpty { null },
       externalModuleReaders = externalModuleReaders,
       externalResourceReaders = externalResourceReaders,
       traceMode = traceMode,

pkl-core/src/main/java/org/pkl/core/evaluatorSettings/PklEvaluatorSettings.java

@@ -1,5 +1,5 @@
 /*
- * Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
+ * Copyright © 2024-2026 Apple Inc. and the Pkl project authors. All rights reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -18,6 +18,7 @@
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.nio.file.Path;
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
@@ -27,13 +28,17 @@
 import java.util.function.BiFunction;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 import org.pkl.core.Duration;
 import org.pkl.core.PNull;
 import org.pkl.core.PObject;
+import org.pkl.core.Pair;
 import org.pkl.core.PklBugException;
 import org.pkl.core.PklException;
 import org.pkl.core.Value;
 import org.pkl.core.util.ErrorMessages;
+import org.pkl.core.util.GlobResolver;
+import org.pkl.core.util.GlobResolver.InvalidGlobPatternException;
 import org.pkl.core.util.Nullable;
 
 /** Java version of {@code pkl.EvaluatorSettings}. */
@@ -126,8 +131,11 @@ public static PklEvaluatorSettings parse(
         traceMode == null ? null : TraceMode.valueOf(traceMode.toUpperCase()));
   }
 
-  public record Http(@Nullable Proxy proxy, @Nullable Map<URI, URI> rewrites) {
-    public static final Http DEFAULT = new Http(null, Collections.emptyMap());
+  public record Http(
+      @Nullable Proxy proxy,
+      @Nullable Map<URI, URI> rewrites,
+      @Nullable List<Pair<Pattern, List<Pair<String, String>>>> headers) {
+    public static final Http DEFAULT = new Http(null, Collections.emptyMap(), null);
 
     @SuppressWarnings("unchecked")
     public static @Nullable Http parse(@Nullable Value input) {
@@ -136,10 +144,9 @@ public record Http(@Nullable Proxy proxy, @Nullable Map<URI, URI> rewrites) {
       } else if (input instanceof PObject http) {
         var proxy = Proxy.parse((Value) http.getProperty("proxy"));
         var rewrites = http.getProperty("rewrites");
-        if (rewrites instanceof PNull) {
-          return new Http(proxy, null);
-        } else {
-          var parsedRewrites = new HashMap<URI, URI>();
+        HashMap<URI, URI> parsedRewrites = null;
+        if (!(rewrites instanceof PNull)) {
+          parsedRewrites = new HashMap<>();
           for (var entry : ((Map<String, String>) rewrites).entrySet()) {
             var key = entry.getKey();
             var value = entry.getValue();
@@ -149,8 +156,37 @@ public record Http(@Nullable Proxy proxy, @Nullable Map<URI, URI> rewrites) {
               throw new PklException(ErrorMessages.create("invalidUri", e.getInput()));
             }
           }
-          return new Http(proxy, parsedRewrites);
         }
+        var headerDefs = http.getProperty("headers");
+        List<Pair<Pattern, List<Pair<String, String>>>> parsedHeaderDefs = null;
+        if (!(headerDefs instanceof PNull)) {
+          parsedHeaderDefs = new ArrayList<>();
+          var headerDefsMap = (Map<String, Map<String, Object>>) headerDefs;
+          for (var entry : headerDefsMap.entrySet()) {
+            var stringPattern = entry.getKey();
+            var headersMap = entry.getValue();
+            try {
+              var urlPattern = GlobResolver.toRegexPattern(stringPattern);
+              var pairs =
+                  headersMap.entrySet().stream()
+                      .flatMap(
+                          header -> {
+                            var value = header.getValue();
+                            if (value instanceof List) {
+                              return ((List<String>) value)
+                                  .stream().map(v -> new Pair(header.getKey(), v));
+                            } else {
+                              return Stream.of(new Pair(header.getKey(), value));
+                            }
+                          })
+                      .toList();
+              parsedHeaderDefs.add(new Pair(urlPattern, pairs));
+            } catch (InvalidGlobPatternException e) {
+              throw new PklException(ErrorMessages.create("invalidUri", stringPattern));
+            }
+          }
+        }
+        return new Http(proxy, parsedRewrites, parsedHeaderDefs);
       } else {
         throw PklBugException.unreachableCode();
       }

pkl-core/src/main/java/org/pkl/core/http/HttpClient.java

@@ -1,5 +1,5 @@
 /*
- * Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
+ * Copyright © 2024-2026 Apple Inc. and the Pkl project authors. All rights reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -23,7 +23,9 @@
 import java.nio.file.Path;
 import java.util.List;
 import java.util.Map;
+import java.util.regex.Pattern;
 import javax.net.ssl.SSLContext;
+import org.pkl.core.Pair;
 import org.pkl.core.util.Nullable;
 
 /**
@@ -150,6 +152,14 @@ interface Builder {
      */
     Builder addRewrite(URI sourcePrefix, URI targetPrefix);
 
+    /**
+     * Sets the HTTP headers for the request, replacing any previously configured headers.
+     *
+     * <p>This method clears all existing headers and replaces them with the contents of the
+     * provided map.
+     */
+    Builder setHeaders(List<Pair<Pattern, List<Pair<String, String>>>> headers);
+
     /**
      * Creates a new {@code HttpClient} from the current state of this builder.
      *

pkl-core/src/main/java/org/pkl/core/http/HttpClientBuilder.java

@@ -1,5 +1,5 @@
 /*
- * Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
+ * Copyright © 2024-2026 Apple Inc. and the Pkl project authors. All rights reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -27,6 +27,8 @@
 import java.util.List;
 import java.util.Map;
 import java.util.function.Supplier;
+import java.util.regex.Pattern;
+import org.pkl.core.Pair;
 import org.pkl.core.Release;
 import org.pkl.core.http.HttpClient.Builder;
 
@@ -39,6 +41,7 @@ final class HttpClientBuilder implements HttpClient.Builder {
   private int testPort = -1;
   private ProxySelector proxySelector;
   private Map<URI, URI> rewrites = new HashMap<>();
+  private List<Pair<Pattern, List<Pair<String, String>>>> headers = new ArrayList<>();
 
   HttpClientBuilder() {
     var release = Release.current();
@@ -110,6 +113,12 @@ public Builder addRewrite(URI sourcePrefix, URI targetPrefix) {
     return this;
   }
 
+  @Override
+  public Builder setHeaders(List<Pair<Pattern, List<Pair<String, String>>>> headers) {
+    this.headers = headers;
+    return this;
+  }
+
   @Override
   public HttpClient build() {
     return doBuild().get();
@@ -128,7 +137,8 @@ private Supplier<HttpClient> doBuild() {
     return () -> {
       var jdkClient =
           new JdkHttpClient(certificateFiles, certificateBytes, connectTimeout, proxySelector);
-      return new RequestRewritingClient(userAgent, requestTimeout, testPort, jdkClient, rewrites);
+      return new RequestRewritingClient(
+          userAgent, requestTimeout, testPort, jdkClient, rewrites, headers);
     };
   }
 }

pkl-core/src/main/java/org/pkl/core/http/RequestRewritingClient.java

@@ -1,5 +1,5 @@
 /*
- * Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
+ * Copyright © 2024-2026 Apple Inc. and the Pkl project authors. All rights reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -28,7 +28,10 @@
 import java.util.Map.Entry;
 import java.util.Objects;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.regex.Pattern;
+import java.util.stream.Stream;
 import javax.annotation.concurrent.ThreadSafe;
+import org.pkl.core.Pair;
 import org.pkl.core.PklBugException;
 import org.pkl.core.util.HttpUtils;
 import org.pkl.core.util.Nullable;
@@ -54,6 +57,7 @@ final class RequestRewritingClient implements HttpClient {
   final int testPort;
   final HttpClient delegate;
   private final List<Entry<URI, URI>> rewrites;
+  private final List<Pair<Pattern, List<Pair<String, String>>>> headers;
 
   private final AtomicBoolean closed = new AtomicBoolean();
 
@@ -62,7 +66,8 @@ final class RequestRewritingClient implements HttpClient {
       Duration requestTimeout,
       int testPort,
       HttpClient delegate,
-      Map<URI, URI> rewrites) {
+      Map<URI, URI> rewrites,
+      List<Pair<Pattern, List<Pair<String, String>>>> headers) {
     this.userAgent = userAgent;
     this.requestTimeout = requestTimeout;
     this.testPort = testPort;
@@ -72,6 +77,7 @@ final class RequestRewritingClient implements HttpClient {
             .map((it) -> Map.entry(normalizeRewrite(it.getKey()), normalizeRewrite(it.getValue())))
             .sorted(Comparator.comparingInt((it) -> -it.getKey().toString().length()))
             .toList();
+    this.headers = headers;
   }
 
   @Override
@@ -112,6 +118,9 @@ private HttpRequest rewriteRequest(HttpRequest original) {
         .map()
         .forEach((name, values) -> values.forEach(value -> builder.header(name, value)));
     builder.setHeader("User-Agent", userAgent);
+    for (var header : this.getHeaders(original.uri())) {
+      builder.header(header.getFirst(), header.getSecond());
+    }
 
     var method = original.method();
     original
@@ -216,6 +225,16 @@ private URI rewriteUri(URI uri) {
     return ret;
   }
 
+  private List<Pair<String, String>> getHeaders(URI uri) {
+    return headers.stream()
+        .flatMap(
+            rule ->
+                rule.getFirst().asPredicate().test(uri.toString())
+                    ? rule.getSecond().stream()
+                    : Stream.empty())
+        .toList();
+  }
+
   private void checkNotClosed(HttpRequest request) {
     if (closed.get()) {
       throw new IllegalStateException(