Skip to content

Commit 813d784

Browse files
netvlnetvl
committed
Added support for external readers in Gradle
This PR adds support for configuring external module and resource readers in the Gradle plugin, for all tasks which may need it. Currently, no external readers are passed to the CliBaseOptions instance which is constructed by every task. Therefore, the only way external readers can be used with Gradle is via PklProject definitions, which sometimes may be limiting. After this change, it will be possible to configure external readers for any tasks, including those which operate on individual modules and not the entire projects. I'm using NamedDomainObjectContainer to store reader definitions. The name of each reader's definition is passed through into the CLI classes as the map key. Therefore, you can define an external reader like this: ```kotlin val myPklTask by pkl.operationKind.creating { val foo by externalModuleReaders.creating { executable = "external-executable" arguments = listOf("--bar", "baz") } } ``` and then use `foo` as the URI schema in the Pkl code: ```pkl import "foo:a/b/c" ``` Properties of each external reader spec are defined as proper properties and are propagated as input dependencies to the task, so the caching and task dependency resolution behaves as expected. I completely forgot that apple#1067 exists and reimplemented this from scratch. But my approach to the interface is slightly different and more in line with Gradle recommendations, IMO, and I really need this functionality so I intend to see this through. As in that PR, there are no tests which validate the actual invocation of the external command because creating one requires writing a command which does msgpack, which I'm unsure of how to do properly within this project.
1 parent 14085c1 commit 813d784

13 files changed

Lines changed: 645 additions & 47 deletions

File tree

‎pkl-gradle/pkl-gradle.gradle.kts‎

Lines changed: 49 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -63,13 +63,59 @@ sourceSets {
6363
}
6464
}
6565

66+
// Support for testing with a real external reader in tests - this builds an additional source set
67+
// into a jar with a main class which provides a simple external reader implementation.
68+
// Then the path to the jar file and the toolchain's `java` binary
69+
// are injected into tests as properties.
70+
71+
val externalReader by sourceSets.creating {}
72+
73+
dependencies { "externalReaderImplementation"(libs.msgpack) }
74+
75+
val externalReaderJar by
76+
tasks.registering(Jar::class) {
77+
description = "Builds an external reader executable jar file"
78+
archiveBaseName = "external-reader"
79+
archiveVersion = ""
80+
81+
// Package all dependencies into the jar (shadow plugin lite).
82+
from(
83+
externalReader.runtimeClasspath.elements.map { locations ->
84+
locations.mapNotNull { location ->
85+
val f = location.asFile
86+
when {
87+
f.isDirectory -> f
88+
f.isFile -> zipTree(f)
89+
else -> null
90+
}
91+
}
92+
}
93+
)
94+
95+
manifest { attributes("Main-Class" to "org.pkl.gradle.test.extreader.Main") }
96+
}
97+
98+
tasks.test {
99+
dependsOn(externalReaderJar)
100+
// Currently the only way to inject system properties from lazy values in Gradle
101+
// is via `jvmArgumentProviders`.
102+
jvmArgumentProviders += CommandLineArgumentProvider {
103+
listOf(
104+
"-DpklGradle.externalReaderJar=" +
105+
externalReaderJar.get().archiveFile.get().asFile.absolutePath,
106+
"-DpklGradle.javaExecutable=" +
107+
javaToolchains.launcherFor(java.toolchain).get().executablePath.asFile.absolutePath,
108+
)
109+
}
110+
}
111+
66112
publishing {
67113
publications {
68114
withType<MavenPublication>().configureEach {
69115
pom {
70-
name.set("pkl-gradle plugin")
71-
url.set("https://github.com/apple/pkl/tree/main/pkl-gradle")
72-
description.set("Gradle plugin for the Pkl configuration language.")
116+
name = "pkl-gradle plugin"
117+
url = "https://github.com/apple/pkl/tree/main/pkl-gradle"
118+
description = "Gradle plugin for the Pkl configuration language."
73119
}
74120
}
75121
}

‎pkl-gradle/src/externalReader/java/org/pkl/gradle/test/extreader/Main.java‎

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
/*
2+
* Copyright © 2026 Apple Inc. and the Pkl project authors. All rights reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* https://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
package org.pkl.gradle.test.extreader;
17+
18+
import java.io.IOException;
19+
import java.nio.charset.StandardCharsets;
20+
import org.msgpack.core.MessagePack;
21+
import org.msgpack.value.Value;
22+
import org.msgpack.value.ValueFactory;
23+
24+
/**
25+
* A minimal external resource reader for Pkl. Uppercases the scheme-specific part of the URI and
26+
* returns it as binary content. Implements the Pkl external reader MessagePack protocol over
27+
* stdin/stdout.
28+
*/
29+
public class Main {
30+
private static final int INITIALIZE_RESOURCE_READER_REQUEST = 0x30;
31+
private static final int INITIALIZE_RESOURCE_READER_RESPONSE = 0x31;
32+
private static final int READ_RESOURCE_REQUEST = 0x26;
33+
private static final int READ_RESOURCE_RESPONSE = 0x27;
34+
private static final int CLOSE_EXTERNAL_PROCESS = 0x32;
35+
36+
private static final Value KEY_REQUEST_ID = ValueFactory.newString("requestId");
37+
private static final Value KEY_EVALUATOR_ID = ValueFactory.newString("evaluatorId");
38+
private static final Value KEY_SCHEME = ValueFactory.newString("scheme");
39+
private static final Value KEY_URI = ValueFactory.newString("uri");
40+
41+
public static void main(String[] args) throws IOException {
42+
var unpacker = MessagePack.newDefaultUnpacker(System.in);
43+
var packer = MessagePack.newDefaultPacker(System.out);
44+
45+
while (unpacker.hasNext()) {
46+
var arrayLen = unpacker.unpackArrayHeader();
47+
if (arrayLen != 2) {
48+
throw new IOException("Expected array of 2, got " + arrayLen);
49+
}
50+
var msgType = unpacker.unpackInt();
51+
var body = unpacker.unpackValue().asMapValue().map();
52+
53+
switch (msgType) {
54+
case INITIALIZE_RESOURCE_READER_REQUEST -> {
55+
var requestId = body.get(KEY_REQUEST_ID).asIntegerValue().asLong();
56+
var scheme = body.get(KEY_SCHEME).asStringValue().asString();
57+
58+
packer.packArrayHeader(2);
59+
packer.packInt(INITIALIZE_RESOURCE_READER_RESPONSE);
60+
packer.packMapHeader(2);
61+
packer.packString("requestId");
62+
packer.packLong(requestId);
63+
packer.packString("spec");
64+
packer.packMapHeader(3);
65+
packer.packString("scheme");
66+
packer.packString(scheme);
67+
packer.packString("hasHierarchicalUris");
68+
packer.packBoolean(false);
69+
packer.packString("isGlobbable");
70+
packer.packBoolean(false);
71+
packer.flush();
72+
}
73+
case READ_RESOURCE_REQUEST -> {
74+
var requestId = body.get(KEY_REQUEST_ID).asIntegerValue().asLong();
75+
var evaluatorId = body.get(KEY_EVALUATOR_ID).asIntegerValue().asLong();
76+
var uri = body.get(KEY_URI).asStringValue().asString();
77+
78+
var colonIndex = uri.indexOf(':');
79+
var schemeSpecific = colonIndex >= 0 ? uri.substring(colonIndex + 1) : uri;
80+
var contents = schemeSpecific.toUpperCase().getBytes(StandardCharsets.UTF_8);
81+
82+
packer.packArrayHeader(2);
83+
packer.packInt(READ_RESOURCE_RESPONSE);
84+
packer.packMapHeader(3);
85+
packer.packString("requestId");
86+
packer.packLong(requestId);
87+
packer.packString("evaluatorId");
88+
packer.packLong(evaluatorId);
89+
packer.packString("contents");
90+
packer.packBinaryHeader(contents.length);
91+
packer.writePayload(contents);
92+
packer.flush();
93+
}
94+
case CLOSE_EXTERNAL_PROCESS -> {
95+
return;
96+
}
97+
default -> {}
98+
}
99+
}
100+
}
101+
}

‎pkl-gradle/src/main/java/org/pkl/gradle/PklPlugin.java‎

Lines changed: 32 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,10 @@
2626
import org.gradle.api.Plugin;
2727
import org.gradle.api.Project;
2828
import org.gradle.api.Transformer;
29+
import org.gradle.api.file.ProjectLayout;
2930
import org.gradle.api.file.SourceDirectorySet;
3031
import org.gradle.api.provider.Provider;
32+
import org.gradle.api.provider.ProviderFactory;
3133
import org.gradle.api.tasks.SourceSet;
3234
import org.gradle.api.tasks.SourceSetContainer;
3335
import org.gradle.api.tasks.TaskProvider;
@@ -138,11 +140,13 @@ private void configureAnalyzeImportsTasks(
138140
configureBaseSpec(project, spec);
139141
spec.getOutputFormat().convention(OutputFormat.PCF.toString());
140142
var analyzeImportsTask = createTask(project, AnalyzeImportsTask.class, spec);
143+
var layout = project.getLayout();
144+
var providers = project.getProviders();
141145
analyzeImportsTask.configure(
142146
task -> {
143147
task.getOutputFormat().set(spec.getOutputFormat());
144148
task.getOutputFile().set(spec.getOutputFile());
145-
configureModulesTask(project, task, spec, null);
149+
configureModulesTask(layout, providers, task, spec, null, null);
146150
});
147151
});
148152
}
@@ -465,8 +469,8 @@ private void configureCodeGenTask(CodeGenTask task, CodeGenSpec spec) {
465469
}
466470

467471
private <T extends BasePklTask, S extends BasePklSpec> void configureBaseTask(
468-
Project project, T task, S spec) {
469-
task.getWorkingDir().set(project.getLayout().getProjectDirectory());
472+
ProjectLayout layout, ProviderFactory providers, T task, S spec) {
473+
task.getWorkingDir().set(layout.getProjectDirectory());
470474
task.getAllowedModules().set(spec.getAllowedModules());
471475
task.getAllowedResources().set(spec.getAllowedResources());
472476
task.getEnvironmentVariables().set(spec.getEnvironmentVariables());
@@ -482,15 +486,20 @@ private <T extends BasePklTask, S extends BasePklSpec> void configureBaseTask(
482486
task.getHttpProxy().set(spec.getHttpProxy());
483487
task.getHttpNoProxy().set(spec.getHttpNoProxy());
484488
task.getHttpRewrites().set(spec.getHttpRewrites());
489+
task.getExternalModuleReaders()
490+
.set(providers.provider(() -> spec.getExternalModuleReaders().getAsMap()));
491+
task.getExternalResourceReaders()
492+
.set(providers.provider(() -> spec.getExternalResourceReaders().getAsMap()));
485493
}
486494

487495
private <T extends ModulesTask, S extends ModulesSpec> void configureModulesTask(
488-
Project project,
496+
ProjectLayout layout,
497+
ProviderFactory providers,
489498
T task,
490499
S spec,
491500
@Nullable TaskProvider<AnalyzeImportsTask> analyzeImportsTask,
492501
@Nullable Transformer<List<?>, List<?>> mapSourceModules) {
493-
configureBaseTask(project, task, spec);
502+
configureBaseTask(layout, providers, task, spec);
494503
if (mapSourceModules != null) {
495504
task.getSourceModules().set(spec.getSourceModules().map(mapSourceModules));
496505
} else {
@@ -513,29 +522,21 @@ private <T extends ModulesTask, S extends ModulesSpec> void configureModulesTask
513522
}
514523
}
515524

516-
private <T extends ModulesTask, S extends ModulesSpec> void configureModulesTask(
517-
Project project,
518-
T task,
519-
S spec,
520-
@Nullable TaskProvider<AnalyzeImportsTask> analyzeImportsTask) {
521-
configureModulesTask(project, task, spec, analyzeImportsTask, null);
522-
}
523-
524-
private TaskProvider<AnalyzeImportsTask> createAnalyzeImportsTask(
525+
private TaskProvider<AnalyzeImportsTask> createGatherImportsTask(
525526
Project project, ModulesSpec spec) {
527+
var layout = project.getLayout();
526528
var outputFile =
527-
project
528-
.getLayout()
529-
.getBuildDirectory()
530-
.file("pkl-gradle/imports/" + spec.getName() + ".json");
529+
layout.getBuildDirectory().file("pkl-gradle/imports/" + spec.getName() + ".json");
530+
var providers = project.getProviders();
531531
return project
532532
.getTasks()
533533
.register(
534534
spec.getName() + "GatherImports",
535535
AnalyzeImportsTask.class,
536536
task -> {
537537
configureModulesTask(
538-
project,
538+
layout,
539+
providers,
539540
task,
540541
spec,
541542
null,
@@ -550,7 +551,10 @@ private TaskProvider<AnalyzeImportsTask> createAnalyzeImportsTask(
550551
(it) ->
551552
it.getScheme() == null || it.getScheme().equalsIgnoreCase("file"))
552553
.toList());
553-
task.setDescription("Compute the set of imports declared by input modules");
554+
task.setDescription(
555+
"Compute the set of imports declared by input modules of "
556+
+ spec.getName()
557+
+ " Pkl operation");
554558
task.setGroup("build");
555559
task.getOutputFormat().set(OutputFormat.JSON.toString());
556560
task.getOutputFile().set(outputFile);
@@ -570,20 +574,25 @@ private TaskProvider<AnalyzeImportsTask> createAnalyzeImportsTask(
570574
*/
571575
private <T extends ModulesTask> TaskProvider<T> createModulesTask(
572576
Project project, Class<T> taskClass, ModulesSpec spec) {
573-
var analyzeImportsTask = createAnalyzeImportsTask(project, spec);
577+
var gatherImportsTask = createGatherImportsTask(project, spec);
578+
var layout = project.getLayout();
579+
var providers = project.getProviders();
574580
return project
575581
.getTasks()
576582
.register(
577583
spec.getName(),
578584
taskClass,
579-
task -> configureModulesTask(project, task, spec, analyzeImportsTask));
585+
task -> configureModulesTask(layout, providers, task, spec, gatherImportsTask, null));
580586
}
581587

582588
private <T extends BasePklTask> TaskProvider<T> createTask(
583589
Project project, Class<T> taskClass, BasePklSpec spec) {
590+
var layout = project.getLayout();
591+
var providers = project.getProviders();
584592
return project
585593
.getTasks()
586-
.register(spec.getName(), taskClass, task -> configureBaseTask(project, task, spec));
594+
.register(
595+
spec.getName(), taskClass, task -> configureBaseTask(layout, providers, task, spec));
587596
}
588597

589598
private <T> Set<T> append(Set<? extends T> set1, T element) {

‎pkl-gradle/src/main/java/org/pkl/gradle/spec/BasePklSpec.java‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
2+
* Copyright © 2024-2026 Apple Inc. and the Pkl project authors. All rights reserved.
33
*
44
* Licensed under the Apache License, Version 2.0 (the "License");
55
* you may not use this file except in compliance with the License.
@@ -17,6 +17,7 @@
1717

1818
import java.net.URI;
1919
import java.time.Duration;
20+
import org.gradle.api.NamedDomainObjectContainer;
2021
import org.gradle.api.file.ConfigurableFileCollection;
2122
import org.gradle.api.file.DirectoryProperty;
2223
import org.gradle.api.provider.ListProperty;
@@ -59,4 +60,8 @@ public interface BasePklSpec {
5960
ListProperty<String> getHttpNoProxy();
6061

6162
MapProperty<URI, URI> getHttpRewrites();
63+
64+
NamedDomainObjectContainer<ExternalReaderSpec> getExternalModuleReaders();
65+
66+
NamedDomainObjectContainer<ExternalReaderSpec> getExternalResourceReaders();
6267
}

‎pkl-gradle/src/main/java/org/pkl/gradle/spec/ExternalReaderSpec.java‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
/*
2+
* Copyright © 2026 Apple Inc. and the Pkl project authors. All rights reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* https://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
package org.pkl.gradle.spec;
17+
18+
import java.util.Collection;
19+
import java.util.Map;
20+
import java.util.stream.Collectors;
21+
import javax.inject.Inject;
22+
import org.gradle.api.Named;
23+
import org.gradle.api.provider.ListProperty;
24+
import org.gradle.api.provider.Property;
25+
import org.gradle.api.tasks.Input;
26+
import org.pkl.core.evaluatorSettings.PklEvaluatorSettings.ExternalReader;
27+
28+
public abstract class ExternalReaderSpec implements Named {
29+
private final String name;
30+
31+
@Inject
32+
public ExternalReaderSpec(String name) {
33+
this.name = name;
34+
}
35+
36+
@Override
37+
@Input
38+
public String getName() {
39+
return name;
40+
}
41+
42+
@Input
43+
public abstract Property<String> getExecutable();
44+
45+
@Input
46+
public abstract ListProperty<String> getArguments();
47+
48+
public ExternalReader toExternalReader() {
49+
return new ExternalReader(getExecutable().get(), getArguments().get());
50+
}
51+
52+
public static Map<String, ExternalReader> toExternalReaderMap(
53+
Collection<? extends ExternalReaderSpec> externalReaderSpecs) {
54+
return externalReaderSpecs.stream()
55+
.collect(
56+
Collectors.toMap(ExternalReaderSpec::getName, ExternalReaderSpec::toExternalReader));
57+
}
58+
}

‎pkl-gradle/src/main/java/org/pkl/gradle/task/AnalyzeImportsTask.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@
1515
*/
1616
package org.pkl.gradle.task;
1717

18+
import static org.pkl.gradle.utils.PluginUtils.mapAndGetOrNull;
19+
1820
import java.io.File;
1921
import org.gradle.api.file.RegularFileProperty;
2022
import org.gradle.api.provider.Property;

0 commit comments

Comments
 (0)