Skip to content

Commit 0f3483f

Browse files
eikekeikek
authored
Merge pull request #3274 from tiborrr/add-env-vars-to-docker-addons
Add env vars to docker addons
2 parents 098a503 + 7d27ee8 commit 0f3483f

12 files changed

Lines changed: 369 additions & 24 deletions

File tree

‎build.sbt‎

Lines changed: 16 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -712,6 +712,19 @@ val joexapi = project
712712
)
713713
.dependsOn(common, loggingScribe, addonlib)
714714

715+
val config = project
716+
.in(file("modules/config"))
717+
.disablePlugins(RevolverPlugin)
718+
.settings(sharedSettings)
719+
.withTestSettings
720+
.settings(
721+
name := "docspell-config",
722+
libraryDependencies ++=
723+
Dependencies.fs2 ++
724+
Dependencies.pureconfig
725+
)
726+
.dependsOn(common, loggingApi, ftspsql, store, addonlib)
727+
715728
val backend = project
716729
.in(file("modules/backend"))
717730
.disablePlugins(RevolverPlugin)
@@ -723,10 +736,12 @@ val backend = project
723736
Dependencies.fs2 ++
724737
Dependencies.bcrypt ++
725738
Dependencies.http4sClient ++
726-
Dependencies.emil
739+
Dependencies.emil ++
740+
Dependencies.pureconfig
727741
)
728742
.dependsOn(
729743
addonlib,
744+
config,
730745
store,
731746
notificationApi,
732747
joexapi,
@@ -771,20 +786,6 @@ val webapp = project
771786
)
772787
.dependsOn(query.js)
773788

774-
// Config project shared among the two applications only
775-
val config = project
776-
.in(file("modules/config"))
777-
.disablePlugins(RevolverPlugin)
778-
.settings(sharedSettings)
779-
.withTestSettings
780-
.settings(
781-
name := "docspell-config",
782-
libraryDependencies ++=
783-
Dependencies.fs2 ++
784-
Dependencies.pureconfig
785-
)
786-
.dependsOn(common, loggingApi, ftspsql, store, addonlib)
787-
788789
// --- Application(s)
789790

790791
val joex = project

‎modules/addonlib/src/main/scala/docspell/addons/AddonExecutor.scala‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,8 @@ object AddonExecutor {
3131

3232
def apply[F[_]: Async: Files](
3333
cfg: AddonExecutorConfig,
34-
urlReader: UrlReader[F]
34+
urlReader: UrlReader[F],
35+
addonEnvResolver: String => Env = _ => Env.empty
3536
): AddonExecutor[F] =
3637
new AddonExecutor[F] with AddonLoggerExtension {
3738
val config = cfg
@@ -99,8 +100,10 @@ object AddonExecutor {
99100
.compile
100101
.drain
101102

103+
addonEnv = addonEnvResolver(ctx.meta.meta.name)
104+
mergedEnv = env.addAll(addonEnv)
102105
runner <- selectRunner(cfg, ctx.meta, ctx.addonDir)
103-
result <- runner.run(logger, env, ctx)
106+
result <- runner.run(logger, mergedEnv, ctx)
104107
} yield result
105108
}
106109

‎modules/addonlib/src/test/scala/docspell/addons/AddonExecutorTest.scala‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import cats.syntax.all._
1212
import docspell.addons.out.AddonOutput
1313
import docspell.common.UrlReader
1414
import docspell.common.bc.{BackendCommand, ItemAction}
15+
import docspell.common.exec.Env
1516
import docspell.logging.{Level, TestLoggingConfig}
1617

1718
import munit._
@@ -109,6 +110,23 @@ class AddonExecutorTest extends CatsEffectSuite with Fixtures with TestLoggingCo
109110
}
110111
}
111112

113+
tempDir.test("inject env vars from addonEnvResolver") { dir =>
114+
val addonEnvResolver: String => Env = {
115+
case "env-test-addon" => Env.of("ADDON_FOO" -> "injected-value")
116+
case _ => Env.empty
117+
}
118+
val cfg = testExecutorConfig(RunnerType.Trivial)
119+
val exec =
120+
AddonExecutor[IO](cfg, UrlReader.defaultReader, addonEnvResolver).execute(logger)
121+
val addon = AddonGenerator.generate("env-test-addon", "1.0", collectOutput = false)(
122+
"""if [ "$ADDON_FOO" = "injected-value" ]; then exit 0; else exit 1; fi"""
123+
)
124+
val result = createInputEnv(dir, addon).use(exec.run)
125+
result.map { res =>
126+
assert(res.isSuccess, clue = res)
127+
}
128+
}
129+
112130
tempDir.test("combine outputs") { dir =>
113131
val cfg = testExecutorConfig(RunnerType.Trivial).copy(failFast = false)
114132
val exec = AddonExecutor[IO](cfg, UrlReader.defaultReader).execute(logger)

‎modules/backend/src/main/scala/docspell/backend/joex/AddonConfig.scala‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
/*
2+
* Copyright 2020 Eike K. & Contributors
3+
*
4+
* SPDX-License-Identifier: AGPL-3.0-or-later
5+
*/
6+
7+
package docspell.backend.joex
8+
9+
import docspell.common.exec.Env
10+
11+
/** Per-addon configuration for custom environment variables. Matched by addon name
12+
* (AddonMeta.Meta.name).
13+
*/
14+
final case class AddonConfig(
15+
name: String,
16+
enabled: Boolean = true,
17+
envs: List[AddonEnvVar] = Nil
18+
) {
19+
20+
/** Resolve all env vars to an Env map. Only applies when enabled. */
21+
def toEnv: Env =
22+
if (!enabled) Env.empty
23+
else
24+
envs.foldLeft(Env.empty) { (acc, ev) =>
25+
ev.resolve.fold(acc)(kv => acc.add(kv._1, kv._2))
26+
}
27+
}
28+
29+
/** A single environment variable to inject, with either direct value or valueFrom. */
30+
final case class AddonEnvVar(
31+
name: String,
32+
value: Option[String] = None,
33+
valueFrom: Option[AddonEnvVarFrom] = None
34+
) {
35+
36+
/** Resolve to Some((name, value)) or None if skipped. */
37+
def resolve: Option[(String, String)] =
38+
value match {
39+
case Some(v) => Some(name -> v)
40+
case None =>
41+
valueFrom.flatMap(_.env) match {
42+
case Some(envVar) =>
43+
val fromEnv = System.getenv(envVar)
44+
if (fromEnv != null) Some(name -> fromEnv)
45+
else if (valueFrom.exists(!_.optional)) Some(name -> "")
46+
else None
47+
case None => None
48+
}
49+
}
50+
}
51+
52+
/** Kubernetes-style valueFrom: read value from another source (e.g. process env). */
53+
final case class AddonEnvVarFrom(
54+
env: Option[String] = None,
55+
optional: Boolean = true
56+
)

‎modules/backend/src/main/scala/docspell/backend/joex/AddonEnvConfig.scala‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,5 +13,6 @@ import docspell.addons.AddonExecutorConfig
1313
final case class AddonEnvConfig(
1414
workingDir: Path,
1515
cacheDir: Path,
16-
executorConfig: AddonExecutorConfig
16+
executorConfig: AddonExecutorConfig,
17+
configs: List[AddonConfig] = Nil
1718
)

‎modules/backend/src/main/scala/docspell/backend/joex/AddonOps.scala‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -170,8 +170,11 @@ object AddonOps {
170170
.ephemeralRun[F]
171171
} yield mm
172172

173-
def getExecutor(cfg: AddonExecutorConfig): F[AddonExecutor[F]] =
174-
Async[F].pure(AddonExecutor(cfg, urlReader))
173+
def getExecutor(execCfg: AddonExecutorConfig): F[AddonExecutor[F]] = {
174+
val addonEnvResolver: String => Env = name =>
175+
cfg.configs.find(_.name == name).fold(Env.empty)(_.toEnv)
176+
Async[F].pure(AddonExecutor(execCfg, urlReader, addonEnvResolver))
177+
}
175178

176179
def findAddonRefs(
177180
collective: CollectiveId,

‎modules/backend/src/test/scala/docspell/backend/joex/AddonConfigTest.scala‎

Lines changed: 188 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,188 @@
1+
/*
2+
* Copyright 2020 Eike K. & Contributors
3+
*
4+
* SPDX-License-Identifier: AGPL-3.0-or-later
5+
*/
6+
7+
package docspell.backend.joex
8+
9+
import docspell.common.exec.Env
10+
import docspell.config.Implicits._
11+
12+
import munit.FunSuite
13+
import pureconfig.ConfigSource
14+
import pureconfig.generic.auto._
15+
16+
class AddonConfigTest extends FunSuite {
17+
18+
test("parse AddonConfig from HOCON with value only") {
19+
val config = ConfigSource.string("""
20+
|name = "my-addon"
21+
|enabled = true
22+
|envs = [
23+
| { name = "FOO", value = "bar" }
24+
|]
25+
|""".stripMargin)
26+
val result = config.at("").load[AddonConfig]
27+
assert(result.isRight, clue = result.left.map(_.toString))
28+
val cfg = result.toOption.get
29+
assertEquals(cfg.name, "my-addon")
30+
assertEquals(cfg.enabled, true)
31+
assertEquals(cfg.envs.size, 1)
32+
assertEquals(cfg.envs.head.name, "FOO")
33+
assertEquals(cfg.envs.head.value, Some("bar"))
34+
assertEquals(cfg.envs.head.valueFrom, None)
35+
}
36+
37+
test("parse AddonConfig from HOCON with valueFrom only") {
38+
val config =
39+
ConfigSource.string("""
40+
|name = "my-addon"
41+
|enabled = true
42+
|envs = [
43+
| {
44+
| name = "SECRET"
45+
| value-from = { env = "DS_SECRET", optional = true }
46+
| }
47+
|]
48+
|""".stripMargin)
49+
val result = config.at("").load[AddonConfig]
50+
assert(result.isRight, clue = result.left.map(_.toString))
51+
val cfg = result.toOption.get
52+
assertEquals(cfg.envs.size, 1)
53+
assertEquals(cfg.envs.head.name, "SECRET")
54+
assertEquals(cfg.envs.head.value, None)
55+
assertEquals(
56+
cfg.envs.head.valueFrom,
57+
Some(AddonEnvVarFrom(env = Some("DS_SECRET"), optional = true))
58+
)
59+
}
60+
61+
test("parse AddonConfig with both value and valueFrom") {
62+
val config = ConfigSource.string(
63+
"""
64+
|name = "my-addon"
65+
|envs = [
66+
| {
67+
| name = "MIXED"
68+
| value = "direct"
69+
| value-from = { env = "DS_MIXED", optional = false }
70+
| }
71+
|]
72+
|""".stripMargin
73+
)
74+
val result = config.at("").load[AddonConfig]
75+
assert(result.isRight, clue = result.left.map(_.toString))
76+
val cfg = result.toOption.get
77+
assertEquals(cfg.envs.head.value, Some("direct"))
78+
assertEquals(
79+
cfg.envs.head.valueFrom,
80+
Some(AddonEnvVarFrom(env = Some("DS_MIXED"), optional = false))
81+
)
82+
}
83+
84+
test("parse AddonEnvConfig with empty addonConfigs") {
85+
val config = ConfigSource.string(
86+
"""
87+
|working-dir = "/tmp/work"
88+
|cache-dir = "/tmp/cache"
89+
|executor-config {
90+
| runner = "trivial"
91+
| run-timeout = "5 minutes"
92+
| fail-fast = true
93+
| nspawn = { enabled = false, sudo-binary = "sudo", nspawn-binary = "nspawn", container-wait = "100 millis" }
94+
| nix-runner = { nix-binary = "nix", build-timeout = "5 minutes" }
95+
| docker-runner = { docker-binary = "docker", build-timeout = "5 minutes" }
96+
|}
97+
|""".stripMargin
98+
)
99+
val result = config.at("").load[AddonEnvConfig]
100+
assert(result.isRight, clue = result.left.map(_.toString))
101+
val cfg = result.toOption.get
102+
assertEquals(cfg.configs, Nil)
103+
}
104+
105+
test("parse AddonEnvConfig with non-empty addonConfigs") {
106+
val config = ConfigSource.string(
107+
"""
108+
|working-dir = "/tmp/work"
109+
|cache-dir = "/tmp/cache"
110+
|executor-config {
111+
| runner = "trivial"
112+
| run-timeout = "5 minutes"
113+
| fail-fast = true
114+
| nspawn = { enabled = false, sudo-binary = "sudo", nspawn-binary = "nspawn", container-wait = "100 millis" }
115+
| nix-runner = { nix-binary = "nix", build-timeout = "5 minutes" }
116+
| docker-runner = { docker-binary = "docker", build-timeout = "5 minutes" }
117+
|}
118+
|configs = [
119+
| {
120+
| name = "postgres-addon"
121+
| enabled = true
122+
| envs = [
123+
| { name = "PG_HOST", value = "localhost" }
124+
| ]
125+
| }
126+
|]
127+
|""".stripMargin
128+
)
129+
val result = config.at("").load[AddonEnvConfig]
130+
assert(result.isRight, clue = result.left.map(_.toString))
131+
val cfg = result.toOption.get
132+
assertEquals(cfg.configs.size, 1)
133+
assertEquals(cfg.configs.head.name, "postgres-addon")
134+
assertEquals(cfg.configs.head.envs.head.name, "PG_HOST")
135+
assertEquals(cfg.configs.head.envs.head.value, Some("localhost"))
136+
}
137+
138+
test("AddonEnvVar.resolve with value") {
139+
val ev = AddonEnvVar(name = "FOO", value = Some("bar"))
140+
assertEquals(ev.resolve, Some("FOO" -> "bar"))
141+
}
142+
143+
test("AddonEnvVar.resolve with valueFrom, optional=true, env unset") {
144+
val ev = AddonEnvVar(
145+
name = "SECRET",
146+
valueFrom = Some(
147+
AddonEnvVarFrom(env = Some("DOCSPELL_ADDON_TEST_UNLIKELY_12345"), optional = true)
148+
)
149+
)
150+
assertEquals(ev.resolve, None)
151+
}
152+
153+
test("AddonEnvVar.resolve with valueFrom, optional=false, env unset") {
154+
val ev = AddonEnvVar(
155+
name = "REQUIRED",
156+
valueFrom = Some(
157+
AddonEnvVarFrom(
158+
env = Some("DOCSPELL_ADDON_TEST_UNLIKELY_67890"),
159+
optional = false
160+
)
161+
)
162+
)
163+
assertEquals(ev.resolve, Some("REQUIRED" -> ""))
164+
}
165+
166+
test("AddonConfig.toEnv when disabled") {
167+
val cfg = AddonConfig(
168+
name = "x",
169+
enabled = false,
170+
envs = List(AddonEnvVar("A", value = Some("a")))
171+
)
172+
assertEquals(cfg.toEnv, Env.empty)
173+
}
174+
175+
test("AddonConfig.toEnv when enabled") {
176+
val cfg = AddonConfig(
177+
name = "x",
178+
enabled = true,
179+
envs = List(
180+
AddonEnvVar("A", value = Some("a")),
181+
AddonEnvVar("B", value = Some("b"))
182+
)
183+
)
184+
val env = cfg.toEnv
185+
assertEquals(env.values.get("A"), Some("a"))
186+
assertEquals(env.values.get("B"), Some("b"))
187+
}
188+
}

‎modules/config/src/test/scala/docspell/config/EnvConfigTest.scala‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,4 +42,22 @@ class EnvConfigTest extends FunSuite {
4242
val cfg = EnvConfig.loadFrom(Map("A_B_C" -> "12").view)
4343
assert(!cfg.hasPath("a.b.c"))
4444
}
45+
46+
test("override addons.configs via env vars") {
47+
val cfg = EnvConfig.loadFrom(
48+
Map(
49+
"DOCSPELL_JOEX_ADDONS_CONFIGS_0_NAME" -> "postgres-addon",
50+
"DOCSPELL_JOEX_ADDONS_CONFIGS_0_ENABLED" -> "true",
51+
"DOCSPELL_JOEX_ADDONS_CONFIGS_0_ENVS_0_NAME" -> "PG_HOST",
52+
"DOCSPELL_JOEX_ADDONS_CONFIGS_0_ENVS_0_VALUE" -> "localhost"
53+
).view
54+
)
55+
assertEquals(cfg.getString("docspell.joex.addons.configs.0.name"), "postgres-addon")
56+
assertEquals(cfg.getBoolean("docspell.joex.addons.configs.0.enabled"), true)
57+
assertEquals(cfg.getString("docspell.joex.addons.configs.0.envs.0.name"), "PG_HOST")
58+
assertEquals(
59+
cfg.getString("docspell.joex.addons.configs.0.envs.0.value"),
60+
"localhost"
61+
)
62+
}
4563
}

0 commit comments

Comments
 (0)