Skip to content

Commit 981489d

Browse files
dacrdacr
committed
Support server script testing through test-with directive
1 parent 7e7a5fc commit 981489d

4 files changed

Lines changed: 103 additions & 50 deletions

File tree

‎README.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -122,6 +122,10 @@ Supported keys in description header are the following :
122122
- Only examples with `@testable` keywords are eligible for automated execution
123123
- on execution the exit code is used to compute execution success or failure
124124
- Use `$file` (or `$scriptFile`) for example filename substitution
125+
- **`test-with`** : Command to test the example
126+
- When `@testable` is set as keyword
127+
- When your example is a "blocking service", you can specify an external command to test it
128+
- for example : `test-with : curl http://127.0.0.1:8080/docs`
125129

126130
## CEM operations
127131

‎src/main/scala/fr/janalyse/cem/Execute.scala‎

Lines changed: 96 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -5,66 +5,108 @@ import zio.stream.*
55
import zio.stream.ZPipeline.{splitLines, utf8Decode}
66
import zio.process.*
77
import zio.json.*
8+
import zio.ZIOAspect.*
89

910
import java.util.concurrent.TimeUnit
1011
import java.time.OffsetDateTime
1112
import java.util.UUID
1213
import fr.janalyse.cem.model.{CodeExample, RunStatus, ExampleIssue}
1314
import zio.nio.file.Path
1415

16+
case class RunFailure(
17+
message: String
18+
)
19+
case class RunResults(
20+
command: List[String],
21+
exitCode: Int,
22+
output: String
23+
)
24+
1525
object Execute {
26+
val timeoutDuration = Duration(100, TimeUnit.SECONDS)
27+
val testStartDelay = Duration(500, TimeUnit.MILLISECONDS)
1628

17-
def runExample(example: CodeExample, runSessionDate: OffsetDateTime, runSessionUUID: UUID) = {
18-
val timeoutDuration = Duration(100, TimeUnit.SECONDS)
19-
val uuid = example.uuid
29+
def makeCommandProcess(command: List[String], workingDir: Path) = {
30+
val results = for {
31+
executable <- ZIO.from(command.headOption).orElseFail(RunFailure(s"Example command is invalid"))
32+
arguments = command.drop(1)
33+
process <- ZIO.acquireRelease(
34+
Command(executable, arguments*)
35+
.redirectErrorStream(true)
36+
.workingDirectory(workingDir.toFile)
37+
.run
38+
.mapError(err => RunFailure(s"Command error ${err.toString}"))
39+
)(process => process.killTreeForcibly.tapError(err => ZIO.logError(err.toString)).ignore)
40+
stream = process.stdout.stream.via(utf8Decode >>> splitLines)
41+
mayBeOutputLines <- stream.runCollect.disconnect.timeout(timeoutDuration).mapError(err => RunFailure(s"Couldn't collect outputs\n${err.toString}"))
42+
outputText = mayBeOutputLines.map(chunks => chunks.mkString("\n")).getOrElse("")
43+
exitCode <- process.exitCode.mapError(err => RunFailure(outputText + "\n" + err.toString))
44+
} yield RunResults(command, exitCode.code, outputText)
2045

21-
val result = for {
22-
exampleFilePath <- ZIO.fromOption(example.filepath).orElseFail(Exception(s"Example $uuid has no path to its content"))
23-
absoluteFileName <- exampleFilePath.toAbsolutePath
46+
ZIO.scoped(results)
47+
}
48+
49+
def makeRunCommandProcess(example: CodeExample) = {
50+
for {
51+
exampleFilePath <- ZIO.fromOption(example.filepath).orElseFail(RunFailure(s"Example has no path for its content"))
52+
workingDir <- ZIO.fromOption(exampleFilePath.parent).orElseFail(RunFailure(s"Example file path content has no parent directory"))
53+
absoluteFileName <- exampleFilePath.toAbsolutePath.orElseFail(RunFailure(s"Example absolute file path error"))
2454
command <- ZIO
25-
.fromOption(
55+
.from(
2656
example.runWith
2757
.map(_.replaceAll("[$]scriptFile", absoluteFileName.toString))
2858
.map(_.replaceAll("[$]file", absoluteFileName.toString))
2959
.map(_.split("\\s+").toList)
3060
)
31-
.orElseFail(Exception(s"Example ${example.uuid} as no run-with directive"))
32-
// _ <- ZIO.log(s"Running $exampleFilePath")
33-
startTimestamp <- Clock.currentDateTime
34-
executable <- ZIO.fromOption(command.headOption).orElseFail(Exception(s"Example $uuid command is invalid : ${command}"))
35-
arguments = command.drop(1)
36-
workingDir <- ZIO.fromOption(exampleFilePath.parent).orElseFail(Exception(s"Example $uuid file path content has no parent"))
37-
process <- Command(executable, arguments*)
38-
.redirectErrorStream(true)
39-
.workingDirectory(workingDir.toFile)
40-
.run
41-
stream = process.stdout.stream.via(utf8Decode >>> splitLines)
42-
outputFiber <- stream.runCollect.fork
43-
exitCodeOption <- process.exitCode.timeout(timeoutDuration)
44-
_ <- process.killTree.ignore
45-
outputLines <- outputFiber.join.catchAll(err => ZIO.succeed(Chunk(err.toString)))
46-
outputText = outputLines.mkString("\n")
47-
duration <- Clock.instant.map(i => i.toEpochMilli - startTimestamp.toInstant.toEpochMilli)
48-
success = exitCodeOption.exists(_.code == 0) || (example.shouldFail && exitCodeOption.exists(_.code > 0))
49-
timeout = exitCodeOption.isEmpty
50-
runState = if timeout then "timeout" else if success then "success" else "failure"
51-
// _ <- ZIO.log(s"Execution state is $runState running ${example.filepath} in ${workingDir} using ${command.mkString(" ")}")
52-
} yield RunStatus(
53-
example = example,
54-
exitCodeOption = exitCodeOption.map(_.code),
55-
Stdout = outputText,
56-
startedTimestamp = startTimestamp,
57-
duration = duration,
58-
runSessionDate = runSessionDate,
59-
runSessionUUID = runSessionUUID,
60-
success = success,
61-
timeout = timeout,
62-
runState = runState
63-
)
61+
.orElseFail(RunFailure(s"Example ${example.uuid} as no run-with directive"))
62+
results <- makeCommandProcess(command, workingDir) @@ annotated("example-run-command" -> command.mkString(" "))
63+
} yield results
64+
}
65+
66+
def makeTestCommandProcess(example: CodeExample) = {
67+
for {
68+
exampleFilePath <- ZIO.fromOption(example.filepath).orElseFail(RunFailure(s"Example has no path for its content"))
69+
workingDir <- ZIO.fromOption(exampleFilePath.parent).orElseFail(RunFailure(s"Example file path content has no parent directory"))
70+
command <- ZIO.succeed(example.testWith.getOrElse(s"sleep ${timeoutDuration.getSeconds()}").trim.split("\\s+").toList)
71+
results <- makeCommandProcess(command, workingDir) @@ annotated("example-test-command" -> command.mkString(" "))
72+
} yield results
73+
}
74+
75+
def runExample(example: CodeExample, runSessionDate: OffsetDateTime, runSessionUUID: UUID) = {
76+
val result =
77+
for {
78+
startTimestamp <- Clock.currentDateTime
79+
runEffect = makeRunCommandProcess(example).disconnect
80+
.timeout(timeoutDuration)
81+
testEffect = makeTestCommandProcess(example)
82+
.filterOrFail(result => result.exitCode == 0)(RunFailure(s"test code is failing"))
83+
.retry(Schedule.exponential(100.millis, 2).jittered && Schedule.recurs(5))
84+
.delay(testStartDelay)
85+
.disconnect
86+
.timeout(timeoutDuration)
87+
results <- runEffect.raceFirst(testEffect).either
88+
duration <- Clock.instant.map(i => i.toEpochMilli - startTimestamp.toInstant.toEpochMilli)
89+
timeout = results.exists(_.isEmpty)
90+
output = results.toOption.flatten.map(_.output).getOrElse("")
91+
exitCodeOption = results.toOption.flatten.map(_.exitCode)
92+
success = exitCodeOption.exists(_ == 0) || (example.shouldFail && exitCodeOption.exists(_ != 0))
93+
runState = if timeout then "timeout" else if success then "success" else "failure"
94+
_ <- if (results.isLeft) ZIO.logError(s"""Couldn't execute either run or test part\n${results.swap.toOption.getOrElse("")}""") else ZIO.succeed(())
95+
_ <- if (!success) ZIO.logWarning(s"example run $runState\nFailed cause:\n$output") else ZIO.log(s"example run success")
96+
} yield RunStatus(
97+
example = example,
98+
exitCodeOption = exitCodeOption,
99+
stdout = output,
100+
startedTimestamp = startTimestamp,
101+
duration = duration,
102+
runSessionDate = runSessionDate,
103+
runSessionUUID = runSessionUUID,
104+
success = success,
105+
timeout = timeout,
106+
runState = runState
107+
)
64108

65109
result
66-
// .tap(status => Console.printLine(s"${example.filename} result state ${status.runState} "))
67-
.tapError(err => ZIO.logError(s"Example $uuid (${example.filename}) has failed with $err"))
68110
}
69111

70112
def runTestableExamples(examples: List[CodeExample]) = {
@@ -76,15 +118,20 @@ object Execute {
76118
runSessionDate <- Clock.currentDateTime
77119
startEpoch <- Clock.instant.map(_.toEpochMilli)
78120
runSessionUUID = UUID.randomUUID()
79-
runStatuses <- ZIO.foreachExec(runnableExamples)(execStrategy)(example => runExample(example, runSessionDate, runSessionUUID))
121+
// runStatuses <- ZIO.foreachExec(runnableExamples)(execStrategy)(example => runExample(example, runSessionDate, runSessionUUID))
122+
runStatuses <- ZIO.foreach(runnableExamples) { example =>
123+
runExample(example, runSessionDate, runSessionUUID) @@ annotated("example-uuid" -> example.uuid.toString, "example-filename" -> example.filename)
124+
}
80125
successes = runStatuses.filter(_.success)
81126
failures = runStatuses.filterNot(_.success)
82-
_ <- ZIO.logError(
83-
failures // runStatuses
84-
.sortBy(s => (s.success, s.example.filepath.map(_.toString)))
85-
.map(state => s"""${if (state.success) "OK" else "KO"} : ${state.example.filepath.get} : ${state.example.summary.getOrElse("")}""")
86-
.mkString("\n", "\n", "")
87-
)
127+
_ <- if (failures.size > 0)
128+
ZIO.logError(
129+
failures // runStatuses
130+
.sortBy(s => (s.success, s.example.filepath.map(_.toString)))
131+
.map(state => s"""${if (state.success) "OK" else "KO"} : ${state.example.filepath.get} : ${state.example.summary.getOrElse("")}""")
132+
.mkString("\n", "\n", "")
133+
)
134+
else ZIO.log("ALL example executed with success :)")
88135
endEpoch <- Clock.instant.map(_.toEpochMilli)
89136
durationSeconds = (endEpoch - startEpoch) / 1000
90137
_ <- ZIO.log(s"${runStatuses.size} runnable examples (with scala-cli) in ${durationSeconds}s")

‎src/main/scala/fr/janalyse/cem/model/CodeExample.scala‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@ case class CodeExample(
5454
publish: List[String] = Nil, // embedded
5555
authors: List[String] = Nil, // embedded
5656
runWith: Option[String] = None, // embedded
57+
testWith: Option[String] = None, // embedded
5758
managedBy: Option[String] = None, // embedded
5859
license: Option[String] = None, // embedded
5960
updatedCount: Option[Int] = None, // computed from GIT history
@@ -155,6 +156,7 @@ object CodeExample {
155156
publish = exampleContentExtractValueList(content, "publish"),
156157
authors = exampleContentExtractValueList(content, "authors"),
157158
runWith = exampleContentExtractValue(content, "run-with"),
159+
testWith = exampleContentExtractValue(content, "test-with"),
158160
managedBy = exampleContentExtractValue(content, "managed-by"),
159161
license = exampleContentExtractValue(content, "license"),
160162
attachments = attachments.toMap

‎src/main/scala/fr/janalyse/cem/model/RunStatus.scala‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ import java.time.OffsetDateTime
77
case class RunStatus(
88
example: CodeExample,
99
exitCodeOption: Option[Int],
10-
Stdout: String,
10+
stdout: String,
1111
startedTimestamp: OffsetDateTime,
1212
duration: Long,
1313
runSessionDate: OffsetDateTime,

0 commit comments

Comments
 (0)