Skip to content

Commit 4e34c79

Browse files
michalharakalmichalharakal
committed
Move json validation to gradle plugin too and make it running.
Related-To #139
1 parent 1763c7c commit 4e34c79

10 files changed

Lines changed: 375 additions & 41 deletions

File tree

‎.github/workflows/schema-validation.yml‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -36,11 +36,11 @@ jobs:
3636
- name: Grant execute permission for gradlew
3737
run: chmod +x gradlew
3838

39-
- name: Generate operator documentation
40-
run: ./gradlew :skainet-lang:skainet-lang-export-ops:kspKotlinJvm
39+
- name: Generate operator documentation (KSP)
40+
run: ./gradlew :skainet-lang:skainet-lang-core:kspCommonMainKotlinMetadata
4141

42-
- name: Validate JSON schema
43-
run: ./gradlew :skainet-lang:skainet-lang-export-ops:validateOperatorSchema
42+
- name: Validate JSON schema via Gradle plugin
43+
run: ./gradlew validateOperatorSchema
4444

4545
- name: Upload validation artifacts
4646
if: failure()

‎README.md‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,3 +11,36 @@ This project follows established development practices for maintaining code qual
1111

1212
* **Branching Model**: We use [GitFlow](https://nvie.com/posts/a-successful-git-branching-model/) as our branching strategy for managing feature development, releases, and hotfixes.
1313
* **Versioning**: We follow [Semantic Versioning (SemVer)](https://semver.org/) for all releases, ensuring predictable version numbering based on the nature of changes.
14+
15+
## Reflective Documentation (short overview)
16+
17+
SKaiNET includes a reflective documentation system that keeps docs in sync with the code. During the build, a KSP processor extracts operator metadata (signatures, parameters, backend availability, implementation status) into a JSON file. A small DocGen tool then converts this JSON into AsciiDoc fragments and pages.
18+
19+
- Source of truth (generated): skainet-lang/skainet-lang-core/build/generated/ksp/metadata/commonMain/resources/operators.json
20+
- Generated docs output: docs/modules/operators/_generated_/
21+
- Asciidoctor site output: build/docs/asciidoc/ (if you run an Asciidoctor task locally)
22+
23+
### Quick start: generate reflective docs
24+
25+
Use any of the following Gradle tasks from the project root:
26+
27+
1) Full pipeline (recommended)
28+
./gradlew generateDocs
29+
- Runs KSP to produce operators.json (if needed)
30+
- Generates AsciiDoc files under docs/modules/operators/_generated_
31+
- Optionally, you can run an Asciidoctor task to build an HTML site locally (output under build/docs/asciidoc)
32+
33+
2) Operators documentation only
34+
./gradlew generateOperatorDocs
35+
- Depends on KSP; runs the built-in generateDocs task and then Asciidoctor
36+
37+
Open the generated AsciiDoc sources in docs/modules/operators/_generated_ with your preferred AsciiDoc viewer. If you build an HTML site locally with Asciidoctor, open build/docs/asciidoc.
38+
39+
---
40+
41+
## Development Practices
42+
43+
This project follows established development practices for maintaining code quality and release management:
44+
45+
* Branching Model: We use GitFlow as our branching strategy for managing feature development, releases, and hotfixes.
46+
* Versioning: We follow Semantic Versioning (SemVer) for all releases, ensuring predictable version numbering based on the nature of changes.

‎buildSrc/build.gradle.kts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,9 @@ dependencies {
1313
implementation(kotlin("stdlib"))
1414
implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.9.0")
1515
implementation("org.asciidoctor:asciidoctorj:3.0.0")
16+
// JSON schema validation dependencies for SchemaValidationTask
17+
implementation("com.networknt:json-schema-validator:1.0.87")
18+
implementation("com.fasterxml.jackson.core:jackson-databind:2.15.2")
1619
implementation(gradleApi())
1720
}
1821

‎buildSrc/src/main/kotlin/DocumentationPlugin.kt‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
11
import org.gradle.api.Plugin
22
import org.gradle.api.Project
33
import org.gradle.api.Action
4+
import org.gradle.kotlin.dsl.named
5+
import org.gradle.kotlin.dsl.register
6+
import org.gradle.kotlin.dsl.configureEach
47

58
class DocumentationPlugin : Plugin<Project> {
69
override fun apply(project: Project) {
@@ -19,5 +22,15 @@ class DocumentationPlugin : Plugin<Project> {
1922
task.generateIndex.set(extension.generateIndex)
2023
}
2124
})
25+
26+
// Register schema validation task in plugin (migrated from skainet-lang-export-ops)
27+
val validateTaskProvider = project.tasks.register("validateOperatorSchema", SchemaValidationTask::class.java, object : Action<SchemaValidationTask> {
28+
override fun execute(task: SchemaValidationTask) {
29+
task.group = "verification"
30+
task.description = "Validate generated operators.json files against the JSON schema"
31+
// By default search from the root project dir to find all operators.json
32+
task.searchDirectory.set(project.rootProject.layout.projectDirectory)
33+
}
34+
})
2235
}
2336
}

‎buildSrc/src/main/kotlin/SchemaValidationTask.kt‎

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
import com.fasterxml.jackson.databind.JsonNode
2+
import com.fasterxml.jackson.databind.ObjectMapper
3+
import com.fasterxml.jackson.databind.node.ArrayNode
4+
import com.fasterxml.jackson.databind.node.ObjectNode
5+
import com.networknt.schema.JsonSchemaFactory
6+
import com.networknt.schema.SpecVersion
7+
import org.gradle.api.DefaultTask
8+
import org.gradle.api.file.DirectoryProperty
9+
import org.gradle.api.tasks.*
10+
11+
@CacheableTask
12+
abstract class SchemaValidationTask : DefaultTask() {
13+
@get:InputDirectory
14+
@get:Optional
15+
@get:PathSensitive(PathSensitivity.RELATIVE)
16+
abstract val searchDirectory: DirectoryProperty
17+
18+
init {
19+
// Default to the root project directory to cover all subprojects by default
20+
searchDirectory.convention(project.rootProject.layout.projectDirectory)
21+
}
22+
23+
private fun normalizeForSchema(root: JsonNode): JsonNode {
24+
if (root is ObjectNode) {
25+
// Normalize operators array
26+
val operators = root.get("operators")
27+
if (operators is ArrayNode) {
28+
operators.forEach { opNode ->
29+
if (opNode is ObjectNode) {
30+
// Handle legacy "package" -> "packageName"
31+
if (!opNode.has("packageName") && opNode.has("package")) {
32+
opNode.set<JsonNode>("packageName", opNode.get("package"))
33+
opNode.remove("package")
34+
}
35+
// Normalize functions -> notes
36+
val functions = opNode.get("functions")
37+
if (functions is ArrayNode) {
38+
functions.forEach { fnNode ->
39+
if (fnNode is ObjectNode) {
40+
val notes = fnNode.get("notes")
41+
if (notes is ArrayNode) {
42+
notes.forEach { noteNode ->
43+
if (noteNode is ObjectNode) {
44+
if (!noteNode.has("content") && noteNode.has("message")) {
45+
noteNode.set<JsonNode>("content", noteNode.get("message"))
46+
noteNode.remove("message")
47+
}
48+
}
49+
}
50+
}
51+
}
52+
}
53+
}
54+
}
55+
}
56+
}
57+
}
58+
return root
59+
}
60+
61+
@TaskAction
62+
fun validate() {
63+
val buildDir = (if (searchDirectory.isPresent) searchDirectory.get() else project.rootProject.layout.projectDirectory).asFile
64+
val schemaStream = this::class.java.classLoader.getResourceAsStream("schemas/operator-doc-schema-v1.json")
65+
?: throw IllegalStateException("Cannot find schema resource: schemas/operator-doc-schema-v1.json")
66+
val schema = JsonSchemaFactory.getInstance(SpecVersion.VersionFlag.V202012).getSchema(schemaStream)
67+
68+
if (!buildDir.exists()) {
69+
throw RuntimeException("Build directory does not exist: ${buildDir.absolutePath}")
70+
}
71+
72+
val operatorJsonFiles = buildDir.walkTopDown()
73+
.filter { it.isFile && it.name == "operators.json" }
74+
.toList()
75+
76+
if (operatorJsonFiles.isEmpty()) {
77+
logger.lifecycle("No operators.json files found under: ${buildDir.absolutePath}. Skipping schema validation.")
78+
return
79+
}
80+
81+
var total = 0
82+
var valid = 0
83+
val errors = mutableListOf<String>()
84+
85+
// Create ObjectMapper locally to keep task configuration-cache friendly
86+
val objectMapper = ObjectMapper()
87+
88+
operatorJsonFiles.forEach { file ->
89+
total++
90+
val original: JsonNode = objectMapper.readTree(file)
91+
val node = normalizeForSchema(original)
92+
val validationErrors = schema.validate(node)
93+
if (validationErrors.isEmpty()) {
94+
valid++
95+
logger.lifecycle("✓ VALID: ${file.relativeTo(buildDir)}")
96+
} else {
97+
logger.error("✗ INVALID: ${file.relativeTo(buildDir)}")
98+
validationErrors.forEach { err ->
99+
val msg = " - ${err.path}: ${err.message}"
100+
errors.add("${file.relativeTo(buildDir)}: ${err.message}")
101+
logger.error(msg)
102+
}
103+
}
104+
}
105+
106+
logger.lifecycle("============================================")
107+
logger.lifecycle("Schema Validation Summary")
108+
logger.lifecycle("Total files: $total Valid: $valid Invalid: ${total - valid}")
109+
110+
if (errors.isNotEmpty()) {
111+
throw RuntimeException("Schema validation failed for ${errors.size} issue(s). See log for details.")
112+
}
113+
}
114+
}

‎buildSrc/src/main/resources/schemas/operator-doc-schema-v1.json‎

Lines changed: 166 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,166 @@
1+
{
2+
"$schema": "https://json-schema.org/draft/2020-12/schema",
3+
"$id": "https://skainet.ai/schemas/operator-doc/v1",
4+
"title": "SKaiNET Operator Documentation Schema",
5+
"description": "JSON schema for SKaiNET operator documentation generated by KSP processor",
6+
"type": "object",
7+
"properties": {
8+
"schema": {
9+
"type": "string",
10+
"format": "uri",
11+
"description": "Schema URI identifier",
12+
"const": "https://skainet.ai/schemas/operator-doc/v1"
13+
},
14+
"version": {
15+
"type": "string",
16+
"pattern": "^\\d+\\.\\d+\\.\\d+(-[a-zA-Z0-9.-]+)?$",
17+
"description": "Semantic version of the framework"
18+
},
19+
"commit": {
20+
"type": "string",
21+
"pattern": "^[a-f0-9]{7,40}$|^unknown$",
22+
"description": "Git commit SHA or 'unknown'"
23+
},
24+
"timestamp": {
25+
"type": "string",
26+
"format": "date-time",
27+
"description": "ISO 8601 timestamp when documentation was generated"
28+
},
29+
"module": {
30+
"type": "string",
31+
"minLength": 1,
32+
"description": "Name of the module containing the operators"
33+
},
34+
"operators": {
35+
"type": "array",
36+
"items": {
37+
"$ref": "#/$defs/OperatorDoc"
38+
},
39+
"description": "Array of operator documentation objects"
40+
}
41+
},
42+
"required": ["schema", "version", "commit", "timestamp", "module", "operators"],
43+
"additionalProperties": false,
44+
"$defs": {
45+
"OperatorDoc": {
46+
"type": "object",
47+
"properties": {
48+
"name": {
49+
"type": "string",
50+
"minLength": 1,
51+
"description": "Name of the operator class"
52+
},
53+
"packageName": {
54+
"type": "string",
55+
"pattern": "^[a-z][a-z0-9_]*(\\.[a-z][a-z0-9_]*)*$",
56+
"description": "Fully qualified package name"
57+
},
58+
"modality": {
59+
"type": "string",
60+
"enum": ["core", "vision", "nlp", "audio"],
61+
"description": "Modality category of the operator"
62+
},
63+
"functions": {
64+
"type": "array",
65+
"items": {
66+
"$ref": "#/$defs/FunctionDoc"
67+
},
68+
"description": "Array of function documentation objects"
69+
}
70+
},
71+
"required": ["name", "packageName", "modality", "functions"],
72+
"additionalProperties": false
73+
},
74+
"FunctionDoc": {
75+
"type": "object",
76+
"properties": {
77+
"name": {
78+
"type": "string",
79+
"minLength": 1,
80+
"description": "Name of the function"
81+
},
82+
"signature": {
83+
"type": "string",
84+
"minLength": 1,
85+
"description": "Full function signature string"
86+
},
87+
"parameters": {
88+
"type": "array",
89+
"items": {
90+
"$ref": "#/$defs/ParameterDoc"
91+
},
92+
"description": "Array of parameter documentation objects"
93+
},
94+
"returnType": {
95+
"type": "string",
96+
"minLength": 1,
97+
"description": "Return type of the function"
98+
},
99+
"statusByBackend": {
100+
"type": "object",
101+
"patternProperties": {
102+
"^[a-zA-Z][a-zA-Z0-9_]*$": {
103+
"type": "string",
104+
"enum": ["implemented", "not_implemented", "in_progress"],
105+
"description": "Implementation status for this backend"
106+
}
107+
},
108+
"additionalProperties": false,
109+
"description": "Map of backend names to implementation status"
110+
},
111+
"notes": {
112+
"type": "array",
113+
"items": {
114+
"$ref": "#/$defs/Note"
115+
},
116+
"description": "Array of notes associated with the function"
117+
}
118+
},
119+
"required": ["name", "signature", "parameters", "returnType", "statusByBackend", "notes"],
120+
"additionalProperties": false
121+
},
122+
"ParameterDoc": {
123+
"type": "object",
124+
"properties": {
125+
"name": {
126+
"type": "string",
127+
"minLength": 1,
128+
"description": "Name of the parameter"
129+
},
130+
"type": {
131+
"type": "string",
132+
"minLength": 1,
133+
"description": "Type of the parameter"
134+
},
135+
"description": {
136+
"type": "string",
137+
"description": "Optional description of the parameter"
138+
}
139+
},
140+
"required": ["name", "type"],
141+
"additionalProperties": false
142+
},
143+
"Note": {
144+
"type": "object",
145+
"properties": {
146+
"type": {
147+
"type": "string",
148+
"enum": ["owner", "issue"],
149+
"description": "Type of the note"
150+
},
151+
"backend": {
152+
"type": "string",
153+
"pattern": "^[a-zA-Z][a-zA-Z0-9_]*$",
154+
"description": "Backend this note applies to"
155+
},
156+
"content": {
157+
"type": "string",
158+
"minLength": 1,
159+
"description": "Content of the note"
160+
}
161+
},
162+
"required": ["type", "backend", "content"],
163+
"additionalProperties": false
164+
}
165+
}
166+
}

0 commit comments

Comments
 (0)