feat(generator): add override mechanism for java and go

STACKITSDK-226

Author: cgoetz-inovex

.github/workflows/ci.yaml

@@ -4,7 +4,7 @@ on: [pull_request, workflow_dispatch]
 
 env:
   GO_VERSION_BUILD: "1.25"
-  JAVA_VERSION: "11"
+  JAVA_VERSION: "25"
 
 jobs:
   main-go:

scripts/generate-sdk/languages/go.sh

@@ -109,9 +109,8 @@ generate_go_sdk() {
 
     # compile custom generator
     cd ${ROOT_DIR}
-    mkdir -p custom/cloud/stackit/codegen
-    javac -cp "${GENERATOR_JAR_PATH}" CustomRegionGenerator.java
-    mv CustomRegionGenerator.class custom/cloud/stackit/codegen/CustomRegionGenerator.class
+    mkdir -p custom
+    javac -cp "${GENERATOR_JAR_PATH}" -d custom $(find ./scripts/generators -type f -name '*.java' | grep -v overrides)
 
     warning=""
 
@@ -167,9 +166,9 @@ generate_go_sdk() {
             cp "${ROOT_DIR}/languages/golang/.openapi-generator-ignore" "${SERVICES_FOLDER}/${service}/${version}api/.openapi-generator-ignore"
 
             # Run the generator for Go
-            java -Dlog.level=${GENERATOR_LOG_LEVEL} -cp "custom:scripts/bin/openapi-generator-cli.jar" \
+            java -Dlog.level=${GENERATOR_LOG_LEVEL} -cp "custom:scripts/bin/openapi-generator-cli.jar:scripts/generators" \
                 org.openapitools.codegen.OpenAPIGenerator generate \
-                -g cloud.stackit.codegen.CustomRegionGenerator \
+                -g GoGenerator \
                 --input-spec "${service_version_json}" \
                 --output "${SERVICES_FOLDER}/${service}/${version}api" \
                 --package-name "${version}api" \

scripts/generate-sdk/languages/java.sh

@@ -75,6 +75,11 @@ generate_java_sdk() {
     rm -rf "${SERVICES_FOLDER}"
     mkdir -p "${SERVICES_FOLDER}"
 
+    # compile custom generator
+    cd ${ROOT_DIR}
+    mkdir -p custom
+    javac -cp "${GENERATOR_JAR_PATH}" -d custom $(find ./scripts/generators -type f -name '*.java' | grep -v overrides)
+
     warning=""
 
     # Generate SDK for each service
@@ -129,13 +134,15 @@ generate_java_sdk() {
             SERVICE_DESCRIPTION=$(cat "${service_version_json}" | jq .info.title --raw-output)
 
             # Run the generator
-            java -Dlog.level="${GENERATOR_LOG_LEVEL}" -jar "${GENERATOR_JAR_PATH}" generate \
-                --generator-name java \
+            java  -Dlog.level="${GENERATOR_LOG_LEVEL}" -cp "custom:${GENERATOR_JAR_PATH}:scripts/generators" \
+                org.openapitools.codegen.OpenAPIGenerator generate \
+                --generator-name JavaGenerator \
                 --input-spec "${service_version_json}" \
                 --output "${SERVICES_FOLDER}/${service}" \
                 --git-host "${GIT_HOST}" \
                 --git-user-id "${GIT_USER_ID}" \
                 --git-repo-id "${GIT_REPO_ID}" \
+                --enable-post-process-file \
                 --global-property apis,models,modelTests=false,modelDocs=false,apiDocs=false,apiTests=true,supportingFiles \
                 --additional-properties="artifactId=${service},artifactDescription=${SERVICE_DESCRIPTION},invokerPackage=cloud.stackit.sdk.${service}.${version}api,modelPackage=cloud.stackit.sdk.${service}.${version}api.model,apiPackage=cloud.stackit.sdk.${service}.${version}api.api,serviceName=${service_pascal_case}" \
                 --inline-schema-options "SKIP_SCHEMA_REUSE=true" \

CustomRegionGenerator.java scripts/generators/GoGenerator.javaCustomRegionGenerator.java renamed to scripts/generators/GoGenerator.java

@@ -1,31 +1,46 @@
-package cloud.stackit.codegen;
-
+import io.swagger.v3.oas.models.media.Schema;
+import io.swagger.v3.oas.models.parameters.Parameter;
+import org.openapitools.codegen.CodegenParameter;
 import org.openapitools.codegen.CodegenProperty;
 import org.openapitools.codegen.languages.GoClientCodegen;
+import shared.PostProcessFileReplace;
 
-import org.openapitools.codegen.CodegenParameter;
-
+import java.io.File;
 import java.util.Set;
-import io.swagger.v3.oas.models.media.Schema;
-import io.swagger.v3.oas.models.parameters.Parameter;
 
-public class CustomRegionGenerator extends GoClientCodegen {
+public class GoGenerator extends GoClientCodegen {
+    private static RegionFix regionFix = new RegionFix();
+    private static PostProcessFileReplace postProcessFileReplace = new PostProcessFileReplace("overrides/go");
 
     @Override
     public String getName() {
-        // This is the name you will pass to the -g flag
-        return "cloud.stackit.codegen.CustomRegionGenerator";
+        return "GoClientCodegen";
     }
 
-    public CustomRegionGenerator() {
+    public GoGenerator(){
         super();
-        System.out.println("=== CUSTOM GO CLIENT GENERATOR INITIALIZED ===");
+        System.out.println("=== Custom Go Generator initialized ===");
     }
 
     @Override
     public CodegenProperty fromProperty(String name, Schema p, boolean required) {
-        CodegenProperty property = super.fromProperty(name, p, required);
+        return regionFix.fromProperty(super.fromProperty(name, p, required));
+    }
 
+    @Override
+    public CodegenParameter fromParameter(Parameter parameter, Set<String> imports) {
+        return regionFix.fromParameter(super.fromParameter(parameter, imports));
+    }
+
+    @Override
+    public void postProcessFile(File file, String fileType) {
+        postProcessFileReplace.process(file);
+        super.postProcessFile(file, fileType);
+    }
+}
+
+class RegionFix {
+    CodegenProperty fromProperty(CodegenProperty property) {
         if (isRegionField(property.name)) {
             property.dataType = "string";
             property.datatypeWithEnum = "string";
@@ -41,13 +56,7 @@ public CodegenProperty fromProperty(String name, Schema p, boolean required) {
         return property;
     }
 
-    /**
-     * Intercepts operation parameters (query, path, header, body).
-     */
-    @Override
-    public CodegenParameter fromParameter(Parameter param, Set<String> imports) {
-        CodegenParameter parameter = super.fromParameter(param, imports);
-
+    CodegenParameter fromParameter(CodegenParameter parameter) {
         if (isRegionField(parameter.paramName)) {
             parameter.dataType = "string";
 
@@ -62,7 +71,7 @@ public CodegenParameter fromParameter(Parameter param, Set<String> imports) {
         return parameter;
     }
 
-    private boolean isRegionField(String name) {
+    private static boolean isRegionField(String name) {
         if (name == null) {
             return false;
         }

scripts/generators/JavaGenerator.java

@@ -0,0 +1,24 @@
+import org.openapitools.codegen.languages.JavaClientCodegen;
+import shared.PostProcessFileReplace;
+
+import java.io.File;
+
+public class JavaGenerator extends JavaClientCodegen {
+    private static PostProcessFileReplace postProcessFileReplace = new PostProcessFileReplace("overrides/java");
+
+    @Override
+    public String getName() {
+        return "JavaClientCodegen";
+    }
+
+    public JavaGenerator(){
+        super();
+        System.out.println("=== Custom Java Generator initialized ===");
+    }
+
+    @Override
+    public void postProcessFile(File file, String fileType) {
+        postProcessFileReplace.process(file);
+        super.postProcessFile(file, fileType);
+    }
+}

scripts/generators/README.md

@@ -0,0 +1,75 @@
+# Custom Generators
+
+Customized templates are painful when updating the generator. Each template has to be compared with upstream and adapted
+accordingly.
+
+We use custom generators for introducing workarounds and fixes to not further customize our templates.
+
+## Structure
+
+Custom generators live in this directory. For each language with a custom generator there should be a `<lang>Generator.java`
+file. Each fix/workaround should live in its own class and be called from the `<lang>Generator.java` file. This allows
+easy removal of such fixes, when they are no longer needed.
+
+Fixes, applicable to multiple languages, should be implemented in the `shared` package.
+
+We try to only use the JDK and no external libraries besides the `openapi-generator` itself to keep a simple build step.
+
+## How Custom Generators are executed
+
+Each language with a custom generator, needs an adapted `<lang>.sh` script. Before building each service the custom
+generator has to be built:
+
+```bash
+# compile custom generator
+cd ${ROOT_DIR}
+mkdir -p custom
+javac -cp "${GENERATOR_JAR_PATH}" -d custom $(find ./scripts/generators -type f -name '*.java' | grep -v overrides)
+```
+
+This compiles all java files into the `custom` directory. Here we also exclude java files in the `overrides` directory.
+These are resources and should not be compiled in this step.
+
+The generator call itself has to be adapted:
+
+```bash
+java  -Dlog.level="${GENERATOR_LOG_LEVEL}" -cp "custom:${GENERATOR_JAR_PATH}:scripts/generators" \
+    org.openapitools.codegen.OpenAPIGenerator generate \
+    --generator-name JavaGenerator \
+    --enable-post-process-file \
+    # more params
+```
+
+We have to:
+- extend the classpath: `custom` makes the compiled custom generator available, `scripts/genrators` the `overrides`
+  resources
+- the `generator-name` needs to be changed to the fully-qualified class name of the custom generator
+- `enable-post-process-file` has to be activated to support `overrides`
+
+## Implemented Fixes
+
+### Go: Region Param Adjustment
+
+- we use `RESOLVE_INLINE_ENUMS`
+- this creates model classes for inline enums
+- this leads to a lot of different types for region enums with the same options
+- which makes the API hard to use
+- we force a string parameter instead
+
+### Go/Java: Overrides
+
+- there are some usages of `oneOf` schemas in our API specs, that are correct schema wise, but the generated code fails
+- example: `{ "type": "cidrv4", "value": string }`, `{ "type": "cidrv6", "value": string }` both, have the same shape
+- but they use different patterns for `value`
+- some language generators do not apply the pattern when validating the `oneOf` and report a false positive violation
+- fixing this upstream would be very involved
+- we fix it by replacing after generating sources, by replacing files
+
+How to configure an override:
+- edit the `overrides.properties` file for the language in question
+- to override a single file you need 3 properties: `<name>.path`, `<name>.replacementPath` and `<name>.hash`
+- `<name>` has to be equal for all 3 properties
+- `path` is used to match the generated file, which should be replaced
+- `hash` is used to check that the generated file still has the same content as when the override was configured
+- `replacementPath` specifies the file in resources, that should be used as replacement
+- mark adjusted sections in the replacement file in resources with an `OVERRIDE:` comment