feat(generator): rework custom generators, support java overrides

Author: cgoetz-inovex

CustomRegionGenerator.java

@@ -109,10 +109,8 @@ generate_go_sdk() {
 
     # compile custom generator
     cd ${ROOT_DIR}
-    mkdir -p custom/cloud/stackit/codegen
-    javac -cp "${GENERATOR_JAR_PATH}" CustomRegionGenerator.java
-    mv ./*.class custom/cloud/stackit/codegen
-    cp languages/golang/overrides/* custom/cloud/stackit/codegen
+    mkdir -p custom
+    javac -cp "${GENERATOR_JAR_PATH}" -d custom $(find ./scripts/generators -type f -name '*.java' | grep -v overrides)
 
     warning=""
 
@@ -168,9 +166,9 @@ generate_go_sdk() {
             cp "${ROOT_DIR}/languages/golang/.openapi-generator-ignore" "${SERVICES_FOLDER}/${service}/${version}api/.openapi-generator-ignore"
 
             # Run the generator for Go
-            java -Dlog.level=${GENERATOR_LOG_LEVEL} -cp "custom:scripts/bin/openapi-generator-cli.jar" \
+            java -Dlog.level=${GENERATOR_LOG_LEVEL} -cp "custom:scripts/bin/openapi-generator-cli.jar:scripts/generators" \
                 org.openapitools.codegen.OpenAPIGenerator generate \
-                -g cloud.stackit.codegen.CustomRegionGenerator \
+                -g GoGenerator \
                 --input-spec "${service_version_json}" \
                 --output "${SERVICES_FOLDER}/${service}/${version}api" \
                 --package-name "${version}api" \

scripts/generate-sdk/languages/go.sh

@@ -75,6 +75,11 @@ generate_java_sdk() {
     rm -rf "${SERVICES_FOLDER}"
     mkdir -p "${SERVICES_FOLDER}"
 
+    # compile custom generator
+    cd ${ROOT_DIR}
+    mkdir -p custom
+    javac -cp "${GENERATOR_JAR_PATH}" -d custom $(find ./scripts/generators -type f -name '*.java' | grep -v overrides)
+
     warning=""
 
     # Generate SDK for each service
@@ -129,13 +134,15 @@ generate_java_sdk() {
             SERVICE_DESCRIPTION=$(cat "${service_version_json}" | jq .info.title --raw-output)
 
             # Run the generator
-            java -Dlog.level="${GENERATOR_LOG_LEVEL}" -jar "${GENERATOR_JAR_PATH}" generate \
-                --generator-name java \
+            java  -Dlog.level="${GENERATOR_LOG_LEVEL}" -cp "custom:${GENERATOR_JAR_PATH}:scripts/generators" \
+                org.openapitools.codegen.OpenAPIGenerator generate \
+                --generator-name JavaGenerator \
                 --input-spec "${service_version_json}" \
                 --output "${SERVICES_FOLDER}/${service}" \
                 --git-host "${GIT_HOST}" \
                 --git-user-id "${GIT_USER_ID}" \
                 --git-repo-id "${GIT_REPO_ID}" \
+                --enable-post-process-file \
                 --global-property apis,models,modelTests=false,modelDocs=false,apiDocs=false,apiTests=true,supportingFiles \
                 --additional-properties="artifactId=${service},artifactDescription=${SERVICE_DESCRIPTION},invokerPackage=cloud.stackit.sdk.${service}.${version}api,modelPackage=cloud.stackit.sdk.${service}.${version}api.model,apiPackage=cloud.stackit.sdk.${service}.${version}api.api,serviceName=${service_pascal_case}" \
                 --inline-schema-options "SKIP_SCHEMA_REUSE=true" \

scripts/generate-sdk/languages/java.sh

@@ -0,0 +1,80 @@
+import io.swagger.v3.oas.models.media.Schema;
+import io.swagger.v3.oas.models.parameters.Parameter;
+import org.openapitools.codegen.CodegenParameter;
+import org.openapitools.codegen.CodegenProperty;
+import org.openapitools.codegen.languages.GoClientCodegen;
+import shared.PostProcessFileReplace;
+
+import java.io.File;
+import java.util.Set;
+
+public class GoGenerator extends GoClientCodegen {
+    private static RegionFix regionFix = new RegionFix();
+    private static PostProcessFileReplace postProcessFileReplace = new PostProcessFileReplace("overrides/go");
+
+    @Override
+    public String getName() {
+        return "GoClientCodegen";
+    }
+
+    public GoGenerator(){
+        super();
+        System.out.println("=== Custom Go Generator initialized ===");
+    }
+
+    @Override
+    public CodegenProperty fromProperty(String name, Schema p, boolean required) {
+        return regionFix.fromProperty(super.fromProperty(name, p, required));
+    }
+
+    @Override
+    public CodegenParameter fromParameter(Parameter parameter, Set<String> imports) {
+        return regionFix.fromParameter(super.fromParameter(parameter, imports));
+    }
+
+    @Override
+    public void postProcessFile(File file, String fileType) {
+        postProcessFileReplace.process(file);
+        super.postProcessFile(file, fileType);
+    }
+}
+
+class RegionFix {
+    CodegenProperty fromProperty(CodegenProperty property) {
+        if (isRegionField(property.name)) {
+            property.dataType = "string";
+            property.datatypeWithEnum = "string";
+            property.baseType = "string";
+
+            // Force template engine to treat this as a string
+            property.isString = true;
+            property.isInteger = false;
+            property.isLong = false;
+            property.isNumber = false;
+            property.isNumeric = false;
+        }
+        return property;
+    }
+
+    CodegenParameter fromParameter(CodegenParameter parameter) {
+        if (isRegionField(parameter.paramName)) {
+            parameter.dataType = "string";
+
+            // Force template engine to treat this as a string
+            parameter.isString = true;
+            parameter.isInteger = false;
+            parameter.isLong = false;
+            parameter.isNumber = false;
+            // If it was previously an enum or another complex type, clear it
+            parameter.isEnum = false;
+        }
+        return parameter;
+    }
+
+    private static boolean isRegionField(String name) {
+        if (name == null) {
+            return false;
+        }
+        return name.equalsIgnoreCase("region") || name.equalsIgnoreCase("regionId");
+    }
+}

scripts/generators/GoGenerator.java

@@ -0,0 +1,24 @@
+import org.openapitools.codegen.languages.JavaClientCodegen;
+import shared.PostProcessFileReplace;
+
+import java.io.File;
+
+public class JavaGenerator extends JavaClientCodegen {
+    private static PostProcessFileReplace postProcessFileReplace = new PostProcessFileReplace("overrides/java");
+
+    @Override
+    public String getName() {
+        return "JavaClientCodegen";
+    }
+
+    public JavaGenerator(){
+        super();
+        System.out.println("=== Custom Java Generator initialized ===");
+    }
+
+    @Override
+    public void postProcessFile(File file, String fileType) {
+        postProcessFileReplace.process(file);
+        super.postProcessFile(file, fileType);
+    }
+}

scripts/generators/JavaGenerator.java

@@ -0,0 +1,74 @@
+# Custom Generators
+
+Customized templates are painful when updating the generator. Each template has to be compared with upstream and adapted
+accordingly.
+
+We use custom generators for introducing workarounds and fixes to not further customize our templates.
+
+## Structure
+
+Custom generators live in this directory. For each language with a custom generator there should be a `<lang>Generator.java`
+file. Each fix/workaround should live in its own class and be called from the `<lang>Generator.java` file. This allows
+easy removal of such fixes, when they are no longer needed.
+
+Fixes, applicable to multiple languages, should be implemented in the `shared` package.
+
+We try to only use the JDK and no external libraries besides the `openapi-generator` itself to keep a simple build step.
+
+## How Custom Generators are executed
+
+Each language with a custom generator, needs an adapted `<lang>.sh` script. Before building each service the custom
+generator has to be built:
+
+```bash
+# compile custom generator
+cd ${ROOT_DIR}
+mkdir -p custom
+javac -cp "${GENERATOR_JAR_PATH}" -d custom $(find ./scripts/generators -type f -name '*.java' | grep -v overrides)
+```
+
+This compiles all java files into the `custom` directory. Here we also exclude java files in the `overrides` directory.
+These are resources and should not be compiled in this step.
+
+The generator call itself has to be adapted:
+
+```bash
+java  -Dlog.level="${GENERATOR_LOG_LEVEL}" -cp "custom:${GENERATOR_JAR_PATH}:scripts/generators" \
+    org.openapitools.codegen.OpenAPIGenerator generate \
+    --generator-name JavaGenerator \
+    --enable-post-process-file \
+    # more params
+```
+
+We have to:
+- extend the classpath: `custom` makes the compiled custom generator available, `scripts/genrators` the `overrides`
+  resources
+- the `generator-name` needs to be changed to the fully-qualified class name of the custom generator
+- `enable-post-process-file` has to be activated to support `overrides`
+
+## Implemented Fixes
+
+### Go: Region Param Adjustment
+
+- we use `RESOLVE_INLINE_ENUMS`
+- this creates model classes for inline enums
+- this leads to a lot of different types for region enums with the same options
+- which makes the API hard to use
+- we force a string parameter instead
+
+### Go/Java: Overrides
+
+- there are some usages of `oneOf` schemas in our API specs, that are correct schema wise, but the generated code fails
+- example: `{ "type": "cidrv4", "value": string }`, `{ "type": "cidrv6", "value": string }` both, have the same shape
+- but they use different patterns for `value`
+- some language generators do not apply the pattern when validating the `oneOf` and report a false positive violation
+- fixing this upstream would be very involved
+- we fix it by replacing after generating sources, by replacing files
+
+How to configure an override:
+- edit the `overrides.properties` file for the language in question
+- to override a single file you need 3 properties: `<name>.path`, `<name>.replacementPath` and `<name>.hash`
+- `<name>` has to be equal for all 3 properties
+- `path` is used to match the generated file, which should be replaced
+- `hash` is used to check that the generated file still has the same content as when the override was configured
+- `replacementPaht` specifies the file in resources, that should be used as replacemen