Merge pull request #16 from cloudogu/feature/validator_refactor

Feature/validator refactor

Author: phaenr

README.md

@@ -682,24 +682,86 @@ node {
         deployViaGitops(gitopsConfig)
     }
 }
+```
+
+### Custom validators
+
+The library offers a convenient base class [`com.cloudogu.gitops.gitopsbuildlib.Validator`](src/com/cloudogu/gitopsbuildlib/Validator.groovy).
+However, this seems impossible to use with neither dynamic library loading via the `library()` nor with `@Library`,
+because the library is loaded after the class is evaluated.  
+If you need to use `library()` or `@Library` then your validator needs to implement the following three methods:
+
+- `void validate(boolean enabled, String targetDirectory, Map config)`
+- `SourceType[] getSupportedSourceTypes()`
+- `GitopsTool[] getSupportedGitopsTools()`
+
+```groovy
+import com.cloudogu.gitopsbuildlib.deployment.GitopsTool
+import com.cloudogu.gitopsbuildlib.deployment.SourceType
+
+class MyValidator extends Validator {
 
-// Simple example that works with dynamic library loading, i.e. the library() step
-class MyValidator {
-    def script
     MyValidator(def script) {
-        this.script = script
+        super(script)
+    }
+
+    @Override
+    void validate(boolean enabled, String targetDirectory, Map validatorConfig, Map gitopsConfig) {
+        script.echo "Enabled: $enabled; targetDirectory: $targetDirectory; validatorConfig: $validatorConfig; gitopsConfig: $gitopsConfig"
     }
 
-    void validate(boolean enabled, String targetDirectory, Map config) {
-        script.echo "Enabled: $enabled; targetDirectory: $targetDirectory; config: $config"
+    @Override
+    SourceType[] getSupportedSourceTypes() {
+        return [SourceType.HELM, SourceType.PLAIN]
+    }
+
+    @Override
+    GitopsTool[] getSupportedGitopsTools() {
+        return [GitopsTool.ARGO, GitopsTool.FLUX]
     }
 }
 ```
 
-### Custom validators
+In general a custom validator may implement the Validator class. You therefore have to implement the following methods:
 
-In general a custom validator must provide this method: `validate(boolean enabled, String targetDirectory, Map config)`
+`void validate(boolean enabled, String targetDirectory, Map config)`
+- Here lies your validation code or the entrypoint for more complex validation processes
 
-The library also offers a convenient base class [`com.cloudogu.gitops.gitopsbuildlib.Validator`](src/com/cloudogu/gitopsbuildlib/Validator.groovy).
-However, this seems impossible to use with neither dynamic library loading via the `library()` nor with `@Library`,
-because the library is loaded after the class is evaluated.
+`SourceType[] getSupportedSourceTypes()`  
+- This method returns a collection of supported Source Types.  
+The SourceType determines which resources are going to be validated.
+There are two locations where resources can be validated.   
+They are differentiated by the resource-type of which there are two right now.
+    - Helm resources
+    - Plain k8s resources
+    
+
+**Visual representation of the folder structure on the Jenkins agent**
+```
+├── jenkinsworkdir/
+   └── .configRepoTempDir/
+      └── ${stage}/
+         └── ${application}/
+            ├── extraResources/
+            ├── generatedResources/
+            ├── deployment.yaml
+            └── ...
+   └── .helmChartTempDir/
+      └── chart/
+         └── ${chartPath}/ (for git repo) or ${chartName}/ (for helm repo)
+      └── mergedValues.yaml
+```
+
+**Helm resources** - `.helmChartTempDir`:  
+This location is only temporary and is being used for the helm chart to be downloaded and the mergedValues.file (values-shared.yaml + values-${stage}.yaml)
+Only Validators which support Helm schemas should operate on this folder
+
+**Plain k8s resources** - `.configRepoTempDir`:  
+This location is for your plain k8s resources. This folder is also the gitops folder which will be pushed to the scm.
+It contains your k8s resources in the root and two extra folders for additional k8s resources:
+`extraResources`: Only needed for a Helm deployment if you whish to deploy plain k8s resources in addition to the helm deployment. See: [Important note in Namespaces](#namespaces)  
+`generatedResources`: If you have files which need to be deployed as a configMap. See: [Extra Files](#extra-files)
+
+`GitopsTool[] getSupportedGitopsTools()`  
+This determins on which GitopsTool the validator will run. We implemented this feature since Argo already uses `helm template` and `kubeval` internally so we don't need `helm kubeval` since it does exactly the same.
+So we defined `HelmKubeval` as only needed to be executed on a `FLUX` operator.

src/com/cloudogu/gitopsbuildlib/deployment/Deployment.groovy

@@ -1,5 +1,7 @@
 package com.cloudogu.gitopsbuildlib.deployment
 
+import com.cloudogu.gitopsbuildlib.docker.DockerWrapper
+
 abstract class Deployment {
 
     protected static String getKubectlImage() { 'lachlanevenson/k8s-kubectl:v1.19.3' }
@@ -10,9 +12,12 @@ abstract class Deployment {
     protected script
     protected Map gitopsConfig
 
+    protected DockerWrapper dockerWrapper
+
     Deployment(def script, def gitopsConfig) {
         this.script = script
         this.gitopsConfig = gitopsConfig
+        dockerWrapper = new DockerWrapper(this.script)
     }
 
     def create(String stage) {
@@ -25,14 +30,7 @@ abstract class Deployment {
 
     abstract preValidation(String stage)
     abstract postValidation(String stage)
-
-
-    def validate(String stage) {
-        gitopsConfig.validators.each { validatorConfig ->
-            script.echo "Executing validator ${validatorConfig.key}"
-            validatorConfig.value.validator.validate(validatorConfig.value.enabled, "${stage}/${gitopsConfig.application}", validatorConfig.value.config, gitopsConfig)
-        }
-    }
+    abstract validate(String stage)
 
     def createFoldersAndCopyK8sResources(String stage) {
         def sourcePath = gitopsConfig.deployments.sourcePath
@@ -57,7 +55,7 @@ abstract class Deployment {
 
     String createConfigMap(String key, String filePath, String name, String namespace) {
         String configMap = ""
-        withKubectl {
+        withDockerImage(kubectlImage) {
             String kubeScript = "KUBECONFIG=${writeKubeConfig()} kubectl create configmap ${name} " +
                 "--from-file=${key}=${filePath} " +
                 "--dry-run=client -o yaml -n ${namespace}"
@@ -67,12 +65,12 @@ abstract class Deployment {
         return configMap
     }
 
-    void withKubectl(Closure body) {
-        script.cesBuildLib.Docker.new(script).image(kubectlImage)
-        // Allow accessing WORKSPACE even when we are in a child dir (using "dir() {}")
-            .inside("${script.pwd().equals(script.env.WORKSPACE) ? '' : "-v ${script.env.WORKSPACE}:${script.env.WORKSPACE}"}") {
-                body()
-            }
+    void withDockerImage(String image, Closure body) {
+        dockerWrapper.withDockerImage(image, body)
+    }
+
+    void withHelm(Closure body) {
+        dockerWrapper.withHelm(body)
     }
 
     // Dummy kubeConfig, so we can use `kubectl --dry-run=client`
@@ -110,4 +108,15 @@ users:
         }
         return namespace
     }
+
+    protected GitopsTool getGitopsTool() {
+        switch (gitopsConfig.gitopsTool) {
+            case 'FLUX':
+                return GitopsTool.FLUX
+            case 'ARGO':
+                return GitopsTool.ARGO
+            default:
+                return null
+        }
+    }
 }

src/com/cloudogu/gitopsbuildlib/deployment/GitopsTool.groovy

@@ -0,0 +1,19 @@
+package com.cloudogu.gitopsbuildlib.deployment
+
+enum GitopsTool {
+    FLUX('flux'), ARGO('argo')
+
+    private final String name
+
+    GitopsTool(String name) {
+        this.name = name
+    }
+
+    String getNameValue() {
+        return name
+    }
+
+    String toString() {
+        return name() + " = " + getNameValue()
+    }
+}

src/com/cloudogu/gitopsbuildlib/deployment/SourceType.groovy

@@ -0,0 +1,19 @@
+package com.cloudogu.gitopsbuildlib.deployment
+
+enum SourceType {
+    HELM('helm'), PLAIN('plain')
+
+    private final String name
+
+    SourceType(String name) {
+        this.name = name
+    }
+
+    String getNameValue() {
+        return name
+    }
+
+    String toString() {
+        return name() + " = " + getNameValue()
+    }
+}

src/com/cloudogu/gitopsbuildlib/deployment/helm/Helm.groovy

@@ -1,18 +1,23 @@
 package com.cloudogu.gitopsbuildlib.deployment.helm
 
 import com.cloudogu.gitopsbuildlib.deployment.Deployment
+import com.cloudogu.gitopsbuildlib.deployment.GitopsTool
 import com.cloudogu.gitopsbuildlib.deployment.helm.helmrelease.ArgoCDRelease
 import com.cloudogu.gitopsbuildlib.deployment.helm.helmrelease.FluxV1Release
 import com.cloudogu.gitopsbuildlib.deployment.helm.helmrelease.HelmRelease
 import com.cloudogu.gitopsbuildlib.deployment.helm.repotype.GitRepo
 import com.cloudogu.gitopsbuildlib.deployment.helm.repotype.HelmRepo
 import com.cloudogu.gitopsbuildlib.deployment.helm.repotype.RepoType
+import com.cloudogu.gitopsbuildlib.deployment.SourceType
 
 class Helm extends Deployment {
 
     protected RepoType chartRepo
     protected HelmRelease helmRelease
 
+    private String helmChartTempDir = ".helmChartTempDir"
+    private String chartRootDir = "chart"
+
     Helm(def script, def gitopsConfig) {
         super(script, gitopsConfig)
         this.extraResourcesFolder = "extraResources"
@@ -34,24 +39,30 @@ class Helm extends Deployment {
         def application = gitopsConfig.application
         def sourcePath = gitopsConfig.deployments.sourcePath
 
+        chartRepo.prepareRepo(helmConfig, helmChartTempDir, chartRootDir)
+
         // writing the merged-values.yaml via writeYaml into a file has the advantage, that it gets formatted as valid yaml
         // This makes it easier to read in and indent for the inline use in the helmRelease.
         // It enables us to reuse the `fileToInlineYaml` function, without writing a complex formatting logic.
-        script.writeFile file: "${stage}/${application}/mergedValues.yaml", text: chartRepo.mergeValues(helmConfig, ["${script.env.WORKSPACE}/${sourcePath}/values-${stage}.yaml", "${script.env.WORKSPACE}/${sourcePath}/values-shared.yaml"] as String[])
-
-        updateYamlValue("${stage}/${application}/mergedValues.yaml", helmConfig)
+        script.writeFile file: "${script.env.WORKSPACE}/${helmChartTempDir}/mergedValues.yaml", text: mergeValuesFiles(helmConfig, ["${script.env.WORKSPACE}/${sourcePath}/values-${stage}.yaml", "${script.env.WORKSPACE}/${sourcePath}/values-shared.yaml"] as String[])
 
-        script.writeFile file: "${stage}/${application}/applicationRelease.yaml", text: helmRelease.create(helmConfig, application, getNamespace(stage), "${stage}/${application}/mergedValues.yaml")
+        updateYamlValue("${script.env.WORKSPACE}/${helmChartTempDir}/mergedValues.yaml", helmConfig)
 
-        // since the values are already inline (helmRelease.yaml) we do not need to commit them into the gitops repo
-        script.sh "rm ${stage}/${application}/mergedValues.yaml"
+        script.writeFile file: "${stage}/${application}/applicationRelease.yaml", text: helmRelease.create(helmConfig, application, getNamespace(stage), "${script.env.WORKSPACE}/${helmChartTempDir}/mergedValues.yaml")
     }
 
     @Override
     def postValidation(String stage) {
-        def helmConfig = gitopsConfig.deployments.helm
-        // clean the gitrepo helm chart folder since the helmRelease.yaml ist now created
-        script.sh "rm -rf chart || true"
+        // clean the helm chart folder since the validation on this helm chart is done
+        script.sh "rm -rf ${script.env.WORKSPACE}/${helmChartTempDir} || true"
+    }
+
+    @Override
+    def validate(String stage) {
+        gitopsConfig.validators.each { validator ->
+            validator.value.validator.validate(validator.value.enabled, getGitopsTool(), SourceType.PLAIN, "${stage}/${gitopsConfig.application}", validator.value.config, gitopsConfig)
+            validator.value.validator.validate(validator.value.enabled, getGitopsTool(), SourceType.HELM, "${script.env.WORKSPACE}/${helmChartTempDir}",validator.value.config, gitopsConfig)
+        }
     }
 
     private void updateYamlValue(String yamlFilePath, Map helmConfig) {
@@ -69,4 +80,29 @@ class Helm extends Deployment {
         }
         script.writeYaml file: yamlFilePath, data: data, overwrite: true
     }
+
+    private String mergeValuesFiles(Map helmConfig, String[] valuesFiles) {
+        String mergedValuesFile = ""
+
+        def chartDir = ''
+        if (helmConfig.containsKey('chartPath') && helmConfig.chartPath) {
+            chartDir = helmConfig.chartPath
+        } else if ( helmConfig.containsKey('chartName')) {
+            chartDir = helmConfig.chartName
+        }
+
+        withHelm {
+            String helmScript = "helm values ${script.env.WORKSPACE}/${helmChartTempDir}/${chartRootDir}/${chartDir} ${valuesFilesWithParameter(valuesFiles)}"
+            mergedValuesFile = script.sh returnStdout: true, script: helmScript
+        }
+        return mergedValuesFile
+    }
+
+    private String valuesFilesWithParameter(String[] valuesFiles) {
+        String valuesFilesWithParameter = ""
+        valuesFiles.each {
+            valuesFilesWithParameter += "-f $it "
+        }
+        return valuesFilesWithParameter
+    }
 }

src/com/cloudogu/gitopsbuildlib/deployment/helm/helmrelease/ArgoCDRelease.groovy

@@ -2,7 +2,7 @@ package com.cloudogu.gitopsbuildlib.deployment.helm.helmrelease
 
 import com.cloudogu.gitopsbuildlib.docker.DockerWrapper
 
-class ArgoCDRelease extends HelmRelease{
+class ArgoCDRelease extends HelmRelease {
 
     protected DockerWrapper dockerWrapper
 
@@ -13,7 +13,6 @@ class ArgoCDRelease extends HelmRelease{
 
     @Override
     String create(Map helmConfig, String application, String namespace, String mergedValuesFile) {
-
         String helmRelease = ""
         if (helmConfig.repoType == 'GIT') {
             helmRelease = createResourcesFromGitRepo(helmConfig, application, mergedValuesFile)
@@ -24,7 +23,6 @@ class ArgoCDRelease extends HelmRelease{
     }
 
     private String createResourcesFromGitRepo(Map helmConfig, String application, String mergedValuesFile) {
-
         def chartPath = ''
         if (helmConfig.containsKey('chartPath')) {
             chartPath = helmConfig.chartPath
@@ -40,7 +38,7 @@ class ArgoCDRelease extends HelmRelease{
     private String createHelmRelease(String chartPath, String application, String mergedValuesFile) {
         String helmRelease = ""
         dockerWrapper.withHelm {
-            String templateScript = "helm template ${application} chart/${chartPath} -f ${mergedValuesFile}"
+            String templateScript = "helm template ${application} ${script.env.WORKSPACE}/.helmChartTempDir/chart/${chartPath} -f ${mergedValuesFile}"
             helmRelease = script.sh returnStdout: true, script: templateScript
         }
 

src/com/cloudogu/gitopsbuildlib/deployment/helm/helmrelease/FluxV1Release.groovy

@@ -1,6 +1,6 @@
 package com.cloudogu.gitopsbuildlib.deployment.helm.helmrelease
 
-class FluxV1Release extends HelmRelease{
+class FluxV1Release extends HelmRelease {
 
     FluxV1Release(def script) {
         super(script)
@@ -26,7 +26,6 @@ ${values}
     }
 
     private String gitRepoChart(Map helmConfig) {
-
         def chartPath = "."
         if (helmConfig.containsKey('chartPath') && helmConfig.chartPath) {
             chartPath = helmConfig.chartPath

src/com/cloudogu/gitopsbuildlib/deployment/helm/repotype/GitRepo.groovy

@@ -7,29 +7,24 @@ class GitRepo extends RepoType {
     }
 
     @Override
-    String mergeValues(Map helmConfig, String[] valuesFiles) {
-        String merge = ""
+    void prepareRepo(Map helmConfig, String helmChartTempDir, String chartRootDir) {
 
-        getHelmChartFromGitRepo(helmConfig)
+        getHelmChartFromGitRepo(helmConfig, helmChartTempDir, chartRootDir)
 
         def chartPath = ''
         if (helmConfig.containsKey('chartPath')) {
             chartPath = helmConfig.chartPath
         }
 
         withHelm {
-            script.sh "helm dep update chart/${chartPath}"
-            String helmScript = "helm values chart/${chartPath} ${valuesFilesWithParameter(valuesFiles)}"
-            merge = script.sh returnStdout: true, script: helmScript
+            script.sh "helm dep update ${script.env.WORKSPACE}/.helmChartTempDir/${chartRootDir}/${chartPath}"
         }
-
-        return merge
     }
 
-    private getHelmChartFromGitRepo(Map helmConfig) {
+    private getHelmChartFromGitRepo(Map helmConfig, String helmChartTempDir, String chartRootDir) {
         def git
 
-        script.dir("chart") {
+        script.dir("${script.env.WORKSPACE}/${helmChartTempDir}/${chartRootDir}/") {
 
             if (helmConfig.containsKey('credentialsId')) {
                 git = script.cesBuildLib.Git.new(script, helmConfig.credentialsId)