updated clair to V4 (devtron-labs#1438)

* updated installations script for clair

* wip

* wip

* wip

* updated clair deployment

* updated postgres chart version - v9.6.0

* wip

* Added a new job for database creation (devtron-labs#1510)

* Update postgresql.yaml

* Update migrator.yaml

* Update postgresql.yaml

* Changed Labels back to 8.6.4

* changes for bitnami quay

* updated image tag

* updated LTAG

* updated migrator image

* updated installation script

* updated installation script

* updated script

* added comment

Co-authored-by: Pawan Kumar <39653409+dheeth@users.noreply.github.com>

Author: kartik-579

manifests/installation-script

@@ -74,7 +74,7 @@ if !defaultCacheBucket {
 argocd_raw = REPO_RAW_URL + LTAG + "/manifests/yamls/argocd.yaml";
 argocdResource_raw = REPO_RAW_URL + LTAG + "/manifests/yamls/argocd-resource.json";
 clair_raw = REPO_RAW_URL + LTAG + "/manifests/yamls/clair.yaml";
-clairConfig_raw = REPO_RAW_URL + LTAG + "/manifests/yamls/clair-config.yaml";
+clairConfig_raw = REPO_RAW_URL + LTAG + "/manifests/yamls/clairv4-config.yaml";
 dashboard_raw = REPO_RAW_URL + LTAG + "/manifests/yamls/dashboard.yaml";
 gitSensor_raw = REPO_RAW_URL + LTAG + "/manifests/yamls/gitsensor.yaml";
 guard_raw = REPO_RAW_URL + LTAG + "/manifests/yamls/guard.yaml";
@@ -356,13 +356,9 @@ externalSecret = kubectl apply -n devtroncd externalSecret -u externalSecretOver
 log("created external secret");
 
 #postgresql
-# if postgres already installed skip installation
-hasPostgresql = kubectl get sts postgresql-postgresql -n devtroncd;
-if !hasPostgresql {
-	kubeYamlEdit(postgresql, "data.postgresql-password", postgresqlPassword, `/Secret//postgresql-postgresql`);
-	postgresql = kubectl apply -n devtroncd postgresql -u postgresqlOverride;
-	log("created postgresql");
-}
+kubeYamlEdit(postgresql, "data.postgresql-password", postgresqlPassword, `/Secret//postgresql-postgresql`);
+postgresql = kubectl apply -n devtroncd postgresql -u postgresqlOverride;
+log("created postgresql");
 #argocd
 hasArgocd = kubectl get deployment argocd-server -n devtroncd;
 hasargocdSecret = kubectl get secret argocd-secret -n devtroncd;
@@ -443,7 +439,7 @@ kubeYamlEdit(lens, "data.PG_PASSWORD", postgresqlPassword, `/Secret//lens-secret
 
 #migrator
 #delete migrator job
-migDelete = kubectl delete -n devtroncd job  postgresql-migrate-devtron postgresql-migrate-casbin postgresql-migrate-gitsensor  postgresql-migrate-lens;
+migDelete = kubectl delete -n devtroncd job  postgresql-migrate-devtron postgresql-migrate-casbin postgresql-migrate-gitsensor  postgresql-migrate-lens postgresql-miscellaneous;
 if !migDelete {
 	log("migration job deletion failed");
 }
@@ -782,8 +778,11 @@ workflow = kubectl apply -n argo workflow -u workflowOverride;
 log("executed workflow setup");
 postgresPlainPwd = base64DecoderPrefix + `echo "` + postgresqlPassword + `" | tr -d ':\n' ` + base64DecoderSuffix;
 postgresPlainPwd = shellScript postgresPlainPwd;
-clairPosrgresUrl = "postgres://postgres:"+ postgresPlainPwd +"@postgresql-postgresql.devtroncd:5432/clair?sslmode=disable";
-yamlEdit(clairConfig, "clair.database.options.source", clairPosrgresUrl, 0);
+clairPostgresUrl = "host=postgresql-postgresql.devtroncd port=5432 dbname=clairv4 user=postgres password="+ postgresPlainPwd +" sslmode=disable"
+log(clairPostgresUrl);
+yamlEdit(clairConfig, "indexer.connstring", clairPostgresUrl, 0);
+yamlEdit(clairConfig, "matcher.connstring", clairPostgresUrl, 0);
+yamlEdit(clairConfig, "notifier.connstring", clairPostgresUrl, 0);
 clairEncodedConfig = base64EncoderPrefix + `echo "` + clairConfig + `"` + base64EncoderSuffix;
 clairEncodedConfig = shellScript clairEncodedConfig;
 kubeYamlEdit(clair, `data.config\.yaml`, clairEncodedConfig, `/Secret//clair`);

manifests/yamls/clair.yaml

@@ -10,7 +10,7 @@ metadata:
 type: Opaque
 data:
   config.yaml: |-
-    Y2xhaXI6CiAgZGF0YWJhc2U6CiAgICAjIERhdGFiYXNlIGRyaXZlci4KICAgIHR5cGU6IHBnc3FsCiAgICBvcHRpb25zOgogICAgICAjIFBvc3RncmVTUUwgQ29ubmVjdGlvbiBzdHJpbmcuCiAgICAgICMgaHR0cHM6Ly93d3cucG9zdGdyZXNxbC5vcmcvZG9jcy9jdXJyZW50L3N0YXRpYy9saWJwcS1jb25uZWN0Lmh0bWwjTElCUFEtQ09OTlNUUklORwogICAgICBzb3VyY2U6ICJwb3N0Z3JlczovL2NsYWlyOmNsYWlyQGNsYWlyLXBnLXBvc3RncmVzcWw6NTQzMi9jbGFpcj9zc2xtb2RlPWRpc2FibGUiCiAgICAgIAoKICAgICAgIyBOdW1iZXIgb2YgZWxlbWVudHMga2VwdCBpbiB0aGUgY2FjaGUuCiAgICAgICMgVmFsdWVzIHVubGlrZWx5IHRvIGNoYW5nZSAoZS5nLiBuYW1lc3BhY2VzKSBhcmUgY2FjaGVkIGluIG9yZGVyIHRvIHNhdmUgcHJldmVudCBuZWVkbGVzcyByb3VuZHRyaXBzIHRvIHRoZSBkYXRhYmFzZS4KICAgICAgY2FjaGVzaXplOiAxNjM4NAoKICAgICAgIyAzMi1iaXQgVVJMLXNhZmUgYmFzZTY0IGtleSB1c2VkIHRvIGVuY3J5cHQgcGFnaW5hdGlvbiB0b2tlbnMuCiAgICAgICMgSWYgb25lIGlzIG5vdCBwcm92aWRlZCwgaXQgd2lsbCBiZSBnZW5lcmF0ZWQuCiAgICAgICMgTXVsdGlwbGUgY2xhaXIgaW5zdGFuY2VzIGluIHRoZSBzYW1lIGNsdXN0ZXIgbmVlZCB0aGUgc2FtZSB2YWx1ZS4KICAgICAgcGFnaW5hdGlvbmtleTogIlh4b1B0Q1V6clV2NEpWNWRTK3lRK01kVzd5TEVKblJNd2lnVlkvYnBndFE9IgogIGFwaToKICAgICMgdjMgZ3JwYy9SRVNUZnVsIEFQSSBzZXJ2ZXIgYWRkcmVzcy4KICAgIGFkZHI6ICIwLjAuMC4wOjYwNjAiCgogICAgIyBIZWFsdGggc2VydmVyIGFkZHJlc3MuCiAgICAjIFRoaXMgaXMgYW4gdW5lbmNyeXB0ZWQgZW5kcG9pbnQgdXNlZnVsIGZvciBsb2FkIGJhbGFuY2VycyB0byBjaGVjayB0byBoZWFsdGhpbmVzcyBvZiB0aGUgY2xhaXIgc2VydmVyLgogICAgaGVhbHRoYWRkcjogIjAuMC4wLjA6NjA2MSIKCiAgICAjIERlYWRsaW5lIGJlZm9yZSBhbiBBUEkgcmVxdWVzdCB3aWxsIHJlc3BvbmQgd2l0aCBhIDUwMy4KICAgIHRpbWVvdXQ6IDkwMHMKCiAgICAjIE9wdGlvbmFsIFBLSSBjb25maWd1cmF0aW9uLgogICAgIyBJZiB5b3Ugd2FudCB0byBlYXNpbHkgZ2VuZXJhdGUgY2xpZW50IGNlcnRpZmljYXRlcyBhbmQgQ0FzLCB0cnkgdGhlIGZvbGxvd2luZyBwcm9qZWN0czoKICAgICMgaHR0cHM6Ly9naXRodWIuY29tL2NvcmVvcy9ldGNkLWNhCiAgICAjIGh0dHBzOi8vZ2l0aHViLmNvbS9jbG91ZGZsYXJlL2Nmc3NsCiAgICBzZXJ2ZXJuYW1lOgogICAgY2FmaWxlOgogICAga2V5ZmlsZToKICAgIGNlcnRmaWxlOgoKICB3b3JrZXI6CiAgICBuYW1lc3BhY2VfZGV0ZWN0b3JzOgogICAgLSBvcy1yZWxlYXNlCiAgICAtIGxzYi1yZWxlYXNlCiAgICAtIGFwdC1zb3VyY2VzCiAgICAtIGFscGluZS1yZWxlYXNlCiAgICAtIHJlZGhhdC1yZWxlYXNlCgogICAgZmVhdHVyZV9saXN0ZXJzOgogICAgLSBhcGsKICAgIC0gZHBrZwogICAgLSBycG0KCiAgdXBkYXRlcjoKICAgICMgRnJlcXVlbmN5IHRoZSBkYXRhYmFzZSB3aWxsIGJlIHVwZGF0ZWQgd2l0aCB2dWxuZXJhYmlsaXRpZXMgZnJvbSB0aGUgZGVmYXVsdCBkYXRhIHNvdXJjZXMuCiAgICAjIFRoZSB2YWx1ZSAwIGRpc2FibGVzIHRoZSB1cGRhdGVyIGVudGlyZWx5LgogICAgaW50ZXJ2YWw6ICIyaCIKICAgIGVuYWJsZWR1cGRhdGVyczoKICAgIC0gZGViaWFuCiAgICAtIHVidW50dQogICAgLSByZWRoYXQKICAgIC0gb3JhY2xlCiAgICAtIGFscGluZQogICAgLSByZWRoYXQKCiAgbm90aWZpZXI6CiAgICAjIE51bWJlciBvZiBhdHRlbXB0cyBiZWZvcmUgdGhlIG5vdGlmaWNhdGlvbiBpcyBtYXJrZWQgYXMgZmFpbGVkIHRvIGJlIHNlbnQuCiAgICBhdHRlbXB0czogMwoKICAgICMgRHVyYXRpb24gYmVmb3JlIGEgZmFpbGVkIG5vdGlmaWNhdGlvbiBpcyByZXRyaWVkLgogICAgcmVub3RpZnlpbnRlcnZhbDogMmgKCiAgICBodHRwOgogICAgICAjIE9wdGlvbmFsIGVuZHBvaW50IHRoYXQgd2lsbCByZWNlaXZlIG5vdGlmaWNhdGlvbnMgdmlhIFBPU1QgcmVxdWVzdHMuCiAgICAgIGVuZHBvaW50OiAiaHR0cHM6Ly9leGFtcGxlLmNvbS9ub3RpZnkvbWUiCgogICAgICAjIE9wdGlvbmFsIFBLSSBjb25maWd1cmF0aW9uLgogICAgICAjIElmIHlvdSB3YW50IHRvIGVhc2lseSBnZW5lcmF0ZSBjbGllbnQgY2VydGlmaWNhdGVzIGFuZCBDQXMsIHRyeSB0aGUgZm9sbG93aW5nIHByb2plY3RzOgogICAgICAjIGh0dHBzOi8vZ2l0aHViLmNvbS9jbG91ZGZsYXJlL2Nmc3NsCiAgICAgICMgaHR0cHM6Ly9naXRodWIuY29tL2NvcmVvcy9ldGNkLWNhCiAgICAgIHNlcnZlcm5hbWU6CiAgICAgIGNhZmlsZToKICAgICAga2V5ZmlsZToKICAgICAgY2VydGZpbGU6CgogICAgICAjIE9wdGlvbmFsIEhUVFAgUHJveHk6IG11c3QgYmUgYSB2YWxpZCBVUkwgKGluY2x1ZGluZyB0aGUgc2NoZW1lKS4KICAgICAgcHJveHk6Cg==
+    aW50cm9zcGVjdGlvbl9hZGRyOiA6NjA2MQpodHRwX2xpc3Rlbl9hZGRyOiA6NjA2MApsb2dfbGV2ZWw6IGRlYnVnCmluZGV4ZXI6CiAgY29ubnN0cmluZzogImhvc3Q9Y2xhaXItcGctcG9zdGdyZXNxbCBwb3J0PTU0MzIgZGJuYW1lPWNsYWlydjQgdXNlcj1jbGFpciBwYXNzd29yZD1jbGFpciBzc2xtb2RlPWRpc2FibGUiCiAgc2NhbmxvY2tfcmV0cnk6IDEwCiAgbGF5ZXJfc2Nhbl9jb25jdXJyZW5jeTogNQogIG1pZ3JhdGlvbnM6IHRydWUKbWF0Y2hlcjoKICBpbmRleGVyX2FkZHI6ICI6NjA2MCIKICBjb25uc3RyaW5nOiAiaG9zdD1jbGFpci1wZy1wb3N0Z3Jlc3FsIHBvcnQ9NTQzMiBkYm5hbWU9Y2xhaXJ2NCB1c2VyPWNsYWlyIHBhc3N3b3JkPWNsYWlyIHNzbG1vZGU9ZGlzYWJsZSIKICBtYXhfY29ubl9wb29sOiAxMDAKICBydW46ICIiCiAgbWlncmF0aW9uczogdHJ1ZQogIHVwZGF0ZXJfc2V0czoKICAgIC0gImFscGluZSIKICAgIC0gImF3cyIKICAgIC0gImRlYmlhbiIKICAgIC0gIm9yYWNsZSIKICAgIC0gInBob3RvbiIKICAgIC0gInB5dXBpbyIKICAgIC0gInJoZWwiCiAgICAtICJzdXNlIgogICAgLSAidWJ1bnR1Igpub3RpZmllcjoKICBjb25uc3RyaW5nOiAiaG9zdD1jbGFpci1wZy1wb3N0Z3Jlc3FsIHBvcnQ9NTQzMiBkYm5hbWU9Y2xhaXJ2NCB1c2VyPWNsYWlyIHBhc3N3b3JkPWNsYWlyIHNzbG1vZGU9ZGlzYWJsZSIKICBkZWxpdmVyeV9pbnRlcnZhbDogMW0KICBwb2xsX2ludGVydmFsOiA1bQogIG1pZ3JhdGlvbnM6IHRydWU=
 ---
 # Source: clair/templates/service.yaml
 apiVersion: v1
@@ -65,11 +65,14 @@ spec:
           "until pg_isready -h postgresql-postgresql.devtroncd -p 5432;
           do echo waiting for database; sleep 1; done;"]
       containers:
-      - name: clair
-        image: "quay.io/coreos/clair:v2.1.7"
+      - env:
+        - name: CLAIR_CONF
+          value: /etc/clair/config.yaml
+        - name: CLAIR_MODE
+          value: combo
+        name: clair
+        image: "quay.io/coreos/clair:v4.3.6"
         imagePullPolicy: IfNotPresent
-        args:
-        - "-log-level=info"
         ports:
         - name: "clair-api"
           containerPort: 6060
@@ -79,11 +82,11 @@ spec:
           protocol: TCP
         livenessProbe:
           httpGet:
-            path: /health
+            path: /healthz
             port: 6061
         readinessProbe:
           httpGet:
-            path: /health
+            path: /healthz
             port: 6061
         volumeMounts:
         - name: "clair-config"

manifests/yamls/clairv4-config.yaml

@@ -0,0 +1,29 @@
+introspection_addr: :6061
+http_listen_addr: :6060
+log_level: debug
+indexer:
+  connstring: "host=clair-pg-postgresql port=5432 dbname=clairv4 user=clair password=clair sslmode=disable"
+  scanlock_retry: 10
+  layer_scan_concurrency: 5
+  migrations: true
+matcher:
+  indexer_addr: ":6060"
+  connstring: "host=clair-pg-postgresql port=5432 dbname=clairv4 user=clair password=clair sslmode=disable"
+  max_conn_pool: 100
+  run: ""
+  migrations: true
+  updater_sets:
+    - "alpine"
+    - "aws"
+    - "debian"
+    - "oracle"
+    - "photon"
+    - "pyupio"
+    - "rhel"
+    - "suse"
+    - "ubuntu"
+notifier:
+  connstring: "host=clair-pg-postgresql port=5432 dbname=clairv4 user=clair password=clair sslmode=disable"
+  delivery_interval: 1m
+  poll_interval: 5m
+  migrations: true

manifests/yamls/migrator.yaml

@@ -172,6 +172,43 @@ spec:
   activeDeadlineSeconds: 1500
 ---
 apiVersion: batch/v1
+#this job is added for creating new database(clairv4).
+#This database is needed for clair upgrade (v2 to v4), since old database does not support migration for new clair.
+#Without this job, database can be created for new users, but not for existing users.
+kind: Job
+metadata:
+  name: postgresql-miscellaneous
+spec:
+  ttlSecondsAfterFinished: 1000
+  template:
+    spec:
+      containers:
+      - name: postgresql-miscellaneous
+        image: quay.io/devtron/postgres:11.9.0-debian-10-r26
+        env:
+        - name: PGPASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: postgresql-postgresql
+              key: postgresql-password
+        - name: PGHOST
+          value: postgresql-postgresql
+        command:
+          - /bin/sh
+          - -c
+          - psql -Upostgres -f /docker-entrypoint-initdb.d/db_create.sql
+        volumeMounts:
+          - name: custom-init-scripts
+            mountPath: /docker-entrypoint-initdb.d/
+      volumes:
+        - name: custom-init-scripts
+          configMap:
+            name: postgresql-postgresql-init-scripts
+      restartPolicy: OnFailure
+  backoffLimit: 20
+  activeDeadlineSeconds: 1500
+---
+apiVersion: batch/v1
 kind: CronJob
 metadata:
   name: app-sync-cronjob

manifests/yamls/postgresql.yaml

@@ -23,7 +23,7 @@ data:
     create database casbin;
     create database git_sensor;
     create database lens;
-    create database clair;
+    create database clairv4;
 ---
 # Source: postgresql/templates/metrics-svc.yaml
 apiVersion: v1
@@ -147,7 +147,7 @@ spec:
 
       containers:
         - name: postgresql-postgresql
-          image: "quay.io/devtron/postgres:11.3.0-debian-9-r28"
+          image: quay.io/devtron/postgres:11.9.0-debian-10-r26
           imagePullPolicy: "IfNotPresent"
           securityContext:
             runAsUser: 1001
@@ -171,6 +171,8 @@ spec:
               value: "orchestrator"
             - name: POSTGRESQL_ENABLE_LDAP
               value: "no"
+            - name: POSTGRESQL_SHARED_PRELOAD_LIBRARIES
+              value: pgaudit, uuid-ossp
           ports:
             - name: tcp-postgresql
               containerPort: 5432