refactor(ci): workflows and base image, add build swtpm (#922)

This PR modernizes our build system by:

Updating GitHub workflow configurations
Restructuring base image management via werf templates
Implementing connect to 3p repos
Updating critical dependencies
Build swtpm package

Rename directory cv to component_versions
---------

Signed-off-by: Nikita Korolev <nikita.korolev@flant.com>

Author: universal-itengineer

.github/workflows/dev_module_build.yml

@@ -24,6 +24,7 @@ env:
   GO_VERSION: "1.22.7"
   GOLANGCI_LINT_VERSION: "1.64.8"
   SOURCE_REPO: "${{secrets.SOURCE_REPO}}"
+  SOURCE_REPO_GIT: "${{secrets.SOURCE_REPO_GIT}}"
 
 on:
   workflow_dispatch:
@@ -35,7 +36,7 @@ on:
         required: false
         type: number
   pull_request:
-    types: [opened, reopened, synchronize, labeled]
+    types: [opened, reopened, synchronize, labeled, unlabeled]
   push:
     branches:
       - main
@@ -325,6 +326,30 @@ jobs:
         if: ${{ !contains(needs.set_vars.outputs.runner_type, 'self-hosted') }}
         uses: ./.github/actions/remove-unwanted-software
 
+      - name: Start ssh-agent
+        uses: webfactory/ssh-agent@v0.9.0
+        with:
+          ssh-private-key: |
+            ${{secrets.SOURCE_REPO_SSH_KEY}}
+
+      - name: Add ssh_known_hosts
+        run: |
+          HOST=$(grep -oP '(?<=@)[^/:]+' <<< ${{secrets.SOURCE_REPO_GIT}})
+          echo "::add-mask::$HOST"
+          IPS=$(nslookup "$HOST" | awk '/^Address: / { print $2 }')
+          for IP in $IPS; do
+            echo "::add-mask::$IP"
+          done
+          mkdir -p ~/.ssh
+          touch ~/.ssh/known_hosts
+          HOST_KEYS=$(ssh-keyscan -H "$HOST" 2>/dev/null)
+          while IFS= read -r KEY_LINE; do
+            CONSTANT_PART=$(awk '{print $2, $3}' <<< "$KEY_LINE")
+            if ! grep -q "$CONSTANT_PART" ~/.ssh/known_hosts; then
+              echo "$KEY_LINE" >> ~/.ssh/known_hosts
+            fi
+          done <<< "$HOST_KEYS"
+
       - uses: deckhouse/modules-actions/setup@v2
         with:
           registry: ${{ vars.DEV_REGISTRY }}

.helmignore

@@ -10,3 +10,4 @@ openapi
 release.yaml
 werf*.yaml
 NOTES.txt
+.git

.prettierignore

@@ -3,6 +3,7 @@
 **/.hg
 **/werf*.yaml
 **/werf*.yml
+.werf/**
 **/templates
 **/ISSUE_TEMPLATE
 CHANGELOG/*.yml

.werf/defines/parse-base-images-map.tmpl

@@ -0,0 +1,33 @@
+{{- define "parse_base_images_map" }}
+  {{- $virtualizationImages := .Files.Get "base-images/virtualization_images.yml" | fromYaml }}
+  {{- $deckhouseImages := .Files.Get "base-images/deckhouse_images.yml" | fromYaml }}
+  
+  # virtualizationImages have image format:
+  # BASE_IMAGE: "<image_name>@sha256abcde12345
+  {{- range $k, $v := $virtualizationImages }}
+    {{ $baseImagePath := (printf "%s%s" $virtualizationImages.REGISTRY_PATH (trimSuffix "/" $v)) }}
+    {{- if ne $k "REGISTRY_PATH" }}
+      {{- $_ := set $virtualizationImages $k $baseImagePath }}
+    {{- end }}
+  {{- end }}
+  {{- $_ := unset $virtualizationImages "REGISTRY_PATH" }}
+  
+  # deckhouse_images has a format
+  # <prefix>/<name>: "sha256:abcde12345
+  {{- range $k, $v := $deckhouseImages }}
+    {{ $baseImagePath := (printf "%s@%s" $deckhouseImages.REGISTRY_PATH (trimSuffix "/" $v)) }}
+    {{- if ne $k "REGISTRY_PATH" }}
+      {{- $_ := set $deckhouseImages $k $baseImagePath }}
+    {{- end }}
+  {{- end }}
+  {{- $_ := unset $deckhouseImages "REGISTRY_PATH" }}
+
+  {{- $_ := set . "Images" (mustMerge $virtualizationImages $deckhouseImages) }}
+  # base images artifacts
+  {{- range $k, $v := .Images }}
+---
+image: {{ $k }}
+from: {{ $v }}
+final: false
+  {{- end }}
+{{- end }}

.werf/images.yaml

@@ -0,0 +1,53 @@
+# Common dirs
+{{- define "module_image_template" }}
+
+  {{- if eq .ImageInstructionType "Dockerfile" }}
+---
+image: images/{{ .ImageName }}
+context: images/{{ .ImageName }}
+dockerfile: Dockerfile
+  {{- else }}
+    {{- tpl .ImageBuildData . | nindent 0 }}
+  {{- end }}
+
+{{- end }}
+
+# Context inside folder images
+{{- $Root := . }}
+
+{{ $ImagesBuildFiles := .Files.Glob "images/*/{Dockerfile,werf.inc.yaml}" }}
+
+{{- range $path, $content := $ImagesBuildFiles }}
+  {{- $ctx := dict }}
+  
+  {{- if regexMatch "/werf.inc.yaml$" $path }}
+    {{- $_ := set $ctx "ImageInstructionType" "Stapel" }}
+  {{- else }}
+    {{- $_ := set $ctx "ImageInstructionType" "Dockerfile" }}
+  {{- end }}
+
+  {{- $ImageData := $path | split "/" }}
+
+  {{- $_ := set $ctx "ImageName" $ImageData._1 }}
+  {{- $_ := set $ctx "ImageBuildData" $content }}
+  {{- $_ := set $ctx "SOURCE_REPO" $Root.SOURCE_REPO }}
+  {{- $_ := set $ctx "SOURCE_REPO_GIT" $Root.SOURCE_REPO_GIT }}
+  {{- $_ := set $ctx "MODULE_EDITION" $Root.MODULE_EDITION }}
+  {{- $_ := set $ctx "Version" $Root.Version }}
+
+  {{- include "module_image_template" $ctx }}
+  
+  {{- range $ImageYamlMainfest := regexSplit "\n?---[ \t]*\n" (include "module_image_template" $ctx) -1 }}
+    {{- $ImageManifest := $ImageYamlMainfest | fromYaml }}
+    
+      {{- if $ImageManifest | dig "final" true }}
+        
+        {{- if $ImageManifest.image }}
+          {{- $_ := set $ "ImagesIDList" (append $.ImagesIDList $ImageManifest.image) }}
+        {{- end }}
+
+      {{- end }}
+    
+  {{- end }}
+
+{{- end }}

.werf/packages.yaml

@@ -0,0 +1,59 @@
+{{- define "packages_template" }}
+
+  {{- if eq .ImageInstructionType "Dockerfile" }}
+---
+image: images/{{ .ImageName }}
+context: images/{{ .ImageName }}
+dockerfile: Dockerfile
+  {{- else }}
+    {{- tpl .ImageBuildData . | nindent 0 }}
+  {{- end }}
+
+{{- end }}
+
+{{- $Root := . }}
+
+
+{{ $ImagePackages := .Files.Glob "images/packages/*/*/{Dockerfile,werf.inc.yaml}" }}
+{{- range $path, $content := $ImagePackages }}
+  {{- $ctx := dict }}
+  
+  {{- if regexMatch "/werf.inc.yaml$" $path }}
+    {{- $_ := set $ctx "ImageInstructionType" "Stapel" }}
+  {{- else }}
+    {{- $_ := set $ctx "ImageInstructionType" "Dockerfile" }}
+  {{- end }}
+
+# Parse variables for images from their path.
+# for example: `packages/binaries/swtpm/werf.inc.yaml` turns into
+# ImageType: packages/binaries
+# ImageName: swtpm
+
+  {{- $ImageData := regexReplaceAll "^images/(packages)/([0-9a-z-_]+)/([0-9a-z-_]+)/(Dockerfile|werf.inc.yaml)$" $path "${1}#${2}#${3}#${4}" | split "#" }}
+
+  {{- $_ := set $ctx "ImageType" (printf "%s/%s" $ImageData._0 $ImageData._1) }}
+  {{- $_ := set $ctx "ImageName" $ImageData._2 }}
+  {{- $_ := set $ctx "ImageBuildData" $content }}
+  {{- $_ := set $ctx "Files" $Root.Files }}
+  {{- $_ := set $ctx "SOURCE_REPO" $Root.SOURCE_REPO }}
+  {{- $_ := set $ctx "SOURCE_REPO_GIT" $Root.SOURCE_REPO_GIT }}
+  {{- $_ := set $ctx "MODULE_EDITION" $Root.MODULE_EDITION }}
+  {{- $_ := set $ctx "Version" $Root.Version }}
+  {{- $_ := set $ctx "Package" $Root.Packages }}
+
+  {{- include "packages_template" $ctx }}
+
+  {{- range $ImageYamlMainfest := regexSplit "\n?---[ \t]*\n" (include "packages_template" $ctx) -1 }}
+    {{- $ImageManifest := $ImageYamlMainfest | fromYaml }}
+    
+      {{- if $ImageManifest | dig "final" true }}
+        
+        {{- if $ImageManifest.image }}
+          {{- $_ := set $ "ImagesIDList" (append $.ImagesIDList $ImageManifest.image) }}
+        {{- end }}
+      
+      {{- end }}
+    
+  {{- end }}
+
+{{- end }}

base-images/alt_image_versions.yml

@@ -0,0 +1,20 @@
+# version v0.4.1
+# REGISTRY_PATH is a special key which is concatenated with other base images
+REGISTRY_PATH: registry.deckhouse.io/base_images
+base/distroless: "sha256:f2ac3320ce677484d7aef4608f99ddf0055d8907cc3509f8cccc8513f314c31a" # fromImage: builder/scratch
+base/nginx-static: "sha256:28a844d4ff777dc704d3808bdd1240108315f4d1ed1668e762627b9c4ddc8d66" # fromImage: base/distroless
+base/python: "sha256:432f19eaeb5035ea38e3fbc721576e6802c679fd0fb7a8e6d441fdb36944c180" # fromImage: base/distroless
+builder/alpine: "sha256:92adb9e8a387c645ba4eeedb08f9a6de0583c5d1d496bf9c9b3a89fe963a6a06" # from: alpine:3.20.6
+builder/alt: "sha256:6fe06774a942e293e28b85cf3307e0f6e79a1593352c9c5ca97a6cbefdce0a55" # from: registry.altlinux.org/p11/alt:20241211
+builder/golang-alpine: "sha256:482aab8cb49614acb14ce9397861b02d05cc222d893d689abd205c9bd2c17b46" # from: golang:1.24.2-alpine3.20
+builder/golang-bookworm: "sha256:3755a67e7d3f16995b1615694a2aae7a95618a1ff9603b23f347764dfa0242b2" # from: golang:1.24.2-bookworm
+builder/golang-bullseye: "sha256:7f2838a3779bb9759accba1cbea7ea7aa086cc67ec763718004febb47948570c" # from: golang:1.24.2-bullseye
+builder/node-alpine: "sha256:f09f957ee8e960c7afee93d7a30f20116e76660c0408d35109ee9a9fa2a4d5b0" # from: node:23.10.0-alpine3.20
+builder/scratch: "sha256:10b5855ae3e20ecc2ba6eb3d7f8365ee2facc98a62c9e4840927ad4074bd2317" # from: registry.werf.io/werf/scratch
+tools/bash: "sha256:892cd63bafb70b529377833c374009db5e9759c0aa72b165aff4ba52b59a45c6" # fromImage: builder/scratch
+tools/cosign: "sha256:d62c8deb048d9d02e5ea3b9f1efa9a940625c746f3c568a283c346c844a8acf8" # fromImage: builder/scratch
+tools/grep: "sha256:48e485161d6cec13d5283d03e69e2d1ff9e1c6146d040dc3bf623d856b62a636" # fromImage: builder/scratch
+tools/jq: "sha256:71aac8b711bef4dbef462d6309e283812358b9920301ad3de23a5734a18a23f2" # fromImage: builder/scratch
+tools/less: "sha256:259d88c1a732e7d3fc8d067ea0324684e1fe73a3bbb3eb7e2a9d88212a4ee2a0" # fromImage: builder/scratch
+tools/vim: "sha256:66db26b1475c389df02b8a8f9b98ff6993b2a2b51085f88a0cf618b915086de6" # fromImage: builder/scratch
+tools/yq4: "sha256:35911ea155806ec3b29c5d6eec90184d3d836bb42037f9865fb5949ab7a4cd6c" # fromImage: builder/scratch