feat: Git sync - add first class support for CAs (#903)

* bump gitsync to v2, adapt test to use ca cert

* added a docs note

* changelog

* switch to tls field

* fixed branch ref

* updated docs

* bump op-rs

* minor cleanup

* use non-privileged nginx as needed for openshift

* remove schemars workarounds

* bump op-rs to 0.110.1

* updated changelog

* removed duplicate comment

* Update docs/modules/nifi/pages/usage_guide/custom-components.adoc

Co-authored-by: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com>

---------

Co-authored-by: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com>

Author: adwk67

CHANGELOG.md

@@ -6,6 +6,7 @@ All notable changes to this project will be documented in this file.
 
 ### Added
 
+- Support for passing CAs to GitSync ([#903]).
 - Added support for NiFi `2.9.0` ([#922]).
 
 ### Changed
@@ -15,6 +16,7 @@ All notable changes to this project will be documented in this file.
   Previously, arbitrary keys were silently accepted but ignored ([#921]).
 - Bump `stackable-operator` to 0.110.1 and `kube` to 3.1.0 ([#921]).
 
+[#903]: https://github.com/stackabletech/nifi-operator/pull/903
 [#916]: https://github.com/stackabletech/nifi-operator/pull/916
 [#921]: https://github.com/stackabletech/nifi-operator/pull/921
 [#922]: https://github.com/stackabletech/nifi-operator/pull/922

Cargo.lock

@@ -58,14 +58,19 @@ spec:
       wait: 10s  # <6>
       credentialsSecret: git-credentials  # <7>
       gitSyncConf:  # <8>
-      --git-config: http.sslCAInfo:/tmp/ca-cert/ca.crt
-    - repo: https://example.com/git/other-nifi-components  # <9>
+      --git-config: pull.rebase=true  # <9>
+      tls:
+        verification:
+          server:
+            caCert:
+              secretClass: git-ca-cert # <10>
+    - repo: https://example.com/git/other-nifi-components  # <11>
   nodes:
     config:
       logging:
         enableVectorAgent: true
         containers:
-          git-sync:  # <10>
+          git-sync:  # <12>
             console:
               level: INFO
             file:
@@ -94,8 +99,14 @@ data:
     The `password` field can either be an actual password (not recommended) or a GitHub token, as described in the {git-sync-docs}[git-sync documentation].
 <8> A map of optional configuration settings that are listed in the {git-sync-docs}[git-sync documentation]. +
     These settings are not verified.
-<9> Multiple repositories can be defined. Only the `repo` field is mandatory.
-<10> Logging can be configured as described in xref:concepts:logging.adoc[].
+<9> Valid `git-config` settings can be passed here N.B. a warning will be triggered if both `http.sslCAInfo` is defined here and the field `caCertSecretName` is specified (as it maps onto the same config setting).
+<10> Optional name of a SecretClass for certificate authorities. These CAs will verify the git server's certificate when running git sync operations.
+    The associated secret must have a key named `ca.crt` whose value is the PEM-encoded certificate bundle.
+    If this field is set to `webPki: {}` or is omitted altogether, then no changes will be made to the gitsync command and it will default to presenting no certificate to the backend.
+    Omitting this field is non-breaking behaviour and as such it does *not* set `http.sslverify` to `false` as disabling security checks should be a last resort and not something activated by default.
+    This can still be achieved by passing `--git-config: http.sslverify=false` explicitly.
+<11> Multiple repositories can be defined. Only the `repo` field is mandatory.
+<12> Logging can be configured as described in xref:concepts:logging.adoc[].
      As git-sync is a command-line tool, just its output is logged and no fine-grained log configuration is possible.
      All git-sync containers are configured via the one `git-sync` field.
 

Cargo.nix

@@ -181,10 +181,34 @@ spec:
 
                             Since git-sync v4.x.x this field is mapped to the flag `--ref`.
                           type: string
-                        credentialsSecret:
+                        credentials:
                           description: An optional secret used for git access.
                           nullable: true
-                          type: string
+                          oneOf:
+                          - required:
+                            - basicAuthSecretName
+                          - required:
+                            - sshPrivateKeySecretName
+                          properties:
+                            basicAuthSecretName:
+                              description: |-
+                                The name of the Secret used to access the repository via Basic Authentication if it is not public.
+
+                                The referenced Secret must include two fields: `user` and `password`.
+                                The `password` field can either be an actual password (not recommended) or a GitHub token,
+                                as described in the git-sync [documentation].
+
+                                [documentation]: https://github.com/kubernetes/git-sync/tree/v4.2.4?tab=readme-ov-file#manual
+                              type: string
+                            sshPrivateKeySecretName:
+                              description: |-
+                                The name of the Secret used for SSH access to the repository.
+
+                                The referenced Secret must include two fields: `key` and `knownHosts`.
+
+                                [documentation]: https://github.com/kubernetes/git-sync/tree/v4.2.4?tab=readme-ov-file#manual
+                              type: string
+                          type: object
                         depth:
                           default: 1
                           description: The depth of syncing, i.e. the number of commits to clone; defaults to 1.

docs/modules/nifi/pages/usage_guide/custom-components.adoc

@@ -157,7 +157,7 @@ pub fn build_nifi_properties(
     auth_config: &NifiAuthenticationConfig,
     overrides: BTreeMap<String, String>,
     product_version: &str,
-    git_sync_resources: &git_sync::v1alpha1::GitSyncResources,
+    git_sync_resources: &git_sync::v1alpha2::GitSyncResources,
 ) -> Result<String, Error> {
     // TODO: Remove once we dropped support for all NiFi 1.x versions
     let is_nifi_1 = product_version.starts_with("1.");

extra/crds.yaml

@@ -230,7 +230,7 @@ pub enum Error {
     FailedToResolveConfig { source: crate::crd::Error },
 
     #[snafu(display("invalid git-sync specification"))]
-    InvalidGitSyncSpec { source: git_sync::v1alpha1::Error },
+    InvalidGitSyncSpec { source: git_sync::v1alpha2::Error },
 
     #[snafu(display("vector agent is enabled but vector aggregator ConfigMap is missing"))]
     VectorAggregatorConfigMapMissing,
@@ -493,7 +493,7 @@ pub async fn reconcile_nifi(
                 .merged_config(&NifiRole::Node, rolegroup_name)
                 .context(FailedToResolveConfigSnafu)?;
 
-            let git_sync_resources = git_sync::v1alpha1::GitSyncResources::new(
+            let git_sync_resources = git_sync::v1alpha2::GitSyncResources::new(
                 &nifi.spec.cluster_config.custom_components_git_sync,
                 &resolved_product_image,
                 &env_vars_from_rolegroup_config(rolegroup_config),
@@ -724,7 +724,7 @@ async fn build_node_rolegroup_config_map(
     rolegroup_config: &HashMap<PropertyNameKind, BTreeMap<String, String>>,
     merged_config: &NifiConfig,
     proxy_hosts: &str,
-    git_sync_resources: &git_sync::v1alpha1::GitSyncResources,
+    git_sync_resources: &git_sync::v1alpha2::GitSyncResources,
 ) -> Result<ConfigMap> {
     tracing::debug!("building rolegroup configmaps");
 
@@ -851,7 +851,7 @@ async fn build_node_rolegroup_statefulset(
     rolling_update_supported: bool,
     replicas: Option<i32>,
     service_account_name: &str,
-    git_sync_resources: &git_sync::v1alpha1::GitSyncResources,
+    git_sync_resources: &git_sync::v1alpha2::GitSyncResources,
 ) -> Result<StatefulSet> {
     tracing::debug!("Building statefulset");
     let role_group = role.role_groups.get(&rolegroup_ref.role_group);
@@ -1238,6 +1238,9 @@ async fn build_node_rolegroup_statefulset(
     pod_builder
         .add_volumes(git_sync_resources.git_content_volumes.to_owned())
         .context(AddVolumeSnafu)?;
+    pod_builder
+        .add_volumes(git_sync_resources.git_ca_cert_volumes.to_owned())
+        .context(AddVolumeSnafu)?;
 
     if let Some(ContainerLogConfig {
         choice:

nix/sources.json

@@ -159,7 +159,7 @@ pub mod versioned {
         /// Learn more in the documentation for
         /// [Loading custom components](DOCS_BASE_URL_PLACEHOLDER/nifi/usage_guide/custom-components.html#git_sync).
         #[serde(default)]
-        pub custom_components_git_sync: Vec<git_sync::v1alpha1::GitSync>,
+        pub custom_components_git_sync: Vec<git_sync::v1alpha2::GitSync>,
 
         /// Extra volumes similar to `.spec.volumes` on a Pod to mount into every container, this can be useful to for
         /// example make client certificates, keytabs or similar things available to processors. These volumes will be