Merge feature branch to master (#7149)

Test report:

https://xenserver.atlassian.net/wiki/spaces/xenserver/pages/1208909838/Quality+Status+Report+for+Secure+LDAP+Support+XS9

Author: changlei-li

doc/content/design/external-auth-ldaps.md

@@ -91,9 +91,11 @@ Given `ldaps` default to `false`, this feature is **NOT** enabled until explicit
 
 #### 3.1.2 Error code
 Following new error codes added to indicate ldaps enable related error
-- POOL_AUTH_ENABLE_FAILED_NO_CERTS,  no certs can be used for ldaps, refer to 4.1.2 for certs finding.
-- POOL_AUTH_ENABLE_FAILED_INVALID_CERTS, found certs, but none of the certs can be used to connect to DC
-**Note**: Current error code handing infrustrucure requires the error code prefix with POOL_AUTH_ENABLE_FAILED
+- `POOL_AUTH_ENABLE_FAILED_NO_TRUSTED_CERTS`: no trusted certs can be used for ldaps, refer to 4.1.2 for trusted certs finding.
+- `POOL_AUTH_ENABLE_FAILED_INVALID_TRUSTED_CERTS`: found trusted certs, but none of the trusted certs can be used to connect to DC.
+- `POOL_AUTH_ENABLE_FAILED_SETUP_TLS_CONNECTION`: failed to set up TLS connection to DC (e.g. GnuTLS handshake failure such as `tstream_tls_sync_setup: GNUTLS ERROR`). The error message contains the underlying details reported by winbind.
+
+**Note**: Current error code handling infrastructure requires the error code prefix with `POOL_AUTH_ENABLE_FAILED`.
 
 ### 3.2 Set/Get Pool LDAPS Status
 
@@ -135,10 +137,11 @@ xe pool-external-auth-set-ldaps uuid=<uuid> ldaps=<true|false>
 
 #### 3.2.1.2 Error code
 This API may raise following errors
-- AUTH_NO_CERTS, no certs found to enable ldaps, refer to 4.1.2 for certs finding
-- AUTH_INVALID_CERTS, found certs, but none of the certs can be used to connect to DC
-- AUTH_IS_DISABLED, AD is not enabled
-- AUTH_SET_LDAPS_FAILED,  Failed to set ldaps, the error message contains the details like ldap query on domain failed
+- `AUTH_NO_TRUSTED_CERTS`: no trusted certs found to enable ldaps, refer to 4.1.2 for trusted certs finding.
+- `AUTH_INVALID_TRUSTED_CERTS`: found trusted certs, but none of the trusted certs can be used to connect to DC.
+- `AUTH_SETUP_TLS_CONNECTION`: failed to set up TLS CONNECTION to DC (e.g. GnuTLS handshake failure such as `tstream_tls_sync_setup: GNUTLS ERROR`). The error message contains the underlying details reported by winbind.
+- `AUTH_IS_DISABLED`: AD is not enabled.
+- `AUTH_SET_LDAPS_FAILED`: Failed to set ldaps, the error message contains the details like ldap query on domain failed.
 
 #### 3.2.2 Get Pool LDAPS Status
 
@@ -268,10 +271,10 @@ alt precheck failed
 client-->>user: precheck failed
 end
 
-Note over client,coor: sync all ldaps certs
-client->>coor: pool.download_trusted_certificate
-coor-->>client:
-client->>join: pool.install_trusted_certificate
+Note over client,coor: sync trusted CA certs from coordinator to joining host
+client->>join: pool.sync_trusted_certificates_from
+join->>coor: pool.exchange_trusted_certificates_on_join
+coor-->>join:
 join-->>client:
 
 user->>client: join domain username/password
@@ -289,15 +292,11 @@ client-->>user: pool.join succeed
 
 **Detailed Steps:**
 
-1. Client find proper `ldaps certs` from pool coordinator as `certs_pool`
-   - a. find all certs `ldaps in purpose`
-   - b. if no LDAPS certs, find all `general` certs
-2. Client find all certs in joining host as `certs_joining_host`
-3. Client identify the certs needs to be synced to joining host as `certs_to_sync = certs_pool - certs_joining_host` (certs in `certs_pool`, but not in `certs_joining_host`), the certs fingerprint should be used to identify the certs
-4. Client download all `certs_to_sync`, `pool.download_trusted_certificate` from coordinator
-5. Client upload all certs to joining pool, `pool.install_trusted_certificate` to joining pool, with the same purpose
-6. Client trigger `pool.join` again with domain username and password
-7. After pool.join:
+1. Client calls `pool.sync_trusted_certificates_from` to joiner host. The call will
+   - a. download all trusted certificates from the pool, and
+   - b. install the trusted certificates into the joiner host.
+2. Client trigger `pool.join` again with domain username and password
+3. After pool.join:
    - If pool.join failed, Client call `pool.uninstall_trusted_certificate` on joining host to revert the certs
    - If pool.join succeed, do nothing as pool.join would sync the certs anyway
 

ocaml/idl/datamodel_certificate.ml

@@ -37,7 +37,13 @@ let certificate_type =
 let certificate_purpose =
   Enum
     ( "certificate_purpose"
-    , [("licensing", "Trusted certificates that are for licensing purpose.")]
+    , [
+        ("licensing", "Trusted certificates that are for licensing purpose.")
+      ; ( "ldaps"
+        , "Trusted certificates that are for LDAPS connections to external \
+           authentication services."
+        )
+      ]
     )
 
 let t =

ocaml/idl/datamodel_errors.ml

@@ -737,28 +737,54 @@ let _ =
   error Api_errors.auth_unknown_type ["type"]
     ~doc:"Unknown type of external authentication." () ;
   error Api_errors.auth_is_disabled []
-    ~doc:"External authentication is disabled, unable to resolve subject name."
+    ~doc:"External authentication is disabled" () ;
+  error Api_errors.auth_invalid_trusted_certs ["message"]
+    ~doc:
+      "The trusted certificates are invalid to setup a TLS connection to \
+       Active Directory."
+    () ;
+  error Api_errors.auth_no_trusted_certs ["message"]
+    ~doc:
+      "No trusted CA certificates found to setup a TLS connection to Active \
+       Directory"
     () ;
   error Api_errors.auth_enable_failed ["message"]
     ~doc:"The host failed to enable external authentication." () ;
   error Api_errors.auth_enable_failed_wrong_credentials ["message"]
-    ~doc:"The host failed to enable external authentication." () ;
+    ~doc:"The host failed to enable external authentication, wrong credentials."
+    () ;
   error Api_errors.auth_enable_failed_permission_denied ["message"]
-    ~doc:"The host failed to enable external authentication." () ;
+    ~doc:"The host failed to enable external authentication, permission denied."
+    () ;
   error Api_errors.auth_enable_failed_domain_lookup_failed ["message"]
     ~doc:"The host failed to enable external authentication." () ;
   error Api_errors.auth_enable_failed_unavailable ["message"]
     ~doc:"The host failed to enable external authentication." () ;
   error Api_errors.auth_enable_failed_invalid_ou ["message"]
-    ~doc:"The host failed to enable external authentication." () ;
+    ~doc:"The host failed to enable external authentication, invalid OU." () ;
   error Api_errors.auth_enable_failed_invalid_account ["message"]
-    ~doc:"The host failed to enable external authentication." () ;
+    ~doc:"The host failed to enable external authentication, invalid account."
+    () ;
+  error Api_errors.auth_enable_failed_invalid_trusted_certs ["message"]
+    ~doc:
+      "The host failed to enable external authentication, invalid trusted \
+       certificates."
+    () ;
+  error Api_errors.auth_enable_failed_no_trusted_certs ["message"]
+    ~doc:
+      "The host failed to enable external authentication, no trusted \
+       certificates"
+    () ;
   error Api_errors.auth_disable_failed ["message"]
     ~doc:"The host failed to disable external authentication." () ;
   error Api_errors.auth_disable_failed_wrong_credentials ["message"]
-    ~doc:"The host failed to disable external authentication." () ;
+    ~doc:
+      "The host failed to disable external authentication, wrong credentials."
+    () ;
   error Api_errors.auth_disable_failed_permission_denied ["message"]
-    ~doc:"The host failed to disable external authentication." () ;
+    ~doc:
+      "The host failed to disable external authentication, permission denied."
+    () ;
   error Api_errors.host_evacuate_in_progress ["host"]
     ~doc:"This host is being evacuated." () ;
 
@@ -827,21 +853,54 @@ let _ =
   error Api_errors.pool_auth_enable_failed ["host"; "message"]
     ~doc:"The pool failed to enable external authentication." () ;
   error Api_errors.pool_auth_enable_failed_wrong_credentials ["host"; "message"]
-    ~doc:"The pool failed to enable external authentication." () ;
+    ~doc:"The pool failed to enable external authentication, wrong credentials."
+    () ;
   error Api_errors.pool_auth_enable_failed_permission_denied ["host"; "message"]
-    ~doc:"The pool failed to enable external authentication." () ;
+    ~doc:"The pool failed to enable external authentication, permission denied."
+    () ;
   error Api_errors.pool_auth_enable_failed_domain_lookup_failed
     ["host"; "message"]
     ~doc:"The pool failed to enable external authentication." () ;
   error Api_errors.pool_auth_enable_failed_unavailable ["host"; "message"]
     ~doc:"The pool failed to enable external authentication." () ;
   error Api_errors.pool_auth_enable_failed_duplicate_hostname
     ["host"; "message"]
-    ~doc:"The pool failed to enable external authentication." () ;
+    ~doc:
+      "The pool failed to enable external authentication, duplicate hostname."
+    () ;
   error Api_errors.pool_auth_enable_failed_invalid_ou ["host"; "message"]
-    ~doc:"The pool failed to enable external authentication." () ;
+    ~doc:"The pool failed to enable external authentication, invalid OU" () ;
   error Api_errors.pool_auth_enable_failed_invalid_account ["host"; "message"]
-    ~doc:"The pool failed to enable external authentication." () ;
+    ~doc:"The pool failed to enable external authentication, invalid account."
+    () ;
+  error Api_errors.pool_auth_enable_failed_invalid_trusted_certs
+    ["host"; "message"]
+    ~doc:
+      "The pool failed to enable external authentication, invalid trusted \
+       certificates."
+    () ;
+  error Api_errors.pool_auth_enable_failed_no_trusted_certs ["host"; "message"]
+    ~doc:
+      "The pool failed to enable external authentication, no trusted \
+       certificates."
+    () ;
+  error Api_errors.pool_auth_enable_failed_no_supp_encrypt_type
+    ["host"; "message"]
+    ~doc:
+      "The pool failed to enable external authentication: domain does not \
+       support encryption type, make sure AES based encryption type is enabled \
+       in kerberos authentication in AD and Administrator is not in use"
+    () ;
+  error Api_errors.pool_auth_enable_failed_setup_tls_connection
+    ["host"; "message"]
+    ~doc:
+      "The pool failed to enable external authentication, failed to set up TLS \
+       connection."
+    () ;
+  error Api_errors.auth_setup_tls_connection ["message"]
+    ~doc:"Failed to set up TLS connection for external authentication." () ;
+  error Api_errors.auth_set_ldaps_failed ["host"; "message"]
+    ~doc:"The pool failed to set LDAPS configuration." () ;
   error Api_errors.pool_auth_disable_failed ["host"; "message"]
     ~doc:
       "The pool failed to disable the external authentication of at least one \

ocaml/idl/datamodel_host.ml

@@ -2044,6 +2044,45 @@ let disable_external_auth =
     ~doc:"This call disables external authentication on the local host"
     ~allowed_roles:_R_POOL_ADMIN ()
 
+let external_auth_set_ldaps =
+  call ~flags:[`Session] ~name:"external_auth_set_ldaps" ~in_oss_since:None
+    ~lifecycle:
+      [
+        ( Published
+        , "26.15.0-next"
+        , "This call enables or disables LDAPS for external authentication on \
+           the host"
+        )
+      ]
+    ~versioned_params:
+      [
+        {
+          param_type= Ref _host
+        ; param_name= "host"
+        ; param_doc= "The host whose LDAPS configuration should be set"
+        ; param_release= numbered_release "26.15.0-next"
+        ; param_default= None
+        }
+      ; {
+          param_type= Bool
+        ; param_name= "ldaps"
+        ; param_doc= "Whether to enable or disable LDAPS"
+        ; param_release= numbered_release "26.15.0-next"
+        ; param_default= None
+        }
+      ; {
+          param_type= Bool
+        ; param_name= "force"
+        ; param_doc= "Force the operation even if already in the desired state"
+        ; param_release= numbered_release "26.15.0-next"
+        ; param_default= Some (VBool false)
+        }
+      ]
+    ~doc:
+      "This call enables or disables LDAPS for external authentication on the \
+       host"
+    ~allowed_roles:_R_POOL_ADMIN ()
+
 let set_license_params =
   call ~name:"set_license_params"
     ~lifecycle:
@@ -2829,6 +2868,7 @@ let t =
       ; disable_binary_storage
       ; enable_external_auth
       ; disable_external_auth
+      ; external_auth_set_ldaps
       ; retrieve_wlb_evacuate_recommendations
       ; install_ca_certificate
       ; uninstall_ca_certificate

ocaml/idl/datamodel_lifecycle.ml

@@ -311,6 +311,8 @@ let prototyped_of_message = function
       Some "26.13.0"
   | "pool", "exchange_trusted_certificates_on_join" ->
       Some "26.13.0"
+  | "pool", "sync_trusted_certificates_from" ->
+      Some "26.15.0-next"
   | "pool", "uninstall_trusted_certificate" ->
       Some "26.13.0"
   | "pool", "install_trusted_certificate" ->

ocaml/idl/datamodel_pool.ml

@@ -660,6 +660,45 @@ let disable_external_auth =
       "This call disables external authentication on all the hosts of the pool"
     ~allowed_roles:_R_POOL_ADMIN ()
 
+let external_auth_set_ldaps =
+  call ~flags:[`Session] ~name:"external_auth_set_ldaps" ~in_oss_since:None
+    ~lifecycle:
+      [
+        ( Published
+        , "26.15.0-next"
+        , "This call enables or disables LDAPS for external authentication on \
+           all hosts in the pool"
+        )
+      ]
+    ~versioned_params:
+      [
+        {
+          param_type= Ref _pool
+        ; param_name= "pool"
+        ; param_doc= "The pool whose LDAPS configuration should be set"
+        ; param_release= numbered_release "26.15.0-next"
+        ; param_default= None
+        }
+      ; {
+          param_type= Bool
+        ; param_name= "ldaps"
+        ; param_doc= "Whether to enable or disable LDAPS"
+        ; param_release= numbered_release "26.15.0-next"
+        ; param_default= None
+        }
+      ; {
+          param_type= Bool
+        ; param_name= "force"
+        ; param_doc= "Force the operation even if already in the desired state"
+        ; param_release= numbered_release "26.15.0-next"
+        ; param_default= Some (VBool false)
+        }
+      ]
+    ~doc:
+      "This call enables or disables LDAPS for external authentication on all \
+       hosts in the pool"
+    ~allowed_roles:_R_POOL_ADMIN ()
+
 let detect_nonhomogeneous_external_auth =
   call ~flags:[`Session] ~name:"detect_nonhomogeneous_external_auth"
     ~in_oss_since:None
@@ -1663,6 +1702,36 @@ let uninstall_trusted_certificate =
     ~allowed_roles:(_R_POOL_OP ++ _R_CLIENT_CERT)
     ~lifecycle:[] ()
 
+let sync_trusted_certificates_from =
+  call ~name:"sync_trusted_certificates_from"
+    ~doc:
+      "Download trusted TLS certificates from a remote pool and install them \
+       in this pool. Certificates already present locally (matched by \
+       fingerprint and purpose) are skipped."
+    ~params:
+      [
+        (Ref _pool, "self", "The pool")
+      ; ( String
+        , "remote_pool"
+        , "The hostname or IP address of the coordinator of the remote pool \
+           from which the certificates are downloaded"
+        )
+      ; ( Ref _session
+        , "remote_session"
+        , "A session obtained from the remote pool, used to authenticate the \
+           download"
+        )
+      ; ( String
+        , "remote_certificate"
+        , "The PEM-encoded TLS certificate of the remote pool's coordinator, \
+           used to verify the TLS connection to the remote pool."
+        )
+      ; (Bool, "ca", "true for 'ca' or false for 'pinned'")
+      ]
+    ~result:(Set (Ref _certificate), "The references of certificates synced.")
+    ~allowed_roles:(_R_POOL_OP ++ _R_CLIENT_CERT)
+    ~lifecycle:[] ()
+
 let trusted_certs = Map (String, Set String)
 
 let exchange_trusted_certificates_on_join =
@@ -1752,6 +1821,7 @@ let t =
       ; disable_binary_storage
       ; enable_external_auth
       ; disable_external_auth
+      ; external_auth_set_ldaps
       ; detect_nonhomogeneous_external_auth
       ; initialize_wlb
       ; deconfigure_wlb
@@ -1814,6 +1884,7 @@ let t =
       ; set_ssh_auto_mode
       ; install_trusted_certificate
       ; uninstall_trusted_certificate
+      ; sync_trusted_certificates_from
       ; exchange_trusted_certificates_on_join
       ; exchange_crls_on_join
       ]

ocaml/idl/schematest.ml

@@ -3,7 +3,7 @@ let hash x = Digest.string x |> Digest.to_hex
 (* BEWARE: if this changes, check that schema has been bumped accordingly in
    ocaml/idl/datamodel_common.ml, usually schema_minor_vsn *)
 
-let last_known_schema_hash = "6147ef4f0f9c3bbbf0c2061e0a0d0010"
+let last_known_schema_hash = "981fd4b2f96e75ee70540759ebd0f37d"
 
 let current_schema_hash : string =
   let open Datamodel_types in

ocaml/libs/stunnel/stunnel.ml

@@ -224,9 +224,9 @@ let config_file ?(accept = None) config host port =
          )
        ; [Printf.sprintf "connect=%s:%d" host port]
        ; [
-           "sslVersion = TLSv1.2"
-         ; "ciphers = " ^ Constants.good_ciphersuites
-         ; "curve = secp384r1"
+           Printf.sprintf "sslVersion = %s" Tls_policy.Openssl.default_version
+         ; Printf.sprintf "ciphers = %s" Tls_policy.Openssl.default_ciphers
+         ; Printf.sprintf "curve = %s" Tls_policy.Openssl.default_curve
          ]
        ; ( match config with
          | None ->