Add ROOT CAs to the trust store

api/src/main/java/org/apache/cloudstack/api/command/admin/ca/ProvisionCertificateCmd.java

@@ -63,6 +63,12 @@ public class ProvisionCertificateCmd extends BaseAsyncCmd {
             description = "Name of the CA service provider, otherwise the default configured provider plugin will be used")
     private String provider;
 
+    @Parameter(name = ApiConstants.FORCED, type = CommandType.BOOLEAN,
+            description = "When true, uses SSH to re-provision the agent's certificate, bypassing the NIO agent connection. " +
+            "Use this when agents are disconnected due to a CA change. Supported for KVM hosts and SystemVMs. Default is false",
+            since = "4.23.0")
+    private Boolean forced;
+
     /////////////////////////////////////////////////////
     /////////////////// Accessors ///////////////////////
     /////////////////////////////////////////////////////
@@ -79,6 +85,10 @@ public String getProvider() {
         return provider;
     }
 
+    public boolean isForced() {
+        return forced != null && forced;
+    }
+
     /////////////////////////////////////////////////////
     /////////////// API Implementation///////////////////
     /////////////////////////////////////////////////////
@@ -90,7 +100,7 @@ public void execute() {
             throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Unable to find host by ID: " + getHostId());
         }
 
-        boolean result = caManager.provisionCertificate(host, getReconnect(), getProvider());
+        boolean result = caManager.provisionCertificate(host, getReconnect(), getProvider(), isForced());
         SuccessResponse response = new SuccessResponse(getCommandName());
         response.setSuccess(result);
         setResponseObject(response);

api/src/main/java/org/apache/cloudstack/ca/CAManager.java

@@ -23,6 +23,8 @@
 import java.util.List;
 import java.util.Map;
 
+import com.trilead.ssh2.Connection;
+
 import org.apache.cloudstack.framework.ca.CAProvider;
 import org.apache.cloudstack.framework.ca.CAService;
 import org.apache.cloudstack.framework.ca.Certificate;
@@ -39,7 +41,10 @@ public interface CAManager extends CAService, Configurable, PluggableService {
     ConfigKey<String> CAProviderPlugin = new ConfigKey<>("Advanced", String.class,
             "ca.framework.provider.plugin",
             "root",
-            "The CA provider plugin that is used for secure CloudStack management server-agent communication for encryption and authentication. Restart management server(s) when changed.", true);
+            "The CA provider plugin used for CloudStack internal certificate management (MS-agent encryption and authentication). " +
+            "The default 'root' provider auto-generates a CA on first startup, but also supports user-provided custom CA material " +
+            "via the ca.plugin.root.private.key, ca.plugin.root.public.key, and ca.plugin.root.ca.certificate settings. " +
+            "Restart management server(s) when changed.", true);
 
     ConfigKey<Integer> CertKeySize = new ConfigKey<>("Advanced", Integer.class,
                                     "ca.framework.cert.keysize",
@@ -85,6 +90,12 @@ public interface CAManager extends CAService, Configurable, PluggableService {
                     "The actual implementation will depend on the configured CA provider.",
             false);
 
+    ConfigKey<Boolean> CaInjectDefaultTruststore = new ConfigKey<>("Advanced", Boolean.class,
+            "ca.framework.inject.default.truststore", "true",
+            "When true, injects the CA provider's certificate into the JVM default truststore on management server startup. " +
+            "This allows outgoing HTTPS connections from the management server to trust servers with certificates signed by the configured CA. " +
+            "Restart management server(s) when changed.", true);
+
     /**
      * Returns a list of available CA provider plugins
      * @return returns list of CAProvider
@@ -130,12 +141,25 @@ public interface CAManager extends CAService, Configurable, PluggableService {
     boolean revokeCertificate(final BigInteger certSerial, final String certCn, final String provider);
 
     /**
-     * Provisions certificate for given active and connected agent host
+     * Provisions certificate for given agent host.
+     * When forced=true, uses SSH to re-provision bypassing the NIO agent connection (for disconnected agents).
      * @param host
+     * @param reconnect
      * @param provider
+     * @param forced when true, provisions via SSH instead of NIO; supports KVM hosts and SystemVMs
      * @return returns success/failure as boolean
      */
-    boolean provisionCertificate(final Host host, final Boolean reconnect, final String provider);
+    boolean provisionCertificate(final Host host, final Boolean reconnect, final String provider, final boolean forced);
+
+    /**
+     * Provisions certificate for a KVM host using an existing SSH connection.
+     * Runs keystore-setup to generate a CSR, issues a certificate, then runs keystore-cert-import.
+     * Used during host discovery and for forced re-provisioning when the NIO agent is unreachable.
+     * @param sshConnection active SSH connection to the KVM host
+     * @param agentIp IP address of the KVM host agent
+     * @param agentHostname hostname of the KVM host agent
+     */
+    void provisionCertificateViaSsh(Connection sshConnection, String agentIp, String agentHostname);
 
     /**
      * Setups up a new keystore and generates CSR for a host

plugins/ca/root-ca/src/main/java/org/apache/cloudstack/ca/provider/RootCAProvider.java

@@ -106,17 +106,21 @@ public final class RootCAProvider extends AdapterBase implements CAProvider, Con
     private static ConfigKey<String> rootCAPrivateKey = new ConfigKey<>("Hidden", String.class,
             "ca.plugin.root.private.key",
             null,
-            "The ROOT CA private key.", true);
+            "The ROOT CA private key in PEM format (PKCS#8: must start with '-----BEGIN PRIVATE KEY-----'). " +
+            "When set along with the public key and certificate, CloudStack uses this custom CA instead of auto-generating one. " +
+            "All three ca.plugin.root.* keys must be set together. Restart management server(s) when changed.", true);
 
     private static ConfigKey<String> rootCAPublicKey = new ConfigKey<>("Hidden", String.class,
             "ca.plugin.root.public.key",
             null,
-            "The ROOT CA public key.", true);
+            "The ROOT CA public key in PEM format (X.509/SPKI: must start with '-----BEGIN PUBLIC KEY-----'). " +
+            "Required when providing a custom CA. Restart management server(s) when changed.", true);
 
     private static ConfigKey<String> rootCACertificate = new ConfigKey<>("Hidden", String.class,
             "ca.plugin.root.ca.certificate",
             null,
-            "The ROOT CA certificate.", true);
+            "The ROOT CA X.509 certificate in PEM format (must start with '-----BEGIN CERTIFICATE-----'). " +
+            "Required when providing a custom CA. Restart management server(s) when changed.", true);
 
     private static ConfigKey<String> rootCAIssuerDN = new ConfigKey<>("Advanced", String.class,
             "ca.plugin.root.issuer.dn",
@@ -422,13 +426,29 @@ protected void addConfiguredManagementIp(List<String> ipList) {
 
 
     private boolean setupCA() {
-        if (!loadRootCAKeyPair() && !saveNewRootCAKeypair()) {
-            logger.error("Failed to save and load root CA keypair");
-            return false;
+        if (!loadRootCAKeyPair()) {
+            if (hasUserProvidedCAKeys()) {
+                logger.error("Failed to load user-provided CA keys from configuration. " +
+                    "Check that ca.plugin.root.private.key, ca.plugin.root.public.key, and " +
+                    "ca.plugin.root.ca.certificate are all set and in the correct PEM format " +
+                    "(private key must be PKCS#8: '-----BEGIN PRIVATE KEY-----'). " +
+                    "Overwriting with auto-generated keys.");
+            }
+            if (!saveNewRootCAKeypair()) {
+                logger.error("Failed to save and load root CA keypair");
+                return false;
+            }
         }
-        if (!loadRootCACertificate() && !saveNewRootCACertificate()) {
-            logger.error("Failed to save and load root CA certificate");
-            return false;
+        if (!loadRootCACertificate()) {
+            if (hasUserProvidedCAKeys()) {
+                logger.error("Failed to load user-provided CA certificate. " +
+                    "Check that ca.plugin.root.ca.certificate is set and in PEM format. " +
+                    "Overwriting with auto-generated certificate.");
+            }
+            if (!saveNewRootCACertificate()) {
+                logger.error("Failed to save and load root CA certificate");
+                return false;
+            }
         }
         if (!loadManagementKeyStore()) {
             logger.error("Failed to check and configure management server keystore");
@@ -437,10 +457,16 @@ private boolean setupCA() {
         return true;
     }
 
+    private boolean hasUserProvidedCAKeys() {
+        return StringUtils.isNotEmpty(rootCAPublicKey.value())
+            || StringUtils.isNotEmpty(rootCAPrivateKey.value())
+            || StringUtils.isNotEmpty(rootCACertificate.value());
+    }
+
     @Override
     public boolean start() {
         managementCertificateCustomSAN = CAManager.CertManagementCustomSubjectAlternativeName.value();
-        return loadRootCAKeyPair() && loadRootCAKeyPair() && loadManagementKeyStore();
+        return loadRootCAKeyPair() && loadRootCACertificate() && loadManagementKeyStore();
     }
 
     @Override

scripts/util/keystore-cert-import

@@ -70,8 +70,8 @@ elif [ ! -f "$CACERT_FILE" ]; then
 fi
 
 # Import cacerts into the keystore
-awk '/-----BEGIN CERTIFICATE-----?/{n++}{print > "cloudca." n }' "$CACERT_FILE"
-for caChain in $(ls cloudca.*); do
+awk 'BEGIN{n=0} /-----BEGIN CERTIFICATE-----/{n++}{print > "cloudca." n }' "$CACERT_FILE"
+for caChain in $(ls cloudca.* 2>/dev/null); do
     keytool -delete -noprompt -alias "$caChain" -keystore "$KS_FILE" -storepass "$KS_PASS" > /dev/null 2>&1 || true
     keytool -import -noprompt -storepass "$KS_PASS" -trustcacerts -alias "$caChain" -file "$caChain" -keystore "$KS_FILE" > /dev/null 2>&1
 done
@@ -137,6 +137,22 @@ if [ -f "$SYSTEM_FILE" ]; then
     chmod 644 /usr/local/share/ca-certificates/cloudstack/ca.crt
     update-ca-certificates > /dev/null 2>&1 || true
 
+    # Import CA cert(s) into realhostip.keystore so the SSVM JVM
+    # (which overrides the truststore via -Djavax.net.ssl.trustStore in _run.sh)
+    # can trust servers signed by the CloudStack CA
+    REALHOSTIP_KS_FILE="$(dirname $(dirname $PROPS_FILE))/certs/realhostip.keystore"
+    REALHOSTIP_PASS="vmops.com"
+    if [ -f "$REALHOSTIP_KS_FILE" ]; then
+        awk 'BEGIN{n=0} /-----BEGIN CERTIFICATE-----/{n++}{print > "cloudca." n }' "$CACERT_FILE"
+        for caChain in $(ls cloudca.* 2>/dev/null); do
+            keytool -delete -noprompt -alias "$caChain" -keystore "$REALHOSTIP_KS_FILE" \
+                -storepass "$REALHOSTIP_PASS" > /dev/null 2>&1 || true
+            keytool -import -noprompt -trustcacerts -alias "$caChain" -file "$caChain" \
+                -keystore "$REALHOSTIP_KS_FILE" -storepass "$REALHOSTIP_PASS" > /dev/null 2>&1
+        done
+        rm -f cloudca.*
+    fi
+
     # Ensure cloud service is running in systemvm
     if [ "$MODE" == "ssh" ]; then
         systemctl start cloud > /dev/null 2>&1

server/src/main/java/com/cloud/hypervisor/kvm/discoverer/LibvirtServerDiscoverer.java

@@ -21,7 +21,6 @@
 import java.net.InetAddress;
 import java.net.URI;
 import java.util.Arrays;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -32,11 +31,8 @@
 
 import org.apache.cloudstack.agent.lb.IndirectAgentLB;
 import org.apache.cloudstack.ca.CAManager;
-import org.apache.cloudstack.ca.SetupCertificateCommand;
 import org.apache.cloudstack.direct.download.DirectDownloadManager;
-import org.apache.cloudstack.framework.ca.Certificate;
 import org.apache.cloudstack.utils.cache.LazyCache;
-import org.apache.cloudstack.utils.security.KeyStoreUtils;
 
 import com.cloud.agent.AgentManager;
 import com.cloud.agent.Listener;
@@ -66,7 +62,6 @@
 import com.cloud.resource.ResourceStateAdapter;
 import com.cloud.resource.ServerResource;
 import com.cloud.resource.UnableDeleteHostException;
-import com.cloud.utils.PasswordGenerator;
 import com.cloud.utils.StringUtils;
 import com.cloud.utils.UuidUtils;
 import com.cloud.utils.exception.CloudRuntimeException;
@@ -174,55 +169,7 @@ private void setupAgentSecurity(final Connection sshConnection, final String age
             throw new CloudRuntimeException("Cannot secure agent communication because SSH connection is invalid for host IP=" + agentIp);
         }
 
-        Integer validityPeriod = CAManager.CertValidityPeriod.value();
-        if (validityPeriod < 1) {
-            validityPeriod = 1;
-        }
-
-        String keystorePassword = PasswordGenerator.generateRandomPassword(16);
-        final SSHCmdHelper.SSHCmdResult keystoreSetupResult = SSHCmdHelper.sshExecuteCmdWithResult(sshConnection,
-                String.format("sudo /usr/share/cloudstack-common/scripts/util/%s " +
-                                "/etc/cloudstack/agent/agent.properties " +
-                                "/etc/cloudstack/agent/%s " +
-                                "%s %d " +
-                                "/etc/cloudstack/agent/%s",
-                        KeyStoreUtils.KS_SETUP_SCRIPT,
-                        KeyStoreUtils.KS_FILENAME,
-                        keystorePassword,
-                        validityPeriod,
-                        KeyStoreUtils.CSR_FILENAME));
-
-        if (!keystoreSetupResult.isSuccess()) {
-            throw new CloudRuntimeException("Failed to setup keystore on the KVM host: " + agentIp);
-        }
-
-        final Certificate certificate = caManager.issueCertificate(keystoreSetupResult.getStdOut(), Arrays.asList(agentHostname, agentIp), Collections.singletonList(agentIp), null, null);
-        if (certificate == null || certificate.getClientCertificate() == null) {
-            throw new CloudRuntimeException("Failed to issue certificates for KVM host agent: " + agentIp);
-        }
-
-        final SetupCertificateCommand certificateCommand = new SetupCertificateCommand(certificate);
-        final SSHCmdHelper.SSHCmdResult setupCertResult = SSHCmdHelper.sshExecuteCmdWithResult(sshConnection,
-                String.format("sudo /usr/share/cloudstack-common/scripts/util/%s " +
-                                "/etc/cloudstack/agent/agent.properties %s " +
-                                "/etc/cloudstack/agent/%s %s " +
-                                "/etc/cloudstack/agent/%s \"%s\" " +
-                                "/etc/cloudstack/agent/%s \"%s\" " +
-                                "/etc/cloudstack/agent/%s \"%s\"",
-                        KeyStoreUtils.KS_IMPORT_SCRIPT,
-                        keystorePassword,
-                        KeyStoreUtils.KS_FILENAME,
-                        KeyStoreUtils.SSH_MODE,
-                        KeyStoreUtils.CERT_FILENAME,
-                        certificateCommand.getEncodedCertificate(),
-                        KeyStoreUtils.CACERT_FILENAME,
-                        certificateCommand.getEncodedCaCertificates(),
-                        KeyStoreUtils.PKEY_FILENAME,
-                        certificateCommand.getEncodedPrivateKey()));
-
-        if (setupCertResult != null && !setupCertResult.isSuccess()) {
-            throw new CloudRuntimeException("Failed to setup certificate in the KVM agent's keystore file, please see logs and configure manually!");
-        }
+        caManager.provisionCertificateViaSsh(sshConnection, agentIp, agentHostname);
 
         if (logger.isDebugEnabled()) {
             logger.debug("Succeeded to import certificate in the keystore for agent on the KVM host: " + agentIp + ". Agent secured and trusted.");