stuff acme certs into appropriate pillar data

bugs->bugs cert
$host->$host cert
loadbalancer->all certs

Author: JacobCoffee

salt/_extensions/pillar/ca.py

@@ -3,6 +3,7 @@
 import binascii
 import datetime
 import os.path
+from pathlib import Path
 
 import salt.loader
 
@@ -295,46 +296,203 @@ def get_ca_signed_cert(cacert_path, ca_name, CN):
     return "\n".join([cert, key])
 
 
-def ext_pillar(minion_id, pillar, base="/etc/ssl", name="PSFCA", cert_opts=None):
-    if cert_opts is None:
-        cert_opts = {}
+def _find_acme_certs(base_path="/etc/letsencrypt/live"):
+    """Read ACME certificates from /etc/letsencrypt/live
+    
+    returns dict with domain name (key) and data (value for each cert.
+    """
+    acme_certs = {}
+    try:
+        if not Path(base_path).exists():
+            print(f"ACME base path {base_path} does not exist")
+            return acme_certs
+
+        print(f"Scanning for certificates in {base_path}")
+        for domain_dir in Path(base_path).iterdir():
+            try:
+                domain_dir_path = Path(base_path) / domain_dir
+                if not domain_dir_path.is_dir() or domain_dir.name == "README":
+                    continue
+
+                domain_name = domain_dir.name
+                print(f"Found certificate directory: {domain_name}")
+
+                # use fullchain.pem instead of just cert.pem to include the full certificate chain
+                cert_file = domain_dir_path / "fullchain.pem"
+                key_file = domain_dir_path / "privkey.pem"
+
+                if not cert_file.exists():
+                    print(f"Certificate file not found: {cert_file}")
+                    continue
+                    
+                if not key_file.exists():
+                    print(f"Key file not found: {key_file}")
+                    continue
+
+                with cert_file.open('r') as f_cert:
+                    cert_data = f_cert.read()
+
+                with key_file.open('r') as f_key:
+                    key_data = f_key.read()
+
+                # Store combined certificate and key
+                combined_data = "\n".join([cert_data, key_data])
+                acme_certs[domain_name] = combined_data
+                # print(f"read certificate for {domain_name}")
+
+            except Exception as e:
+                print(f"Error processing certificate for {domain_dir.name}: {e}")
+    
+    except Exception as e:
+        print(f"Error scanning ACME certificates directory: {e}")
+
+    print(f"Found {len(acme_certs)} ACME certificates")
+    return acme_certs
+
+
+def _process_ca_certificates(minion_id, pillar, base="/etc/ssl", name="PSFCA", cert_opts=None):
+    ca_data = {
+        "ca": {},
+        "certs": {},
+    }
+
+    try:
+        if cert_opts is None:
+            cert_opts = {}
+
+        # Create CA certificate
+        opts = cert_opts.copy()
+        opts["CN"] = name
+        create_ca(base, name, **opts)
+
+        ca_data["ca"][name] = get_ca_cert(base, name)
+
+        # Process CA-signed certificates (gen_certs)
+        gen_certs = pillar.get("tls", {}).get("gen_certs", {})
+        for certificate, config in gen_certs.items():
+            role_patterns = [
+                role.get("pattern")
+                for role in [
+                    pillar.get("roles", {}).get(r) for r in config.get("roles", "")
+                ]
+                if role and role.get("pattern") is not None
+            ]
+
+            if any(compound(pat, minion_id) for pat in role_patterns):
+                # Create the options
+                opts = cert_opts.copy()
+                opts["CN"] = certificate
+                opts["days"] = config.get("days", 1)
+
+                create_ca_signed_cert(base, name, **opts)
+
+                # Add the signed certificates to the pillar data
+                cert_data = get_ca_signed_cert(base, name, certificate)
+                ca_data["certs"][certificate] = cert_data
+    except Exception as e:
+        print(f"Error processing CA certificates: {e}")
+
+    return ca_data
 
-    # Ensure we have a CA created.
-    opts = cert_opts.copy()
-    opts["CN"] = name
-    create_ca(base, name, **opts)
 
-    # Start our pillar with just the ca certificate.
+def _process_acme_certificates(minion_id, pillar):
+    """Process ACME certificates
+    
+    Reads ACME certificates and determines which ones should be available
+    to the specified minion based on access rules.
+    """
+    acme_certs = {}
+    
+    try:
+        print(f"Processing ACME certificates for minion: {minion_id}")
+        all_acme_certs = _find_acme_certs()
+        
+        # Check if this is a loadbalancer (gets all certs)
+        # todo: clean up all but the one that works
+        is_loadbalancer = False
+        try:
+            if 'loadbalancer' in minion_id.lower():
+                is_loadbalancer = True
+                print(f"Minion {minion_id} identified as loadbalancer by name")
+            
+            # Also check via roles grain if that doesn't work
+            elif compound('G@roles:loadbalancer', minion_id):
+                is_loadbalancer = True
+                print(f"Minion {minion_id} identified as loadbalancer by grain")
+            
+            # Additional check - look for the loadbalancer role in the hostname
+            elif (minion_id.startswith('lb.') or minion_id.startswith('loadbalancer.')):
+                is_loadbalancer = True
+                print(f"Minion {minion_id} identified as loadbalancer by hostname pattern")
+                
+            if is_loadbalancer:
+                print(f"Minion {minion_id} is a loadbalancer, providing all certificates")
+        except Exception as e:
+            print(f"Error checking loadbalancer role: {e}")
+        
+        # Process each certificate
+        for domain_name, cert_data in all_acme_certs.items():
+            should_include = False
+            
+            # Loadbalancer gets all certs
+            if is_loadbalancer:
+                should_include = True
+                reason = "loadbalancer role"
+            
+            # Minion name matches domain name
+            if minion_id.startswith(domain_name.split('.')[0]):
+                should_include = True
+                reason = "name match"
+
+            # Add certificate if allowed
+            if should_include:
+                acme_certs[domain_name] = cert_data
+                print(f"Added ACME certificate {domain_name} to pillar data (reason: {reason})")
+            else:
+                print(f"Skipping certificate {domain_name} for minion {minion_id} (no access)")
+    
+    except Exception as e:
+        print(f"Error processing ACME certificates: {e}")
+    
+    return acme_certs
+
+
+def ext_pillar(minion_id, pillar, base="/etc/ssl", name="PSFCA", cert_opts=None):
+    """Pillar extension to provide TLS certificates from internal PSFCA and acme.cert generated certs"""
+    print(f"Processing pillar data for minion: {minion_id}")
+    
+    # initial data structure for certs
     data = {
         "tls": {
-            "ca": {
-                name: get_ca_cert(base, name),
-            },
+            "ca": {},
             "certs": {},
+            "certs_acme": {},
         },
     }
-
-    # Create all of the certificates required by this minion
-    gen_certs = pillar.get("tls", {}).get("gen_certs", {})
-    for certificate, config in gen_certs.items():
-        role_patterns = [
-            role.get("pattern")
-            for role in [
-                pillar.get("roles", {}).get(r) for r in config.get("roles", "")
-            ]
-            if role and role.get("pattern") is not None
-        ]
-        if any([compound(pat, minion_id) for pat in role_patterns]):
-            # Create the options
-            opts = cert_opts.copy()
-            opts["CN"] = certificate
-            opts["days"] = config.get("days", 1)
-
-            # Create the signed certificates
-            create_ca_signed_cert(base, name, **opts)
-
-            # Add the signed certificates to the pillar data
-            cert_data = get_ca_signed_cert(base, name, certificate)
-            data["tls"]["certs"][certificate] = cert_data
-
+    
+    # Process CA certificates and CA-signed certificates
+    ca_data = _process_ca_certificates(minion_id, pillar, base, name, cert_opts)
+    data["tls"]["ca"] = ca_data["ca"]
+    for cert_name, cert_data in ca_data["certs"].items():
+        data["tls"]["certs"][cert_name] = cert_data
+    
+    # process ACME certificates
+    acme_certs = _process_acme_certificates(minion_id, pillar)
+    
+    # Add ACME certificates to both certs and certs_acme sections
+    for cert_name, cert_data in acme_certs.items():
+        # Store in certs_acme section (dedicated for ACME certificates)
+        data["tls"]["certs_acme"][cert_name] = cert_data
+        
+        # Also store in general certs section for backward compatibility
+        # Only if not already present from CA-signed certs
+        if cert_name not in data["tls"]["certs"]:
+            data["tls"]["certs"][cert_name] = cert_data
+    
+    # Check if we have ACME certificates for debugging
+    if not acme_certs:
+        print(f"No ACME certificates were included for minion: {minion_id}")
+    else:
+        print(f"Included {len(acme_certs)} ACME certificates for minion: {minion_id}")
+    
     return data

salt/tls/init.sls

@@ -45,7 +45,27 @@ certbot:
       - pkg: ssl-cert
 {% endfor %}
 
-{% if grains['id'] == 'salt.nyc1.psf.io' or grains['id'] == 'salt-master.vagrant.psf.io' %}
+{% if salt["match.compound"](pillar["roles"]["salt-master"]["pattern"]) %}
+# HTTP-validated domains
+{% for domain in [
+    'pypa.io',
+    'www.pycon.org',
+    'speed.pypy.org',
+    'salt-public.psf.io',
+    'planetpython.org',
+    'bugs.python.org'
+] %}
+{{ domain }}:
+  acme.cert:
+    - email: infrastructure-staff@python.org
+    - webroot: /etc/lego
+    - renew: 14
+    {% if pillar["dc"] == "vagrant" %}
+    - server: https://salt-master.vagrant.psf.io:14000/dir
+    {% endif %}
+    - require:
+      - sls: tls.lego
+{% endfor %}
 
 # DNS-validated domains
 # dns plugins do not exist yet for route53 & gandi
@@ -85,27 +105,6 @@ certbot:
 {#    - require:#}
 {#      - sls: tls.lego#}
 
-# HTTP-validated domains
-{% for domain in [
-    'pypa.io',
-    'www.pycon.org',
-    'speed.pypy.org',
-    'salt-public.psf.io',
-    'planetpython.org',
-    'bugs.python.org'
-] %}
-{{ domain }}:
-  acme.cert:
-    - email: infrastructure-staff@python.org
-    - webroot: /etc/lego
-    - renew: 14
-    {% if pillar["dc"] == "vagrant" %}
-    - server: https://salt-master.vagrant.psf.io:14000/dir
-    {% endif %}
-    - require:
-      - sls: tls.lego
-{% endfor %}
-
 # Multi-domain certificates
 {#jython.org:#}
 {#  acme.cert:#}

salt/tls/pebble.sls

@@ -1,3 +1,4 @@
+{% if salt["match.compound"](pillar["roles"]["salt-master"]["pattern"]) %}
 {% if pillar.get('pebble', {'enabled': False}).enabled %}
 pebble-build-deps:
   pkg.installed:
@@ -60,3 +61,4 @@ pebble-service:
       - file: /etc/ssl/certs/PSF_CA.pem
       - file: /etc/ssl/private/salt-master.vagrant.psf.io.pem
 {% endif %}
+{% endif %}