feat(ocf_www): add nginx reverse proxy in front of apache

Deploy nginx on test ports (8080/8443) proxying to apache as a
slowloris mitigation layer. Includes rate limiting, request
buffering, SSL termination, and dynamic user vhost generation.

Author: oliver-ni

modules/ocf_www/files/build-vhosts

@@ -22,6 +22,10 @@ from ocflib.vhost.web import get_vhosts
 
 APACHE_SITE_CONFIG = '/etc/apache2/ocf-vhost.conf'
 NGINX_SITE_CONFIG = '/etc/nginx/sites-enabled/virtual'
+NGINX_WEB_SITE_CONFIG = '/etc/nginx/ocf-vhost.conf'
+
+# Must match $backend_port in ocf_www::init
+BACKEND_PORT = 16767
 
 LETS_ENCRYPT_SSL = Path('/services/http/ssl')
 SYSTEM_SSL = Path('/etc/ssl/private')
@@ -259,15 +263,15 @@ def build_config(src_vhosts, template, dev_config=False):
             ))
 
     return '\n\n'.join(
-        template.render(vhost=vhost)
+        template.render(vhost=vhost, backend_port=BACKEND_PORT)
         for vhost in sorted(
             vhosts,
             key=lambda vhost: (vhost.user, vhost.fqdn, bool(vhost.ssl)),
         )
     )
 
 
-def test_and_overwrite_config(config_path, new_config, target):
+def test_and_overwrite_config(config_path, new_config, test_cmd):
     """Diffs and tests the new config and overwrites the old config if
     the test passes.
 
@@ -306,10 +310,7 @@ def test_and_overwrite_config(config_path, new_config, target):
         os.rename(new_path, config_path)
 
         report('Performing config test.')
-        if target == 'web':
-            ret = subprocess.call(('apachectl', 'configtest'))
-        else:
-            ret = subprocess.call(('nginx', '-t'))
+        ret = subprocess.call(test_cmd)
 
         if ret != 0:
             report('Test failed!')
@@ -404,7 +405,6 @@ def main():
         changed |= process_app_vhosts()
 
     if args.target == 'web':
-        site_cfg = APACHE_SITE_CONFIG
         # Build app vhosts so that they can get proxied to apphost.o.b.e
         # Placed before regular vhosts so they take priority in domain matching
         # (sometimes hosts have entries in both vhost.conf and vhost-app.conf)
@@ -414,17 +414,58 @@ def main():
             for domain, conf in get_app_vhosts().items()
             if 'dev' not in conf['flags']
         }
-        config = build_config(
+
+        web_vhosts = get_vhosts()
+
+        # Apache config (existing behavior)
+        apache_config = build_config(
             prod_app_vhosts,
             jinja_env.get_template('vhost-web.jinja'),
             dev_config=args.dev,
         )
-        config += '\n\n'
-        config += build_config(
-            get_vhosts(),
+        apache_config += '\n\n'
+        apache_config += build_config(
+            web_vhosts,
             jinja_env.get_template('vhost-web.jinja'),
             dev_config=args.dev,
         )
+
+        # Nginx frontend config
+        nginx_config = build_config(
+            prod_app_vhosts,
+            jinja_env.get_template('vhost-web-nginx.jinja'),
+            dev_config=args.dev,
+        )
+        nginx_config += '\n\n'
+        nginx_config += build_config(
+            web_vhosts,
+            jinja_env.get_template('vhost-web-nginx.jinja'),
+            dev_config=args.dev,
+        )
+
+        if args.dry_run:
+            report('=== Apache config ===')
+            report(apache_config)
+            report('\n=== Nginx config ===')
+            report(nginx_config)
+            return 0
+
+        changed |= test_and_overwrite_config(
+            APACHE_SITE_CONFIG, apache_config, ('apachectl', 'configtest'),
+        )
+        changed |= test_and_overwrite_config(
+            NGINX_WEB_SITE_CONFIG, nginx_config, ('nginx', '-t'),
+        )
+
+        if changed:
+            if not args.no_reload:
+                report('Things changed, reloading.')
+                subprocess.check_call(('systemctl', 'reload', 'apache2'))
+                subprocess.check_call(('systemctl', 'reload', 'nginx'))
+            else:
+                report('Not reloading, as you requested.')
+        else:
+            report('Nothing changed, not doing anything.')
     else:
         site_cfg = NGINX_SITE_CONFIG
         config = build_config(
@@ -433,22 +474,22 @@ def main():
             dev_config=args.dev,
         )
 
-    if args.dry_run:
-        report(config)
-        return 0
+        if args.dry_run:
+            report(config)
+            return 0
 
-    changed |= test_and_overwrite_config(site_cfg, config, args.target)
-    if changed:
-        if not args.no_reload:
-            report('Things changed, reloading.')
-            if args.target == 'web':
-                subprocess.check_call(('systemctl', 'reload', 'apache2'))
-            else:
+        changed |= test_and_overwrite_config(
+            site_cfg, config, ('nginx', '-t'),
+        )
+
+        if changed:
+            if not args.no_reload:
+                report('Things changed, reloading.')
                 subprocess.check_call(('systemctl', 'reload', 'nginx'))
+            else:
+                report('Not reloading, as you requested.')
         else:
-            report('Not reloading, as you requested.')
-    else:
-        report('Nothing changed, not doing anything.')
+            report('Nothing changed, not doing anything.')
 
 
 if __name__ == '__main__':

modules/ocf_www/files/vhost-web-nginx.jinja

@@ -0,0 +1,66 @@
+# {{vhost.comment}}
+# CR-soon oliverni: move to 80/443
+
+{% if vhost.ssl %}
+server {
+    listen 8443 ssl http2;
+    listen [::]:8443 ssl http2;
+    server_name "{{vhost.fqdn}}";
+
+    ssl_certificate {{vhost.ssl.bundle}};
+    ssl_certificate_key {{vhost.ssl.key}};
+
+    add_header Strict-Transport-Security "max-age=31536000" always;
+
+    limit_req zone=web_ratelimit burst=20 nodelay;
+    limit_conn web_connlimit 50;
+
+    {% for name, value in vhost.response_headers.items() %}
+    add_header {{name}} "{{value}}" always;
+    {% endfor %}
+
+    location /.well-known/ {
+        alias /var/lib/lets-encrypt/.well-known/;
+    }
+
+    location / {
+        {% if vhost.is_redirect %}
+        return {{vhost.redirect_type}} {{vhost.redirect_dest}}$request_uri;
+        {% else %}
+        proxy_pass http://127.0.0.1:{{backend_port}};
+        proxy_set_header Host $host;
+        proxy_set_header X-Forwarded-For $remote_addr;
+        proxy_set_header X-Forwarded-Proto $scheme;
+        proxy_set_header X-Real-IP $remote_addr;
+        {% endif %}
+    }
+
+    access_log /var/log/nginx/vhost-access.log vhost;
+}
+{% endif %}
+
+{% if not vhost.ssl or vhost.is_redirect %}
+# HTTP (redirect or non-SSL)
+server {
+    listen 8080;
+    listen [::]:8080;
+    server_name "{{vhost.fqdn}}";
+
+    location /.well-known/ {
+        alias /var/lib/lets-encrypt/.well-known/;
+    }
+
+    location / {
+        {% if vhost.is_redirect %}
+        return {{vhost.redirect_type}} {{vhost.redirect_dest}}$request_uri;
+        {% elif vhost.ssl %}
+        return 301 {{vhost.canonical_url}}$request_uri;
+        {% else %}
+        proxy_pass http://127.0.0.1:{{backend_port}};
+        proxy_set_header Host $host;
+        proxy_set_header X-Forwarded-For $remote_addr;
+        proxy_set_header X-Real-IP $remote_addr;
+        {% endif %}
+    }
+}
+{% endif %}

modules/ocf_www/files/vhost-web.jinja

@@ -1,9 +1,11 @@
 # {{vhost.comment}}
-<VirtualHost *:{{vhost.port}}>
+{% set ports = [vhost.port, backend_port] if vhost.ssl else [vhost.port] %}
+{% for port in ports %}
+<VirtualHost *:{{port}}>
     ServerName {{vhost.fqdn}}
     ServerAdmin {{vhost.contact_email}}
 
-    {% if vhost.ssl %}
+    {% if vhost.ssl and port != backend_port %}
         # SSL
         SSLEngine on
         SSLCertificateFile {{vhost.ssl.bundle}}
@@ -27,7 +29,7 @@
         # Proxy to the local "unavailable" vhost, which serves up a friendly
         # "your website is rekt" page.
         RequestHeader set Host unavailable.ocf.berkeley.edu
-        ProxyPass / http://localhost/
+        ProxyPass / http://127.0.0.1/
     {% else %}
         DocumentRoot {{vhost.docroot}}
 
@@ -59,3 +61,4 @@
 
     UserDir disabled
 </VirtualHost>
+{% endfor %}

modules/ocf_www/manifests/init.pp

@@ -10,7 +10,14 @@
 #
 # The interesting config is in ocf_www::site::www, which sets up
 # www.ocf.berkeley.edu, which is by far the most complicated domain.
+#
+# Nginx sits in front of Apache for slowloris protection.
+# CR-soon oliverni: swap nginx to 80/443, apache to 127.0.0.1:$backend_port only
 class ocf_www {
+  # Port Apache listens on as nginx's backend (plain HTTP on localhost).
+  # Must match BACKEND_PORT in build-vhosts.
+  # Phase 2: make this the only Apache port and bind to 127.0.0.1.
+  $backend_port = 16767
   include ocf::acct
   include ocf::extrapackages
   include ocf::firewall::allow_web
@@ -26,6 +33,9 @@
     web  => false,
   }
 
+  # nginx reverse proxy (test ports for now)
+  include ocf_www::nginx
+
   class {
     '::apache':
       log_formats => {