Generates TLS certificates dynamically in CI (#505)

close #500

Author: anarthal

.drone.star

@@ -75,6 +75,27 @@ def _find_package_b2_command(source_dir, generator):
                 '--generator="{}" '.format(generator)
 
 
+def _make_entrypoint(db):
+    if db.startswith('mysql:'):
+        # MySQL generic. Sanitize UNIX socket permissions and launch the server with the adequate TLS files
+        res = "chown -R mysql:mysql /var/run/mysqld && /usr/local/bin/docker-entrypoint.sh mysqld " + \
+                    "--ssl-ca=/tls/ca-cert.pem " + \
+                    "--ssl-cert=/tls/server-cert.pem " + \
+                    "--ssl-key=/tls/server-key.pem "
+        if db.startswith('mysql:8.'):
+            # v8.x needs this flag to enable mysql_native_password
+            res += "--mysql-native-password=ON"
+    else:
+        # MariaDB changed the default socket path, so we provide it explicitly
+        res = "chown -R mysql:mysql /var/run/mysqld && /usr/local/bin/docker-entrypoint.sh mariadbd " + \
+                    "--ssl-ca=/tls/ca-cert.pem " + \
+                    "--ssl-cert=/tls/server-cert.pem " + \
+                    "--ssl-key=/tls/server-key.pem " + \
+                    "--socket=/var/run/mysqld/mysqld.sock"
+
+    return res
+
+
 def _pipeline(
     name,
     image,
@@ -85,6 +106,20 @@ def _pipeline(
     disable_aslr=False
 ):
     steps = []
+
+    # Volumes, common to all steps
+    volumes = [
+        {
+            "name": "mysql-socket",
+            "path": "/var/run/mysqld"
+        },
+        {
+            "name": "tls-certificates",
+            "path": "/tls"
+        }
+    ] if db != None else []
+
+    # Disable ASLR
     if disable_aslr:
         steps.append({
             "name": "Disable ASLR",
@@ -93,15 +128,60 @@ def _pipeline(
             "privileged": True,
             "commands": ["echo 0 | tee /proc/sys/kernel/randomize_va_space"]
         })
+
+    # Set up the database and certificates
+    cert_dir = "C:\\\\ssl\\\\" if os == "windows" else "/tls/"
+    if os == "windows":
+        # Generate certificates
+        steps.append({
+            "name": "Generate certificates",
+            "image": image,
+            "pull": "if-not-exists",
+            "commands": [
+                "python tools/ci/gen-certificates.py {}".format(cert_dir)
+            ]
+        })
+
+    elif db != None:
+        # Generate certificates
+        steps.append({
+            "name": "Generate certificates",
+            "image": image,
+            "pull": "if-not-exists",
+            "volumes": volumes,
+            "commands": [
+                "python tools/ci/gen-certificates.py {}".format(cert_dir)
+            ]
+        })
+
+        # Database step
+        steps.append({
+            "name": "mysql",
+            "image": db,
+            "pull": "if-not-exists",
+            "detach": True,
+            "environment": {
+                "MYSQL_ALLOW_EMPTY_PASSWORD": "1",
+                "MYSQL_ROOT_PASSWORD": ""
+            },
+            "entrypoint": [
+                "/bin/bash",
+                "-c",
+                _make_entrypoint(db)
+            ],
+            "volumes": volumes
+        })
+    
+    # Run the build
     steps.append({
         "name": "Build and run",
         "image": image,
         "pull": "if-not-exists",
         "privileged": arch == "arm64", # TSAN tests fail otherwise (personality syscall)
-        "volumes":[{
-            "name": "mysql-socket",
-            "path": "/var/run/mysqld"
-        }] if db != None else [],
+        "volumes": volumes,
+        "environment": {
+            "BOOST_MYSQL_CA_CERTIFICATE": cert_dir + "ca-cert.pem"
+        },
         "commands": [command]
     })
 
@@ -119,18 +199,16 @@ def _pipeline(
         },
         "node": {},
         "steps": steps,
-        "services": [{
-            "name": "mysql",
-            "image": "ghcr.io/anarthal/cpp-ci-containers/{}".format(db),
-            "volumes": [{
+        "volumes": [
+            {
                 "name": "mysql-socket",
-                "path": "/var/run/mysqld"
-            }]
-        }] if db != None else [],
-        "volumes": [{
-            "name": "mysql-socket",
-            "temp": {}
-        }] if db != None else []
+                "temp": {}
+            },
+            {
+                "name": "tls-certificates",
+                "temp": {}
+            }
+        ]
     }
 
 
@@ -149,7 +227,7 @@ def linux_b2(
     valgrind=0,
     arch='amd64',
     fail_if_no_openssl=1,
-    db='mysql-8_4_1:1',
+    db='mysql:8.4.1',
 ):
     command = _b2_command(
         source_dir='$(pwd)',
@@ -201,7 +279,7 @@ def windows_b2(
 def linux_cmake(
     name,
     image,
-    db='mysql-8_4_1:1',
+    db='mysql:8.4.1',
     build_shared_libs=0,
     cmake_build_type='Debug',
     cxxstd='20',
@@ -270,7 +348,7 @@ def bench(name):
                 '--server-host=mysql ' + \
                 '--connection-pool-iters=1 ' + \
                 '--protocol-iters=1 '
-    return _pipeline(name=name, image=_image('build-bench:1'), os='linux', command=command, db='mysql-8_4_1:1')
+    return _pipeline(name=name, image=_image('build-bench:1'), os='linux', command=command, db='mysql:8.4.1')
 
 
 def docs(name):
@@ -286,8 +364,8 @@ def docs(name):
 def main(ctx):
     return [
         # CMake Linux
-        linux_cmake('Linux CMake MySQL 5.x',      _image('build-gcc14:1'), db='mysql-5_7_41:1',   build_shared_libs=0),
-        linux_cmake('Linux CMake MariaDB',        _image('build-gcc14:1'), db='mariadb-11_4_2:1', build_shared_libs=1),
+        linux_cmake('Linux CMake MySQL 5.x',      _image('build-gcc14:1'), db='mysql:5.7.41',   build_shared_libs=0),
+        linux_cmake('Linux CMake MariaDB',        _image('build-gcc14:1'), db='mariadb:11.4.2', build_shared_libs=1),
         linux_cmake('Linux CMake cmake 3.8',      _image('build-cmake3_8:3'), cxxstd='11', install_test=0),
         linux_cmake('Linux CMake gcc Release',    _image('build-gcc14:1'), cmake_build_type='Release'),
         linux_cmake('Linux CMake gcc MinSizeRel', _image('build-gcc14:1'), cmake_build_type='MinSizeRel'),
@@ -311,14 +389,14 @@ def main(ctx):
         #   Ubuntu 24.04: gcc13, clang 18
         linux_b2('Linux B2 clang-4',              _image('build-clang4:1'),        toolset='clang-4',   cxxstd='14'),
         linux_b2('Linux B2 clang-5-honly-dbg',    _image('build-clang5:1'),        toolset='clang-5',   cxxstd='14', separate_compilation=0),
-        linux_b2('Linux B2 clang-6',              _image('build-clang5:1'),        toolset='clang-5',   cxxstd='14'),
+        linux_b2('Linux B2 clang-6',              _image('build-clang6:1'),        toolset='clang-6',   cxxstd='14'),
         linux_b2('Linux B2 clang-7',              _image('build-clang7:2'),        toolset='clang-7',   cxxstd='14,17'),
         linux_b2('Linux B2 clang-8',              _image('build-clang8:2'),        toolset='clang-8',   cxxstd='14', variant='debug', address_sanitizer=1, undefined_sanitizer=1),
         linux_b2('Linux B2 clang-9',              _image('build-clang9:2'),        toolset='clang-9',   cxxstd='17', variant='release'),
         linux_b2('Linux B2 clang-10',             _image('build-clang10:2'),       toolset='clang-10',  cxxstd='17,20', variant='debug'),
         linux_b2('Linux B2 clang-11',             _image('build-clang11:2'),       toolset='clang-11',  cxxstd='20'),
         linux_b2('Linux B2 clang-12',             _image('build-clang12:2'),       toolset='clang-12',  cxxstd='20', variant='debug', stdlib='libc++', address_sanitizer=1, undefined_sanitizer=1),
-        linux_b2('Linux B2 clang-13',             _image('build-clang13:1'),       toolset='clang-13',  cxxstd='20', db='mysql-9_4_0:1'),
+        linux_b2('Linux B2 clang-13',             _image('build-clang13:1'),       toolset='clang-13',  cxxstd='20', db='mysql:9.4.0'),
         linux_b2('Linux B2 clang-14',             _image('build-clang14:1'),       toolset='clang-14',  cxxstd='20', variant='debug'),
         linux_b2('Linux B2 clang-15',             _image('build-clang15:1'),       toolset='clang-15',  cxxstd='20', variant='debug'),
         linux_b2('Linux B2 clang-16',             _image('build-clang16:1'),       toolset='clang-16',  cxxstd='20', variant='debug', address_sanitizer=1, undefined_sanitizer=1),
@@ -338,7 +416,7 @@ def main(ctx):
         linux_b2('Linux B2 gcc-10',               _image('build-gcc10:1'),         toolset='gcc-10',    cxxstd='17'),
         linux_b2('Linux B2 gcc-11',               _image('build-gcc11:1'),         toolset='gcc-11',    cxxstd='20'),
         linux_b2('Linux B2 gcc-12',               _image('build-gcc12:1'),         toolset='gcc-12',    cxxstd='20,23', variant='debug'),
-        linux_b2('Linux B2 gcc-13',               _image('build-gcc13:1'),         toolset='gcc-13',    cxxstd='20', db='mysql-9_4_0:1'),
+        linux_b2('Linux B2 gcc-13',               _image('build-gcc13:1'),         toolset='gcc-13',    cxxstd='20', db='mysql:9.4.0'),
         linux_b2('Linux B2 gcc-14',               _image('build-gcc14:1'),         toolset='gcc-14',    cxxstd='23'),
         linux_b2('Linux B2 gcc-15',               _image('build-gcc15:1'),         toolset='gcc-15',    cxxstd='23'),
         linux_b2('Linux B2 gcc-sanit',            _image('build-gcc14:1'),         toolset='gcc-14',    cxxstd='23', variant='debug', address_sanitizer=1, undefined_sanitizer=1),

.github/workflows/build-code.yml

@@ -10,16 +10,15 @@ name: Build
 on:
   push:
     branches: [develop, master]
-    tags: ['*']
+    tags: ["*"]
   pull_request:
   workflow_dispatch:
 
-
 jobs:
   osx:
     runs-on: macos-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
       - run: |
           unlink /usr/local/bin/python || echo "/usr/local/bin/python not found"
           ln -s /usr/local/bin/python3 /usr/local/bin/python

.github/workflows/coverage.yml

@@ -17,27 +17,26 @@ on:
 jobs:
   coverage:
     runs-on: ubuntu-latest
-    container:
-      image: ghcr.io/anarthal/cpp-ci-containers/build-gcc14-lcov:1
-      volumes:
-        - /var/run/mysqld:/var/run/mysqld
-    services:
-      mysql:
-        image: ghcr.io/anarthal/cpp-ci-containers/mysql-8_4_1:1
-        ports:
-          - 3306:3306
-        volumes:
-          - /var/run/mysqld:/var/run/mysqld
     steps:
       - name: Fetch code
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
+
+      - name: Generate certificates
+        run: python tools/ci/gen-certificates.py /tmp/mysql-tls
+
+      - name: Start containers
+        uses: hoverkraft-tech/compose-action@v2.6.0
+        with:
+          compose-file: ./tools/ci/docker-compose.yml
+        env:
+          BUILDER_IMAGE: ghcr.io/anarthal/cpp-ci-containers/build-gcc14-lcov:1
 
       - name: Build code
         run: |
-          python tools/ci/main.py \
-            --source-dir=$(pwd) \
+          docker exec builder python /boost-mysql/tools/ci/main.py \
+            --source-dir=/boost-mysql \
             b2 \
-            --server-host=mysql \
+            --server-host=localhost \
             --toolset=gcc \
             --cxxstd=20 \
             --variant=debug \
@@ -47,29 +46,29 @@ jobs:
       - name: Generate coverage reports
         shell: bash
         run: |
-          cd ~/boost-root/bin.v2
-          lcov \
+          docker exec builder lcov \
             --rc branch_coverage=0 \
             --rc geninfo_unexecuted_blocks=1 \
             --ignore-errors mismatch \
             --gcov-tool gcov-14 \
-            --directory . \
+            --directory /root/boost-root/bin.v2 \
             --capture \
             --output-file all.info
-          lcov \
+          docker exec builder lcov \
             --rc branch_coverage=0 \
             --output-file coverage.info \
             --extract all.info '*/boost/mysql*'
-          sed "s|^SF:$HOME/boost-root/|SF:include/|g" coverage.info > $GITHUB_WORKSPACE/coverage.info
+          docker exec builder sed -i "s|^SF:/root/boost-root/|SF:include/|g" coverage.info
+          docker exec builder mv coverage.info /boost-mysql/coverage.info
 
       - name: Upload coverage reports
-        uses: codecov/codecov-action@v4
+        uses: codecov/codecov-action@v6
         with:
           verbose: true
           fail_ci_if_error: true
           token: ${{ secrets.CODECOV_TOKEN }}
           plugins: noop # Don't run gcov again, codecov doesn't know about the filtering we perform
-          file: coverage.info
+          files: coverage.info
           disable_search: true # Don't upload unwanted files
           disable_file_fixes: true # Default fixes make reports unusable
 

.github/workflows/fuzz.yml

@@ -10,7 +10,7 @@ name: fuzz
 on:
   push:
     branches: [develop, master]
-    tags: ['*']
+    tags: ["*"]
   pull_request:
   workflow_dispatch:
   schedule:
@@ -19,43 +19,46 @@ on:
 jobs:
   fuzz:
     runs-on: ubuntu-latest
-    container:
-      image: ghcr.io/anarthal/cpp-ci-containers/build-clang18:1
-      volumes:
-        - /var/run/mysqld:/var/run/mysqld
-    services:
-      mysql:
-        image: ghcr.io/anarthal/cpp-ci-containers/mysql-8_4_1:1
-        ports:
-          - 3306:3306
-        volumes:
-          - /var/run/mysqld:/var/run/mysqld
     steps:
       - name: Fetch code
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
+
+      - name: Generate certificates
+        run: python tools/ci/gen-certificates.py /tmp/mysql-tls
+
+      - name: Start containers
+        uses: hoverkraft-tech/compose-action@v2.6.0
+        with:
+          compose-file: ./tools/ci/docker-compose.yml
+        env:
+          BUILDER_IMAGE: ghcr.io/anarthal/cpp-ci-containers/build-clang18:1
 
       - name: Restore corpus
-        uses: actions/cache@v4
+        uses: actions/cache@v5
         with:
           path: /tmp/corpus.tar.gz
           key: corpus-${{ github.run_id }}
           restore-keys: corpus-
-        
+
       # Note: this will take care of using the corpus and updating it
       - name: Build and run the fuzzer
         run: |
-          python tools/ci/main.py \
-            --source-dir=$(pwd) \
-            fuzz \
-            --server-host=mysql
+          docker exec builder python /boost-mysql/tools/ci/main.py \
+            --source-dir=/boost-mysql \
+            fuzz
+
+      - name: Copy crashes from container
+        if: always()
+        run: |
+          docker exec builder bash -c 'cp /root/boost-root/crash-* /root/boost-root/leak-* /root/boost-root/timeout-* /boost-mysql/ || true'
 
       - name: Archive any crashes as an artifact
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v7
         if: always()
         with:
           name: crashes
           path: |
-            ~/boost-root/crash-*
-            ~/boost-root/leak-*
-            ~/boost-root/timeout-*
+            crash-*
+            leak-*
+            timeout-*
           if-no-files-found: ignore