Reorg

.github/workflows/test.yaml

@@ -8,9 +8,14 @@ jobs:
     strategy:
       matrix:
         include:
-          - dir: python-web-app
-            deploy-file: deploy.py
+          - dir: full-deploys/python-web-app
+            deploy-file: deploy_app.py
             inventory-file: inventories/docker.py
+
+          - dir: full-deploys/foundationdb-cluster
+            deploy-file: deploy_foundationdb.py
+            inventory-file: inventories/docker.py
+
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
@@ -19,10 +24,19 @@ jobs:
           python-version: "3.12"
       - uses: docker/setup-buildx-action@v2
       - run: pip install pyinfra --pre
+
       - run: ./docker-start.sh
         working-directory: ${{ matrix.dir }}
+
       - name: Run pyinfra deploy
         run: pyinfra -y ${{ matrix.inventory-file }} ${{ matrix.deploy-file }}
         working-directory: ${{ matrix.dir }}
+        if: ${{ !runner.debug }}
+
+      - name: Run pyinfra deploy (debug mode)
+        run: pyinfra -y -vvv ${{ matrix.inventory-file }} ${{ matrix.deploy-file }}
+        working-directory: ${{ matrix.dir }}
+        if: ${{ runner.debug }}
+
       - run: ./docker-stop.sh
         working-directory: ${{ matrix.dir }}

.gitignore

@@ -1 +1,2 @@
 .vagrant*
+settings.local.json

README.md

@@ -2,18 +2,32 @@
 
 A set of documented & tested pyinfra deploys.
 
-## Complete Deploys
+## Full Deploys
 
-### [`python-web-app`](./python-web-app)
+These are complete examples along with Docker test containers (acting as servers accessible via SSH)
+and scripts to execute them. These deploys are all tested as part of CI.
+
+### [`python-web-app`](./full-deploys/python-web-app)
 
 Simple: deploys two servers: one database and one web running a Python app fetched using Git.
 
-## Specific Features
+### [`foundationdb-cluster`](./full-deploys/foundationdb-cluster)
+
+Advanced: sets up a five node FoundationDB cluster from scratch. Uses nested operations (callbacks) to bootstrap.
+
+## Snippets
 
-### [`deploy-functions`](./deploy-functions)
+Shorter collections of code snippets that don't always function on their own but demonstrate various
+parts of pyinfra functionality.
+
+### [`deploy-functions`](./snippets/deploy-functions)
 
 Shows how to execute Python functions as operations directly from the CLI.
 
-### [`inventory-functions`](./inventory-functions)
+### [`inventory-functions`](./snippets/inventory-functions)
 
 Look at using Python functions, and external packages, to generate inventory.
+
+### [`nested-operations`](./snippets/nested-operations)
+
+Execute operations using output or results of other operations.

full-deploys/foundationdb-cluster/README.md

@@ -0,0 +1,35 @@
+# Example: FoundationDB Cluster
+
+This example deploys a 5-node FoundationDB cluster using pyinfra and Docker containers. Usage:
+
+```sh
+# Start Docker containers
+./docker-start.sh
+
+# Run pyinfra against them
+pyinfra inventories/docker.py deploy_foundationdb.py
+
+# Verify the cluster is health
+ssh -p 9022 -i ../../.docker/insecure_private_key pyinfra@localhost
+fdbcli --exec "status details"
+
+# Delete Docker containers
+./docker-stop.sh
+```
+
+## File Layout
+
+```
+foundationdb-cluster/
+├── deploy_foundationdb.py       # Main deployment entry point
+├── tasks/
+│   ├── install.py               # Package download and installation
+│   ├── configure.py             # Config files and service management
+│   └── bootstrap.py             # Cluster initialization
+├── templates/
+│   └── foundationdb.conf.j2     # fdbmonitor configuration template
+├── group_data/
+│   └── all.py                   # Shared settings for all hosts
+└── inventories/
+    └── docker.py                # Docker inventory with host data
+```

full-deploys/foundationdb-cluster/deploy_foundationdb.py

@@ -0,0 +1,40 @@
+"""
+FoundationDB Cluster Deployment
+
+Deploys a 5-node FoundationDB cluster with:
+- 3 coordinator nodes (nodes 1-3) for quorum
+- 5 storage nodes (all nodes)
+- Triple redundancy mode
+
+Usage:
+    pyinfra inventories/docker.py deploy_foundationdb.py
+"""
+
+from os import path
+
+from pyinfra import host, local
+from pyinfra.operations import apt
+
+# Update apt cache (with 1 hour cache time to avoid unnecessary updates)
+apt.packages(
+    name="Update apt cache",
+    packages=["wget"],
+    cache_time=3600,
+    update=True,
+)
+
+# Install FoundationDB packages
+local.include(
+    filename=path.join("tasks", "install.py"),
+)
+
+# Configure FoundationDB (config files, cluster file, service)
+local.include(
+    filename=path.join("tasks", "configure.py"),
+)
+
+# Bootstrap cluster (only on the designated bootstrap node)
+if host.data.get("is_bootstrap_node"):
+    local.include(
+        filename=path.join("tasks", "bootstrap.py"),
+    )

full-deploys/foundationdb-cluster/docker-start.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source "$(realpath "$(realpath "$(dirname "${BASH_SOURCE[0]}")")/../../utils.sh")"
+ensure_test_container
+
+export DOCKER_TEST_NETWORK_NAME="pyinfra-examples-foundationdb-cluster"
+
+echo "Create Docker network..."
+docker network create "$DOCKER_TEST_NETWORK_NAME"
+
+echo "Starting Docker containers..."
+run_test_container pyinfra-example-foundationdb-node-1 -p 9022:22 -p 4500:4500
+run_test_container pyinfra-example-foundationdb-node-2 -p 9023:22
+run_test_container pyinfra-example-foundationdb-node-3 -p 9024:22
+run_test_container pyinfra-example-foundationdb-node-4 -p 9025:22
+run_test_container pyinfra-example-foundationdb-node-5 -p 9026:22
+
+echo
+echo "Doker containers are now ready to run the pyinfra deploy, you can do this by running:"
+echo
+echo "    pyinfra inventories/docker.py deploy_foundationdb.py"
+echo
+echo "Once complete, don't forget to remove the Docker containers and network using the ./docker-stop.sh script!"

full-deploys/foundationdb-cluster/docker-stop.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source "$(realpath "$(realpath "$(dirname "${BASH_SOURCE[0]}")")/../../utils.sh")"
+
+export DOCKER_TEST_NETWORK_NAME="pyinfra-examples-foundationdb-cluster"
+
+docker rm -f pyinfra-example-foundationdb-node-1
+docker rm -f pyinfra-example-foundationdb-node-2
+docker rm -f pyinfra-example-foundationdb-node-3
+docker rm -f pyinfra-example-foundationdb-node-4
+docker rm -f pyinfra-example-foundationdb-node-5
+docker network rm "$DOCKER_TEST_NETWORK_NAME"

full-deploys/foundationdb-cluster/group_data/all.py

@@ -0,0 +1,12 @@
+fdb_version = "7.3.61"
+fdb_port = 4500
+fdb_coordinators = [
+    f"pyinfra-example-foundationdb-node-1.pyinfra-examples-foundationdb-cluster:{fdb_port + 1}",
+    f"pyinfra-example-foundationdb-node-2.pyinfra-examples-foundationdb-cluster:{fdb_port + 1}",
+    f"pyinfra-example-foundationdb-node-3.pyinfra-examples-foundationdb-cluster:{fdb_port + 1}",
+]
+
+fdb_cluster_name = "docker"
+fdb_cluster_id = "pyinfra123456789"  # Random cluster identifier
+fdb_redundancy_mode = "triple"
+fdb_storage_engine = "ssd-redwood-1"

full-deploys/foundationdb-cluster/inventories/docker.py

@@ -0,0 +1,24 @@
+inventory = (
+    # Individual host list with host-specific data
+    [
+        (
+            "node-1",
+            {"ssh_port": 9022, "is_coordinator": True, "is_bootstrap_node": True},
+        ),
+        ("node-2", {"ssh_port": 9023, "is_coordinator": True}),
+        ("node-3", {"ssh_port": 9024, "is_coordinator": True}),
+        ("node-4", {"ssh_port": 9025}),
+        ("node-5", {"ssh_port": 9026}),
+    ],
+    # Shared data for all the hosts in the group
+    {
+        "_sudo": True,  # use sudo for all operations
+        # SSH details matching the Docker container started in ./docker-start.sh
+        "ssh_hostname": "localhost",
+        "ssh_user": "pyinfra",
+        "ssh_key": "../../.docker/insecure_private_key",
+        "ssh_known_hosts_file": "/dev/null",
+        # This is insecure, don't use in production!
+        "ssh_strict_host_key_checking": "off",
+    },
+)

full-deploys/foundationdb-cluster/tasks/bootstrap.py

@@ -0,0 +1,46 @@
+"""
+FoundationDB cluster bootstrap tasks.
+Initializes the cluster configuration (runs only on bootstrap node).
+"""
+
+import json
+
+from pyinfra import host
+from pyinfra.operations import python, server
+
+
+def bootstrap_cluster():
+    """
+    Initialize the FoundationDB cluster if not already configured.
+    """
+
+    # Note: this command will take a while to execute because it has to timeout trying to get the
+    # status, I have yet to find a quick way to do this check.
+    check_status = server.shell(
+        name="Check cluster status",
+        commands="fdbcli --no-status --exec 'status json' || true",
+    )
+
+    status_json = json.loads(check_status.stdout)
+    status_client = status_json.get("client", {})
+
+    # Note this is, so far, the best way I can tell to determine if FDB cluster is bootstrapped,
+    # despite that for production I'd recommend any configure new commands are executed by hand.
+    if (
+        status_client.get("coordinators", {}).get("quorum_reachable") is True
+        and status_client.get("database_status", {}).get("available") is False
+    ):
+        redundancy_mode = host.data.fdb_redundancy_mode
+        storage_engine = host.data.fdb_storage_engine
+        server.shell(
+            name="Configure cluster",
+            commands=f"fdbcli --no-status --exec 'configure new {redundancy_mode} {storage_engine}'",
+        )
+
+
+# Use python.call to execute the bootstrap function at runtime
+# This ensures proper state checking during deployment
+python.call(
+    name="Bootstrap FoundationDB cluster",
+    function=bootstrap_cluster,
+)

full-deploys/foundationdb-cluster/tasks/configure.py

@@ -0,0 +1,74 @@
+"""
+FoundationDB configuration tasks.
+Manages config files, cluster file, and service state.
+"""
+
+from io import StringIO
+
+from pyinfra import host
+from pyinfra.operations import files, systemd
+
+# Build the cluster file connection string
+# Format: description:id@coordinator1,coordinator2,coordinator3
+cluster_description = host.data.fdb_cluster_name
+cluster_id = host.data.fdb_cluster_id
+coordinators = ",".join(host.data.fdb_coordinators)
+cluster_string = f"{cluster_description}:{cluster_id}@{coordinators}"
+
+# Ensure data directory exists with correct permissions
+files.directory(
+    name="Ensure FDB data directory exists",
+    path="/var/lib/foundationdb/data",
+    user="foundationdb",
+    group="foundationdb",
+    mode="0755",
+    present=True,
+)
+
+# Ensure log directory exists with correct permissions
+files.directory(
+    name="Ensure FDB log directory exists",
+    path="/var/log/foundationdb",
+    user="foundationdb",
+    group="foundationdb",
+    mode="0755",
+    present=True,
+)
+
+# Deploy foundationdb.conf from template
+config_changed = files.template(
+    name="Deploy foundationdb.conf",
+    src="templates/foundationdb.conf.j2",
+    dest="/etc/foundationdb/foundationdb.conf",
+    user="root",
+    group="root",
+    mode="0644",
+    fdb_port=host.data.fdb_port,
+    is_coordinator=host.data.get("is_coordinator", False),
+)
+
+# Deploy cluster file
+cluster_file_changed = files.put(
+    name="Deploy fdb.cluster file",
+    src=StringIO(cluster_string),
+    dest="/etc/foundationdb/fdb.cluster",
+    user="foundationdb",
+    group="foundationdb",
+    mode="0644",
+)
+
+# Ensure service is enabled and started
+systemd.service(
+    name="Enable and start foundationdb service",
+    service="foundationdb",
+    running=True,
+    enabled=True,
+)
+
+# Restart service if config changed
+systemd.service(
+    name="Restart foundationdb if config changed",
+    service="foundationdb",
+    restarted=True,
+    _if=lambda: config_changed.did_change() or cluster_file_changed.did_change(),
+)

full-deploys/foundationdb-cluster/tasks/install.py

@@ -0,0 +1,38 @@
+"""
+FoundationDB package installation tasks.
+Downloads and installs FDB .deb packages from GitHub releases.
+"""
+
+from pyinfra import host
+from pyinfra.facts.server import Arch
+from pyinfra.operations import apt, files, server
+
+fdb_version = host.data.fdb_version
+fdb_base_url = f"https://github.com/apple/foundationdb/releases/download/{fdb_version}"
+arch = host.get_fact(Arch)
+if arch == "x86_64":
+    arch = "amd64"
+
+# Download FDB packages
+files.download(
+    name="Download foundationdb-clients package",
+    src=f"{fdb_base_url}/foundationdb-clients_{fdb_version}-1_{arch}.deb",
+    dest=f"/tmp/foundationdb-clients_{fdb_version}-1_{arch}.deb",
+)
+
+files.download(
+    name="Download foundationdb-server package",
+    src=f"{fdb_base_url}/foundationdb-server_{fdb_version}-1_{arch}.deb",
+    dest=f"/tmp/foundationdb-server_{fdb_version}-1_{arch}.deb",
+)
+
+# Install packages (clients first, then server)
+apt.deb(
+    name="Install foundationdb-clients",
+    src=f"/tmp/foundationdb-clients_{fdb_version}-1_{arch}.deb",
+)
+
+apt.deb(
+    name="Install foundationdb-server",
+    src=f"/tmp/foundationdb-server_{fdb_version}-1_{arch}.deb",
+)