diff --git a/.asf.yaml b/.asf.yaml index 58590f16..34573b81 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -19,7 +19,7 @@ github: features: issues: true projects: false - + ghp_branch: gh-pages ghp_path: / enabled_merge_buttons: @@ -40,7 +40,7 @@ github: required_approving_review_count: 1 collaborators: - daniel-hutao - + notifications: commits: commits@devlake.apache.org issues: commits@devlake.apache.org diff --git a/.github/.licenserc.yaml b/.github/.licenserc.yaml new file mode 100644 index 00000000..80d0db6d --- /dev/null +++ b/.github/.licenserc.yaml @@ -0,0 +1,26 @@ +header: + license: + spdx-id: Apache-2.0 + copyright-owner: Apache Software Foundation + + paths-ignore: + - '**/*.md' + - '**/*.json' + - '**/*.txt' + - '**/LICENSE' + - '**/NOTICE' + - '.gitignore' + - '.gitmodules' + - 'charts/**/charts/**' + - 'charts/**/Chart.lock' + - '.github/.licenserc.yaml' + - '.github/dependabot.yml' + - '.github/labeler.yml' + - '.github/actions/**/action.yml' + - '.licenserc.yaml' + - '.mise.toml' + - '.pre-commit-config.yaml' + - '.yamllint' + - '**/*.gotmpl' + + comment: on-failure diff --git a/.github/actions/chart-releaser-action b/.github/actions/chart-releaser-action deleted file mode 160000 index be16258d..00000000 --- a/.github/actions/chart-releaser-action +++ /dev/null @@ -1 +0,0 @@ -Subproject commit be16258da8010256c6e82849661221415f031968 diff --git a/.github/actions/kind-cluster/action.yml b/.github/actions/kind-cluster/action.yml new file mode 100644 index 00000000..07680e8d --- /dev/null +++ b/.github/actions/kind-cluster/action.yml @@ -0,0 +1,35 @@ +name: Create kind Cluster +description: Creates a kind Kubernetes cluster with diagnostics on failure + +inputs: + cluster_name: + description: 'Name of the kind cluster to create' + required: true + namespace: + description: 'Kubernetes namespace to create (optional)' + required: false + default: '' + +runs: + using: composite + steps: + - name: Create kind cluster + run: | + kind create cluster --name ${{ inputs.cluster_name }} + shell: bash + + - name: Cluster information + run: | + kubectl cluster-info + kubectl get nodes + kubectl get pods -n kube-system + helm version + kubectl version + kubectl get storageclasses + shell: bash + + - name: Create namespace + if: inputs.namespace != '' + run: | + kubectl create namespace ${{ inputs.namespace }} || true + shell: bash diff --git a/.github/actions/setup-environment/action.yml b/.github/actions/setup-environment/action.yml new file mode 100644 index 00000000..f73c37e4 --- /dev/null +++ b/.github/actions/setup-environment/action.yml @@ -0,0 +1,39 @@ +name: Setup DevLake Environment +description: Sets up mise, Helm, and chart dependencies for DevLake workflows + +inputs: + charts_dir: + description: 'Path to charts directory' + required: false + default: 'charts/devlake' + +runs: + using: composite + steps: + - name: Install mise + uses: jdx/mise-action@1648a7812b9aeae629881980618f079932869151 # v4 + + - name: Install tools via mise + run: mise install + shell: bash + + - name: Cache Helm dependencies + uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4 + with: + path: | + ~/.cache/helm + ${{ inputs.charts_dir }}/charts/ + key: helm-deps-${{ hashFiles(format('{0}/Chart.lock', inputs.charts_dir)) }} + restore-keys: | + helm-deps- + + - name: Add Helm repositories + run: | + helm repo add grafana https://grafana.github.io/helm-charts + helm repo update + shell: bash + + - name: Build chart dependencies + run: | + helm dependency build ${{ inputs.charts_dir }} + shell: bash diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..a37bbee4 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,9 @@ +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + groups: + actions: + patterns: ["*"] diff --git a/.github/labeler.yml b/.github/labeler.yml new file mode 100644 index 00000000..8c18fbca --- /dev/null +++ b/.github/labeler.yml @@ -0,0 +1,15 @@ +'area/charts': + - changed-files: + - any-glob-to-any-file: 'charts/**' + +'area/workflows': + - changed-files: + - any-glob-to-any-file: '.github/workflows/**' + +'area/actions': + - changed-files: + - any-glob-to-any-file: '.github/actions/**' + +'area/docs': + - changed-files: + - any-glob-to-any-file: '**/*.md' diff --git a/.github/scripts/e2e-smoke-test.sh b/.github/scripts/e2e-smoke-test.sh new file mode 100755 index 00000000..da932bcf --- /dev/null +++ b/.github/scripts/e2e-smoke-test.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash +# E2E smoke test - verify DevLake API and UI endpoints are accessible +set -euo pipefail + +NAMESPACE="${NAMESPACE:-default}" +SERVICE_PORT="${SERVICE_PORT:-30000}" +TIMEOUT="${TIMEOUT:-300}" +MAX_RETRIES="${MAX_RETRIES:-10}" +RETRY_INTERVAL="${RETRY_INTERVAL:-3}" + +echo "=== DevLake E2E Smoke Test ===" +echo "Namespace: $NAMESPACE" +echo "Service Port: $SERVICE_PORT" +echo "Timeout: ${TIMEOUT}s" +echo "Max Retries: $MAX_RETRIES" +echo "" + +# Get node IP +echo "๐Ÿ” Getting node IP address..." +NODE_IP=$(kubectl get nodes --namespace "$NAMESPACE" -o jsonpath="{.items[0].status.addresses[0].address}") +echo "โœ… Node IP: $NODE_IP" +echo "" + +BASE_URL="http://${NODE_IP}:${SERVICE_PORT}" + +# Test function with retry logic +test_endpoint() { + local name="$1" + local url="$2" + local check_pattern="${3:-}" + + echo "๐Ÿ“ก Testing: $name" + echo " URL: $url" + + for attempt in $(seq 1 "$MAX_RETRIES"); do + if [[ $attempt -gt 1 ]]; then + echo " Retry $((attempt-1))/$((MAX_RETRIES-1)) after ${RETRY_INTERVAL}s..." + sleep "$RETRY_INTERVAL" + fi + + if response=$(curl --silent --show-error --fail --max-time 10 "$url" 2>&1); then + # If check_pattern provided, verify response contains it + if [[ -n "$check_pattern" ]]; then + if echo "$response" | grep -q "$check_pattern"; then + echo "โœ… $name: OK (pattern matched)" + return 0 + else + echo " โš ๏ธ Response received but pattern not found" + continue + fi + else + echo "โœ… $name: OK" + return 0 + fi + else + echo " โš ๏ธ Attempt $attempt failed" + fi + done + + echo "โŒ $name: FAILED after $MAX_RETRIES attempts" + return 1 +} + +# Test 1: Home page +echo "Test 1: DevLake Home Page" +test_endpoint "Home Page" "$BASE_URL" +echo "" + +# Test 2: DevLake API health +echo "Test 2: DevLake API - Blueprints Endpoint" +test_endpoint "Blueprints API" "$BASE_URL/api/blueprints" +echo "" + +# Test 3: Grafana API health +echo "Test 3: Grafana API Health" +test_endpoint "Grafana Health" "$BASE_URL/grafana/api/health" "database" +echo "" + +# Test 4: Additional DevLake API endpoints +echo "Test 4: DevLake Version Info" +test_endpoint "Version Info" "$BASE_URL/api/version" +echo "" + +echo "Test 5: DevLake Plugins Endpoint" +test_endpoint "Plugins API" "$BASE_URL/api/plugins" +echo "" + +echo "๐ŸŽ‰ All E2E smoke tests passed!" diff --git a/.github/scripts/mysql-smoke-test.sh b/.github/scripts/mysql-smoke-test.sh new file mode 100755 index 00000000..3ab7716a --- /dev/null +++ b/.github/scripts/mysql-smoke-test.sh @@ -0,0 +1,106 @@ +#!/usr/bin/env bash +# MySQL smoke test - verify basic MySQL connectivity and operations +set -euo pipefail + +RELEASE_NAME="${RELEASE_NAME:-devlake-mysql}" +NAMESPACE="${NAMESPACE:-devlake-mysql}" +TIMEOUT="${TIMEOUT:-300}" + +echo "=== MySQL Smoke Test ===" +echo "Release: $RELEASE_NAME" +echo "Namespace: $NAMESPACE" +echo "Timeout: ${TIMEOUT}s" +echo "" + +# Wait for MySQL pod to be ready +echo "โณ Waiting for MySQL pod to be ready..." +kubectl wait --for=condition=ready pod \ + -l "app.kubernetes.io/instance=$RELEASE_NAME,devlakeComponent=mysql" \ + -n "$NAMESPACE" \ + --timeout="${TIMEOUT}s" + +POD_NAME=$(kubectl get pod -l "app.kubernetes.io/instance=$RELEASE_NAME,devlakeComponent=mysql" \ + -n "$NAMESPACE" -o jsonpath='{.items[0].metadata.name}') + +echo "โœ… Pod ready: $POD_NAME" +echo "" + +# Get MySQL root password from secret +echo "๐Ÿ” Retrieving MySQL credentials..." +ROOT_PASSWORD=$(kubectl get secret "${RELEASE_NAME}-db-auth" \ + -n "$NAMESPACE" \ + -o jsonpath='{.data.MYSQL_ROOT_PASSWORD}' | base64 -d) + +# Test 1: Basic connectivity +echo "๐Ÿ“ก Test 1: Basic MySQL connectivity..." +kubectl exec -n "$NAMESPACE" "$POD_NAME" -- \ + mysqladmin ping -uroot -p"$ROOT_PASSWORD" 2>&1 | grep -q "mysqld is alive" +echo "โœ… MySQL is alive" +echo "" + +# Test 2: Query system info +echo "๐Ÿ” Test 2: Query MySQL version..." +VERSION=$(kubectl exec -n "$NAMESPACE" "$POD_NAME" -- \ + mysql -uroot -p"$ROOT_PASSWORD" -e "SELECT VERSION();" -sN) +echo "โœ… MySQL version: $VERSION" +echo "" + +# Test 3: Check character set +echo "๐Ÿ”ค Test 3: Verify character set configuration..." +CHARSET=$(kubectl exec -n "$NAMESPACE" "$POD_NAME" -- \ + mysql -uroot -p"$ROOT_PASSWORD" -e "SHOW VARIABLES LIKE 'character_set_server';" -sN | awk '{print $2}') +COLLATION=$(kubectl exec -n "$NAMESPACE" "$POD_NAME" -- \ + mysql -uroot -p"$ROOT_PASSWORD" -e "SHOW VARIABLES LIKE 'collation_server';" -sN | awk '{print $2}') + +if [[ "$CHARSET" != "utf8mb4" ]]; then + echo "โŒ Character set is $CHARSET, expected utf8mb4" + exit 1 +fi + +if [[ "$COLLATION" != "utf8mb4_bin" ]]; then + echo "โŒ Collation is $COLLATION, expected utf8mb4_bin" + exit 1 +fi + +echo "โœ… Character set: $CHARSET, Collation: $COLLATION" +echo "" + +# Test 4: Create test database +echo "๐Ÿ—„๏ธ Test 4: Create and drop test database..." +kubectl exec -n "$NAMESPACE" "$POD_NAME" -- \ + mysql -uroot -p"$ROOT_PASSWORD" -e "CREATE DATABASE IF NOT EXISTS smoke_test;" + +kubectl exec -n "$NAMESPACE" "$POD_NAME" -- \ + mysql -uroot -p"$ROOT_PASSWORD" -e "SHOW DATABASES;" | grep -q "smoke_test" + +kubectl exec -n "$NAMESPACE" "$POD_NAME" -- \ + mysql -uroot -p"$ROOT_PASSWORD" -e "DROP DATABASE smoke_test;" + +echo "โœ… Database operations successful" +echo "" + +# Test 5: Check devlake database exists +echo "๐Ÿ” Test 5: Check devlake database..." +DB_USER=$(kubectl get secret "${RELEASE_NAME}-db-auth" \ + -n "$NAMESPACE" \ + -o jsonpath='{.data.MYSQL_USER}' | base64 -d) + +kubectl exec -n "$NAMESPACE" "$POD_NAME" -- \ + mysql -uroot -p"$ROOT_PASSWORD" -e "SHOW DATABASES;" | grep -q "lake" + +echo "โœ… Devlake database 'lake' exists" +echo "" + +# Test 6: Verify user can connect +echo "๐Ÿ‘ค Test 6: Verify devlake user connectivity..." +DB_PASSWORD=$(kubectl get secret "${RELEASE_NAME}-db-auth" \ + -n "$NAMESPACE" \ + -o jsonpath='{.data.MYSQL_PASSWORD}' | base64 -d) + +kubectl exec -n "$NAMESPACE" "$POD_NAME" -- \ + mysql -u"$DB_USER" -p"$DB_PASSWORD" -D lake -e "SELECT 1;" -sN | grep -q "1" + +echo "โœ… User $DB_USER can connect to database" +echo "" + +echo "๐ŸŽ‰ All MySQL smoke tests passed!" diff --git a/.github/scripts/postgresql-smoke-test.sh b/.github/scripts/postgresql-smoke-test.sh new file mode 100755 index 00000000..a175588c --- /dev/null +++ b/.github/scripts/postgresql-smoke-test.sh @@ -0,0 +1,111 @@ +#!/usr/bin/env bash +# PostgreSQL smoke test - verify basic PostgreSQL connectivity and operations +set -euo pipefail + +RELEASE_NAME="${RELEASE_NAME:-devlake-postgresql}" +NAMESPACE="${NAMESPACE:-devlake-postgresql}" +TIMEOUT="${TIMEOUT:-300}" + +echo "=== PostgreSQL Smoke Test ===" +echo "Release: $RELEASE_NAME" +echo "Namespace: $NAMESPACE" +echo "Timeout: ${TIMEOUT}s" +echo "" + +# Wait for PostgreSQL pod to be ready +echo "โณ Waiting for PostgreSQL pod to be ready..." +kubectl wait --for=condition=ready pod \ + -l "app.kubernetes.io/instance=$RELEASE_NAME,devlakeComponent=postgresql" \ + -n "$NAMESPACE" \ + --timeout="${TIMEOUT}s" + +POD_NAME=$(kubectl get pod -l "app.kubernetes.io/instance=$RELEASE_NAME,devlakeComponent=postgresql" \ + -n "$NAMESPACE" -o jsonpath='{.items[0].metadata.name}') + +echo "โœ… Pod ready: $POD_NAME" +echo "" + +# Get PostgreSQL password from secret +echo "๐Ÿ” Retrieving PostgreSQL credentials..." +DB_PASSWORD=$(kubectl get secret "${RELEASE_NAME}-db-auth" \ + -n "$NAMESPACE" \ + -o jsonpath='{.data.POSTGRES_PASSWORD}' | base64 -d) + +DB_USER=$(kubectl get secret "${RELEASE_NAME}-db-auth" \ + -n "$NAMESPACE" \ + -o jsonpath='{.data.POSTGRES_USER}' | base64 -d) + +# Test 1: Basic connectivity +echo "๐Ÿ“ก Test 1: Basic PostgreSQL connectivity..." +kubectl exec -n "$NAMESPACE" "$POD_NAME" -- \ + pg_isready -U "$DB_USER" 2>&1 | grep -q "accepting connections" +echo "โœ… PostgreSQL is accepting connections" +echo "" + +# Test 2: Query system info +echo "๐Ÿ” Test 2: Query PostgreSQL version..." +VERSION=$(kubectl exec -n "$NAMESPACE" "$POD_NAME" -- \ + psql -U "$DB_USER" -d lake -t -c "SELECT version();" | head -1 | xargs) +echo "โœ… PostgreSQL version: ${VERSION:0:60}..." +echo "" + +# Test 3: Check encoding +echo "๐Ÿ”ค Test 3: Verify database encoding..." +ENCODING=$(kubectl exec -n "$NAMESPACE" "$POD_NAME" -- \ + psql -U "$DB_USER" -d lake -t -c "SHOW server_encoding;" | xargs) + +if [[ "$ENCODING" != "UTF8" ]]; then + echo "โŒ Encoding is $ENCODING, expected UTF8" + exit 1 +fi + +echo "โœ… Encoding: $ENCODING" +echo "" + +# Test 4: Create test table and data +echo "๐Ÿ—„๏ธ Test 4: Create and drop test table..." +kubectl exec -n "$NAMESPACE" "$POD_NAME" -- \ + psql -U "$DB_USER" -d lake -c "CREATE TABLE IF NOT EXISTS smoke_test (id SERIAL PRIMARY KEY, data TEXT);" + +kubectl exec -n "$NAMESPACE" "$POD_NAME" -- \ + psql -U "$DB_USER" -d lake -c "INSERT INTO smoke_test (data) VALUES ('test');" + +COUNT=$(kubectl exec -n "$NAMESPACE" "$POD_NAME" -- \ + psql -U "$DB_USER" -d lake -t -c "SELECT COUNT(*) FROM smoke_test;" | xargs) + +if [[ "$COUNT" != "1" ]]; then + echo "โŒ Expected 1 row, got $COUNT" + exit 1 +fi + +kubectl exec -n "$NAMESPACE" "$POD_NAME" -- \ + psql -U "$DB_USER" -d lake -c "DROP TABLE smoke_test;" + +echo "โœ… Table operations successful" +echo "" + +# Test 5: Check devlake database exists +echo "๐Ÿ” Test 5: Check devlake database..." +kubectl exec -n "$NAMESPACE" "$POD_NAME" -- \ + psql -U "$DB_USER" -d postgres -t -c "\l lake" | grep -q "lake" + +echo "โœ… Devlake database 'lake' exists" +echo "" + +# Test 6: Verify user permissions +echo "๐Ÿ‘ค Test 6: Verify user has necessary permissions..." +kubectl exec -n "$NAMESPACE" "$POD_NAME" -- \ + psql -U "$DB_USER" -d lake -t -c "SELECT 1;" | grep -q "1" + +echo "โœ… User $DB_USER has database access" +echo "" + +# Test 7: Check connection limit +echo "๐Ÿ”Œ Test 7: Check connection settings..." +MAX_CONN=$(kubectl exec -n "$NAMESPACE" "$POD_NAME" -- \ + psql -U "$DB_USER" -d lake -t -c "SHOW max_connections;" | xargs) + +echo "โœ… Max connections: $MAX_CONN" +echo "" + +echo "๐ŸŽ‰ All PostgreSQL smoke tests passed!" diff --git a/.github/workflows/deploy-test.yml b/.github/workflows/deploy-test.yml index fc2ea272..d34bb0c4 100644 --- a/.github/workflows/deploy-test.yml +++ b/.github/workflows/deploy-test.yml @@ -32,54 +32,160 @@ on: - "!**.md" workflow_dispatch: +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: deploy-with-helm: runs-on: ubuntu-latest + timeout-minutes: 45 strategy: fail-fast: false matrix: - database_type: ["mysql-builtin", "mysql-external"] + database_type: ["mysql-builtin", "mysql-external", "postgresql-builtin", "postgresql-external"] steps: - - name: Creating kind cluster - uses: container-tools/kind-action@v1 + - name: Checkout + uses: actions/checkout@v6.0.2 + + - name: Setup environment + uses: ./.github/actions/setup-environment - - name: Cluster information + - name: Create kind cluster + uses: ./.github/actions/kind-cluster + with: + cluster_name: devlake-${{ matrix.database_type }} + + - name: Install external MySQL + if: matrix.database_type == 'mysql-external' run: | - kubectl cluster-info - kubectl get nodes - kubectl get pods -n kube-system - helm version - kubectl version - kubectl get storageclasses + DB_ROOT_PASSWORD=$(openssl rand -base64 16) + DB_PASSWORD=$(openssl rand -base64 16) + echo "::add-mask::${DB_ROOT_PASSWORD}" + echo "::add-mask::${DB_PASSWORD}" - - name: Checkout - uses: actions/checkout@v2 + # Retry logic for helm repo add (network flakiness) + for i in {1..3}; do + if helm repo add bitnami https://charts.bitnami.com/bitnami; then + break + else + sleep $((2**i)) + fi + done + + helm install mysql bitnami/mysql --version 9.19.1 \ + --set auth.rootPassword="${DB_ROOT_PASSWORD}" \ + --set auth.database=lake \ + --set auth.username=devlake \ + --set auth.password="${DB_PASSWORD}" + + - name: Install external PostgreSQL + if: matrix.database_type == 'postgresql-external' + run: | + DB_ROOT_PASSWORD=$(openssl rand -base64 16) + DB_PASSWORD=$(openssl rand -base64 16) + echo "::add-mask::${DB_ROOT_PASSWORD}" + echo "::add-mask::${DB_PASSWORD}" + + # Retry logic for helm repo add (network flakiness) + for i in {1..3}; do + if helm repo add bitnami https://charts.bitnami.com/bitnami; then + break + else + sleep $((2**i)) + fi + done + + helm install postgresql bitnami/postgresql --version 12.12.10 \ + --set auth.postgresPassword="${DB_ROOT_PASSWORD}" \ + --set auth.database=lake \ + --set auth.username=devlake \ + --set auth.password="${DB_PASSWORD}" - - name: Helm install devlake + - name: Install DevLake (MySQL external) if: matrix.database_type == 'mysql-external' run: | - helm repo add bitnami https://charts.bitnami.com/bitnami - helm repo add grafana https://grafana.github.io/helm-charts - helm install mysql bitnami/mysql --version 9.19.1 --set auth.rootPassword=admin --set auth.database=lake --set auth.username=merico --set auth.password=merico - # external mysql at service: mysql - helm dep build charts/devlake + DB_PASSWORD=$(openssl rand -base64 16) + ENCRYPTION_SECRET=$(openssl rand -base64 2000 | tr -dc '[:upper:]' | fold -w 128 | head -n 1) + echo "::add-mask::${DB_PASSWORD}" + echo "::add-mask::${ENCRYPTION_SECRET}" + NODE_IP=$(kubectl get nodes --namespace default -o jsonpath="{.items[0].status.addresses[0].address}") + echo "Node IP: ${NODE_IP}" helm install --debug --wait --timeout 2400s deploy-test charts/devlake \ --set service.uiPort=30000 \ - --set mysql.useExternal=true \ - --set mysql.externalServer=mysql \ - --set lake.encryptionSecret.secret=$(openssl rand -base64 2000 | tr -dc 'A-Z' | fold -w 128 | head -n 1) + --set database.type=mysql \ + --set database.useExternal=true \ + --set database.externalServer=mysql \ + --set database.externalPort=3306 \ + --set database.username=devlake \ + --set database.password="${DB_PASSWORD}" \ + --set database.database=lake \ + --set lake.encryptionSecret.secret="${ENCRYPTION_SECRET}" - - name: Helm install devlake + - name: Install DevLake (MySQL builtin) if: matrix.database_type == 'mysql-builtin' run: | - helm repo add grafana https://grafana.github.io/helm-charts - helm dep build charts/devlake - export NODE_IP=$(kubectl get nodes --namespace default -o jsonpath="{.items[0].status.addresses[0].address}") - echo Node IP: ${NODE_IP} + DB_ROOT_PASSWORD=$(openssl rand -base64 16) + DB_PASSWORD=$(openssl rand -base64 16) + ENCRYPTION_SECRET=$(openssl rand -base64 2000 | tr -dc '[:upper:]' | fold -w 128 | head -n 1) + echo "::add-mask::${DB_ROOT_PASSWORD}" + echo "::add-mask::${DB_PASSWORD}" + echo "::add-mask::${ENCRYPTION_SECRET}" + NODE_IP=$(kubectl get nodes --namespace default -o jsonpath="{.items[0].status.addresses[0].address}") + echo "Node IP: ${NODE_IP}" + helm install --debug --wait --timeout 2400s deploy-test charts/devlake \ + --set service.uiPort=30000 \ + --set database.type=mysql \ + --set database.useExternal=false \ + --set database.username=devlake \ + --set database.password="${DB_PASSWORD}" \ + --set database.database=lake \ + --set database.mysql.rootPassword="${DB_ROOT_PASSWORD}" \ + --set database.image.tag=8-debian \ + --set lake.encryptionSecret.secret="${ENCRYPTION_SECRET}" + + - name: Install DevLake (PostgreSQL external) + if: matrix.database_type == 'postgresql-external' + run: | + DB_PASSWORD=$(openssl rand -base64 16) + ENCRYPTION_SECRET=$(openssl rand -base64 2000 | tr -dc '[:upper:]' | fold -w 128 | head -n 1) + echo "::add-mask::${DB_PASSWORD}" + echo "::add-mask::${ENCRYPTION_SECRET}" + NODE_IP=$(kubectl get nodes --namespace default -o jsonpath="{.items[0].status.addresses[0].address}") + echo "Node IP: ${NODE_IP}" helm install --debug --wait --timeout 2400s deploy-test charts/devlake \ --set service.uiPort=30000 \ - --set mysql.image.tag=8-debian \ - --set lake.encryptionSecret.secret=$(openssl rand -base64 2000 | tr -dc 'A-Z' | fold -w 128 | head -n 1) + --set database.type=postgresql \ + --set database.useExternal=true \ + --set database.externalServer=postgresql \ + --set database.externalPort=5432 \ + --set database.username=devlake \ + --set database.password="${DB_PASSWORD}" \ + --set database.database=lake \ + --set grafana.enabled=false \ + --set lake.encryptionSecret.secret="${ENCRYPTION_SECRET}" + + - name: Install DevLake (PostgreSQL builtin) + if: matrix.database_type == 'postgresql-builtin' + run: | + DB_PASSWORD=$(openssl rand -base64 16) + ENCRYPTION_SECRET=$(openssl rand -base64 2000 | tr -dc '[:upper:]' | fold -w 128 | head -n 1) + echo "::add-mask::${DB_PASSWORD}" + echo "::add-mask::${ENCRYPTION_SECRET}" + NODE_IP=$(kubectl get nodes --namespace default -o jsonpath="{.items[0].status.addresses[0].address}") + echo "Node IP: ${NODE_IP}" + helm install --debug --wait --timeout 2400s deploy-test charts/devlake \ + --set service.uiPort=30000 \ + --set database.type=postgresql \ + --set database.useExternal=false \ + --set database.username=devlake \ + --set database.password="${DB_PASSWORD}" \ + --set database.database=lake \ + --set grafana.enabled=false \ + --set lake.encryptionSecret.secret="${ENCRYPTION_SECRET}" - name: List cluster resources if: ${{ always() }} @@ -91,36 +197,37 @@ jobs: kubectl get secrets kubectl get pvc - # TODO: using some e2e test code to replace it - - name: Curl with endpoints + - name: Run E2E smoke tests + run: .github/scripts/e2e-smoke-test.sh + + - name: Dump diagnostics on failure + if: failure() run: | - export NODE_IP=$(kubectl get nodes --namespace default -o jsonpath="{.items[0].status.addresses[0].address}") - failed=0 - for retry in {1..10} ; do - failed=0 - # home - curl --fail http://${NODE_IP}:30000 || failed=1 - # API for devlake - curl --fail http://${NODE_IP}:30000/api/blueprints || failed=1 - # API for grafana - curl --fail http://${NODE_IP}:30000/grafana/api/health || failed=1 - if [ $failed -eq 0 ] ; then - break - else - sleep 3 - fi + echo "=== Helm status for release ===" + helm status deploy-test || true + echo "=== Kubernetes resources (all) ===" + kubectl get all -o wide || true + echo "=== Kubernetes events (latest 200) ===" + kubectl get events --sort-by=.lastTimestamp | tail -n 200 || true + echo "=== Describe deployments/statefulsets ===" + kubectl describe deploy || true + kubectl describe statefulset || true + echo "=== Describe pods ===" + kubectl describe pods || true + echo "=== Logs from all pods (current and previous, last 200 lines) ===" + for pod in $(kubectl get pods -o jsonpath='{.items[*].metadata.name}'); do + echo "----- logs for ${pod} -----" + kubectl logs "$pod" --all-containers --tail=200 || true + echo "----- previous logs for ${pod} -----" + kubectl logs "$pod" --all-containers --previous --tail=200 || true done - if [ $failed -ne 0 ] ; then - echo 'Test apis failed, please check logs from the PODS' - exit 1 - fi - name: Show logs for pods if: ${{ always() }} run: | for pod in $(kubectl get pods -o jsonpath='{.items[*].metadata.name}') ; do - echo describe for $pod - kubectl describe pod $pod - echo logs for $pod - kubectl logs $pod || echo "" + echo "describe for ${pod}" + kubectl describe pod "$pod" + echo "logs for ${pod}" + kubectl logs "$pod" || echo "" done diff --git a/.github/workflows/license-check.yml b/.github/workflows/license-check.yml new file mode 100644 index 00000000..d8802587 --- /dev/null +++ b/.github/workflows/license-check.yml @@ -0,0 +1,38 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: License Check + +on: + pull_request: + +permissions: + contents: read + +jobs: + license-header-check: + name: Apache License Header Validation + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v6.0.2 + + - name: Check License Headers + uses: apache/skywalking-eyes/header@v0.8.0 + with: + mode: check + config: .github/.licenserc.yaml diff --git a/.github/workflows/pr-automation.yml b/.github/workflows/pr-automation.yml new file mode 100644 index 00000000..05f9bc93 --- /dev/null +++ b/.github/workflows/pr-automation.yml @@ -0,0 +1,57 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: PR Automation + +on: + pull_request: + types: [opened, synchronize, reopened] + +permissions: + contents: read + pull-requests: write + +jobs: + label-pr: + name: Auto-label PR + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v6.0.2 + + - name: Label by path + uses: actions/labeler@v6.1.0 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + configuration-path: .github/labeler.yml + + - name: Label by size + uses: codelytv/pr-size-labeler@v1.10.4 + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + xs_label: 'size/XS' + xs_max_size: 10 + s_label: 'size/S' + s_max_size: 100 + m_label: 'size/M' + m_max_size: 500 + l_label: 'size/L' + l_max_size: 1000 + xl_label: 'size/XL' + +# Note: Reviewer assignment handled automatically by GitHub via CODEOWNERS file +# Create .github/CODEOWNERS to enable automatic reviewer requests on PR creation diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index e169df80..13d57cda 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -1,3 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + name: Release Charts on: @@ -13,15 +30,19 @@ on: permissions: contents: write packages: write + id-token: write + +concurrency: + group: release + cancel-in-progress: false jobs: release: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: - submodules: recursive fetch-depth: 0 - name: Configure Git @@ -30,29 +51,57 @@ jobs: git config user.email "$GITHUB_ACTOR@users.noreply.github.com" - name: Login to GitHub Container Registry - uses: docker/login-action@v3 + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Install Helm - run: | - echo "installing helm 3..." - curl -sSL https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | bash + - name: Install mise + uses: jdx/mise-action@1648a7812b9aeae629881980618f079932869151 # v4 + + - name: Install tools via mise + run: mise install - name: Add repositories run: | helm repo add grafana https://grafana.github.io/helm-charts - name: Run chart-releaser - uses: ./.github/actions/chart-releaser-action + uses: helm/chart-releaser-action@a0d2dc62c5e491af8ef6ba64a2e02bcf3fb33aa1 # v1.7.0 with: charts_dir: charts env: CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}" CR_SKIP_EXISTING: false + - name: Generate SBOM + uses: anchore/sbom-action@61119d458adab75f756bc0b9e4bde25725f86a7a # v0.17.2 + continue-on-error: true + with: + path: charts/devlake + format: spdx-json + output-file: devlake-sbom.spdx.json + + - name: Upload SBOM to release + continue-on-error: true + run: | + if [ -f devlake-sbom.spdx.json ]; then + for pkg in .cr-release-packages/*; do + if [ -z "${pkg:-}" ]; then + break + fi + TAG=$(basename "$pkg" | sed 's/.*-\([0-9].*\)\.tgz/\1/') + gh release upload "devlake-${TAG}" devlake-sbom.spdx.json --clobber || true + done + fi + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Install cosign + uses: sigstore/cosign-installer@dc72c7d5c4d10cd6bcb8cf6e3fd625a9e5e537da # v3.7.0 + continue-on-error: true + - name: Push Chart to GHCR run: | for pkg in .cr-release-packages/*; do @@ -61,3 +110,21 @@ jobs: fi helm push "${pkg}" oci://ghcr.io/${{ github.repository }} done + + - name: Sign chart with cosign + continue-on-error: true + run: | + for pkg in .cr-release-packages/*; do + if [ -z "${pkg:-}" ]; then + break + fi + CHART_NAME=$(basename "$pkg" | sed 's/-[0-9].*//') + TAG=$(basename "$pkg" | sed 's/.*-\([0-9].*\)\.tgz/\1/') + cosign sign --yes "ghcr.io/${{ github.repository }}/${CHART_NAME}:${TAG}" || true + done + + - name: Generate SLSA provenance + uses: actions/attest-build-provenance@1c608d11d69870c2092266b3f9a6f3abbf17002c # v1.4.3 + continue-on-error: true + with: + subject-path: '.cr-release-packages/*.tgz' diff --git a/.github/workflows/secret-scan.yml b/.github/workflows/secret-scan.yml new file mode 100644 index 00000000..ba3885a5 --- /dev/null +++ b/.github/workflows/secret-scan.yml @@ -0,0 +1,45 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: Secret Scan + +on: + pull_request: + push: + branches: + - main + +permissions: + contents: read + +jobs: + trufflehog-scan: + name: TruffleHog Secret Detection + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v6.0.2 + with: + fetch-depth: 0 + + - name: Run TruffleHog scan + uses: trufflesecurity/trufflehog@v3.95.3 + with: + path: ./ + base: ${{ github.event.repository.default_branch }} + head: HEAD + extra_args: --only-verified diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml new file mode 100644 index 00000000..06568bb9 --- /dev/null +++ b/.github/workflows/security-scan.yml @@ -0,0 +1,99 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: Security Scan + +on: + pull_request: + schedule: + - cron: '0 0 * * 0' # Weekly on Sunday at midnight UTC + +permissions: + contents: read + security-events: write + +jobs: + trivy-config-scan: + name: Trivy Chart Configuration Scan + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v6.0.2 + + - name: Run Trivy config scan + uses: aquasecurity/trivy-action@v0.36.0 + with: + scan-type: 'config' + scan-ref: 'charts/devlake' + severity: 'HIGH,CRITICAL' + format: 'sarif' + output: 'trivy-results.sarif' + exit-code: '1' # Fail on HIGH/CRITICAL findings (enforcement mode) + + - name: Upload Trivy results to Security tab + uses: github/codeql-action/upload-sarif@v4 + if: always() + with: + sarif_file: 'trivy-results.sarif' + + kubescape-scan: + name: Kubescape Security Best Practices + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v6.0.2 + + - name: Install mise + uses: jdx/mise-action@v4.0.1 + + - name: Install tools via mise + run: mise install + + - name: Add Helm repos + run: | + helm repo add grafana https://grafana.github.io/helm-charts + helm repo update + + - name: Build chart dependencies + run: | + helm dependency build charts/devlake + + - name: Template Helm chart + run: | + ENCRYPTION_SECRET=$(openssl rand -base64 2000 | tr -dc '[:upper:]' | fold -w 128 | head -n 1) + DB_PASSWORD=$(openssl rand -base64 32) + DB_ROOT_PASSWORD=$(openssl rand -base64 32) + + helm template devlake charts/devlake \ + --set lake.encryptionSecret.secret="${ENCRYPTION_SECRET}" \ + --set database.password="${DB_PASSWORD}" \ + --set database.mysql.rootPassword="${DB_ROOT_PASSWORD}" \ + > rendered-manifests.yaml + + - name: Run Kubescape scan + uses: kubescape/github-action@v3.0.21 + with: + format: sarif + outputFile: kubescape-results.sarif + files: rendered-manifests.yaml + # Enforcement mode enabled: scan will fail on security violations + + - name: Upload Kubescape results to Security tab + uses: github/codeql-action/upload-sarif@v4 + if: always() + with: + sarif_file: 'kubescape-results.sarif' diff --git a/.github/workflows/smoke-test.yml b/.github/workflows/smoke-test.yml index c5220264..1bf35d9f 100644 --- a/.github/workflows/smoke-test.yml +++ b/.github/workflows/smoke-test.yml @@ -1,64 +1,98 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + name: DevLake Helm Smoke Test on: push: - branches: [ "**" ] + branches: ["**"] pull_request: - branches: [ "**" ] + branches: ["**"] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true jobs: smoke-test: - name: Smoke Test + name: Smoke Test (${{ matrix.database }}) runs-on: ubuntu-latest timeout-minutes: 50 + strategy: + fail-fast: false + matrix: + database: [mysql, postgresql] env: - RELEASE_NAME: devlake - NAMESPACE: devlake + RELEASE_NAME: devlake-${{ matrix.database }} + NAMESPACE: devlake-${{ matrix.database }} steps: - name: Checkout - uses: actions/checkout@v5 - - - name: Set up kubectl - run: | - KVER="$(curl -sL https://dl.k8s.io/release/stable.txt)" - curl -L -o kubectl "https://dl.k8s.io/release/${KVER}/bin/linux/amd64/kubectl" - chmod +x kubectl && sudo mv kubectl /usr/local/bin/ - - - name: Set up Helm - run: | - HELM_VER="v3.18.6" - curl -L "https://get.helm.sh/helm-${HELM_VER}-linux-amd64.tar.gz" | tar xz - sudo mv linux-amd64/helm /usr/local/bin/helm - - - name: Set up kind - run: | - KIND_VER="v0.30.0" - curl -L -o kind "https://kind.sigs.k8s.io/dl/${KIND_VER}/kind-linux-amd64" - chmod +x kind && sudo mv kind /usr/local/bin/kind + uses: actions/checkout@v6.0.2 - - name: Create cluster - run: | - kind create cluster --name devlake-smoke-test + - name: Setup environment + uses: ./.github/actions/setup-environment - - name: Add Helm repos - run: | - helm repo add grafana https://grafana.github.io/helm-charts - helm repo update - - - name: Build chart dependencies - run: | - helm dependency build charts/devlake + - name: Create kind cluster + uses: ./.github/actions/kind-cluster + with: + cluster_name: devlake-${{ matrix.database }} + namespace: devlake-${{ matrix.database }} - name: Install DevLake chart run: | kubectl get events -n "$NAMESPACE" -w & + EVENTS_PID=$! kubectl get pods -n "$NAMESPACE" -w & - ENCRYPTION_SECRET=$(openssl rand -base64 2000 | tr -dc 'A-Z' | fold -w 128 | head -n 1) + PODS_PID=$! + trap 'kill $EVENTS_PID $PODS_PID 2>/dev/null || true' EXIT + + ENCRYPTION_SECRET=$(openssl rand -base64 2000 | tr -dc '[:upper:]' | fold -w 128 | head -n 1) + DB_PASSWORD=$(openssl rand -base64 32) + DB_ROOT_PASSWORD=$(openssl rand -base64 32) + echo "::add-mask::${ENCRYPTION_SECRET}" + echo "::add-mask::${DB_PASSWORD}" + echo "::add-mask::${DB_ROOT_PASSWORD}" + helm install "$RELEASE_NAME" ./charts/devlake \ --namespace "$NAMESPACE" \ --create-namespace \ - --wait \ - --set lake.encryptionSecret.secret="${ENCRYPTION_SECRET}" + --timeout 10m \ + --set database.type="${{ matrix.database }}" \ + --set lake.encryptionSecret.secret="${ENCRYPTION_SECRET}" \ + --set database.password="${DB_PASSWORD}" \ + --set database.mysql.rootPassword="${DB_ROOT_PASSWORD}" + + # Wait specifically for database pod (not full application stack) + echo "Waiting for ${{ matrix.database }} database pod to be ready..." + kubectl wait --for=condition=ready pod \ + -l "devlakeComponent=${{ matrix.database }}" \ + -n "$NAMESPACE" \ + --timeout=5m + + - name: Run database smoke test + run: | + if [ "${{ matrix.database }}" = "mysql" ]; then + .github/scripts/mysql-smoke-test.sh + elif [ "${{ matrix.database }}" = "postgresql" ]; then + .github/scripts/postgresql-smoke-test.sh + fi - name: Dump diagnostics on failure if: failure() diff --git a/.github/workflows/yaml-lint.yml b/.github/workflows/yaml-lint.yml index b3f301c4..05bf8f1c 100644 --- a/.github/workflows/yaml-lint.yml +++ b/.github/workflows/yaml-lint.yml @@ -21,17 +21,24 @@ on: branches: - main pull_request: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: helm-lint: name: lint for helm chart runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - name: install latest helm - run: curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash - - name: Add repositories - run: | - helm repo add grafana https://grafana.github.io/helm-charts - helm dep build charts/devlake + - name: Checkout + uses: actions/checkout@v6.0.2 + + - name: Setup environment + uses: ./.github/actions/setup-environment + - name: lint helm chart run: helm lint charts/devlake --strict diff --git a/.gitignore b/.gitignore index 4a66fe91..a51e744d 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,5 @@ *.tgz charts/*/charts +**/.omc.github/WORKFLOWS.md +.github/RUNBOOK.md diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index d6b3681a..00000000 --- a/.gitmodules +++ /dev/null @@ -1,8 +0,0 @@ -[submodule ".github/actions/setup-kind"] - path = .github/actions/setup-kind - url = https://github.com/engineerd/setup-kind.git - branch = v0.5.0 -[submodule ".github/actions/chart-releaser-action"] - path = .github/actions/chart-releaser-action - url = https://github.com/helm/chart-releaser-action.git - branch = v1.5.0 diff --git a/.licenserc.yaml b/.licenserc.yaml new file mode 100644 index 00000000..251296f0 --- /dev/null +++ b/.licenserc.yaml @@ -0,0 +1,20 @@ +header: + license: + spdx-id: Apache-2.0 + copyright-owner: Apache Software Foundation + + paths-ignore: + - '**/Chart.lock' + - '**/*.md' + - '**/*.json' + - '.gitignore' + - '.trivyignore' + - 'LICENSE' + - 'NOTICE' + - '.licenserc.yaml' + - '.mise.toml' + - '.pre-commit-config.yaml' + - '.yamllint' + - '**/*.gotmpl' + + comment: on-failure diff --git a/.mise.toml b/.mise.toml new file mode 100644 index 00000000..890b8b41 --- /dev/null +++ b/.mise.toml @@ -0,0 +1,9 @@ +[tools] +kubectl = "1.36.0" +helm = "3.20.2" +kind = "0.31.0" +trivy = "0.70.0" +actionlint = "1.7.6" +yamllint = "1.35.1" +pre-commit = "4.0.1" +helm-docs = "1.14.2" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..b15f8c73 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,75 @@ +# See https://pre-commit.com for more information +# See https://pre-commit.com/hooks.html for more hooks +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v5.0.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + args: ['--unsafe'] # Allow custom YAML tags in Helm templates + exclude: ^charts/.*/templates/.*\.yaml$ # Skip Helm templates (contain Go template syntax) + - id: check-added-large-files + args: ['--maxkb=1000'] + - id: check-merge-conflict + - id: detect-private-key + + - repo: https://github.com/sirosen/texthooks + rev: 0.7.1 + hooks: + - id: fix-smartquotes + - id: fix-ligatures + + - repo: https://github.com/jumanjihouse/pre-commit-hooks + rev: 3.0.0 + hooks: + - id: shellcheck + args: ['--severity=warning'] + + - repo: https://github.com/trufflesecurity/trufflehog + rev: v3.87.2 + hooks: + - id: trufflehog + name: TruffleHog secret scan + entry: trufflehog filesystem --no-update --fail + args: ['--only-verified', '.'] + language: golang + pass_filenames: false + + - repo: https://github.com/gruntwork-io/pre-commit + rev: v0.1.24 + hooks: + - id: helmlint + + + - repo: https://github.com/rhysd/actionlint + rev: v1.7.4 + hooks: + - id: actionlint + + - repo: local + hooks: + - id: helm-docs + name: Helm Docs + entry: bash -c 'mise exec -- helm-docs --chart-search-root=charts' + language: system + pass_filenames: false + files: (Chart\.yaml|values\.yaml|README\.md)$ + - id: apache-license-check + name: Apache License Header Check + entry: bash -c 'docker run --rm -v "$PWD:/github/workspace" apache/skywalking-eyes:latest header check' + language: system + pass_filenames: false + always_run: true + +ci: + autofix_commit_msg: | + [pre-commit.ci] auto fixes from pre-commit hooks + + for more information, see https://pre-commit.ci + autofix_prs: true + autoupdate_branch: '' + autoupdate_commit_msg: '[pre-commit.ci] pre-commit autoupdate' + autoupdate_schedule: weekly + skip: [] + submodules: false diff --git a/.trivyignore b/.trivyignore new file mode 100644 index 00000000..32f18546 --- /dev/null +++ b/.trivyignore @@ -0,0 +1,33 @@ +# Trivy Ignore File +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Trivy False Positive Suppressions +# +# Format: AVD-ID or CVE-ID # Rationale | Mitigation | Review date +# +# Guidelines: +# - Only suppress findings that are verified false positives or accepted risks +# - Include detailed rationale explaining why suppression is safe +# - Add mitigation steps if applicable +# - Review suppressions quarterly (every 3 months) +# - Remove suppressions when underlying issue is fixed +# +# Example: +# AVD-KSV-0012 # Grafana container runs as UID 472 by design (upstream chart default). Non-root UID with read-only filesystem. | Review: 2026-05-06 + +# Add suppressions below this line diff --git a/.yamllint b/.yamllint new file mode 100644 index 00000000..b09299e6 --- /dev/null +++ b/.yamllint @@ -0,0 +1,12 @@ +--- +extends: default + +rules: + line-length: + max: 120 + level: warning + document-start: disable + truthy: + allowed-values: ['true', 'false', 'on'] + comments: + min-spaces-from-content: 1 diff --git a/HelmSetup.md b/HelmSetup.md deleted file mode 100644 index 435389a5..00000000 --- a/HelmSetup.md +++ /dev/null @@ -1,307 +0,0 @@ ---- -title: "Install via Helm" -description: > - The steps to install Apache DevLake via Helm for Kubernetes -sidebar_position: 2 ---- - -## 1 Prerequisites - -- Helm >= 3.6.0 -- Kubernetes >= 1.19.0 - ---- - -## 2 Quick Start - -**Check https://github.com/apache/incubator-devlake-helm-chart to contribute!** - -### 2.1 Install - -To install the chart with release name `devlake`: - -```shell -helm repo add devlake https://apache.github.io/incubator-devlake-helm-chart -helm repo update -ENCRYPTION_SECRET=$(openssl rand -base64 2000 | tr -dc 'A-Z' | fold -w 128 | head -n 1) -helm install devlake devlake/devlake --version=1.0.3-beta10 --set lake.encryptionSecret.secret=$ENCRYPTION_SECRET -``` - -Visit your devlake from the node port (32001 by default): http://YOUR-NODE-IP:32001. - -_Notes for mac users with minikube:_ - -- forward the port: `kubectl port-forward svc/devlake-ui 4000:4000` -- access config-ui: `http://YOUR-NODE-IP:4000/` -- access Grafana dashboard: click the dashboard button in config-ui, or visit `http://YOUR-NODE-IP:4000/grafana/` - -### 2.2 Upgrade - -**Note:** - -**If you're upgrading from DevLake v0.17.x or earlier versions to v0.18.x or later versions:** - -1. Copy the ENCODE_KEY value from /app/config/.env of the lake pod (e.g. devlake-lake-0), and replace the in the upgrade command below. - -2. You may encounter the below error when upgrading because the built-in grafana has been replaced by the official grafana dependency. So you may need to delete the grafana deployment first. - -> Error: UPGRADE FAILED: cannot patch "devlake-grafana" with kind Deployment: Deployment.apps "devlake-grafana" is invalid: spec.selector: Invalid value: v1.LabelSelector{MatchLabels:map[string]string{"app.kubernetes.io/instance":"devlake", "app.kubernetes.io/name":"grafana"}, MatchExpressions:[]v1.LabelSelectorRequirement(nil)}: field is immutable - -```shell -helm repo update -helm upgrade devlake devlake/devlake --version=1.0.3-beta10 --set lake.encryptionSecret.secret= -``` - -**If you're upgrading from DevLake v0.18.x or later versions:** - -```shell -helm repo update -helm upgrade devlake devlake/devlake --version=1.0.3-beta10 -``` - -### 2.3 Uninstall - -To uninstall/delete the `devlake` release: - -```shell -helm uninstall devlake -``` - ---- - -## 3 Example Deployments - -### 3.1 Deploy with NodePort - -Conditions: - -- IP Address of Kubernetes node: 192.168.0.6 -- Want to visit devlake with port 30000. - -``` -ENCRYPTION_SECRET=$(openssl rand -base64 2000 | tr -dc 'A-Z' | fold -w 128 | head -n 1) -helm install devlake devlake/devlake --set service.uiPort=30000 --set lake.encryptionSecret.secret=$ENCRYPTION_SECRET -``` - -After deployed, visit devlake: http://192.168.0.6:30000 - -### 3.2 Deploy with Ingress - -Conditions: - -- I have already configured default ingress for the Kubernetes cluster -- I want to use http://devlake.example.com for visiting devlake - -``` -ENCRYPTION_SECRET=$(openssl rand -base64 2000 | tr -dc 'A-Z' | fold -w 128 | head -n 1) -helm install devlake devlake/devlake --set "ingress.enabled=true,ingress.hostname=devlake.example.com,lake.encryptionSecret.secret=$ENCRYPTION_SECRET" -``` - -After deployed, visit devlake: http://devlake.example.com, and grafana at http://devlake.example.com/grafana - -### 3.3 Deploy with Ingress (Https) - -Conditions: - -- I have already configured ingress(class: nginx) for the Kubernetes cluster, and the https using 8443 port. -- I want to use https://devlake-0.example.com:8443 for visiting devlake. -- The https certificates are generated by letsencrypt.org, and the certificate and key files: `cert.pem` and `key.pem` - -First, create the secret: - -``` -kubectl create secret tls ssl-certificate --cert cert.pem --key secret.pem -``` - -Then, deploy the devlake: - -``` -ENCRYPTION_SECRET=$(openssl rand -base64 2000 | tr -dc 'A-Z' | fold -w 128 | head -n 1) -helm install devlake devlake/devlake \ - --set "ingress.enabled=true,ingress.enableHttps=true,ingress.hostname=devlake-0.example.com" \ - --set "ingress.className=nginx,ingress.httpsPort=8443" \ - --set "ingress.tlsSecretName=ssl-certificate" - --set "lake.encryptionSecret.secret=$ENCRYPTION_SECRET" -``` - -After deployed, visit devlake: https://devlake-0.example.com:8443, and grafana at https://devlake-0.example.com:8443/grafana - -### 3.4 Specify grafana admin password - -``` -ENCRYPTION_SECRET=$(openssl rand -base64 2000 | tr -dc 'A-Z' | fold -w 128 | head -n 1) -helm install devlake devlake/devlake \ - --set grafana.adminPassword= \ - --set lake.encryptionSecret.secret=$ENCRYPTION_SECRET -``` - ---- - -## 4 Parameters - -Some useful parameters for the chart, you could also check them in values.yaml - -| Parameter | Description | Default | -| -------------------------------------- | ------------------------------------------------------------------------------------- | ------------------------ | -| replicaCount | Replica Count for devlake, currently not used (removed) | 1 | -| imageTag | The version tag for all images | see Values.yaml | -| imagePullSecrets | Name of the Secret for accessing private image registries | [] | -| commonEnvs | The common envs for all pods except grafana | {TZ: "UTC"} | -| mysql.replicaCount | Replica count can only be 0 or 1 | 1 | -| mysql.useExternal | If use external mysql server, set true | false | -| mysql.externalServer | External mysql server address | 127.0.0.1 | -| mysql.externalPort | External mysql server port | 3306 | -| mysql.username | username for mysql | merico | -| mysql.password | password for mysql | merico | -| mysql.database | database for mysql | lake | -| mysql.rootPassword | root password for mysql | admin | -| mysql.storage.type | storage type, pvc or hostpath | pvc | -| mysql.storage.class | storage class for mysql's volume | "" | -| mysql.storage.size | volume size for mysql's data | 5Gi | -| mysql.storage.hostPath | the host path if mysql.storage.type is hostpath | /devlake/mysql/data | -| mysql.image.repository | repository for mysql's image | mysql | -| mysql.image.tag | image tag for mysql's image | 8 | -| mysql.image.pullPolicy | pullPolicy for mysql's image | IfNotPresent | -| mysql.initContainers | init containers to run to complete before mysql | [] | -| mysql.extraLabels | extra labels for mysql's statefulset | {} | -| mysql.securityContext | pod security context values | {} | -| mysql.containerSecurityContext | container security context values | {} | -| mysql.service.type | mysql service type | ClusterIP | -| mysql.service.nodePort | specify mysql nodeport | "" | -| grafana | dashboard, datasource, etc. settings for grafana, installed by grafana official chart | | -| lake.replicaCount | Replica count can only be 0 or 1 | 1 | -| lake.image.repository | repository for lake's image | apache/devlake | -| lake.image.pullPolicy | pullPolicy for lake's image | Always | -| lake.port | the port of devlake backend | 8080 | -| lake.envs | initial envs for lake | see Values.yaml | -| lake.extraEnvsFromSecret | existing secret name of extra envs | "" | -| lake.encryptionSecret.secretName | the k8s secret name for ENCRYPTION_SECRET | "" | -| lake.encryptionSecret.secret | the secret for ENCRYPTION_SECRET | "" | -| lake.encryptionSecret.autoCreateSecret | whether let the helm chart create the secret | true | -| lake.extraLabels | extra labels for lake's deployment template | {} | -| lake.securityContext | pod security context values | {} | -| lake.strategy | pod update strategy | {} | -| lake.containerSecurityContext | container security context values | {} | -| lake.livenessProbe | container livenessprobe | see Values.yaml | -| lake.readinessProbe | container readinessProbe | see Values.yaml | -| lake.deployment.extraLabels | extra labels for lake's deployment metadata | {} | -| ui.replicaCount | Replica count for ui | 1 | -| ui.image.repository | repository for ui's image | apache/devlake-config-ui | -| ui.image.pullPolicy | pullPolicy for ui's image | Always | -| ui.basicAuth.enabled | If the basic auth in ui is enabled | false | -| ui.basicAuth.user | The user name for the basic auth | "admin" | -| ui.basicAuth.password | The password for the basic auth | "admin" | -| ui.basicAuth.autoCreateSecret | If let the helm chart create the secret | true | -| ui.basicAuth.secretName | The basic auth secret name | "" | -| ui.extraLabels | extra labels for ui's deployment template | {} | -| ui.securityContext | pod security context values | {} | -| ui.strategy | pod update strategy | {} | -| ui.containerSecurityContext | container security context values | {} | -| ui.livenessProbe | container livenessprobe | see Values.yaml | -| ui.readinessProbe | container readinessProbe | see Values.yaml | -| ui.deployment.extraLabels | extra labels for ui's deployment metadata | {} | -| service.type | Service type for exposed service | NodePort | -| service.uiPort | Node port for config ui | 32001 | -| ingress.enabled | If enable ingress | false | -| ingress.enableHttps | If enable https | false | -| ingress.useDefaultNginx | If use nginx ingress controller | true | -| ingress.className | Name for ingressClass. leave empty for using default | "" | -| ingress.annotations | The ingress annotations | {} | -| ingress.hostname | The hostname/domainname for ingress | localhost | -| ingress.prefix | The prefix for endpoints, currently not used | / | -| ingress.tlsSecretName | The secret name for tls's certificate for https | "" | -| ingress.httpPort | The http port for ingress | 80 | -| ingress.httpsPort | The https port for ingress | 443 | -| ingress.extraPaths | The extra paths for ingress | [] | -| option.database | The database type, valids: mysql | mysql | -| option.connectionSecretName | The database connection details secret name | devlake-mysql-auth | -| option.autoCreateSecret | If let the helm chart create the secret | true | - ---- - -## 5 FAQ - -1. Can I use a managed Cloud database service instead of running database in docker? - -Yes, it just set useExternal value to true while you deploy devlake with helm chart. Below we'll use MySQL on AWS RDS as an example. - -a. (Optional) Create a MySQL instance on AWS RDS following this [doc](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/CHAP_GettingStarted.CreatingConnecting.MySQL.html), skip this step if you'd like to use an existing instance -b. Proviede below values while install from helm: - - - `mysql.useExternal`: this should be `true` - - `mysql.externalServer`: use your RDS instance's IP address or domain name. - - `mysql.externalPort`: use your RDS instance's database port. - - `mysql.username`: use your `username` for access RDS instance's DB - - `mysql.password`: use your `password` for access RDS instance's DB - - `mysql.database`: use your RDS instance's DB name, you may need to create a database first with `CREATE DATABASE ;` - -Here is the example: - -``` -helm repo add devlake https://apache.github.io/incubator-devlake-helm-chart -helm repo update -ENCRYPTION_SECRET=$(openssl rand -base64 2000 | tr -dc 'A-Z' | fold -w 128 | head -n 1) -helm install devlake devlake/devlake \ - --set mysql.useExternal=true \ - --set mysql.externalServer=db.example.com \ - --set mysql.externalPort=3306 \ - --set mysql.username=admin \ - --set mysql.password=password_4_admin \ - --set mysql.database=devlake - --set lake.encryptionSecret.secret=$ENCRYPTION_SECRET - -``` - -2. Can I use a secret to store the database connection details? - -Yes, to do so, you need to have a secret in your Kubernetes Cluster that contains the following values: - -- `MYSQL_USER`: The user to connect to your DB. -- `MYSQL_PASSWORD`: The password to connect to your DB. -- `MYSQL_DATABASE`: The database to connect to your DB. -- `MYSQL_ROOT_PASSWORD`: The root password to connect to your DB. -- `DB_URL`: mysql://`username`:`password`@`dbserver`:`port`/`database`?charset=utf8mb4&parseTime=True - -The secret name needs to be the same as the value `option.connectionSecretName` - -3. Can I use an external Grafana instead of running built-in Grafana? - -Yes, the devlake helm chart supports using an external Grafana. You can set the following values while installing from helm: - -- `grafana.enabled`: this should be `false` -- `grafana.external.url`: use your Grafana's URL, e.g. `https://grafana.example.com` - -Here is the example: - -``` -helm repo add devlake https://apache.github.io/incubator-devlake-helm-chart -helm repo update -ENCRYPTION_SECRET=$(openssl rand -base64 2000 | tr -dc 'A-Z' | fold -w 128 | head -n 1) -helm install devlake devlake/devlake \ - --set grafana.enabled=false \ - --set grafana.external.url=https://grafana.example.com - --set lake.encryptionSecret.secret=$ENCRYPTION_SECRET - -``` - -4. How to set the Grafana admin password? If not explicitly set, a random password will be generated and saved in a Kubernetes Secret - -- `grafana.adminPassword`: your password - -Here is the example: - -``` -helm repo add devlake https://apache.github.io/incubator-devlake-helm-chart -helm repo update -ENCRYPTION_SECRET=$(openssl rand -base64 2000 | tr -dc 'A-Z' | fold -w 128 | head -n 1) -helm install devlake devlake/devlake \ - --set grafana.adminPassword= \ - --set lake.encryptionSecret.secret=$ENCRYPTION_SECRET - -``` - ---- - -## 6 Troubleshooting - -If you run into any problem, please check the [Troubleshooting](/Troubleshooting/Installation.md) or [create an issue](https://github.com/apache/incubator-devlake/issues) diff --git a/README.md b/README.md index 50d33016..19065e3e 100644 --- a/README.md +++ b/README.md @@ -1,100 +1,584 @@ -# Apache Incubator DevLake Helm Chart +# Apache DevLake Helm Chart -Thanks to @matrixji who initiated all content in `apache/incubator-devlake`, this repo is copied from directory deployment/helm in repo `apache/incubator-devlake`! Also thanks to @lshmouse, @shubham-cmyk and @SnowMoon-Dev for the contribution for devlake helm deployment. +[![Release Charts](https://github.com/apache/incubator-devlake-helm-chart/actions/workflows/release.yaml/badge.svg)](https://github.com/apache/incubator-devlake-helm-chart/actions/workflows/release.yaml) +[![Helm Version](https://img.shields.io/badge/Helm-v3.6%2B-blue)](https://helm.sh) +[![Kubernetes Version](https://img.shields.io/badge/Kubernetes-v1.19%2B-blue)](https://kubernetes.io) + +Production-ready Helm chart for deploying [Apache DevLake](https://devlake.apache.org/), an open-source dev data platform that ingests, analyzes, and visualizes fragmented data from DevOps tools to distill insights for engineering productivity. + +## Table of Contents + +- [Apache DevLake Helm Chart](#apache-devlake-helm-chart) + - [Table of Contents](#table-of-contents) + - [Features](#features) + - [Core Capabilities](#core-capabilities) + - [Security](#security) + - [Operations](#operations) + - [Enterprise Features](#enterprise-features) + - [Prerequisites](#prerequisites) + - [Quick Start](#quick-start) + - [Installation](#installation) + - [Upgrade](#upgrade) + - [Uninstallation](#uninstallation) + - [Configuration](#configuration) + - [Database Options](#database-options) + - [MySQL (Default)](#mysql-default) + - [PostgreSQL](#postgresql) + - [Storage Configuration](#storage-configuration) + - [Security Configuration](#security-configuration) + - [External Secrets Operator](#external-secrets-operator) + - [Network Policies](#network-policies) + - [Image Digest Pinning](#image-digest-pinning) + - [Deployment Scenarios](#deployment-scenarios) + - [NodePort (Simple Access)](#nodeport-simple-access) + - [Ingress (HTTP)](#ingress-http) + - [Ingress (HTTPS with TLS)](#ingress-https-with-tls) + - [Production Deployment (Full Security)](#production-deployment-full-security) + - [Operations](#operations-1) + - [Backup and Restore](#backup-and-restore) + - [Enable Automated Backups](#enable-automated-backups) + - [Manual Backup](#manual-backup) + - [Restore](#restore) + - [Monitoring](#monitoring) + - [Prometheus Metrics](#prometheus-metrics) + - [Health Checks](#health-checks) + - [Scaling](#scaling) + - [Horizontal Pod Autoscaling](#horizontal-pod-autoscaling) + - [Multi-AZ Deployment](#multi-az-deployment) + - [Development](#development) + - [Local Development](#local-development) + - [Contributing](#contributing) + - [License](#license) + - [Acknowledgments](#acknowledgments) + +## Features + +### Core Capabilities +- **Multi-Database Support**: MySQL 8.x and PostgreSQL 14.x with embedded or external database options +- **Production-Ready Defaults**: Secure by default with ClusterIP service, non-root containers, read-only filesystems +- **Flexible Deployment**: NodePort, LoadBalancer, or Ingress (HTTP/HTTPS) with optional basic authentication + +### Security +- **Network Isolation**: NetworkPolicy support for database, backend, and UI layers +- **Secrets Management**: Native Kubernetes Secrets with optional External Secrets Operator integration +- **RBAC**: ServiceAccount with least-privilege configuration +- **Security Hardening**: Read-only root filesystems, dropped capabilities, seccomp profiles + +### Operations +- **Automated Backups**: CronJob-based database backups with configurable retention +- **High Availability**: HorizontalPodAutoscaler support for backend and UI components +- **Observability**: Prometheus metrics, structured JSON logging, health probes +- **Migration Support**: Built-in database migration job for schema management -## Install +### Enterprise Features +- **External Database**: Connect to managed RDS, Cloud SQL, or Azure Database services +- **External Grafana**: Integration with existing Grafana instances +- **Custom Images**: SHA256 digest pinning for supply chain security +- **Pod Scheduling**: Anti-affinity rules and topology spread constraints -1. Install the latest stable version with release name `devlake` +## Prerequisites -```shell +- **Kubernetes**: 1.19.0 or higher +- **Helm**: 3.6.0 or higher +- **Storage**: PersistentVolume provisioner support (or hostPath for development) +- **Resources**: Minimum 2 CPU cores and 4GB RAM available in cluster + +## Quick Start + +### Installation + +1. **Add the Helm repository**: + +```bash helm repo add devlake https://apache.github.io/incubator-devlake-helm-chart helm repo update -ENCRYPTION_SECRET=$(openssl rand -base64 2000 | tr -dc 'A-Z' | fold -w 128 | head -n 1) -helm install devlake devlake/devlake --set lake.encryptionSecret.secret=$ENCRYPTION_SECRET ``` -2. Install the latest development version with release name `devlake`: +2. **Generate encryption secret** (required): -```shell -helm repo add devlake https://apache.github.io/incubator-devlake-helm-chart -helm repo update +```bash ENCRYPTION_SECRET=$(openssl rand -base64 2000 | tr -dc 'A-Z' | fold -w 128 | head -n 1) -helm install devlake devlake/devlake --version=1.0.3-beta10 --set lake.encryptionSecret.secret=$ENCRYPTION_SECRET ``` -Helm chart are also published to GitHub container registry as OCI artifact. +3. **Install with MySQL** (default): -If you are using minikube inside your mac, please use the following command to forward the port: +```bash +helm install devlake devlake/devlake \ + --set database.password= \ + --set database.mysql.rootPassword= \ + --set lake.encryptionSecret.secret=$ENCRYPTION_SECRET +``` -```shell -kubectl port-forward service/devlake-ui 30090:4000 +4. **Install with PostgreSQL**: + +```bash +helm install devlake devlake/devlake \ + --set database.type=postgresql \ + --set database.password= \ + --set lake.encryptionSecret.secret=$ENCRYPTION_SECRET \ + --set grafana.enabled=false ``` -and open another terminal: +5. **Access the application**: -```shell -kubectl port-forward service/devlake-grafana 30091:3000 +For minikube users: +```bash +kubectl port-forward service/devlake-ui 4000:4000 ``` -Then you can visit: -config-ui by url `http://YOUR-NODE-IP:30090` -grafana by url `http://YOUR-NODE-IP:30091` +Then visit: http://localhost:4000 + +### Verifying Signed Releases + +All Helm chart releases are signed with [Sigstore cosign](https://docs.sigstore.dev/cosign/overview/) for supply chain security. + +**Verify OCI chart signature**: -## Upgrade +```bash +# Install cosign +brew install cosign -**Note:** +# Verify chart signature (example for version 0.1.0) +cosign verify ghcr.io/apache/incubator-devlake-helm-chart/devlake:0.1.0 \ + --certificate-identity-regexp="https://github.com/apache/incubator-devlake-helm-chart/.*" \ + --certificate-oidc-issuer="https://token.actions.githubusercontent.com" +``` + +**Verify SLSA provenance**: -**If you're upgrading from DevLake v0.17.x or earlier versions to v0.18.x or later versions:** +```bash +# Verify chart provenance attestation +gh attestation verify devlake-0.1.0.tgz --repo apache/incubator-devlake-helm-chart +``` -1. Copy the ENCODE_KEY value from /app/config/.env of the lake pod (e.g. devlake-lake-0), and replace the in the upgrade command below. +All releases include: +- **SBOM (SPDX)**: Software Bill of Materials listing chart dependencies +- **Cosign signature**: Keyless signature using GitHub OIDC +- **SLSA provenance**: Build provenance attestation -2. You may encounter the below error when upgrading because the built-in grafana has been replaced by the official grafana dependency. So you may need to delete the grafana deployment first. +### Upgrade -> Error: UPGRADE FAILED: cannot patch "devlake-grafana" with kind Deployment: Deployment.apps "devlake-grafana" is invalid: spec.selector: Invalid value: v1.LabelSelector{MatchLabels:map[string]string{"app.kubernetes.io/instance":"devlake", "app.kubernetes.io/name":"grafana"}, MatchExpressions:[]v1.LabelSelectorRequirement(nil)}: field is immutable +**From v0.18.x or later**: -```shell +```bash helm repo update -helm upgrade devlake devlake/devlake --version=1.0.3-beta10 --set lake.encryptionSecret.secret= +helm upgrade devlake devlake/devlake ``` -**If you're upgrading from DevLake v0.18.x or later versions:** +**From v0.17.x or earlier** (requires ENCRYPTION_SECRET): + +1. Extract existing encryption key: +```bash +kubectl exec -it devlake-lake-0 -- cat /app/config/.env | grep ENCODE_KEY +``` -```shell +2. Upgrade with the extracted key: +```bash helm repo update -helm upgrade devlake devlake/devlake --version=1.0.3-beta10 +helm upgrade devlake devlake/devlake \ + --set lake.encryptionSecret.secret= ``` -## Uninstall +**Note**: Upgrading from v0.17.x may require deleting the Grafana deployment first if you encounter selector immutability errors. See [HelmSetup.md](HelmSetup.md#22-upgrade) for details. -To uninstall/delete the `devlake` release: +### Uninstallation -```shell +```bash helm uninstall devlake ``` -## Original pr in apache/incubator-devlake +**Warning**: This does not delete PersistentVolumeClaims. To completely remove data: + +```bash +kubectl delete pvc -l app.kubernetes.io/instance=devlake +``` + +## Configuration + +### Database Options + +#### MySQL (Default) + +**Embedded MySQL**: +```yaml +database: + type: mysql + useExternal: false + password: "strong-password" + mysql: + rootPassword: "strong-root-password" +``` + +**External MySQL** (AWS RDS, Cloud SQL, etc.): +```yaml +database: + type: mysql + useExternal: true + externalServer: "db.example.com" + externalPort: 3306 + username: "devlake" + password: "strong-password" + database: "lake" +``` + +#### PostgreSQL + +**Embedded PostgreSQL**: +```yaml +database: + type: postgresql + useExternal: false + externalPort: 5432 + password: "strong-password" +``` + +**External PostgreSQL**: +```yaml +database: + type: postgresql + useExternal: true + externalServer: "postgres.example.com" + externalPort: 5432 + username: "devlake" + password: "strong-password" + database: "lake" +``` + +**Important**: Grafana dashboards currently only support MySQL datasources. When using PostgreSQL, set `grafana.enabled=false` and provide an external Grafana instance via `grafana.external.url`, or use the UI without Grafana dashboards. + +### Storage Configuration + +**PersistentVolumeClaim** (Production): +```yaml +database: + storage: + type: pvc + class: "fast-ssd" # your storage class + size: 50Gi +``` + +**HostPath** (Development only): +```yaml +database: + storage: + type: hostpath + hostPath: /data/devlake + size: 50Gi +``` + +### Security Configuration + +#### External Secrets Operator + +```yaml +option: + externalSecrets: + enabled: true + secretStoreRef: + name: vault-backend + kind: SecretStore +``` + +#### Network Policies + +```yaml +networkPolicy: + enabled: true + externalEgressCIDRs: + - "192.30.252.0/22" # GitHub API + - "140.82.112.0/20" # GitHub API + ui: + ingressNamespaceSelector: + matchLabels: + name: ingress-nginx +``` + +#### Image Digest Pinning + +```yaml +imageDigests: + lake: "sha256:abc123..." + ui: "sha256:def456..." +``` + +## Deployment Scenarios + +### NodePort (Simple Access) + +```bash +helm install devlake devlake/devlake \ + --set service.type=NodePort \ + --set service.uiPort=30000 \ + --set database.password= \ + --set database.mysql.rootPassword= \ + --set lake.encryptionSecret.secret=$ENCRYPTION_SECRET +``` + +Access at: `http://:30000` + +### Ingress (HTTP) + +```bash +helm install devlake devlake/devlake \ + --set ingress.enabled=true \ + --set ingress.hostname=devlake.example.com \ + --set database.password= \ + --set database.mysql.rootPassword= \ + --set lake.encryptionSecret.secret=$ENCRYPTION_SECRET +``` + +Access at: `http://devlake.example.com` + +### Ingress (HTTPS with TLS) + +1. Create TLS secret: +```bash +kubectl create secret tls devlake-tls --cert=cert.pem --key=key.pem +``` + +2. Install with HTTPS: +```bash +helm install devlake devlake/devlake \ + --set ingress.enabled=true \ + --set ingress.enableHttps=true \ + --set ingress.hostname=devlake.example.com \ + --set ingress.tlsSecretName=devlake-tls \ + --set database.password= \ + --set database.mysql.rootPassword= \ + --set lake.encryptionSecret.secret=$ENCRYPTION_SECRET +``` + +Access at: `https://devlake.example.com` + +### Gateway API HTTPRoute + +For Kubernetes Gateway API integration: + +```bash +helm install devlake devlake/devlake \ + --set httpRoute.enabled=true \ + --set httpRoute.gatewayName=my-gateway \ + --set httpRoute.gatewayNamespace=gateway-system \ + --set httpRoute.hostnames[0]=devlake.example.com \ + --set database.password= \ + --set database.mysql.rootPassword= \ + --set lake.encryptionSecret.secret=$ENCRYPTION_SECRET +``` + +**Configuration options**: +- `httpRoute.prefix`: Path prefix (default: `/`) +- `httpRoute.sectionName`: Gateway listener name (optional) +- `httpRoute.annotations`: Additional HTTPRoute annotations +- `httpRoute.extraPaths`: Additional path-based routes + +Access at: `http://devlake.example.com` (HTTPS via Gateway TLS configuration) + +### Production Deployment (Full Security) + +```bash +helm install devlake devlake/devlake \ + --set database.type=mysql \ + --set database.password=$DB_PASSWORD \ + --set database.mysql.rootPassword=$DB_ROOT_PASSWORD \ + --set lake.encryptionSecret.secret=$ENCRYPTION_SECRET \ + --set networkPolicy.enabled=true \ + --set networkPolicy.externalEgressCIDRs='{0.0.0.0/0}' \ + --set lake.autoscaling.enabled=true \ + --set ui.autoscaling.enabled=true \ + --set ingress.enabled=true \ + --set ingress.enableHttps=true \ + --set ingress.hostname=devlake.example.com \ + --set ingress.tlsSecretName=devlake-tls \ + --set backup.enabled=true \ + --set backup.schedule="0 2 * * *" +``` + +## Operations + +### Backup and Restore + +#### Enable Automated Backups + +```yaml +backup: + enabled: true + schedule: "0 2 * * *" # Daily at 2 AM UTC + retentionDays: 7 + pvc: + size: 10Gi + storageClassName: "fast-ssd" +``` + +#### Manual Backup + +**MySQL**: +```bash +kubectl exec -it devlake-mysql-0 -- mysqldump -uroot -p lake > backup.sql +``` + +**PostgreSQL**: +```bash +kubectl exec -it devlake-postgresql-0 -- pg_dump -U merico lake > backup.sql +``` + +#### Restore + +**MySQL**: +```bash +kubectl exec -i devlake-mysql-0 -- mysql -uroot -p lake < backup.sql +``` + +**PostgreSQL**: +```bash +kubectl exec -i devlake-postgresql-0 -- psql -U merico -d lake < backup.sql +``` + +### Monitoring + +#### Prometheus Metrics + +```yaml +metrics: + enabled: true + serviceMonitor: + enabled: true + namespace: monitoring + interval: 30s +``` + +#### Health Checks + +DevLake includes built-in health probes: +- **Liveness**: `/ping` endpoint on port 8080 +- **Readiness**: `/ping` endpoint with stricter thresholds +- **Database**: Type-specific probes (`mysqladmin ping` or `pg_isready`) + +### Scaling + +#### Horizontal Pod Autoscaling + +```yaml +lake: + autoscaling: + enabled: true + minReplicas: 2 + maxReplicas: 10 + targetCPUUtilizationPercentage: 70 + targetMemoryUtilizationPercentage: 80 + +ui: + autoscaling: + enabled: true + minReplicas: 2 + maxReplicas: 5 + targetCPUUtilizationPercentage: 70 +``` + +#### Multi-AZ Deployment + +```yaml +lake: + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + devlakeComponent: lake + +ui: + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + devlakeComponent: ui +``` + +## Development + +### Local Development + +1. **Clone repository**: +```bash +git clone https://github.com/apache/incubator-devlake-helm-chart.git +cd incubator-devlake-helm-chart +``` + +2. **Install dependencies**: +```bash +helm repo add grafana https://grafana.github.io/helm-charts +helm dependency update charts/devlake +``` + +3. **Validate chart**: +```bash +helm lint charts/devlake +``` + +4. **Test rendering**: +```bash +helm template test charts/devlake \ + --set database.password=test \ + --set database.mysql.rootPassword=test \ + --set lake.encryptionSecret.secret=$(openssl rand -base64 128) +``` + +5. **Run tests**: +```bash +helm install test charts/devlake \ + --set database.password=test \ + --set database.mysql.rootPassword=test \ + --set lake.encryptionSecret.secret=$(openssl rand -base64 128) + +helm test test +``` + +### Contributing + +Contributions welcome! This chart was originally created by [@matrixji](https://github.com/matrixji) with contributions from [@lshmouse](https://github.com/lshmouse), [@shubham-cmyk](https://github.com/shubham-cmyk), and [@SnowMoon-Dev](https://github.com/SnowMoon-Dev). + +**Guidelines**: +- Follow [Helm Best Practices](https://helm.sh/docs/chart_best_practices/) +- Maintain backward compatibility for minor versions +- Add tests for new features +- Update documentation + + +## License + +Licensed under the [Apache License 2.0](LICENSE). + +## Acknowledgments + +Special thanks to the Apache DevLake community and all contributors who have helped improve this Helm chart. + +**Original Contributors**: +- [@matrixji](https://github.com/matrixji) - Initial helm chart implementation +- [@lshmouse](https://github.com/lshmouse) - Helm deployment features +- [@shubham-cmyk](https://github.com/shubham-cmyk) - Configuration improvements +- [@SnowMoon-Dev](https://github.com/SnowMoon-Dev) - Bug fixes and enhancements -https://github.com/apache/incubator-devlake/pulls?q=is%3Apr+helm+is%3Aclosed +**Source**: This repository is derived from the `deployment/helm` directory in [apache/incubator-devlake](https://github.com/apache/incubator-devlake). -## More +--- -You could find more examples and details in [HelmSetup.md](HelmSetup.md) +**Chart Version**: 2.0.0 | **App Version**: v1.0.3-beta12 | **Maintained by**: Apache DevLake Community diff --git a/ReleaseSOP.md b/ReleaseSOP.md deleted file mode 100644 index 878fa7a9..00000000 --- a/ReleaseSOP.md +++ /dev/null @@ -1,8 +0,0 @@ -## How to upgrade helm chart after releasing new devlake images - -1. In [values.yaml](https://github.com/apache/incubator-devlake-helm-chart/blob/main/charts/devlake/values.yaml), change {{ imageTag }} to current image tag -2. In [chart.yaml](https://github.com/apache/incubator-devlake-helm-chart/blob/main/charts/devlake/Chart.yaml), change {{ version }}, {{ appVersion }} to current image tag -3. If we want to release a new chart without new release of devlake, we should increase both chart version and image tag. - - For example, right now both versions are 0.16.1-beta1, if we make change on chart, we should set chart-version to 0.16.1-beta1, also, we need to crate new images for devlake with tag 0.16.1-beta1 -4. If we release any new image for devlake, we just need to set a new version for chart. - diff --git a/charts/devlake/.helmignore b/charts/devlake/.helmignore index 3133ff8a..c37657cb 100644 --- a/charts/devlake/.helmignore +++ b/charts/devlake/.helmignore @@ -38,3 +38,5 @@ .idea/ *.tmproj .vscode/ +# helm-docs template +README.md.gotmpl diff --git a/charts/devlake/Chart.yaml b/charts/devlake/Chart.yaml index 8e5aefac..5aedac04 100644 --- a/charts/devlake/Chart.yaml +++ b/charts/devlake/Chart.yaml @@ -28,10 +28,10 @@ keywords: type: application # Chart version -version: 1.0.3-beta10 +version: 2.0.0 # devlake version -appVersion: v1.0.3-beta10 +appVersion: v1.0.3-beta12 dependencies: - condition: grafana.enabled diff --git a/charts/devlake/README.md b/charts/devlake/README.md new file mode 100644 index 00000000..0008d1c2 --- /dev/null +++ b/charts/devlake/README.md @@ -0,0 +1,271 @@ +# devlake + +Apache DevLake is an open-source dev data platform that ingests, analyzes, and visualizes the fragmented data from DevOps tools to distill insights for engineering productivity. + +![Version: 2.0.0](https://img.shields.io/badge/Version-2.0.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v1.0.3-beta12](https://img.shields.io/badge/AppVersion-v1.0.3--beta12-informational?style=flat-square) + +## Requirements + +| Repository | Name | Version | +|------------|------|---------| +| https://grafana.github.io/helm-charts | grafana | 9.3.2 | + +## Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| alpine.image.pullPolicy | string | `"IfNotPresent"` | | +| alpine.image.repository | string | `"alpine"` | | +| alpine.image.tag | float | `3.16` | | +| backup.backoffLimit | int | `2` | | +| backup.enabled | bool | `false` | | +| backup.failedJobsHistoryLimit | int | `1` | | +| backup.image.postgresTag | string | `"14"` | | +| backup.image.pullPolicy | string | `"IfNotPresent"` | | +| backup.image.repository | string | `"mysql"` | | +| backup.image.tag | string | `"8"` | | +| backup.pvc.existingClaim | string | `""` | | +| backup.pvc.size | string | `"10Gi"` | | +| backup.pvc.storageClassName | string | `""` | | +| backup.retentionDays | int | `7` | | +| backup.schedule | string | `"0 2 * * *"` | | +| backup.successfulJobsHistoryLimit | int | `3` | | +| commonEnvs.TZ | string | `"UTC"` | | +| database.affinity | object | `{}` | | +| database.containerSecurityContext.allowPrivilegeEscalation | bool | `false` | | +| database.containerSecurityContext.capabilities.drop[0] | string | `"ALL"` | | +| database.containerSecurityContext.readOnlyRootFilesystem | bool | `true` | | +| database.containerSecurityContext.runAsNonRoot | bool | `true` | | +| database.database | string | `"lake"` | | +| database.externalPort | int | `3306` | | +| database.externalServer | string | `"127.0.0.1"` | | +| database.extraLabels | object | `{}` | | +| database.image.pullPolicy | string | `"IfNotPresent"` | | +| database.image.repository | string | `""` | | +| database.image.tag | string | `""` | | +| database.initContainers | list | `[]` | | +| database.mysql.enableFixPermissions | bool | `true` | | +| database.mysql.extraParams | string | `""` | | +| database.mysql.rootPassword | string | `""` | | +| database.nodeSelector | object | `{}` | | +| database.password | string | `""` | | +| database.podAnnotations | object | `{}` | | +| database.postgresql.enableFixPermissions | bool | `true` | | +| database.postgresql.extraParams | string | `""` | | +| database.replicaCount | int | `1` | | +| database.resources.limits.cpu | string | `"2000m"` | | +| database.resources.limits.memory | string | `"4Gi"` | | +| database.resources.requests.cpu | string | `"500m"` | | +| database.resources.requests.memory | string | `"1Gi"` | | +| database.securityContext.runAsNonRoot | bool | `true` | | +| database.securityContext.seccompProfile.type | string | `"RuntimeDefault"` | | +| database.service.extraLabels | object | `{}` | | +| database.service.loadBalancerIP | string | `""` | | +| database.service.nodePort | string | `""` | | +| database.service.type | string | `"ClusterIP"` | | +| database.storage.class | string | `""` | | +| database.storage.hostPath | string | `"/devlake/database/data"` | | +| database.storage.size | string | `"50Gi"` | | +| database.storage.type | string | `"pvc"` | | +| database.tolerations | list | `[]` | | +| database.type | string | `"mysql"` | | +| database.useExternal | bool | `false` | | +| database.username | string | `"merico"` | | +| externalSecrets.enabled | bool | `false` | | +| externalSecrets.remoteRefs.dbPassword | string | `""` | | +| externalSecrets.remoteRefs.dbRootPassword | string | `""` | | +| externalSecrets.remoteRefs.encryptionSecret | string | `""` | | +| externalSecrets.remoteRefs.uiPassword | string | `""` | | +| externalSecrets.secretName | string | `""` | | +| externalSecrets.secretStoreRef.kind | string | `"SecretStore"` | | +| externalSecrets.secretStoreRef.name | string | `""` | | +| extraResources | list | `[]` | | +| grafana."grafana.ini".server.root_url | string | `"%(protocol)s://%(domain)s/grafana"` | | +| grafana."grafana.ini".server.serve_from_subpath | string | `"true"` | | +| grafana.adminPassword | string | `""` | | +| grafana.deploymentStrategy.type | string | `"Recreate"` | | +| grafana.enabled | bool | `true` | | +| grafana.env.TZ | string | `"UTC"` | | +| grafana.envFromConfigMaps[0].name | string | `"devlake-db-auth-config"` | | +| grafana.envFromSecrets[0].name | string | `"devlake-db-auth"` | | +| grafana.external.url | string | `""` | | +| grafana.image.registry | string | `"devlake.docker.scarf.sh"` | | +| grafana.image.repository | string | `"apache/devlake-dashboard"` | | +| grafana.image.tag | string | `"v1.0.3-beta10"` | | +| grafana.ingressServiceName | string | `""` | | +| grafana.ingressServicePort | string | `""` | | +| grafana.persistence.enabled | bool | `true` | | +| grafana.persistence.size | string | `"4Gi"` | | +| grafana.serviceAccount.create | bool | `false` | | +| grafana.serviceAccount.nameOverride | string | `"{{ include \"devlake.serviceAccountName\" . }}"` | | +| httpRoute.annotations | object | `{}` | | +| httpRoute.enabled | bool | `false` | | +| httpRoute.extraLabels | object | `{}` | | +| httpRoute.extraPaths | list | `[]` | | +| httpRoute.gatewayName | string | `""` | | +| httpRoute.gatewayNamespace | string | `""` | | +| httpRoute.hostnames | list | `[]` | | +| httpRoute.prefix | string | `"/"` | | +| httpRoute.sectionName | string | `""` | | +| imageDigests.database.mysql | string | `""` | | +| imageDigests.database.postgresql | string | `""` | | +| imageDigests.grafana | string | `""` | | +| imageDigests.lake | string | `""` | | +| imageDigests.ui | string | `""` | | +| imagePullSecrets | list | `[]` | | +| imageTag | string | `"v1.0.3-beta12"` | | +| ingress.annotations | object | `{}` | | +| ingress.className | string | `nil` | | +| ingress.enableHttps | bool | `false` | | +| ingress.enabled | bool | `false` | | +| ingress.extraLabels | object | `{}` | | +| ingress.hostname | string | `"localhost"` | | +| ingress.httpPort | int | `80` | | +| ingress.httpsPort | int | `443` | | +| ingress.prefix | string | `"/"` | | +| ingress.rateLimit | string | `"100"` | | +| ingress.tlsSecretName | string | `""` | | +| ingress.useDefaultNginx | bool | `false` | | +| lake.affinity | object | `{}` | | +| lake.containerSecurityContext.allowPrivilegeEscalation | bool | `false` | | +| lake.containerSecurityContext.capabilities.drop[0] | string | `"ALL"` | | +| lake.containerSecurityContext.readOnlyRootFilesystem | bool | `true` | | +| lake.containerSecurityContext.runAsNonRoot | bool | `true` | | +| lake.containerSecurityContext.runAsUser | int | `1000` | | +| lake.deployment.extraLabels | object | `{}` | | +| lake.encryptionSecret.autoCreateSecret | bool | `true` | | +| lake.encryptionSecret.secret | string | `""` | | +| lake.encryptionSecret.secretName | string | `""` | | +| lake.envs.API_REQUESTS_PER_HOUR | string | `"10000"` | | +| lake.envs.API_RETRY | string | `"3"` | | +| lake.envs.API_TIMEOUT | string | `"120s"` | | +| lake.envs.IN_SECURE_SKIP_VERIFY | string | `"false"` | | +| lake.envs.JIRA_JQL_AUTO_FULL_REFRESH | string | `"true"` | | +| lake.envs.LOGGING_DIR | string | `"/app/logs"` | | +| lake.envs.LOGGING_FORMAT | string | `"json"` | | +| lake.envs.LOGGING_LEVEL | string | `"info"` | | +| lake.envs.PIPELINE_MAX_PARALLEL | string | `"1"` | | +| lake.extraEnvsFromSecret | string | `""` | | +| lake.extraLabels | object | `{}` | | +| lake.hpa.enabled | bool | `false` | | +| lake.hpa.maxReplicas | int | `10` | | +| lake.hpa.minReplicas | int | `1` | | +| lake.hpa.targetCPUUtilizationPercentage | int | `70` | | +| lake.image.pullPolicy | string | `"IfNotPresent"` | | +| lake.image.repository | string | `"devlake.docker.scarf.sh/apache/devlake"` | | +| lake.initContainerSecurityContext.allowPrivilegeEscalation | bool | `false` | | +| lake.initContainerSecurityContext.capabilities.drop[0] | string | `"ALL"` | | +| lake.initContainerSecurityContext.readOnlyRootFilesystem | bool | `true` | | +| lake.initContainerSecurityContext.runAsNonRoot | bool | `true` | | +| lake.initContainerSecurityContext.runAsUser | int | `1000` | | +| lake.initContainers | list | `[]` | | +| lake.livenessProbe.failureThreshold | int | `5` | | +| lake.livenessProbe.httpGet.path | string | `"/ping"` | | +| lake.livenessProbe.httpGet.port | int | `8080` | | +| lake.livenessProbe.httpGet.scheme | string | `"HTTP"` | | +| lake.livenessProbe.initialDelaySeconds | int | `30` | | +| lake.livenessProbe.periodSeconds | int | `5` | | +| lake.livenessProbe.successThreshold | int | `1` | | +| lake.livenessProbe.timeoutSeconds | int | `5` | | +| lake.nodeSelector | object | `{}` | | +| lake.podAnnotations | object | `{}` | | +| lake.podAntiAffinity.enabled | bool | `false` | | +| lake.podAntiAffinity.type | string | `"preferred"` | | +| lake.podDisruptionBudget.enabled | bool | `true` | | +| lake.podDisruptionBudget.minAvailable | int | `1` | | +| lake.port | int | `8080` | | +| lake.readinessProbe.failureThreshold | int | `3` | | +| lake.readinessProbe.httpGet.path | string | `"/ping"` | | +| lake.readinessProbe.httpGet.port | int | `8080` | | +| lake.readinessProbe.httpGet.scheme | string | `"HTTP"` | | +| lake.readinessProbe.initialDelaySeconds | int | `5` | | +| lake.readinessProbe.periodSeconds | int | `5` | | +| lake.readinessProbe.successThreshold | int | `1` | | +| lake.readinessProbe.timeoutSeconds | int | `5` | | +| lake.replicaCount | int | `1` | | +| lake.resources.limits.cpu | string | `"2000m"` | | +| lake.resources.limits.memory | string | `"2Gi"` | | +| lake.resources.requests.cpu | string | `"500m"` | | +| lake.resources.requests.memory | string | `"512Mi"` | | +| lake.revisionHistoryLimit | int | `10` | | +| lake.securityContext.fsGroup | int | `1000` | | +| lake.securityContext.runAsNonRoot | bool | `true` | | +| lake.securityContext.runAsUser | int | `1000` | | +| lake.securityContext.seccompProfile.type | string | `"RuntimeDefault"` | | +| lake.service.extraLabels | object | `{}` | | +| lake.strategy.type | string | `"Recreate"` | | +| lake.tolerations | list | `[]` | | +| lake.topologySpreadConstraints | list | `[]` | | +| lake.volumeMounts | list | `[]` | | +| lake.volumes | list | `[]` | | +| metrics.enabled | bool | `false` | | +| metrics.serviceMonitor.enabled | bool | `false` | | +| networkPolicy.enabled | bool | `false` | | +| networkPolicy.externalDatabaseCIDRs | list | `[]` | | +| networkPolicy.externalEgressCIDRs | list | `[]` | | +| networkPolicy.ui.ingressNamespaceSelector | object | `{}` | | +| networkPolicy.ui.ingressPodSelector | object | `{}` | | +| option.assembleDbUrl | bool | `true` | | +| option.autoCreateSecret | bool | `true` | | +| option.connectionConfigmapName | string | `"devlake-db-auth-config"` | | +| option.connectionSecretName | string | `"devlake-db-auth"` | | +| replicaCount | int | `1` | | +| service.type | string | `"ClusterIP"` | | +| serviceAccount.create | bool | `true` | | +| serviceAccount.name | string | `""` | | +| ui.affinity | object | `{}` | | +| ui.basicAuth.autoCreateSecret | bool | `true` | | +| ui.basicAuth.enabled | bool | `false` | | +| ui.basicAuth.password | string | `""` | | +| ui.basicAuth.secretName | string | `""` | | +| ui.basicAuth.user | string | `"admin"` | | +| ui.containerSecurityContext.allowPrivilegeEscalation | bool | `false` | | +| ui.containerSecurityContext.capabilities.drop[0] | string | `"ALL"` | | +| ui.containerSecurityContext.readOnlyRootFilesystem | bool | `false` | | +| ui.containerSecurityContext.runAsNonRoot | bool | `true` | | +| ui.containerSecurityContext.runAsUser | int | `1001` | | +| ui.deployment.extraLabels | object | `{}` | | +| ui.extraContainers | list | `[]` | | +| ui.extraLabels | object | `{}` | | +| ui.hpa.enabled | bool | `false` | | +| ui.hpa.maxReplicas | int | `10` | | +| ui.hpa.minReplicas | int | `1` | | +| ui.hpa.targetCPUUtilizationPercentage | int | `70` | | +| ui.image.pullPolicy | string | `"IfNotPresent"` | | +| ui.image.repository | string | `"devlake.docker.scarf.sh/apache/devlake-config-ui"` | | +| ui.livenessProbe.failureThreshold | int | `5` | | +| ui.livenessProbe.httpGet.path | string | `"/health/"` | | +| ui.livenessProbe.httpGet.port | int | `4000` | | +| ui.livenessProbe.httpGet.scheme | string | `"HTTP"` | | +| ui.livenessProbe.initialDelaySeconds | int | `15` | | +| ui.livenessProbe.periodSeconds | int | `5` | | +| ui.livenessProbe.successThreshold | int | `1` | | +| ui.livenessProbe.timeoutSeconds | int | `5` | | +| ui.nodeSelector | object | `{}` | | +| ui.podAnnotations | object | `{}` | | +| ui.podAntiAffinity.enabled | bool | `false` | | +| ui.podAntiAffinity.type | string | `"preferred"` | | +| ui.podDisruptionBudget.enabled | bool | `true` | | +| ui.podDisruptionBudget.minAvailable | int | `1` | | +| ui.readinessProbe.failureThreshold | int | `3` | | +| ui.readinessProbe.httpGet.path | string | `"/health/"` | | +| ui.readinessProbe.httpGet.port | int | `4000` | | +| ui.readinessProbe.httpGet.scheme | string | `"HTTP"` | | +| ui.readinessProbe.initialDelaySeconds | int | `5` | | +| ui.readinessProbe.periodSeconds | int | `5` | | +| ui.readinessProbe.successThreshold | int | `1` | | +| ui.readinessProbe.timeoutSeconds | int | `5` | | +| ui.replicaCount | int | `1` | | +| ui.resources.limits.cpu | string | `"500m"` | | +| ui.resources.limits.memory | string | `"512Mi"` | | +| ui.resources.requests.cpu | string | `"100m"` | | +| ui.resources.requests.memory | string | `"128Mi"` | | +| ui.revisionHistoryLimit | int | `10` | | +| ui.securityContext.fsGroup | int | `1001` | | +| ui.securityContext.runAsNonRoot | bool | `true` | | +| ui.securityContext.runAsUser | int | `1001` | | +| ui.securityContext.seccompProfile.type | string | `"RuntimeDefault"` | | +| ui.service.extraLabels | object | `{}` | | +| ui.strategy | object | `{}` | | +| ui.tolerations | list | `[]` | | +| ui.topologySpreadConstraints | list | `[]` | | diff --git a/charts/devlake/README.md.gotmpl b/charts/devlake/README.md.gotmpl new file mode 100644 index 00000000..28ee45a1 --- /dev/null +++ b/charts/devlake/README.md.gotmpl @@ -0,0 +1,9 @@ +{{ template "chart.header" . }} + +{{ template "chart.description" . }} + +{{ template "chart.versionBadge" . }}{{ template "chart.typeBadge" . }}{{ template "chart.appVersionBadge" . }} + +{{ template "chart.requirementsSection" . }} + +{{ template "chart.valuesSection" . }} diff --git a/charts/devlake/templates/NOTES.txt b/charts/devlake/templates/NOTES.txt index 2ed09576..80cb1573 100644 --- a/charts/devlake/templates/NOTES.txt +++ b/charts/devlake/templates/NOTES.txt @@ -15,14 +15,91 @@ # limitations under the License. # -Welcome to use devlake. +Thank you for installing {{ .Chart.Name }} v{{ .Chart.Version }}! {{- if .Values.ingress.enabled }} -Now please visit: + +DevLake UI is accessible at: {{ include "devlake.uiEndpoint" . }} {{- else if contains "NodePort" .Values.service.type }} -Now please get the URL by running these commands: - export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "devlake.fullname" . }}-ui) - export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + +To access DevLake UI, get the URL: + export NODE_PORT=$(kubectl get -n {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" svc {{ include "devlake.fullname" . }}-ui) + export NODE_IP=$(kubectl get nodes -n {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") echo http://$NODE_IP:$NODE_PORT +{{- else }} + +To access DevLake UI, forward the port: + kubectl port-forward -n {{ .Release.Namespace }} svc/{{ include "devlake.fullname" . }}-ui 4000:4000 + +Then visit http://localhost:4000 {{- end }} + +================================================================================ +SECURITY & PRODUCTION READINESS +================================================================================ +{{- if not .Values.networkPolicy.enabled }} + +โš ๏ธ NetworkPolicy is DISABLED (not production-safe) + +For production, enable NetworkPolicy to isolate traffic: + + helm upgrade {{ .Release.Name }} {{ .Chart.Name }} \ + --set networkPolicy.enabled=true \ + --set networkPolicy.externalEgressCIDRs[0]=140.82.112.0/20 # GitHub \ + --set networkPolicy.externalEgressCIDRs[1]=35.231.145.0/24 # GitLab + +Add CIDRs for your integrations (Jira, Jenkins, etc.) +{{- end }} +{{- if not .Values.externalSecrets.enabled }} +{{- if .Values.option.autoCreateSecret }} + +โš ๏ธ Using PLAINTEXT secrets from values.yaml + +For production, use External Secrets Operator: + helm upgrade {{ .Release.Name }} {{ .Chart.Name }} \ + --set externalSecrets.enabled=true \ + --set externalSecrets.secretStoreRef.name= +{{- end }} +{{- end }} +{{- if or (eq .Values.database.storage.type "hostpath") .Values.lake.hostNetwork }} + +โš ๏ธ DEPRECATED configuration detected + +โ€ข database.storage.type=hostpath โ†’ DEPRECATED in v2.1.0, REMOVED in v2.2.0 +โ€ข lake.hostNetwork=true โ†’ DEPRECATED in v2.1.0, REMOVED in v2.2.0 + +Migrate before upgrading to v2.2.0 to avoid installation failure. +{{- end }} + +================================================================================ +POD SECURITY STANDARDS (PSS) +================================================================================ + +This chart supports PSS "restricted" profile. + +To enforce PSS at namespace level: + + kubectl label namespace {{ .Release.Namespace }} \ + pod-security.kubernetes.io/enforce=restricted \ + pod-security.kubernetes.io/audit=restricted \ + pod-security.kubernetes.io/warn=restricted + +================================================================================ +TROUBLESHOOTING +================================================================================ + +If pre-install validation Job fails: + +1. Check logs: + kubectl logs -n {{ .Release.Namespace }} job/{{ include "devlake.fullname" . }}-pre-install-validation + +2. Fix configuration, then retry + +3. Emergency bypass (NOT for production): + helm install {{ .Release.Name }} {{ .Chart.Name }} --no-hooks + +================================================================================ + +Documentation: https://devlake.apache.org/docs/ +GitHub: https://github.com/apache/incubator-devlake diff --git a/charts/devlake/templates/_helpers.tpl b/charts/devlake/templates/_helpers.tpl index 731af593..5bb7b725 100644 --- a/charts/devlake/templates/_helpers.tpl +++ b/charts/devlake/templates/_helpers.tpl @@ -1,9 +1,28 @@ +{{/* +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +*/}} + {{/* Expand the name of the chart. */}} {{- define "devlake.name" -}} {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} -{{- end }} +{{- end -}} {{/* Create a default fully qualified app name. @@ -21,14 +40,14 @@ If release name contains chart name it will be used as a full name. {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} {{- end }} {{- end }} -{{- end }} +{{- end -}} {{/* Create chart name and version as used by the chart label. */}} {{- define "devlake.chart" -}} {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} -{{- end }} +{{- end -}} {{/* Common labels @@ -40,26 +59,47 @@ helm.sh/chart: {{ include "devlake.chart" . }} app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} {{- end }} app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- end }} +app.kubernetes.io/part-of: devlake +{{- end -}} {{/* -Selector labels +Common annotations +*/}} +{{- define "devlake.annotations" -}} +meta.helm.sh/release-name: {{ .Release.Name }} +meta.helm.sh/release-namespace: {{ .Release.Namespace }} +{{- end -}} + +{{/* +Selector labels (returns YAML string) */}} {{- define "devlake.selectorLabels" -}} app.kubernetes.io/name: {{ include "devlake.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} -{{- end }} +{{- end -}} + +{{/* +Selector labels as dict (for matchLabels) +*/}} +{{- define "devlake.selectorLabelsDict" -}} +{{- $labels := dict }} +{{- $_ := set $labels "app.kubernetes.io/name" (include "devlake.name" .) }} +{{- $_ := set $labels "app.kubernetes.io/instance" .Release.Name }} +{{- toYaml $labels }} +{{- end -}} {{/* Create the name of the service account to use */}} {{- define "devlake.serviceAccountName" -}} -{{- if .Values.serviceAccount.create }} -{{- default (include "devlake.fullname" .) .Values.serviceAccount.name }} -{{- else }} -{{- default "default" .Values.serviceAccount.name }} -{{- end }} -{{- end }} +{{- if .Values.serviceAccount.name -}} +{{- .Values.serviceAccount.name -}} +{{- else if .Values.serviceAccount.create -}} +{{- include "devlake.fullname" . }}-sa +{{- else -}} +default +{{- end -}} +{{- end -}} {{/* @@ -67,14 +107,14 @@ The ui endpoint prefix */}} {{- define "devlake.grafanaEndpointPrefix" -}} {{- print .Values.ingress.prefix "/grafana" | replace "//" "/" | trimAll "/" -}} -{{- end }} +{{- end -}} {{/* The ui endpoint prefix */}} {{- define "devlake.uiEndpointPrefix" -}} {{- print .Values.ingress.prefix "/" | replace "//" "/" | trimAll "/" -}} -{{- end }} +{{- end -}} {{/* The ui endpoint @@ -94,84 +134,148 @@ The ui endpoint {{- printf "http://%s%s/%s" .Values.ingress.hostname $uiPortString (include "devlake.uiEndpointPrefix" .) }} {{- end }} {{- end }} -{{- end }} +{{- end -}} -{{- define "devlake.mysql.secret" -}} +{{/* +Database secret name (renamed from devlake.mysql.secret in v2.0.0) +*/}} +{{- define "devlake.db.secret" -}} +{{- if .Values.externalSecrets.enabled -}} +{{- .Values.externalSecrets.secretName -}} +{{- else -}} {{- if .Values.option.connectionSecretName -}} {{- .Values.option.connectionSecretName -}} {{- else -}} -{{ include "devlake.fullname" . }}-db-connection +{{ include "devlake.fullname" . }}-db-auth +{{- end -}} {{- end -}} {{- end -}} -{{- define "devlake.mysql.configmap" -}} +{{/* +Database configmap name (renamed from devlake.mysql.configmap in v2.0.0) +*/}} +{{- define "devlake.db.configmap" -}} {{- if .Values.option.connectionConfigmapName -}} {{- .Values.option.connectionConfigmapName -}} {{- else -}} -{{ include "devlake.fullname" . }}-config +{{ include "devlake.fullname" . }}-db-config {{- end -}} {{- end -}} {{- define "devlake.ui.auth.secret" -}} +{{- if .Values.externalSecrets.enabled -}} +{{- .Values.externalSecrets.secretName -}} +{{- else -}} {{- if .Values.ui.basicAuth.secretName -}} {{- .Values.ui.basicAuth.secretName -}} {{- else -}} {{ include "devlake.fullname" . }}-ui-auth {{- end -}} {{- end -}} +{{- end -}} {{- define "devlake.lake.encryption.secret" -}} +{{- if .Values.externalSecrets.enabled -}} +{{- .Values.externalSecrets.secretName -}} +{{- else -}} {{- if .Values.lake.encryptionSecret.secretName -}} {{- .Values.lake.encryptionSecret.secretName -}} {{- else -}} {{ include "devlake.fullname" . }}-encryption-secret {{- end -}} {{- end -}} +{{- end -}} {{/* -The mysql server +The database server (v2.0.0: refactored for database.type selector) */}} -{{- define "mysql.server" -}} -{{- if .Values.mysql.useExternal }} -{{- .Values.mysql.externalServer }} +{{- define "database.server" -}} +{{- if .Values.database.useExternal }} +{{- .Values.database.externalServer }} {{- else }} -{{- print (include "devlake.fullname" . ) "-mysql" }} -{{- end }} +{{- printf "%s-%s" (include "devlake.fullname" .) .Values.database.type }} {{- end }} - +{{- end -}} {{/* -The mysql port +The database port (v2.0.0: refactored for database.type selector) */}} -{{- define "mysql.port" -}} -{{- if .Values.mysql.useExternal }} -{{- .Values.mysql.externalPort }} -{{- else }} -{{- 3306 }} -{{- end }} -{{- end }} +{{- define "database.port" -}} +{{- if .Values.database.useExternal -}} +{{- .Values.database.externalPort -}} +{{- else -}} +{{- if eq .Values.database.type "mysql" -}} +{{- "3306" -}} +{{- else if eq .Values.database.type "postgresql" -}} +{{- "5432" -}} +{{- end -}} +{{- end -}} +{{- end -}} +{{/* +The database image (v2.0.0: new helper for database.type selector with digest support) +*/}} +{{- define "database.image" -}} +{{- $image := "" -}} +{{- if and .Values.database.image.repository (ne .Values.database.image.repository "") -}} + {{- $image = printf "%s:%s" .Values.database.image.repository (.Values.database.image.tag | toString) -}} +{{- else -}} + {{- if eq .Values.database.type "mysql" -}} + {{- $image = "mysql:8" -}} + {{- else if eq .Values.database.type "postgresql" -}} + {{- $image = "postgres:14" -}} + {{- end -}} +{{- end -}} +{{- if eq .Values.database.type "mysql" -}} + {{- if .Values.imageDigests.database.mysql -}} + {{- printf "%s@%s" $image .Values.imageDigests.database.mysql -}} + {{- else -}} + {{- $image -}} + {{- end -}} +{{- else if eq .Values.database.type "postgresql" -}} + {{- if .Values.imageDigests.database.postgresql -}} + {{- printf "%s@%s" $image .Values.imageDigests.database.postgresql -}} + {{- else -}} + {{- $image -}} + {{- end -}} +{{- end -}} +{{- end -}} {{/* -The database server +Database uid based on type (mysql=999, postgresql=70) */}} -{{- define "database.server" -}} -{{- if eq .Values.option.database "mysql" }} -{{- include "mysql.server" . }} -{{- end }} -{{- end }} - +{{- define "database.uid" -}} +{{- if eq .Values.database.type "mysql" -}} +{{- "999" -}} +{{- else if eq .Values.database.type "postgresql" -}} +{{- "70" -}} +{{- end -}} +{{- end -}} {{/* -The database port +Lake image with optional digest */}} -{{- define "database.port" -}} -{{- if eq .Values.option.database "mysql" }} -{{- include "mysql.port" . }} -{{- end }} -{{- end }} +{{- define "devlake.lake.image" -}} +{{- $image := printf "%s:%s" .Values.lake.image.repository (.Values.lake.image.tag | default .Values.imageTag) -}} +{{- if .Values.imageDigests.lake -}} +{{- printf "%s@%s" $image .Values.imageDigests.lake -}} +{{- else -}} +{{- $image -}} +{{- end -}} +{{- end -}} +{{/* +UI image with optional digest +*/}} +{{- define "devlake.ui.image" -}} +{{- $image := printf "%s:%s" .Values.ui.image.repository (.Values.ui.image.tag | default .Values.imageTag) -}} +{{- if .Values.imageDigests.ui -}} +{{- printf "%s@%s" $image .Values.imageDigests.ui -}} +{{- else -}} +{{- $image -}} +{{- end -}} +{{- end -}} {{/* The probe for check database connection @@ -189,4 +293,46 @@ The probe for check database connection sleep 2 done echo database is ready +{{- end -}} + +{{/* +Pod Anti-Affinity helper +*/}} +{{- define "devlake.podAntiAffinity" -}} +{{- if .enabled }} +affinity: + podAntiAffinity: + {{- if eq .type "required" }} + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app.kubernetes.io/name: {{ include "devlake.name" .root }} + app.kubernetes.io/instance: {{ .root.Release.Name }} + devlakeComponent: {{ .componentName }} + topologyKey: kubernetes.io/hostname + {{- else }} + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchLabels: + app.kubernetes.io/name: {{ include "devlake.name" .root }} + app.kubernetes.io/instance: {{ .root.Release.Name }} + devlakeComponent: {{ .componentName }} + topologyKey: kubernetes.io/hostname + {{- end }} {{- end }} +{{- end -}} + +{{/* +PodDisruptionBudget helper - uses maxUnavailable when replicaCount=1, minAvailable otherwise +*/}} +{{- define "devlake.podDisruptionBudget" -}} +{{- if .pdb.enabled }} +{{- if eq (int .replicaCount) 1 }} +maxUnavailable: 1 +{{- else }} +minAvailable: {{ .pdb.minAvailable }} +{{- end }} +{{- end }} +{{- end -}} diff --git a/charts/devlake/templates/backup-cronjob.yaml b/charts/devlake/templates/backup-cronjob.yaml new file mode 100644 index 00000000..32e1a963 --- /dev/null +++ b/charts/devlake/templates/backup-cronjob.yaml @@ -0,0 +1,108 @@ +{{/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{- if .Values.backup.enabled }} +apiVersion: batch/v1 +kind: CronJob +metadata: + name: {{ include "devlake.fullname" . }}-backup + namespace: {{ .Release.Namespace }} + labels: + {{- include "devlake.labels" . | nindent 4 }} + app.kubernetes.io/component: backup + annotations: + {{- include "devlake.annotations" . | nindent 4 }} +spec: + schedule: "{{ .Values.backup.schedule }}" + successfulJobsHistoryLimit: {{ .Values.backup.successfulJobsHistoryLimit }} + failedJobsHistoryLimit: {{ .Values.backup.failedJobsHistoryLimit }} + concurrencyPolicy: Forbid + jobTemplate: + spec: + backoffLimit: {{ .Values.backup.backoffLimit }} + template: + metadata: + labels: + {{- include "devlake.selectorLabels" . | nindent 12 }} + app.kubernetes.io/component: backup + spec: + restartPolicy: OnFailure + serviceAccountName: {{ include "devlake.serviceAccountName" . }} + securityContext: + runAsNonRoot: true + runAsUser: 65534 + fsGroup: 65534 + seccompProfile: + type: RuntimeDefault + containers: + - name: backup + {{- if eq .Values.database.type "mysql" }} + image: "{{ .Values.backup.image.repository }}:{{ .Values.backup.image.tag }}" + {{- else if eq .Values.database.type "postgresql" }} + image: "postgres:{{ .Values.backup.image.postgresTag | default "14" }}" + {{- end }} + imagePullPolicy: {{ .Values.backup.image.pullPolicy }} + securityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + envFrom: + - configMapRef: + name: {{ include "devlake.db.configmap" . }} + - secretRef: + name: {{ include "devlake.db.secret" . }} + command: + - sh + - -c + - | + set -e + TIMESTAMP=$(date +%Y%m%d-%H%M%S) + BACKUP_FILE="/backup/devlake-${TIMESTAMP}.sql.gz" + + echo "Starting backup at ${TIMESTAMP}..." + + {{- if eq .Values.database.type "mysql" }} + mysqldump -h"${MYSQL_SERVER}" -P"${MYSQL_PORT}" -u"${MYSQL_USER}" -p"${MYSQL_PASSWORD}" "${MYSQL_DATABASE}" \ + --single-transaction --quick --lock-tables=false | gzip > "${BACKUP_FILE}" + {{- else if eq .Values.database.type "postgresql" }} + PGPASSWORD="${DB_PASSWORD}" pg_dump -h "${DB_SERVER}" -p "${DB_PORT}" -U "${DB_USER}" "${DB_DATABASE}" \ + --no-owner --no-acl | gzip > "${BACKUP_FILE}" + {{- end }} + + echo "Backup completed: ${BACKUP_FILE}" + ls -lh "${BACKUP_FILE}" + + # Retention: delete backups older than specified days + {{- if .Values.backup.retentionDays }} + echo "Cleaning backups older than {{ .Values.backup.retentionDays }} days..." + find /backup -name "devlake-*.sql.gz" -type f -mtime +{{ .Values.backup.retentionDays }} -delete + {{- end }} + + echo "Backup job finished successfully" + volumeMounts: + - name: backup + mountPath: /backup + - name: tmp + mountPath: /tmp + volumes: + - name: backup + persistentVolumeClaim: + claimName: {{ .Values.backup.pvc.existingClaim | default (printf "%s-backup" (include "devlake.fullname" .)) }} + - name: tmp + emptyDir: {} +{{- end }} diff --git a/charts/devlake/templates/backup-pvc.yaml b/charts/devlake/templates/backup-pvc.yaml new file mode 100644 index 00000000..72a589c6 --- /dev/null +++ b/charts/devlake/templates/backup-pvc.yaml @@ -0,0 +1,37 @@ +{{/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{- if and .Values.backup.enabled (not .Values.backup.pvc.existingClaim) }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "devlake.fullname" . }}-backup + namespace: {{ .Release.Namespace }} + labels: + {{- include "devlake.labels" . | nindent 4 }} + app.kubernetes.io/component: storage + annotations: + {{- include "devlake.annotations" . | nindent 4 }} +spec: + accessModes: + - ReadWriteOnce + {{- with .Values.backup.pvc.storageClassName }} + storageClassName: "{{ . }}" + {{- end }} + resources: + requests: + storage: {{ .Values.backup.pvc.size | default "10Gi" }} +{{- end }} diff --git a/charts/devlake/templates/configmap.yaml b/charts/devlake/templates/configmap.yaml index 207e6d49..5c74e55e 100644 --- a/charts/devlake/templates/configmap.yaml +++ b/charts/devlake/templates/configmap.yaml @@ -1,36 +1,56 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# +{{/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} --- apiVersion: v1 kind: ConfigMap metadata: - name: {{ include "devlake.mysql.configmap" . }} + name: {{ include "devlake.db.configmap" . }} + namespace: {{ .Release.Namespace }} labels: {{- include "devlake.labels" . | nindent 4 }} + app.kubernetes.io/component: config + annotations: + {{- include "devlake.annotations" . | nindent 4 }} data: # Database connection configuration (non-sensitive) -{{- if (eq .Values.option.database "mysql") }} - MYSQL_USER: "{{ .Values.mysql.username }}" - MYSQL_DATABASE: "{{ .Values.mysql.database }}" - MYSQL_URL: "{{ include "mysql.server" . }}:{{ include "mysql.port" . }}" - MYSQL_SERVER: "{{ include "mysql.server" . }}" - MYSQL_PORT: "{{ include "mysql.port" . }}" + # Note: Environment variable naming differs by database type: + # - MySQL: MYSQL_* prefix (standard MySQL client convention) + # - PostgreSQL: DB_* prefix (PostgreSQL client convention) + # This follows each database's standard naming to avoid conflicts +{{- if eq .Values.database.type "mysql" }} + MYSQL_USER: "{{ .Values.database.username }}" + MYSQL_DATABASE: "{{ .Values.database.database }}" + MYSQL_URL: "{{ include "database.server" . }}:{{ include "database.port" . }}" + MYSQL_SERVER: "{{ include "database.server" . }}" + MYSQL_PORT: "{{ include "database.port" . }}" DB_CHARSET: "utf8mb4" DB_PARSE_TIME: "True" DB_LOCATION: "{{ .Values.commonEnvs.TZ }}" - DB_CUSTOM_PARAMS: "{{ .Values.mysql.extraParams }}" + DB_CUSTOM_PARAMS: "{{ .Values.database.mysql.extraParams }}" +{{- else if eq .Values.database.type "postgresql" }} + # Lake envs (DB_* prefix) + DB_USER: "{{ .Values.database.username }}" + DB_DATABASE: "{{ .Values.database.database }}" + DB_URL: "{{ include "database.server" . }}:{{ include "database.port" . }}" + DB_SERVER: "{{ include "database.server" . }}" + DB_PORT: "{{ include "database.port" . }}" + DB_CUSTOM_PARAMS: "{{ .Values.database.postgresql.extraParams }}" + # Grafana envs (POSTGRES_* prefix for entrypoint.sh) + POSTGRES_USER: "{{ .Values.database.username }}" + POSTGRES_DATABASE: "{{ .Values.database.database }}" + POSTGRES_URL: "{{ include "database.server" . }}:{{ include "database.port" . }}" {{- end }} diff --git a/charts/devlake/templates/deployments.yaml b/charts/devlake/templates/deployments.yaml index a81c8c55..a40e14b6 100644 --- a/charts/devlake/templates/deployments.yaml +++ b/charts/devlake/templates/deployments.yaml @@ -1,30 +1,34 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# +{{/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} --- # devlake-ui apiVersion: apps/v1 kind: Deployment metadata: name: {{ include "devlake.fullname" . }}-ui + namespace: {{ .Release.Namespace }} labels: {{- include "devlake.labels" . | nindent 4 }} + app.kubernetes.io/component: ui {{- with .Values.ui.deployment.extraLabels }} {{- toYaml . | nindent 4 }} {{- end }} + annotations: + {{- include "devlake.annotations" . | nindent 4 }} spec: replicas: {{ .Values.ui.replicaCount }} revisionHistoryLimit: {{ .Values.ui.revisionHistoryLimit }} @@ -51,23 +55,23 @@ spec: imagePullSecrets: {{- toYaml . | nindent 8 }} {{- end }} - {{- with .Values.ui.serviceAccount.name }} - serviceAccountName: {{ . }} - {{- end }} + serviceAccountName: {{ include "devlake.serviceAccountName" . }} + automountServiceAccountToken: false {{- with .Values.ui.securityContext }} securityContext: {{- toYaml . | nindent 8 }} {{- end }} + {{- include "devlake.podAntiAffinity" (dict "enabled" .Values.ui.podAntiAffinity.enabled "type" .Values.ui.podAntiAffinity.type "componentName" "ui" "root" .) | nindent 6 }} + {{- with .Values.ui.topologySpreadConstraints }} + topologySpreadConstraints: + {{- toYaml . | nindent 8 }} + {{- end }} containers: {{- with .Values.ui.extraContainers }} {{- tpl (toYaml .) $ | nindent 8 }} {{- end }} - name: config-ui -{{- if .Values.ui.image.tag }} - image: "{{ .Values.ui.image.repository }}:{{ .Values.ui.image.tag }}" -{{- else }} - image: "{{ .Values.ui.image.repository }}:{{ .Values.imageTag }}" -{{- end }} + image: {{ include "devlake.ui.image" . }} imagePullPolicy: {{ .Values.ui.image.pullPolicy }} ports: - containerPort: 4000 @@ -127,13 +131,17 @@ apiVersion: apps/v1 kind: Deployment metadata: name: {{ include "devlake.fullname" . }}-lake + namespace: {{ .Release.Namespace }} labels: - {{- include "devlake.labels" . | nindent 4 }} - {{- with .Values.lake.deployment.extraLabels }} - {{- toYaml . | nindent 4 }} - {{- end }} + {{- include "devlake.labels" . | nindent 4 }} + app.kubernetes.io/component: lake + {{- with .Values.lake.deployment.extraLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} + annotations: + {{- include "devlake.annotations" . | nindent 4 }} spec: - replicas: {{ if gt (int .Values.lake.replicaCount) 1 }}1{{ else }}{{ .Values.lake.replicaCount }}{{ end }} + replicas: {{ .Values.lake.replicaCount }} revisionHistoryLimit: {{ .Values.lake.revisionHistoryLimit }} {{- with .Values.lake.strategy }} strategy: @@ -158,13 +166,17 @@ spec: imagePullSecrets: {{- toYaml . | nindent 8 }} {{- end }} - {{- with .Values.lake.serviceAccount.name }} - serviceAccountName: {{ . }} - {{- end }} + serviceAccountName: {{ include "devlake.serviceAccountName" . }} + automountServiceAccountToken: false {{- with .Values.lake.securityContext }} securityContext: {{- toYaml . | nindent 8 }} {{- end }} + {{- include "devlake.podAntiAffinity" (dict "enabled" .Values.lake.podAntiAffinity.enabled "type" .Values.lake.podAntiAffinity.type "componentName" "lake" "root" .) | nindent 6 }} + {{- with .Values.lake.topologySpreadConstraints }} + topologySpreadConstraints: + {{- toYaml . | nindent 8 }} + {{- end }} initContainers: {{- if .Values.lake.initContainers }} {{- toYaml .Values.lake.initContainers | nindent 8 }} @@ -176,11 +188,7 @@ spec: {{- end }} containers: - name: lake - {{- if .Values.lake.image.tag }} - image: "{{ .Values.lake.image.repository }}:{{ .Values.lake.image.tag }}" - {{- else }} - image: "{{ .Values.lake.image.repository }}:{{ .Values.imageTag }}" - {{- end }} + image: {{ include "devlake.lake.image" . }} imagePullPolicy: {{ .Values.lake.image.pullPolicy }} ports: - containerPort: {{ .Values.lake.port }} @@ -194,9 +202,9 @@ spec: {{- end }} envFrom: - configMapRef: - name: {{ include "devlake.mysql.configmap" . }} + name: {{ include "devlake.db.configmap" . }} - secretRef: - name: {{ include "devlake.mysql.secret" . }} + name: {{ include "devlake.db.secret" . }} - secretRef: name: {{ include "devlake.lake.encryption.secret" . }} {{- if .Values.lake.extraEnvsFromSecret }} @@ -206,9 +214,14 @@ spec: env: - name: PORT value: "{{ .Values.lake.port }}" - {{- if and (eq .Values.option.database "mysql") (.Values.option.assembleDbUrl) }} + {{- if .Values.option.assembleDbUrl }} + {{- if eq .Values.database.type "mysql" }} + - name: DB_URL + value: "mysql://$(MYSQL_USER):$(MYSQL_PASSWORD)@$(MYSQL_SERVER):$(MYSQL_PORT)/$(MYSQL_DATABASE)?charset=$(DB_CHARSET)&parseTime=$(DB_PARSE_TIME)&loc=$(DB_LOCATION){{ .Values.database.mysql.extraParams }}" + {{- else if eq .Values.database.type "postgresql" }} - name: DB_URL - value: "mysql://$(MYSQL_USER):$(MYSQL_PASSWORD)@$(MYSQL_SERVER):$(MYSQL_PORT)/$(MYSQL_DATABASE)?charset=$(DB_CHARSET)&parseTime=$(DB_PARSE_TIME)&loc=$(DB_LOCATION){{ .Values.mysql.extraParams }}" + value: "postgres://$(DB_USER):$(DB_PASSWORD)@$(DB_SERVER):$(DB_PORT)/$(DB_DATABASE){{ .Values.database.postgresql.extraParams }}" + {{- end }} {{- end }} {{- range $key1, $value1 := .Values.lake.envs }} - name: "{{ tpl $key1 $ }}" @@ -222,8 +235,16 @@ spec: resources: {{- toYaml . | nindent 12 }} {{- end }} - {{- if .Values.lake.volumeMounts }} volumeMounts: + - name: tmp + mountPath: /tmp + - name: logs + mountPath: /app/logs + - name: cache + mountPath: /.cache + - name: shadow-azuredevops + mountPath: /app/python/plugins/azuredevops + {{- if .Values.lake.volumeMounts }} {{- range $volumeMount := .Values.lake.volumeMounts }} - {{- $volumeMount | toYaml | nindent 14 }} {{- end }} @@ -231,17 +252,21 @@ spec: {{- with .Values.lake.containerSecurityContext }} securityContext: {{- toYaml . | nindent 12 }} - {{- end }} - {{- if .Values.lake.volumes }} + {{- end }} volumes: + - name: tmp + emptyDir: {} + - name: logs + emptyDir: {} + - name: cache + emptyDir: {} + - name: shadow-azuredevops + emptyDir: {} + {{- if .Values.lake.volumes }} {{- range $volume := .Values.lake.volumes }} - {{- $volume | toYaml | nindent 10 }} {{- end }} {{- end }} - {{- if .Values.lake.hostNetwork }} - hostNetwork: true - dnsPolicy: ClusterFirstWithHostNet - {{- end }} {{- with .Values.lake.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} @@ -253,6 +278,4 @@ spec: {{- with .Values.lake.tolerations }} tolerations: {{- toYaml . | nindent 8 }} - {{- end }} - - + {{- end }} diff --git a/charts/devlake/templates/external-secret.yaml b/charts/devlake/templates/external-secret.yaml new file mode 100644 index 00000000..a0d7cdd7 --- /dev/null +++ b/charts/devlake/templates/external-secret.yaml @@ -0,0 +1,68 @@ +{{/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{- if .Values.externalSecrets.enabled }} +--- +apiVersion: external-secrets.io/v1beta1 +kind: ExternalSecret +metadata: + name: {{ include "devlake.fullname" . }}-external-secret + namespace: {{ .Release.Namespace }} + labels: + {{- include "devlake.labels" . | nindent 4 }} + app.kubernetes.io/component: secret + annotations: + {{- include "devlake.annotations" . | nindent 4 }} +spec: + secretStoreRef: + name: {{ .Values.externalSecrets.secretStoreRef.name }} + kind: {{ .Values.externalSecrets.secretStoreRef.kind }} + target: + name: {{ .Values.externalSecrets.secretName }} + creationPolicy: Owner + data: + {{- if .Values.externalSecrets.remoteRefs.dbPassword }} + {{- if eq .Values.database.type "mysql" }} + - secretKey: MYSQL_PASSWORD + remoteRef: + key: {{ .Values.externalSecrets.remoteRefs.dbPassword }} + {{- else if eq .Values.database.type "postgresql" }} + - secretKey: DB_PASSWORD + remoteRef: + key: {{ .Values.externalSecrets.remoteRefs.dbPassword }} + {{- end }} + {{- end }} + {{- if and (eq .Values.database.type "mysql") .Values.externalSecrets.remoteRefs.dbRootPassword }} + - secretKey: MYSQL_ROOT_PASSWORD + remoteRef: + key: {{ .Values.externalSecrets.remoteRefs.dbRootPassword }} + {{- end }} + {{- if .Values.externalSecrets.remoteRefs.encryptionSecret }} + - secretKey: ENCRYPTION_SECRET + remoteRef: + key: {{ .Values.externalSecrets.remoteRefs.encryptionSecret }} + {{- end }} + {{- if and .Values.ui.basicAuth.enabled .Values.externalSecrets.remoteRefs.uiPassword }} + - secretKey: ADMIN_USER + remoteRef: + key: {{ .Values.externalSecrets.remoteRefs.uiPassword }} + property: username + - secretKey: ADMIN_PASS + remoteRef: + key: {{ .Values.externalSecrets.remoteRefs.uiPassword }} + property: password + {{- end }} +{{- end }} diff --git a/charts/devlake/templates/extraresources.yaml b/charts/devlake/templates/extraresources.yaml index 39338a8d..a646c7d7 100644 --- a/charts/devlake/templates/extraresources.yaml +++ b/charts/devlake/templates/extraresources.yaml @@ -1,3 +1,20 @@ +{{/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + {{- if .Values.extraResources }} {{- range .Values.extraResources }} --- @@ -13,4 +30,4 @@ {{ toYaml $resource | nindent 0 }} {{- end }} -{{- end }} \ No newline at end of file +{{- end }} diff --git a/charts/devlake/templates/hpa.yaml b/charts/devlake/templates/hpa.yaml new file mode 100644 index 00000000..672dd85b --- /dev/null +++ b/charts/devlake/templates/hpa.yaml @@ -0,0 +1,70 @@ +{{/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{- if .Values.lake.hpa.enabled }} +--- +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "devlake.fullname" . }}-lake + namespace: {{ .Release.Namespace }} + labels: + {{- include "devlake.labels" . | nindent 4 }} + app.kubernetes.io/component: lake + annotations: + {{- include "devlake.annotations" . | nindent 4 }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "devlake.fullname" . }}-lake + minReplicas: {{ .Values.lake.hpa.minReplicas }} + maxReplicas: {{ .Values.lake.hpa.maxReplicas }} + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: {{ .Values.lake.hpa.targetCPUUtilizationPercentage }} +{{- end }} +{{- if .Values.ui.hpa.enabled }} +--- +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "devlake.fullname" . }}-ui + namespace: {{ .Release.Namespace }} + labels: + {{- include "devlake.labels" . | nindent 4 }} + app.kubernetes.io/component: ui + annotations: + {{- include "devlake.annotations" . | nindent 4 }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "devlake.fullname" . }}-ui + minReplicas: {{ .Values.ui.hpa.minReplicas }} + maxReplicas: {{ .Values.ui.hpa.maxReplicas }} + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: {{ .Values.ui.hpa.targetCPUUtilizationPercentage }} +{{- end }} diff --git a/charts/devlake/templates/httproute.yaml b/charts/devlake/templates/httproute.yaml new file mode 100644 index 00000000..0bcd0683 --- /dev/null +++ b/charts/devlake/templates/httproute.yaml @@ -0,0 +1,69 @@ +{{/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{- if .Values.httpRoute.enabled -}} +{{- $fullName := include "devlake.fullname" . -}} +{{- $uiServiceName := printf "%s-%s" $fullName "ui" -}} +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: {{ $fullName }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "devlake.labels" . | nindent 4 }} + app.kubernetes.io/component: httproute + {{- with .Values.httpRoute.extraLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} + annotations: + {{- include "devlake.annotations" . | nindent 4 }} + {{- with .Values.httpRoute.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + parentRefs: + - name: {{ .Values.httpRoute.gatewayName }} + {{- if .Values.httpRoute.gatewayNamespace }} + namespace: {{ .Values.httpRoute.gatewayNamespace }} + {{- end }} + {{- if .Values.httpRoute.sectionName }} + sectionName: {{ .Values.httpRoute.sectionName }} + {{- end }} + {{- if .Values.httpRoute.hostnames }} + hostnames: + {{- range .Values.httpRoute.hostnames }} + - {{ . | quote }} + {{- end }} + {{- end }} + rules: + {{- if .Values.grafana.enabled }} + - matches: + - path: + type: PathPrefix + value: /{{ include "devlake.grafanaEndpointPrefix" . }} + backendRefs: + - name: {{ .Values.grafana.ingressServiceName | default ( include "grafana.fullname" (dict "Values" .Values.grafana "Chart" (dict "Name" "grafana") "Release" .Release ) ) }} + port: {{ .Values.grafana.ingressServicePort | default .Values.grafana.service.port | default 80 }} + {{- end }} + - matches: + - path: + type: PathPrefix + value: {{ .Values.httpRoute.prefix | default "/" }} + backendRefs: + - name: {{ $uiServiceName }} + port: 4000 +{{- end }} diff --git a/charts/devlake/templates/ingresses.yaml b/charts/devlake/templates/ingresses.yaml index 3bce0c44..cf45bcdf 100644 --- a/charts/devlake/templates/ingresses.yaml +++ b/charts/devlake/templates/ingresses.yaml @@ -1,19 +1,19 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# +{{/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} {{- if .Values.ingress.enabled -}} {{- $fullName := include "devlake.fullname" . -}} {{- $uiServiceName := printf "%s-%s" $fullName "ui" -}} @@ -33,12 +33,18 @@ apiVersion: extensions/v1beta1 kind: Ingress metadata: name: {{ $fullName }} + namespace: {{ .Release.Namespace }} labels: {{- include "devlake.labels" . | nindent 4 }} + app.kubernetes.io/component: ingress {{- with .Values.ingress.extraLabels }} {{- toYaml . | nindent 4 }} {{- end }} annotations: + {{- include "devlake.annotations" . | nindent 4 }} + {{- if .Values.ingress.useDefaultNginx }} + nginx.ingress.kubernetes.io/rate-limit: {{ .Values.ingress.rateLimit | quote }} + {{- end }} {{- with .Values.ingress.annotations }} {{- toYaml . | nindent 4 }} {{- end }} diff --git a/charts/devlake/templates/networkpolicy-database.yaml b/charts/devlake/templates/networkpolicy-database.yaml new file mode 100644 index 00000000..7568f5cb --- /dev/null +++ b/charts/devlake/templates/networkpolicy-database.yaml @@ -0,0 +1,54 @@ +{{/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +--- +{{- if .Values.networkPolicy.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "devlake.fullname" . }}-database + namespace: {{ .Release.Namespace }} + labels: + {{- include "devlake.labels" . | nindent 4 }} + app.kubernetes.io/component: network-policy + annotations: + {{- include "devlake.annotations" . | nindent 4 }} +spec: + podSelector: + matchLabels: + devlakeComponent: {{ if eq .Values.database.type "mysql" }}mysql{{ else }}postgresql{{ end }} + policyTypes: + - Ingress + ingress: + # Lake pods access + - from: + - podSelector: + matchLabels: + devlakeComponent: lake + ports: + - protocol: TCP + port: {{ if eq .Values.database.type "mysql" }}3306{{ else }}5432{{ end }} +{{- if and .Values.grafana.enabled (eq .Values.database.type "mysql") }} + # Grafana dashboard access (only for MySQL - Grafana dashboards don't support PostgreSQL datasources) + - from: + - podSelector: + matchLabels: + app.kubernetes.io/name: grafana + ports: + - protocol: TCP + port: 3306 +{{- end }} +{{- end }} diff --git a/charts/devlake/templates/networkpolicy-lake.yaml b/charts/devlake/templates/networkpolicy-lake.yaml new file mode 100644 index 00000000..37f384f3 --- /dev/null +++ b/charts/devlake/templates/networkpolicy-lake.yaml @@ -0,0 +1,93 @@ +{{/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +--- +{{- if .Values.networkPolicy.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "devlake.fullname" . }}-lake + namespace: {{ .Release.Namespace }} + labels: + {{- include "devlake.labels" . | nindent 4 }} + app.kubernetes.io/component: network-policy + annotations: + {{- include "devlake.annotations" . | nindent 4 }} +spec: + podSelector: + matchLabels: + devlakeComponent: lake + policyTypes: + - Ingress + - Egress + ingress: + - from: + - podSelector: + matchLabels: + devlakeComponent: ui + ports: + - protocol: TCP + port: 8080 + egress: + # Database access +{{- if .Values.database.useExternal }} + # External database access + - to: +{{- if .Values.networkPolicy.externalDatabaseCIDRs }} +{{- range .Values.networkPolicy.externalDatabaseCIDRs }} + - ipBlock: + cidr: {{ . }} +{{- end }} +{{- else }} + - ipBlock: + cidr: 0.0.0.0/0 +{{- end }} + ports: + - protocol: TCP + port: {{ include "database.port" . }} +{{- else }} + # Internal database access + - to: + - podSelector: + matchLabels: + devlakeComponent: {{ if eq .Values.database.type "mysql" }}mysql{{ else }}postgresql{{ end }} + ports: + - protocol: TCP + port: {{ if eq .Values.database.type "mysql" }}3306{{ else }}5432{{ end }} +{{- end }} + # DNS resolution + # Allow DNS to kube-system namespace (supports both kube-dns and CoreDNS) + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + ports: + - protocol: UDP + port: 53 + - protocol: TCP + port: 53 +{{- if .Values.networkPolicy.externalEgressCIDRs }} + # External API access (e.g., GitHub, GitLab, Jira) +{{- range .Values.networkPolicy.externalEgressCIDRs }} + - to: + - ipBlock: + cidr: {{ . }} + ports: + - protocol: TCP + port: 443 +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/devlake/templates/networkpolicy-ui.yaml b/charts/devlake/templates/networkpolicy-ui.yaml new file mode 100644 index 00000000..079b97c8 --- /dev/null +++ b/charts/devlake/templates/networkpolicy-ui.yaml @@ -0,0 +1,56 @@ +{{/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +--- +{{- if .Values.networkPolicy.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "devlake.fullname" . }}-ui + namespace: {{ .Release.Namespace }} + labels: + {{- include "devlake.labels" . | nindent 4 }} + app.kubernetes.io/component: network-policy + annotations: + {{- include "devlake.annotations" . | nindent 4 }} +spec: + podSelector: + matchLabels: + devlakeComponent: ui + policyTypes: + - Ingress + ingress: +{{- if or .Values.networkPolicy.ui.ingressNamespaceSelector .Values.networkPolicy.ui.ingressPodSelector }} + # Restricted ingress based on configured selectors + - from: +{{- if .Values.networkPolicy.ui.ingressNamespaceSelector }} + - namespaceSelector: + {{- toYaml .Values.networkPolicy.ui.ingressNamespaceSelector | nindent 12 }} +{{- end }} +{{- if .Values.networkPolicy.ui.ingressPodSelector }} + podSelector: + {{- toYaml .Values.networkPolicy.ui.ingressPodSelector | nindent 12 }} +{{- end }} + ports: + - protocol: TCP + port: 4000 +{{- else }} + # Open ingress (no from: selector) - restrict via networkPolicy.ui.ingressNamespaceSelector or ingressPodSelector + - ports: + - protocol: TCP + port: 4000 +{{- end }} +{{- end }} diff --git a/charts/devlake/templates/poddisruptionbudget.yaml b/charts/devlake/templates/poddisruptionbudget.yaml new file mode 100644 index 00000000..39e7c44c --- /dev/null +++ b/charts/devlake/templates/poddisruptionbudget.yaml @@ -0,0 +1,54 @@ +{{/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{- if .Values.lake.podDisruptionBudget.enabled }} +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ include "devlake.fullname" . }}-lake + namespace: {{ .Release.Namespace }} + labels: + {{- include "devlake.labels" . | nindent 4 }} + app.kubernetes.io/component: lake + annotations: + {{- include "devlake.annotations" . | nindent 4 }} +spec: + {{- include "devlake.podDisruptionBudget" (dict "pdb" .Values.lake.podDisruptionBudget "replicaCount" .Values.lake.replicaCount) | nindent 2 }} + selector: + matchLabels: + {{- include "devlake.selectorLabels" . | nindent 6 }} + devlakeComponent: lake +{{- end }} +{{- if .Values.ui.podDisruptionBudget.enabled }} +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ include "devlake.fullname" . }}-ui + namespace: {{ .Release.Namespace }} + labels: + {{- include "devlake.labels" . | nindent 4 }} + app.kubernetes.io/component: ui + annotations: + {{- include "devlake.annotations" . | nindent 4 }} +spec: + {{- include "devlake.podDisruptionBudget" (dict "pdb" .Values.ui.podDisruptionBudget "replicaCount" .Values.ui.replicaCount) | nindent 2 }} + selector: + matchLabels: + {{- include "devlake.selectorLabels" . | nindent 6 }} + devlakeComponent: ui +{{- end }} diff --git a/charts/devlake/templates/pre-install-validation.yaml b/charts/devlake/templates/pre-install-validation.yaml new file mode 100644 index 00000000..fb120922 --- /dev/null +++ b/charts/devlake/templates/pre-install-validation.yaml @@ -0,0 +1,142 @@ +{{/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "devlake.fullname" . }}-pre-install-validation + namespace: {{ .Release.Namespace }} + labels: + {{- include "devlake.labels" . | nindent 4 }} + app.kubernetes.io/component: validation + annotations: + {{- include "devlake.annotations" . | nindent 4 }} + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-weight: "-5" + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded,hook-failed +spec: + backoffLimit: 1 + activeDeadlineSeconds: 60 + template: + metadata: + labels: + {{- include "devlake.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: pre-install-validation + spec: + restartPolicy: Never + automountServiceAccountToken: false + securityContext: + runAsNonRoot: true + runAsUser: 65534 + seccompProfile: + type: RuntimeDefault + containers: + - name: validation + image: "{{ .Values.alpine.image.repository }}:{{ .Values.alpine.image.tag }}" + imagePullPolicy: {{ .Values.alpine.image.pullPolicy }} + resources: + requests: + cpu: 10m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + securityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + command: + - sh + - -c + - | + #!/bin/sh + set -e + + echo "Validating DevLake configuration..." + + {{- if not .Values.externalSecrets.enabled }} + # Only validate when using internal secrets + {{- if .Values.option.autoCreateSecret }} + + # Validation happens at Helm template time via validate.yaml + # This job just confirms template validation passed + echo "โœ“ Template validation passed" + echo "โœ“ database.type={{ .Values.database.type }}" + echo "โœ“ database.password is set" + {{- if eq .Values.database.type "mysql" }} + {{- if not .Values.database.useExternal }} + echo "โœ“ database.mysql.rootPassword is set" + {{- end }} + {{- end }} + {{- if .Values.lake.encryptionSecret.autoCreateSecret }} + echo "โœ“ lake.encryptionSecret.secret is set" + {{- end }} + + {{- end }} + + # Check UI basic auth password + {{- if .Values.ui.basicAuth.enabled }} + {{- if .Values.ui.basicAuth.autoCreateSecret }} + echo "โœ“ ui.basicAuth.password is set" + {{- end }} + {{- end }} + + {{- end }} + + # Check ingress hostname + {{- if .Values.ingress.enabled }} + if [ -z "{{ .Values.ingress.hostname }}" ]; then + echo "ERROR: ingress.hostname is required when ingress.enabled=true" + exit 1 + fi + echo "โœ“ ingress.hostname is set: {{ .Values.ingress.hostname }}" + {{- end }} + + # Security warnings (non-blocking) + {{- if not .Values.externalSecrets.enabled }} + {{- if .Values.option.autoCreateSecret }} + echo "" + echo "โš ๏ธ WARNING: Using plaintext secrets in values.yaml" + echo " For production, enable externalSecrets or provide pre-created secrets" + echo " Example: helm install --set externalSecrets.enabled=true" + {{- end }} + {{- end }} + + {{- if not .Values.networkPolicy.enabled }} + echo "" + echo "โš ๏ธ WARNING: NetworkPolicy is disabled" + echo " For production, set networkPolicy.enabled=true and configure externalEgressCIDRs" + echo " See NOTES.txt after installation for configuration guidance" + {{- end }} + + {{- if .Values.lake.hostNetwork }} + echo "" + echo "โš ๏ธ DEPRECATION WARNING: lake.hostNetwork=true" + echo " hostNetwork is DEPRECATED in v2.1.0 and will be REMOVED in v2.2.0" + echo " Migrate to standard pod networking before upgrading to v2.2.0" + {{- end }} + + {{- if eq .Values.database.storage.type "hostpath" }} + echo "" + echo "โš ๏ธ DEPRECATION WARNING: database.storage.type=hostpath" + echo " hostPath is DEPRECATED in v2.1.0 and will be REMOVED in v2.2.0" + echo " Migrate to PVC storage before upgrading to v2.2.0" + {{- end }} + + echo "" + echo "All validation checks passed successfully!" diff --git a/charts/devlake/templates/secrets.yaml b/charts/devlake/templates/secrets.yaml index bc4db555..5d0d7c17 100644 --- a/charts/devlake/templates/secrets.yaml +++ b/charts/devlake/templates/secrets.yaml @@ -1,29 +1,47 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# +{{/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{- if not .Values.externalSecrets.enabled }} {{- if .Values.option.autoCreateSecret }} +# โš ๏ธ SECURITY IMPLICATION: This Secret is created from plaintext values in values.yaml +# In production environments: +# 1. Use externalSecrets.enabled=true with External Secrets Operator, OR +# 2. Pre-create this Secret manually before install, OR +# 3. Use sealed-secrets or similar secret management solutions +# 4. Never commit values.yaml with populated passwords to version control --- apiVersion: v1 kind: Secret metadata: - name: {{ include "devlake.mysql.secret" . }} + name: {{ include "devlake.db.secret" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "devlake.labels" . | nindent 4 }} + app.kubernetes.io/component: secret + annotations: + {{- include "devlake.annotations" . | nindent 4 }} stringData: -{{- if (eq .Values.option.database "mysql") }} - MYSQL_PASSWORD: "{{ .Values.mysql.password }}" - MYSQL_ROOT_PASSWORD: "{{ .Values.mysql.rootPassword }}" +{{- if eq .Values.database.type "mysql" }} + MYSQL_PASSWORD: "{{ .Values.database.password }}" + MYSQL_ROOT_PASSWORD: "{{ .Values.database.mysql.rootPassword }}" +{{- else if eq .Values.database.type "postgresql" }} + # Lake env (DB_* prefix) + DB_PASSWORD: "{{ .Values.database.password }}" + # Grafana env (POSTGRES_* prefix for entrypoint.sh) + POSTGRES_PASSWORD: "{{ .Values.database.password }}" {{- end }} {{- end }} @@ -33,6 +51,12 @@ apiVersion: v1 kind: Secret metadata: name: {{ include "devlake.ui.auth.secret" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "devlake.labels" . | nindent 4 }} + app.kubernetes.io/component: secret + annotations: + {{- include "devlake.annotations" . | nindent 4 }} stringData: ADMIN_USER: "{{ .Values.ui.basicAuth.user }}" ADMIN_PASS: "{{ .Values.ui.basicAuth.password }}" @@ -44,6 +68,13 @@ apiVersion: v1 kind: Secret metadata: name: {{ include "devlake.lake.encryption.secret" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "devlake.labels" . | nindent 4 }} + app.kubernetes.io/component: secret + annotations: + {{- include "devlake.annotations" . | nindent 4 }} stringData: ENCRYPTION_SECRET: {{ .Values.lake.encryptionSecret.secret }} {{- end }} +{{- end }} diff --git a/charts/devlake/templates/serviceaccount.yaml b/charts/devlake/templates/serviceaccount.yaml new file mode 100644 index 00000000..2f59f6ac --- /dev/null +++ b/charts/devlake/templates/serviceaccount.yaml @@ -0,0 +1,28 @@ +{{/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{- if .Values.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "devlake.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "devlake.labels" . | nindent 4 }} + app.kubernetes.io/component: rbac + annotations: + {{- include "devlake.annotations" . | nindent 4 }} +{{- end }} diff --git a/charts/devlake/templates/servicemonitor.yaml b/charts/devlake/templates/servicemonitor.yaml new file mode 100644 index 00000000..9d536c39 --- /dev/null +++ b/charts/devlake/templates/servicemonitor.yaml @@ -0,0 +1,57 @@ +{{/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{- if .Values.metrics.serviceMonitor.enabled }} +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "devlake.fullname" . }}-lake + namespace: {{ .Release.Namespace }} + labels: + {{- include "devlake.labels" . | nindent 4 }} + app.kubernetes.io/component: monitoring + annotations: + {{- include "devlake.annotations" . | nindent 4 }} +spec: + selector: + matchLabels: + app.kubernetes.io/name: {{ include "devlake.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + devlakeComponent: lake + endpoints: + - port: http + path: /metrics +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "devlake.fullname" . }}-ui + labels: + {{- include "devlake.labels" . | nindent 4 }} + app.kubernetes.io/component: monitoring + annotations: + {{- include "devlake.annotations" . | nindent 4 }} +spec: + selector: + matchLabels: + app.kubernetes.io/name: {{ include "devlake.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + devlakeComponent: ui + endpoints: + - port: http + path: /metrics +{{- end }} diff --git a/charts/devlake/templates/services.yaml b/charts/devlake/templates/services.yaml index eb8c6ec5..806cc01d 100644 --- a/charts/devlake/templates/services.yaml +++ b/charts/devlake/templates/services.yaml @@ -1,69 +1,81 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# database services +{{/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +# database services (v2.0.0: refactored for database.type selector) --- -{{- if eq .Values.option.database "mysql" }} -{{- if not .Values.mysql.useExternal }} +{{- if not .Values.database.useExternal }} apiVersion: v1 kind: Service metadata: - name: {{ include "devlake.fullname" . }}-mysql + name: {{ include "devlake.fullname" . }}-{{ .Values.database.type }} + namespace: {{ .Release.Namespace }} labels: {{- include "devlake.labels" . | nindent 4 }} - {{- with .Values.mysql.service.extraLabels }} + app.kubernetes.io/component: database + {{- with .Values.database.service.extraLabels }} {{- toYaml . | nindent 4 }} {{- end }} + annotations: + {{- include "devlake.annotations" . | nindent 4 }} spec: - type: {{ .Values.mysql.service.type }} - {{- if and (eq .Values.mysql.service.type "LoadBalancer") .Values.mysql.service.loadBalancerIP }} - loadBalancerIP: {{ .Values.mysql.service.loadBalancerIP }} + type: {{ .Values.database.service.type }} + {{- if and (eq .Values.database.service.type "LoadBalancer") .Values.database.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.database.service.loadBalancerIP }} {{- end }} selector: {{- include "devlake.selectorLabels" . | nindent 4 }} - devlakeComponent: mysql + devlakeComponent: {{ .Values.database.type }} ports: - protocol: TCP - name: mysql - port: 3306 - targetPort: 3306 - {{- if and (eq .Values.mysql.service.type "NodePort") .Values.mysql.service.nodePort }} - nodePort: {{ .Values.mysql.service.nodePort }} + name: {{ .Values.database.type }} + port: {{ include "database.port" . }} + targetPort: {{ include "database.port" . }} + {{- if and (eq .Values.database.service.type "NodePort") .Values.database.service.nodePort }} + nodePort: {{ .Values.database.service.nodePort }} {{- end }} {{- end }} -{{- end }} # devlake services --- apiVersion: v1 kind: Service metadata: name: {{ include "devlake.fullname" . }}-lake + namespace: {{ .Release.Namespace }} labels: {{- include "devlake.labels" . | nindent 4 }} + app.kubernetes.io/component: lake + devlakeComponent: lake {{- with .Values.lake.service.extraLabels }} {{- toYaml . | nindent 4 }} {{- end }} + annotations: + {{- include "devlake.annotations" . | nindent 4 }} + {{- if .Values.metrics.enabled }} + prometheus.io/scrape: "true" + prometheus.io/port: "8080" + prometheus.io/path: "/metrics" + {{- end }} spec: selector: {{- include "devlake.selectorLabels" . | nindent 4 }} devlakeComponent: lake ports: - protocol: TCP - name: devlake + name: http port: {{ .Values.lake.port }} targetPort: {{ .Values.lake.port }} @@ -73,11 +85,21 @@ apiVersion: v1 kind: Service metadata: name: {{ include "devlake.fullname" . }}-ui + namespace: {{ .Release.Namespace }} labels: {{- include "devlake.labels" . | nindent 4 }} + app.kubernetes.io/component: ui + devlakeComponent: ui {{- with .Values.ui.service.extraLabels }} {{- toYaml . | nindent 4 }} {{- end }} + annotations: + {{- include "devlake.annotations" . | nindent 4 }} + {{- if .Values.metrics.enabled }} + prometheus.io/scrape: "true" + prometheus.io/port: "4000" + prometheus.io/path: "/metrics" + {{- end }} spec: type: {{ .Values.service.type }} selector: @@ -85,9 +107,6 @@ spec: devlakeComponent: ui ports: - protocol: TCP - name: ui + name: http port: 4000 targetPort: 4000 - {{- if eq .Values.service.type "NodePort" }} - nodePort: {{ .Values.service.uiPort }} - {{- end }} diff --git a/charts/devlake/templates/statefulset-mysql.yaml b/charts/devlake/templates/statefulset-mysql.yaml new file mode 100644 index 00000000..8e8ccdc0 --- /dev/null +++ b/charts/devlake/templates/statefulset-mysql.yaml @@ -0,0 +1,195 @@ +{{/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +--- +# MySQL database statefulset +{{- if eq .Values.database.type "mysql" }} +{{- if not .Values.database.useExternal }} +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "devlake.fullname" . }}-mysql + namespace: {{ .Release.Namespace }} + labels: + {{- include "devlake.labels" . | nindent 4 }} + app.kubernetes.io/component: database + annotations: + {{- include "devlake.annotations" . | nindent 4 }} +spec: + replicas: {{ if gt (int .Values.database.replicaCount) 1 }}1{{ else }}{{ .Values.database.replicaCount }}{{ end }} + serviceName: {{ include "devlake.fullname" . }}-mysql + selector: + matchLabels: + app.kubernetes.io/name: {{ include "devlake.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + template: + metadata: + labels: + app.kubernetes.io/name: {{ include "devlake.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + devlakeComponent: mysql + {{- with .Values.database.extraLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + annotations: + {{- toYaml .Values.database.podAnnotations | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + automountServiceAccountToken: false + securityContext: + runAsNonRoot: {{ .Values.database.securityContext.runAsNonRoot }} + runAsUser: {{ include "database.uid" . }} + fsGroup: {{ include "database.uid" . }} + fsGroupChangePolicy: "OnRootMismatch" + seccompProfile: + type: {{ .Values.database.securityContext.seccompProfile.type }} + {{- if or .Values.database.mysql.enableFixPermissions .Values.database.initContainers }} + initContainers: + {{- if .Values.database.mysql.enableFixPermissions }} + - name: fix-permissions + image: {{ include "database.image" . }} + command: ['sh', '-c'] + args: + - | + chown -R 999:999 /var/lib/mysql + chmod 755 /var/lib/mysql + volumeMounts: + - name: data + mountPath: /var/lib/mysql + securityContext: + runAsUser: 0 + runAsNonRoot: false + {{- end }} + {{- with .Values.database.initContainers}} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + containers: + - name: mysql + image: {{ include "database.image" . }} + imagePullPolicy: {{ .Values.database.image.pullPolicy }} + args: + - "mysqld" + - "--character-set-server=utf8mb4" + - "--collation-server=utf8mb4_bin" + - "--skip-log-bin" + {{- with .Values.database.mysql.extraArgs }} + {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - name: mysql + containerPort: 3306 + protocol: TCP + startupProbe: + exec: + command: + - sh + - -c + - mysqladmin ping -uroot -p$MYSQL_ROOT_PASSWORD + initialDelaySeconds: 10 + periodSeconds: 5 + failureThreshold: 60 + livenessProbe: + exec: + command: + - sh + - -c + - mysqladmin ping -uroot -p$MYSQL_ROOT_PASSWORD + periodSeconds: 10 + failureThreshold: 3 + readinessProbe: + exec: + command: + - sh + - -c + - mysqladmin ping -uroot -p$MYSQL_ROOT_PASSWORD + periodSeconds: 5 + failureThreshold: 3 + {{- with .Values.database.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + envFrom: + - configMapRef: + name: {{ include "devlake.db.configmap" . }} + - secretRef: + name: {{ include "devlake.db.secret" . }} + env: + {{- range $key, $value := .Values.commonEnvs }} + - name: "{{ tpl $key $ }}" + value: "{{ tpl (print $value) $ }}" + {{- end }} + volumeMounts: + {{- if or (eq .Values.database.storage.type "pvc") (eq .Values.database.storage.type "hostpath") }} + - mountPath: /var/lib/mysql + name: data + {{- end }} + - mountPath: /var/lib/mysql-files + name: mysql-files + - mountPath: /var/run/mysqld + name: run + - mountPath: /tmp + name: tmp + securityContext: + runAsNonRoot: {{ .Values.database.containerSecurityContext.runAsNonRoot }} + runAsUser: {{ include "database.uid" . }} + readOnlyRootFilesystem: {{ .Values.database.containerSecurityContext.readOnlyRootFilesystem }} + allowPrivilegeEscalation: {{ .Values.database.containerSecurityContext.allowPrivilegeEscalation }} + capabilities: + drop: {{ toYaml .Values.database.containerSecurityContext.capabilities.drop | nindent 16 }} + {{- with .Values.database.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.database.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.database.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + {{- if eq .Values.database.storage.type "hostpath" }} + - name: data + hostPath: + path: {{ .Values.database.storage.hostPath }} + type: DirectoryOrCreate + {{- end }} + - name: mysql-files + emptyDir: {} + - name: run + emptyDir: {} + - name: tmp + emptyDir: {} + {{- if eq .Values.database.storage.type "pvc" }} + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: ["ReadWriteOnce"] + {{- with .Values.database.storage.class }} + storageClassName: "{{ . }}" + {{- end }} + resources: + requests: + storage: "{{ .Values.database.storage.size }}" + {{- end }} +{{- end }} +{{- end }} diff --git a/charts/devlake/templates/statefulset-postgresql.yaml b/charts/devlake/templates/statefulset-postgresql.yaml new file mode 100644 index 00000000..76dbaba0 --- /dev/null +++ b/charts/devlake/templates/statefulset-postgresql.yaml @@ -0,0 +1,208 @@ +{{/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +--- +# PostgreSQL database statefulset +{{- if eq .Values.database.type "postgresql" }} +{{- if not .Values.database.useExternal }} +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "devlake.fullname" . }}-postgresql + namespace: {{ .Release.Namespace }} + labels: + {{- include "devlake.labels" . | nindent 4 }} + app.kubernetes.io/component: database + annotations: + {{- include "devlake.annotations" . | nindent 4 }} +spec: + replicas: {{ if gt (int .Values.database.replicaCount) 1 }}1{{ else }}{{ .Values.database.replicaCount }}{{ end }} + serviceName: {{ include "devlake.fullname" . }}-postgresql + selector: + matchLabels: + app.kubernetes.io/name: {{ include "devlake.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + template: + metadata: + labels: + app.kubernetes.io/name: {{ include "devlake.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + devlakeComponent: postgresql + {{- with .Values.database.extraLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + annotations: + {{- toYaml .Values.database.podAnnotations | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + automountServiceAccountToken: false + securityContext: + runAsNonRoot: {{ .Values.database.securityContext.runAsNonRoot }} + runAsUser: {{ include "database.uid" . }} + fsGroup: {{ include "database.uid" . }} + fsGroupChangePolicy: "OnRootMismatch" + seccompProfile: + type: {{ .Values.database.securityContext.seccompProfile.type }} + {{- if or .Values.database.postgresql.enableFixPermissions .Values.database.initContainers }} + initContainers: + {{- if .Values.database.postgresql.enableFixPermissions }} + - name: fix-permissions + image: {{ include "database.image" . }} + command: ['sh', '-c'] + args: + - | + chown -R 70:70 /var/lib/postgresql/data + chmod 700 /var/lib/postgresql/data + volumeMounts: + - name: data + mountPath: /var/lib/postgresql/data + securityContext: + runAsUser: 0 + runAsNonRoot: false + {{- end }} + {{- with .Values.database.initContainers}} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + containers: + - name: postgresql + image: {{ include "database.image" . }} + imagePullPolicy: {{ .Values.database.image.pullPolicy }} + {{- with .Values.database.postgresql.extraArgs }} + args: + {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - name: postgresql + containerPort: 5432 + protocol: TCP + startupProbe: + exec: + command: + - sh + - -c + - pg_isready -U $POSTGRES_USER + initialDelaySeconds: 10 + periodSeconds: 5 + failureThreshold: 30 + livenessProbe: + exec: + command: + - sh + - -c + - pg_isready -U $POSTGRES_USER + periodSeconds: 10 + failureThreshold: 3 + readinessProbe: + exec: + command: + - sh + - -c + - pg_isready -U $POSTGRES_USER + periodSeconds: 5 + failureThreshold: 3 + {{- with .Values.database.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + envFrom: + - configMapRef: + name: {{ include "devlake.db.configmap" . }} + - secretRef: + name: {{ include "devlake.db.secret" . }} + env: + - name: POSTGRES_USER + valueFrom: + configMapKeyRef: + name: {{ include "devlake.db.configmap" . }} + key: DB_USER + - name: POSTGRES_DB + valueFrom: + configMapKeyRef: + name: {{ include "devlake.db.configmap" . }} + key: DB_DATABASE + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "devlake.db.secret" . }} + key: DB_PASSWORD + - name: PGDATA + value: /var/lib/postgresql/data/pgdata + {{- range $key, $value := .Values.commonEnvs }} + - name: "{{ tpl $key $ }}" + value: "{{ tpl (print $value) $ }}" + {{- end }} + volumeMounts: + - mountPath: /var/lib/postgresql + name: pgdata-parent + {{- if or (eq .Values.database.storage.type "pvc") (eq .Values.database.storage.type "hostpath") }} + - mountPath: /var/lib/postgresql/data + name: data + {{- end }} + - mountPath: /var/run/postgresql + name: run + - mountPath: /tmp + name: tmp + securityContext: + runAsNonRoot: {{ .Values.database.containerSecurityContext.runAsNonRoot }} + runAsUser: {{ include "database.uid" . }} + readOnlyRootFilesystem: {{ .Values.database.containerSecurityContext.readOnlyRootFilesystem }} + allowPrivilegeEscalation: {{ .Values.database.containerSecurityContext.allowPrivilegeEscalation }} + capabilities: + drop: {{ toYaml .Values.database.containerSecurityContext.capabilities.drop | nindent 16 }} + {{- with .Values.database.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.database.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.database.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + {{- if eq .Values.database.storage.type "hostpath" }} + - name: data + hostPath: + path: {{ .Values.database.storage.hostPath }} + type: DirectoryOrCreate + {{- end }} + - name: pgdata-parent + emptyDir: {} + - name: run + emptyDir: {} + - name: tmp + emptyDir: {} + {{- if eq .Values.database.storage.type "pvc" }} + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: ["ReadWriteOnce"] + {{- with .Values.database.storage.class }} + storageClassName: "{{ . }}" + {{- end }} + resources: + requests: + storage: "{{ .Values.database.storage.size }}" + {{- end }} +{{- end }} +{{- end }} diff --git a/charts/devlake/templates/statefulsets.yaml b/charts/devlake/templates/statefulsets.yaml deleted file mode 100644 index 1b49d088..00000000 --- a/charts/devlake/templates/statefulsets.yaml +++ /dev/null @@ -1,142 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- -# database statefulset -{{- if eq .Values.option.database "mysql" }} -{{- if not .Values.mysql.useExternal }} -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: {{ include "devlake.fullname" . }}-mysql - labels: - {{- include "devlake.labels" . | nindent 4 }} -spec: - replicas: {{ if gt (int .Values.mysql.replicaCount) 1 }}1{{ else }}{{ .Values.mysql.replicaCount }}{{ end }} - serviceName: {{ include "devlake.fullname" . }}-mysql - selector: - matchLabels: - {{- include "devlake.selectorLabels" . | nindent 6 }} - template: - metadata: - labels: - {{- include "devlake.selectorLabels" . | nindent 8 }} - devlakeComponent: mysql - {{- with .Values.mysql.extraLabels }} - {{- toYaml . | nindent 8 }} - {{- end }} - annotations: - {{- toYaml .Values.mysql.podAnnotations | nindent 8 }} - spec: - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.mysql.securityContext }} - securityContext: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.mysql.initContainers}} - initContainers: - {{- toYaml . | nindent 8 }} - {{- end }} - containers: - - name: mysql - image: "{{ .Values.mysql.image.repository }}:{{ .Values.mysql.image.tag }}" - imagePullPolicy: {{ .Values.mysql.image.pullPolicy }} - args: - - "mysqld" - - "--character-set-server=utf8mb4" - - "--collation-server=utf8mb4_bin" - - "--skip-log-bin" - {{- with .Values.mysql.extraArgs }} - {{- toYaml . | nindent 12 }} - {{- end }} - ports: - - name: mysql - containerPort: 3306 - protocol: TCP - {{- with .Values.mysql.startupProbe }} - startupProbe: - {{- toYaml . | nindent 12 }} - {{- end }} - {{- with .Values.mysql.livenessProbe }} - livenessProbe: - {{- toYaml . | nindent 12 }} - {{- end }} - {{- with .Values.mysql.readinessProbe }} - readinessProbe: - {{- toYaml . | nindent 12 }} - {{- end }} - {{- with .Values.mysql.resources }} - resources: - {{- toYaml . | nindent 12 }} - {{- end }} - envFrom: - - configMapRef: - name: {{ include "devlake.mysql.configmap" . }} - - secretRef: - name: {{ include "devlake.mysql.secret" . }} - env: - {{- range $key, $value := .Values.commonEnvs }} - - name: "{{ tpl $key $ }}" - value: "{{ tpl (print $value) $ }}" - {{- end }} - {{- if or (eq .Values.mysql.storage.type "pvc") (eq .Values.mysql.storage.type "hostpath") }} - volumeMounts: - - mountPath: /var/lib/mysql - name: {{ include "devlake.fullname" . }}-mysql-data - {{- end }} - {{- with .Values.mysql.containerSecurityContext }} - securityContext: - {{- toYaml . | nindent 12 }} - {{- end }} - {{- with .Values.mysql.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.mysql.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.mysql.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- if eq .Values.mysql.storage.type "hostpath" }} - volumes: - - name: {{ include "devlake.fullname" . }}-mysql-data - hostPath: - path: {{ .Values.mysql.storage.hostPath }} - type: DirectoryOrCreate - {{- end }} - {{- if eq .Values.mysql.storage.type "pvc" }} - volumeClaimTemplates: - - metadata: - name: {{ include "devlake.fullname" . }}-mysql-data - spec: - accessModes: ["ReadWriteOnce"] - {{- with .Values.mysql.storage.class }} - storageClassName: "{{ . }}" - {{- end }} - resources: - requests: - storage: "{{ .Values.mysql.storage.size }}" - {{- end }} -{{- end }} -{{- end }} - - diff --git a/charts/devlake/templates/tests/test-connection.yaml b/charts/devlake/templates/tests/test-connection.yaml new file mode 100644 index 00000000..5c959cab --- /dev/null +++ b/charts/devlake/templates/tests/test-connection.yaml @@ -0,0 +1,109 @@ +{{/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +apiVersion: v1 +kind: Pod +metadata: + name: {{ include "devlake.fullname" . }}-test-connection + namespace: {{ .Release.Namespace }} + labels: + {{- include "devlake.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test + "helm.sh/hook-delete-policy": before-hook-creation +spec: + serviceAccountName: {{ include "devlake.serviceAccountName" . }} + securityContext: + runAsNonRoot: true + runAsUser: 65534 + seccompProfile: + type: RuntimeDefault + initContainers: + - name: test-database + {{- if not .Values.database.useExternal }} + image: {{ include "database.image" . }} + {{- else }} + image: {{ if eq .Values.database.type "mysql" }}mysql:8{{ else }}postgres:14{{ end }} + {{- end }} + imagePullPolicy: IfNotPresent + resources: + requests: + cpu: 10m + memory: 64Mi + limits: + cpu: 200m + memory: 128Mi + envFrom: + - configMapRef: + name: {{ include "devlake.db.configmap" . }} + - secretRef: + name: {{ include "devlake.db.secret" . }} + command: ['sh', '-c'] + args: + - | + echo "Testing database connection..." + {{- if eq .Values.database.type "mysql" }} + mysql -h$(MYSQL_SERVER) -P$(MYSQL_PORT) -u$(MYSQL_USER) -p$(MYSQL_PASSWORD) -e "SELECT 1 as test" $(MYSQL_DATABASE) + echo "MySQL connection: OK" + {{- else if eq .Values.database.type "postgresql" }} + PGPASSWORD=$(DB_PASSWORD) psql -h $(DB_SERVER) -p $(DB_PORT) -U $(DB_USER) -d $(DB_DATABASE) -c "SELECT 1 as test" + echo "PostgreSQL connection: OK" + {{- end }} + securityContext: + runAsNonRoot: true + {{- if eq .Values.database.type "mysql" }} + runAsUser: 999 + {{- else }} + runAsUser: 70 + {{- end }} + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + volumeMounts: + - name: tmp + mountPath: /tmp + containers: + - name: test-http + image: {{ .Values.alpine.image.repository }}:{{ .Values.alpine.image.tag }} + imagePullPolicy: {{ .Values.alpine.image.pullPolicy }} + resources: + requests: + cpu: 10m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + command: ['sh', '-c'] + args: + - | + echo "Testing HTTP endpoints..." + wget --spider --timeout=5 http://{{ include "devlake.fullname" . }}-lake:8080/ping + echo "Lake endpoint: OK" + wget --spider --timeout=5 http://{{ include "devlake.fullname" . }}-ui:4000/health/ + echo "UI endpoint: OK" + echo "All tests passed!" + securityContext: + runAsNonRoot: true + runAsUser: 65534 + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + volumes: + - name: tmp + emptyDir: {} + restartPolicy: Never diff --git a/charts/devlake/templates/validate.yaml b/charts/devlake/templates/validate.yaml index c74296db..72977251 100644 --- a/charts/devlake/templates/validate.yaml +++ b/charts/devlake/templates/validate.yaml @@ -1,21 +1,64 @@ -{{- if and .Values.lake.encryptionSecret.autoCreateSecret (not .Values.lake.encryptionSecret.secret) }} +{{/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and (not .Values.externalSecrets.enabled) .Values.lake.encryptionSecret.autoCreateSecret (not .Values.lake.encryptionSecret.secret) }} {{- fail "Helm test requires lake.encryptionSecret.secret.\n\n - If you're upgrading from DevLake v0.17.x or earlier versions, please get the encryption secret by copying the ENCODE_KEY value from /app/config/.env of the lake pod (e.g. devlake-lake-0);\n - If upgrading from v0.18.0+, get the original secret in k8s secret and decode it\n - If new installation, get the encryption secret via command `openssl rand -base64 2000 | tr -dc 'A-Z' | fold -w 128 | head -n 1`.\n\nFor more information, please check https://github.com/apache/incubator-devlake-helm-chart" }} {{- end }} -{{- if and .Values.ui.basicAuth.enabled .Values.ui.basicAuth.autoCreateSecret (or (not .Values.ui.basicAuth.user) (not .Values.ui.basicAuth.password)) }} +{{- if and (not .Values.externalSecrets.enabled) .Values.ui.basicAuth.enabled .Values.ui.basicAuth.autoCreateSecret (or (not .Values.ui.basicAuth.user) (not .Values.ui.basicAuth.password)) }} {{- fail "Helm test requires ui.basicAuth.user and ui.basicAuth.password" }} {{- end }} -{{- if and (eq .Values.option.database "mysql") .Values.option.autoCreateSecret (or (not .Values.mysql.username) (not .Values.mysql.password) (not .Values.mysql.database)) }} -{{- fail "Helm test requires mysql.username, mysql.password and mysql.database" }} +{{- if not (has .Values.database.type (list "mysql" "postgresql")) }} +{{- fail "database.type must be 'mysql' or 'postgresql'" }} +{{- end }} + +{{- if and (not .Values.externalSecrets.enabled) .Values.option.autoCreateSecret (or (not .Values.database.username) (not .Values.database.password) (not .Values.database.database)) }} +{{- fail "Helm test requires database.username, database.password and database.database" }} +{{- end }} + +{{- if and (not .Values.externalSecrets.enabled) .Values.option.autoCreateSecret (eq .Values.database.type "mysql") (not .Values.database.useExternal) (not .Values.database.mysql.rootPassword) }} +{{- fail "Helm test requires database.mysql.rootPassword when using embedded MySQL" }} +{{- end }} + +{{- if and .Values.database.useExternal (or (not .Values.database.externalServer) (eq .Values.database.externalServer "127.0.0.1")) }} +{{- fail "database.externalServer must be set to a valid hostname/IP when database.useExternal=true" }} +{{- end }} + +{{- if and .Values.backup.enabled .Values.database.useExternal }} +{{- fail "backup.enabled=true is not supported with database.useExternal=true. Backups must be configured on the external database server directly." }} +{{- end }} + +{{- if and (not .Values.grafana.enabled) (or (not .Values.grafana.external.url) (eq .Values.grafana.external.url "")) }} +{{- fail "When grafana.enabled=false, grafana.external.url must be set to a valid Grafana endpoint URL (e.g., http://grafana.example.com). Alternatively, set grafana.enabled=true to use the built-in Grafana instance." }} +{{- end }} + +{{- if or (not .Values.lake.resources.requests) (not .Values.lake.resources.requests.cpu) (not .Values.lake.resources.requests.memory) }} +{{- fail "lake.resources.requests with cpu and memory is required. Set cpu and memory requests for production deployments (e.g., requests: { cpu: 500m, memory: 512Mi })" }} {{- end }} -{{- if and .Values.option.autoCreateSecret (eq .Values.option.database "mysql") (not .Values.mysql.useExternal) (not .Values.mysql.rootPassword) }} -{{- fail "Helm test requires mysql.rootPassword" }} +{{- if or (not .Values.ui.resources.requests) (not .Values.ui.resources.requests.cpu) (not .Values.ui.resources.requests.memory) }} +{{- fail "ui.resources.requests with cpu and memory is required. Set cpu and memory requests for production deployments (e.g., requests: { cpu: 100m, memory: 128Mi })" }} {{- end }} -{{- if and (not .Values.grafana.enabled) (not .Values.grafana.external.url) }} -{{- fail "Helm test requires grafana.enabled to be true or grafana.external.url to be provided" }} +{{- if not .Values.database.useExternal }} +{{- if or (not .Values.database.resources.requests) (not .Values.database.resources.requests.cpu) (not .Values.database.resources.requests.memory) }} +{{- fail "database.resources.requests with cpu and memory is required when using embedded database. Set cpu and memory requests (e.g., requests: { cpu: 500m, memory: 1Gi })" }} +{{- end }} {{- end }} diff --git a/charts/devlake/values.schema.json b/charts/devlake/values.schema.json new file mode 100644 index 00000000..a71c1663 --- /dev/null +++ b/charts/devlake/values.schema.json @@ -0,0 +1,368 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "DevLake Helm Chart Values Schema", + "type": "object", + "required": ["database", "lake", "ui"], + "properties": { + "database": { + "type": "object", + "required": ["type", "username", "database"], + "properties": { + "type": { + "type": "string", + "enum": ["mysql", "postgresql"], + "description": "Database type - must be mysql or postgresql" + }, + "username": { + "type": "string", + "minLength": 1, + "description": "Database username" + }, + "password": { + "type": "string", + "description": "Database password - leave empty when using externalSecrets" + }, + "database": { + "type": "string", + "minLength": 1, + "description": "Database name" + }, + "useExternal": { + "type": "boolean", + "description": "Use external database server" + }, + "externalServer": { + "type": "string", + "description": "External database server hostname" + }, + "externalPort": { + "type": "integer", + "minimum": 1, + "maximum": 65535, + "description": "External database server port" + }, + "replicaCount": { + "type": "integer", + "minimum": 1, + "maximum": 1, + "description": "Database replica count (must be 1 for embedded mode)" + } + } + }, + "externalSecrets": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean", + "description": "Enable External Secrets Operator integration" + }, + "secretName": { + "type": "string", + "description": "Name of the external secret" + }, + "secretStoreRef": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "kind": { + "type": "string", + "enum": ["SecretStore", "ClusterSecretStore"] + } + } + } + }, + "if": { + "properties": { + "enabled": { + "const": true + } + } + }, + "then": { + "required": ["secretName", "secretStoreRef"], + "properties": { + "secretStoreRef": { + "required": ["name"], + "properties": { + "name": { + "minLength": 1 + } + } + } + } + } + }, + "lake": { + "type": "object", + "required": ["replicaCount", "image", "resources"], + "properties": { + "replicaCount": { + "type": "integer", + "minimum": 1, + "description": "Number of lake replicas" + }, + "hostNetwork": { + "type": "boolean", + "const": false, + "description": "hostNetwork must be false for security" + }, + "resources": { + "type": "object", + "required": ["requests"], + "properties": { + "requests": { + "type": "object", + "required": ["cpu", "memory"], + "properties": { + "cpu": { + "type": "string", + "pattern": "^[0-9]+(m|[0-9]*\\.?[0-9]+)?$" + }, + "memory": { + "type": "string", + "pattern": "^[0-9]+(Mi|Gi|Ki)?$" + } + } + } + } + }, + "encryptionSecret": { + "type": "object", + "properties": { + "secret": { + "type": "string", + "description": "Encryption secret must be at least 32 characters when provided" + }, + "autoCreateSecret": { + "type": "boolean" + } + } + } + } + }, + "ui": { + "type": "object", + "required": ["replicaCount", "resources"], + "properties": { + "replicaCount": { + "type": "integer", + "minimum": 1, + "description": "Number of UI replicas" + }, + "resources": { + "type": "object", + "required": ["requests"], + "properties": { + "requests": { + "type": "object", + "required": ["cpu", "memory"], + "properties": { + "cpu": { + "type": "string", + "pattern": "^[0-9]+(m|[0-9]*\\.?[0-9]+)?$" + }, + "memory": { + "type": "string", + "pattern": "^[0-9]+(Mi|Gi|Ki)?$" + } + } + } + } + } + } + }, + "ingress": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + }, + "hostname": { + "type": "string", + "description": "Ingress hostname - should be a valid FQDN when enabled" + } + }, + "if": { + "properties": { + "enabled": { + "const": true + } + } + }, + "then": { + "required": ["hostname"], + "properties": { + "hostname": { + "pattern": "^([a-z0-9-]+\\.)*[a-z0-9-]+\\.[a-z]{2,}$|^localhost$" + } + } + } + }, + "httpRoute": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean", + "description": "Enable Gateway API HTTPRoute" + }, + "gatewayName": { + "type": "string", + "description": "Name of the Gateway to attach to" + }, + "gatewayNamespace": { + "type": "string", + "description": "Namespace of the Gateway (optional)" + }, + "sectionName": { + "type": "string", + "description": "Listener/section name on the Gateway (optional)" + }, + "hostnames": { + "type": "array", + "items": { + "type": "string", + "pattern": "^([a-z0-9-]+\\.)*[a-z0-9-]+\\.[a-z]{2,}$", + "description": "Valid hostname for HTTPRoute" + }, + "description": "List of hostnames for the HTTPRoute" + }, + "prefix": { + "type": "string", + "pattern": "^/.*$", + "description": "Path prefix (must start with /)" + }, + "extraLabels": { + "type": "object" + }, + "annotations": { + "type": "object" + } + }, + "if": { + "properties": { + "enabled": { + "const": true + } + } + }, + "then": { + "required": ["gatewayName"], + "properties": { + "gatewayName": { + "minLength": 1 + } + } + } + }, + "networkPolicy": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + }, + "externalEgressCIDRs": { + "type": "array", + "items": { + "type": "string", + "pattern": "^([0-9]{1,3}\\.){3}[0-9]{1,3}/[0-9]{1,2}$", + "description": "Must be valid CIDR notation" + } + }, + "externalDatabaseCIDRs": { + "type": "array", + "items": { + "type": "string", + "pattern": "^([0-9]{1,3}\\.){3}[0-9]{1,3}/[0-9]{1,2}$", + "description": "CIDR blocks for external database access" + } + } + } + }, + "backup": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + }, + "schedule": { + "type": "string", + "pattern": "^(@(annually|yearly|monthly|weekly|daily|hourly|reboot))|(@every (\\d+(ns|us|ยตs|ms|s|m|h))+)|((((\\d+,)+\\d+|(\\d+([/-])\\d+)|\\d+|\\*) ?){5,7})$", + "description": "Cron schedule expression" + }, + "retentionDays": { + "type": "integer", + "minimum": 1, + "description": "Number of days to retain backups" + }, + "pvc": { + "type": "object", + "properties": { + "existingClaim": { + "type": "string", + "description": "Name of existing PVC for backups" + }, + "storageClassName": { + "type": "string" + }, + "size": { + "type": "string", + "pattern": "^[0-9]+(Mi|Gi|Ti)?$" + } + } + } + } + }, + "serviceAccount": { + "type": "object", + "properties": { + "create": { + "type": "boolean" + }, + "name": { + "type": "string" + } + } + }, + "grafana": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean", + "description": "Enable built-in Grafana instance" + }, + "external": { + "type": "object", + "properties": { + "url": { + "type": "string", + "pattern": "^(https?://[^\\s]+)?$", + "description": "External Grafana URL (required when enabled=false with MySQL)" + } + } + }, + "adminPassword": { + "type": "string", + "description": "Grafana admin password" + } + } + }, + "imageTag": { + "type": "string", + "description": "Global image tag override" + }, + "imagePullSecrets": { + "type": "array", + "items": { + "type": "object" + }, + "description": "Image pull secrets" + }, + "replicaCount": { + "type": "integer", + "minimum": 1, + "description": "Global replica count (deprecated - use lake.replicaCount and ui.replicaCount)" + } + } +} diff --git a/charts/devlake/values.yaml b/charts/devlake/values.yaml index 6c05c526..060e5d76 100644 --- a/charts/devlake/values.yaml +++ b/charts/devlake/values.yaml @@ -17,116 +17,176 @@ # replica count replicaCount: 1 -imageTag: v1.0.3-beta10 +imageTag: v1.0.3-beta12 # image pull secrets imagePullSecrets: [] +# NetworkPolicy configuration +networkPolicy: + enabled: false # Production: set to true. See NOTES.txt for required egress config. + externalEgressCIDRs: [] + # External database access CIDRs (required when database.useExternal=true and networkPolicy.enabled=true) + # Example: ["10.0.0.0/8"] or leave empty to allow all destinations (0.0.0.0/0) + externalDatabaseCIDRs: [] + ui: + # Restrict UI ingress sources (optional) + # When not set, UI accepts traffic from all sources + ingressNamespaceSelector: {} + ingressPodSelector: {} + +# Optional SHA256 digests for image pinning +imageDigests: + lake: "" + ui: "" + grafana: "" + database: + mysql: "" + postgresql: "" + +# ServiceAccount configuration for lake, ui, and grafana pods +serviceAccount: + # Specifies whether a service account should be created + create: true + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template with -sa suffix + name: "" + #the common environments for all pods except grafana, grafana needs to be set in grafana section seperately commonEnvs: TZ: "UTC" -mysql: +# BREAKING CHANGE (v2.0.0): mysql.* has been replaced with database.* +# Migration guide: https://github.com/apache/incubator-devlake-helm-chart/blob/main/MIGRATION.md +# +# Old (v1.x): New (v2.0.0): +# mysql.username โ†’ database.username +# mysql.password โ†’ database.password +# mysql.database โ†’ database.database +# mysql.useExternal โ†’ database.useExternal +# mysql.rootPassword โ†’ database.mysql.rootPassword +# mysql.extraParams โ†’ database.mysql.extraParams +# +# You must also add: database.type: mysql (or postgresql) + +database: + # Database type selector - supported: mysql, postgresql + # REQUIRED FOR POSTGRESQL: Set type to 'postgresql' + type: mysql + replicaCount: 1 - # if use external mysql server, please set true - # by default using false, chart will create a single mysql instance + + # if use external database server, please set true + # by default using false, chart will create a single database instance useExternal: false - # the external mysql server address + # the external database server address externalServer: 127.0.0.1 - # external mysql port + # external database port (default: 3306 for mysql, 5432 for postgresql) externalPort: 3306 # the username for devlake database username: merico # the password for devlake database - password: merico + # โš ๏ธ SECURITY WARNING: Never commit plaintext passwords to Git! + # For production, use externalSecrets.enabled=true or pass via --set + # Example: helm install --set database.password=$DB_PASSWORD + password: "" - # the database for devlake + # the database name for devlake database: lake - # extra MySQL DSN query params appended to DB_URL. - # Note: include a leading '&' yourself. Empty string means no change. - # example: "&tls=skip-verify" or "&tls=skip-verify&autocommit=true" - extraParams: "" - - # root password for mysql, only used when use_external=false - rootPassword: admin - - # storage for mysql + # storage for database (embedded mode only) storage: # pvc or hostpath type: pvc # the storage class for pv, leave empty will using default class: "" size: 50Gi - hostPath: /devlake/mysql/data + hostPath: /devlake/database/data - # image for mysql + # image for database (embedded mode only) + # defaults: mysql:8 for type=mysql, postgres:14 for type=postgresql + # Leave repository and tag empty to use type-driven defaults from _helpers.tpl image: - repository: mysql - tag: 8 + repository: "" + tag: "" pullPolicy: IfNotPresent - # init containers for mysql if have + # init containers for database if have initContainers: [] - # resources config for mysql if have - resources: {} + # resources config for database if have + resources: + requests: + cpu: 500m + memory: 1Gi + limits: + cpu: 2000m + memory: 4Gi - # nodeSelector config for mysql if have + # nodeSelector config for database if have nodeSelector: {} - # tolerations config for mysql if have + # tolerations config for database if have tolerations: [] - # affinity config for mysql if have + # affinity config for database if have affinity: {} - extraArgs: [] - extraLabels: {} - securityContext: {} + # Security context (runAsUser and fsGroup are computed based on database.type) + # mysql: uid=999, postgresql: uid=70 + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - containerSecurityContext: {} + containerSecurityContext: + runAsNonRoot: true + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] podAnnotations: {} - # Probes for MySQL container - startupProbe: - exec: &mysql_ping_exec - command: - - "sh" - - "-c" - - "mysqladmin ping --protocol=TCP -h 127.0.0.1 -u root -p$MYSQL_ROOT_PASSWORD" - initialDelaySeconds: 120 - periodSeconds: 10 - timeoutSeconds: 10 - failureThreshold: 60 - - livenessProbe: - exec: *mysql_ping_exec - initialDelaySeconds: 60 - periodSeconds: 10 - timeoutSeconds: 30 - failureThreshold: 5 - - readinessProbe: - exec: *mysql_ping_exec - initialDelaySeconds: 5 - periodSeconds: 5 - timeoutSeconds: 10 - failureThreshold: 3 - service: type: "ClusterIP" nodePort: "" loadBalancerIP: "" extraLabels: {} + # MySQL-specific settings (only used when database.type: mysql) + mysql: + # root password for mysql, only used when useExternal=false + rootPassword: "" + + # Enable root initContainer to fix permissions + # Set to false for Pod Security Standards "restricted" compliance + # When false, ensure PVC/volume permissions are correct (fsGroup may handle this) + enableFixPermissions: true + + # extra MySQL DSN query params appended to DB_URL. + # Note: include a leading '&' yourself. Empty string means no change. + # example: "&tls=skip-verify" or "&tls=skip-verify&autocommit=true" + extraParams: "" + + # PostgreSQL-specific settings (only used when database.type: postgresql) + postgresql: + # Enable root initContainer to fix permissions + # Set to false for Pod Security Standards "restricted" compliance + # When false, ensure PVC/volume permissions are correct (fsGroup may handle this) + enableFixPermissions: true + + # extra PostgreSQL DSN query params appended to DB_URL. + # Note: must include a leading '?' for first param. Empty string means no change. + # example: "?sslmode=disable" or "?sslmode=disable&search_path=public" + extraParams: "" + # dependency chart values grafana: enabled: true @@ -136,7 +196,11 @@ grafana: image: registry: devlake.docker.scarf.sh repository: apache/devlake-dashboard - tag: v1.0.3-beta10 + tag: v1.0.3-beta12 + serviceAccount: + create: false + nameOverride: "{{ include \"devlake.serviceAccountName\" . }}" + automountServiceAccountToken: false deploymentStrategy: type: Recreate adminPassword: "" @@ -146,16 +210,38 @@ grafana: root_url: "%(protocol)s://%(domain)s/grafana" # the Secret name should be the same as .Values.option.connectionSecretName envFromSecrets: - - name: &devlake_mysql_auth "devlake-mysql-auth" + - name: &devlake_db_auth "devlake-db-auth" # the ConfigMap name should be the same as .Values.option.connectionConfigmapName envFromConfigMaps: - - name: &devlake_mysql_auth_config "devlake-mysql-auth-config" + - name: &devlake_db_auth_config "devlake-db-auth-config" #keep grafana timezone same as other pods, which is set by .Values.commonEnvs.TZ env: TZ: "UTC" persistence: enabled: true size: 4Gi + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 500m + memory: 512Mi + testFramework: + resources: + requests: + cpu: 10m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + securityContext: + runAsNonRoot: true + runAsUser: 65534 + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] ingressServiceName: "" ingressServicePort: "" @@ -164,7 +250,7 @@ lake: revisionHistoryLimit: 10 image: repository: devlake.docker.scarf.sh/apache/devlake - pullPolicy: Always + pullPolicy: IfNotPresent # defaults to imageTag; if set, lake.image.tag will override imageTag # tag: # storage for config @@ -178,6 +264,7 @@ lake: LOGGING_DIR: "/app/logs" # debug, info, warn, error LOGGING_LEVEL: "info" + LOGGING_FORMAT: json JIRA_JQL_AUTO_FULL_REFRESH: "true" ########################## # ENABLE_SUBTASKS_BY_DEFAULT: This environment variable is used to enable or disable the execution of subtasks. @@ -192,14 +279,20 @@ lake: # The name of secret which contains keys named ENCRYPTION_SECRET secretName: "" # if secretName is empty, secret should be set - # you can generate the encryption secret via cmd `openssl rand -base64 2000 | tr -dc 'A-Z' | fold -w 128 | head -n 1` + # โš ๏ธ SECURITY WARNING: Never commit encryption secrets to Git! + # For production, use externalSecrets.enabled=true or pass via --set + # Generate: openssl rand -base64 2000 | tr -dc 'A-Z' | fold -w 128 | head -n 1 + # Example: helm install --set lake.encryptionSecret.secret=$ENCRYPTION_SECRET secret: "" autoCreateSecret: true - # If hostNetwork is true, then dnsPolicy is set to ClusterFirstWithHostNet - hostNetwork: false - - resources: {} + resources: + requests: + cpu: 500m + memory: 512Mi + limits: + cpu: 2000m + memory: 2Gi strategy: type: Recreate @@ -212,11 +305,28 @@ lake: extraLabels: {} - securityContext: {} - - containerSecurityContext: {} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + fsGroup: 1000 + seccompProfile: + type: RuntimeDefault - initContainerSecurityContext: {} + containerSecurityContext: + runAsNonRoot: true + runAsUser: 1000 + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + + initContainerSecurityContext: + runAsNonRoot: true + runAsUser: 1000 + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] podAnnotations: {} @@ -265,20 +375,41 @@ lake: # subPath: test_file.yaml volumeMounts: [] - # Name of an existing ServiceAccount to use for the lake pod. - # If empty, the pod uses the namespace default ServiceAccount. - serviceAccount: - name: "" + # PodDisruptionBudget for high availability + podDisruptionBudget: + enabled: true + minAvailable: 1 + + # HorizontalPodAutoscaler configuration + hpa: + enabled: false + minReplicas: 1 + maxReplicas: 10 + targetCPUUtilizationPercentage: 70 + + # Pod Anti-Affinity configuration + podAntiAffinity: + enabled: false + type: preferred + + # Topology spread constraints + topologySpreadConstraints: [] ui: replicaCount: 1 revisionHistoryLimit: 10 image: repository: devlake.docker.scarf.sh/apache/devlake-config-ui - pullPolicy: Always + pullPolicy: IfNotPresent # defaults to imageTag; if set, lake.image.tag will override imageTag # tag: - resources: {} + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 500m + memory: 512Mi strategy: {} @@ -313,7 +444,10 @@ ui: basicAuth: enabled: false user: admin - password: admin + # โš ๏ธ SECURITY WARNING: Never commit plaintext passwords to Git! + # For production, use externalSecrets.enabled=true or pass via --set + # Example: helm install --set ui.basicAuth.password=$UI_PASSWORD + password: "" autoCreateSecret: true secretName: "" @@ -322,23 +456,23 @@ ui: podAnnotations: {} ## SecurityContext holds pod-level security attributes and common container settings. - ## This defaults to non root user with uid 101 and gid 1000. *v1.PodSecurityContext false ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ securityContext: - {} - # fsGroup: 101 - # runAsGroup: 1000 - # runAsNonRoot: true - # runAsUser: 101 + runAsNonRoot: true + runAsUser: 1001 + fsGroup: 1001 + seccompProfile: + type: RuntimeDefault ## K8s containers' Security Context ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container containerSecurityContext: - {} - # allowPrivilegeEscalation: false - # capabilities: - # drop: - # - all + runAsNonRoot: true + runAsUser: 1001 + readOnlyRootFilesystem: false + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] deployment: extraLabels: {} @@ -349,10 +483,25 @@ ui: ## Side Contaainer Configuration extraContainers: [] - # Name of an existing ServiceAccount to use for the config-ui pod. - # If empty, the pod uses the namespace default ServiceAccount. - serviceAccount: - name: "" + # PodDisruptionBudget for high availability + podDisruptionBudget: + enabled: true + minAvailable: 1 + + # HorizontalPodAutoscaler configuration + hpa: + enabled: false + minReplicas: 1 + maxReplicas: 10 + targetCPUUtilizationPercentage: 70 + + # Pod Anti-Affinity configuration + podAntiAffinity: + enabled: false + type: preferred + + # Topology spread constraints + topologySpreadConstraints: [] # - name: vault-agent # image: vault:1.6.2 # args: @@ -374,11 +523,15 @@ alpine: tag: 3.16 pullPolicy: IfNotPresent +# Prometheus metrics configuration +metrics: + enabled: false + serviceMonitor: + enabled: false + service: # service type: NodePort/ClusterIP - type: NodePort - # node port for devlake-ui if NodePort is enabled - uiPort: 32001 + type: ClusterIP ingress: enabled: false @@ -412,6 +565,27 @@ ingress: httpPort: 80 # ingress https port httpsPort: 443 + # rate limiting for default nginx ingress controller + rateLimit: "100" + +# Gateway API HTTPRoute configuration +httpRoute: + enabled: false + # Gateway name to attach to + gatewayName: "" + # Gateway namespace (if different from release namespace) + gatewayNamespace: "" + # Listener/section name on the Gateway + sectionName: "" + # Hostnames for the HTTPRoute + hostnames: [] + # - devlake.example.com + # Path prefix + prefix: / + # Extra labels + extraLabels: {} + # Annotations + annotations: {} extraPaths: [] # extraPaths: @@ -424,19 +598,33 @@ ingress: # name: use-annotation option: - # database type, supported: [mysql] - database: mysql - # the existing k8s secret name of db connection auth. The secret name should be as same as .Values.grafana.envFromSecret - connectionSecretName: *devlake_mysql_auth + # DEPRECATED (v2.0.0): option.database removed - use database.type instead + # database type now controlled by database.type field above + + # the existing k8s secret name of db connection auth + connectionSecretName: *devlake_db_auth # Optional: override the ConfigMap name for non-sensitive DB envs used across components # Default is a fixed name to align references from subcharts - connectionConfigmapName: *devlake_mysql_auth_config + connectionConfigmapName: *devlake_db_auth_config autoCreateSecret: true - # If true, the chart assembles DB_URL automatically for MySQL. Set to false + # If true, the chart assembles DB_URL automatically. Set to false # to disable auto-assembly and provide DB_URL yourself via `lake.envs` or # an external secret referenced by `lake.extraEnvsFromSecret`. assembleDbUrl: true +# External Secrets Operator integration +externalSecrets: + enabled: false + secretStoreRef: + name: "" + kind: SecretStore + secretName: "" + remoteRefs: + dbPassword: "" + dbRootPassword: "" + encryptionSecret: "" + uiPassword: "" + # Define some extra resources to be created # This section is useful when you need ExternalResource or Secrets, etc. extraResources: [] @@ -449,3 +637,27 @@ extraResources: [] # stringData: # username: admin # password: mypassword + +# Database backup configuration (CronJob) +backup: + enabled: false + schedule: "0 2 * * *" # Daily at 2 AM + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 1 + backoffLimit: 2 + retentionDays: 7 + image: + # Image for MySQL backups (used when database.type=mysql) + repository: mysql + tag: "8" + # Image tag for PostgreSQL backups (used when database.type=postgresql) + # Uses postgres: + postgresTag: "14" + pullPolicy: IfNotPresent + pvc: + # Use existing PVC (leave empty to create new PVC with name: -backup) + existingClaim: "" + # Storage class for backup PVC (leave empty for default) + storageClassName: "" + # Storage size for backup PVC + size: 10Gi