Skip to content

Commit a41f4cb

Browse files
fabiolucianofabioluciano
committed
feat: add postgresql support
1 parent 37978b2 commit a41f4cb

55 files changed

Lines changed: 3872 additions & 845 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.asf.yaml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ github:
1919
features:
2020
issues: true
2121
projects: false
22-
22+
2323
ghp_branch: gh-pages
2424
ghp_path: /
2525
enabled_merge_buttons:
@@ -40,7 +40,7 @@ github:
4040
required_approving_review_count: 1
4141
collaborators:
4242
- daniel-hutao
43-
43+
4444
notifications:
4545
commits: commits@devlake.apache.org
4646
issues: commits@devlake.apache.org

‎.github/.licenserc.yaml‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
header:
2+
license:
3+
spdx-id: Apache-2.0
4+
copyright-owner: Apache Software Foundation
5+
6+
paths-ignore:
7+
- '**/*.md'
8+
- '**/*.json'
9+
- '**/*.txt'
10+
- '**/LICENSE'
11+
- '**/NOTICE'
12+
- '.gitignore'
13+
- '.gitmodules'
14+
- 'charts/**/charts/**'
15+
- 'charts/**/Chart.lock'
16+
17+
comment: on-failure

‎.github/actions/chart-releaser-action‎

Lines changed: 0 additions & 1 deletion
This file was deleted.

‎.github/actions/kind-cluster/action.yml‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
name: Create kind Cluster
2+
description: Creates a kind Kubernetes cluster with diagnostics on failure
3+
4+
inputs:
5+
cluster_name:
6+
description: 'Name of the kind cluster to create'
7+
required: true
8+
namespace:
9+
description: 'Kubernetes namespace to create (optional)'
10+
required: false
11+
default: ''
12+
13+
runs:
14+
using: composite
15+
steps:
16+
- name: Create kind cluster
17+
run: |
18+
kind create cluster --name ${{ inputs.cluster_name }}
19+
shell: bash
20+
21+
- name: Cluster information
22+
run: |
23+
kubectl cluster-info
24+
kubectl get nodes
25+
kubectl get pods -n kube-system
26+
helm version
27+
kubectl version
28+
kubectl get storageclasses
29+
shell: bash
30+
31+
- name: Create namespace
32+
if: inputs.namespace != ''
33+
run: |
34+
kubectl create namespace ${{ inputs.namespace }} || true
35+
shell: bash

‎.github/actions/setup-environment/action.yml‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
name: Setup DevLake Environment
2+
description: Sets up mise, Helm, and chart dependencies for DevLake workflows
3+
4+
inputs:
5+
charts_dir:
6+
description: 'Path to charts directory'
7+
required: false
8+
default: 'charts/devlake'
9+
10+
runs:
11+
using: composite
12+
steps:
13+
- name: Install mise
14+
uses: jdx/mise-action@1648a7812b9aeae629881980618f079932869151 # v4
15+
16+
- name: Install tools via mise
17+
run: mise install
18+
shell: bash
19+
20+
- name: Cache Helm dependencies
21+
uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4
22+
with:
23+
path: |
24+
~/.cache/helm
25+
${{ inputs.charts_dir }}/charts/
26+
key: helm-deps-${{ hashFiles(format('{0}/Chart.lock', inputs.charts_dir)) }}
27+
restore-keys: |
28+
helm-deps-
29+
30+
- name: Add Helm repositories
31+
run: |
32+
helm repo add grafana https://grafana.github.io/helm-charts
33+
helm repo update
34+
shell: bash
35+
36+
- name: Build chart dependencies
37+
run: |
38+
helm dependency build ${{ inputs.charts_dir }}
39+
shell: bash

‎.github/dependabot.yml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
version: 2
2+
updates:
3+
- package-ecosystem: "github-actions"
4+
directory: "/"
5+
schedule:
6+
interval: "weekly"
7+
groups:
8+
actions:
9+
patterns: ["*"]

‎.github/labeler.yml‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
'area/charts':
2+
- changed-files:
3+
- any-glob-to-any-file: 'charts/**'
4+
5+
'area/workflows':
6+
- changed-files:
7+
- any-glob-to-any-file: '.github/workflows/**'
8+
9+
'area/actions':
10+
- changed-files:
11+
- any-glob-to-any-file: '.github/actions/**'
12+
13+
'area/docs':
14+
- changed-files:
15+
- any-glob-to-any-file: '**/*.md'

‎.github/workflows/deploy-test.yml‎

Lines changed: 165 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -32,54 +32,160 @@ on:
3232
- "!**.md"
3333
workflow_dispatch:
3434

35+
permissions:
36+
contents: read
37+
38+
concurrency:
39+
group: ${{ github.workflow }}-${{ github.ref }}
40+
cancel-in-progress: true
41+
3542
jobs:
3643
deploy-with-helm:
3744
runs-on: ubuntu-latest
45+
timeout-minutes: 45
3846
strategy:
3947
fail-fast: false
4048
matrix:
41-
database_type: ["mysql-builtin", "mysql-external"]
49+
database_type: ["mysql-builtin", "mysql-external", "postgresql-builtin", "postgresql-external"]
4250
steps:
43-
- name: Creating kind cluster
44-
uses: container-tools/kind-action@v1
51+
- name: Checkout
52+
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
53+
54+
- name: Setup environment
55+
uses: ./.github/actions/setup-environment
4556

46-
- name: Cluster information
57+
- name: Create kind cluster
58+
uses: ./.github/actions/kind-cluster
59+
with:
60+
cluster_name: devlake-${{ matrix.database_type }}
61+
62+
- name: Install external MySQL
63+
if: matrix.database_type == 'mysql-external'
4764
run: |
48-
kubectl cluster-info
49-
kubectl get nodes
50-
kubectl get pods -n kube-system
51-
helm version
52-
kubectl version
53-
kubectl get storageclasses
65+
DB_ROOT_PASSWORD=$(openssl rand -base64 16)
66+
DB_PASSWORD=$(openssl rand -base64 16)
67+
echo "::add-mask::${DB_ROOT_PASSWORD}"
68+
echo "::add-mask::${DB_PASSWORD}"
5469
55-
- name: Checkout
56-
uses: actions/checkout@v2
70+
# Retry logic for helm repo add (network flakiness)
71+
for i in {1..3}; do
72+
if helm repo add bitnami https://charts.bitnami.com/bitnami; then
73+
break
74+
else
75+
sleep $((2**i))
76+
fi
77+
done
78+
79+
helm install mysql bitnami/mysql --version 9.19.1 \
80+
--set auth.rootPassword="${DB_ROOT_PASSWORD}" \
81+
--set auth.database=lake \
82+
--set auth.username=devlake \
83+
--set auth.password="${DB_PASSWORD}"
5784
58-
- name: Helm install devlake
85+
- name: Install external PostgreSQL
86+
if: matrix.database_type == 'postgresql-external'
87+
run: |
88+
DB_ROOT_PASSWORD=$(openssl rand -base64 16)
89+
DB_PASSWORD=$(openssl rand -base64 16)
90+
echo "::add-mask::${DB_ROOT_PASSWORD}"
91+
echo "::add-mask::${DB_PASSWORD}"
92+
93+
# Retry logic for helm repo add (network flakiness)
94+
for i in {1..3}; do
95+
if helm repo add bitnami https://charts.bitnami.com/bitnami; then
96+
break
97+
else
98+
sleep $((2**i))
99+
fi
100+
done
101+
102+
helm install postgresql bitnami/postgresql --version 12.12.10 \
103+
--set auth.postgresPassword="${DB_ROOT_PASSWORD}" \
104+
--set auth.database=lake \
105+
--set auth.username=devlake \
106+
--set auth.password="${DB_PASSWORD}"
107+
108+
- name: Install DevLake (MySQL external)
59109
if: matrix.database_type == 'mysql-external'
60110
run: |
61-
helm repo add bitnami https://charts.bitnami.com/bitnami
62-
helm repo add grafana https://grafana.github.io/helm-charts
63-
helm install mysql bitnami/mysql --version 9.19.1 --set auth.rootPassword=admin --set auth.database=lake --set auth.username=merico --set auth.password=merico
64-
# external mysql at service: mysql
65-
helm dep build charts/devlake
111+
DB_PASSWORD=$(openssl rand -base64 16)
112+
ENCRYPTION_SECRET=$(openssl rand -base64 2000 | tr -dc '[:upper:]' | fold -w 128 | head -n 1)
113+
echo "::add-mask::${DB_PASSWORD}"
114+
echo "::add-mask::${ENCRYPTION_SECRET}"
115+
NODE_IP=$(kubectl get nodes --namespace default -o jsonpath="{.items[0].status.addresses[0].address}")
116+
echo "Node IP: ${NODE_IP}"
66117
helm install --debug --wait --timeout 2400s deploy-test charts/devlake \
67118
--set service.uiPort=30000 \
68-
--set mysql.useExternal=true \
69-
--set mysql.externalServer=mysql \
70-
--set lake.encryptionSecret.secret=$(openssl rand -base64 2000 | tr -dc 'A-Z' | fold -w 128 | head -n 1)
119+
--set database.type=mysql \
120+
--set database.useExternal=true \
121+
--set database.externalServer=mysql \
122+
--set database.externalPort=3306 \
123+
--set database.username=devlake \
124+
--set database.password="${DB_PASSWORD}" \
125+
--set database.database=lake \
126+
--set lake.encryptionSecret.secret="${ENCRYPTION_SECRET}"
71127
72-
- name: Helm install devlake
128+
- name: Install DevLake (MySQL builtin)
73129
if: matrix.database_type == 'mysql-builtin'
74130
run: |
75-
helm repo add grafana https://grafana.github.io/helm-charts
76-
helm dep build charts/devlake
77-
export NODE_IP=$(kubectl get nodes --namespace default -o jsonpath="{.items[0].status.addresses[0].address}")
78-
echo Node IP: ${NODE_IP}
131+
DB_ROOT_PASSWORD=$(openssl rand -base64 16)
132+
DB_PASSWORD=$(openssl rand -base64 16)
133+
ENCRYPTION_SECRET=$(openssl rand -base64 2000 | tr -dc '[:upper:]' | fold -w 128 | head -n 1)
134+
echo "::add-mask::${DB_ROOT_PASSWORD}"
135+
echo "::add-mask::${DB_PASSWORD}"
136+
echo "::add-mask::${ENCRYPTION_SECRET}"
137+
NODE_IP=$(kubectl get nodes --namespace default -o jsonpath="{.items[0].status.addresses[0].address}")
138+
echo "Node IP: ${NODE_IP}"
139+
helm install --debug --wait --timeout 2400s deploy-test charts/devlake \
140+
--set service.uiPort=30000 \
141+
--set database.type=mysql \
142+
--set database.useExternal=false \
143+
--set database.username=devlake \
144+
--set database.password="${DB_PASSWORD}" \
145+
--set database.database=lake \
146+
--set database.mysql.rootPassword="${DB_ROOT_PASSWORD}" \
147+
--set database.image.tag=8-debian \
148+
--set lake.encryptionSecret.secret="${ENCRYPTION_SECRET}"
149+
150+
- name: Install DevLake (PostgreSQL external)
151+
if: matrix.database_type == 'postgresql-external'
152+
run: |
153+
DB_PASSWORD=$(openssl rand -base64 16)
154+
ENCRYPTION_SECRET=$(openssl rand -base64 2000 | tr -dc '[:upper:]' | fold -w 128 | head -n 1)
155+
echo "::add-mask::${DB_PASSWORD}"
156+
echo "::add-mask::${ENCRYPTION_SECRET}"
157+
NODE_IP=$(kubectl get nodes --namespace default -o jsonpath="{.items[0].status.addresses[0].address}")
158+
echo "Node IP: ${NODE_IP}"
79159
helm install --debug --wait --timeout 2400s deploy-test charts/devlake \
80160
--set service.uiPort=30000 \
81-
--set mysql.image.tag=8-debian \
82-
--set lake.encryptionSecret.secret=$(openssl rand -base64 2000 | tr -dc 'A-Z' | fold -w 128 | head -n 1)
161+
--set database.type=postgresql \
162+
--set database.useExternal=true \
163+
--set database.externalServer=postgresql \
164+
--set database.externalPort=5432 \
165+
--set database.username=devlake \
166+
--set database.password="${DB_PASSWORD}" \
167+
--set database.database=lake \
168+
--set grafana.enabled=false \
169+
--set lake.encryptionSecret.secret="${ENCRYPTION_SECRET}"
170+
171+
- name: Install DevLake (PostgreSQL builtin)
172+
if: matrix.database_type == 'postgresql-builtin'
173+
run: |
174+
DB_PASSWORD=$(openssl rand -base64 16)
175+
ENCRYPTION_SECRET=$(openssl rand -base64 2000 | tr -dc '[:upper:]' | fold -w 128 | head -n 1)
176+
echo "::add-mask::${DB_PASSWORD}"
177+
echo "::add-mask::${ENCRYPTION_SECRET}"
178+
NODE_IP=$(kubectl get nodes --namespace default -o jsonpath="{.items[0].status.addresses[0].address}")
179+
echo "Node IP: ${NODE_IP}"
180+
helm install --debug --wait --timeout 2400s deploy-test charts/devlake \
181+
--set service.uiPort=30000 \
182+
--set database.type=postgresql \
183+
--set database.useExternal=false \
184+
--set database.username=devlake \
185+
--set database.password="${DB_PASSWORD}" \
186+
--set database.database=lake \
187+
--set grafana.enabled=false \
188+
--set lake.encryptionSecret.secret="${ENCRYPTION_SECRET}"
83189
84190
- name: List cluster resources
85191
if: ${{ always() }}
@@ -94,16 +200,16 @@ jobs:
94200
# TODO: using some e2e test code to replace it
95201
- name: Curl with endpoints
96202
run: |
97-
export NODE_IP=$(kubectl get nodes --namespace default -o jsonpath="{.items[0].status.addresses[0].address}")
203+
NODE_IP=$(kubectl get nodes --namespace default -o jsonpath="{.items[0].status.addresses[0].address}")
98204
failed=0
99-
for retry in {1..10} ; do
205+
for _ in {1..10} ; do
100206
failed=0
101207
# home
102-
curl --fail http://${NODE_IP}:30000 || failed=1
208+
curl --fail "http://${NODE_IP}:30000" || failed=1
103209
# API for devlake
104-
curl --fail http://${NODE_IP}:30000/api/blueprints || failed=1
210+
curl --fail "http://${NODE_IP}:30000/api/blueprints" || failed=1
105211
# API for grafana
106-
curl --fail http://${NODE_IP}:30000/grafana/api/health || failed=1
212+
curl --fail "http://${NODE_IP}:30000/grafana/api/health" || failed=1
107213
if [ $failed -eq 0 ] ; then
108214
break
109215
else
@@ -115,12 +221,34 @@ jobs:
115221
exit 1
116222
fi
117223
224+
- name: Dump diagnostics on failure
225+
if: failure()
226+
run: |
227+
echo "=== Helm status for release ==="
228+
helm status deploy-test || true
229+
echo "=== Kubernetes resources (all) ==="
230+
kubectl get all -o wide || true
231+
echo "=== Kubernetes events (latest 200) ==="
232+
kubectl get events --sort-by=.lastTimestamp | tail -n 200 || true
233+
echo "=== Describe deployments/statefulsets ==="
234+
kubectl describe deploy || true
235+
kubectl describe statefulset || true
236+
echo "=== Describe pods ==="
237+
kubectl describe pods || true
238+
echo "=== Logs from all pods (current and previous, last 200 lines) ==="
239+
for pod in $(kubectl get pods -o jsonpath='{.items[*].metadata.name}'); do
240+
echo "----- logs for ${pod} -----"
241+
kubectl logs "$pod" --all-containers --tail=200 || true
242+
echo "----- previous logs for ${pod} -----"
243+
kubectl logs "$pod" --all-containers --previous --tail=200 || true
244+
done
245+
118246
- name: Show logs for pods
119247
if: ${{ always() }}
120248
run: |
121249
for pod in $(kubectl get pods -o jsonpath='{.items[*].metadata.name}') ; do
122-
echo describe for $pod
123-
kubectl describe pod $pod
124-
echo logs for $pod
125-
kubectl logs $pod || echo ""
250+
echo "describe for ${pod}"
251+
kubectl describe pod "$pod"
252+
echo "logs for ${pod}"
253+
kubectl logs "$pod" || echo ""
126254
done

0 commit comments

Comments
 (0)