Skip to content

Commit 6eb306c

Browse files
committed
Prepare a container with psql and rclone
Signed-off-by: Nikolai Rodionov <iam@allanger.xyz>
1 parent 3c6ece8 commit 6eb306c

8 files changed

Lines changed: 313 additions & 0 deletions

File tree

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
name: Publish a container image
2+
3+
on:
4+
push:
5+
release:
6+
types: [published, edited]
7+
8+
jobs:
9+
build:
10+
runs-on: ubuntu-24.04
11+
permissions:
12+
packages: write
13+
steps:
14+
- uses: actions/checkout@v6
15+
- uses: docker/setup-qemu-action@v3
16+
- run: ./build/set_image_metadata
17+
- uses: docker/setup-compose-action@v1
18+
- name: Build an image
19+
id: build-image
20+
uses: redhat-actions/buildah-build@v2
21+
with:
22+
image: ${{ github.event.repository.name }}-dev
23+
tags: ${{ env.TAGS }}
24+
platforms: linux/amd64, linux/arm64/v8
25+
containerfiles: |
26+
./Containerfile
27+
labels: ${{ env.ANNOTATIONS }}
28+
29+
- name: Push the image to GHCR
30+
id: push-to-ghcr
31+
uses: redhat-actions/push-to-registry@v2
32+
with:
33+
image: ${{ steps.build-image.outputs.image }}
34+
tags: ${{ steps.build-image.outputs.tags }}
35+
registry: ghcr.io/${{ github.repository_owner }}
36+
username: ${{ github.actor }}
37+
password: ${{ github.token }}
38+
39+
40+
- name: Set the version tag (only if released)
41+
id: retag-version
42+
if: (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'edited'))
43+
run: |
44+
buildah tag \
45+
${{ steps.build-image.outputs.image }}:${{ github.sha }} \
46+
ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}:latest
47+
buildah tag \
48+
${{ steps.build-image.outputs.image }}:${{ github.sha }} \
49+
ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}:${{ github.event.release.tag_name }}
50+
51+
- name: Push the release image to GHCR
52+
if: (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'edited'))
53+
uses: redhat-actions/push-to-registry@v2
54+
with:
55+
tags: ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}:latest ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}:${{ github.event.release.tag_name }}
56+
username: ${{ github.actor }}
57+
password: ${{ github.token }}

Containerfile

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
FROM postgres:18-alpine
2+
3+
RUN apk --no-cache add \
4+
curl \
5+
bash \
6+
rclone
7+
8+
COPY entrypoint.sh /
9+
RUN chmod +x /entrypoint.sh
10+
11+
WORKDIR /backup
12+
ENTRYPOINT /entrypoint.sh

README.md

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
# pgdump-gcs
2+
3+
Small docker container for creating a backup of a psql database and upload the dump to an external storage using rclone.
4+
5+
## How to use
6+
TO BE DONE ...
7+
8+
## monitoring
9+
10+
Simple curl pushing some basic parameter to a prometheus push gateway.
11+
12+
### metrics
13+
* timestamp
14+
* duration
15+
* size
16+
17+
### labels
18+
* job = pgdump-gcs
19+
* source_type = postgresql
20+
* source_name = `${DB_NAME}`

build/set_image_metadata

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
#! /usr/bin/env bash
2+
# --------------------------------------------
3+
# -- Should be used in Github Actions
4+
# --------------------------------------------
5+
#
6+
# --------------------------------------------
7+
# -- To have a multi-line env var in github,
8+
# -- we must define the EOF
9+
# --------------------------------------------
10+
EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64)
11+
echo "ANNOTATIONS<<$EOF" >> "$GITHUB_ENV"
12+
13+
ANNOTATIONS=$(cat << EOF
14+
org.opencontainers.image.created=$(date +"%Y-%m-%d %T")
15+
org.opencontainers.image.authors=$GITHUB_ACTOR
16+
org.opencontainers.image.url=$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID
17+
org.opencontainers.image.documentation=$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/blob/main/README.md
18+
org.opencontainers.image.source=$GITHUB_SERVER_URL/$GITHUB_REPOSITORY
19+
org.opencontainers.image.version=$GITHUB_SHA
20+
org.opencontainers.image.revision=$GITHUB_SHA
21+
org.opencontainers.image.vendor=$GITHUB_REPOSITORY_OWNER
22+
org.opencontainers.image.license=GNU GENERAL PUBLIC LICENSE v3
23+
org.opencontainers.image.title=$GITHUB_REPOSITORY
24+
org.opencontainers.image.description=Backup databases using pg_dump and upload backups using rclone
25+
EOF
26+
)
27+
28+
echo "${ANNOTATIONS}" >> "${GITHUB_ENV}"
29+
echo "$EOF" >> "$GITHUB_ENV"
30+
# --------------------------------------------
31+
# -- Set the image tag by commit sha
32+
# --------------------------------------------
33+
echo "TAGS=${GITHUB_SHA}" >> "${GITHUB_ENV}"
34+
35+

entrypoint.sh

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
#!/bin/bash
2+
set -e
3+
4+
echo "Prepare configuration for script"
5+
TIMESTAMP=$(date +%F_%R)
6+
START_TIMESTAMP=$(date +%s)
7+
BACKUP_FILE=${DB_NAME}-${TIMESTAMP}.sql.gz
8+
BACKUP_FILE_LATEST=${DB_NAME}-latest.sql.gz
9+
DB_HOST=${DB_HOST:-localhost}
10+
DB_PASSWORD=$(cat ${DB_PASSWORD_FILE})
11+
DB_USER=$(cat ${DB_USERNAME_FILE})
12+
PROM_NAMESPACE=${PROM_NAMESPACE:-dboperator}
13+
14+
if [[ -z "${STORAGE_BUCKET}" ]]; then
15+
echo "Variable STORAGE_BUCKET must be set"
16+
exit 1
17+
fi
18+
19+
# create login credential file
20+
(umask 377 && echo *:5432:*:${DB_USER}:${DB_PASSWORD} >> ~/.pgpass)
21+
22+
echo "Start create backup"
23+
pg_dump -F c -Z 9 -h ${DB_HOST} -p 5432 -U ${DB_USER} ${DB_NAME} -f ${BACKUP_FILE}
24+
BACKUP_SIZE=$(du ${BACKUP_FILE} | awk '{print $1}')
25+
echo "End backup"
26+
27+
## copy to destination
28+
echo "Copy to gcs"
29+
rclone copyto "./${BACKUP_FILE}" "storage://${STORAGE_BUCKET}/${DB_NAME}/${BACKUP_FILE}"
30+
rclone copyto "./${BACKUP_FILE}" "storage://${STORAGE_BUCKET}/${DB_NAME}/${BACKUP_FILE_LATEST}"
31+
32+
END_TIMESTAMP=$(date +%s)
33+
BACKUP_DURATION=$((END_TIMESTAMP - START_TIMESTAMP))
34+
if [[ ! -z "$PROMETHEUS_PUSH_GATEWAY" ]];
35+
then
36+
echo "sending monitoring metrics to ${PROMETHEUS_PUSH_GATEWAY}"
37+
cat <<EOF | curl -s --data-binary @- http://${PROMETHEUS_PUSH_GATEWAY}/metrics/job/pgdump-rclone/source_type/postgresql/source_name/${DB_NAME}
38+
# TYPE ${PROM_NAMESPACE}_backup_timestamp counter
39+
# HELP ${PROM_NAMESPACE}_backup_timestamp Timestamp of last backup run
40+
${PROM_NAMESPACE}_backup_timestamp $END_TIMESTAMP
41+
# TYPE ${PROM_NAMESPACE}_backup_duration gauge
42+
# HELP ${PROM_NAMESPACE}_backup_duration Time the backup run take until finished
43+
${PROM_NAMESPACE}_backup_duration $BACKUP_DURATION
44+
# TYPE ${PROM_NAMESPACE}_backup_size gauge
45+
# HELP ${PROM_NAMESPACE}_backup_size Backup Size in bytes
46+
${PROM_NAMESPACE}_backup_size $BACKUP_SIZE
47+
EOF
48+
fi

test/docker-compose.yaml

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
version: "3.3"
2+
services:
3+
postgres:
4+
image: postgres:18
5+
ports:
6+
- "5432:5432"
7+
environment:
8+
POSTGRES_PASSWORD: "test1234
9+

test/init.sql

Lines changed: 123 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,123 @@
1+
-- =========================================================
2+
-- Random Test Data Generator for PostgreSQL Backup Testing
3+
-- =========================================================
4+
5+
-- Optional: speed up bulk inserts
6+
SET synchronous_commit = OFF;
7+
SET maintenance_work_mem = '512MB';
8+
9+
-- =========================================================
10+
-- Drop existing tables (safe reset)
11+
-- =========================================================
12+
13+
DROP TABLE IF EXISTS order_items;
14+
DROP TABLE IF EXISTS orders;
15+
DROP TABLE IF EXISTS products;
16+
DROP TABLE IF EXISTS customers;
17+
18+
-- =========================================================
19+
-- Create tables
20+
-- =========================================================
21+
22+
CREATE TABLE customers (
23+
id BIGSERIAL PRIMARY KEY,
24+
first_name TEXT NOT NULL,
25+
last_name TEXT NOT NULL,
26+
email TEXT NOT NULL,
27+
created_at TIMESTAMP NOT NULL DEFAULT now()
28+
);
29+
30+
CREATE TABLE products (
31+
id BIGSERIAL PRIMARY KEY,
32+
name TEXT NOT NULL,
33+
price NUMERIC(10,2) NOT NULL,
34+
created_at TIMESTAMP NOT NULL DEFAULT now()
35+
);
36+
37+
CREATE TABLE orders (
38+
id BIGSERIAL PRIMARY KEY,
39+
customer_id BIGINT NOT NULL REFERENCES customers(id),
40+
order_date TIMESTAMP NOT NULL,
41+
status TEXT NOT NULL
42+
);
43+
44+
CREATE TABLE order_items (
45+
id BIGSERIAL PRIMARY KEY,
46+
order_id BIGINT NOT NULL REFERENCES orders(id),
47+
product_id BIGINT NOT NULL REFERENCES products(id),
48+
quantity INT NOT NULL,
49+
unit_price NUMERIC(10,2) NOT NULL
50+
);
51+
52+
-- =========================================================
53+
-- Insert customers
54+
-- Adjust the number in generate_series for scale
55+
-- =========================================================
56+
57+
INSERT INTO customers (first_name, last_name, email, created_at)
58+
SELECT
59+
'First' || gs,
60+
'Last' || gs,
61+
'user' || gs || '@example.com',
62+
NOW() - (random() * interval '365 days')
63+
FROM generate_series(1, 100000) AS gs;
64+
65+
-- =========================================================
66+
-- Insert products
67+
-- =========================================================
68+
69+
INSERT INTO products (name, price, created_at)
70+
SELECT
71+
'Product ' || gs,
72+
ROUND((random() * 500 + 5)::numeric, 2),
73+
NOW() - (random() * interval '365 days')
74+
FROM generate_series(1, 5000) AS gs;
75+
76+
-- =========================================================
77+
-- Insert orders
78+
-- =========================================================
79+
80+
INSERT INTO orders (customer_id, order_date, status)
81+
SELECT
82+
(random() * 99999 + 1)::BIGINT,
83+
NOW() - (random() * interval '365 days'),
84+
(ARRAY['pending','shipped','delivered','cancelled'])[floor(random()*4)+1]
85+
FROM generate_series(1, 300000);
86+
87+
-- =========================================================
88+
-- Insert order items
89+
-- Each order gets 1–5 items
90+
-- =========================================================
91+
92+
INSERT INTO order_items (order_id, product_id, quantity, unit_price)
93+
SELECT
94+
o.id,
95+
(random() * 4999 + 1)::BIGINT,
96+
(random() * 4 + 1)::INT,
97+
ROUND((random() * 500 + 5)::numeric, 2)
98+
FROM orders o
99+
CROSS JOIN LATERAL generate_series(1, (random()*4 + 1)::INT);
100+
101+
-- =========================================================
102+
-- Indexes (important for realistic backup size)
103+
-- =========================================================
104+
105+
CREATE INDEX idx_orders_customer ON orders(customer_id);
106+
CREATE INDEX idx_order_items_order ON order_items(order_id);
107+
CREATE INDEX idx_order_items_product ON order_items(product_id);
108+
109+
-- =========================================================
110+
-- Analyze for realistic planner stats
111+
-- =========================================================
112+
113+
ANALYZE;
114+
115+
-- =========================================================
116+
-- Summary
117+
-- =========================================================
118+
119+
SELECT
120+
(SELECT count(*) FROM customers) AS customers,
121+
(SELECT count(*) FROM products) AS products,
122+
(SELECT count(*) FROM orders) AS orders,
123+
(SELECT count(*) FROM order_items) AS order_items;

test/init_database.sh

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
#!/usr/bin/env bash
2+
3+
export PGHOST=localhost
4+
export PHGPORT=5432
5+
export PGUSER=postgres
6+
export PGPASSWORD=test1234
7+
export PGDATABASE=postgres
8+
9+
psql -f ./test/init.sql

0 commit comments

Comments
 (0)