Skip to content

Commit ae278e8

Browse files
peterSirotnakpeterSirotnak
authored
PMM-7: Fixes for PMM 3.3.0 (#148)
* PMM-7: Fix pmm server port * PMM-7: Fix pmm server port * PMM-7: 8.4 as default ps * PMM-7: 8.4 as default ps * PMM-7: road on pdpgsql * PMM-7: road on pdpgsql * PMM-7: road on pdpgsql * PMM-7: load on pdpgsql * PMM-7: load on pdpgsql * PMM-7: load on pdpgsql * PMM-7: load on pdpgsql * PMM-7: load on pdpgsql * PMM-7: Correct name for client instalation file * PMM-7: minio credentials * PMM-7: minio credentials * PMM-7: minio credentials * PMM-7: minio credentials * PMM-7: minio credentials * PMM-7: minio credentials * PMM-7: New buckets * PMM-7: Configure agents * PMM-7: Configure agents * PMM-7: Minio * PMM-7: Minio * PMM-7: Minio * PMM-7: Minio * PMM-7: Minio * PMM-7: Minio * PMM-7: Minio * PMM-7: Minio * PMM-7: Minio * PMM-7: Minio * PMM-7: Minio * PMM-7: Minio * PMM-7: Minio * PMM-7: Minio * PMM-7: Minio * PMM-7: Minio * PMM-7: Minio * PMM-7: Minio * PMM-7: Minio * PMM-7: Minio * PMM-7: Minio * PMM-7: Minio * PMM-7: Minio * PMM-7: Minio * PMM-7: Minio * PMM-7: Code cleanup * PMM-7: Code cleanup * PMM-7: Default value for bucket * PMM-7: Default value for bucket * PMM-7: Default value for bucket * PMM-7: Default value for bucket * PMM-7: Default value for bucket * PMM-7: Default value for bucket * PMM-7: Default value for bucket * PMM-7: Default value for bucket * PMM-7: Default value for bucket * PMM-7: Default value for bucket * PMM-7: Default value for bucket * PMM-7: Add load to pgsql replication * PMM-7: Add load to pgsql replication * PMM-7: Add load to pgsql replication * PMM-7: Add load to pgsql replication * PMM-7: Add load to pgsql replication * PMM-7: Add load to pgsql replication * PMM-7: background load * PMM-7: background load * PMM-7: background load * PMM-7: background load * PMM-7: background load * PMM-7: background load * PMM-7: background load * PMM-7: cluster and env * PMM-7: cluster and env * PMM-7: cluster and env * PMM-7: cluster and env * PMM-7: cluster and env * PMM-7: cluster and env * PMM-7: cluster and env * PMM-7: cluster and env * PMM-7: cluster and env * PMM-7: cluster and env * PMM-7: cluster and env * PMM-7: cluster and env * PMM-7: cluster and env * PMM-7: cluster and env * PMM-7: cluster and env * PMM-7: cluster and env * PMM-7: cluster and env * PMM-7: cluster and env * PMM-7: cluster and env * PMM-7: cluster and env * PMM-7: cluster and env * PMM-7: cluster and env * PMM-7: cluster and env * PMM-7: cluster and env * PMM-7: cluster and env * PMM-7: cluster and env * PMM-7: cluster and env * PMM-7: cluster and env * PMM-7: install pgsm * PMM-7: install pgsm * PMM-7: install pgsm * PMM-7: log * PMM-7: log * PMM-7: log * PMM-7: log * PMM-7: log * PMM-7: log * PMM-7: log * PMM-7: log * PMM-7: log * PMM-7: log * PMM-7: log * PMM-7: log * PMM-7: log * PMM-7: log * PMM-7: log * PMM-7: log * PMM-7: log * PMM-7: log * PMM-7: log * PMM-7: log * PMM-7: log * PMM-7: log * PMM-7: log * PMM-7: log * PMM-7: log * PMM-7: log * PMM-7: log * PMM-7: log * PMM-7: log * PMM-7: log * PMM-7: log * PMM-7: log * PMM-7: log * PMM-7: log * PMM-7: log * PMM-7: log * PMM-7: log * PMM-7: log * PMM-7: log * PMM-7: log * PMM-7: cleanup * PMM-7: cleanup * PMM-7: cleanup * PMM-7: cleanup * PMM-7: cleanup * PMM-7: cleanup * PMM-7: cleanup * PMM-7: cleanup * PMM-7: cleanup * PMM-7: cleanup * PMM-7: cleanup * PMM-7: cleanup
1 parent 5d5ebb5 commit ae278e8

7 files changed

Lines changed: 201 additions & 51 deletions

File tree

‎pmm_psmdb_diffauth_setup/test-auth.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ docker compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server systemctl re
5555
set +e
5656
i=1
5757
while [ $i -le 3 ]; do
58-
output=$(docker compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server pmm-agent setup 2>&1)
58+
output=$(docker compose -f docker-compose-pmm-psmdb.yml exec -T psmdb-server pmm-agent setup --config-file=/usr/local/percona/pmm/config/pmm-agent.yaml --server-address=pmm-server:8443 --metrics-mode=auto --server-username=admin --server-password=admin --server-insecure-tls)
5959
exit_code=$?
6060

6161
if [ $exit_code -ne 0 ] && [[ $output == *"500 Internal Server Error"* ]]; then

‎pmm_qa/data/load_pgsql.sql‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
-- Step 1: Show initial buffers_alloc value
2+
SELECT 'Initial buffers_alloc' AS info, buffers_alloc FROM pg_stat_bgwriter;
3+
4+
-- Step 2: Drop and create a large test table (~500MB+)
5+
DROP TABLE IF EXISTS buffer_test;
6+
7+
CREATE TABLE buffer_test AS
8+
SELECT
9+
generate_series(1, 1_000_000) AS id,
10+
md5(random()::text) AS filler;
11+
12+
ANALYZE buffer_test;
13+
14+
-- Step 3: Perform repeated full-table scans to stress buffer allocation
15+
DO $$
16+
BEGIN
17+
FOR i IN 1..10 LOOP
18+
RAISE NOTICE 'Running scan iteration %', i;
19+
PERFORM COUNT(*) FROM buffer_test;
20+
END LOOP;
21+
END $$;
22+
23+
---- Step 4: Show final buffers_alloc value
24+
--SELECT 'Final buffers_alloc' AS info, buffers_alloc FROM pg_stat_bgwriter;
25+
VACUUM;
26+
SELECT pg_switch_wal();
27+
28+
-- Step 5: Query the data
29+
SELECT * FROM buffer_test;
30+
31+
-- Step 6: Delete the data
32+
DELETE FROM buffer_test;

‎pmm_qa/percona-distribution-postgresql/data/background_load.sql‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
-- Step 1: Create a test table
2+
CREATE TABLE IF NOT EXISTS test_users (
3+
id SERIAL PRIMARY KEY,
4+
name VARCHAR(100),
5+
email VARCHAR(150),
6+
created_at TIMESTAMP DEFAULT NOW(),
7+
bio TEXT
8+
);
9+
10+
-- Step 2: Insert 10,000 rows
11+
INSERT INTO test_users (name, email, bio)
12+
SELECT
13+
'User_' || gs::TEXT AS name,
14+
'user_' || gs::TEXT || '@example.com' AS email,
15+
'This is a generated bio for user #' || gs::TEXT
16+
FROM generate_series(1, 10000) AS gs;
17+
18+
-- Step 3: Query the data
19+
SELECT * FROM test_users;
20+
21+
-- Step 4: Delete the data
22+
DELETE FROM test_users;
23+
24+
-- Step 5: Drop the table
25+
DROP TABLE test_users;

‎pmm_qa/percona-distribution-postgresql/data/postgresql-primary.conf‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
wal_level = replica
1+
wal_level = logical
22
max_wal_senders = 10
33
wal_keep_size = 64MB
44
hot_standby = on

‎pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-patroni-setup.yml‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,9 @@
22
set_fact:
33
nodes_count: 3
44
when: nodes_count | int < 3
5+
- name: Set external facing port in patroni setup
6+
set_fact:
7+
pdpgsql_port: 6432
58

69
- name: Remove old data folders
710
shell: 'rm -fr {{ data_dir }}'
@@ -173,14 +176,19 @@
173176
- name: Add service to pmm server
174177
community.docker.docker_container_exec:
175178
container: "pdpgsql_pmm_patroni_{{ pg_version }}_{{ item }}"
176-
command: pmm-admin add postgresql --username=postgres --password=postgres --query-source=pgstatmonitor pdpgsql_pmm_patroni_{{ pg_version }}_{{ item }}{{ random_service_name_value }} --debug 127.0.0.1:5432
179+
command: pmm-admin add postgresql --username=postgres --cluster=pdpgsql_patroni_cluster --environment=pdpgsql_patroni_environment --password=postgres --query-source=pgstatmonitor pdpgsql_pmm_patroni_{{ pg_version }}_{{ item }}{{ random_service_name_value }} --debug 127.0.0.1:5432
177180
loop: "{{ range(1, nodes_count | int + 1) | list }}"
178181

182+
- name: Add patroni service to pmm server
183+
community.docker.docker_container_exec:
184+
container: "pdpgsql_pmm_patroni_{{ pg_version }}_1"
185+
command: pmm-admin add external --listen-port=8008 --service-name=patroni_service_1{{ random_service_name_value }}
186+
179187
- name: Add patroni service to pmm server
180188
community.docker.docker_container_exec:
181189
container: "pdpgsql_pmm_patroni_{{ pg_version }}_{{ item }}"
182-
command: pmm-admin add external --listen-port=8008 --service-name=patroni_service_{{ item }}{{ random_service_name_value }}
183-
loop: "{{ range(1, nodes_count | int + 1) | list }}"
190+
command: pmm-admin add external --listen-port=8008 --cluster=pdpgsql_patroni_service_cluster --environment=pdpgsql_patroni_service_environment --service-name=patroni_service_{{ item }}{{ random_service_name_value }}
191+
loop: "{{ range(2, nodes_count | int + 1) | list }}"
184192

185193
- name: Log Patroni cluster
186194
community.docker.docker_container_exec:

‎pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml‎

Lines changed: 66 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -36,15 +36,15 @@
3636

3737
- name: Remove old PostgreSQL primary container
3838
community.docker.docker_container:
39-
name: "pdpgsql_pmm_{{ pg_version }}_1"
39+
name: "pdpgsql_pmm_replication_{{ pg_version }}_1"
4040
image: "{{ docker_repo }}:{{ pg_version }}"
4141
restart_policy: always
4242
state: absent
4343
ignore_errors: yes
4444

4545
- name: Start PostgreSQL primary container
4646
community.docker.docker_container:
47-
name: "pdpgsql_pmm_{{ pg_version }}_1"
47+
name: "pdpgsql_pmm_replication_{{ pg_version }}_1"
4848
image: "{{ docker_repo }}:{{ pg_version }}"
4949
restart_policy: always
5050
state: started
@@ -70,7 +70,7 @@
7070

7171
- name: Create replication user
7272
community.docker.docker_container_exec:
73-
container: "pdpgsql_pmm_{{ pg_version }}_1"
73+
container: "pdpgsql_pmm_replication_{{ pg_version }}_1"
7474
user: postgres
7575
command: >
7676
psql -c "
@@ -79,14 +79,14 @@
7979
8080
- name: Stop and remove replica if exists
8181
community.docker.docker_container:
82-
name: "pdpgsql_pmm_{{ pg_version }}_{{ item }}"
82+
name: "pdpgsql_pmm_replication_{{ pg_version }}_{{ item }}"
8383
state: absent
8484
loop: "{{ range(2, nodes_count | int + 1) | list }}"
8585
ignore_errors: yes
8686

8787
- name: Start Percona Distribution PostgreSQL replica container
8888
community.docker.docker_container:
89-
name: "pdpgsql_pmm_{{ pg_version }}_{{ item }}"
89+
name: "pdpgsql_pmm_replication_{{ pg_version }}_{{ item }}"
9090
image: "{{ docker_repo }}:{{ pg_version }}"
9191
restart_policy: "no"
9292
state: started
@@ -104,14 +104,14 @@
104104

105105
- name: Wipe replica data directory before basebackup
106106
community.docker.docker_container_exec:
107-
container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}"
107+
container: "pdpgsql_pmm_replication_{{ pg_version }}_{{ item }}"
108108
user: root
109109
command: rm -rf /data/db/*
110110
loop: "{{ range(2, nodes_count | int + 1) | list }}"
111111

112112
- name: Create PostgreSQL user 'pmm' with password
113113
community.docker.docker_container_exec:
114-
container: "pdpgsql_pmm_{{ pg_version }}_1"
114+
container: "pdpgsql_pmm_replication_{{ pg_version }}_1"
115115
user: postgres
116116
command: >
117117
bash -c '
@@ -121,16 +121,16 @@
121121
"
122122
'
123123
124-
- name: Create custom database for pgbench
124+
- name: Create custom database
125125
community.docker.docker_container_exec:
126-
container: "pdpgsql_pmm_{{ pg_version }}_1"
126+
container: "pdpgsql_pmm_replication_{{ pg_version }}_1"
127127
user: postgres
128128
command: >
129129
bash -c "
130130
echo \"
131-
CREATE DATABASE pgbench;
132-
\\c pgbench
133-
GRANT CONNECT ON DATABASE pgbench TO pmm;
131+
CREATE DATABASE test_database;
132+
\\c test_database
133+
GRANT CONNECT ON DATABASE test_database TO pmm;
134134
GRANT USAGE ON SCHEMA public TO pmm;
135135
GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public TO pmm;
136136
ALTER DEFAULT PRIVILEGES IN SCHEMA public
@@ -140,20 +140,20 @@
140140
141141
- name: Run pg_basebackup from primary to replica
142142
community.docker.docker_container_exec:
143-
container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}"
143+
container: "pdpgsql_pmm_replication_{{ pg_version }}_{{ item }}"
144144
user: root
145145
command: >
146146
bash -c "
147147
export PGPASSWORD='{{ replication_password }}' && \
148148
timeout 120s \
149149
pg_basebackup --pgdata=/data/db -R -v -Fp -Xs -P \
150-
--host=pdpgsql_pmm_{{ pg_version }}_1 --port=5432 -U {{ replication_user }} > /tmp/pg_basebackup.log 2>&1
150+
--host=pdpgsql_pmm_replication_{{ pg_version }}_1 --port=5432 -U {{ replication_user }} > /tmp/pg_basebackup.log 2>&1
151151
"
152152
loop: "{{ range(2, nodes_count | int + 1) | list }}"
153153

154154
- name: Remove temporary backup container
155155
docker_container:
156-
name: "pdpgsql_pmm_{{ pg_version }}_{{ item }}"
156+
name: "pdpgsql_pmm_replication_{{ pg_version }}_{{ item }}"
157157
state: absent
158158
loop: "{{ range(2, nodes_count | int + 1) | list }}"
159159

@@ -168,7 +168,7 @@
168168

169169
- name: Restart Percona Distribution PostgreSQL container with custom command
170170
community.docker.docker_container:
171-
name: "pdpgsql_pmm_{{ pg_version }}_{{ item }}"
171+
name: "pdpgsql_pmm_replication_{{ pg_version }}_{{ item }}"
172172
image: "{{ docker_repo }}:{{ pg_version }}"
173173
restart: true
174174
state: started
@@ -188,17 +188,17 @@
188188
- name: Install pg stat monitor.
189189
include_tasks: ./tasks/install_pg_stat-monitor.yml
190190
vars:
191-
container_name: "pdpgsql_pmm_{{ pg_version }}"
191+
container_name: "pdpgsql_pmm_replication_{{ pg_version }}"
192192

193193
- name: Install and add pmm client.
194194
include_tasks: ../tasks/install_pmm_client.yml
195195
vars:
196-
container_name: "pdpgsql_pmm_{{ pg_version }}_{{ item }}"
196+
container_name: "pdpgsql_pmm_replication_{{ pg_version }}_{{ item }}"
197197
loop: "{{ range(1, nodes_count | int + 1) | list }}"
198198

199199
- name: Get already connected services to pmm server
200200
community.docker.docker_container_exec:
201-
container: "pdpgsql_pmm_{{ pg_version }}_1"
201+
container: "pdpgsql_pmm_replication_{{ pg_version }}_1"
202202
command: >
203203
sh -c 'curl --location --insecure -u"admin:{{ admin_password }}" -s --request GET "http://{{ pmm_server_ip }}:{{ '80' if pmm_server_ip is ansible.utils.ipv4 else '8080' }}/v1/management/services" | jq -r ".services[].service_name"'
204204
register: pmm_server_services
@@ -217,20 +217,57 @@
217217

218218
- name: Add service to pmm server
219219
community.docker.docker_container_exec:
220-
container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}"
221-
command: pmm-admin add postgresql --username=pmm --password=pmm --query-source=pgstatmonitor pdpgsql_pmm_{{ pg_version }}_{{ item }}{{ random_service_name_value }} --debug 127.0.0.1:5432
220+
container: "pdpgsql_pmm_replication_{{ pg_version }}_{{ item }}"
221+
command: pmm-admin add postgresql --username=pmm --password=pmm --cluster=pdpgsql_replication_cluster --environment=pdpgsql_replication_environment --query-source=pgstatmonitor pdpgsql_pmm_replication_{{ pg_version }}_{{ item }}{{ random_service_name_value }} --debug 127.0.0.1:5432
222222
loop: "{{ range(1, nodes_count | int + 1) | list }}"
223223

224-
- name: Run load on primary node.
225-
include_tasks: ../postgresql/tasks/run_load_pgsql.yml
226-
vars:
227-
node_name: "pdpgsql_pmm_{{ pg_version }}_1"
228-
pgbench_clients: 10
229-
pgbench_time: 120
230-
pgbench_scale: 1000
224+
- name: Create pg_custom_publication view using psql
225+
community.docker.docker_container_exec:
226+
container: "pdpgsql_pmm_replication_{{ pg_version }}_1"
227+
user: postgres
228+
command: >
229+
bash -c 'psql -U postgres -d postgres <<SQL
230+
CREATE OR REPLACE VIEW pg_custom_publication AS
231+
SELECT
232+
pubname,
233+
puballtables,
234+
pubinsert,
235+
pubupdate,
236+
pubdelete
237+
FROM
238+
pg_publication;
239+
SQL'
240+
241+
- name: Create pg_stat_statements extension
242+
community.docker.docker_container_exec:
243+
container: "pdpgsql_pmm_replication_{{ pg_version }}_1"
244+
user: postgres
245+
command: >
246+
psql -U postgres -d test_database -c "
247+
CREATE EXTENSION IF NOT EXISTS pg_stat_monitor;
248+
SELECT pg_stat_monitor_version();
249+
"
250+
251+
- name: Copy sample load into container
252+
community.docker.docker_container_copy_into:
253+
container: "pdpgsql_pmm_replication_{{ pg_version }}_1"
254+
path: ../data/load_pgsql.sql
255+
container_path: /load_pgsql.sql
256+
257+
- name: Start permanent SQL load in background
258+
community.docker.docker_container_exec:
259+
container: "pdpgsql_pmm_replication_{{ pg_version }}_1"
260+
command: >
261+
sh -c "nohup bash -c 'while true; do echo Starting insert at $(date +\"%Y-%m-%d %H:%M:%S\"); psql -U postgres -d test_database -f /load_pgsql.sql; sleep 30; done' > /tmp/sql_loop.log 2>&1 &"
262+
263+
- name: Create logical replication slots
264+
community.docker.docker_container_exec:
265+
container: "pdpgsql_pmm_replication_{{ pg_version }}_1"
266+
user: postgres
267+
command: psql -U postgres -d postgres -c "SELECT * FROM pg_create_logical_replication_slot('test_slot', 'test_decoding');"
231268

232269
- name: Check replication status on primary
233270
community.docker.docker_container_exec:
234-
container: "pdpgsql_pmm_{{ pg_version }}_1"
271+
container: "pdpgsql_pmm_replication_{{ pg_version }}_1"
235272
user: postgres
236273
command: psql -c "SELECT * FROM pg_stat_replication;"

0 commit comments

Comments
 (0)