From fdc6a09b5284a9fd8f28916b879463fbf702223c Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 28 Apr 2025 15:04:17 +0200 Subject: [PATCH 001/340] PMM-12153: Async replication for Percona Server (#129) --- .../data/init-async-replication.sql.j2 | 13 ++ .../data/my-async-replication.cnf.j2 | 42 ++++ .../percona_server/percona-server-setup.yml | 214 +++++------------- ...percona-server-async-replication-setup.yml | 129 +++++++++++ ...percona-server-group-replication-setup.yml | 127 +++++++++++ .../tasks/percona-server-setup.yml | 24 ++ pmm_qa/pmm-framework.py | 12 + 7 files changed, 402 insertions(+), 159 deletions(-) create mode 100644 pmm_qa/percona_server/data/init-async-replication.sql.j2 create mode 100644 pmm_qa/percona_server/data/my-async-replication.cnf.j2 create mode 100644 pmm_qa/percona_server/tasks/percona-server-async-replication-setup.yml create mode 100644 pmm_qa/percona_server/tasks/percona-server-group-replication-setup.yml create mode 100644 pmm_qa/percona_server/tasks/percona-server-setup.yml diff --git a/pmm_qa/percona_server/data/init-async-replication.sql.j2 b/pmm_qa/percona_server/data/init-async-replication.sql.j2 new file mode 100644 index 00000000..a96ef137 --- /dev/null +++ b/pmm_qa/percona_server/data/init-async-replication.sql.j2 @@ -0,0 +1,13 @@ +-- Create replication user and grant necessary privileges +SET SQL_LOG_BIN=0; +CREATE USER '{{ replication_user }}'@'%' IDENTIFIED WITH 'caching_sha2_password' BY '{{ replication_password }}' REQUIRE NONE; +GRANT REPLICATION SLAVE ON *.* TO '{{ replication_user }}'@'%'; +GRANT CONNECTION_ADMIN ON *.* TO '{{ replication_user }}'@'%'; +GRANT BACKUP_ADMIN ON *.* TO '{{ replication_user }}'@'%'; +FLUSH PRIVILEGES; +SET SQL_LOG_BIN=1; + +{% if item == 1 %} +-- Primary server: enable binary logging for replication +FLUSH BINARY LOGS; +{% endif %} diff --git a/pmm_qa/percona_server/data/my-async-replication.cnf.j2 b/pmm_qa/percona_server/data/my-async-replication.cnf.j2 new file mode 100644 index 00000000..a0b341e4 --- /dev/null +++ b/pmm_qa/percona_server/data/my-async-replication.cnf.j2 @@ -0,0 +1,42 @@ +[mysqld] +# General server configuration +server_id={{ item }} +bind-address=0.0.0.0 +port={{ mysql_listen_port }} + +# Authentication settings for caching_sha2_password +caching_sha2_password_auto_generate_rsa_keys=ON +# The following two parameters tell MySQL where to store the RSA key pair +caching_sha2_password_private_key_path=private_key.pem +caching_sha2_password_public_key_path=public_key.pem + +# Replication settings +gtid_mode=ON +enforce_gtid_consistency=ON +log_bin=binlog +log_replica_updates=ON +sync_binlog=1 +binlog_checksum=NONE +disabled_storage_engines="MyISAM,BLACKHOLE,FEDERATED,ARCHIVE,MEMORY" +# MacOS-specific, where table names are case-sensitive +lower_case_table_names=2 + +# MySQL 8.4 compatibility settings +report_host=ps_pmm_{{ ps_version }}_{{ item }} + +# Replica configuration - applies to all nodes except primary (they'll be able to become replicas) +{% if item != 1 %} +# Replica specific settings +replica_parallel_workers=4 +replica_parallel_type=LOGICAL_CLOCK +replica_preserve_commit_order=1 +{% endif %} + +# Crash-safe replication settings +relay-log=ps_pmm_{{ ps_version }}_{{ item }}-relay-bin +relay_log_recovery=ON +relay_log_purge=ON + +# Performance and connection settings +max_connections=1000 +innodb_buffer_pool_size=256M diff --git a/pmm_qa/percona_server/percona-server-setup.yml b/pmm_qa/percona_server/percona-server-setup.yml index 896af31e..7d7cb4fa 100644 --- a/pmm_qa/percona_server/percona-server-setup.yml +++ b/pmm_qa/percona_server/percona-server-setup.yml @@ -22,8 +22,19 @@ admin_password: "{{ lookup('vars', 'extra_admin_password', default=lookup('env','ADMIN_PASSWORD') | default('admin', true) ) }}" query_source: "{{ lookup('env', 'QUERY_SOURCE') | default('perfschema', true) }}" setup_type: "{{ lookup('env', 'SETUP_TYPE') }}" + random_service_name_value: "" tasks: + - name: Chance to correct nodes count for group replication + set_fact: + nodes_count: 3 + when: nodes_count | int < 3 and setup_type == "gr" + + - name: Chance to correct nodes count for async replication + set_fact: + nodes_count: 2 + when: nodes_count | int < 2 and setup_type == "replication" + - name: Create Docker network community.docker.docker_network: name: "{{ network_name }}" @@ -40,20 +51,6 @@ mode: '0755' loop: "{{ range(1, nodes_count | int + 1) | list }}" - - name: Generate my.cnf for each node - template: - src: my.cnf.j2 - dest: "{{ data_dir }}/node{{ item }}/my.cnf" - loop: "{{ range(1, nodes_count | int + 1) | list }}" - when: setup_type == "gr" - - - name: Create initialization script for each node - template: - src: init.sql.j2 - dest: "{{ data_dir }}/node{{ item }}/init.sql" - loop: "{{ range(1, nodes_count | int + 1) | list }}" - when: setup_type == "gr" - - name: Remove old percona server containers community.docker.docker_container: name: "ps_pmm_{{ ps_version }}_{{ item }}" @@ -67,64 +64,17 @@ shell: "sudo chown -R 1001:1001 {{ data_dir }}/node{{ item }}/data" loop: "{{ range(1, nodes_count | int + 1) | list }}" - - name: Start Percona Server containers with group replication - community.docker.docker_container: - name: "ps_pmm_{{ ps_version }}_{{ item }}" - image: "percona/percona-server:{{ ps_version }}" - restart_policy: always - state: started - networks: - - name: "{{ network_name }}" - env: - MYSQL_ROOT_PASSWORD: "{{ root_password }}" - ports: - - "{{ mysql_port + item - 1 }}:{{ mysql_listen_port }}" - - "{{ group_seeds_port + item - 1 }}:{{ group_seeds_port }}" - volumes: - - "{{ data_dir }}/node{{ item }}/data:/var/lib/mysql" - - "{{ data_dir }}/node{{ item }}/my.cnf:/etc/mysql/my.cnf" - - "{{ data_dir }}/node{{ item }}/init.sql:/docker-entrypoint-initdb.d/init.sql" - loop: "{{ range(1, nodes_count | int + 1) | list }}" + - name: Setup Percona Server group replication + include_tasks: ./tasks/percona-server-group-replication-setup.yml when: setup_type == "gr" - - name: Start Percona Server containers - community.docker.docker_container: - name: "ps_pmm_{{ ps_version }}_{{ item }}" - image: "percona/percona-server:{{ ps_version }}" - restart_policy: always - state: started - networks: - - name: "{{ network_name }}" - env: - MYSQL_ROOT_PASSWORD: "{{ root_password }}" - ports: - - "{{ mysql_port + item - 1 }}:{{ mysql_listen_port }}" - - "{{ group_seeds_port + item - 1 }}:{{ group_seeds_port }}" - volumes: - - "{{ data_dir }}/node{{ item }}/data:/var/lib/mysql" - loop: "{{ range(1, nodes_count | int + 1) | list }}" - when: setup_type != "gr" + - name: Setup Percona Server with async replication + include_tasks: ./tasks/percona-server-async-replication-setup.yml + when: setup_type == "replication" - - name: Wait for MySQL to be available - wait_for: - host: localhost - port: "{{ mysql_port + item - 1 }}" - delay: 10 - timeout: 300 - loop: "{{ range(1, nodes_count | int + 1) | list }}" - - - name: Reset configuration for all nodes - community.docker.docker_container_exec: - container: "ps_pmm_{{ ps_version }}_{{ item }}" - command: > - mysql -uroot -p{{ root_password }} -e " - RESET BINARY LOGS AND GTIDS; - RESET REPLICA ALL; - SET GLOBAL gtid_purged=''; - " - loop: "{{ range(1, nodes_count | int + 1) | list }}" - ignore_errors: yes - when: setup_type == "gr" + - name: Setup Percona Server + include_tasks: ./tasks/percona-server-setup.yml + when: setup_type != "gr" and setup_type != "replication" - name: Create slowlog configuration for mysql nodes community.docker.docker_container_exec: @@ -139,102 +89,48 @@ loop: "{{ range(1, nodes_count | int + 1) | list }}" when: query_source == "slowlog" - - name: Bootstrap first node in the cluster - community.docker.docker_container_exec: - container: "ps_pmm_{{ ps_version }}_1" - command: > - mysql -uroot -p{{ root_password }} -e " - SET GLOBAL group_replication_bootstrap_group=ON; - START GROUP_REPLICATION; - SET GLOBAL group_replication_bootstrap_group=OFF;" - when: setup_type == "gr" - retries: 5 - delay: 10 + - name: Install and add pmm client. + include_tasks: ../tasks/install_pmm_client_centos.yml + vars: + container_name: "ps_pmm_{{ ps_version }}_{{ item }}" + loop: "{{ range(1, nodes_count | int + 1) | list }}" - - name: Wait 5 seconds for bootstrap to complete - pause: - seconds: 5 - when: setup_type == "gr" + - name: Get already connected services to pmm server + shell: curl --location --insecure -u"admin:admin" -s --request GET "http://172.105.74.90/v1/management/services" | jq -r '.services[].service_name' + register: pmm_server_services - - name: Start group replication on other nodes - community.docker.docker_container_exec: - container: "ps_pmm_{{ ps_version }}_{{ item }}" - command: mysql -uroot -p{{ root_password }} -e "START GROUP_REPLICATION;" - loop: "{{ range(2, nodes_count | int + 1) | list }}" - ignore_errors: yes - when: setup_type == "gr" + - name: Display already connected services to pmm server + debug: + msg: "{{ pmm_server_services.stdout | split('\n') }}" - - name: Wait 10 seconds for the other nodes to join - pause: - seconds: 10 - when: setup_type == "gr" + - name: Find out if service is already connected to pmm server + block: + - name: Loop through percona servers + set_fact: + random_service_name_value: "_{{ 9999 | random + 1 }}" + loop: "{{ range(1, nodes_count | int + 1) | list }}" + when: "('ps_pmm_' ~ ps_version ~ '_' ~ item) in pmm_server_services.stdout" - - name: Create and seed a test database on primary + - name: Add service to pmm server community.docker.docker_container_exec: - container: "ps_pmm_{{ ps_version }}_1" - command: > - mysql -uroot -p{{ root_password}} -e " - CREATE DATABASE testdb; - USE testdb; - CREATE TABLE testdb (id INT PRIMARY KEY, data VARCHAR(100)); - INSERT INTO testdb VALUES (1, 'Initial data from node mysql1');" + container: "ps_pmm_{{ ps_version }}_{{ item }}" + command: pmm-admin add mysql --query-source={{ query_source }} --username=root --password={{ root_password }} --environment=ps-gr-dev --cluster=ps-gr-dev-cluster --replication-set=ps-gr-replication ps_pmm_{{ ps_version }}_{{ item }}{{ random_service_name_value }} --debug 127.0.0.1:3306 + loop: "{{ range(1, nodes_count | int + 1) | list }}" when: setup_type == "gr" - - name: Check replication status on first node + - name: Add service to pmm server community.docker.docker_container_exec: - container: "ps_pmm_{{ ps_version }}_1" - command: mysql -uroot -p{{ root_password }} -e "SELECT * FROM performance_schema.replication_group_members;" - register: replication_status - when: setup_type == "gr" - - - name: Display replication status - debug: - var: replication_status.stdout - when: setup_type == "gr" + container: "ps_pmm_{{ ps_version }}_{{ item }}" + command: pmm-admin add mysql --query-source={{ query_source }} --username=root --password={{ root_password }} --environment=ps-replication-dev --cluster=ps-replication-dev-cluster --replication-set=ps-async-replication ps_pmm_{{ ps_version }}_{{ item }}{{ random_service_name_value }} --debug 127.0.0.1:3306 + loop: "{{ range(1, nodes_count | int + 1) | list }}" + when: setup_type == "replication" - - name: Check replication group members count + - name: Add service to pmm server community.docker.docker_container_exec: - container: "ps_pmm_{{ ps_version }}_1" - command: mysql -uroot -p{{ root_password }} -e "SELECT COUNT(*) AS count FROM performance_schema.replication_group_members;" - register: member_count - when: setup_type == "gr" - - - name: Display member count - debug: - var: member_count.stdout - when: setup_type == "gr" - - - name: Set verification instructions - set_fact: - verification_msg: | - MySQL Cluster setup complete! - - To verify replication is working: - 1. Connect to the first node: - docker exec -it ps_pmm_{{ ps_version }}_1 mysql -uroot -p{{ root_password }} - - 2. Insert data in the test database: - USE testdb; - INSERT INTO testdb VALUES (100, 'Test replication'); - - 3. Connect to other nodes and verify data is replicated: - docker exec -it ps_pmm_{{ ps_version }}_2 mysql -uroot -p{{ root_password }} - USE testdb; - SELECT * FROM testdb; - when: setup_type == "gr" - - - name: Display verification instructions - debug: - msg: "{{ verification_msg | split('\n') }}" - when: setup_type == "gr" - - - name: Install pmm client and connect to pmm server - include_tasks: ../tasks/add_mysql_to_pmm_server.yml - vars: - container_name: "ps_pmm_{{ ps_version }}_{{ item }}" + container: "ps_pmm_{{ ps_version }}_{{ item }}" + command: pmm-admin add mysql --query-source={{ query_source }} --username=root --password={{ root_password }} --environment=ps-dev ps_pmm_{{ ps_version }}_{{ item }}{{ random_service_name_value }} --debug 127.0.0.1:3306 loop: "{{ range(1, nodes_count | int + 1) | list }}" - - + when: setup_type != "gr" and setup_type != "replication" - name: Install sysbench inside of all percona server nodes community.docker.docker_container_exec: @@ -242,7 +138,7 @@ user: "root" command: > /bin/sh -c " - wget -O epel-release.rpm -nv https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && + wget -O epel-release.rpm --progress=dot:giga https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && rpm -i epel-release.rpm && microdnf install -y sysbench " @@ -271,7 +167,7 @@ FLUSH PRIVILEGES; " loop: "{{ range(1, nodes_count | int + 1) | list }}" - when: setup_type != "gr" + when: setup_type != "gr" and setup_type != "replication" - name: Prepare sysbench inside of all percona server nodes community.docker.docker_container_exec: @@ -285,7 +181,7 @@ GRANT ALL PRIVILEGES ON *.* TO 'sbtest'@'127.0.0.1'; FLUSH PRIVILEGES; " - when: setup_type == "gr" + when: setup_type == "gr" or setup_type == "replication" - name: Prepare data for sysbench inside of all percona server nodes community.docker.docker_container_exec: @@ -300,7 +196,7 @@ --tables=10 --table-size=100000 prepare - when: setup_type != "gr" + when: setup_type != "gr" and setup_type != "replication" loop: "{{ range(1, nodes_count | int + 1) | list }}" - name: Prepare data for sysbench inside of first percona server nodes @@ -316,7 +212,7 @@ --tables=10 --table-size=100000 prepare - when: setup_type == "gr" + when: setup_type == "gr" or setup_type == "replication" - name: Run load for sysbench inside of all percona server nodes community.docker.docker_container_exec: diff --git a/pmm_qa/percona_server/tasks/percona-server-async-replication-setup.yml b/pmm_qa/percona_server/tasks/percona-server-async-replication-setup.yml new file mode 100644 index 00000000..67cebb47 --- /dev/null +++ b/pmm_qa/percona_server/tasks/percona-server-async-replication-setup.yml @@ -0,0 +1,129 @@ +- name: Generate my.cnf for each node + template: + src: ./data/my-async-replication.cnf.j2 + dest: "{{ data_dir }}/node{{ item }}/my.cnf" + loop: "{{ range(1, nodes_count | int + 1) | list }}" + +- name: Create initialization script for each node + template: + src: ./data/init-async-replication.sql.j2 + dest: "{{ data_dir }}/node{{ item }}/init.sql" + loop: "{{ range(1, nodes_count | int + 1) | list }}" + +- name: Start Percona Server containers with async replication + community.docker.docker_container: + name: "ps_pmm_{{ ps_version }}_{{ item }}" + image: "percona/percona-server:{{ ps_version }}" + restart_policy: always + state: started + networks: + - name: "{{ network_name }}" + env: + MYSQL_ROOT_PASSWORD: "{{ root_password }}" + ports: + - "{{ mysql_port + item - 1 }}:{{ mysql_listen_port }}" + volumes: + - "{{ data_dir }}/node{{ item }}/data:/var/lib/mysql" + - "{{ data_dir }}/node{{ item }}/my.cnf:/etc/mysql/my.cnf" + - "{{ data_dir }}/node{{ item }}/init.sql:/docker-entrypoint-initdb.d/init.sql" + loop: "{{ range(1, nodes_count | int + 1) | list }}" + +- name: Wait for MySQL to be available + wait_for: + host: localhost + port: "{{ mysql_port + item - 1 }}" + delay: 10 + timeout: 300 + loop: "{{ range(1, nodes_count | int + 1) | list }}" + +- name: Wait 5 seconds for percona server start to complete + pause: + seconds: 5 + +- name: Reset configuration for all nodes + community.docker.docker_container_exec: + container: "ps_pmm_{{ ps_version }}_{{ item }}" + command: > + mysql -uroot -p{{ root_password }} -e " + RESET BINARY LOGS AND GTIDS; + RESET REPLICA ALL; + " + loop: "{{ range(1, nodes_count | int + 1) | list }}" + ignore_errors: yes + +- name: Get primary ps_pmm_{{ ps_version }}_1 binary log status + community.docker.docker_container_exec: + container: "ps_pmm_{{ ps_version }}_1" + command: > + mysql -uroot -p{{ root_password }} -e " + SHOW BINARY LOG STATUS\G + " + register: primary_status + changed_when: false + + +- name: Display binary log status for primary + debug: + msg: "{{ primary_status.stdout | split('\n') }}" + +- name: Configure replica servers (container2-containerN) + community.docker.docker_container_exec: + container: "ps_pmm_{{ ps_version }}_{{ item }}" + command: > + mysql -uroot -p{{ root_password }} -e " + CHANGE REPLICATION SOURCE TO + SOURCE_HOST='ps_pmm_{{ ps_version }}_1', + SOURCE_PORT={{ mysql_listen_port }}, + SOURCE_USER='{{ replication_user }}', + SOURCE_PASSWORD='{{ replication_password }}', + SOURCE_AUTO_POSITION=1, + SOURCE_PUBLIC_KEY_PATH='', + GET_SOURCE_PUBLIC_KEY=1; + START REPLICA; + " + loop: "{{ range(2, nodes_count | int + 1) | list }}" + +- name: Create and seed a test database on primary + community.docker.docker_container_exec: + container: "ps_pmm_{{ ps_version }}_1" + command: > + mysql -uroot -p{{ root_password}} -e " + CREATE DATABASE testdb; + USE testdb; + CREATE TABLE testdb (id INT PRIMARY KEY, data VARCHAR(100)); + INSERT INTO testdb VALUES (1, 'Initial data from node mysql1');" + +- name: Check replication status on replica nodes + community.docker.docker_container_exec: + container: "ps_pmm_{{ ps_version }}_{{ item }}" + command: mysql -uroot -p{{ root_password }} -e "SHOW REPLICA STATUS\G" + register: replication_status + loop: "{{ range(2, nodes_count | int + 1) | list }}" + changed_when: false + +- name: Display replication status for each replica + debug: + msg: "{{ replication_status.results[item - 2].stdout_lines }}" + loop: "{{ range(2, nodes_count | int + 1) | list }}" + +- name: Set verification instructions + set_fact: + verification_msg: | + MySQL Cluster setup complete with asynchronous replication! + + To verify replication is working: + 1. Connect to the primary (ps_pmm_{{ ps_version }}_1): + docker exec -it ps_pmm_{{ ps_version }}_1 mysql -uroot -p{{ root_password }} + + 2. Insert data in the test database: + USE testdb; + INSERT INTO testdb VALUES (100, 'Test replication'); + + 3. Connect to replicas and verify data is replicated: + docker exec -it ps_pmm_{{ ps_version }}_2 mysql -uroot -p{{ root_password }} + USE testdb; + SELECT * FROM testdb; + +- name: Display verification instructions + debug: + msg: "{{ verification_msg | split('\n') }}" diff --git a/pmm_qa/percona_server/tasks/percona-server-group-replication-setup.yml b/pmm_qa/percona_server/tasks/percona-server-group-replication-setup.yml new file mode 100644 index 00000000..0f6d72f9 --- /dev/null +++ b/pmm_qa/percona_server/tasks/percona-server-group-replication-setup.yml @@ -0,0 +1,127 @@ +- name: Generate my.cnf for each node + template: + src: my.cnf.j2 + dest: "{{ data_dir }}/node{{ item }}/my.cnf" + loop: "{{ range(1, nodes_count | int + 1) | list }}" + +- name: Create initialization script for each node + template: + src: init.sql.j2 + dest: "{{ data_dir }}/node{{ item }}/init.sql" + loop: "{{ range(1, nodes_count | int + 1) | list }}" + +- name: Start Percona Server containers with group replication + community.docker.docker_container: + name: "ps_pmm_{{ ps_version }}_{{ item }}" + image: "percona/percona-server:{{ ps_version }}" + restart_policy: always + state: started + networks: + - name: "{{ network_name }}" + env: + MYSQL_ROOT_PASSWORD: "{{ root_password }}" + ports: + - "{{ mysql_port + item - 1 }}:{{ mysql_listen_port }}" + - "{{ group_seeds_port + item - 1 }}:{{ group_seeds_port }}" + volumes: + - "{{ data_dir }}/node{{ item }}/data:/var/lib/mysql" + - "{{ data_dir }}/node{{ item }}/my.cnf:/etc/mysql/my.cnf" + - "{{ data_dir }}/node{{ item }}/init.sql:/docker-entrypoint-initdb.d/init.sql" + loop: "{{ range(1, nodes_count | int + 1) | list }}" + +- name: Wait for MySQL to be available + wait_for: + host: localhost + port: "{{ mysql_port + item - 1 }}" + delay: 10 + timeout: 300 + loop: "{{ range(1, nodes_count | int + 1) | list }}" + +- name: Reset configuration for all nodes + community.docker.docker_container_exec: + container: "ps_pmm_{{ ps_version }}_{{ item }}" + command: > + mysql -uroot -p{{ root_password }} -e " + RESET BINARY LOGS AND GTIDS; + RESET REPLICA ALL; + SET GLOBAL gtid_purged=''; + " + loop: "{{ range(1, nodes_count | int + 1) | list }}" + +- name: Bootstrap first node in the cluster + community.docker.docker_container_exec: + container: "ps_pmm_{{ ps_version }}_1" + command: > + mysql -uroot -p{{ root_password }} -e " + SET GLOBAL group_replication_bootstrap_group=ON; + START GROUP_REPLICATION; + SET GLOBAL group_replication_bootstrap_group=OFF;" + retries: 5 + delay: 10 + +- name: Wait 5 seconds for bootstrap to complete + pause: + seconds: 5 + +- name: Start group replication on other nodes + community.docker.docker_container_exec: + container: "ps_pmm_{{ ps_version }}_{{ item }}" + command: mysql -uroot -p{{ root_password }} -e "START GROUP_REPLICATION;" + loop: "{{ range(2, nodes_count | int + 1) | list }}" + ignore_errors: yes + +- name: Wait 10 seconds for the other nodes to join + pause: + seconds: 10 + +- name: Create and seed a test database on primary + community.docker.docker_container_exec: + container: "ps_pmm_{{ ps_version }}_1" + command: > + mysql -uroot -p{{ root_password}} -e " + CREATE DATABASE testdb; + USE testdb; + CREATE TABLE testdb (id INT PRIMARY KEY, data VARCHAR(100)); + INSERT INTO testdb VALUES (1, 'Initial data from node mysql1');" + +- name: Check replication status on first node + community.docker.docker_container_exec: + container: "ps_pmm_{{ ps_version }}_1" + command: mysql -uroot -p{{ root_password }} -e "SELECT * FROM performance_schema.replication_group_members;" + register: replication_status + +- name: Display replication status + debug: + var: replication_status.stdout + +- name: Check replication group members count + community.docker.docker_container_exec: + container: "ps_pmm_{{ ps_version }}_1" + command: mysql -uroot -p{{ root_password }} -e "SELECT COUNT(*) AS count FROM performance_schema.replication_group_members;" + register: member_count + +- name: Display member count + debug: + var: member_count.stdout + +- name: Set verification instructions + set_fact: + verification_msg: | + MySQL Cluster setup complete! + + To verify replication is working: + 1. Connect to the first node: + docker exec -it ps_pmm_{{ ps_version }}_1 mysql -uroot -p{{ root_password }} + + 2. Insert data in the test database: + USE testdb; + INSERT INTO testdb VALUES (100, 'Test replication'); + + 3. Connect to other nodes and verify data is replicated: + docker exec -it ps_pmm_{{ ps_version }}_2 mysql -uroot -p{{ root_password }} + USE testdb; + SELECT * FROM testdb; + +- name: Display verification instructions + debug: + msg: "{{ verification_msg | split('\n') }}" diff --git a/pmm_qa/percona_server/tasks/percona-server-setup.yml b/pmm_qa/percona_server/tasks/percona-server-setup.yml new file mode 100644 index 00000000..366ed37f --- /dev/null +++ b/pmm_qa/percona_server/tasks/percona-server-setup.yml @@ -0,0 +1,24 @@ +- name: Start Percona Server containers + community.docker.docker_container: + name: "ps_pmm_{{ ps_version }}_{{ item }}" + image: "percona/percona-server:{{ ps_version }}" + restart_policy: always + state: started + networks: + - name: "{{ network_name }}" + env: + MYSQL_ROOT_PASSWORD: "{{ root_password }}" + ports: + - "{{ mysql_port + item - 1 }}:{{ mysql_listen_port }}" + - "{{ group_seeds_port + item - 1 }}:{{ group_seeds_port }}" + volumes: + - "{{ data_dir }}/node{{ item }}/data:/var/lib/mysql" + loop: "{{ range(1, nodes_count | int + 1) | list }}" + +- name: Wait for MySQL to be available + wait_for: + host: localhost + port: "{{ mysql_port + item - 1 }}" + delay: 10 + timeout: 300 + loop: "{{ range(1, nodes_count | int + 1) | list }}" diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index cfce689a..1ea30cda 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -90,6 +90,12 @@ def run_ansible_playbook(playbook_filename, env_vars, args): script_path = os.path.abspath(sys.argv[0]) script_dir = os.path.dirname(script_path) playbook_path = script_dir + "/" + playbook_filename + verbosity_level = 1 + + + + if args.verbosity_level is not None: + verbosity_level = int(args.verbosity_level) if args.verbose: print(f'Options set after considering Defaults: {env_vars}') @@ -101,6 +107,7 @@ def run_ansible_playbook(playbook_filename, env_vars, args): cmdline='-l localhost, --connection=local', envvars=env_vars, suppress_env_files=True, + verbosity=verbosity_level, ) print(f'{playbook_filename} playbook execution {r.status}') @@ -886,8 +893,13 @@ def setup_database(db_type, db_version=None, db_config=None, args=None): parser.add_argument("--pmm-server-password", nargs='?', help='PMM Server password') parser.add_argument("--client-version", nargs='?', help='PMM Client version/tarball') parser.add_argument("--verbose", "--v", action='store_true', help='Display verbose information') + parser.add_argument("--verbosity-level", nargs='?', help='Display verbose information level') args = parser.parse_args() + if args.verbosity_level is not None and not args.verbosity_level.isnumeric(): + print(f"Option {args.verbosity_level} is valid verbosity level option, please provide number 1-5") + exit(1) + # Parse arguments try: for db in args.database: From e565496937a838945497176cb2f0c565833d68e2 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 28 Apr 2025 15:12:13 +0200 Subject: [PATCH 002/340] PMM-12153: Split group replication --- pmm_qa/percona_server/percona-server-setup.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pmm_qa/percona_server/percona-server-setup.yml b/pmm_qa/percona_server/percona-server-setup.yml index 7d7cb4fa..88834d13 100644 --- a/pmm_qa/percona_server/percona-server-setup.yml +++ b/pmm_qa/percona_server/percona-server-setup.yml @@ -89,16 +89,16 @@ loop: "{{ range(1, nodes_count | int + 1) | list }}" when: query_source == "slowlog" + - name: Get already connected services to pmm server + shell: curl --location --insecure -u"admin:admin" -s --request GET "http://{{ pmm_server_ip }}/v1/management/services" | jq -r '.services[].service_name' + register: pmm_server_services + - name: Install and add pmm client. include_tasks: ../tasks/install_pmm_client_centos.yml vars: container_name: "ps_pmm_{{ ps_version }}_{{ item }}" loop: "{{ range(1, nodes_count | int + 1) | list }}" - - name: Get already connected services to pmm server - shell: curl --location --insecure -u"admin:admin" -s --request GET "http://172.105.74.90/v1/management/services" | jq -r '.services[].service_name' - register: pmm_server_services - - name: Display already connected services to pmm server debug: msg: "{{ pmm_server_services.stdout | split('\n') }}" From 5e016437fa3fd0204598fc24a55201ed8b4a4dbc Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 28 Apr 2025 15:24:46 +0200 Subject: [PATCH 003/340] PMM-12153: Split group replication --- pmm_qa/percona_server/percona-server-setup.yml | 10 ++++++---- pmm_qa/pmm3-client-setup-centos.sh | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/pmm_qa/percona_server/percona-server-setup.yml b/pmm_qa/percona_server/percona-server-setup.yml index 88834d13..5ed26ca4 100644 --- a/pmm_qa/percona_server/percona-server-setup.yml +++ b/pmm_qa/percona_server/percona-server-setup.yml @@ -89,10 +89,6 @@ loop: "{{ range(1, nodes_count | int + 1) | list }}" when: query_source == "slowlog" - - name: Get already connected services to pmm server - shell: curl --location --insecure -u"admin:admin" -s --request GET "http://{{ pmm_server_ip }}/v1/management/services" | jq -r '.services[].service_name' - register: pmm_server_services - - name: Install and add pmm client. include_tasks: ../tasks/install_pmm_client_centos.yml vars: @@ -103,6 +99,12 @@ debug: msg: "{{ pmm_server_services.stdout | split('\n') }}" + - name: Get already connected services to pmm server + community.docker.docker_container_exec: + container: "ps_pmm_{{ ps_version }}_1" + command: curl --location --insecure -u"admin:{{ admin_password }}" -s --request GET "http://{{ pmm_server_ip }}/v1/management/services" | jq -r '.services[].service_name' + register: pmm_server_services + - name: Find out if service is already connected to pmm server block: - name: Loop through percona servers diff --git a/pmm_qa/pmm3-client-setup-centos.sh b/pmm_qa/pmm3-client-setup-centos.sh index 8a9440a4..8b2ed8b2 100644 --- a/pmm_qa/pmm3-client-setup-centos.sh +++ b/pmm_qa/pmm3-client-setup-centos.sh @@ -45,7 +45,7 @@ if [[ "$pmm_server_ip" =~ ^([0-9]{1,3}\.){3}[0-9]{1,3}$ ]]; then port=443 fi -microdnf install -y wget gnupg2 +microdnf install -y wget gnupg2 jq wget https://repo.percona.com/yum/percona-release-latest.noarch.rpm rpm -i ./percona-release-latest.noarch.rpm export PMM_AGENT_SETUP_NODE_NAME=client_container_$(echo $((1 + $RANDOM % 9999))) From 35dd0f6b29d651c0247185e2d917bad0c582efa4 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 28 Apr 2025 15:28:28 +0200 Subject: [PATCH 004/340] PMM-12153: Split group replication --- pmm_qa/percona_server/percona-server-setup.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pmm_qa/percona_server/percona-server-setup.yml b/pmm_qa/percona_server/percona-server-setup.yml index 5ed26ca4..73ba5379 100644 --- a/pmm_qa/percona_server/percona-server-setup.yml +++ b/pmm_qa/percona_server/percona-server-setup.yml @@ -95,16 +95,16 @@ container_name: "ps_pmm_{{ ps_version }}_{{ item }}" loop: "{{ range(1, nodes_count | int + 1) | list }}" - - name: Display already connected services to pmm server - debug: - msg: "{{ pmm_server_services.stdout | split('\n') }}" - - name: Get already connected services to pmm server community.docker.docker_container_exec: container: "ps_pmm_{{ ps_version }}_1" command: curl --location --insecure -u"admin:{{ admin_password }}" -s --request GET "http://{{ pmm_server_ip }}/v1/management/services" | jq -r '.services[].service_name' register: pmm_server_services + - name: Display already connected services to pmm server + debug: + msg: "{{ pmm_server_services.stdout | split('\n') }}" + - name: Find out if service is already connected to pmm server block: - name: Loop through percona servers From 2ae7fdaff0db8a5d0f6db007750a5624354f7b85 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 28 Apr 2025 15:37:19 +0200 Subject: [PATCH 005/340] PMM-7: Install dependencies --- pmm_qa/tasks/install_pmm_client_centos.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/pmm_qa/tasks/install_pmm_client_centos.yml b/pmm_qa/tasks/install_pmm_client_centos.yml index ea8f9f38..dcc604df 100644 --- a/pmm_qa/tasks/install_pmm_client_centos.yml +++ b/pmm_qa/tasks/install_pmm_client_centos.yml @@ -27,3 +27,15 @@ container: "{{ container_name }}" user: "root" command: bash -x /pmm3-client-setup.sh --pmm_server_ip {{ pmm_server_ip }} --client_version {{ client_version }} --admin_password {{ admin_password }} --use_metrics_mode no + +- name: Install dependencies + package: + name: + - wget + - gnupg2 + - jq + state: present +- name: Download a file from URL + get_url: + url: "https://example.com/myfile.tar.gz" + dest: "/tmp/myfile.tar.gz" \ No newline at end of file From 6e87a27971f953494135c848a8020c87d19ba73f Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 28 Apr 2025 15:37:45 +0200 Subject: [PATCH 006/340] PMM-12153: Correct pmm server port --- pmm_qa/percona_server/percona-server-setup.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/percona_server/percona-server-setup.yml b/pmm_qa/percona_server/percona-server-setup.yml index 73ba5379..55922da6 100644 --- a/pmm_qa/percona_server/percona-server-setup.yml +++ b/pmm_qa/percona_server/percona-server-setup.yml @@ -98,7 +98,7 @@ - name: Get already connected services to pmm server community.docker.docker_container_exec: container: "ps_pmm_{{ ps_version }}_1" - command: curl --location --insecure -u"admin:{{ admin_password }}" -s --request GET "http://{{ pmm_server_ip }}/v1/management/services" | jq -r '.services[].service_name' + command: curl --location --insecure -u"admin:{{ admin_password }}" -s --request GET "http://{{ pmm_server_ip }}:8080/v1/management/services" | jq -r '.services[].service_name' register: pmm_server_services - name: Display already connected services to pmm server From 430ff75d952927ec1a27034e629779963cffb0a3 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 28 Apr 2025 15:45:03 +0200 Subject: [PATCH 007/340] PMM-7: Install dependencies --- pmm_qa/tasks/install_pmm_client_centos.yml | 38 ++++++++++++++++++---- 1 file changed, 31 insertions(+), 7 deletions(-) diff --git a/pmm_qa/tasks/install_pmm_client_centos.yml b/pmm_qa/tasks/install_pmm_client_centos.yml index dcc604df..2460e8f4 100644 --- a/pmm_qa/tasks/install_pmm_client_centos.yml +++ b/pmm_qa/tasks/install_pmm_client_centos.yml @@ -28,13 +28,37 @@ user: "root" command: bash -x /pmm3-client-setup.sh --pmm_server_ip {{ pmm_server_ip }} --client_version {{ client_version }} --admin_password {{ admin_password }} --use_metrics_mode no -- name: Install dependencies - package: - name: - - wget - - gnupg2 - - jq - state: present +- name: Detect OS inside the container + community.docker.docker_container_exec: + container: "{{ container_name }}" + command: cat /etc/os-release + register: container_os_info + +- name: Set distro family (debian/rhel) + set_fact: + distro_family: >- + {% if 'debian' in container_os_info.stdout | lower %} + debian + {% elif 'rhel' in container_os_info.stdout | lower or 'centos' in container_os_info.stdout | lower or 'fedora' in container_os_info.stdout | lower %} + rhel + {% else %} + unknown + {% endif %} + +- name: Install wget inside docker container + community.docker.docker_container_exec: + container: "{{ container_name }}" + command: > + apt-get install -y wget gnupg2 jq + when: distro_family == "debian" + +- name: Install wget on RHEL-family containers + community.docker.docker_container_exec: + container: "{{ container_name }}" + command: | + microdnf install -y wget gnupg2 jq + when: distro_family == "rhel" + - name: Download a file from URL get_url: url: "https://example.com/myfile.tar.gz" From 1e890db4d0374bafa9e7c6dc7572db50238d5634 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 28 Apr 2025 15:47:57 +0200 Subject: [PATCH 008/340] PMM-12153: Refactor --- pmm_qa/percona_server/percona-server-setup.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/percona_server/percona-server-setup.yml b/pmm_qa/percona_server/percona-server-setup.yml index 55922da6..50c0becd 100644 --- a/pmm_qa/percona_server/percona-server-setup.yml +++ b/pmm_qa/percona_server/percona-server-setup.yml @@ -98,7 +98,7 @@ - name: Get already connected services to pmm server community.docker.docker_container_exec: container: "ps_pmm_{{ ps_version }}_1" - command: curl --location --insecure -u"admin:{{ admin_password }}" -s --request GET "http://{{ pmm_server_ip }}:8080/v1/management/services" | jq -r '.services[].service_name' + command: curl --location --insecure -u"admin:{{ admin_password }}" -s --request GET "http://{{ pmm_server_ip }}:8080/v1/management/services" | jq -r ".services[].service_name" register: pmm_server_services - name: Display already connected services to pmm server From 2d7074819f5c9a1fed0507fb646d790baf077e67 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 28 Apr 2025 15:51:09 +0200 Subject: [PATCH 009/340] PMM-7: download percona release and print client version --- pmm_qa/tasks/install_pmm_client_centos.yml | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/pmm_qa/tasks/install_pmm_client_centos.yml b/pmm_qa/tasks/install_pmm_client_centos.yml index 2460e8f4..924839ed 100644 --- a/pmm_qa/tasks/install_pmm_client_centos.yml +++ b/pmm_qa/tasks/install_pmm_client_centos.yml @@ -45,7 +45,7 @@ unknown {% endif %} -- name: Install wget inside docker container +- name: Install wget inside Debian-family container community.docker.docker_container_exec: container: "{{ container_name }}" command: > @@ -59,7 +59,18 @@ microdnf install -y wget gnupg2 jq when: distro_family == "rhel" -- name: Download a file from URL +- name: Download percona release on RHEL based OS get_url: - url: "https://example.com/myfile.tar.gz" - dest: "/tmp/myfile.tar.gz" \ No newline at end of file + url: "https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb" + dest: "/percona-release_latest.$(lsb_release -sc)_all.deb" + when: distro_family == "debian" + +- name: Download percona release on RHEL based OS + get_url: + url: "https://repo.percona.com/yum/percona-release-latest.noarch.rpm" + dest: "/percona-release-latest.noarch.rpm" + when: distro_family == "rhel" + +- name: Print expected client version + debug: + msg: "{{ client_version }}" From 46b557121404005c55917ef2d66323ba28413c04 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 28 Apr 2025 15:54:02 +0200 Subject: [PATCH 010/340] PMM-7: download percona release and print client version --- pmm_qa/percona_server/percona-server-setup.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pmm_qa/percona_server/percona-server-setup.yml b/pmm_qa/percona_server/percona-server-setup.yml index 50c0becd..28669f0e 100644 --- a/pmm_qa/percona_server/percona-server-setup.yml +++ b/pmm_qa/percona_server/percona-server-setup.yml @@ -98,7 +98,8 @@ - name: Get already connected services to pmm server community.docker.docker_container_exec: container: "ps_pmm_{{ ps_version }}_1" - command: curl --location --insecure -u"admin:{{ admin_password }}" -s --request GET "http://{{ pmm_server_ip }}:8080/v1/management/services" | jq -r ".services[].service_name" + command: > + curl --location --insecure -u"admin:{{ admin_password }}" -s --request GET "http://{{ pmm_server_ip }}:8080/v1/management/services" | jq -r ".services[].service_name" register: pmm_server_services - name: Display already connected services to pmm server From 5b4dbeb37c5bc7c44e6fa4f10d776a0ba14d1c9e Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 28 Apr 2025 15:58:58 +0200 Subject: [PATCH 011/340] PMM-7: download percona release and print client version --- pmm_qa/percona_server/percona-server-setup.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/percona_server/percona-server-setup.yml b/pmm_qa/percona_server/percona-server-setup.yml index 28669f0e..fb0945ae 100644 --- a/pmm_qa/percona_server/percona-server-setup.yml +++ b/pmm_qa/percona_server/percona-server-setup.yml @@ -99,7 +99,7 @@ community.docker.docker_container_exec: container: "ps_pmm_{{ ps_version }}_1" command: > - curl --location --insecure -u"admin:{{ admin_password }}" -s --request GET "http://{{ pmm_server_ip }}:8080/v1/management/services" | jq -r ".services[].service_name" + sh -c 'curl --location --insecure -u"admin:{{ admin_password }}" -s --request GET "http://{{ pmm_server_ip }}:8080/v1/management/services" | jq -r ".services[].service_name"' register: pmm_server_services - name: Display already connected services to pmm server From 242eb1d76d9a6c093a680010a00239a5dae3ef79 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 28 Apr 2025 16:07:03 +0200 Subject: [PATCH 012/340] PMM-7: reorder steps --- pmm_qa/tasks/install_pmm_client_centos.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pmm_qa/tasks/install_pmm_client_centos.yml b/pmm_qa/tasks/install_pmm_client_centos.yml index 924839ed..407564c4 100644 --- a/pmm_qa/tasks/install_pmm_client_centos.yml +++ b/pmm_qa/tasks/install_pmm_client_centos.yml @@ -22,12 +22,6 @@ debug: msg: "{{ pmm_server_ip }} is pmm server IP" -- name: Install pmm-client - community.docker.docker_container_exec: - container: "{{ container_name }}" - user: "root" - command: bash -x /pmm3-client-setup.sh --pmm_server_ip {{ pmm_server_ip }} --client_version {{ client_version }} --admin_password {{ admin_password }} --use_metrics_mode no - - name: Detect OS inside the container community.docker.docker_container_exec: container: "{{ container_name }}" @@ -74,3 +68,9 @@ - name: Print expected client version debug: msg: "{{ client_version }}" + +- name: Install pmm-client + community.docker.docker_container_exec: + container: "{{ container_name }}" + user: "root" + command: bash -x /pmm3-client-setup.sh --pmm_server_ip {{ pmm_server_ip }} --client_version {{ client_version }} --admin_password {{ admin_password }} --use_metrics_mode no \ No newline at end of file From 316161540478d9cedae079fbce5cf99389e0ccfd Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 28 Apr 2025 16:12:48 +0200 Subject: [PATCH 013/340] PMM-7: reorder steps --- pmm_qa/tasks/install_pmm_client_centos.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pmm_qa/tasks/install_pmm_client_centos.yml b/pmm_qa/tasks/install_pmm_client_centos.yml index 407564c4..35c5779b 100644 --- a/pmm_qa/tasks/install_pmm_client_centos.yml +++ b/pmm_qa/tasks/install_pmm_client_centos.yml @@ -31,13 +31,13 @@ - name: Set distro family (debian/rhel) set_fact: distro_family: >- - {% if 'debian' in container_os_info.stdout | lower %} - debian - {% elif 'rhel' in container_os_info.stdout | lower or 'centos' in container_os_info.stdout | lower or 'fedora' in container_os_info.stdout | lower %} - rhel - {% else %} - unknown - {% endif %} + {{ + ( + 'debian' if 'debian' in container_os_info.stdout | lower else + 'rhel' if 'rhel' in container_os_info.stdout | lower or 'centos' in container_os_info.stdout | lower or 'fedora' in container_os_info.stdout | lower + else 'unknown' + ) + }} | trim - name: Install wget inside Debian-family container community.docker.docker_container_exec: From 6b46cd29732e09c967d38004d9d633fc5cc2f94e Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 28 Apr 2025 16:15:06 +0200 Subject: [PATCH 014/340] PMM-7: reorder steps --- pmm_qa/tasks/install_pmm_client_centos.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pmm_qa/tasks/install_pmm_client_centos.yml b/pmm_qa/tasks/install_pmm_client_centos.yml index 35c5779b..c6b33a2d 100644 --- a/pmm_qa/tasks/install_pmm_client_centos.yml +++ b/pmm_qa/tasks/install_pmm_client_centos.yml @@ -36,8 +36,8 @@ 'debian' if 'debian' in container_os_info.stdout | lower else 'rhel' if 'rhel' in container_os_info.stdout | lower or 'centos' in container_os_info.stdout | lower or 'fedora' in container_os_info.stdout | lower else 'unknown' - ) - }} | trim + ) | trim + }} - name: Install wget inside Debian-family container community.docker.docker_container_exec: From f83a07f76665baa42b7f571c0a84ccfcba32ae15 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 28 Apr 2025 16:16:35 +0200 Subject: [PATCH 015/340] PMM-7: reorder steps --- pmm_qa/tasks/install_pmm_client_centos.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/pmm_qa/tasks/install_pmm_client_centos.yml b/pmm_qa/tasks/install_pmm_client_centos.yml index c6b33a2d..c786bbe0 100644 --- a/pmm_qa/tasks/install_pmm_client_centos.yml +++ b/pmm_qa/tasks/install_pmm_client_centos.yml @@ -42,15 +42,13 @@ - name: Install wget inside Debian-family container community.docker.docker_container_exec: container: "{{ container_name }}" - command: > - apt-get install -y wget gnupg2 jq + command: apt-get install -y wget gnupg2 jq when: distro_family == "debian" - name: Install wget on RHEL-family containers community.docker.docker_container_exec: container: "{{ container_name }}" - command: | - microdnf install -y wget gnupg2 jq + command: microdnf install -y wget gnupg2 jq when: distro_family == "rhel" - name: Download percona release on RHEL based OS From 31fd312f21244093d9f800b9400b9f6abdadcef6 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 28 Apr 2025 16:19:02 +0200 Subject: [PATCH 016/340] PMM-7: reorder steps --- pmm_qa/tasks/install_pmm_client_centos.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/pmm_qa/tasks/install_pmm_client_centos.yml b/pmm_qa/tasks/install_pmm_client_centos.yml index c786bbe0..72c1703d 100644 --- a/pmm_qa/tasks/install_pmm_client_centos.yml +++ b/pmm_qa/tasks/install_pmm_client_centos.yml @@ -43,12 +43,14 @@ community.docker.docker_container_exec: container: "{{ container_name }}" command: apt-get install -y wget gnupg2 jq + user: "root" when: distro_family == "debian" - name: Install wget on RHEL-family containers community.docker.docker_container_exec: container: "{{ container_name }}" command: microdnf install -y wget gnupg2 jq + user: "root" when: distro_family == "rhel" - name: Download percona release on RHEL based OS @@ -67,6 +69,23 @@ debug: msg: "{{ client_version }}" +- name: Install wget on RHEL-family containers + community.docker.docker_container_exec: + container: "{{ container_name }}" + command: > + percona-release enable-only pmm3-client experimental + apt-get update + apt-get -y install pmm-client + when: distro_family == "debian" and client_version == "3-dev-latest" + +- name: Install wget on RHEL-family containers + community.docker.docker_container_exec: + container: "{{ container_name }}" + command: > + percona-release enable-only pmm3-client experimental + microdnf install -y pmm-client + when: distro_family == "rhel" and client_version == "3-dev-latest" + - name: Install pmm-client community.docker.docker_container_exec: container: "{{ container_name }}" From 6b2f4b9f5557b6d0e1af5b22557df3421974e1a7 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 28 Apr 2025 16:23:45 +0200 Subject: [PATCH 017/340] PMM-7: reorder steps --- pmm_qa/tasks/install_pmm_client_centos.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pmm_qa/tasks/install_pmm_client_centos.yml b/pmm_qa/tasks/install_pmm_client_centos.yml index 72c1703d..b4ceac50 100644 --- a/pmm_qa/tasks/install_pmm_client_centos.yml +++ b/pmm_qa/tasks/install_pmm_client_centos.yml @@ -72,6 +72,7 @@ - name: Install wget on RHEL-family containers community.docker.docker_container_exec: container: "{{ container_name }}" + user: "root" command: > percona-release enable-only pmm3-client experimental apt-get update @@ -81,6 +82,7 @@ - name: Install wget on RHEL-family containers community.docker.docker_container_exec: container: "{{ container_name }}" + user: "root" command: > percona-release enable-only pmm3-client experimental microdnf install -y pmm-client From 19bb43ac4ecdf0ca941728148679f8d5102a1821 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 28 Apr 2025 16:25:32 +0200 Subject: [PATCH 018/340] PMM-7: reorder steps --- pmm_qa/tasks/install_pmm_client_centos.yml | 40 ++++++++++++++++++++-- 1 file changed, 38 insertions(+), 2 deletions(-) diff --git a/pmm_qa/tasks/install_pmm_client_centos.yml b/pmm_qa/tasks/install_pmm_client_centos.yml index b4ceac50..8c86952d 100644 --- a/pmm_qa/tasks/install_pmm_client_centos.yml +++ b/pmm_qa/tasks/install_pmm_client_centos.yml @@ -69,7 +69,7 @@ debug: msg: "{{ client_version }}" -- name: Install wget on RHEL-family containers +- name: Install pmm client experimental on RHEL-family containers community.docker.docker_container_exec: container: "{{ container_name }}" user: "root" @@ -79,7 +79,7 @@ apt-get -y install pmm-client when: distro_family == "debian" and client_version == "3-dev-latest" -- name: Install wget on RHEL-family containers +- name: Install pmm client experimental on RHEL-family containers community.docker.docker_container_exec: container: "{{ container_name }}" user: "root" @@ -88,6 +88,42 @@ microdnf install -y pmm-client when: distro_family == "rhel" and client_version == "3-dev-latest" +- name: Install pmm client testing on RHEL-family containers + community.docker.docker_container_exec: + container: "{{ container_name }}" + user: "root" + command: > + percona-release enable-only pmm3-client testing + apt-get update + apt-get -y install pmm-client + when: distro_family == "debian" and client_version == "pmm3-rc" + +- name: Install pmm client testing on RHEL-family containers + community.docker.docker_container_exec: + container: "{{ container_name }}" + user: "root" + command: > + percona-release enable-only pmm3-client testing + microdnf install -y pmm-client + when: distro_family == "rhel" and client_version == "pmm3-rc" + +- name: Install pmm client testing on RHEL-family containers + community.docker.docker_container_exec: + container: "{{ container_name }}" + user: "root" + command: > + apt-get update + apt-get -y install pmm-client + when: distro_family == "debian" and client_version == "pmm3-latest" + +- name: Install pmm client testing on RHEL-family containers + community.docker.docker_container_exec: + container: "{{ container_name }}" + user: "root" + command: > + microdnf install -y pmm-client + when: distro_family == "rhel" and client_version == "pmm3-latest" + - name: Install pmm-client community.docker.docker_container_exec: container: "{{ container_name }}" From d08d77d2bd995219bf5ed9cf8b2d98555e0cddc2 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 28 Apr 2025 16:27:17 +0200 Subject: [PATCH 019/340] PMM-7: reorder steps --- pmm_qa/tasks/install_pmm_client_centos.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/pmm_qa/tasks/install_pmm_client_centos.yml b/pmm_qa/tasks/install_pmm_client_centos.yml index 8c86952d..1d18a524 100644 --- a/pmm_qa/tasks/install_pmm_client_centos.yml +++ b/pmm_qa/tasks/install_pmm_client_centos.yml @@ -123,9 +123,3 @@ command: > microdnf install -y pmm-client when: distro_family == "rhel" and client_version == "pmm3-latest" - -- name: Install pmm-client - community.docker.docker_container_exec: - container: "{{ container_name }}" - user: "root" - command: bash -x /pmm3-client-setup.sh --pmm_server_ip {{ pmm_server_ip }} --client_version {{ client_version }} --admin_password {{ admin_password }} --use_metrics_mode no \ No newline at end of file From 8c4c958133a1ae0d7d54b6824ce9530642874580 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 28 Apr 2025 16:30:00 +0200 Subject: [PATCH 020/340] PMM-7: reorder steps --- pmm_qa/tasks/install_pmm_client_centos.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/pmm_qa/tasks/install_pmm_client_centos.yml b/pmm_qa/tasks/install_pmm_client_centos.yml index 1d18a524..1a8330f8 100644 --- a/pmm_qa/tasks/install_pmm_client_centos.yml +++ b/pmm_qa/tasks/install_pmm_client_centos.yml @@ -123,3 +123,20 @@ command: > microdnf install -y pmm-client when: distro_family == "rhel" and client_version == "pmm3-latest" + +- name: Install pmm client released version on RHEL-family containers + community.docker.docker_container_exec: + container: "{{ container_name }}" + user: "root" + command: > + wget -O pmm-client.deb https://repo.percona.com/pmm3-client/apt/pool/main/p/pmm-client/pmm-client_${client_version}-7.$(lsb_release -sc)_amd64.deb + dpkg -i pmm-client.deb + when: "'{{ client_version }}' | regex_match('^3\\.[0-9]+\\.[0-9]+$') and distro_family == 'debian'" + +- name: Install pmm client released version on RHEL-family containers + community.docker.docker_container_exec: + container: "{{ container_name }}" + user: "root" + command: > + microdnf install -y pmm-client + when: "'{{ client_version }}' | regex_match('^3\\.[0-9]+\\.[0-9]+$') and distro_family == 'rhel'" From 5be9fb397a77058ffd094b21b4a0b838715c04d0 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 28 Apr 2025 16:35:10 +0200 Subject: [PATCH 021/340] PMM-7: reorder steps --- pmm_qa/percona_server/percona-server-setup.yml | 1 + pmm_qa/tasks/install_pmm_client_centos.yml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/pmm_qa/percona_server/percona-server-setup.yml b/pmm_qa/percona_server/percona-server-setup.yml index fb0945ae..261b6d11 100644 --- a/pmm_qa/percona_server/percona-server-setup.yml +++ b/pmm_qa/percona_server/percona-server-setup.yml @@ -21,6 +21,7 @@ client_version: "{{ lookup('vars', 'extra_client_version', default=lookup('env','CLIENT_VERSION') | default('3-dev-latest', true) ) }}" admin_password: "{{ lookup('vars', 'extra_admin_password', default=lookup('env','ADMIN_PASSWORD') | default('admin', true) ) }}" query_source: "{{ lookup('env', 'QUERY_SOURCE') | default('perfschema', true) }}" + metrics_mode: "{{ lookup('env', 'metrics_mode') | default('auto', true) }}" setup_type: "{{ lookup('env', 'SETUP_TYPE') }}" random_service_name_value: "" diff --git a/pmm_qa/tasks/install_pmm_client_centos.yml b/pmm_qa/tasks/install_pmm_client_centos.yml index 1a8330f8..ab92bc29 100644 --- a/pmm_qa/tasks/install_pmm_client_centos.yml +++ b/pmm_qa/tasks/install_pmm_client_centos.yml @@ -67,7 +67,7 @@ - name: Print expected client version debug: - msg: "{{ client_version }}" + msg: "{{ metrics_mode }}" - name: Install pmm client experimental on RHEL-family containers community.docker.docker_container_exec: From 4ea09766dc7d43ebe064a5d9fd0f488ab59113a7 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 28 Apr 2025 16:38:05 +0200 Subject: [PATCH 022/340] PMM-7: reorder steps --- .../percona_server/percona-server-setup.yml | 2 +- pmm_qa/tasks/install_pmm_client_centos.yml | 37 +++++++++++-------- 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/pmm_qa/percona_server/percona-server-setup.yml b/pmm_qa/percona_server/percona-server-setup.yml index 261b6d11..e098a52e 100644 --- a/pmm_qa/percona_server/percona-server-setup.yml +++ b/pmm_qa/percona_server/percona-server-setup.yml @@ -21,7 +21,7 @@ client_version: "{{ lookup('vars', 'extra_client_version', default=lookup('env','CLIENT_VERSION') | default('3-dev-latest', true) ) }}" admin_password: "{{ lookup('vars', 'extra_admin_password', default=lookup('env','ADMIN_PASSWORD') | default('admin', true) ) }}" query_source: "{{ lookup('env', 'QUERY_SOURCE') | default('perfschema', true) }}" - metrics_mode: "{{ lookup('env', 'metrics_mode') | default('auto', true) }}" + metrics_mode: "{{ lookup('env', 'metrics_mode') }}" setup_type: "{{ lookup('env', 'SETUP_TYPE') }}" random_service_name_value: "" diff --git a/pmm_qa/tasks/install_pmm_client_centos.yml b/pmm_qa/tasks/install_pmm_client_centos.yml index ab92bc29..e25af00f 100644 --- a/pmm_qa/tasks/install_pmm_client_centos.yml +++ b/pmm_qa/tasks/install_pmm_client_centos.yml @@ -124,19 +124,24 @@ microdnf install -y pmm-client when: distro_family == "rhel" and client_version == "pmm3-latest" -- name: Install pmm client released version on RHEL-family containers - community.docker.docker_container_exec: - container: "{{ container_name }}" - user: "root" - command: > - wget -O pmm-client.deb https://repo.percona.com/pmm3-client/apt/pool/main/p/pmm-client/pmm-client_${client_version}-7.$(lsb_release -sc)_amd64.deb - dpkg -i pmm-client.deb - when: "'{{ client_version }}' | regex_match('^3\\.[0-9]+\\.[0-9]+$') and distro_family == 'debian'" - -- name: Install pmm client released version on RHEL-family containers - community.docker.docker_container_exec: - container: "{{ container_name }}" - user: "root" - command: > - microdnf install -y pmm-client - when: "'{{ client_version }}' | regex_match('^3\\.[0-9]+\\.[0-9]+$') and distro_family == 'rhel'" +#- name: Install pmm client released version on RHEL-family containers +# community.docker.docker_container_exec: +# container: "{{ container_name }}" +# user: "root" +# command: > +# wget -O pmm-client.deb https://repo.percona.com/pmm3-client/apt/pool/main/p/pmm-client/pmm-client_${client_version}-7.$(lsb_release -sc)_amd64.deb +# dpkg -i pmm-client.deb +# when: "'{{ client_version }}' | regex_match('^3\\.[0-9]+\\.[0-9]+$') and distro_family == 'debian'" +# +#- name: Install pmm client released version on RHEL-family containers +# community.docker.docker_container_exec: +# container: "{{ container_name }}" +# user: "root" +# command: > +# microdnf install -y pmm-client +# when: "'{{ client_version }}' | regex_match('^3\\.[0-9]+\\.[0-9]+$') and distro_family == 'rhel'" + +- name: Connect pmm client to pmm server using metrics mode + user: "root" + command: pmm-agent setup --config-file=/usr/local/percona/pmm/config/pmm-agent.yaml --server-address=${pmm_server_ip}:${port} --server-insecure-tls --metrics-mode=${metrics_mode} --server-username=admin --server-password=${admin_password} + when: metrics_mode is defined From c59da8c6612a7752a9460d118e801dfe5d05f91e Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 28 Apr 2025 16:39:25 +0200 Subject: [PATCH 023/340] PMM-7: reorder steps --- pmm_qa/tasks/install_pmm_client_centos.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pmm_qa/tasks/install_pmm_client_centos.yml b/pmm_qa/tasks/install_pmm_client_centos.yml index e25af00f..38e5e71f 100644 --- a/pmm_qa/tasks/install_pmm_client_centos.yml +++ b/pmm_qa/tasks/install_pmm_client_centos.yml @@ -143,5 +143,10 @@ - name: Connect pmm client to pmm server using metrics mode user: "root" - command: pmm-agent setup --config-file=/usr/local/percona/pmm/config/pmm-agent.yaml --server-address=${pmm_server_ip}:${port} --server-insecure-tls --metrics-mode=${metrics_mode} --server-username=admin --server-password=${admin_password} + command: pmm-agent setup --config-file=/usr/local/percona/pmm/config/pmm-agent.yaml --server-address={{ pmm_server_ip }}:${port} --server-insecure-tls --metrics-mode=${metrics_mode} --server-username=admin --server-password={{ admin_password }} when: metrics_mode is defined + +- name: Connect pmm client to pmm server using metrics mode + user: "root" + command: pmm-agent setup --config-file=/usr/local/percona/pmm/config/pmm-agent.yaml --server-address={{ pmm_server_ip }}:${port} --server-insecure-tls --server-username=admin --server-password={{ admin_password }} + when: metrics_mode is not defined \ No newline at end of file From 703c620a5cf9c0fe634de38c14344c28c83cdd02 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 28 Apr 2025 16:39:47 +0200 Subject: [PATCH 024/340] PMM-7: reorder steps --- pmm_qa/tasks/install_pmm_client_centos.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/tasks/install_pmm_client_centos.yml b/pmm_qa/tasks/install_pmm_client_centos.yml index 38e5e71f..77c76fbd 100644 --- a/pmm_qa/tasks/install_pmm_client_centos.yml +++ b/pmm_qa/tasks/install_pmm_client_centos.yml @@ -149,4 +149,4 @@ - name: Connect pmm client to pmm server using metrics mode user: "root" command: pmm-agent setup --config-file=/usr/local/percona/pmm/config/pmm-agent.yaml --server-address={{ pmm_server_ip }}:${port} --server-insecure-tls --server-username=admin --server-password={{ admin_password }} - when: metrics_mode is not defined \ No newline at end of file + when: metrics_mode is undefined \ No newline at end of file From 26a0f0e560bdbc72b9ae19c6fb3f8b711162f764 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 28 Apr 2025 16:42:55 +0200 Subject: [PATCH 025/340] PMM-7: reorder steps --- pmm_qa/tasks/install_pmm_client_centos.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/pmm_qa/tasks/install_pmm_client_centos.yml b/pmm_qa/tasks/install_pmm_client_centos.yml index 77c76fbd..71ce08e9 100644 --- a/pmm_qa/tasks/install_pmm_client_centos.yml +++ b/pmm_qa/tasks/install_pmm_client_centos.yml @@ -142,11 +142,15 @@ # when: "'{{ client_version }}' | regex_match('^3\\.[0-9]+\\.[0-9]+$') and distro_family == 'rhel'" - name: Connect pmm client to pmm server using metrics mode - user: "root" - command: pmm-agent setup --config-file=/usr/local/percona/pmm/config/pmm-agent.yaml --server-address={{ pmm_server_ip }}:${port} --server-insecure-tls --metrics-mode=${metrics_mode} --server-username=admin --server-password={{ admin_password }} + community.docker.docker_container_exec: + container: "{{ container_name }}" + user: "root" + command: pmm-agent setup --config-file=/usr/local/percona/pmm/config/pmm-agent.yaml --server-address={{ pmm_server_ip }}:${port} --server-insecure-tls --metrics-mode=${metrics_mode} --server-username=admin --server-password={{ admin_password }} when: metrics_mode is defined - name: Connect pmm client to pmm server using metrics mode - user: "root" - command: pmm-agent setup --config-file=/usr/local/percona/pmm/config/pmm-agent.yaml --server-address={{ pmm_server_ip }}:${port} --server-insecure-tls --server-username=admin --server-password={{ admin_password }} + community.docker.docker_container_exec: + container: "{{ container_name }}" + user: "root" + command: pmm-agent setup --config-file=/usr/local/percona/pmm/config/pmm-agent.yaml --server-address={{ pmm_server_ip }}:${port} --server-insecure-tls --server-username=admin --server-password={{ admin_password }} when: metrics_mode is undefined \ No newline at end of file From b93b078c345213ecfe17a53f5e405838aaafb15a Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 28 Apr 2025 16:45:37 +0200 Subject: [PATCH 026/340] PMM-7: reorder steps --- pmm_qa/tasks/install_pmm_client_centos.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/pmm_qa/tasks/install_pmm_client_centos.yml b/pmm_qa/tasks/install_pmm_client_centos.yml index 71ce08e9..60b58eac 100644 --- a/pmm_qa/tasks/install_pmm_client_centos.yml +++ b/pmm_qa/tasks/install_pmm_client_centos.yml @@ -22,6 +22,16 @@ debug: msg: "{{ pmm_server_ip }} is pmm server IP" +- name: Set correct pmm server port + set_fact: + pmm_server_port: 80 + when: pmm_server_ip is ansible.utils.ipv4 + +- name: Set correct pmm server port + set_fact: + pmm_server_port: 8080 + when: pmm_server_ip is not ansible.utils.ipv4 + - name: Detect OS inside the container community.docker.docker_container_exec: container: "{{ container_name }}" @@ -145,7 +155,7 @@ community.docker.docker_container_exec: container: "{{ container_name }}" user: "root" - command: pmm-agent setup --config-file=/usr/local/percona/pmm/config/pmm-agent.yaml --server-address={{ pmm_server_ip }}:${port} --server-insecure-tls --metrics-mode=${metrics_mode} --server-username=admin --server-password={{ admin_password }} + command: pmm-agent setup --config-file=/usr/local/percona/pmm/config/pmm-agent.yaml --server-address={{ pmm_server_ip }}:${port} --server-insecure-tls --metrics-mode={{ metrics_mode }} --server-username=admin --server-password={{ admin_password }} when: metrics_mode is defined - name: Connect pmm client to pmm server using metrics mode From 87609c106cc4c1dd0e084711c3b832bbbf432c76 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 28 Apr 2025 16:47:00 +0200 Subject: [PATCH 027/340] PMM-7: reorder steps --- pmm_qa/tasks/install_pmm_client_centos.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pmm_qa/tasks/install_pmm_client_centos.yml b/pmm_qa/tasks/install_pmm_client_centos.yml index 60b58eac..bf897eb6 100644 --- a/pmm_qa/tasks/install_pmm_client_centos.yml +++ b/pmm_qa/tasks/install_pmm_client_centos.yml @@ -155,12 +155,12 @@ community.docker.docker_container_exec: container: "{{ container_name }}" user: "root" - command: pmm-agent setup --config-file=/usr/local/percona/pmm/config/pmm-agent.yaml --server-address={{ pmm_server_ip }}:${port} --server-insecure-tls --metrics-mode={{ metrics_mode }} --server-username=admin --server-password={{ admin_password }} + command: pmm-agent setup --config-file=/usr/local/percona/pmm/config/pmm-agent.yaml --server-address={{ pmm_server_ip }}:{{ pmm_server_port }} --server-insecure-tls --metrics-mode={{ metrics_mode }} --server-username=admin --server-password={{ admin_password }} when: metrics_mode is defined - name: Connect pmm client to pmm server using metrics mode community.docker.docker_container_exec: container: "{{ container_name }}" user: "root" - command: pmm-agent setup --config-file=/usr/local/percona/pmm/config/pmm-agent.yaml --server-address={{ pmm_server_ip }}:${port} --server-insecure-tls --server-username=admin --server-password={{ admin_password }} + command: pmm-agent setup --config-file=/usr/local/percona/pmm/config/pmm-agent.yaml --server-address={{ pmm_server_ip }}:{{ pmm_server_port }} --server-insecure-tls --server-username=admin --server-password={{ admin_password }} when: metrics_mode is undefined \ No newline at end of file From d0dd4cf623f4310c59fc68fd9f1fd32c49b8edb9 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 28 Apr 2025 16:49:53 +0200 Subject: [PATCH 028/340] PMM-7: reorder steps --- pmm_qa/tasks/install_pmm_client_centos.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pmm_qa/tasks/install_pmm_client_centos.yml b/pmm_qa/tasks/install_pmm_client_centos.yml index bf897eb6..fb1cbc01 100644 --- a/pmm_qa/tasks/install_pmm_client_centos.yml +++ b/pmm_qa/tasks/install_pmm_client_centos.yml @@ -84,8 +84,8 @@ container: "{{ container_name }}" user: "root" command: > - percona-release enable-only pmm3-client experimental - apt-get update + percona-release enable-only pmm3-client experimental && + apt-get update && apt-get -y install pmm-client when: distro_family == "debian" and client_version == "3-dev-latest" @@ -94,7 +94,7 @@ container: "{{ container_name }}" user: "root" command: > - percona-release enable-only pmm3-client experimental + percona-release enable-only pmm3-client experimental && microdnf install -y pmm-client when: distro_family == "rhel" and client_version == "3-dev-latest" From d640ebae95a2f6bf932cd09c2bcec9ee8acad789 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 28 Apr 2025 16:55:41 +0200 Subject: [PATCH 029/340] PMM-7: reorder steps --- pmm_qa/tasks/install_pmm_client_centos.yml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/pmm_qa/tasks/install_pmm_client_centos.yml b/pmm_qa/tasks/install_pmm_client_centos.yml index fb1cbc01..6633a1d8 100644 --- a/pmm_qa/tasks/install_pmm_client_centos.yml +++ b/pmm_qa/tasks/install_pmm_client_centos.yml @@ -84,9 +84,11 @@ container: "{{ container_name }}" user: "root" command: > - percona-release enable-only pmm3-client experimental && - apt-get update && - apt-get -y install pmm-client + sh -c ' + percona-release enable-only pmm3-client experimental && + apt-get update && + apt-get -y install pmm-client + ' when: distro_family == "debian" and client_version == "3-dev-latest" - name: Install pmm client experimental on RHEL-family containers @@ -94,8 +96,10 @@ container: "{{ container_name }}" user: "root" command: > - percona-release enable-only pmm3-client experimental && - microdnf install -y pmm-client + sh -c ' + percona-release enable-only pmm3-client experimental && + microdnf install -y pmm-client + ' when: distro_family == "rhel" and client_version == "3-dev-latest" - name: Install pmm client testing on RHEL-family containers From 92d6aca5108bc188386eac4c34499ccde44629eb Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 28 Apr 2025 16:59:07 +0200 Subject: [PATCH 030/340] PMM-7: reorder steps --- pmm_qa/tasks/install_pmm_client_centos.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pmm_qa/tasks/install_pmm_client_centos.yml b/pmm_qa/tasks/install_pmm_client_centos.yml index 6633a1d8..2d9576aa 100644 --- a/pmm_qa/tasks/install_pmm_client_centos.yml +++ b/pmm_qa/tasks/install_pmm_client_centos.yml @@ -160,11 +160,11 @@ container: "{{ container_name }}" user: "root" command: pmm-agent setup --config-file=/usr/local/percona/pmm/config/pmm-agent.yaml --server-address={{ pmm_server_ip }}:{{ pmm_server_port }} --server-insecure-tls --metrics-mode={{ metrics_mode }} --server-username=admin --server-password={{ admin_password }} - when: metrics_mode is defined + when: metrics_mode | length > 0 - name: Connect pmm client to pmm server using metrics mode community.docker.docker_container_exec: container: "{{ container_name }}" user: "root" command: pmm-agent setup --config-file=/usr/local/percona/pmm/config/pmm-agent.yaml --server-address={{ pmm_server_ip }}:{{ pmm_server_port }} --server-insecure-tls --server-username=admin --server-password={{ admin_password }} - when: metrics_mode is undefined \ No newline at end of file + when: metrics_mode | length == 0 \ No newline at end of file From ce77a7f2b061ccb97af69e5cede99ad569edac05 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 28 Apr 2025 17:01:01 +0200 Subject: [PATCH 031/340] PMM-7: reorder steps --- pmm_qa/tasks/install_pmm_client_centos.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pmm_qa/tasks/install_pmm_client_centos.yml b/pmm_qa/tasks/install_pmm_client_centos.yml index 2d9576aa..274375d6 100644 --- a/pmm_qa/tasks/install_pmm_client_centos.yml +++ b/pmm_qa/tasks/install_pmm_client_centos.yml @@ -24,12 +24,12 @@ - name: Set correct pmm server port set_fact: - pmm_server_port: 80 + pmm_server_port: 443 when: pmm_server_ip is ansible.utils.ipv4 - name: Set correct pmm server port set_fact: - pmm_server_port: 8080 + pmm_server_port: 8433 when: pmm_server_ip is not ansible.utils.ipv4 - name: Detect OS inside the container From f70908ad998eec43c447e5ba705b19714909accf Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 28 Apr 2025 17:06:05 +0200 Subject: [PATCH 032/340] PMM-7: reorder steps --- pmm_qa/tasks/install_pmm_client_centos.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/tasks/install_pmm_client_centos.yml b/pmm_qa/tasks/install_pmm_client_centos.yml index 274375d6..0e9f8dfc 100644 --- a/pmm_qa/tasks/install_pmm_client_centos.yml +++ b/pmm_qa/tasks/install_pmm_client_centos.yml @@ -29,7 +29,7 @@ - name: Set correct pmm server port set_fact: - pmm_server_port: 8433 + pmm_server_port: 8443 when: pmm_server_ip is not ansible.utils.ipv4 - name: Detect OS inside the container From bddb854cf769edc4e7926337a6927ac06bb677dd Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 6 May 2025 13:26:13 +0200 Subject: [PATCH 033/340] PMM-7: Fix psmdb pmm client setup --- pmm_qa/tasks/install_pmm_client_centos.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/pmm_qa/tasks/install_pmm_client_centos.yml b/pmm_qa/tasks/install_pmm_client_centos.yml index 0e9f8dfc..45b6f66e 100644 --- a/pmm_qa/tasks/install_pmm_client_centos.yml +++ b/pmm_qa/tasks/install_pmm_client_centos.yml @@ -167,4 +167,14 @@ container: "{{ container_name }}" user: "root" command: pmm-agent setup --config-file=/usr/local/percona/pmm/config/pmm-agent.yaml --server-address={{ pmm_server_ip }}:{{ pmm_server_port }} --server-insecure-tls --server-username=admin --server-password={{ admin_password }} - when: metrics_mode | length == 0 \ No newline at end of file + when: metrics_mode | length == 0 + +- name: Wait 10 seconds for connection to complete + pause: + seconds: 10 + +- name: Start pmm client + community.docker.docker_container_exec: + container: "{{ container_name }}" + user: "root" + command: pmm-agent --config-file=/usr/local/percona/pmm/config/pmm-agent.yaml > pmm-agent.log 2>&1 & From fa60885d1f3224019a99a4445fa8c2438d85605a Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 6 May 2025 13:29:00 +0200 Subject: [PATCH 034/340] PMM-7: Fix psmdb pmm client setup --- pmm_qa/tasks/install_pmm_client_centos.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pmm_qa/tasks/install_pmm_client_centos.yml b/pmm_qa/tasks/install_pmm_client_centos.yml index 45b6f66e..55a61c31 100644 --- a/pmm_qa/tasks/install_pmm_client_centos.yml +++ b/pmm_qa/tasks/install_pmm_client_centos.yml @@ -177,4 +177,5 @@ community.docker.docker_container_exec: container: "{{ container_name }}" user: "root" - command: pmm-agent --config-file=/usr/local/percona/pmm/config/pmm-agent.yaml > pmm-agent.log 2>&1 & + detach: true + command: pmm-agent --config-file=/usr/local/percona/pmm/config/pmm-agent.yaml From e195d0ac009bc05e04719c2222dba9aeaf8ab72f Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 09:56:21 +0200 Subject: [PATCH 035/340] PMM-7: Fix psmdb pmm client setup --- pmm_qa/tasks/install_pmm_client_centos.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pmm_qa/tasks/install_pmm_client_centos.yml b/pmm_qa/tasks/install_pmm_client_centos.yml index 55a61c31..6fe2d391 100644 --- a/pmm_qa/tasks/install_pmm_client_centos.yml +++ b/pmm_qa/tasks/install_pmm_client_centos.yml @@ -169,9 +169,9 @@ command: pmm-agent setup --config-file=/usr/local/percona/pmm/config/pmm-agent.yaml --server-address={{ pmm_server_ip }}:{{ pmm_server_port }} --server-insecure-tls --server-username=admin --server-password={{ admin_password }} when: metrics_mode | length == 0 -- name: Wait 10 seconds for connection to complete +- name: Wait 5 seconds for connection to complete pause: - seconds: 10 + seconds: 5 - name: Start pmm client community.docker.docker_container_exec: @@ -179,3 +179,7 @@ user: "root" detach: true command: pmm-agent --config-file=/usr/local/percona/pmm/config/pmm-agent.yaml + +- name: Wait 5 seconds for start to complete + pause: + seconds: 5 \ No newline at end of file From 5d879633ff85cdf2e816efb7a260527bd2c0ede9 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 10:30:13 +0200 Subject: [PATCH 036/340] PMM-7: Fix psmdb pmm client setup --- pmm_qa/tasks/install_pmm_client_centos.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pmm_qa/tasks/install_pmm_client_centos.yml b/pmm_qa/tasks/install_pmm_client_centos.yml index 6fe2d391..3fdf3956 100644 --- a/pmm_qa/tasks/install_pmm_client_centos.yml +++ b/pmm_qa/tasks/install_pmm_client_centos.yml @@ -177,8 +177,8 @@ community.docker.docker_container_exec: container: "{{ container_name }}" user: "root" - detach: true - command: pmm-agent --config-file=/usr/local/percona/pmm/config/pmm-agent.yaml + command: > + sh -c 'nohup pmm-agent --config-file=/usr/local/percona/pmm/config/pmm-agent.yaml > /var/log/pmm-agent.log 2>&1 &' - name: Wait 5 seconds for start to complete pause: From 423e287d246ccc29c62cbfd5aaf17edd5e2e76cd Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 10:36:11 +0200 Subject: [PATCH 037/340] PMM-7: Fix psmdb pmm client setup --- pmm_qa/tasks/install_pmm_client_centos.yml | 40 +++++++++++++--------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/pmm_qa/tasks/install_pmm_client_centos.yml b/pmm_qa/tasks/install_pmm_client_centos.yml index 3fdf3956..d5855bef 100644 --- a/pmm_qa/tasks/install_pmm_client_centos.yml +++ b/pmm_qa/tasks/install_pmm_client_centos.yml @@ -138,22 +138,30 @@ microdnf install -y pmm-client when: distro_family == "rhel" and client_version == "pmm3-latest" -#- name: Install pmm client released version on RHEL-family containers -# community.docker.docker_container_exec: -# container: "{{ container_name }}" -# user: "root" -# command: > -# wget -O pmm-client.deb https://repo.percona.com/pmm3-client/apt/pool/main/p/pmm-client/pmm-client_${client_version}-7.$(lsb_release -sc)_amd64.deb -# dpkg -i pmm-client.deb -# when: "'{{ client_version }}' | regex_match('^3\\.[0-9]+\\.[0-9]+$') and distro_family == 'debian'" -# -#- name: Install pmm client released version on RHEL-family containers -# community.docker.docker_container_exec: -# container: "{{ container_name }}" -# user: "root" -# command: > -# microdnf install -y pmm-client -# when: "'{{ client_version }}' | regex_match('^3\\.[0-9]+\\.[0-9]+$') and distro_family == 'rhel'" +- name: Install specific PMM client version on Debian-family containers + community.docker.docker_container_exec: + container: "{{ container_name }}" + user: "root" + command: > + sh -c ' + wget -O /pmm-client.deb https://repo.percona.com/pmm3-client/apt/pool/main/p/pmm-client/pmm-client_{{ client_version }}-7.bullseye_amd64.deb && + dpkg -i /pmm-client.deb + ' + when: + - distro_family == 'debian' + - client_version is match('^3\\.[0-9]+\\.[0-9]+$') + +- name: Install specific PMM client version on RHEL-family containers + community.docker.docker_container_exec: + container: "{{ container_name }}" + user: "root" + command: > + sh -c ' + yum install -y https://repo.percona.com/pmm3-client/yum/release/7/x86_64/pmm-client-{{ client_version }}-7.el7.x86_64.rpm + ' + when: + - distro_family == 'rhel' + - client_version is match('^3\\.[0-9]+\\.[0-9]+$') - name: Connect pmm client to pmm server using metrics mode community.docker.docker_container_exec: From ede64f96f40341a063f5ea17cee575cd1b0deb21 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 10:37:55 +0200 Subject: [PATCH 038/340] PMM-7: Fix psmdb pmm client setup --- pmm_qa/tasks/install_pmm_client_centos.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pmm_qa/tasks/install_pmm_client_centos.yml b/pmm_qa/tasks/install_pmm_client_centos.yml index d5855bef..7379f0ae 100644 --- a/pmm_qa/tasks/install_pmm_client_centos.yml +++ b/pmm_qa/tasks/install_pmm_client_centos.yml @@ -144,7 +144,7 @@ user: "root" command: > sh -c ' - wget -O /pmm-client.deb https://repo.percona.com/pmm3-client/apt/pool/main/p/pmm-client/pmm-client_{{ client_version }}-7.bullseye_amd64.deb && + wget -O /pmm-client.deb https://repo.percona.com/pmm3-client/apt/pool/main/p/pmm-client/pmm-client_{{ client_version }}-7.$(lsb_release -sc)_amd64.deb && dpkg -i /pmm-client.deb ' when: @@ -157,7 +157,7 @@ user: "root" command: > sh -c ' - yum install -y https://repo.percona.com/pmm3-client/yum/release/7/x86_64/pmm-client-{{ client_version }}-7.el7.x86_64.rpm + dnf install -y https://repo.percona.com/pmm3-client/yum/release/7/x86_64/pmm-client-{{ client_version }}-7.el9.x86_64.rpm ' when: - distro_family == 'rhel' From e04becc6e47e86efc1e1f84c09fb0898f01cdd99 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 10:38:56 +0200 Subject: [PATCH 039/340] PMM-7: Install pmm client --- pmm_qa/tasks/install_pmm_client_centos.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/tasks/install_pmm_client_centos.yml b/pmm_qa/tasks/install_pmm_client_centos.yml index 7379f0ae..bcdcc90f 100644 --- a/pmm_qa/tasks/install_pmm_client_centos.yml +++ b/pmm_qa/tasks/install_pmm_client_centos.yml @@ -157,7 +157,7 @@ user: "root" command: > sh -c ' - dnf install -y https://repo.percona.com/pmm3-client/yum/release/7/x86_64/pmm-client-{{ client_version }}-7.el9.x86_64.rpm + microdnf install -y https://repo.percona.com/pmm3-client/yum/release/7/x86_64/pmm-client-{{ client_version }}-7.el9.x86_64.rpm ' when: - distro_family == 'rhel' From 16ce2dab1c742effe9962c9992ea5720ef75f04f Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 10:46:24 +0200 Subject: [PATCH 040/340] PMM-7: Install pmm client --- pmm_qa/tasks/install_pmm_client_centos.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/tasks/install_pmm_client_centos.yml b/pmm_qa/tasks/install_pmm_client_centos.yml index bcdcc90f..3c714172 100644 --- a/pmm_qa/tasks/install_pmm_client_centos.yml +++ b/pmm_qa/tasks/install_pmm_client_centos.yml @@ -157,7 +157,7 @@ user: "root" command: > sh -c ' - microdnf install -y https://repo.percona.com/pmm3-client/yum/release/7/x86_64/pmm-client-{{ client_version }}-7.el9.x86_64.rpm + microdnf install -y https://repo.percona.com/pmm3-client/yum/release/9/RPMS/x86_64/pmm-client-{{ client_version }}-7.el9.x86_64.rpm ' when: - distro_family == 'rhel' From b67229576b86c704da9bca36d9b9944e0c9875b4 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 10:53:34 +0200 Subject: [PATCH 041/340] PMM-7: Install pmm client --- pmm_qa/tasks/install_pmm_client_centos.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pmm_qa/tasks/install_pmm_client_centos.yml b/pmm_qa/tasks/install_pmm_client_centos.yml index 3c714172..11c77b41 100644 --- a/pmm_qa/tasks/install_pmm_client_centos.yml +++ b/pmm_qa/tasks/install_pmm_client_centos.yml @@ -157,12 +157,14 @@ user: "root" command: > sh -c ' - microdnf install -y https://repo.percona.com/pmm3-client/yum/release/9/RPMS/x86_64/pmm-client-{{ client_version }}-7.el9.x86_64.rpm + wget -O /pmm-client.rpm https://repo.percona.com/pmm3-client/yum/release/9/RPMS/x86_64/pmm-client-{{ client_version }}-7.el9.x86_64.rpm && + microdnf install -y /pmm-client.rpm ' when: - distro_family == 'rhel' - client_version is match('^3\\.[0-9]+\\.[0-9]+$') - +#https://repo.percona.com/pmm3-client/yum/release/9/RPMS/x86_64/pmm-client-3.1.0-7.el9.x86_64.rpm +#https://repo.percona.com/pmm3-client/yum/release/9/RPMS/x86_64/pmm-client-3.1.0-7.el9.x86_64.rpm - name: Connect pmm client to pmm server using metrics mode community.docker.docker_container_exec: container: "{{ container_name }}" From 69ff82fd34ecb5a218baf18ac534e9e902a661af Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 10:58:08 +0200 Subject: [PATCH 042/340] PMM-7: Install pmm client --- pmm_qa/tasks/install_pmm_client_centos.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/tasks/install_pmm_client_centos.yml b/pmm_qa/tasks/install_pmm_client_centos.yml index 11c77b41..36c274ef 100644 --- a/pmm_qa/tasks/install_pmm_client_centos.yml +++ b/pmm_qa/tasks/install_pmm_client_centos.yml @@ -158,7 +158,7 @@ command: > sh -c ' wget -O /pmm-client.rpm https://repo.percona.com/pmm3-client/yum/release/9/RPMS/x86_64/pmm-client-{{ client_version }}-7.el9.x86_64.rpm && - microdnf install -y /pmm-client.rpm + rpm -i /pmm-client.rpm ' when: - distro_family == 'rhel' From 6f404aaa315bb0536e9d89eb730e7276d8253cd3 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 11:18:40 +0200 Subject: [PATCH 043/340] PMM-7: Install pmm client --- pmm_qa/tasks/install_pmm_client_centos.yml | 41 ++++++++++++++++++++-- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/pmm_qa/tasks/install_pmm_client_centos.yml b/pmm_qa/tasks/install_pmm_client_centos.yml index 36c274ef..19f46cfc 100644 --- a/pmm_qa/tasks/install_pmm_client_centos.yml +++ b/pmm_qa/tasks/install_pmm_client_centos.yml @@ -163,8 +163,45 @@ when: - distro_family == 'rhel' - client_version is match('^3\\.[0-9]+\\.[0-9]+$') -#https://repo.percona.com/pmm3-client/yum/release/9/RPMS/x86_64/pmm-client-3.1.0-7.el9.x86_64.rpm -#https://repo.percona.com/pmm3-client/yum/release/9/RPMS/x86_64/pmm-client-3.1.0-7.el9.x86_64.rpm + +- name: Install tarball PMM client version on Debian-family containers + community.docker.docker_container_exec: + container: "{{ container_name }}" + user: "root" + command: > + sh -c ' + wget -O /pmm-client.tar.gz "{{ client_version }}" && + dpkg -i /pmm-client.deb + ' + when: + - distro_family == 'debian' + - client_version | regex_search('^https?://.*\\.tar\\.gz$') is not none + +- name: Install tarball PMM client version on RHEL-family containers + community.docker.docker_container_exec: + container: "{{ container_name }}" + user: "root" + command: > + sh -c ' + wget -O /pmm-client.tar.gz "{{ client_version }}" && + tar -zxpf /pmm-client.tar.gz && + PMM_CLIENT=`ls -1td pmm-client* 2>/dev/null | grep -v ".tar" | grep -v ".sh" | head -n1` && + echo ${PMM_CLIENT} && + rm -rf pmm-client && + mv ${PMM_CLIENT} pmm-client && + rm -rf /usr/local/bin/pmm-client && + mv -f pmm-client /usr/local/bin && + pushd /usr/local/bin/pmm-client && + ## only setting up all binaries in default path /usr/local/percona/pmm && + bash -x ./install_tarball ${upgrade} && + pwd && + popd && + pmm-admin --version && + ' + when: + - distro_family == 'rhel' + - client_version | regex_search('^https?://.*\\.tar\\.gz$') is not none + - name: Connect pmm client to pmm server using metrics mode community.docker.docker_container_exec: container: "{{ container_name }}" From d8dff0cde588a224b4ef2b2d3b807bb946eb612d Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 11:20:01 +0200 Subject: [PATCH 044/340] PMM-7: Install pmm client --- pmm_qa/tasks/install_pmm_client_centos.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/tasks/install_pmm_client_centos.yml b/pmm_qa/tasks/install_pmm_client_centos.yml index 19f46cfc..77a522fc 100644 --- a/pmm_qa/tasks/install_pmm_client_centos.yml +++ b/pmm_qa/tasks/install_pmm_client_centos.yml @@ -196,7 +196,7 @@ bash -x ./install_tarball ${upgrade} && pwd && popd && - pmm-admin --version && + pmm-admin --version ' when: - distro_family == 'rhel' From 9e93301123a82c91d42e444bd5ebc65655e279d8 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 11:28:28 +0200 Subject: [PATCH 045/340] PMM-7: Install pmm client --- pmm_qa/tasks/install_pmm_client_centos.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pmm_qa/tasks/install_pmm_client_centos.yml b/pmm_qa/tasks/install_pmm_client_centos.yml index 77a522fc..839ca87a 100644 --- a/pmm_qa/tasks/install_pmm_client_centos.yml +++ b/pmm_qa/tasks/install_pmm_client_centos.yml @@ -193,7 +193,9 @@ mv -f pmm-client /usr/local/bin && pushd /usr/local/bin/pmm-client && ## only setting up all binaries in default path /usr/local/percona/pmm && - bash -x ./install_tarball ${upgrade} && + bash -x ./install_tarball && + ln -sf /usr/local/percona/pmm/bin/pmm-admin /usr/local/bin/pmm-admin && + ln -sf /usr/local/percona/pmm/bin/pmm-agent /usr/local/bin/pmm-agent && pwd && popd && pmm-admin --version From b7d529041dc8433dfec648ea4c0bda76b452089f Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 11:51:43 +0200 Subject: [PATCH 046/340] PMM-7: Add pgsql --- pmm_qa/pmm-framework.py | 44 ++++++++++++++++++-------- pmm_qa/postgresql/postgresql-setup.yml | 41 ++++++++++++++++++++++++ 2 files changed, 72 insertions(+), 13 deletions(-) create mode 100644 pmm_qa/postgresql/postgresql-setup.yml diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 5538e53b..17e07946 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -369,21 +369,39 @@ def setup_pgsql(db_type, db_version=None, db_config=None, args=None): # Gather Version details pgsql_version = os.getenv('PGSQL_VERSION') or db_version or database_configs[db_type]["versions"][-1] + setup_type_value = get_value('SETUP_TYPE', db_type, args, db_config).lower() - # Define environment variables for playbook - env_vars = { - 'PGSQL_VERSION': pgsql_version, - 'PMM_SERVER_IP': args.pmm_server_ip or container_name or '127.0.0.1', - 'PGSQL_PGSS_CONTAINER': 'pgsql_pgss_pmm_' + str(pgsql_version), - 'CLIENT_VERSION': get_value('CLIENT_VERSION', db_type, args, db_config), - 'USE_SOCKET': get_value('USE_SOCKET', db_type, args, db_config), - 'ADMIN_PASSWORD': os.getenv('ADMIN_PASSWORD') or args.pmm_server_password or 'admin', - 'PGSQL_PGSS_PORT': 5448, - 'PMM_QA_GIT_BRANCH': os.getenv('PMM_QA_GIT_BRANCH') or 'v3' - } + if setup_type_value in ("replication", "replica"): + # Define environment variables for playbook + env_vars = { + 'PGSQL_VERSION': pgsql_version, + 'PMM_SERVER_IP': args.pmm_server_ip or container_name or '127.0.0.1', + 'PGSQL_PGSS_CONTAINER': 'pgsql_pgss_pmm_' + str(pgsql_version), + 'CLIENT_VERSION': get_value('CLIENT_VERSION', db_type, args, db_config), + 'USE_SOCKET': get_value('USE_SOCKET', db_type, args, db_config), + 'ADMIN_PASSWORD': os.getenv('ADMIN_PASSWORD') or args.pmm_server_password or 'admin', + 'PGSQL_PGSS_PORT': 5448, + 'PMM_QA_GIT_BRANCH': os.getenv('PMM_QA_GIT_BRANCH') or 'v3', + 'SETUP_TYPE': setup_type_value + } - # Ansible playbook filename - playbook_filename = 'pgsql_pgss_setup.yml' + # Ansible playbook filename + playbook_filename = 'postgresql/postgresql-setup.yml' + else: + # Define environment variables for playbook + env_vars = { + 'PGSQL_VERSION': pgsql_version, + 'PMM_SERVER_IP': args.pmm_server_ip or container_name or '127.0.0.1', + 'PGSQL_PGSS_CONTAINER': 'pgsql_pgss_pmm_' + str(pgsql_version), + 'CLIENT_VERSION': get_value('CLIENT_VERSION', db_type, args, db_config), + 'USE_SOCKET': get_value('USE_SOCKET', db_type, args, db_config), + 'ADMIN_PASSWORD': os.getenv('ADMIN_PASSWORD') or args.pmm_server_password or 'admin', + 'PGSQL_PGSS_PORT': 5448, + 'PMM_QA_GIT_BRANCH': os.getenv('PMM_QA_GIT_BRANCH') or 'v3' + } + + # Ansible playbook filename + playbook_filename = 'pgsql_pgss_setup.yml' # Call the function to run the Ansible playbook run_ansible_playbook(playbook_filename, env_vars, args) diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml new file mode 100644 index 00000000..04b8235d --- /dev/null +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -0,0 +1,41 @@ +--- +# Postgresql Replication +- name: Setup Postgresql replication + hosts: localhost + connection: local + gather_facts: yes + vars: + pg_version: "{{ lookup('env', 'PGSQL_VERSION') | default('17', true) }}" +# cluster_name: "mysql_cluster" +# replication_user: "repl_user" +# replication_password: "GRgrO9301RuF" + root_password: "GRgrO9301RuF" +# mysql_port: 33066 +# mysql_listen_port: 3306 +# group_seeds_port: 34061 +# nodes_count: "{{ (lookup('env', 'NODES_COUNT') | default('3', true)) | int }}" +# network_name: "pmm-qa" +# data_dir: "{{ lookup('env', 'HOME') }}/mysql_cluster_data" +# server_id_start: 1 +# pmm_server_ip: "{{ lookup('vars', 'extra_pmm_server_ip', default=lookup('env','PMM_SERVER_IP') | default('127.0.0.1', true) ) }}" +# client_version: "{{ lookup('vars', 'extra_client_version', default=lookup('env','CLIENT_VERSION') | default('3-dev-latest', true) ) }}" +# admin_password: "{{ lookup('vars', 'extra_admin_password', default=lookup('env','ADMIN_PASSWORD') | default('admin', true) ) }}" +# query_source: "{{ lookup('env', 'QUERY_SOURCE') | default('perfschema', true) }}" +# metrics_mode: "{{ lookup('env', 'metrics_mode') }}" +# setup_type: "{{ lookup('env', 'SETUP_TYPE') }}" +# random_service_name_value: "" + +- name: Start PostgreSQL primary container + community.docker.docker_container: + name: "pgsql_pmm_{{ ps_version }}_1" + image: "postgres:{{ pg_version }}-bookworm" + restart_policy: always + state: started + networks: + - name: "{{ network_name }}" + env: + POSTGRES_PASSWORD: root_password + volumes: + - pg_primary_data:/var/lib/postgresql/data + ports: + - "5432:5432" \ No newline at end of file From 4fd69cb67f87dfd59c83d8d41e96eec96069363d Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 11:53:22 +0200 Subject: [PATCH 047/340] PMM-7: Add pgsql --- pmm_qa/pmm-framework.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 17e07946..f9fb7b51 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -48,7 +48,8 @@ }, "PGSQL": { "versions": ["11", "12", "13", "14", "15", "16", "17"], - "configurations": {"QUERY_SOURCE": "pgstatements", "CLIENT_VERSION": "3-dev-latest", "USE_SOCKET": ""} + "configurations": {"QUERY_SOURCE": "pgstatements", "CLIENT_VERSION": "3-dev-latest", "USE_SOCKET": "", + "SETUP_TYPE": ""} }, "PDPGSQL": { "versions": ["11", "12", "13", "14", "15", "16", "17"], @@ -371,6 +372,8 @@ def setup_pgsql(db_type, db_version=None, db_config=None, args=None): pgsql_version = os.getenv('PGSQL_VERSION') or db_version or database_configs[db_type]["versions"][-1] setup_type_value = get_value('SETUP_TYPE', db_type, args, db_config).lower() + print(f"Setup type is {setup_type_value}") + if setup_type_value in ("replication", "replica"): # Define environment variables for playbook env_vars = { From c37c9e3bb14bf70e0c516baaa287fd0fde9063d4 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 11:54:37 +0200 Subject: [PATCH 048/340] PMM-7: Add pgsql --- pmm_qa/postgresql/postgresql-setup.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index 04b8235d..2efc3510 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -34,8 +34,8 @@ networks: - name: "{{ network_name }}" env: - POSTGRES_PASSWORD: root_password - volumes: - - pg_primary_data:/var/lib/postgresql/data + POSTGRES_PASSWORD: "{{ root_password }}" +# volumes: +# - pg_primary_data:/var/lib/postgresql/data ports: - "5432:5432" \ No newline at end of file From f9b475020cf0c9a78081c697ab8e755e0c63eb9d Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 11:56:03 +0200 Subject: [PATCH 049/340] PMM-7: Add pgsql --- pmm_qa/postgresql/postgresql-setup.yml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index 2efc3510..310902cd 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -24,18 +24,18 @@ # metrics_mode: "{{ lookup('env', 'metrics_mode') }}" # setup_type: "{{ lookup('env', 'SETUP_TYPE') }}" # random_service_name_value: "" - -- name: Start PostgreSQL primary container - community.docker.docker_container: - name: "pgsql_pmm_{{ ps_version }}_1" - image: "postgres:{{ pg_version }}-bookworm" - restart_policy: always - state: started - networks: - - name: "{{ network_name }}" - env: - POSTGRES_PASSWORD: "{{ root_password }}" + tasks: + - name: Start PostgreSQL primary container + community.docker.docker_container: + name: "pgsql_pmm_{{ ps_version }}_1" + image: "postgres:{{ pg_version }}-bookworm" + restart_policy: always + state: started + networks: + - name: "{{ network_name }}" + env: + POSTGRES_PASSWORD: "{{ root_password }}" # volumes: # - pg_primary_data:/var/lib/postgresql/data - ports: - - "5432:5432" \ No newline at end of file + ports: + - "5432:5432" \ No newline at end of file From 60967611e7c9960fe7b5ae6d19dc5eaaa91e10a7 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 11:57:12 +0200 Subject: [PATCH 050/340] PMM-7: Add pgsql --- pmm_qa/postgresql/postgresql-setup.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index 310902cd..dedd4ded 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -25,9 +25,17 @@ # setup_type: "{{ lookup('env', 'SETUP_TYPE') }}" # random_service_name_value: "" tasks: + - name: Remove old PostgreSQL primary container + community.docker.docker_container: + name: "pgsql_pmm_{{ pg_version }}_1" + image: "postgres:{{ pg_version }}-bookworm" + restart_policy: always + state: absent + ignore_errors: yes + - name: Start PostgreSQL primary container community.docker.docker_container: - name: "pgsql_pmm_{{ ps_version }}_1" + name: "pgsql_pmm_{{ pg_version }}_1" image: "postgres:{{ pg_version }}-bookworm" restart_policy: always state: started From 76557902a05bf27e73ac15cb22a3cd988b259671 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 11:57:36 +0200 Subject: [PATCH 051/340] PMM-7: Add pgsql --- pmm_qa/postgresql/postgresql-setup.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index dedd4ded..4b8ca321 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -14,7 +14,7 @@ # mysql_listen_port: 3306 # group_seeds_port: 34061 # nodes_count: "{{ (lookup('env', 'NODES_COUNT') | default('3', true)) | int }}" -# network_name: "pmm-qa" + network_name: "pmm-qa" # data_dir: "{{ lookup('env', 'HOME') }}/mysql_cluster_data" # server_id_start: 1 # pmm_server_ip: "{{ lookup('vars', 'extra_pmm_server_ip', default=lookup('env','PMM_SERVER_IP') | default('127.0.0.1', true) ) }}" From d6f7547296f0ead3be73dfc9662292389533ca9c Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 12:03:20 +0200 Subject: [PATCH 052/340] PMM-7: Add pgsql --- pmm_qa/postgresql/postgresql-setup.yml | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index 4b8ca321..bbca7dd3 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -15,7 +15,7 @@ # group_seeds_port: 34061 # nodes_count: "{{ (lookup('env', 'NODES_COUNT') | default('3', true)) | int }}" network_name: "pmm-qa" -# data_dir: "{{ lookup('env', 'HOME') }}/mysql_cluster_data" + data_dir: "{{ lookup('env', 'HOME') }}/pgsql_cluster_data" # server_id_start: 1 # pmm_server_ip: "{{ lookup('vars', 'extra_pmm_server_ip', default=lookup('env','PMM_SERVER_IP') | default('127.0.0.1', true) ) }}" # client_version: "{{ lookup('vars', 'extra_client_version', default=lookup('env','CLIENT_VERSION') | default('3-dev-latest', true) ) }}" @@ -43,7 +43,14 @@ - name: "{{ network_name }}" env: POSTGRES_PASSWORD: "{{ root_password }}" -# volumes: -# - pg_primary_data:/var/lib/postgresql/data + volumes: + - "{{ data_dir }}/node1/data:/var/lib/postgresql" ports: - - "5432:5432" \ No newline at end of file + - "5432:5432" + + - name: Wait for PgSQL to be available + wait_for: + host: localhost + port: "5432" + delay: 10 + timeout: 300 From e63709e7a79eacf88c2c4d53804ed01504146efb Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 12:04:09 +0200 Subject: [PATCH 053/340] PMM-7: Add pgsql --- pmm_qa/postgresql/postgresql-setup.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index bbca7dd3..52959850 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -25,6 +25,15 @@ # setup_type: "{{ lookup('env', 'SETUP_TYPE') }}" # random_service_name_value: "" tasks: + - name: Create Docker network + community.docker.docker_network: + name: "{{ network_name }}" + state: present + ignore_errors: yes + + - name: "Remove old data folders" + shell: 'rm -fr {{ data_dir }}' + - name: Remove old PostgreSQL primary container community.docker.docker_container: name: "pgsql_pmm_{{ pg_version }}_1" From d460faa155d100a91af94263837c89ddf8d836c0 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 12:05:29 +0200 Subject: [PATCH 054/340] PMM-7: Add pgsql --- pmm_qa/postgresql/postgresql-setup.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index 52959850..7fca20c4 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -53,7 +53,7 @@ env: POSTGRES_PASSWORD: "{{ root_password }}" volumes: - - "{{ data_dir }}/node1/data:/var/lib/postgresql" + - "{{ data_dir }}/node1/data:/var/lib/postgresql/data" ports: - "5432:5432" From 971224886111beaa88e1a92fee49487e5987023c Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 12:08:03 +0200 Subject: [PATCH 055/340] PMM-7: Add pgsql --- pmm_qa/postgresql/data/pg-primary-conf/pg_hba.conf | 3 +++ pmm_qa/postgresql/data/pg-primary-conf/postgresql.conf | 4 ++++ pmm_qa/postgresql/postgresql-setup.yml | 3 +++ 3 files changed, 10 insertions(+) create mode 100644 pmm_qa/postgresql/data/pg-primary-conf/pg_hba.conf create mode 100644 pmm_qa/postgresql/data/pg-primary-conf/postgresql.conf diff --git a/pmm_qa/postgresql/data/pg-primary-conf/pg_hba.conf b/pmm_qa/postgresql/data/pg-primary-conf/pg_hba.conf new file mode 100644 index 00000000..a103c5c7 --- /dev/null +++ b/pmm_qa/postgresql/data/pg-primary-conf/pg_hba.conf @@ -0,0 +1,3 @@ +# Allow replication from any host using md5 +host replication replicator 0.0.0.0/0 md5 +host all all 0.0.0.0/0 md5 \ No newline at end of file diff --git a/pmm_qa/postgresql/data/pg-primary-conf/postgresql.conf b/pmm_qa/postgresql/data/pg-primary-conf/postgresql.conf new file mode 100644 index 00000000..d8fc5ef9 --- /dev/null +++ b/pmm_qa/postgresql/data/pg-primary-conf/postgresql.conf @@ -0,0 +1,4 @@ +wal_level = replica +max_wal_senders = 10 +wal_keep_size = 64MB +hot_standby = on \ No newline at end of file diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index 7fca20c4..c038a255 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -54,6 +54,9 @@ POSTGRES_PASSWORD: "{{ root_password }}" volumes: - "{{ data_dir }}/node1/data:/var/lib/postgresql/data" + - "./data/pg-primary-conf/postgresql.conf:/etc/postgresql/postgresql.conf:ro" + - "./data/pg-primary-conf/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro" + command: -c config_file=/etc/postgresql/postgresql.conf ports: - "5432:5432" From 95054f66fb402e6a489c6152a11a72e72dc01ad5 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 12:10:49 +0200 Subject: [PATCH 056/340] PMM-7: Add pgsql --- pmm_qa/postgresql/postgresql-setup.yml | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index c038a255..ae21fbbb 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -7,8 +7,8 @@ vars: pg_version: "{{ lookup('env', 'PGSQL_VERSION') | default('17', true) }}" # cluster_name: "mysql_cluster" -# replication_user: "repl_user" -# replication_password: "GRgrO9301RuF" + replication_user: "repl_user" + replication_password: "GRgrO9301RuF" root_password: "GRgrO9301RuF" # mysql_port: 33066 # mysql_listen_port: 3306 @@ -66,3 +66,16 @@ port: "5432" delay: 10 timeout: 300 + + - name: Create replication user + community.docker.docker_container_exec: + container: "pgsql_pmm_{{ pg_version }}_1" +# user: postgres + command: > + psql -c " + CREATE ROLE {{ replication_user }} WITH REPLICATION LOGIN ENCRYPTED PASSWORD '{{ replication_password }}'; + " + - name: Restart PostgreSQL primary + community.docker.docker_container: + container: "pgsql_pmm_{{ pg_version }}_1" + state: restarted \ No newline at end of file From 5135e759df1f3430d2206f923001ff83a976aa4a Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 12:11:24 +0200 Subject: [PATCH 057/340] PMM-7: Add pgsql --- pmm_qa/postgresql/postgresql-setup.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index ae21fbbb..f8c758d5 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -70,7 +70,7 @@ - name: Create replication user community.docker.docker_container_exec: container: "pgsql_pmm_{{ pg_version }}_1" -# user: postgres + user: postgres command: > psql -c " CREATE ROLE {{ replication_user }} WITH REPLICATION LOGIN ENCRYPTED PASSWORD '{{ replication_password }}'; From 8c6b5fb0665e17950aa6f4eea6611a49e04a8799 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 12:12:19 +0200 Subject: [PATCH 058/340] PMM-7: Add pgsql --- pmm_qa/postgresql/postgresql-setup.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index f8c758d5..760dc59d 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -77,5 +77,5 @@ " - name: Restart PostgreSQL primary community.docker.docker_container: - container: "pgsql_pmm_{{ pg_version }}_1" + name: "pgsql_pmm_{{ pg_version }}_1" state: restarted \ No newline at end of file From 8aa76b94cbc2b7afe5705423adb02db2bff75b74 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 12:15:04 +0200 Subject: [PATCH 059/340] PMM-7: Add pgsql --- pmm_qa/postgresql/postgresql-setup.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index 760dc59d..00f14195 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -48,6 +48,7 @@ image: "postgres:{{ pg_version }}-bookworm" restart_policy: always state: started + recreate: true networks: - name: "{{ network_name }}" env: @@ -75,7 +76,10 @@ psql -c " CREATE ROLE {{ replication_user }} WITH REPLICATION LOGIN ENCRYPTED PASSWORD '{{ replication_password }}'; " - - name: Restart PostgreSQL primary + + - name: Stop and remove replica if exists community.docker.docker_container: - name: "pgsql_pmm_{{ pg_version }}_1" - state: restarted \ No newline at end of file + name: "pgsql_pmm_{{ pg_version }}_{{ item }}" + state: absent + loop: "{{ range(2, nodes_count | int + 1) | list }}" + ignore_errors: yes From ebc8ea6d59c82cc2cc2621c34f35f630ec26b1a8 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 12:16:09 +0200 Subject: [PATCH 060/340] PMM-7: Add pgsql --- pmm_qa/postgresql/postgresql-setup.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index 00f14195..9678f0c4 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -13,7 +13,7 @@ # mysql_port: 33066 # mysql_listen_port: 3306 # group_seeds_port: 34061 -# nodes_count: "{{ (lookup('env', 'NODES_COUNT') | default('3', true)) | int }}" + nodes_count: "{{ (lookup('env', 'NODES_COUNT') | default('3', true)) | int }}" network_name: "pmm-qa" data_dir: "{{ lookup('env', 'HOME') }}/pgsql_cluster_data" # server_id_start: 1 @@ -22,9 +22,12 @@ # admin_password: "{{ lookup('vars', 'extra_admin_password', default=lookup('env','ADMIN_PASSWORD') | default('admin', true) ) }}" # query_source: "{{ lookup('env', 'QUERY_SOURCE') | default('perfschema', true) }}" # metrics_mode: "{{ lookup('env', 'metrics_mode') }}" -# setup_type: "{{ lookup('env', 'SETUP_TYPE') }}" + setup_type: "{{ lookup('env', 'SETUP_TYPE') }}" # random_service_name_value: "" tasks: + - debug: + msg: "{{ nodes_count }}" + - name: Create Docker network community.docker.docker_network: name: "{{ network_name }}" From 294b695a2a8d67e2a476b5b602ee124504a7759c Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 12:16:51 +0200 Subject: [PATCH 061/340] PMM-7: Add pgsql --- pmm_qa/postgresql/postgresql-setup.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index 9678f0c4..adeb397e 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -13,7 +13,7 @@ # mysql_port: 33066 # mysql_listen_port: 3306 # group_seeds_port: 34061 - nodes_count: "{{ (lookup('env', 'NODES_COUNT') | default('3', true)) | int }}" + nodes_count: "{{ (lookup('env', 'NODES_COUNT') | default(2, true)) | int }}" network_name: "pmm-qa" data_dir: "{{ lookup('env', 'HOME') }}/pgsql_cluster_data" # server_id_start: 1 @@ -24,6 +24,7 @@ # metrics_mode: "{{ lookup('env', 'metrics_mode') }}" setup_type: "{{ lookup('env', 'SETUP_TYPE') }}" # random_service_name_value: "" + tasks: - debug: msg: "{{ nodes_count }}" From 7912dcef919c059e1375973ad127680fb4088f90 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 12:16:59 +0200 Subject: [PATCH 062/340] PMM-7: Add pgsql --- pmm_qa/postgresql/postgresql-setup.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index adeb397e..b9338f94 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -13,7 +13,7 @@ # mysql_port: 33066 # mysql_listen_port: 3306 # group_seeds_port: 34061 - nodes_count: "{{ (lookup('env', 'NODES_COUNT') | default(2, true)) | int }}" + nodes_count: "{{ (lookup('env', 'NODES_COUNT') | default('2', true)) | int }}" network_name: "pmm-qa" data_dir: "{{ lookup('env', 'HOME') }}/pgsql_cluster_data" # server_id_start: 1 From c8a7e9d55540eeb464856a9cf5e01cb909af5c3c Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 12:20:53 +0200 Subject: [PATCH 063/340] PMM-7: Add pgsql --- .../percona_server/percona-server-setup.yml | 2 +- pmm_qa/postgresql/postgresql-setup.yml | 31 ++++++++++++++++--- 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/pmm_qa/percona_server/percona-server-setup.yml b/pmm_qa/percona_server/percona-server-setup.yml index b72b08e7..b58b2760 100644 --- a/pmm_qa/percona_server/percona-server-setup.yml +++ b/pmm_qa/percona_server/percona-server-setup.yml @@ -41,7 +41,7 @@ name: "{{ network_name }}" state: present - - name: "Remove old data folders" + - name: Remove old data folders shell: 'rm -fr {{ data_dir }}' loop: "{{ range(1, nodes_count | int + 1) | list }}" diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index b9338f94..5365c6f8 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -26,18 +26,22 @@ # random_service_name_value: "" tasks: - - debug: - msg: "{{ nodes_count }}" - - name: Create Docker network community.docker.docker_network: name: "{{ network_name }}" state: present ignore_errors: yes - - name: "Remove old data folders" + - name: Remove old data folders shell: 'rm -fr {{ data_dir }}' + - name: Create data directories + file: + path: "{{ data_dir }}/node{{ item }}/data" + state: directory + mode: '0755' + loop: "{{ range(1, nodes_count | int + 1) | list }}" + - name: Remove old PostgreSQL primary container community.docker.docker_container: name: "pgsql_pmm_{{ pg_version }}_1" @@ -87,3 +91,22 @@ state: absent loop: "{{ range(2, nodes_count | int + 1) | list }}" ignore_errors: yes + + - name: Start PostgreSQL replica container + community.docker.docker_container: + name: "pgsql_pmm_{{ pg_version }}_{{ item }}" + image: "postgres:{{ pg_version }}-bookworm" + restart_policy: always + state: started + recreate: true + env: + POSTGRES_PASSWORD: "{{ root_password }}" + volumes: + - "{{ data_dir }}/node{{ item }}/data:/var/lib/postgresql/data" + loop: "{{ range(2, nodes_count | int + 1) | list }}" + + - name: Check replication status on primary + community.docker.docker_container_exec: + container: "pgsql_pmm_{{ pg_version }}_1" + user: postgres + command: psql -c "SELECT * FROM pg_stat_replication;" From b9912ec7a754c0552785903efbcf09edfb6dc3c0 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 12:24:55 +0200 Subject: [PATCH 064/340] PMM-7: Add pgsql --- pmm_qa/postgresql/postgresql-setup.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index 5365c6f8..9be1f1f9 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -105,6 +105,14 @@ - "{{ data_dir }}/node{{ item }}/data:/var/lib/postgresql/data" loop: "{{ range(2, nodes_count | int + 1) | list }}" + - name: Perform base backup from primary to replica + community.docker.docker_container_exec: + container: "pgsql_pmm_{{ pg_version }}_{{ item }}" + user: postgres + command: > + bash -c "PGPASSWORD={{ replication_password }} pg_basebackup -h pgsql_pmm_{{ pg_version }}_1 -D $PGDATA -U replicator -Fp -Xs -P -R" + loop: "{{ range(2, nodes_count | int + 1) | list }}" + - name: Check replication status on primary community.docker.docker_container_exec: container: "pgsql_pmm_{{ pg_version }}_1" From 5a08d1556550299c8a4836f216ad7cf76844125e Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 12:26:26 +0200 Subject: [PATCH 065/340] PMM-7: Add pgsql --- pmm_qa/postgresql/postgresql-setup.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index 9be1f1f9..1495ebc1 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -99,6 +99,8 @@ restart_policy: always state: started recreate: true + networks: + - name: "{{ network_name }}" env: POSTGRES_PASSWORD: "{{ root_password }}" volumes: From 0c809c90bd236707e286698d92db44542349602a Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 12:30:55 +0200 Subject: [PATCH 066/340] PMM-7: Add pgsql --- .../data/pg-primary-conf/postgresql.conf | 3 +- pmm_qa/postgresql/tasks/run_load_pgsql.yml | 37 +++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) create mode 100644 pmm_qa/postgresql/tasks/run_load_pgsql.yml diff --git a/pmm_qa/postgresql/data/pg-primary-conf/postgresql.conf b/pmm_qa/postgresql/data/pg-primary-conf/postgresql.conf index d8fc5ef9..6d581e09 100644 --- a/pmm_qa/postgresql/data/pg-primary-conf/postgresql.conf +++ b/pmm_qa/postgresql/data/pg-primary-conf/postgresql.conf @@ -1,4 +1,5 @@ wal_level = replica max_wal_senders = 10 wal_keep_size = 64MB -hot_standby = on \ No newline at end of file +hot_standby = on +listen_addresses = '*' \ No newline at end of file diff --git a/pmm_qa/postgresql/tasks/run_load_pgsql.yml b/pmm_qa/postgresql/tasks/run_load_pgsql.yml new file mode 100644 index 00000000..3cbcd6cc --- /dev/null +++ b/pmm_qa/postgresql/tasks/run_load_pgsql.yml @@ -0,0 +1,37 @@ +# This ansible file run load on docker container of pgsql with variable name node_name + +- name: Ensure pgbench is installed (Debian-based container) + community.docker.docker_container_exec: + container: "{{ node_name }}" + user: root + command: apt-get update && apt-get install -y postgresql-contrib + when: > + "'Debian' in (lookup('community.docker.docker_container_info', container_name).container.Config.Image | default('')).title()" + +- name: Ensure pgbench is installed (RHEL-based container) + community.docker.docker_container_exec: + container: "{{ node_name }}" + user: root + command: microdnf install -y postgresql-contrib + when: > + "'Centos' in (lookup('community.docker.docker_container_info', container_name).container.Config.Image | default('')).title() or + 'Redhat' in (lookup('community.docker.docker_container_info', container_name).container.Config.Image | default('')).title()" + +- name: Initialize pgbench database + community.docker.docker_container_exec: + container: "{{ node_name }}" + user: postgres + command: > + pgbench -i -s {{ pgbench_scale }} {{ pgbench_db }} + +- name: Run pgbench benchmark + community.docker.docker_container_exec: + container: "{{ node_name }}" + user: postgres + command: > + pgbench -c {{ pgbench_clients }} -T {{ pgbench_time }} {{ pgbench_db }} + register: pgbench_result + +- name: Print pgbench results + debug: + var: pgbench_result.stdout_lines \ No newline at end of file From cd5783e871f922cf3a5a4e8b70320c1b338c2b5e Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 12:35:03 +0200 Subject: [PATCH 067/340] PMM-7: Add pgsql --- pmm_qa/postgresql/data/pg-primary-conf/postgresql.conf | 3 ++- pmm_qa/postgresql/postgresql-setup.yml | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/pmm_qa/postgresql/data/pg-primary-conf/postgresql.conf b/pmm_qa/postgresql/data/pg-primary-conf/postgresql.conf index 6d581e09..7916060a 100644 --- a/pmm_qa/postgresql/data/pg-primary-conf/postgresql.conf +++ b/pmm_qa/postgresql/data/pg-primary-conf/postgresql.conf @@ -2,4 +2,5 @@ wal_level = replica max_wal_senders = 10 wal_keep_size = 64MB hot_standby = on -listen_addresses = '*' \ No newline at end of file +listen_addresses = '*' +hba_file = '/etc/postgresql/pg_hba.conf' \ No newline at end of file diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index 1495ebc1..e2cca53d 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -120,3 +120,8 @@ container: "pgsql_pmm_{{ pg_version }}_1" user: postgres command: psql -c "SELECT * FROM pg_stat_replication;" + + - name: Run load on primary node. + include_tasks: ./tasks/run_load_pgsql.yml + vars: + node_name: "pgsql_pmm_{{ pg_version }}_1" \ No newline at end of file From 46fed1419e6a368f6ffd407035ae70c2adadfd20 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 12:37:11 +0200 Subject: [PATCH 068/340] PMM-7: Add pgsql --- pmm_qa/postgresql/data/pg-primary-conf/pg_hba.conf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pmm_qa/postgresql/data/pg-primary-conf/pg_hba.conf b/pmm_qa/postgresql/data/pg-primary-conf/pg_hba.conf index a103c5c7..0b71a3f9 100644 --- a/pmm_qa/postgresql/data/pg-primary-conf/pg_hba.conf +++ b/pmm_qa/postgresql/data/pg-primary-conf/pg_hba.conf @@ -1,3 +1,4 @@ # Allow replication from any host using md5 host replication replicator 0.0.0.0/0 md5 -host all all 0.0.0.0/0 md5 \ No newline at end of file +host all all 0.0.0.0/0 md5 +local all postgres trust \ No newline at end of file From d9ab9a4f93210f1e1b8dea616e3a9934f32e928c Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 12:38:23 +0200 Subject: [PATCH 069/340] PMM-7: Add pgsql --- pmm_qa/postgresql/postgresql-setup.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index e2cca53d..e0d0f3c7 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -112,7 +112,7 @@ container: "pgsql_pmm_{{ pg_version }}_{{ item }}" user: postgres command: > - bash -c "PGPASSWORD={{ replication_password }} pg_basebackup -h pgsql_pmm_{{ pg_version }}_1 -D $PGDATA -U replicator -Fp -Xs -P -R" + bash -c "PGPASSWORD={{ replication_password }} pg_basebackup -h pgsql_pmm_{{ pg_version }}_1 -D $PGDATA -U {{ replication_user }} -Fp -Xs -P -R" loop: "{{ range(2, nodes_count | int + 1) | list }}" - name: Check replication status on primary From 2182c945837936ff8bbd0e13722d113433b3e132 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 12:40:47 +0200 Subject: [PATCH 070/340] PMM-7: Add pgsql --- .../data/pg-primary-conf/{pg_hba.conf => pg_hba.conf.j2} | 2 +- pmm_qa/postgresql/postgresql-setup.yml | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) rename pmm_qa/postgresql/data/pg-primary-conf/{pg_hba.conf => pg_hba.conf.j2} (70%) diff --git a/pmm_qa/postgresql/data/pg-primary-conf/pg_hba.conf b/pmm_qa/postgresql/data/pg-primary-conf/pg_hba.conf.j2 similarity index 70% rename from pmm_qa/postgresql/data/pg-primary-conf/pg_hba.conf rename to pmm_qa/postgresql/data/pg-primary-conf/pg_hba.conf.j2 index 0b71a3f9..c16a4787 100644 --- a/pmm_qa/postgresql/data/pg-primary-conf/pg_hba.conf +++ b/pmm_qa/postgresql/data/pg-primary-conf/pg_hba.conf.j2 @@ -1,4 +1,4 @@ # Allow replication from any host using md5 -host replication replicator 0.0.0.0/0 md5 +host replication {{ replication_user }} 0.0.0.0/0 md5 host all all 0.0.0.0/0 md5 local all postgres trust \ No newline at end of file diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index e0d0f3c7..0cf6f730 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -42,6 +42,11 @@ mode: '0755' loop: "{{ range(1, nodes_count | int + 1) | list }}" + - name: Generate my.cnf for each node + template: + src: ./data/pg-primary-conf/pg_hba.conf.j2 + dest: "{{ data_dir }}/node{{ item }}/pg_hba.conf" + - name: Remove old PostgreSQL primary container community.docker.docker_container: name: "pgsql_pmm_{{ pg_version }}_1" @@ -64,7 +69,7 @@ volumes: - "{{ data_dir }}/node1/data:/var/lib/postgresql/data" - "./data/pg-primary-conf/postgresql.conf:/etc/postgresql/postgresql.conf:ro" - - "./data/pg-primary-conf/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro" + - "./data/pg-primary-conf/pg_hba.conf.j2:/etc/postgresql/pg_hba.conf.j2:ro" command: -c config_file=/etc/postgresql/postgresql.conf ports: - "5432:5432" From b8f5c59031f7e73cd5c42568343a82bef3e8f202 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 12:41:06 +0200 Subject: [PATCH 071/340] PMM-7: Add pgsql --- pmm_qa/postgresql/postgresql-setup.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index 0cf6f730..c95507e0 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -69,7 +69,7 @@ volumes: - "{{ data_dir }}/node1/data:/var/lib/postgresql/data" - "./data/pg-primary-conf/postgresql.conf:/etc/postgresql/postgresql.conf:ro" - - "./data/pg-primary-conf/pg_hba.conf.j2:/etc/postgresql/pg_hba.conf.j2:ro" + - "{{ data_dir }}/node1/pg_hba.conf.j2:/etc/postgresql/pg_hba.conf.j2:ro" command: -c config_file=/etc/postgresql/postgresql.conf ports: - "5432:5432" From 09935d1c63b1f583a375912916dfa4211e588c1b Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 12:41:47 +0200 Subject: [PATCH 072/340] PMM-7: Add pgsql --- pmm_qa/postgresql/postgresql-setup.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index c95507e0..12549f77 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -42,10 +42,10 @@ mode: '0755' loop: "{{ range(1, nodes_count | int + 1) | list }}" - - name: Generate my.cnf for each node + - name: Generate pg_hba.conf for primary node template: src: ./data/pg-primary-conf/pg_hba.conf.j2 - dest: "{{ data_dir }}/node{{ item }}/pg_hba.conf" + dest: "{{ data_dir }}/node1/pg_hba.conf" - name: Remove old PostgreSQL primary container community.docker.docker_container: From 869130c4a0d6b2acdb025fed6cdfdf2b01832a80 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 12:43:36 +0200 Subject: [PATCH 073/340] PMM-7: Add pgsql --- pmm_qa/postgresql/data/pg-primary-conf/pg_hba.conf.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/postgresql/data/pg-primary-conf/pg_hba.conf.j2 b/pmm_qa/postgresql/data/pg-primary-conf/pg_hba.conf.j2 index c16a4787..0886a04c 100644 --- a/pmm_qa/postgresql/data/pg-primary-conf/pg_hba.conf.j2 +++ b/pmm_qa/postgresql/data/pg-primary-conf/pg_hba.conf.j2 @@ -1,4 +1,4 @@ # Allow replication from any host using md5 host replication {{ replication_user }} 0.0.0.0/0 md5 host all all 0.0.0.0/0 md5 -local all postgres trust \ No newline at end of file +local all postgres trust \ No newline at end of file From ee30e625145f5c0d7cda6d4939802bdbcefd81b0 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 12:44:20 +0200 Subject: [PATCH 074/340] PMM-7: Add pgsql --- pmm_qa/postgresql/data/pg-primary-conf/pg_hba.conf.j2 | 1 - pmm_qa/postgresql/postgresql-setup.yml | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/pmm_qa/postgresql/data/pg-primary-conf/pg_hba.conf.j2 b/pmm_qa/postgresql/data/pg-primary-conf/pg_hba.conf.j2 index 0886a04c..7d58f0c8 100644 --- a/pmm_qa/postgresql/data/pg-primary-conf/pg_hba.conf.j2 +++ b/pmm_qa/postgresql/data/pg-primary-conf/pg_hba.conf.j2 @@ -1,4 +1,3 @@ -# Allow replication from any host using md5 host replication {{ replication_user }} 0.0.0.0/0 md5 host all all 0.0.0.0/0 md5 local all postgres trust \ No newline at end of file diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index 12549f77..953cefc8 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -69,7 +69,7 @@ volumes: - "{{ data_dir }}/node1/data:/var/lib/postgresql/data" - "./data/pg-primary-conf/postgresql.conf:/etc/postgresql/postgresql.conf:ro" - - "{{ data_dir }}/node1/pg_hba.conf.j2:/etc/postgresql/pg_hba.conf.j2:ro" + - "{{ data_dir }}/node1/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro" command: -c config_file=/etc/postgresql/postgresql.conf ports: - "5432:5432" From 7c7e0c77b8573f879e03789cba99e7323d71e9a3 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 12:48:30 +0200 Subject: [PATCH 075/340] PMM-7: Add pgsql --- pmm_qa/postgresql/postgresql-setup.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index 953cefc8..6325c87d 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -112,6 +112,13 @@ - "{{ data_dir }}/node{{ item }}/data:/var/lib/postgresql/data" loop: "{{ range(2, nodes_count | int + 1) | list }}" + - name: Clear replica data directory before base backup + community.docker.docker_container_exec: + container: "pgsql_pmm_{{ pg_version }}_{{ item }}" + user: root + command: rm -rf /var/lib/postgresql/data/* + loop: "{{ range(2, nodes_count | int + 1) | list }}" + - name: Perform base backup from primary to replica community.docker.docker_container_exec: container: "pgsql_pmm_{{ pg_version }}_{{ item }}" From 1c0a17b6aea8084df50ce4f40d0a7abdfe62bcbe Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 12:50:03 +0200 Subject: [PATCH 076/340] PMM-7: Add pgsql --- pmm_qa/postgresql/postgresql-setup.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index 6325c87d..be228a86 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -116,7 +116,7 @@ community.docker.docker_container_exec: container: "pgsql_pmm_{{ pg_version }}_{{ item }}" user: root - command: rm -rf /var/lib/postgresql/data/* + command: rm -rf /var/lib/postgresql/data/ loop: "{{ range(2, nodes_count | int + 1) | list }}" - name: Perform base backup from primary to replica From f29dc48c6b50bf67922f3301a9c86877eae1ade8 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 12:51:59 +0200 Subject: [PATCH 077/340] PMM-7: Add pgsql --- pmm_qa/postgresql/postgresql-setup.yml | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index be228a86..1cc9ee1b 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -112,11 +112,22 @@ - "{{ data_dir }}/node{{ item }}/data:/var/lib/postgresql/data" loop: "{{ range(2, nodes_count | int + 1) | list }}" - - name: Clear replica data directory before base backup - community.docker.docker_container_exec: - container: "pgsql_pmm_{{ pg_version }}_{{ item }}" - user: root - command: rm -rf /var/lib/postgresql/data/ + - name: Stop PostgreSQL replica temporarily + community.docker.docker_container: + name: "pgsql_pmm_{{ pg_version }}_{{ item }}" + state: stopped + loop: "{{ range(2, nodes_count | int + 1) | list }}" + + - name: Clear replica data directory + ansible.builtin.shell: > + rm -rf {{ data_dir }}/node{{ item }}/data/* + loop: "{{ range(2, nodes_count | int + 1) | list }}" + become: true + + - name: Stop PostgreSQL replica temporarily + community.docker.docker_container: + name: "pgsql_pmm_{{ pg_version }}_{{ item }}" + state: started loop: "{{ range(2, nodes_count | int + 1) | list }}" - name: Perform base backup from primary to replica From e7089293279abf6a4c71196663e008f8397e8094 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 12:53:15 +0200 Subject: [PATCH 078/340] PMM-7: Add pgsql --- pmm_qa/postgresql/postgresql-setup.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index 1cc9ee1b..17d68b4a 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -124,12 +124,6 @@ loop: "{{ range(2, nodes_count | int + 1) | list }}" become: true - - name: Stop PostgreSQL replica temporarily - community.docker.docker_container: - name: "pgsql_pmm_{{ pg_version }}_{{ item }}" - state: started - loop: "{{ range(2, nodes_count | int + 1) | list }}" - - name: Perform base backup from primary to replica community.docker.docker_container_exec: container: "pgsql_pmm_{{ pg_version }}_{{ item }}" From 43f1499c6efff49368e0cd45e95ea7f315ec25db Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 12:55:35 +0200 Subject: [PATCH 079/340] PMM-7: Add pgsql --- pmm_qa/postgresql/postgresql-setup.yml | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index 17d68b4a..13f6e13d 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -107,22 +107,23 @@ networks: - name: "{{ network_name }}" env: + POSTGRES_INITDB_SKIP: "yes" POSTGRES_PASSWORD: "{{ root_password }}" volumes: - "{{ data_dir }}/node{{ item }}/data:/var/lib/postgresql/data" loop: "{{ range(2, nodes_count | int + 1) | list }}" - - name: Stop PostgreSQL replica temporarily - community.docker.docker_container: - name: "pgsql_pmm_{{ pg_version }}_{{ item }}" - state: stopped - loop: "{{ range(2, nodes_count | int + 1) | list }}" - - - name: Clear replica data directory - ansible.builtin.shell: > - rm -rf {{ data_dir }}/node{{ item }}/data/* - loop: "{{ range(2, nodes_count | int + 1) | list }}" - become: true +# - name: Stop PostgreSQL replica temporarily +# community.docker.docker_container: +# name: "pgsql_pmm_{{ pg_version }}_{{ item }}" +# state: stopped +# loop: "{{ range(2, nodes_count | int + 1) | list }}" +# +# - name: Clear replica data directory +# ansible.builtin.shell: > +# rm -rf {{ data_dir }}/node{{ item }}/data/* +# loop: "{{ range(2, nodes_count | int + 1) | list }}" +# become: true - name: Perform base backup from primary to replica community.docker.docker_container_exec: From 3284f5f7a2ee28b121a838356498bd6c66c7b518 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 12:58:35 +0200 Subject: [PATCH 080/340] PMM-7: Add pgsql --- pmm_qa/postgresql/postgresql-setup.yml | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index 13f6e13d..27b6a98f 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -104,6 +104,10 @@ restart_policy: always state: started recreate: true + command: > + bash -c " + PGPASSWORD={{ replication_password }} pg_basebackup -h pgsql_pmm_{{ pg_version }}_1 -D $PGDATA -U {{ replication_user }} -Fp -Xs -P -R + " networks: - name: "{{ network_name }}" env: @@ -125,13 +129,13 @@ # loop: "{{ range(2, nodes_count | int + 1) | list }}" # become: true - - name: Perform base backup from primary to replica - community.docker.docker_container_exec: - container: "pgsql_pmm_{{ pg_version }}_{{ item }}" - user: postgres - command: > - bash -c "PGPASSWORD={{ replication_password }} pg_basebackup -h pgsql_pmm_{{ pg_version }}_1 -D $PGDATA -U {{ replication_user }} -Fp -Xs -P -R" - loop: "{{ range(2, nodes_count | int + 1) | list }}" +# - name: Perform base backup from primary to replica +# community.docker.docker_container_exec: +# container: "pgsql_pmm_{{ pg_version }}_{{ item }}" +# user: postgres +# command: > +# bash -c "PGPASSWORD={{ replication_password }} pg_basebackup -h pgsql_pmm_{{ pg_version }}_1 -D $PGDATA -U {{ replication_user }} -Fp -Xs -P -R" +# loop: "{{ range(2, nodes_count | int + 1) | list }}" - name: Check replication status on primary community.docker.docker_container_exec: From e56eff4553b6116ce6b008a71f8f9b0980af1989 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 13:00:10 +0200 Subject: [PATCH 081/340] PMM-7: Add pgsql --- pmm_qa/postgresql/tasks/run_load_pgsql.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pmm_qa/postgresql/tasks/run_load_pgsql.yml b/pmm_qa/postgresql/tasks/run_load_pgsql.yml index 3cbcd6cc..e0e84ad3 100644 --- a/pmm_qa/postgresql/tasks/run_load_pgsql.yml +++ b/pmm_qa/postgresql/tasks/run_load_pgsql.yml @@ -5,8 +5,8 @@ container: "{{ node_name }}" user: root command: apt-get update && apt-get install -y postgresql-contrib - when: > - "'Debian' in (lookup('community.docker.docker_container_info', container_name).container.Config.Image | default('')).title()" + when: > + "'Debian' in (lookup('community.docker.docker_container_info', container_name).container.Config.Image | default('')).title()" - name: Ensure pgbench is installed (RHEL-based container) community.docker.docker_container_exec: From bd984fa3ae4b35c2145ffca84373d7060fed9a3e Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 13:02:03 +0200 Subject: [PATCH 082/340] PMM-7: Add pgsql --- pmm_qa/postgresql/tasks/run_load_pgsql.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pmm_qa/postgresql/tasks/run_load_pgsql.yml b/pmm_qa/postgresql/tasks/run_load_pgsql.yml index e0e84ad3..7cce1b73 100644 --- a/pmm_qa/postgresql/tasks/run_load_pgsql.yml +++ b/pmm_qa/postgresql/tasks/run_load_pgsql.yml @@ -4,7 +4,11 @@ community.docker.docker_container_exec: container: "{{ node_name }}" user: root - command: apt-get update && apt-get install -y postgresql-contrib + command: > + /bin/sh -c " + apt-get update && + apt-get install -y postgresql-contrib + " when: > "'Debian' in (lookup('community.docker.docker_container_info', container_name).container.Config.Image | default('')).title()" From d815a1eb50bf9641977fd3431f3defa91e590f53 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 13:05:18 +0200 Subject: [PATCH 083/340] PMM-7: Add pgsql --- pmm_qa/postgresql/tasks/run_load_pgsql.yml | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/pmm_qa/postgresql/tasks/run_load_pgsql.yml b/pmm_qa/postgresql/tasks/run_load_pgsql.yml index 7cce1b73..c59d4c6d 100644 --- a/pmm_qa/postgresql/tasks/run_load_pgsql.yml +++ b/pmm_qa/postgresql/tasks/run_load_pgsql.yml @@ -1,5 +1,16 @@ # This ansible file run load on docker container of pgsql with variable name node_name +- name: Set distro family (debian/rhel) + set_fact: + distro_family: >- + {{ + ( + 'debian' if 'debian' in container_os_info.stdout | lower else + 'rhel' if 'rhel' in container_os_info.stdout | lower or 'centos' in container_os_info.stdout | lower or 'fedora' in container_os_info.stdout | lower + else 'unknown' + ) | trim + }} + - name: Ensure pgbench is installed (Debian-based container) community.docker.docker_container_exec: container: "{{ node_name }}" @@ -9,17 +20,14 @@ apt-get update && apt-get install -y postgresql-contrib " - when: > - "'Debian' in (lookup('community.docker.docker_container_info', container_name).container.Config.Image | default('')).title()" + when: distro_family == "debian" - name: Ensure pgbench is installed (RHEL-based container) community.docker.docker_container_exec: container: "{{ node_name }}" user: root command: microdnf install -y postgresql-contrib - when: > - "'Centos' in (lookup('community.docker.docker_container_info', container_name).container.Config.Image | default('')).title() or - 'Redhat' in (lookup('community.docker.docker_container_info', container_name).container.Config.Image | default('')).title()" + when: distro_family == "rhel" - name: Initialize pgbench database community.docker.docker_container_exec: From 3443753e559b836f54ce04c92475171a59621ff6 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 13:06:17 +0200 Subject: [PATCH 084/340] PMM-7: Add pgsql --- pmm_qa/postgresql/tasks/run_load_pgsql.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pmm_qa/postgresql/tasks/run_load_pgsql.yml b/pmm_qa/postgresql/tasks/run_load_pgsql.yml index c59d4c6d..0c6f0852 100644 --- a/pmm_qa/postgresql/tasks/run_load_pgsql.yml +++ b/pmm_qa/postgresql/tasks/run_load_pgsql.yml @@ -1,5 +1,11 @@ # This ansible file run load on docker container of pgsql with variable name node_name +- name: Detect OS inside the container + community.docker.docker_container_exec: + container: "{{ node_name }}" + command: cat /etc/os-release + register: container_os_info + - name: Set distro family (debian/rhel) set_fact: distro_family: >- From 5a6c7999ebf120c5b469f1910dc3d17477d9813b Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 13:08:09 +0200 Subject: [PATCH 085/340] PMM-7: Add pgsql --- pmm_qa/postgresql/postgresql-setup.yml | 26 +++++--------------------- 1 file changed, 5 insertions(+), 21 deletions(-) diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index 27b6a98f..f46696fa 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -117,26 +117,6 @@ - "{{ data_dir }}/node{{ item }}/data:/var/lib/postgresql/data" loop: "{{ range(2, nodes_count | int + 1) | list }}" -# - name: Stop PostgreSQL replica temporarily -# community.docker.docker_container: -# name: "pgsql_pmm_{{ pg_version }}_{{ item }}" -# state: stopped -# loop: "{{ range(2, nodes_count | int + 1) | list }}" -# -# - name: Clear replica data directory -# ansible.builtin.shell: > -# rm -rf {{ data_dir }}/node{{ item }}/data/* -# loop: "{{ range(2, nodes_count | int + 1) | list }}" -# become: true - -# - name: Perform base backup from primary to replica -# community.docker.docker_container_exec: -# container: "pgsql_pmm_{{ pg_version }}_{{ item }}" -# user: postgres -# command: > -# bash -c "PGPASSWORD={{ replication_password }} pg_basebackup -h pgsql_pmm_{{ pg_version }}_1 -D $PGDATA -U {{ replication_user }} -Fp -Xs -P -R" -# loop: "{{ range(2, nodes_count | int + 1) | list }}" - - name: Check replication status on primary community.docker.docker_container_exec: container: "pgsql_pmm_{{ pg_version }}_1" @@ -146,4 +126,8 @@ - name: Run load on primary node. include_tasks: ./tasks/run_load_pgsql.yml vars: - node_name: "pgsql_pmm_{{ pg_version }}_1" \ No newline at end of file + node_name: "pgsql_pmm_{{ pg_version }}_1" + pgbench_clients: 10 + pgbench_time: 60 + pgbench_scale: 10 + pgbench_db: postgres \ No newline at end of file From 60cb79312eb86b5ce7927e818a112763bf31de96 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 13:10:53 +0200 Subject: [PATCH 086/340] PMM-7: Add pgsql --- pmm_qa/percona_server/percona-server-setup.yml | 2 +- pmm_qa/postgresql/postgresql-setup.yml | 18 ++++++++++++------ pmm_qa/tasks/add_mysql_to_pmm_server.yml | 2 +- ...lient_centos.yml => install_pmm_client.yml} | 0 4 files changed, 14 insertions(+), 8 deletions(-) rename pmm_qa/tasks/{install_pmm_client_centos.yml => install_pmm_client.yml} (100%) diff --git a/pmm_qa/percona_server/percona-server-setup.yml b/pmm_qa/percona_server/percona-server-setup.yml index b58b2760..a7f4d8c4 100644 --- a/pmm_qa/percona_server/percona-server-setup.yml +++ b/pmm_qa/percona_server/percona-server-setup.yml @@ -91,7 +91,7 @@ when: query_source == "slowlog" - name: Install and add pmm client. - include_tasks: ../tasks/install_pmm_client_centos.yml + include_tasks: ../tasks/install_pmm_client.yml vars: container_name: "ps_pmm_{{ ps_version }}_{{ item }}" loop: "{{ range(1, nodes_count | int + 1) | list }}" diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index f46696fa..6161e5e9 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -117,11 +117,11 @@ - "{{ data_dir }}/node{{ item }}/data:/var/lib/postgresql/data" loop: "{{ range(2, nodes_count | int + 1) | list }}" - - name: Check replication status on primary - community.docker.docker_container_exec: - container: "pgsql_pmm_{{ pg_version }}_1" - user: postgres - command: psql -c "SELECT * FROM pg_stat_replication;" + - name: Install and add pmm client. + include_tasks: ../tasks/install_pmm_client.yml + vars: + container_name: "ps_pmm_{{ ps_version }}_{{ item }}" + loop: "{{ range(1, nodes_count | int + 1) | list }}" - name: Run load on primary node. include_tasks: ./tasks/run_load_pgsql.yml @@ -130,4 +130,10 @@ pgbench_clients: 10 pgbench_time: 60 pgbench_scale: 10 - pgbench_db: postgres \ No newline at end of file + pgbench_db: postgres + + - name: Check replication status on primary + community.docker.docker_container_exec: + container: "pgsql_pmm_{{ pg_version }}_1" + user: postgres + command: psql -c "SELECT * FROM pg_stat_replication;" diff --git a/pmm_qa/tasks/add_mysql_to_pmm_server.yml b/pmm_qa/tasks/add_mysql_to_pmm_server.yml index 1dd6b42d..55d54ebf 100644 --- a/pmm_qa/tasks/add_mysql_to_pmm_server.yml +++ b/pmm_qa/tasks/add_mysql_to_pmm_server.yml @@ -1,5 +1,5 @@ - name: Install and add pmm client - include_tasks: ./install_pmm_client_centos.yml + include_tasks: install_pmm_client.yml - name: Set unique service name include_tasks: ./set_unique_service_name.yml diff --git a/pmm_qa/tasks/install_pmm_client_centos.yml b/pmm_qa/tasks/install_pmm_client.yml similarity index 100% rename from pmm_qa/tasks/install_pmm_client_centos.yml rename to pmm_qa/tasks/install_pmm_client.yml From 78b71c19154d78e85475c283bef4bba3345724dd Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 13:13:38 +0200 Subject: [PATCH 087/340] PMM-7: Add pgsql --- pmm_qa/postgresql/postgresql-setup.yml | 31 +++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index 6161e5e9..a522e705 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -17,9 +17,9 @@ network_name: "pmm-qa" data_dir: "{{ lookup('env', 'HOME') }}/pgsql_cluster_data" # server_id_start: 1 -# pmm_server_ip: "{{ lookup('vars', 'extra_pmm_server_ip', default=lookup('env','PMM_SERVER_IP') | default('127.0.0.1', true) ) }}" -# client_version: "{{ lookup('vars', 'extra_client_version', default=lookup('env','CLIENT_VERSION') | default('3-dev-latest', true) ) }}" -# admin_password: "{{ lookup('vars', 'extra_admin_password', default=lookup('env','ADMIN_PASSWORD') | default('admin', true) ) }}" + pmm_server_ip: "{{ lookup('vars', 'extra_pmm_server_ip', default=lookup('env','PMM_SERVER_IP') | default('127.0.0.1', true) ) }}" + client_version: "{{ lookup('vars', 'extra_client_version', default=lookup('env','CLIENT_VERSION') | default('3-dev-latest', true) ) }}" + admin_password: "{{ lookup('vars', 'extra_admin_password', default=lookup('env','ADMIN_PASSWORD') | default('admin', true) ) }}" # query_source: "{{ lookup('env', 'QUERY_SOURCE') | default('perfschema', true) }}" # metrics_mode: "{{ lookup('env', 'metrics_mode') }}" setup_type: "{{ lookup('env', 'SETUP_TYPE') }}" @@ -123,6 +123,31 @@ container_name: "ps_pmm_{{ ps_version }}_{{ item }}" loop: "{{ range(1, nodes_count | int + 1) | list }}" + - name: Get already connected services to pmm server + community.docker.docker_container_exec: + container: "pgsql_pmm_{{ ps_version }}_1" + command: > + sh -c 'curl --location --insecure -u"admin:{{ admin_password }}" -s --request GET "http://{{ pmm_server_ip }}:{{ '80' if pmm_server_ip is ansible.utils.ipv4 else '8080' }}/v1/management/services" | jq -r ".services[].service_name"' + register: pmm_server_services + + - name: Display already connected services to pmm server + debug: + msg: "{{ pmm_server_services.stdout | split('\n') }}" + + - name: Find out if service is already connected to pmm server + block: + - name: Loop through percona servers + set_fact: + random_service_name_value: "_{{ 9999 | random + 1 }}" + loop: "{{ range(1, nodes_count | int + 1) | list }}" + when: "('pgsql_pmm_' ~ pg_version ~ '_' ~ item) in pmm_server_services.stdout" + + - name: Add service to pmm server + community.docker.docker_container_exec: + container: "pgsql_pmm_{{ ps_version }}_{{ item }}" + command: pmm-admin add postgresql --username=pmm --password=pmm ps_pmm_{{ ps_version }}_{{ item }}{{ random_service_name_value }} --debug 127.0.0.1:5432 + loop: "{{ range(1, nodes_count | int + 1) | list }}" + - name: Run load on primary node. include_tasks: ./tasks/run_load_pgsql.yml vars: From d4b71a1ee24731a314ce361f13144778e4340af9 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 13:14:50 +0200 Subject: [PATCH 088/340] PMM-7: Add pgsql --- pmm_qa/postgresql/postgresql-setup.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index a522e705..3ed5dae7 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -120,12 +120,12 @@ - name: Install and add pmm client. include_tasks: ../tasks/install_pmm_client.yml vars: - container_name: "ps_pmm_{{ ps_version }}_{{ item }}" + container_name: "ps_pmm_{{ pg_version }}_{{ item }}" loop: "{{ range(1, nodes_count | int + 1) | list }}" - name: Get already connected services to pmm server community.docker.docker_container_exec: - container: "pgsql_pmm_{{ ps_version }}_1" + container: "pgsql_pmm_{{ pg_version }}_1" command: > sh -c 'curl --location --insecure -u"admin:{{ admin_password }}" -s --request GET "http://{{ pmm_server_ip }}:{{ '80' if pmm_server_ip is ansible.utils.ipv4 else '8080' }}/v1/management/services" | jq -r ".services[].service_name"' register: pmm_server_services @@ -144,8 +144,8 @@ - name: Add service to pmm server community.docker.docker_container_exec: - container: "pgsql_pmm_{{ ps_version }}_{{ item }}" - command: pmm-admin add postgresql --username=pmm --password=pmm ps_pmm_{{ ps_version }}_{{ item }}{{ random_service_name_value }} --debug 127.0.0.1:5432 + container: "pgsql_pmm_{{ pg_version }}_{{ item }}" + command: pmm-admin add postgresql --username=pmm --password=pmm ps_pmm_{{ pg_version }}_{{ item }}{{ random_service_name_value }} --debug 127.0.0.1:5432 loop: "{{ range(1, nodes_count | int + 1) | list }}" - name: Run load on primary node. From 8a5349356f4184b40334261d2d72781699dc786c Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 13:16:03 +0200 Subject: [PATCH 089/340] PMM-7: Add pgsql --- pmm_qa/postgresql/postgresql-setup.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index 3ed5dae7..3834c272 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -120,7 +120,7 @@ - name: Install and add pmm client. include_tasks: ../tasks/install_pmm_client.yml vars: - container_name: "ps_pmm_{{ pg_version }}_{{ item }}" + container_name: "pgsql_pmm_{{ pg_version }}_{{ item }}" loop: "{{ range(1, nodes_count | int + 1) | list }}" - name: Get already connected services to pmm server From b5ad42ba6d8b4fbe0f395d0e61ef1c9728b37fb0 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 13:17:57 +0200 Subject: [PATCH 090/340] PMM-7: Add pgsql --- pmm_qa/tasks/install_pmm_client.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pmm_qa/tasks/install_pmm_client.yml b/pmm_qa/tasks/install_pmm_client.yml index 839ca87a..532bd399 100644 --- a/pmm_qa/tasks/install_pmm_client.yml +++ b/pmm_qa/tasks/install_pmm_client.yml @@ -52,7 +52,11 @@ - name: Install wget inside Debian-family container community.docker.docker_container_exec: container: "{{ container_name }}" - command: apt-get install -y wget gnupg2 jq + command: > + sh -c ' + apt-get update && + apt-get install -y wget gnupg2 jq + ' user: "root" when: distro_family == "debian" From 6153740321354decba1ffa51cabc89f0875bc285 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 13:21:01 +0200 Subject: [PATCH 091/340] PMM-7: Add pgsql --- pmm_qa/tasks/install_pmm_client.yml | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/pmm_qa/tasks/install_pmm_client.yml b/pmm_qa/tasks/install_pmm_client.yml index 532bd399..cc4f294a 100644 --- a/pmm_qa/tasks/install_pmm_client.yml +++ b/pmm_qa/tasks/install_pmm_client.yml @@ -67,10 +67,17 @@ user: "root" when: distro_family == "rhel" -- name: Download percona release on RHEL based OS +- name: get lsb_release + community.docker.docker_container_exec: + container: "{{ container_name }}" + command: lsb_release -sc + user: "root" + register: lsb_release + +- name: Download percona release on debian based OS get_url: - url: "https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb" - dest: "/percona-release_latest.$(lsb_release -sc)_all.deb" + url: "https://repo.percona.com/apt/percona-release_latest.{{ lsb_release }}_all.deb" + dest: "/percona-release_latest.{{ lsb_release }}_all.deb" when: distro_family == "debian" - name: Download percona release on RHEL based OS From ce1fe963fbdce42c224a6994c4159526c2913c14 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 13:22:14 +0200 Subject: [PATCH 092/340] PMM-7: Add pgsql --- pmm_qa/tasks/install_pmm_client.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pmm_qa/tasks/install_pmm_client.yml b/pmm_qa/tasks/install_pmm_client.yml index cc4f294a..df47bce0 100644 --- a/pmm_qa/tasks/install_pmm_client.yml +++ b/pmm_qa/tasks/install_pmm_client.yml @@ -67,13 +67,17 @@ user: "root" when: distro_family == "rhel" -- name: get lsb_release +- name: Get lsb_release community.docker.docker_container_exec: container: "{{ container_name }}" command: lsb_release -sc user: "root" register: lsb_release +- name: Set lsb_release + set_fact: + lsb_release: lsb_release.stdout + - name: Download percona release on debian based OS get_url: url: "https://repo.percona.com/apt/percona-release_latest.{{ lsb_release }}_all.deb" From a1f2f541e20f1100608458070619a98e43fcc78e Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 13:22:30 +0200 Subject: [PATCH 093/340] PMM-7: Add pgsql --- pmm_qa/tasks/install_pmm_client.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/tasks/install_pmm_client.yml b/pmm_qa/tasks/install_pmm_client.yml index df47bce0..6eca6d35 100644 --- a/pmm_qa/tasks/install_pmm_client.yml +++ b/pmm_qa/tasks/install_pmm_client.yml @@ -72,7 +72,7 @@ container: "{{ container_name }}" command: lsb_release -sc user: "root" - register: lsb_release + register: lsb_release - name: Set lsb_release set_fact: From 03c4335b6fd5c966ce73f0ee8e6a962cf7ef72a6 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 13:24:17 +0200 Subject: [PATCH 094/340] PMM-7: Add pgsql --- pmm_qa/tasks/install_pmm_client.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/tasks/install_pmm_client.yml b/pmm_qa/tasks/install_pmm_client.yml index 6eca6d35..d4c10e6a 100644 --- a/pmm_qa/tasks/install_pmm_client.yml +++ b/pmm_qa/tasks/install_pmm_client.yml @@ -55,7 +55,7 @@ command: > sh -c ' apt-get update && - apt-get install -y wget gnupg2 jq + apt-get install -y wget gnupg2 jq lsb ' user: "root" when: distro_family == "debian" From 1a4239d1e0f81bfc226b096426c94f1a2b109a70 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 13:26:06 +0200 Subject: [PATCH 095/340] PMM-7: Add pgsql --- pmm_qa/tasks/install_pmm_client.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/tasks/install_pmm_client.yml b/pmm_qa/tasks/install_pmm_client.yml index d4c10e6a..c4b430f0 100644 --- a/pmm_qa/tasks/install_pmm_client.yml +++ b/pmm_qa/tasks/install_pmm_client.yml @@ -55,7 +55,7 @@ command: > sh -c ' apt-get update && - apt-get install -y wget gnupg2 jq lsb + apt-get install -y wget gnupg2 jq lsb-base lsb-release ' user: "root" when: distro_family == "debian" From 3f416c1746c4408ff187bb127baa8c7119dc0d56 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 13:27:19 +0200 Subject: [PATCH 096/340] PMM-7: Add pgsql --- pmm_qa/tasks/install_pmm_client.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/tasks/install_pmm_client.yml b/pmm_qa/tasks/install_pmm_client.yml index c4b430f0..c7ed271c 100644 --- a/pmm_qa/tasks/install_pmm_client.yml +++ b/pmm_qa/tasks/install_pmm_client.yml @@ -76,7 +76,7 @@ - name: Set lsb_release set_fact: - lsb_release: lsb_release.stdout + lsb_release: "{{ lsb_release.stdout }}" - name: Download percona release on debian based OS get_url: From eb6b3f5377588f4389c7cd8e64c04abb60723ef7 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 13:32:07 +0200 Subject: [PATCH 097/340] PMM-7: Add pgsql --- pmm_qa/postgresql/data/pg-primary-conf/postgresql.conf | 3 ++- pmm_qa/postgresql/postgresql-setup.yml | 10 +++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/pmm_qa/postgresql/data/pg-primary-conf/postgresql.conf b/pmm_qa/postgresql/data/pg-primary-conf/postgresql.conf index 7916060a..8356b760 100644 --- a/pmm_qa/postgresql/data/pg-primary-conf/postgresql.conf +++ b/pmm_qa/postgresql/data/pg-primary-conf/postgresql.conf @@ -3,4 +3,5 @@ max_wal_senders = 10 wal_keep_size = 64MB hot_standby = on listen_addresses = '*' -hba_file = '/etc/postgresql/pg_hba.conf' \ No newline at end of file +hba_file = '/etc/postgresql/pg_hba.conf' +shared_preload_libraries = 'pg_stat_statements' \ No newline at end of file diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index 3834c272..7b5f7573 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -20,7 +20,7 @@ pmm_server_ip: "{{ lookup('vars', 'extra_pmm_server_ip', default=lookup('env','PMM_SERVER_IP') | default('127.0.0.1', true) ) }}" client_version: "{{ lookup('vars', 'extra_client_version', default=lookup('env','CLIENT_VERSION') | default('3-dev-latest', true) ) }}" admin_password: "{{ lookup('vars', 'extra_admin_password', default=lookup('env','ADMIN_PASSWORD') | default('admin', true) ) }}" -# query_source: "{{ lookup('env', 'QUERY_SOURCE') | default('perfschema', true) }}" + query_source: "{{ lookup('env', 'QUERY_SOURCE') | default('pgstatements', true) }}" # metrics_mode: "{{ lookup('env', 'metrics_mode') }}" setup_type: "{{ lookup('env', 'SETUP_TYPE') }}" # random_service_name_value: "" @@ -117,6 +117,14 @@ - "{{ data_dir }}/node{{ item }}/data:/var/lib/postgresql/data" loop: "{{ range(2, nodes_count | int + 1) | list }}" + - name: Create pg_stat_statements extension if not exists + community.docker.docker_container_exec: + name: "pgsql_pmm_{{ pg_version }}_{{ item }}" + user: postgres + command: > + psql -d {{ db_name | default('postgres') }} -c "CREATE EXTENSION IF NOT EXISTS pg_stat_statements;" + loop: "{{ range(1, nodes_count | int + 1) | list }}" + - name: Install and add pmm client. include_tasks: ../tasks/install_pmm_client.yml vars: From ad2624ff529cc74e1acc27decc9c283f669985b2 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 13:32:57 +0200 Subject: [PATCH 098/340] PMM-7: Add pgsql --- pmm_qa/postgresql/postgresql-setup.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index 7b5f7573..9e2074d1 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -119,7 +119,7 @@ - name: Create pg_stat_statements extension if not exists community.docker.docker_container_exec: - name: "pgsql_pmm_{{ pg_version }}_{{ item }}" + container: "pgsql_pmm_{{ pg_version }}_{{ item }}" user: postgres command: > psql -d {{ db_name | default('postgres') }} -c "CREATE EXTENSION IF NOT EXISTS pg_stat_statements;" From c3c0aa61c8dd92c188a3f9fb68b6d79918b30a0f Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 13:34:57 +0200 Subject: [PATCH 099/340] PMM-7: Add pgsql --- pmm_qa/postgresql/postgresql-setup.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index 9e2074d1..c10e9b4f 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -117,13 +117,13 @@ - "{{ data_dir }}/node{{ item }}/data:/var/lib/postgresql/data" loop: "{{ range(2, nodes_count | int + 1) | list }}" - - name: Create pg_stat_statements extension if not exists - community.docker.docker_container_exec: - container: "pgsql_pmm_{{ pg_version }}_{{ item }}" - user: postgres - command: > - psql -d {{ db_name | default('postgres') }} -c "CREATE EXTENSION IF NOT EXISTS pg_stat_statements;" - loop: "{{ range(1, nodes_count | int + 1) | list }}" +# - name: Create pg_stat_statements extension if not exists +# community.docker.docker_container_exec: +# container: "pgsql_pmm_{{ pg_version }}_{{ item }}" +# user: postgres +# command: > +# psql -d {{ db_name | default('postgres') }} -c "CREATE EXTENSION IF NOT EXISTS pg_stat_statements;" +# loop: "{{ range(1, nodes_count | int + 1) | list }}" - name: Install and add pmm client. include_tasks: ../tasks/install_pmm_client.yml From 5d042b0ceaedc48d7b182aa570f68b471cc65a10 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 13:36:12 +0200 Subject: [PATCH 100/340] PMM-7: Add pgsql --- pmm_qa/postgresql/postgresql-setup.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index c10e9b4f..18a8d128 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -21,7 +21,7 @@ client_version: "{{ lookup('vars', 'extra_client_version', default=lookup('env','CLIENT_VERSION') | default('3-dev-latest', true) ) }}" admin_password: "{{ lookup('vars', 'extra_admin_password', default=lookup('env','ADMIN_PASSWORD') | default('admin', true) ) }}" query_source: "{{ lookup('env', 'QUERY_SOURCE') | default('pgstatements', true) }}" -# metrics_mode: "{{ lookup('env', 'metrics_mode') }}" + metrics_mode: "no" setup_type: "{{ lookup('env', 'SETUP_TYPE') }}" # random_service_name_value: "" From 5ec2c4f2ebb6c4166b95d64d7e8b8765372378cf Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 13:38:11 +0200 Subject: [PATCH 101/340] PMM-7: Add pgsql --- pmm_qa/postgresql/postgresql-setup.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index 18a8d128..355db412 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -106,7 +106,14 @@ recreate: true command: > bash -c " - PGPASSWORD={{ replication_password }} pg_basebackup -h pgsql_pmm_{{ pg_version }}_1 -D $PGDATA -U {{ replication_user }} -Fp -Xs -P -R + PGPASSWORD={{ replication_password }} until pg_basebackup -h pgsql_pmm_{{ pg_version }}_1 -D $PGDATA -U {{ replication_user }} -Fp -Xs -P -R + do + echo 'Waiting for primary to connect...' + sleep 1s + done + echo 'Backup done, starting replica...' + chmod 0700 /var/lib/postgresql/data + postgres " networks: - name: "{{ network_name }}" From fb3f881135f4aa1686341c76f59371ff4665036f Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 13:39:36 +0200 Subject: [PATCH 102/340] PMM-7: Add pgsql --- pmm_qa/postgresql/postgresql-setup.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index 355db412..118709d0 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -106,7 +106,7 @@ recreate: true command: > bash -c " - PGPASSWORD={{ replication_password }} until pg_basebackup -h pgsql_pmm_{{ pg_version }}_1 -D $PGDATA -U {{ replication_user }} -Fp -Xs -P -R + until PGPASSWORD={{ replication_password }} pg_basebackup -h pgsql_pmm_{{ pg_version }}_1 -D $PGDATA -U {{ replication_user }} -Fp -Xs -P -R do echo 'Waiting for primary to connect...' sleep 1s From 8fb6327b3dff976f9e6b8ab3b823d54128875f44 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 13:40:00 +0200 Subject: [PATCH 103/340] PMM-7: Add pgsql --- pmm_qa/postgresql/postgresql-setup.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index 118709d0..eedab658 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -106,7 +106,7 @@ recreate: true command: > bash -c " - until PGPASSWORD={{ replication_password }} pg_basebackup -h pgsql_pmm_{{ pg_version }}_1 -D $PGDATA -U {{ replication_user }} -Fp -Xs -P -R + until pg_basebackup -h pgsql_pmm_{{ pg_version }}_1 -D $PGDATA -U {{ replication_user }} -Fp -Xs -P -R do echo 'Waiting for primary to connect...' sleep 1s @@ -120,6 +120,7 @@ env: POSTGRES_INITDB_SKIP: "yes" POSTGRES_PASSWORD: "{{ root_password }}" + PGPASSWORD: "{{ replication_password }}" volumes: - "{{ data_dir }}/node{{ item }}/data:/var/lib/postgresql/data" loop: "{{ range(2, nodes_count | int + 1) | list }}" From fb6d8e5deb18449d71d504e759a2052908eadd9a Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 13:42:39 +0200 Subject: [PATCH 104/340] PMM-7: Add pgsql --- pmm_qa/postgresql/postgresql-setup.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index eedab658..8654d7fa 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -106,7 +106,7 @@ recreate: true command: > bash -c " - until pg_basebackup -h pgsql_pmm_{{ pg_version }}_1 -D $PGDATA -U {{ replication_user }} -Fp -Xs -P -R + until pg_basebackup --pgdata=/var/lib/postgresql/data -R --slot=replication_slot --host=pgsql_pmm_{{ pg_version }}_1 -U {{ replication_user }} --port=5432 do echo 'Waiting for primary to connect...' sleep 1s From dcb76cb4af423c5acab854c523efab97dbb424e2 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 13:45:52 +0200 Subject: [PATCH 105/340] PMM-7: Add pgsql --- pmm_qa/postgresql/postgresql-setup.yml | 2 +- pmm_qa/tasks/install_pmm_client.yml | 16 +--------------- 2 files changed, 2 insertions(+), 16 deletions(-) diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index 8654d7fa..3907a230 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -106,7 +106,7 @@ recreate: true command: > bash -c " - until pg_basebackup --pgdata=/var/lib/postgresql/data -R --slot=replication_slot --host=pgsql_pmm_{{ pg_version }}_1 -U {{ replication_user }} --port=5432 + until pg_basebackup --pgdata=/var/lib/postgresql/data -R --host=pgsql_pmm_{{ pg_version }}_1 -U {{ replication_user }} --port=5432 do echo 'Waiting for primary to connect...' sleep 1s diff --git a/pmm_qa/tasks/install_pmm_client.yml b/pmm_qa/tasks/install_pmm_client.yml index c7ed271c..7f7e4a4b 100644 --- a/pmm_qa/tasks/install_pmm_client.yml +++ b/pmm_qa/tasks/install_pmm_client.yml @@ -179,20 +179,7 @@ - distro_family == 'rhel' - client_version is match('^3\\.[0-9]+\\.[0-9]+$') -- name: Install tarball PMM client version on Debian-family containers - community.docker.docker_container_exec: - container: "{{ container_name }}" - user: "root" - command: > - sh -c ' - wget -O /pmm-client.tar.gz "{{ client_version }}" && - dpkg -i /pmm-client.deb - ' - when: - - distro_family == 'debian' - - client_version | regex_search('^https?://.*\\.tar\\.gz$') is not none - -- name: Install tarball PMM client version on RHEL-family containers +- name: Install tarball PMM client version community.docker.docker_container_exec: container: "{{ container_name }}" user: "root" @@ -216,7 +203,6 @@ pmm-admin --version ' when: - - distro_family == 'rhel' - client_version | regex_search('^https?://.*\\.tar\\.gz$') is not none - name: Connect pmm client to pmm server using metrics mode From 0e28b22388eb65663a64a6877ccc2cf42f179c0a Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 13:50:00 +0200 Subject: [PATCH 106/340] PMM-7: Add pgsql --- pmm_qa/postgresql/postgresql-setup.yml | 32 +++++++++++++++----------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index 3907a230..de39c75b 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -101,20 +101,9 @@ community.docker.docker_container: name: "pgsql_pmm_{{ pg_version }}_{{ item }}" image: "postgres:{{ pg_version }}-bookworm" - restart_policy: always + restart_policy: "no" state: started - recreate: true - command: > - bash -c " - until pg_basebackup --pgdata=/var/lib/postgresql/data -R --host=pgsql_pmm_{{ pg_version }}_1 -U {{ replication_user }} --port=5432 - do - echo 'Waiting for primary to connect...' - sleep 1s - done - echo 'Backup done, starting replica...' - chmod 0700 /var/lib/postgresql/data - postgres - " + command: sleep infinity networks: - name: "{{ network_name }}" env: @@ -125,6 +114,23 @@ - "{{ data_dir }}/node{{ item }}/data:/var/lib/postgresql/data" loop: "{{ range(2, nodes_count | int + 1) | list }}" + - name: Wipe replica data directory before basebackup + community.docker.docker_container_exec: + container: "pgsql_pmm_{{ pg_version }}_{{ item }}" + user: root + command: rm -rf /var/lib/postgresql/data/* + loop: "{{ range(2, nodes_count | int + 1) | list }}" + + - name: Run pg_basebackup from primary to replica + community.docker.docker_container_exec: + container: "pgsql_pmm_{{ pg_version }}_{{ item }}" + user: postgres + command: > + pg_basebackup --pgdata=/var/lib/postgresql/data -R + --host=pgsql_pmm_{{ pg_version }}_1 --port=5432 + -U {{ replication_user }} + loop: "{{ range(2, nodes_count | int + 1) | list }}" + # - name: Create pg_stat_statements extension if not exists # community.docker.docker_container_exec: # container: "pgsql_pmm_{{ pg_version }}_{{ item }}" From 6fa4c05b084058c0b2d111c74a7948058c0f20f9 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 13:50:52 +0200 Subject: [PATCH 107/340] PMM-7: Add pgsql --- pmm_qa/tasks/install_pmm_client.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/pmm_qa/tasks/install_pmm_client.yml b/pmm_qa/tasks/install_pmm_client.yml index 7f7e4a4b..f4032df5 100644 --- a/pmm_qa/tasks/install_pmm_client.yml +++ b/pmm_qa/tasks/install_pmm_client.yml @@ -193,13 +193,9 @@ mv ${PMM_CLIENT} pmm-client && rm -rf /usr/local/bin/pmm-client && mv -f pmm-client /usr/local/bin && - pushd /usr/local/bin/pmm-client && - ## only setting up all binaries in default path /usr/local/percona/pmm && - bash -x ./install_tarball && + bash -x /usr/local/bin/pmm-client/install_tarball && ln -sf /usr/local/percona/pmm/bin/pmm-admin /usr/local/bin/pmm-admin && ln -sf /usr/local/percona/pmm/bin/pmm-agent /usr/local/bin/pmm-agent && - pwd && - popd && pmm-admin --version ' when: From 1574082e64a88ed49492bffb119d7c925e59a165 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 13:51:35 +0200 Subject: [PATCH 108/340] PMM-7: Add pgsql --- pmm_qa/postgresql/postgresql-setup.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index de39c75b..7b62fb1f 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -124,7 +124,7 @@ - name: Run pg_basebackup from primary to replica community.docker.docker_container_exec: container: "pgsql_pmm_{{ pg_version }}_{{ item }}" - user: postgres + user: root command: > pg_basebackup --pgdata=/var/lib/postgresql/data -R --host=pgsql_pmm_{{ pg_version }}_1 --port=5432 From 6a7f934e82e6bbb57968fc3840aa17e168a4c512 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 13:53:09 +0200 Subject: [PATCH 109/340] PMM-7: Add pgsql --- pmm_qa/postgresql/postgresql-setup.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index 7b62fb1f..efe269e4 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -21,7 +21,7 @@ client_version: "{{ lookup('vars', 'extra_client_version', default=lookup('env','CLIENT_VERSION') | default('3-dev-latest', true) ) }}" admin_password: "{{ lookup('vars', 'extra_admin_password', default=lookup('env','ADMIN_PASSWORD') | default('admin', true) ) }}" query_source: "{{ lookup('env', 'QUERY_SOURCE') | default('pgstatements', true) }}" - metrics_mode: "no" + metrics_mode: "auto" setup_type: "{{ lookup('env', 'SETUP_TYPE') }}" # random_service_name_value: "" From da7eec140b57f5638d30af71c2718b147866fcdd Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 13:55:29 +0200 Subject: [PATCH 110/340] PMM-7: Add pgsql --- pmm_qa/postgresql/postgresql-setup.yml | 53 ++++++++++++++------------ 1 file changed, 29 insertions(+), 24 deletions(-) diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index efe269e4..c4da8be7 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -23,7 +23,7 @@ query_source: "{{ lookup('env', 'QUERY_SOURCE') | default('pgstatements', true) }}" metrics_mode: "auto" setup_type: "{{ lookup('env', 'SETUP_TYPE') }}" -# random_service_name_value: "" + random_service_name_value: "" tasks: - name: Create Docker network @@ -97,22 +97,22 @@ loop: "{{ range(2, nodes_count | int + 1) | list }}" ignore_errors: yes - - name: Start PostgreSQL replica container - community.docker.docker_container: - name: "pgsql_pmm_{{ pg_version }}_{{ item }}" - image: "postgres:{{ pg_version }}-bookworm" - restart_policy: "no" - state: started - command: sleep infinity - networks: - - name: "{{ network_name }}" - env: - POSTGRES_INITDB_SKIP: "yes" - POSTGRES_PASSWORD: "{{ root_password }}" - PGPASSWORD: "{{ replication_password }}" - volumes: - - "{{ data_dir }}/node{{ item }}/data:/var/lib/postgresql/data" - loop: "{{ range(2, nodes_count | int + 1) | list }}" +# - name: Start PostgreSQL replica container +# community.docker.docker_container: +# name: "pgsql_pmm_{{ pg_version }}_{{ item }}" +# image: "postgres:{{ pg_version }}-bookworm" +# restart_policy: "no" +# state: started +# command: sleep infinity +# networks: +# - name: "{{ network_name }}" +# env: +# POSTGRES_INITDB_SKIP: "yes" +# POSTGRES_PASSWORD: "{{ root_password }}" +# PGPASSWORD: "{{ replication_password }}" +# volumes: +# - "{{ data_dir }}/node{{ item }}/data:/var/lib/postgresql/data" +# loop: "{{ range(2, nodes_count | int + 1) | list }}" - name: Wipe replica data directory before basebackup community.docker.docker_container_exec: @@ -131,13 +131,18 @@ -U {{ replication_user }} loop: "{{ range(2, nodes_count | int + 1) | list }}" -# - name: Create pg_stat_statements extension if not exists -# community.docker.docker_container_exec: -# container: "pgsql_pmm_{{ pg_version }}_{{ item }}" -# user: postgres -# command: > -# psql -d {{ db_name | default('postgres') }} -c "CREATE EXTENSION IF NOT EXISTS pg_stat_statements;" -# loop: "{{ range(1, nodes_count | int + 1) | list }}" + - name: Start replica with PostgreSQL server + community.docker.docker_container: + name: "pgsql_pmm_{{ pg_version }}_{{ item }}" + image: "postgres:{{ pg_version }}-bookworm" + state: started + restart_policy: always + env: + POSTGRES_PASSWORD: "{{ root_password }}" + volumes: + - "{{ data_dir }}/node{{ item }}/data:/var/lib/postgresql/data" + networks: + - name: "{{ network_name }}" - name: Install and add pmm client. include_tasks: ../tasks/install_pmm_client.yml From 72614ce3a6d46d97c966918619798c86a042f5d3 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 13:56:36 +0200 Subject: [PATCH 111/340] PMM-7: Add pgsql --- pmm_qa/postgresql/postgresql-setup.yml | 45 +++++++++----------------- 1 file changed, 16 insertions(+), 29 deletions(-) diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index c4da8be7..292ec20e 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -97,22 +97,22 @@ loop: "{{ range(2, nodes_count | int + 1) | list }}" ignore_errors: yes -# - name: Start PostgreSQL replica container -# community.docker.docker_container: -# name: "pgsql_pmm_{{ pg_version }}_{{ item }}" -# image: "postgres:{{ pg_version }}-bookworm" -# restart_policy: "no" -# state: started -# command: sleep infinity -# networks: -# - name: "{{ network_name }}" -# env: -# POSTGRES_INITDB_SKIP: "yes" -# POSTGRES_PASSWORD: "{{ root_password }}" -# PGPASSWORD: "{{ replication_password }}" -# volumes: -# - "{{ data_dir }}/node{{ item }}/data:/var/lib/postgresql/data" -# loop: "{{ range(2, nodes_count | int + 1) | list }}" + - name: Start PostgreSQL replica container + community.docker.docker_container: + name: "pgsql_pmm_{{ pg_version }}_{{ item }}" + image: "postgres:{{ pg_version }}-bookworm" + restart_policy: "no" + state: started + command: sleep infinity + networks: + - name: "{{ network_name }}" + env: + POSTGRES_INITDB_SKIP: "yes" + POSTGRES_PASSWORD: "{{ root_password }}" + PGPASSWORD: "{{ replication_password }}" + volumes: + - "{{ data_dir }}/node{{ item }}/data:/var/lib/postgresql/data" + loop: "{{ range(2, nodes_count | int + 1) | list }}" - name: Wipe replica data directory before basebackup community.docker.docker_container_exec: @@ -131,19 +131,6 @@ -U {{ replication_user }} loop: "{{ range(2, nodes_count | int + 1) | list }}" - - name: Start replica with PostgreSQL server - community.docker.docker_container: - name: "pgsql_pmm_{{ pg_version }}_{{ item }}" - image: "postgres:{{ pg_version }}-bookworm" - state: started - restart_policy: always - env: - POSTGRES_PASSWORD: "{{ root_password }}" - volumes: - - "{{ data_dir }}/node{{ item }}/data:/var/lib/postgresql/data" - networks: - - name: "{{ network_name }}" - - name: Install and add pmm client. include_tasks: ../tasks/install_pmm_client.yml vars: From 9b1f1f53eaa5b77a4b1e40db77b0a62ce547991c Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 14:00:05 +0200 Subject: [PATCH 112/340] PMM-7: Add pgsql --- .../data/pg-primary-conf/postgres-replica.conf | 1 + pmm_qa/postgresql/postgresql-setup.yml | 18 +++++++++++++++++- 2 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 pmm_qa/postgresql/data/pg-primary-conf/postgres-replica.conf diff --git a/pmm_qa/postgresql/data/pg-primary-conf/postgres-replica.conf b/pmm_qa/postgresql/data/pg-primary-conf/postgres-replica.conf new file mode 100644 index 00000000..172f4210 --- /dev/null +++ b/pmm_qa/postgresql/data/pg-primary-conf/postgres-replica.conf @@ -0,0 +1 @@ +shared_preload_libraries = 'pg_stat_statements' \ No newline at end of file diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index 292ec20e..5783e729 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -103,7 +103,7 @@ image: "postgres:{{ pg_version }}-bookworm" restart_policy: "no" state: started - command: sleep infinity + command: sleep infinity && -c config_file=/etc/postgresql/postgresql.conf networks: - name: "{{ network_name }}" env: @@ -112,6 +112,7 @@ PGPASSWORD: "{{ replication_password }}" volumes: - "{{ data_dir }}/node{{ item }}/data:/var/lib/postgresql/data" + - "./data/pg-primary-conf/postgresql-replica.conf:/etc/postgresql/postgresql.conf:ro" loop: "{{ range(2, nodes_count | int + 1) | list }}" - name: Wipe replica data directory before basebackup @@ -131,6 +132,21 @@ -U {{ replication_user }} loop: "{{ range(2, nodes_count | int + 1) | list }}" + - name: Create PostgreSQL user 'pmm' with password + community.docker.docker_container_exec: + container: "{{ container_name }}" + user: postgres + command: > + psql -U postgres -d {{ db_name | default('postgres') }} -c + "DO \$\$ + BEGIN + IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = 'pmm') THEN + CREATE ROLE pmm LOGIN PASSWORD 'pmm'; + END IF; + END + \$\$;" + loop: "{{ range(1, nodes_count | int + 1) | list }}" + - name: Install and add pmm client. include_tasks: ../tasks/install_pmm_client.yml vars: From 6ca638ed85eea253bae88446d4082786fce2a7f3 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 14:02:59 +0200 Subject: [PATCH 113/340] PMM-7: Add pgsql --- pmm_qa/postgresql/postgresql-setup.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index 5783e729..0b308896 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -103,7 +103,7 @@ image: "postgres:{{ pg_version }}-bookworm" restart_policy: "no" state: started - command: sleep infinity && -c config_file=/etc/postgresql/postgresql.conf + command: sleep infinity && postgres -c config_file=/etc/postgresql/postgresql.conf networks: - name: "{{ network_name }}" env: From a5eab0527cefc71779207399d929314203bb26c4 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 14:05:01 +0200 Subject: [PATCH 114/340] PMM-7: Add pgsql --- pmm_qa/postgresql/postgresql-setup.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index 0b308896..1e628d6c 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -103,7 +103,11 @@ image: "postgres:{{ pg_version }}-bookworm" restart_policy: "no" state: started - command: sleep infinity && postgres -c config_file=/etc/postgresql/postgresql.conf + command: > + bash -c " + sleep infinity && + postgres -c config_file=/etc/postgresql/postgresql.conf + " networks: - name: "{{ network_name }}" env: From fc63082526bb399f4b00d912e88229f4dbb95a56 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 14:05:16 +0200 Subject: [PATCH 115/340] PMM-7: Add pgsql --- pmm_qa/postgresql/postgresql-setup.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index 1e628d6c..3a09c3a9 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -105,8 +105,8 @@ state: started command: > bash -c " - sleep infinity && - postgres -c config_file=/etc/postgresql/postgresql.conf + postgres -c config_file=/etc/postgresql/postgresql.conf && + sleep infinity " networks: - name: "{{ network_name }}" From ca8c0ef3dbcf3201c35acdbe2315c9427c0696e6 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 14:06:56 +0200 Subject: [PATCH 116/340] PMM-7: Add pgsql --- pmm_qa/postgresql/postgresql-setup.yml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index 3a09c3a9..95ba5488 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -103,17 +103,12 @@ image: "postgres:{{ pg_version }}-bookworm" restart_policy: "no" state: started - command: > - bash -c " - postgres -c config_file=/etc/postgresql/postgresql.conf && - sleep infinity - " + command: sleep infinity networks: - name: "{{ network_name }}" env: POSTGRES_INITDB_SKIP: "yes" POSTGRES_PASSWORD: "{{ root_password }}" - PGPASSWORD: "{{ replication_password }}" volumes: - "{{ data_dir }}/node{{ item }}/data:/var/lib/postgresql/data" - "./data/pg-primary-conf/postgresql-replica.conf:/etc/postgresql/postgresql.conf:ro" From 0f576ccff9a4611403f97f17d368b7c3334b4177 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 14:08:44 +0200 Subject: [PATCH 117/340] PMM-7: Add pgsql --- pmm_qa/postgresql/postgresql-setup.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index 95ba5488..90d21441 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -131,6 +131,14 @@ -U {{ replication_user }} loop: "{{ range(2, nodes_count | int + 1) | list }}" + - name: Restart PostgreSQL container with custom command + community.docker.docker_container: + name: "pgsql_pmm_{{ pg_version }}_{{ item }}" + restart: true + state: started + command: -c config_file=/etc/postgresql/postgresql.conf + loop: "{{ range(2, nodes_count | int + 1) | list }}" + - name: Create PostgreSQL user 'pmm' with password community.docker.docker_container_exec: container: "{{ container_name }}" From 4de0d0d31536db93315ddfd4d8dec726ceb33273 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 14:15:13 +0200 Subject: [PATCH 118/340] PMM-7: Add pgsql --- pmm_qa/postgresql/postgresql-setup.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index 90d21441..790f4c64 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -125,10 +125,7 @@ community.docker.docker_container_exec: container: "pgsql_pmm_{{ pg_version }}_{{ item }}" user: root - command: > - pg_basebackup --pgdata=/var/lib/postgresql/data -R - --host=pgsql_pmm_{{ pg_version }}_1 --port=5432 - -U {{ replication_user }} + command: pg_basebackup --pgdata=/var/lib/postgresql/data -R -v -P --host=pgsql_pmm_{{ pg_version }}_1 --port=5432 -U {{ replication_user }} loop: "{{ range(2, nodes_count | int + 1) | list }}" - name: Restart PostgreSQL container with custom command From bae176a0fe6b63fb99310d72276d1e0e2b732d10 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 14:17:33 +0200 Subject: [PATCH 119/340] PMM-7: Add pgsql --- pmm_qa/postgresql/postgresql-setup.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index 790f4c64..beb45f5b 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -125,7 +125,9 @@ community.docker.docker_container_exec: container: "pgsql_pmm_{{ pg_version }}_{{ item }}" user: root - command: pg_basebackup --pgdata=/var/lib/postgresql/data -R -v -P --host=pgsql_pmm_{{ pg_version }}_1 --port=5432 -U {{ replication_user }} + command: > + timeout 60s + pg_basebackup --pgdata=/var/lib/postgresql/data -R -v -P --host=pgsql_pmm_{{ pg_version }}_1 --port=5432 -U {{ replication_user }} loop: "{{ range(2, nodes_count | int + 1) | list }}" - name: Restart PostgreSQL container with custom command From 9e954a55e1840bb3599c3c070b6f8a50f372c7f4 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 14:24:02 +0200 Subject: [PATCH 120/340] PMM-7: Add pgsql --- pmm_qa/postgresql/postgresql-setup.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index beb45f5b..6691675d 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -126,6 +126,7 @@ container: "pgsql_pmm_{{ pg_version }}_{{ item }}" user: root command: > + PGPASSWORD='{{ replication_password }}' timeout 60s pg_basebackup --pgdata=/var/lib/postgresql/data -R -v -P --host=pgsql_pmm_{{ pg_version }}_1 --port=5432 -U {{ replication_user }} loop: "{{ range(2, nodes_count | int + 1) | list }}" From 677ed0e4e951f98ed4002808665027ccdb1ad351 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 14:27:03 +0200 Subject: [PATCH 121/340] PMM-7: Add pgsql --- pmm_qa/postgresql/postgresql-setup.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index 6691675d..347f4707 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -126,9 +126,11 @@ container: "pgsql_pmm_{{ pg_version }}_{{ item }}" user: root command: > - PGPASSWORD='{{ replication_password }}' - timeout 60s - pg_basebackup --pgdata=/var/lib/postgresql/data -R -v -P --host=pgsql_pmm_{{ pg_version }}_1 --port=5432 -U {{ replication_user }} + bash -c " + PGPASSWORD='{{ replication_password }}' + timeout 60s \ + pg_basebackup --pgdata=/var/lib/postgresql/data -R -v -P --host=pgsql_pmm_{{ pg_version }}_1 --port=5432 -U {{ replication_user }} + " loop: "{{ range(2, nodes_count | int + 1) | list }}" - name: Restart PostgreSQL container with custom command From b9d598ddc119b08f640ce08128fc277a108c15b9 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 14:30:13 +0200 Subject: [PATCH 122/340] PMM-7: Add pgsql --- pmm_qa/postgresql/postgresql-setup.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index 347f4707..0d17380c 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -127,9 +127,10 @@ user: root command: > bash -c " - PGPASSWORD='{{ replication_password }}' timeout 60s \ - pg_basebackup --pgdata=/var/lib/postgresql/data -R -v -P --host=pgsql_pmm_{{ pg_version }}_1 --port=5432 -U {{ replication_user }} + PGPASSWORD='{{ replication_password }}' \ + pg_basebackup --pgdata=/var/lib/postgresql/data -R -v -P \ + --host=pgsql_pmm_{{ pg_version }}_1 --port=5432 -U {{ replication_user }} " loop: "{{ range(2, nodes_count | int + 1) | list }}" From b3deee059a3b81171f5786a90054ac95a5962c4d Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 14:32:05 +0200 Subject: [PATCH 123/340] PMM-7: Add pgsql --- pmm_qa/postgresql/postgresql-setup.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index 0d17380c..ecaa8a8e 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -127,8 +127,8 @@ user: root command: > bash -c " + export PGPASSWORD='{{ replication_password }}' && \ timeout 60s \ - PGPASSWORD='{{ replication_password }}' \ pg_basebackup --pgdata=/var/lib/postgresql/data -R -v -P \ --host=pgsql_pmm_{{ pg_version }}_1 --port=5432 -U {{ replication_user }} " From 34c72e8a8471352f6a4a777b3874f061a83a0eeb Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 14:33:17 +0200 Subject: [PATCH 124/340] PMM-7: Add pgsql --- pmm_qa/postgresql/postgresql-setup.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index ecaa8a8e..235424f6 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -144,7 +144,7 @@ - name: Create PostgreSQL user 'pmm' with password community.docker.docker_container_exec: - container: "{{ container_name }}" + name: "pgsql_pmm_{{ pg_version }}_{{ item }}" user: postgres command: > psql -U postgres -d {{ db_name | default('postgres') }} -c From 31434a2031a4ef008aa975ac4a386d760a95fdf5 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 14:34:30 +0200 Subject: [PATCH 125/340] PMM-7: Add pgsql --- pmm_qa/postgresql/postgresql-setup.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index 235424f6..28bc5f90 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -144,7 +144,7 @@ - name: Create PostgreSQL user 'pmm' with password community.docker.docker_container_exec: - name: "pgsql_pmm_{{ pg_version }}_{{ item }}" + container: "pgsql_pmm_{{ pg_version }}_{{ item }}" user: postgres command: > psql -U postgres -d {{ db_name | default('postgres') }} -c From 16a2a01a932c2ffbe64a3925d9d1ff3407f76dac Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 14:36:02 +0200 Subject: [PATCH 126/340] PMM-7: Add pgsql --- pmm_qa/postgresql/postgresql-setup.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index 28bc5f90..130054ba 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -140,6 +140,14 @@ restart: true state: started command: -c config_file=/etc/postgresql/postgresql.conf + networks: + - name: "{{ network_name }}" + env: + POSTGRES_INITDB_SKIP: "yes" + POSTGRES_PASSWORD: "{{ root_password }}" + volumes: + - "{{ data_dir }}/node{{ item }}/data:/var/lib/postgresql/data" + - "./data/pg-primary-conf/postgresql-replica.conf:/etc/postgresql/postgresql.conf:ro" loop: "{{ range(2, nodes_count | int + 1) | list }}" - name: Create PostgreSQL user 'pmm' with password From 50ab9a96be0de266afd51d9da63b9b36719da5d5 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 14:36:11 +0200 Subject: [PATCH 127/340] PMM-7: Add pgsql --- pmm_qa/postgresql/postgresql-setup.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index 130054ba..d8f952fc 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -143,7 +143,6 @@ networks: - name: "{{ network_name }}" env: - POSTGRES_INITDB_SKIP: "yes" POSTGRES_PASSWORD: "{{ root_password }}" volumes: - "{{ data_dir }}/node{{ item }}/data:/var/lib/postgresql/data" From 481e91990ab3d2758a79280f93fe54cd2747ae20 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 14:38:25 +0200 Subject: [PATCH 128/340] PMM-7: Add pgsql --- pmm_qa/postgresql/postgresql-setup.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index d8f952fc..d00d48a2 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -139,7 +139,7 @@ name: "pgsql_pmm_{{ pg_version }}_{{ item }}" restart: true state: started - command: -c config_file=/etc/postgresql/postgresql.conf +# command: -c config_file=/etc/postgresql/postgresql.conf networks: - name: "{{ network_name }}" env: From 442deef5baa4875e58110b969612fe8fb283f327 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 14:41:31 +0200 Subject: [PATCH 129/340] PMM-7: Add pgsql --- .../postgresql/data/{pg-primary-conf => }/postgres-replica.conf | 0 pmm_qa/postgresql/postgresql-setup.yml | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename pmm_qa/postgresql/data/{pg-primary-conf => }/postgres-replica.conf (100%) diff --git a/pmm_qa/postgresql/data/pg-primary-conf/postgres-replica.conf b/pmm_qa/postgresql/data/postgres-replica.conf similarity index 100% rename from pmm_qa/postgresql/data/pg-primary-conf/postgres-replica.conf rename to pmm_qa/postgresql/data/postgres-replica.conf diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index d00d48a2..fe97273f 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -146,7 +146,7 @@ POSTGRES_PASSWORD: "{{ root_password }}" volumes: - "{{ data_dir }}/node{{ item }}/data:/var/lib/postgresql/data" - - "./data/pg-primary-conf/postgresql-replica.conf:/etc/postgresql/postgresql.conf:ro" + - "./data/postgres-replica.conf:/etc/postgresql/postgresql.conf:ro" loop: "{{ range(2, nodes_count | int + 1) | list }}" - name: Create PostgreSQL user 'pmm' with password From 5cc107ff368d5a03488fc0f6d27c3517e6f7a291 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 14:42:52 +0200 Subject: [PATCH 130/340] PMM-7: Add pgsql --- pmm_qa/postgresql/postgresql-setup.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index fe97273f..0888cb31 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -139,7 +139,7 @@ name: "pgsql_pmm_{{ pg_version }}_{{ item }}" restart: true state: started -# command: -c config_file=/etc/postgresql/postgresql.conf + command: -c config_file=/etc/postgresql/postgresql.conf networks: - name: "{{ network_name }}" env: From 9008c7fe9adf8672f8b6e14bf398b2df42736ec3 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 14:45:02 +0200 Subject: [PATCH 131/340] PMM-7: Add pgsql --- pmm_qa/postgresql/postgresql-setup.yml | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index 0888cb31..3953a2bf 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -154,14 +154,16 @@ container: "pgsql_pmm_{{ pg_version }}_{{ item }}" user: postgres command: > - psql -U postgres -d {{ db_name | default('postgres') }} -c - "DO \$\$ - BEGIN - IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = 'pmm') THEN - CREATE ROLE pmm LOGIN PASSWORD 'pmm'; - END IF; - END - \$\$;" + bash -c ' + psql -U postgres -d {{ db_name | default("postgres") }} -c " + DO \$$ + BEGIN + IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = '\''pmm'\'') THEN + CREATE ROLE pmm LOGIN PASSWORD '\''pmm'\''; + END IF; + END + \$$;" + ' loop: "{{ range(1, nodes_count | int + 1) | list }}" - name: Install and add pmm client. From 1cdacdf349255660c888390f789db48b03502c09 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 14:50:51 +0200 Subject: [PATCH 132/340] PMM-7: Add pgsql --- pmm_qa/postgresql/postgresql-setup.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index 3953a2bf..b2016e26 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -166,6 +166,16 @@ ' loop: "{{ range(1, nodes_count | int + 1) | list }}" + - name: Create pg_stat_statements extension + community.docker.docker_container_exec: + container: "pgsql_pmm_{{ pg_version }}_{{ item }}" + user: postgres + command: > + psql -U postgres -d 'postgres' -c " + CREATE EXTENSION IF NOT EXISTS pg_stat_statements; + " + loop: "{{ range(1, nodes_count | int + 1) | list }}" + - name: Install and add pmm client. include_tasks: ../tasks/install_pmm_client.yml vars: From 12c31e98240b91a21c01ea6781b7e8639a4e06f8 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 14:53:32 +0200 Subject: [PATCH 133/340] PMM-7: Add pgsql --- pmm_qa/postgresql/postgresql-setup.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index b2016e26..1bb146e2 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -168,13 +168,12 @@ - name: Create pg_stat_statements extension community.docker.docker_container_exec: - container: "pgsql_pmm_{{ pg_version }}_{{ item }}" + container: "pgsql_pmm_{{ pg_version }}_1" user: postgres command: > psql -U postgres -d 'postgres' -c " CREATE EXTENSION IF NOT EXISTS pg_stat_statements; " - loop: "{{ range(1, nodes_count | int + 1) | list }}" - name: Install and add pmm client. include_tasks: ../tasks/install_pmm_client.yml From 752426a3b4e397550a25682e3f2770766e745bea Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 14:57:11 +0200 Subject: [PATCH 134/340] PMM-7: Add pgsql --- pmm_qa/postgresql/postgresql-setup.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index 1bb146e2..2e7dbd5c 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -203,7 +203,7 @@ - name: Add service to pmm server community.docker.docker_container_exec: container: "pgsql_pmm_{{ pg_version }}_{{ item }}" - command: pmm-admin add postgresql --username=pmm --password=pmm ps_pmm_{{ pg_version }}_{{ item }}{{ random_service_name_value }} --debug 127.0.0.1:5432 + command: pmm-admin add postgresql --username=pmm --password=pmm pgsql_pmm_{{ pg_version }}_{{ item }}{{ random_service_name_value }} --debug 127.0.0.1:5432 loop: "{{ range(1, nodes_count | int + 1) | list }}" - name: Run load on primary node. From 3807955a6f83ab0f13acde4d0bacb90c5ca68b3f Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 15:04:44 +0200 Subject: [PATCH 135/340] PMM-7: Add pgsql --- pmm_qa/postgresql/postgresql-setup.yml | 1 - pmm_qa/postgresql/tasks/run_load_pgsql.yml | 11 +++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index 2e7dbd5c..ce02b90e 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -213,7 +213,6 @@ pgbench_clients: 10 pgbench_time: 60 pgbench_scale: 10 - pgbench_db: postgres - name: Check replication status on primary community.docker.docker_container_exec: diff --git a/pmm_qa/postgresql/tasks/run_load_pgsql.yml b/pmm_qa/postgresql/tasks/run_load_pgsql.yml index 0c6f0852..c5a325b0 100644 --- a/pmm_qa/postgresql/tasks/run_load_pgsql.yml +++ b/pmm_qa/postgresql/tasks/run_load_pgsql.yml @@ -17,6 +17,13 @@ ) | trim }} +- name: Create custom database for pgbench + community.docker.docker_container_exec: + container: "{{ node_name }}" + user: postgres + command: > + psql -U postgres -c "CREATE DATABASE pgbench;" + - name: Ensure pgbench is installed (Debian-based container) community.docker.docker_container_exec: container: "{{ node_name }}" @@ -40,14 +47,14 @@ container: "{{ node_name }}" user: postgres command: > - pgbench -i -s {{ pgbench_scale }} {{ pgbench_db }} + pgbench -i -s {{ pgbench_scale }} pgbench - name: Run pgbench benchmark community.docker.docker_container_exec: container: "{{ node_name }}" user: postgres command: > - pgbench -c {{ pgbench_clients }} -T {{ pgbench_time }} {{ pgbench_db }} + pgbench -c {{ pgbench_clients }} -T {{ pgbench_time }} pgbench register: pgbench_result - name: Print pgbench results From 26ecacb42409e9ef278c688ee5b759a9e6160a96 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 15:13:47 +0200 Subject: [PATCH 136/340] PMM-7: Add pgsql --- pmm_qa/postgresql/data/sample_data.sql | 14 +++++++++++ pmm_qa/postgresql/tasks/run_load_pgsql.yml | 28 ++++++++++++++++------ 2 files changed, 35 insertions(+), 7 deletions(-) create mode 100644 pmm_qa/postgresql/data/sample_data.sql diff --git a/pmm_qa/postgresql/data/sample_data.sql b/pmm_qa/postgresql/data/sample_data.sql new file mode 100644 index 00000000..82749246 --- /dev/null +++ b/pmm_qa/postgresql/data/sample_data.sql @@ -0,0 +1,14 @@ +-- Populate sample branches +INSERT INTO pgbench_branches (bid, bbalance, bname) +SELECT i, 100000, 'Branch ' || i +FROM generate_series(1, 10) AS i; + +-- Populate sample tellers +INSERT INTO pgbench_tellers (tid, bid, tbalance, tname) +SELECT i, (i % 10) + 1, 10000, 'Teller ' || i +FROM generate_series(1, 50) AS i; + +-- Populate sample accounts +INSERT INTO pgbench_accounts (aid, bid, abalance) +SELECT i, (i % 10) + 1, 1000 +FROM generate_series(1, 10000) AS i; \ No newline at end of file diff --git a/pmm_qa/postgresql/tasks/run_load_pgsql.yml b/pmm_qa/postgresql/tasks/run_load_pgsql.yml index c5a325b0..7100da8c 100644 --- a/pmm_qa/postgresql/tasks/run_load_pgsql.yml +++ b/pmm_qa/postgresql/tasks/run_load_pgsql.yml @@ -17,12 +17,12 @@ ) | trim }} -- name: Create custom database for pgbench - community.docker.docker_container_exec: - container: "{{ node_name }}" - user: postgres - command: > - psql -U postgres -c "CREATE DATABASE pgbench;" +#- name: Create custom database for pgbench +# community.docker.docker_container_exec: +# container: "{{ node_name }}" +# user: postgres +# command: > +# psql -U postgres -c "CREATE DATABASE pgbench;" - name: Ensure pgbench is installed (Debian-based container) community.docker.docker_container_exec: @@ -49,12 +49,26 @@ command: > pgbench -i -s {{ pgbench_scale }} pgbench +- name: Copy populate script into container + community.docker.docker_container_copy_into: + container: "{{ node_name }}" + path: ./data/sample_data.sql + container_path: /populate_sample_data.sql + +- name: Populate pgbench tables with sample data + community.docker.docker_container_exec: + container: "{{ container_name }}" + user: postgres + command: > + psql -U postgres -d pgbench -f /sample_data.sql + - name: Run pgbench benchmark community.docker.docker_container_exec: container: "{{ node_name }}" user: postgres command: > - pgbench -c {{ pgbench_clients }} -T {{ pgbench_time }} pgbench + pgbench -f /tmp/sample_data.sql + -c {{ pgbench_clients }} -T {{ pgbench_time }} pgbench register: pgbench_result - name: Print pgbench results From f3211c101f359848335f37aa2842fc474fc71599 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 15:15:19 +0200 Subject: [PATCH 137/340] PMM-7: Add pgsql --- pmm_qa/postgresql/data/workload.sql | 29 ++++++++++++++++++++++ pmm_qa/postgresql/tasks/run_load_pgsql.yml | 8 +++++- 2 files changed, 36 insertions(+), 1 deletion(-) create mode 100644 pmm_qa/postgresql/data/workload.sql diff --git a/pmm_qa/postgresql/data/workload.sql b/pmm_qa/postgresql/data/workload.sql new file mode 100644 index 00000000..81efcc2b --- /dev/null +++ b/pmm_qa/postgresql/data/workload.sql @@ -0,0 +1,29 @@ +-- begin a transaction +BEGIN; + +-- simulate user account lookup +SELECT abalance +FROM pgbench_accounts +WHERE aid = :aid; + +-- simulate joining branch and teller info +SELECT b.bid, b.bname, t.tid, t.tname +FROM pgbench_branches b +JOIN pgbench_tellers t ON b.bid = t.bid +WHERE b.bid = :bid AND t.tid = :tid; + +-- conditional balance transfer +UPDATE pgbench_accounts +SET abalance = abalance - :delta +WHERE aid = :aid AND abalance >= :delta; + +-- deposit to another account +UPDATE pgbench_accounts +SET abalance = abalance + :delta +WHERE aid = :aid + 1; + +-- insert transaction log +INSERT INTO pgbench_history (tid, bid, aid, delta, mtime) +VALUES (:tid, :bid, :aid, :delta, CURRENT_TIMESTAMP); + +COMMIT; \ No newline at end of file diff --git a/pmm_qa/postgresql/tasks/run_load_pgsql.yml b/pmm_qa/postgresql/tasks/run_load_pgsql.yml index 7100da8c..05883827 100644 --- a/pmm_qa/postgresql/tasks/run_load_pgsql.yml +++ b/pmm_qa/postgresql/tasks/run_load_pgsql.yml @@ -53,7 +53,7 @@ community.docker.docker_container_copy_into: container: "{{ node_name }}" path: ./data/sample_data.sql - container_path: /populate_sample_data.sql + container_path: /sample_data.sql - name: Populate pgbench tables with sample data community.docker.docker_container_exec: @@ -62,6 +62,12 @@ command: > psql -U postgres -d pgbench -f /sample_data.sql +- name: Copy realistic workload SQL into container + community.docker.docker_container_copy_into: + container: "{{ node_name }}" + path: ./data/workload.sql + container_path: /workload.sql + - name: Run pgbench benchmark community.docker.docker_container_exec: container: "{{ node_name }}" From 74bd536d4a997c31b7f45d965b728332424712aa Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 15:16:43 +0200 Subject: [PATCH 138/340] PMM-7: Add pgsql --- pmm_qa/postgresql/tasks/run_load_pgsql.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/postgresql/tasks/run_load_pgsql.yml b/pmm_qa/postgresql/tasks/run_load_pgsql.yml index 05883827..56916801 100644 --- a/pmm_qa/postgresql/tasks/run_load_pgsql.yml +++ b/pmm_qa/postgresql/tasks/run_load_pgsql.yml @@ -74,7 +74,7 @@ user: postgres command: > pgbench -f /tmp/sample_data.sql - -c {{ pgbench_clients }} -T {{ pgbench_time }} pgbench + -c {{ pgbench_clients }} -T {{ pgbench_time }} -f /workload.sql pgbench register: pgbench_result - name: Print pgbench results From 39d070926e4ad1593548cc302fd827f8d355cb97 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 15:17:04 +0200 Subject: [PATCH 139/340] PMM-7: Add pgsql --- pmm_qa/postgresql/tasks/run_load_pgsql.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pmm_qa/postgresql/tasks/run_load_pgsql.yml b/pmm_qa/postgresql/tasks/run_load_pgsql.yml index 56916801..c0c7f29f 100644 --- a/pmm_qa/postgresql/tasks/run_load_pgsql.yml +++ b/pmm_qa/postgresql/tasks/run_load_pgsql.yml @@ -73,8 +73,10 @@ container: "{{ node_name }}" user: postgres command: > - pgbench -f /tmp/sample_data.sql - -c {{ pgbench_clients }} -T {{ pgbench_time }} -f /workload.sql pgbench + pgbench -c {{ pgbench_clients }} + -T {{ pgbench_time }} + -f /workload.sql + pgbench register: pgbench_result - name: Print pgbench results From 2e4d8278df9a49fc58d9eb41ea84ccbce18d3a35 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 15:17:16 +0200 Subject: [PATCH 140/340] PMM-7: Add pgsql --- pmm_qa/postgresql/tasks/run_load_pgsql.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pmm_qa/postgresql/tasks/run_load_pgsql.yml b/pmm_qa/postgresql/tasks/run_load_pgsql.yml index c0c7f29f..3667daa5 100644 --- a/pmm_qa/postgresql/tasks/run_load_pgsql.yml +++ b/pmm_qa/postgresql/tasks/run_load_pgsql.yml @@ -17,12 +17,12 @@ ) | trim }} -#- name: Create custom database for pgbench -# community.docker.docker_container_exec: -# container: "{{ node_name }}" -# user: postgres -# command: > -# psql -U postgres -c "CREATE DATABASE pgbench;" +- name: Create custom database for pgbench + community.docker.docker_container_exec: + container: "{{ node_name }}" + user: postgres + command: > + psql -U postgres -c "CREATE DATABASE pgbench;" - name: Ensure pgbench is installed (Debian-based container) community.docker.docker_container_exec: From 6f5caa58f37aa44261a484de591621c37428112d Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 15:19:34 +0200 Subject: [PATCH 141/340] PMM-7: Add pgsql --- pmm_qa/tasks/install_pmm_client.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pmm_qa/tasks/install_pmm_client.yml b/pmm_qa/tasks/install_pmm_client.yml index f4032df5..6521ac2d 100644 --- a/pmm_qa/tasks/install_pmm_client.yml +++ b/pmm_qa/tasks/install_pmm_client.yml @@ -55,7 +55,7 @@ command: > sh -c ' apt-get update && - apt-get install -y wget gnupg2 jq lsb-base lsb-release + apt-get install -y wget gnupg2 jq lsb-base lsb-release curl ' user: "root" when: distro_family == "debian" @@ -63,7 +63,7 @@ - name: Install wget on RHEL-family containers community.docker.docker_container_exec: container: "{{ container_name }}" - command: microdnf install -y wget gnupg2 jq + command: microdnf install -y wget gnupg2 jq curl user: "root" when: distro_family == "rhel" From d9bcbfc74273f325961c7ad90d225686eb90ee38 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 15:21:29 +0200 Subject: [PATCH 142/340] PMM-7: Add pgsql --- pmm_qa/postgresql/tasks/run_load_pgsql.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/postgresql/tasks/run_load_pgsql.yml b/pmm_qa/postgresql/tasks/run_load_pgsql.yml index 3667daa5..10c8b189 100644 --- a/pmm_qa/postgresql/tasks/run_load_pgsql.yml +++ b/pmm_qa/postgresql/tasks/run_load_pgsql.yml @@ -57,7 +57,7 @@ - name: Populate pgbench tables with sample data community.docker.docker_container_exec: - container: "{{ container_name }}" + container: "{{ node_name }}" user: postgres command: > psql -U postgres -d pgbench -f /sample_data.sql From 379937583c60dafc9a30ca0e81a0d3a7ea754483 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 15:26:26 +0200 Subject: [PATCH 143/340] PMM-7: Add pgsql --- pmm_qa/postgresql/tasks/run_load_pgsql.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/pmm_qa/postgresql/tasks/run_load_pgsql.yml b/pmm_qa/postgresql/tasks/run_load_pgsql.yml index 10c8b189..f4de3665 100644 --- a/pmm_qa/postgresql/tasks/run_load_pgsql.yml +++ b/pmm_qa/postgresql/tasks/run_load_pgsql.yml @@ -73,10 +73,7 @@ container: "{{ node_name }}" user: postgres command: > - pgbench -c {{ pgbench_clients }} - -T {{ pgbench_time }} - -f /workload.sql - pgbench + pgbench -c {{ pgbench_clients }} -T {{ pgbench_time }} -f /workload.sql pgbench register: pgbench_result - name: Print pgbench results From a484f8259c3c5b131d9a250fb68ba6c53fdacc79 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 15:29:26 +0200 Subject: [PATCH 144/340] PMM-7: Add pgsql --- pmm_qa/postgresql/data/workload.sql | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/pmm_qa/postgresql/data/workload.sql b/pmm_qa/postgresql/data/workload.sql index 81efcc2b..74dc9f2f 100644 --- a/pmm_qa/postgresql/data/workload.sql +++ b/pmm_qa/postgresql/data/workload.sql @@ -1,28 +1,27 @@ --- begin a transaction +\set aid random(1, 10000) +\set delta random(-100, 100) +\set bid random(1, 10) +\set tid random(1, 50) + BEGIN; --- simulate user account lookup SELECT abalance -FROM pgbench_accounts -WHERE aid = :aid; + FROM pgbench_accounts + WHERE aid = :aid; --- simulate joining branch and teller info SELECT b.bid, b.bname, t.tid, t.tname -FROM pgbench_branches b -JOIN pgbench_tellers t ON b.bid = t.bid -WHERE b.bid = :bid AND t.tid = :tid; + FROM pgbench_branches b + JOIN pgbench_tellers t ON b.bid = t.bid + WHERE b.bid = :bid AND t.tid = :tid; --- conditional balance transfer UPDATE pgbench_accounts -SET abalance = abalance - :delta -WHERE aid = :aid AND abalance >= :delta; + SET abalance = abalance - :delta + WHERE aid = :aid AND abalance >= :delta; --- deposit to another account UPDATE pgbench_accounts -SET abalance = abalance + :delta -WHERE aid = :aid + 1; + SET abalance = abalance + :delta + WHERE aid = :aid + 1; --- insert transaction log INSERT INTO pgbench_history (tid, bid, aid, delta, mtime) VALUES (:tid, :bid, :aid, :delta, CURRENT_TIMESTAMP); From 8bc5b0c28cea9652593ebfbfc3ed272342aa6cf4 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 15:32:12 +0200 Subject: [PATCH 145/340] PMM-7: Add pgsql --- pmm_qa/postgresql/data/workload.sql | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pmm_qa/postgresql/data/workload.sql b/pmm_qa/postgresql/data/workload.sql index 74dc9f2f..6e0ba0bf 100644 --- a/pmm_qa/postgresql/data/workload.sql +++ b/pmm_qa/postgresql/data/workload.sql @@ -3,6 +3,9 @@ \set bid random(1, 10) \set tid random(1, 50) +ALTER TABLE pgbench_branches ADD COLUMN bname TEXT DEFAULT 'Main Branch'; +ALTER TABLE pgbench_tellers ADD COLUMN tname TEXT DEFAULT 'Teller X'; + BEGIN; SELECT abalance From 2dfd91623a596f429bae58dc74c7f72bd92b7a5f Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 15:34:41 +0200 Subject: [PATCH 146/340] PMM-7: Add pgsql --- pmm_qa/postgresql/data/workload.sql | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/pmm_qa/postgresql/data/workload.sql b/pmm_qa/postgresql/data/workload.sql index 6e0ba0bf..77af92c7 100644 --- a/pmm_qa/postgresql/data/workload.sql +++ b/pmm_qa/postgresql/data/workload.sql @@ -3,8 +3,29 @@ \set bid random(1, 10) \set tid random(1, 50) -ALTER TABLE pgbench_branches ADD COLUMN bname TEXT DEFAULT 'Main Branch'; -ALTER TABLE pgbench_tellers ADD COLUMN tname TEXT DEFAULT 'Teller X'; +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM information_schema.columns + WHERE table_name = 'pgbench_branches' + AND column_name = 'bname' + ) THEN + ALTER TABLE pgbench_branches ADD COLUMN bname TEXT DEFAULT 'Branch'; + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM information_schema.columns + WHERE table_name = 'pgbench_tellers' + AND column_name = 'tname' + ) THEN + ALTER TABLE pgbench_tellers ADD COLUMN tname TEXT DEFAULT 'Teller'; + END IF; +END$$; BEGIN; From 82e01f65a7389872e5910afa1fd8a70a94e47570 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 15:34:56 +0200 Subject: [PATCH 147/340] PMM-7: Add pgsql --- pmm_qa/postgresql/data/sample_data.sql | 26 +++++++++++++++++++++++++- pmm_qa/postgresql/data/workload.sql | 24 ------------------------ 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/pmm_qa/postgresql/data/sample_data.sql b/pmm_qa/postgresql/data/sample_data.sql index 82749246..a1e5c59f 100644 --- a/pmm_qa/postgresql/data/sample_data.sql +++ b/pmm_qa/postgresql/data/sample_data.sql @@ -11,4 +11,28 @@ FROM generate_series(1, 50) AS i; -- Populate sample accounts INSERT INTO pgbench_accounts (aid, bid, abalance) SELECT i, (i % 10) + 1, 1000 -FROM generate_series(1, 10000) AS i; \ No newline at end of file +FROM generate_series(1, 10000) AS i; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM information_schema.columns + WHERE table_name = 'pgbench_branches' + AND column_name = 'bname' + ) THEN + ALTER TABLE pgbench_branches ADD COLUMN bname TEXT DEFAULT 'Branch'; + END IF; +END$$; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM information_schema.columns + WHERE table_name = 'pgbench_tellers' + AND column_name = 'tname' + ) THEN + ALTER TABLE pgbench_tellers ADD COLUMN tname TEXT DEFAULT 'Teller'; + END IF; +END$$; \ No newline at end of file diff --git a/pmm_qa/postgresql/data/workload.sql b/pmm_qa/postgresql/data/workload.sql index 77af92c7..74dc9f2f 100644 --- a/pmm_qa/postgresql/data/workload.sql +++ b/pmm_qa/postgresql/data/workload.sql @@ -3,30 +3,6 @@ \set bid random(1, 10) \set tid random(1, 50) -DO $$ -BEGIN - IF NOT EXISTS ( - SELECT 1 - FROM information_schema.columns - WHERE table_name = 'pgbench_branches' - AND column_name = 'bname' - ) THEN - ALTER TABLE pgbench_branches ADD COLUMN bname TEXT DEFAULT 'Branch'; - END IF; -END$$; - -DO $$ -BEGIN - IF NOT EXISTS ( - SELECT 1 - FROM information_schema.columns - WHERE table_name = 'pgbench_tellers' - AND column_name = 'tname' - ) THEN - ALTER TABLE pgbench_tellers ADD COLUMN tname TEXT DEFAULT 'Teller'; - END IF; -END$$; - BEGIN; SELECT abalance From 7b40c52764e501355ec10a7c9a6c7c7506aa91d6 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 15:41:04 +0200 Subject: [PATCH 148/340] PMM-7: Add pgsql --- pmm_qa/postgresql/tasks/run_load_pgsql.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pmm_qa/postgresql/tasks/run_load_pgsql.yml b/pmm_qa/postgresql/tasks/run_load_pgsql.yml index f4de3665..160068f7 100644 --- a/pmm_qa/postgresql/tasks/run_load_pgsql.yml +++ b/pmm_qa/postgresql/tasks/run_load_pgsql.yml @@ -22,7 +22,13 @@ container: "{{ node_name }}" user: postgres command: > - psql -U postgres -c "CREATE DATABASE pgbench;" + psql -U postgres -c " + CREATE DATABASE pgbench; + GRANT CONNECT ON DATABASE pgbench TO pmm; + GRANT USAGE ON SCHEMA public TO pmm; + GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public TO pmm; + ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO pmm;" + " - name: Ensure pgbench is installed (Debian-based container) community.docker.docker_container_exec: From 694f858784e090b0c1f939038ba999553d069759 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 15:43:37 +0200 Subject: [PATCH 149/340] PMM-7: Add pgsql --- pmm_qa/postgresql/tasks/run_load_pgsql.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pmm_qa/postgresql/tasks/run_load_pgsql.yml b/pmm_qa/postgresql/tasks/run_load_pgsql.yml index 160068f7..18e26947 100644 --- a/pmm_qa/postgresql/tasks/run_load_pgsql.yml +++ b/pmm_qa/postgresql/tasks/run_load_pgsql.yml @@ -22,12 +22,16 @@ container: "{{ node_name }}" user: postgres command: > - psql -U postgres -c " + bash -c " + psql -U postgres -v ON_ERROR_STOP=1 <<'SQL' CREATE DATABASE pgbench; + \connect pgbench GRANT CONNECT ON DATABASE pgbench TO pmm; GRANT USAGE ON SCHEMA public TO pmm; GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public TO pmm; - ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO pmm;" + ALTER DEFAULT PRIVILEGES IN SCHEMA public + GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO pmm; + SQL " - name: Ensure pgbench is installed (Debian-based container) From d119cf76a575d06b4950fb4ae544aaa754a69ab3 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 15:47:00 +0200 Subject: [PATCH 150/340] PMM-7: Add pgsql --- pmm_qa/postgresql/tasks/run_load_pgsql.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/pmm_qa/postgresql/tasks/run_load_pgsql.yml b/pmm_qa/postgresql/tasks/run_load_pgsql.yml index 18e26947..5ee47daf 100644 --- a/pmm_qa/postgresql/tasks/run_load_pgsql.yml +++ b/pmm_qa/postgresql/tasks/run_load_pgsql.yml @@ -23,16 +23,15 @@ user: postgres command: > bash -c " - psql -U postgres -v ON_ERROR_STOP=1 <<'SQL' + echo \" CREATE DATABASE pgbench; - \connect pgbench + \\c pgbench GRANT CONNECT ON DATABASE pgbench TO pmm; GRANT USAGE ON SCHEMA public TO pmm; GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public TO pmm; ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO pmm; - SQL - " + \" | psql -U postgres -v - name: Ensure pgbench is installed (Debian-based container) community.docker.docker_container_exec: From c227fb3c8c70bf9d778722fa7732a0f43107466b Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 15:49:22 +0200 Subject: [PATCH 151/340] PMM-7: Add pgsql --- pmm_qa/postgresql/tasks/run_load_pgsql.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/pmm_qa/postgresql/tasks/run_load_pgsql.yml b/pmm_qa/postgresql/tasks/run_load_pgsql.yml index 5ee47daf..b5691414 100644 --- a/pmm_qa/postgresql/tasks/run_load_pgsql.yml +++ b/pmm_qa/postgresql/tasks/run_load_pgsql.yml @@ -32,6 +32,7 @@ ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO pmm; \" | psql -U postgres -v + " - name: Ensure pgbench is installed (Debian-based container) community.docker.docker_container_exec: From 4cea3f7027f4a7579be3211818f59897dcf024c4 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 15:52:37 +0200 Subject: [PATCH 152/340] PMM-7: Add pgsql --- pmm_qa/postgresql/tasks/run_load_pgsql.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/postgresql/tasks/run_load_pgsql.yml b/pmm_qa/postgresql/tasks/run_load_pgsql.yml index b5691414..b5a4aea9 100644 --- a/pmm_qa/postgresql/tasks/run_load_pgsql.yml +++ b/pmm_qa/postgresql/tasks/run_load_pgsql.yml @@ -31,7 +31,7 @@ GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public TO pmm; ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO pmm; - \" | psql -U postgres -v + \" | psql -U postgres -v ON_ERROR_STOP=1 " - name: Ensure pgbench is installed (Debian-based container) From c49bbdacc84b48203644958cb3c35899594c3010 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 15:56:40 +0200 Subject: [PATCH 153/340] PMM-7: Add pgsql --- pmm_qa/tasks/install_pmm_client.yml | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/pmm_qa/tasks/install_pmm_client.yml b/pmm_qa/tasks/install_pmm_client.yml index 6521ac2d..d02fbff2 100644 --- a/pmm_qa/tasks/install_pmm_client.yml +++ b/pmm_qa/tasks/install_pmm_client.yml @@ -94,7 +94,23 @@ debug: msg: "{{ metrics_mode }}" -- name: Install pmm client experimental on RHEL-family containers +- name: Install percona release on Debian-family containers + community.docker.docker_container_exec: + container: "{{ container_name }}" + user: "root" + command: dpkg -i /percona-release_latest.{{ lsb_release }}_all.deb + when: + - distro_family == 'debian' + +- name: Install specific PMM client version on RHEL-family containers + community.docker.docker_container_exec: + container: "{{ container_name }}" + user: "root" + command: rpm -i /percona-release-latest.noarch.rpm + when: + - distro_family == 'rhel' + +- name: Install pmm client experimental on Debian-family containers community.docker.docker_container_exec: container: "{{ container_name }}" user: "root" From 50a04782027e235a5f19d14e7f98cab83a549366 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 16:01:26 +0200 Subject: [PATCH 154/340] PMM-7: Add pgsql --- pmm_qa/tasks/install_pmm_client.yml | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/pmm_qa/tasks/install_pmm_client.yml b/pmm_qa/tasks/install_pmm_client.yml index d02fbff2..3fa0501f 100644 --- a/pmm_qa/tasks/install_pmm_client.yml +++ b/pmm_qa/tasks/install_pmm_client.yml @@ -78,18 +78,6 @@ set_fact: lsb_release: "{{ lsb_release.stdout }}" -- name: Download percona release on debian based OS - get_url: - url: "https://repo.percona.com/apt/percona-release_latest.{{ lsb_release }}_all.deb" - dest: "/percona-release_latest.{{ lsb_release }}_all.deb" - when: distro_family == "debian" - -- name: Download percona release on RHEL based OS - get_url: - url: "https://repo.percona.com/yum/percona-release-latest.noarch.rpm" - dest: "/percona-release-latest.noarch.rpm" - when: distro_family == "rhel" - - name: Print expected client version debug: msg: "{{ metrics_mode }}" @@ -98,7 +86,11 @@ community.docker.docker_container_exec: container: "{{ container_name }}" user: "root" - command: dpkg -i /percona-release_latest.{{ lsb_release }}_all.deb + command: > + sh -c ' + wget -O /percona-release_latest.{{ lsb_release }}_all.deb https://repo.percona.com/apt/percona-release_latest.{{ lsb_release }}_all.deb && + dpkg -i /percona-release_latest.{{ lsb_release }}_all.deb + ' when: - distro_family == 'debian' @@ -106,7 +98,11 @@ community.docker.docker_container_exec: container: "{{ container_name }}" user: "root" - command: rpm -i /percona-release-latest.noarch.rpm + command: > + sh -c ' + wget -O /percona-release-latest.noarch.rpm https://repo.percona.com/yum/percona-release-latest.noarch.rpm && + rpm -i /percona-release-latest.noarch.rpm + ' when: - distro_family == 'rhel' From ce640ad968e373fbb8ca469a5bb4eb2071058fcb Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 16:09:25 +0200 Subject: [PATCH 155/340] PMM-7: Add pgsql --- pmm_qa/tasks/install_pmm_client.yml | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/pmm_qa/tasks/install_pmm_client.yml b/pmm_qa/tasks/install_pmm_client.yml index 3fa0501f..1d19d90c 100644 --- a/pmm_qa/tasks/install_pmm_client.yml +++ b/pmm_qa/tasks/install_pmm_client.yml @@ -129,14 +129,16 @@ ' when: distro_family == "rhel" and client_version == "3-dev-latest" -- name: Install pmm client testing on RHEL-family containers +- name: Install pmm client testing on Debian-family containers community.docker.docker_container_exec: container: "{{ container_name }}" user: "root" command: > - percona-release enable-only pmm3-client testing - apt-get update - apt-get -y install pmm-client + sh -c ' + percona-release enable-only pmm3-client testing && + apt-get update && + apt-get -y install pmm-client + ' when: distro_family == "debian" and client_version == "pmm3-rc" - name: Install pmm client testing on RHEL-family containers @@ -144,25 +146,28 @@ container: "{{ container_name }}" user: "root" command: > - percona-release enable-only pmm3-client testing - microdnf install -y pmm-client + sh -c ' + percona-release enable-only pmm3-client testing && + microdnf install -y pmm-client + ' when: distro_family == "rhel" and client_version == "pmm3-rc" -- name: Install pmm client testing on RHEL-family containers +- name: Install pmm client release on Debian-family containers community.docker.docker_container_exec: container: "{{ container_name }}" user: "root" command: > - apt-get update - apt-get -y install pmm-client + sh -c ' + apt-get update && + apt-get -y install pmm-client + ' when: distro_family == "debian" and client_version == "pmm3-latest" -- name: Install pmm client testing on RHEL-family containers +- name: Install pmm client release on RHEL-family containers community.docker.docker_container_exec: container: "{{ container_name }}" user: "root" - command: > - microdnf install -y pmm-client + command: microdnf install -y pmm-client when: distro_family == "rhel" and client_version == "pmm3-latest" - name: Install specific PMM client version on Debian-family containers From aa7d97251add8fea794ef09e2f69b8bf537e4b6d Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 16:16:31 +0200 Subject: [PATCH 156/340] PMM-7: Add pdpgsql --- pmm_qa/pmm-framework.py | 53 ++++++++---- .../percona-distribution-postgres-setup.yml | 82 +++++++++++++++++++ 2 files changed, 119 insertions(+), 16 deletions(-) create mode 100644 pmm_qa/postgresql/percona-distribution-postgres-setup.yml diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index f9fb7b51..1d059295 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -53,7 +53,7 @@ }, "PDPGSQL": { "versions": ["11", "12", "13", "14", "15", "16", "17"], - "configurations": {"CLIENT_VERSION": "3-dev-latest", "USE_SOCKET": ""} + "configurations": {"CLIENT_VERSION": "3-dev-latest", "USE_SOCKET": "", "SETUP_TYPE": ""} }, "SSL_PDPGSQL": { "versions": ["11", "12", "13", "14", "15", "16", "17"], @@ -310,23 +310,44 @@ def setup_pdpgsql(db_type, db_version=None, db_config=None, args=None): # Gather Version details pdpgsql_version = os.getenv('PDPGSQL_VERSION') or db_version or database_configs[db_type]["versions"][-1] + setup_type_value = get_value('SETUP_TYPE', db_type, args, db_config).lower() + print(f"Setup type is {setup_type_value}") - # Define environment variables for playbook - env_vars = { - 'PGSTAT_MONITOR_BRANCH': 'main', - 'PDPGSQL_VERSION': pdpgsql_version, - 'PMM_SERVER_IP': args.pmm_server_ip or container_name or '127.0.0.1', - 'PDPGSQL_PGSM_CONTAINER': 'pdpgsql_pgsm_pmm_' + str(pdpgsql_version), - 'CLIENT_VERSION': get_value('CLIENT_VERSION', db_type, args, db_config), - 'USE_SOCKET': get_value('USE_SOCKET', db_type, args, db_config), - 'ADMIN_PASSWORD': os.getenv('ADMIN_PASSWORD') or args.pmm_server_password or 'admin', - 'PDPGSQL_PGSM_PORT': 5447, - 'DISTRIBUTION': '', - 'PMM_QA_GIT_BRANCH': os.getenv('PMM_QA_GIT_BRANCH') or 'v3' - } + if setup_type_value in ("replication", "replica"): + # Define environment variables for playbook + env_vars = { + 'PGSTAT_MONITOR_BRANCH': 'main', + 'PDPGSQL_VERSION': pdpgsql_version, + 'PMM_SERVER_IP': args.pmm_server_ip or container_name or '127.0.0.1', + 'PDPGSQL_PGSM_CONTAINER': 'pdpgsql_pgsm_pmm_' + str(pdpgsql_version), + 'CLIENT_VERSION': get_value('CLIENT_VERSION', db_type, args, db_config), + 'USE_SOCKET': get_value('USE_SOCKET', db_type, args, db_config), + 'ADMIN_PASSWORD': os.getenv('ADMIN_PASSWORD') or args.pmm_server_password or 'admin', + 'PDPGSQL_PGSM_PORT': 5447, + 'DISTRIBUTION': '', + 'PMM_QA_GIT_BRANCH': os.getenv('PMM_QA_GIT_BRANCH') or 'v3', + 'SETUP_TYPE': setup_type_value + } - # Ansible playbook filename - playbook_filename = 'pdpgsql_pgsm_setup.yml' + # Ansible playbook filename + playbook_filename = './postgresql/percona-distribution-postgres-setup.yml' + else: + # Define environment variables for playbook + env_vars = { + 'PGSTAT_MONITOR_BRANCH': 'main', + 'PDPGSQL_VERSION': pdpgsql_version, + 'PMM_SERVER_IP': args.pmm_server_ip or container_name or '127.0.0.1', + 'PDPGSQL_PGSM_CONTAINER': 'pdpgsql_pgsm_pmm_' + str(pdpgsql_version), + 'CLIENT_VERSION': get_value('CLIENT_VERSION', db_type, args, db_config), + 'USE_SOCKET': get_value('USE_SOCKET', db_type, args, db_config), + 'ADMIN_PASSWORD': os.getenv('ADMIN_PASSWORD') or args.pmm_server_password or 'admin', + 'PDPGSQL_PGSM_PORT': 5447, + 'DISTRIBUTION': '', + 'PMM_QA_GIT_BRANCH': os.getenv('PMM_QA_GIT_BRANCH') or 'v3' + } + + # Ansible playbook filename + playbook_filename = 'pdpgsql_pgsm_setup.yml' # Call the function to run the Ansible playbook run_ansible_playbook(playbook_filename, env_vars, args) diff --git a/pmm_qa/postgresql/percona-distribution-postgres-setup.yml b/pmm_qa/postgresql/percona-distribution-postgres-setup.yml new file mode 100644 index 00000000..5c194444 --- /dev/null +++ b/pmm_qa/postgresql/percona-distribution-postgres-setup.yml @@ -0,0 +1,82 @@ +--- +# Percona Distribution Postgresql Replication +- name: Setup Postgresql replication + hosts: localhost + connection: local + gather_facts: yes + vars: + pg_version: "{{ lookup('env', 'PGSQL_VERSION') | default('17', true) }}" +# cluster_name: "mysql_cluster" + replication_user: "repl_user" + replication_password: "GRgrO9301RuF" + root_password: "GRgrO9301RuF" +# mysql_port: 33066 +# mysql_listen_port: 3306 +# group_seeds_port: 34061 + nodes_count: "{{ (lookup('env', 'NODES_COUNT') | default('2', true)) | int }}" + network_name: "pmm-qa" + data_dir: "{{ lookup('env', 'HOME') }}/pgsql_cluster_data" +# server_id_start: 1 + pmm_server_ip: "{{ lookup('vars', 'extra_pmm_server_ip', default=lookup('env','PMM_SERVER_IP') | default('127.0.0.1', true) ) }}" + client_version: "{{ lookup('vars', 'extra_client_version', default=lookup('env','CLIENT_VERSION') | default('3-dev-latest', true) ) }}" + admin_password: "{{ lookup('vars', 'extra_admin_password', default=lookup('env','ADMIN_PASSWORD') | default('admin', true) ) }}" + query_source: "{{ lookup('env', 'QUERY_SOURCE') | default('pgstatements', true) }}" + metrics_mode: "auto" + setup_type: "{{ lookup('env', 'SETUP_TYPE') }}" + random_service_name_value: "" + + tasks: + - name: Create Docker network + community.docker.docker_network: + name: "{{ network_name }}" + state: present + ignore_errors: yes + + - name: Remove old data folders + shell: 'rm -fr {{ data_dir }}' + + - name: Create data directories + file: + path: "{{ data_dir }}/node{{ item }}/data" + state: directory + mode: '0755' + loop: "{{ range(1, nodes_count | int + 1) | list }}" + + - name: Generate pg_hba.conf for primary node + template: + src: ./data/pg-primary-conf/pg_hba.conf.j2 + dest: "{{ data_dir }}/node1/pg_hba.conf" + + - name: Remove old Percona Distribution PostgreSQL primary container + community.docker.docker_container: + name: "pgsql_pmm_{{ pg_version }}_1" + image: "postgres:{{ pg_version }}-bookworm" + restart_policy: always + state: absent + ignore_errors: yes + + - name: Start Percona Distribution PostgreSQL primary container + community.docker.docker_container: + name: "pdpgsql_pmm_{{ pg_version }}_1" + image: "percona/percona-distribution-postgresql:{{ pg_version }}" + restart_policy: always + state: started + recreate: true + networks: + - name: "{{ network_name }}" + env: + POSTGRES_PASSWORD: "{{ root_password }}" + volumes: + - "{{ data_dir }}/node1/data:/var/lib/postgresql/data" + - "./data/pg-primary-conf/postgresql.conf:/etc/postgresql/postgresql.conf:ro" + - "{{ data_dir }}/node1/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro" + command: -c config_file=/etc/postgresql/postgresql.conf + ports: + - "5432:5432" + + - name: Wait for PgSQL to be available + wait_for: + host: localhost + port: "5432" + delay: 10 + timeout: 300 From 40d71e2477492f8e1e611ce5776068f76920246a Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 16:23:40 +0200 Subject: [PATCH 157/340] PMM-7: Add pdpgsql --- .../percona-distribution-postgres-setup.yml | 123 ++++++++++++++++++ 1 file changed, 123 insertions(+) diff --git a/pmm_qa/postgresql/percona-distribution-postgres-setup.yml b/pmm_qa/postgresql/percona-distribution-postgres-setup.yml index 5c194444..44eadb0f 100644 --- a/pmm_qa/postgresql/percona-distribution-postgres-setup.yml +++ b/pmm_qa/postgresql/percona-distribution-postgres-setup.yml @@ -80,3 +80,126 @@ port: "5432" delay: 10 timeout: 300 + - name: Create replication user + community.docker.docker_container_exec: + container: "pgsql_pmm_{{ pg_version }}_1" + user: postgres + command: > + psql -c " + CREATE ROLE {{ replication_user }} WITH REPLICATION LOGIN ENCRYPTED PASSWORD '{{ replication_password }}'; + " + - name: Stop and remove replica if exists + community.docker.docker_container: + name: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" + state: absent + loop: "{{ range(2, nodes_count | int + 1) | list }}" + ignore_errors: yes + + - name: Start PostgreSQL replica container + community.docker.docker_container: + name: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" + image: "percona/percona-distribution-postgresql:{{ pg_version }}" + restart_policy: "no" + state: started + command: sleep infinity + networks: + - name: "{{ network_name }}" + env: + POSTGRES_INITDB_SKIP: "yes" + POSTGRES_PASSWORD: "{{ root_password }}" + volumes: + - "{{ data_dir }}/node{{ item }}/data:/var/lib/postgresql/data" + - "./data/pg-primary-conf/postgresql-replica.conf:/etc/postgresql/postgresql.conf:ro" + loop: "{{ range(2, nodes_count | int + 1) | list }}" + + - name: Wipe replica data directory before basebackup + community.docker.docker_container_exec: + container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" + user: root + command: rm -rf /var/lib/postgresql/data/* + loop: "{{ range(2, nodes_count | int + 1) | list }}" + + - name: Run pg_basebackup from primary to replica + community.docker.docker_container_exec: + container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" + user: root + command: > + bash -c " + export PGPASSWORD='{{ replication_password }}' && \ + timeout 60s \ + pg_basebackup --pgdata=/var/lib/postgresql/data -R -v -P \ + --host=pgsql_pmm_{{ pg_version }}_1 --port=5432 -U {{ replication_user }} + " + loop: "{{ range(2, nodes_count | int + 1) | list }}" + + - name: Restart PostgreSQL container with custom command + community.docker.docker_container: + name: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" + restart: true + state: started + command: -c config_file=/etc/postgresql/postgresql.conf + networks: + - name: "{{ network_name }}" + env: + POSTGRES_PASSWORD: "{{ root_password }}" + volumes: + - "{{ data_dir }}/node{{ item }}/data:/var/lib/postgresql/data" + - "./data/postgres-replica.conf:/etc/postgresql/postgresql.conf:ro" + loop: "{{ range(2, nodes_count | int + 1) | list }}" + + - name: Create PostgreSQL user 'pmm' with password + community.docker.docker_container_exec: + container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" + user: postgres + command: > + bash -c ' + psql -U postgres -d {{ db_name | default("postgres") }} -c " + DO \$$ + BEGIN + IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = '\''pmm'\'') THEN + CREATE ROLE pmm LOGIN PASSWORD '\''pmm'\''; + END IF; + END + \$$;" + ' + loop: "{{ range(1, nodes_count | int + 1) | list }}" + + - name: Create pg_stat_statements extension + community.docker.docker_container_exec: + container: "pdpgsql_pmm_{{ pg_version }}_1" + user: postgres + command: > + psql -U postgres -d 'postgres' -c " + CREATE EXTENSION IF NOT EXISTS pg_stat_statements; + " + + - name: Install and add pmm client. + include_tasks: ../tasks/install_pmm_client.yml + vars: + container_name: "pgsql_pmm_{{ pg_version }}_{{ item }}" + loop: "{{ range(1, nodes_count | int + 1) | list }}" + + - name: Get already connected services to pmm server + community.docker.docker_container_exec: + container: "pdpgsql_pmm_{{ pg_version }}_1" + command: > + sh -c 'curl --location --insecure -u"admin:{{ admin_password }}" -s --request GET "http://{{ pmm_server_ip }}:{{ '80' if pmm_server_ip is ansible.utils.ipv4 else '8080' }}/v1/management/services" | jq -r ".services[].service_name"' + register: pmm_server_services + + - name: Display already connected services to pmm server + debug: + msg: "{{ pmm_server_services.stdout | split('\n') }}" + + - name: Find out if service is already connected to pmm server + block: + - name: Loop through percona servers + set_fact: + random_service_name_value: "_{{ 9999 | random + 1 }}" + loop: "{{ range(1, nodes_count | int + 1) | list }}" + when: "('pgsql_pmm_' ~ pg_version ~ '_' ~ item) in pmm_server_services.stdout" + + - name: Add service to pmm server + community.docker.docker_container_exec: + container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" + command: pmm-admin add postgresql --username=pmm --password=pmm pgsql_pmm_{{ pg_version }}_{{ item }}{{ random_service_name_value }} --debug 127.0.0.1:5432 + loop: "{{ range(1, nodes_count | int + 1) | list }}" From 33bc5aeb50c54f6107ca47a694f8eb8da1ac0f69 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 16:25:16 +0200 Subject: [PATCH 158/340] PMM-7: Add pdpgsql --- pmm_qa/postgresql/percona-distribution-postgres-setup.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pmm_qa/postgresql/percona-distribution-postgres-setup.yml b/pmm_qa/postgresql/percona-distribution-postgres-setup.yml index 44eadb0f..cad2c3a3 100644 --- a/pmm_qa/postgresql/percona-distribution-postgres-setup.yml +++ b/pmm_qa/postgresql/percona-distribution-postgres-setup.yml @@ -49,8 +49,8 @@ - name: Remove old Percona Distribution PostgreSQL primary container community.docker.docker_container: - name: "pgsql_pmm_{{ pg_version }}_1" - image: "postgres:{{ pg_version }}-bookworm" + name: "pdpgsql_pmm_{{ pg_version }}_1" + image: "percona/percona-distribution-postgresql:{{ pg_version }}" restart_policy: always state: absent ignore_errors: yes @@ -82,7 +82,7 @@ timeout: 300 - name: Create replication user community.docker.docker_container_exec: - container: "pgsql_pmm_{{ pg_version }}_1" + container: "pdpgsql_pmm_{{ pg_version }}_1" user: postgres command: > psql -c " @@ -176,7 +176,7 @@ - name: Install and add pmm client. include_tasks: ../tasks/install_pmm_client.yml vars: - container_name: "pgsql_pmm_{{ pg_version }}_{{ item }}" + container_name: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" loop: "{{ range(1, nodes_count | int + 1) | list }}" - name: Get already connected services to pmm server From 7e36402815fdddcfd4f5d8e23a64f9bcc9fb9e4c Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 16:26:34 +0200 Subject: [PATCH 159/340] PMM-7: Add pdpgsql --- pmm_qa/postgresql/percona-distribution-postgres-setup.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/postgresql/percona-distribution-postgres-setup.yml b/pmm_qa/postgresql/percona-distribution-postgres-setup.yml index cad2c3a3..cd21726a 100644 --- a/pmm_qa/postgresql/percona-distribution-postgres-setup.yml +++ b/pmm_qa/postgresql/percona-distribution-postgres-setup.yml @@ -128,7 +128,7 @@ export PGPASSWORD='{{ replication_password }}' && \ timeout 60s \ pg_basebackup --pgdata=/var/lib/postgresql/data -R -v -P \ - --host=pgsql_pmm_{{ pg_version }}_1 --port=5432 -U {{ replication_user }} + --host=pdpgsql_pmm_{{ pg_version }}_1 --port=5432 -U {{ replication_user }} " loop: "{{ range(2, nodes_count | int + 1) | list }}" From 25ab540cd3e86d6dccc259430ef5a54d95cd99a8 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 16:32:49 +0200 Subject: [PATCH 160/340] PMM-7: Add pdpgsql --- pmm_qa/postgresql/postgresql-setup.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index ce02b90e..c25d8138 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -10,7 +10,7 @@ replication_user: "repl_user" replication_password: "GRgrO9301RuF" root_password: "GRgrO9301RuF" -# mysql_port: 33066 + pgsql_port: 6432 # mysql_listen_port: 3306 # group_seeds_port: 34061 nodes_count: "{{ (lookup('env', 'NODES_COUNT') | default('2', true)) | int }}" @@ -72,7 +72,7 @@ - "{{ data_dir }}/node1/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro" command: -c config_file=/etc/postgresql/postgresql.conf ports: - - "5432:5432" + - "{{ pgsql_port }}:5432" - name: Wait for PgSQL to be available wait_for: @@ -147,6 +147,8 @@ volumes: - "{{ data_dir }}/node{{ item }}/data:/var/lib/postgresql/data" - "./data/postgres-replica.conf:/etc/postgresql/postgresql.conf:ro" + ports: + - "{{ pgsql_port + item - 1 }}:5432" loop: "{{ range(2, nodes_count | int + 1) | list }}" - name: Create PostgreSQL user 'pmm' with password From 6f3ab43bc9a4011167ed3c4464786b9d2af865cc Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 16:34:05 +0200 Subject: [PATCH 161/340] PMM-7: Add pdpgsql --- pmm_qa/postgresql/postgresql-setup.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index c25d8138..1fc2341f 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -77,7 +77,7 @@ - name: Wait for PgSQL to be available wait_for: host: localhost - port: "5432" + port: "{{ pgsql_port }}" delay: 10 timeout: 300 From d4176cac306e6f1397d2bb2ef6092eaf1c8bbd6c Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 16:39:01 +0200 Subject: [PATCH 162/340] PMM-7: Add pdpgsql --- pmm_qa/postgresql/percona-distribution-postgres-setup.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pmm_qa/postgresql/percona-distribution-postgres-setup.yml b/pmm_qa/postgresql/percona-distribution-postgres-setup.yml index cd21726a..c5e29b85 100644 --- a/pmm_qa/postgresql/percona-distribution-postgres-setup.yml +++ b/pmm_qa/postgresql/percona-distribution-postgres-setup.yml @@ -10,7 +10,7 @@ replication_user: "repl_user" replication_password: "GRgrO9301RuF" root_password: "GRgrO9301RuF" -# mysql_port: 33066 + pdpgsql_port: 5432 # mysql_listen_port: 3306 # group_seeds_port: 34061 nodes_count: "{{ (lookup('env', 'NODES_COUNT') | default('2', true)) | int }}" @@ -72,7 +72,7 @@ - "{{ data_dir }}/node1/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro" command: -c config_file=/etc/postgresql/postgresql.conf ports: - - "5432:5432" + - "{{ pdpgsql_port }}:5432" - name: Wait for PgSQL to be available wait_for: @@ -145,6 +145,8 @@ volumes: - "{{ data_dir }}/node{{ item }}/data:/var/lib/postgresql/data" - "./data/postgres-replica.conf:/etc/postgresql/postgresql.conf:ro" + ports: + - "{{ pdpgsql_port + item - 1}}:5432" loop: "{{ range(2, nodes_count | int + 1) | list }}" - name: Create PostgreSQL user 'pmm' with password From 69b6d8df375503e66d98e5cd61277edb47ab9be3 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 16:41:57 +0200 Subject: [PATCH 163/340] PMM-7: Add pdpgsql --- pmm_qa/postgresql/postgresql-setup.yml | 6 +++--- pmm_qa/postgresql/tasks/run_load_pgsql.yml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index 1fc2341f..a49262bb 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -212,9 +212,9 @@ include_tasks: ./tasks/run_load_pgsql.yml vars: node_name: "pgsql_pmm_{{ pg_version }}_1" - pgbench_clients: 10 - pgbench_time: 60 - pgbench_scale: 10 + pgbench_clients: 100 + pgbench_time: 120 + pgbench_scale: 100 - name: Check replication status on primary community.docker.docker_container_exec: diff --git a/pmm_qa/postgresql/tasks/run_load_pgsql.yml b/pmm_qa/postgresql/tasks/run_load_pgsql.yml index b5a4aea9..0984a64b 100644 --- a/pmm_qa/postgresql/tasks/run_load_pgsql.yml +++ b/pmm_qa/postgresql/tasks/run_load_pgsql.yml @@ -83,7 +83,7 @@ container: "{{ node_name }}" user: postgres command: > - pgbench -c {{ pgbench_clients }} -T {{ pgbench_time }} -f /workload.sql pgbench + pgbench -c {{ pgbench_clients }} -T {{ pgbench_time }} -j 4 -f /workload.sql pgbench register: pgbench_result - name: Print pgbench results From 99dfb9652e6a9e4ae4f848a0c767c3df1c109ab0 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 16:42:10 +0200 Subject: [PATCH 164/340] PMM-7: Add pdpgsql --- pmm_qa/postgresql/tasks/run_load_pgsql.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pmm_qa/postgresql/tasks/run_load_pgsql.yml b/pmm_qa/postgresql/tasks/run_load_pgsql.yml index 0984a64b..ea72bff4 100644 --- a/pmm_qa/postgresql/tasks/run_load_pgsql.yml +++ b/pmm_qa/postgresql/tasks/run_load_pgsql.yml @@ -83,7 +83,8 @@ container: "{{ node_name }}" user: postgres command: > - pgbench -c {{ pgbench_clients }} -T {{ pgbench_time }} -j 4 -f /workload.sql pgbench + pgbench -c {{ pgbench_clients }} -T {{ pgbench_time }} -j 4 pgbench +# -f /workload.sql pgbench register: pgbench_result - name: Print pgbench results From f2fea7b65ec2402227090067080cc5c5bc3f072b Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 16:46:54 +0200 Subject: [PATCH 165/340] PMM-7: Add pdpgsql --- pmm_qa/postgresql/postgresql-setup.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index a49262bb..a62eb1b5 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -212,9 +212,9 @@ include_tasks: ./tasks/run_load_pgsql.yml vars: node_name: "pgsql_pmm_{{ pg_version }}_1" - pgbench_clients: 100 + pgbench_clients: 10 pgbench_time: 120 - pgbench_scale: 100 + pgbench_scale: 1000 - name: Check replication status on primary community.docker.docker_container_exec: From f2db6724fc73ef56356a860ac36c941474f2101f Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 16:47:39 +0200 Subject: [PATCH 166/340] PMM-7: Add pdpgsql --- pmm_qa/postgresql/percona-distribution-postgres-setup.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/postgresql/percona-distribution-postgres-setup.yml b/pmm_qa/postgresql/percona-distribution-postgres-setup.yml index c5e29b85..574a0382 100644 --- a/pmm_qa/postgresql/percona-distribution-postgres-setup.yml +++ b/pmm_qa/postgresql/percona-distribution-postgres-setup.yml @@ -77,7 +77,7 @@ - name: Wait for PgSQL to be available wait_for: host: localhost - port: "5432" + port: "{{ pdpgsql_port }}" delay: 10 timeout: 300 - name: Create replication user From be237299d6675e69786ed96737e33d4bb62cd563 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 16:49:33 +0200 Subject: [PATCH 167/340] PMM-7: Add pdpgsql --- pmm_qa/postgresql/percona-distribution-postgres-setup.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/postgresql/percona-distribution-postgres-setup.yml b/pmm_qa/postgresql/percona-distribution-postgres-setup.yml index 574a0382..a95f6d23 100644 --- a/pmm_qa/postgresql/percona-distribution-postgres-setup.yml +++ b/pmm_qa/postgresql/percona-distribution-postgres-setup.yml @@ -155,7 +155,7 @@ user: postgres command: > bash -c ' - psql -U postgres -d {{ db_name | default("postgres") }} -c " + psql -h localhost -U postgres -d {{ db_name | default("postgres") }} -c " DO \$$ BEGIN IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = '\''pmm'\'') THEN From 2c03315c2b1b73b298414975381da6465ea2c3b9 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 16:58:27 +0200 Subject: [PATCH 168/340] PMM-7: Add pdpgsql --- pmm_qa/postgresql/data/pdpgsql.conf | 1 + pmm_qa/postgresql/data/pdpgsql_hba.conf | 2 ++ pmm_qa/postgresql/percona-distribution-postgres-setup.yml | 5 +++-- 3 files changed, 6 insertions(+), 2 deletions(-) create mode 100644 pmm_qa/postgresql/data/pdpgsql.conf create mode 100644 pmm_qa/postgresql/data/pdpgsql_hba.conf diff --git a/pmm_qa/postgresql/data/pdpgsql.conf b/pmm_qa/postgresql/data/pdpgsql.conf new file mode 100644 index 00000000..a4abdb84 --- /dev/null +++ b/pmm_qa/postgresql/data/pdpgsql.conf @@ -0,0 +1 @@ +hba_file = '/etc/postgresql/pg_hba.conf' \ No newline at end of file diff --git a/pmm_qa/postgresql/data/pdpgsql_hba.conf b/pmm_qa/postgresql/data/pdpgsql_hba.conf new file mode 100644 index 00000000..9b76134e --- /dev/null +++ b/pmm_qa/postgresql/data/pdpgsql_hba.conf @@ -0,0 +1,2 @@ +host all all 0.0.0.0/0 md5 +local all postgres trust \ No newline at end of file diff --git a/pmm_qa/postgresql/percona-distribution-postgres-setup.yml b/pmm_qa/postgresql/percona-distribution-postgres-setup.yml index a95f6d23..276c7929 100644 --- a/pmm_qa/postgresql/percona-distribution-postgres-setup.yml +++ b/pmm_qa/postgresql/percona-distribution-postgres-setup.yml @@ -74,7 +74,7 @@ ports: - "{{ pdpgsql_port }}:5432" - - name: Wait for PgSQL to be available + - name: Wait for Percona Distribution PgSQL to be available wait_for: host: localhost port: "{{ pdpgsql_port }}" @@ -144,7 +144,8 @@ POSTGRES_PASSWORD: "{{ root_password }}" volumes: - "{{ data_dir }}/node{{ item }}/data:/var/lib/postgresql/data" - - "./data/postgres-replica.conf:/etc/postgresql/postgresql.conf:ro" + - "./data/pdpgsql.conf:/etc/postgresql/postgresql.conf:ro" + - "./data/pdpgsql_hba.conf:/etc/postgresql/pg_hba.conf:ro" ports: - "{{ pdpgsql_port + item - 1}}:5432" loop: "{{ range(2, nodes_count | int + 1) | list }}" From 6d7e6fc0e4cef45f785e29cf0b00f61d8809a799 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 17:05:23 +0200 Subject: [PATCH 169/340] PMM-7: Add pdpgsql --- pmm_qa/postgresql/percona-distribution-postgres-setup.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pmm_qa/postgresql/percona-distribution-postgres-setup.yml b/pmm_qa/postgresql/percona-distribution-postgres-setup.yml index 276c7929..18f637f2 100644 --- a/pmm_qa/postgresql/percona-distribution-postgres-setup.yml +++ b/pmm_qa/postgresql/percona-distribution-postgres-setup.yml @@ -110,6 +110,7 @@ volumes: - "{{ data_dir }}/node{{ item }}/data:/var/lib/postgresql/data" - "./data/pg-primary-conf/postgresql-replica.conf:/etc/postgresql/postgresql.conf:ro" + - "{{ data_dir }}/node1/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro" loop: "{{ range(2, nodes_count | int + 1) | list }}" - name: Wipe replica data directory before basebackup @@ -156,7 +157,7 @@ user: postgres command: > bash -c ' - psql -h localhost -U postgres -d {{ db_name | default("postgres") }} -c " + psql -h 127.0.0.1 -U postgres -d {{ db_name | default("postgres") }} -c " DO \$$ BEGIN IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = '\''pmm'\'') THEN From 7dea9c06c1efa47aff48c075e95f9ca4c4a45661 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 17:10:14 +0200 Subject: [PATCH 170/340] PMM-7: Add pdpgsql --- pmm_qa/postgresql/percona-distribution-postgres-setup.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/pmm_qa/postgresql/percona-distribution-postgres-setup.yml b/pmm_qa/postgresql/percona-distribution-postgres-setup.yml index 18f637f2..561bdadd 100644 --- a/pmm_qa/postgresql/percona-distribution-postgres-setup.yml +++ b/pmm_qa/postgresql/percona-distribution-postgres-setup.yml @@ -109,8 +109,6 @@ POSTGRES_PASSWORD: "{{ root_password }}" volumes: - "{{ data_dir }}/node{{ item }}/data:/var/lib/postgresql/data" - - "./data/pg-primary-conf/postgresql-replica.conf:/etc/postgresql/postgresql.conf:ro" - - "{{ data_dir }}/node1/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro" loop: "{{ range(2, nodes_count | int + 1) | list }}" - name: Wipe replica data directory before basebackup @@ -136,6 +134,7 @@ - name: Restart PostgreSQL container with custom command community.docker.docker_container: name: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" + image: "percona/percona-distribution-postgresql:{{ pg_version }}" restart: true state: started command: -c config_file=/etc/postgresql/postgresql.conf @@ -145,8 +144,6 @@ POSTGRES_PASSWORD: "{{ root_password }}" volumes: - "{{ data_dir }}/node{{ item }}/data:/var/lib/postgresql/data" - - "./data/pdpgsql.conf:/etc/postgresql/postgresql.conf:ro" - - "./data/pdpgsql_hba.conf:/etc/postgresql/pg_hba.conf:ro" ports: - "{{ pdpgsql_port + item - 1}}:5432" loop: "{{ range(2, nodes_count | int + 1) | list }}" From a7e24e85f3107a75cd44857417f1511e182d80fa Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 17:12:05 +0200 Subject: [PATCH 171/340] PMM-7: Add pdpgsql --- pmm_qa/postgresql/percona-distribution-postgres-setup.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/pmm_qa/postgresql/percona-distribution-postgres-setup.yml b/pmm_qa/postgresql/percona-distribution-postgres-setup.yml index 561bdadd..f173d661 100644 --- a/pmm_qa/postgresql/percona-distribution-postgres-setup.yml +++ b/pmm_qa/postgresql/percona-distribution-postgres-setup.yml @@ -144,6 +144,7 @@ POSTGRES_PASSWORD: "{{ root_password }}" volumes: - "{{ data_dir }}/node{{ item }}/data:/var/lib/postgresql/data" + - "./data/postgres-replica.conf:/etc/postgresql/postgresql.conf:ro" ports: - "{{ pdpgsql_port + item - 1}}:5432" loop: "{{ range(2, nodes_count | int + 1) | list }}" From 13413d8db3c018aa623ed3a3356998ee868720e4 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 17:12:50 +0200 Subject: [PATCH 172/340] PMM-7: Add pdpgsql --- pmm_qa/postgresql/percona-distribution-postgres-setup.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/postgresql/percona-distribution-postgres-setup.yml b/pmm_qa/postgresql/percona-distribution-postgres-setup.yml index f173d661..7b3fdbe9 100644 --- a/pmm_qa/postgresql/percona-distribution-postgres-setup.yml +++ b/pmm_qa/postgresql/percona-distribution-postgres-setup.yml @@ -155,7 +155,7 @@ user: postgres command: > bash -c ' - psql -h 127.0.0.1 -U postgres -d {{ db_name | default("postgres") }} -c " + PGPASSWORD="{{ root_password }}" psql -h 127.0.0.1 -U postgres -d {{ db_name | default("postgres") }} -c " DO \$$ BEGIN IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = '\''pmm'\'') THEN From 216249466515311cea40bb0cda48de054fa78c10 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 17:14:56 +0200 Subject: [PATCH 173/340] PMM-7: Add pdpgsql --- pmm_qa/postgresql/data/postgres-replica.conf | 3 ++- pmm_qa/postgresql/percona-distribution-postgres-setup.yml | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/pmm_qa/postgresql/data/postgres-replica.conf b/pmm_qa/postgresql/data/postgres-replica.conf index 172f4210..a26d04bd 100644 --- a/pmm_qa/postgresql/data/postgres-replica.conf +++ b/pmm_qa/postgresql/data/postgres-replica.conf @@ -1 +1,2 @@ -shared_preload_libraries = 'pg_stat_statements' \ No newline at end of file +shared_preload_libraries = 'pg_stat_statements' +hba_file = '/etc/postgresql/pg_hba.conf' \ No newline at end of file diff --git a/pmm_qa/postgresql/percona-distribution-postgres-setup.yml b/pmm_qa/postgresql/percona-distribution-postgres-setup.yml index 7b3fdbe9..4f65f2b5 100644 --- a/pmm_qa/postgresql/percona-distribution-postgres-setup.yml +++ b/pmm_qa/postgresql/percona-distribution-postgres-setup.yml @@ -145,6 +145,7 @@ volumes: - "{{ data_dir }}/node{{ item }}/data:/var/lib/postgresql/data" - "./data/postgres-replica.conf:/etc/postgresql/postgresql.conf:ro" + - "{{ data_dir }}/node1/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro" ports: - "{{ pdpgsql_port + item - 1}}:5432" loop: "{{ range(2, nodes_count | int + 1) | list }}" From 247afaae878e63dfbe7e98056ebe941593b11573 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 17:17:30 +0200 Subject: [PATCH 174/340] PMM-7: Add pdpgsql --- pmm_qa/postgresql/percona-distribution-postgres-setup.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pmm_qa/postgresql/percona-distribution-postgres-setup.yml b/pmm_qa/postgresql/percona-distribution-postgres-setup.yml index 4f65f2b5..b689b5dd 100644 --- a/pmm_qa/postgresql/percona-distribution-postgres-setup.yml +++ b/pmm_qa/postgresql/percona-distribution-postgres-setup.yml @@ -109,6 +109,8 @@ POSTGRES_PASSWORD: "{{ root_password }}" volumes: - "{{ data_dir }}/node{{ item }}/data:/var/lib/postgresql/data" + - "./data/postgres-replica.conf:/etc/postgresql/postgresql.conf:ro" + - "{{ data_dir }}/node1/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro" loop: "{{ range(2, nodes_count | int + 1) | list }}" - name: Wipe replica data directory before basebackup From 04f0dcaf97433a18750aeb1573494ff8df12d7e1 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 17:20:00 +0200 Subject: [PATCH 175/340] PMM-7: Add pdpgsql --- pmm_qa/percona-distribution-postgresql/data/pg_hba.conf.j2 | 3 +++ .../data/postgresql-primary.conf | 7 +++++++ .../percona-distribution-postgres-setup.yml | 4 ++-- pmm_qa/pmm-framework.py | 2 +- 4 files changed, 13 insertions(+), 3 deletions(-) create mode 100644 pmm_qa/percona-distribution-postgresql/data/pg_hba.conf.j2 create mode 100644 pmm_qa/percona-distribution-postgresql/data/postgresql-primary.conf rename pmm_qa/{postgresql => percona-distribution-postgresql}/percona-distribution-postgres-setup.yml (98%) diff --git a/pmm_qa/percona-distribution-postgresql/data/pg_hba.conf.j2 b/pmm_qa/percona-distribution-postgresql/data/pg_hba.conf.j2 new file mode 100644 index 00000000..7d58f0c8 --- /dev/null +++ b/pmm_qa/percona-distribution-postgresql/data/pg_hba.conf.j2 @@ -0,0 +1,3 @@ +host replication {{ replication_user }} 0.0.0.0/0 md5 +host all all 0.0.0.0/0 md5 +local all postgres trust \ No newline at end of file diff --git a/pmm_qa/percona-distribution-postgresql/data/postgresql-primary.conf b/pmm_qa/percona-distribution-postgresql/data/postgresql-primary.conf new file mode 100644 index 00000000..8356b760 --- /dev/null +++ b/pmm_qa/percona-distribution-postgresql/data/postgresql-primary.conf @@ -0,0 +1,7 @@ +wal_level = replica +max_wal_senders = 10 +wal_keep_size = 64MB +hot_standby = on +listen_addresses = '*' +hba_file = '/etc/postgresql/pg_hba.conf' +shared_preload_libraries = 'pg_stat_statements' \ No newline at end of file diff --git a/pmm_qa/postgresql/percona-distribution-postgres-setup.yml b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml similarity index 98% rename from pmm_qa/postgresql/percona-distribution-postgres-setup.yml rename to pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml index b689b5dd..3ff3129b 100644 --- a/pmm_qa/postgresql/percona-distribution-postgres-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml @@ -44,7 +44,7 @@ - name: Generate pg_hba.conf for primary node template: - src: ./data/pg-primary-conf/pg_hba.conf.j2 + src: ./data/pg_hba.conf.j2 dest: "{{ data_dir }}/node1/pg_hba.conf" - name: Remove old Percona Distribution PostgreSQL primary container @@ -68,7 +68,7 @@ POSTGRES_PASSWORD: "{{ root_password }}" volumes: - "{{ data_dir }}/node1/data:/var/lib/postgresql/data" - - "./data/pg-primary-conf/postgresql.conf:/etc/postgresql/postgresql.conf:ro" + - "./data/postgresql-primary.conf:/etc/postgresql/postgresql.conf:ro" - "{{ data_dir }}/node1/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro" command: -c config_file=/etc/postgresql/postgresql.conf ports: diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 1d059295..68c92d12 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -330,7 +330,7 @@ def setup_pdpgsql(db_type, db_version=None, db_config=None, args=None): } # Ansible playbook filename - playbook_filename = './postgresql/percona-distribution-postgres-setup.yml' + playbook_filename = 'percona-distribution-postgresql/percona-distribution-postgres-setup.yml' else: # Define environment variables for playbook env_vars = { From 50c152956bb975894a666398046c6459f1826261 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 17:20:47 +0200 Subject: [PATCH 176/340] PMM-7: Add pdpgsql --- .../percona-distribution-postgres-setup.yml | 10 ++--- pmm_qa/pg_stat_monitor_setup.sh | 2 +- pmm_qa/pg_stat_statements_setup.sh | 2 +- pmm_qa/postgresql/data/pdpgsql.conf | 1 - pmm_qa/postgresql/data/pdpgsql_hba.conf | 2 - .../data/{pg-primary-conf => }/pg_hba.conf.j2 | 0 ...ostgresql.conf => postgresql-primary.conf} | 0 pmm_qa/postgresql/data/sample_data.sql | 38 ------------------- pmm_qa/postgresql/data/workload.sql | 28 -------------- pmm_qa/postgresql/postgresql-setup.yml | 12 +++--- 10 files changed, 13 insertions(+), 82 deletions(-) delete mode 100644 pmm_qa/postgresql/data/pdpgsql.conf delete mode 100644 pmm_qa/postgresql/data/pdpgsql_hba.conf rename pmm_qa/postgresql/data/{pg-primary-conf => }/pg_hba.conf.j2 (100%) rename pmm_qa/postgresql/data/{pg-primary-conf/postgresql.conf => postgresql-primary.conf} (100%) delete mode 100644 pmm_qa/postgresql/data/sample_data.sql delete mode 100644 pmm_qa/postgresql/data/workload.sql diff --git a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml index 3ff3129b..facfcd3b 100644 --- a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml @@ -68,9 +68,9 @@ POSTGRES_PASSWORD: "{{ root_password }}" volumes: - "{{ data_dir }}/node1/data:/var/lib/postgresql/data" - - "./data/postgresql-primary.conf:/etc/postgresql/postgresql.conf:ro" + - "./data/postgresql-primary.conf:/etc/postgresql/postgresql-primary.conf:ro" - "{{ data_dir }}/node1/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro" - command: -c config_file=/etc/postgresql/postgresql.conf + command: -c config_file=/etc/postgresql/postgresql-primary.conf ports: - "{{ pdpgsql_port }}:5432" @@ -109,7 +109,7 @@ POSTGRES_PASSWORD: "{{ root_password }}" volumes: - "{{ data_dir }}/node{{ item }}/data:/var/lib/postgresql/data" - - "./data/postgres-replica.conf:/etc/postgresql/postgresql.conf:ro" + - "./data/postgres-replica.conf:/etc/postgresql/postgresql-primary.conf:ro" - "{{ data_dir }}/node1/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro" loop: "{{ range(2, nodes_count | int + 1) | list }}" @@ -139,14 +139,14 @@ image: "percona/percona-distribution-postgresql:{{ pg_version }}" restart: true state: started - command: -c config_file=/etc/postgresql/postgresql.conf + command: -c config_file=/etc/postgresql/postgresql-primary.conf networks: - name: "{{ network_name }}" env: POSTGRES_PASSWORD: "{{ root_password }}" volumes: - "{{ data_dir }}/node{{ item }}/data:/var/lib/postgresql/data" - - "./data/postgres-replica.conf:/etc/postgresql/postgresql.conf:ro" + - "./data/postgres-replica.conf:/etc/postgresql/postgresql-primary.conf:ro" - "{{ data_dir }}/node1/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro" ports: - "{{ pdpgsql_port + item - 1}}:5432" diff --git a/pmm_qa/pg_stat_monitor_setup.sh b/pmm_qa/pg_stat_monitor_setup.sh index cc6fb54e..010f7838 100644 --- a/pmm_qa/pg_stat_monitor_setup.sh +++ b/pmm_qa/pg_stat_monitor_setup.sh @@ -95,7 +95,7 @@ make USE_PGXS=1 # Install built PGSM library into server make USE_PGXS=1 install -# Stop server and edit postgresql.conf to load PGSM library with required configurations +# Stop server and edit postgresql-primary.conf to load PGSM library with required configurations service postgresql stop echo "shared_preload_libraries = 'pg_stat_monitor'" >> /etc/postgresql/${pgsql_version}/main/postgresql.conf echo "track_activity_query_size=2048" >> /etc/postgresql/${pgsql_version}/main/postgresql.conf diff --git a/pmm_qa/pg_stat_statements_setup.sh b/pmm_qa/pg_stat_statements_setup.sh index 550d8131..c04f2493 100644 --- a/pmm_qa/pg_stat_statements_setup.sh +++ b/pmm_qa/pg_stat_statements_setup.sh @@ -71,7 +71,7 @@ export PATH="/usr/lib/postgresql/${pgsql_version}/bin:$PATH" echo $PATH cp /usr/lib/postgresql/${pgsql_version}/bin/pg_config /usr/bin -# Stop server and edit postgresql.conf to load pg_stat_sstatement library with required configurations +# Stop server and edit postgresql-primary.conf to load pg_stat_sstatement library with required configurations service postgresql stop echo "shared_preload_libraries = 'pg_stat_statements'" >> /etc/postgresql/${pgsql_version}/main/postgresql.conf echo "track_activity_query_size=2048" >> /etc/postgresql/${pgsql_version}/main/postgresql.conf diff --git a/pmm_qa/postgresql/data/pdpgsql.conf b/pmm_qa/postgresql/data/pdpgsql.conf deleted file mode 100644 index a4abdb84..00000000 --- a/pmm_qa/postgresql/data/pdpgsql.conf +++ /dev/null @@ -1 +0,0 @@ -hba_file = '/etc/postgresql/pg_hba.conf' \ No newline at end of file diff --git a/pmm_qa/postgresql/data/pdpgsql_hba.conf b/pmm_qa/postgresql/data/pdpgsql_hba.conf deleted file mode 100644 index 9b76134e..00000000 --- a/pmm_qa/postgresql/data/pdpgsql_hba.conf +++ /dev/null @@ -1,2 +0,0 @@ -host all all 0.0.0.0/0 md5 -local all postgres trust \ No newline at end of file diff --git a/pmm_qa/postgresql/data/pg-primary-conf/pg_hba.conf.j2 b/pmm_qa/postgresql/data/pg_hba.conf.j2 similarity index 100% rename from pmm_qa/postgresql/data/pg-primary-conf/pg_hba.conf.j2 rename to pmm_qa/postgresql/data/pg_hba.conf.j2 diff --git a/pmm_qa/postgresql/data/pg-primary-conf/postgresql.conf b/pmm_qa/postgresql/data/postgresql-primary.conf similarity index 100% rename from pmm_qa/postgresql/data/pg-primary-conf/postgresql.conf rename to pmm_qa/postgresql/data/postgresql-primary.conf diff --git a/pmm_qa/postgresql/data/sample_data.sql b/pmm_qa/postgresql/data/sample_data.sql deleted file mode 100644 index a1e5c59f..00000000 --- a/pmm_qa/postgresql/data/sample_data.sql +++ /dev/null @@ -1,38 +0,0 @@ --- Populate sample branches -INSERT INTO pgbench_branches (bid, bbalance, bname) -SELECT i, 100000, 'Branch ' || i -FROM generate_series(1, 10) AS i; - --- Populate sample tellers -INSERT INTO pgbench_tellers (tid, bid, tbalance, tname) -SELECT i, (i % 10) + 1, 10000, 'Teller ' || i -FROM generate_series(1, 50) AS i; - --- Populate sample accounts -INSERT INTO pgbench_accounts (aid, bid, abalance) -SELECT i, (i % 10) + 1, 1000 -FROM generate_series(1, 10000) AS i; - -DO $$ -BEGIN - IF NOT EXISTS ( - SELECT 1 - FROM information_schema.columns - WHERE table_name = 'pgbench_branches' - AND column_name = 'bname' - ) THEN - ALTER TABLE pgbench_branches ADD COLUMN bname TEXT DEFAULT 'Branch'; - END IF; -END$$; - -DO $$ -BEGIN - IF NOT EXISTS ( - SELECT 1 - FROM information_schema.columns - WHERE table_name = 'pgbench_tellers' - AND column_name = 'tname' - ) THEN - ALTER TABLE pgbench_tellers ADD COLUMN tname TEXT DEFAULT 'Teller'; - END IF; -END$$; \ No newline at end of file diff --git a/pmm_qa/postgresql/data/workload.sql b/pmm_qa/postgresql/data/workload.sql deleted file mode 100644 index 74dc9f2f..00000000 --- a/pmm_qa/postgresql/data/workload.sql +++ /dev/null @@ -1,28 +0,0 @@ -\set aid random(1, 10000) -\set delta random(-100, 100) -\set bid random(1, 10) -\set tid random(1, 50) - -BEGIN; - -SELECT abalance - FROM pgbench_accounts - WHERE aid = :aid; - -SELECT b.bid, b.bname, t.tid, t.tname - FROM pgbench_branches b - JOIN pgbench_tellers t ON b.bid = t.bid - WHERE b.bid = :bid AND t.tid = :tid; - -UPDATE pgbench_accounts - SET abalance = abalance - :delta - WHERE aid = :aid AND abalance >= :delta; - -UPDATE pgbench_accounts - SET abalance = abalance + :delta - WHERE aid = :aid + 1; - -INSERT INTO pgbench_history (tid, bid, aid, delta, mtime) -VALUES (:tid, :bid, :aid, :delta, CURRENT_TIMESTAMP); - -COMMIT; \ No newline at end of file diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index a62eb1b5..e20f2250 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -44,7 +44,7 @@ - name: Generate pg_hba.conf for primary node template: - src: ./data/pg-primary-conf/pg_hba.conf.j2 + src: data/pg_hba.conf.j2 dest: "{{ data_dir }}/node1/pg_hba.conf" - name: Remove old PostgreSQL primary container @@ -68,9 +68,9 @@ POSTGRES_PASSWORD: "{{ root_password }}" volumes: - "{{ data_dir }}/node1/data:/var/lib/postgresql/data" - - "./data/pg-primary-conf/postgresql.conf:/etc/postgresql/postgresql.conf:ro" + - "./data/pg-primary-conf/postgresql-primary.conf:/etc/postgresql/postgresql-primary.conf:ro" - "{{ data_dir }}/node1/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro" - command: -c config_file=/etc/postgresql/postgresql.conf + command: -c config_file=/etc/postgresql/postgresql-primary.conf ports: - "{{ pgsql_port }}:5432" @@ -111,7 +111,7 @@ POSTGRES_PASSWORD: "{{ root_password }}" volumes: - "{{ data_dir }}/node{{ item }}/data:/var/lib/postgresql/data" - - "./data/pg-primary-conf/postgresql-replica.conf:/etc/postgresql/postgresql.conf:ro" + - "./data/pg-primary-conf/postgresql-replica.conf:/etc/postgresql/postgresql-primary.conf:ro" loop: "{{ range(2, nodes_count | int + 1) | list }}" - name: Wipe replica data directory before basebackup @@ -139,14 +139,14 @@ name: "pgsql_pmm_{{ pg_version }}_{{ item }}" restart: true state: started - command: -c config_file=/etc/postgresql/postgresql.conf + command: -c config_file=/etc/postgresql/postgresql-primary.conf networks: - name: "{{ network_name }}" env: POSTGRES_PASSWORD: "{{ root_password }}" volumes: - "{{ data_dir }}/node{{ item }}/data:/var/lib/postgresql/data" - - "./data/postgres-replica.conf:/etc/postgresql/postgresql.conf:ro" + - "./data/postgres-replica.conf:/etc/postgresql/postgresql-primary.conf:ro" ports: - "{{ pgsql_port + item - 1 }}:5432" loop: "{{ range(2, nodes_count | int + 1) | list }}" From da17ac3ff08ffe30ca3f4dfd6841cb5947f17b9d Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 17:24:27 +0200 Subject: [PATCH 177/340] PMM-7: Add pdpgsql --- pmm_qa/postgresql/postgresql-setup.yml | 6 +++--- pmm_qa/postgresql/tasks/run_load_pgsql.yml | 19 ------------------- 2 files changed, 3 insertions(+), 22 deletions(-) diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index e20f2250..7404fcf7 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -68,9 +68,9 @@ POSTGRES_PASSWORD: "{{ root_password }}" volumes: - "{{ data_dir }}/node1/data:/var/lib/postgresql/data" - - "./data/pg-primary-conf/postgresql-primary.conf:/etc/postgresql/postgresql-primary.conf:ro" + - "./data/postgresql-primary.conf:/etc/postgresql/postgresql.conf:ro" - "{{ data_dir }}/node1/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro" - command: -c config_file=/etc/postgresql/postgresql-primary.conf + command: -c config_file=/etc/postgresql/postgresql.conf ports: - "{{ pgsql_port }}:5432" @@ -111,7 +111,7 @@ POSTGRES_PASSWORD: "{{ root_password }}" volumes: - "{{ data_dir }}/node{{ item }}/data:/var/lib/postgresql/data" - - "./data/pg-primary-conf/postgresql-replica.conf:/etc/postgresql/postgresql-primary.conf:ro" + - "./data/postgresql-replica.conf:/etc/postgresql/postgresql-primary.conf:ro" loop: "{{ range(2, nodes_count | int + 1) | list }}" - name: Wipe replica data directory before basebackup diff --git a/pmm_qa/postgresql/tasks/run_load_pgsql.yml b/pmm_qa/postgresql/tasks/run_load_pgsql.yml index ea72bff4..a22cba70 100644 --- a/pmm_qa/postgresql/tasks/run_load_pgsql.yml +++ b/pmm_qa/postgresql/tasks/run_load_pgsql.yml @@ -59,25 +59,6 @@ command: > pgbench -i -s {{ pgbench_scale }} pgbench -- name: Copy populate script into container - community.docker.docker_container_copy_into: - container: "{{ node_name }}" - path: ./data/sample_data.sql - container_path: /sample_data.sql - -- name: Populate pgbench tables with sample data - community.docker.docker_container_exec: - container: "{{ node_name }}" - user: postgres - command: > - psql -U postgres -d pgbench -f /sample_data.sql - -- name: Copy realistic workload SQL into container - community.docker.docker_container_copy_into: - container: "{{ node_name }}" - path: ./data/workload.sql - container_path: /workload.sql - - name: Run pgbench benchmark community.docker.docker_container_exec: container: "{{ node_name }}" From e5a7b5867e9b209c2349893b36412265a6f2f154 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 17:26:42 +0200 Subject: [PATCH 178/340] PMM-7: Add pdpgsql --- pmm_qa/postgresql/data/postgres-replica.conf | 1 - 1 file changed, 1 deletion(-) diff --git a/pmm_qa/postgresql/data/postgres-replica.conf b/pmm_qa/postgresql/data/postgres-replica.conf index a26d04bd..13346e28 100644 --- a/pmm_qa/postgresql/data/postgres-replica.conf +++ b/pmm_qa/postgresql/data/postgres-replica.conf @@ -1,2 +1 @@ shared_preload_libraries = 'pg_stat_statements' -hba_file = '/etc/postgresql/pg_hba.conf' \ No newline at end of file From cd01e5b70574b949e8a6de85b455a33d3ea7747d Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 17:35:37 +0200 Subject: [PATCH 179/340] PMM-7: Add pdpgsql --- .../data/postgres-replica.conf | 2 ++ .../percona-distribution-postgres-setup.yml | 9 +++++---- 2 files changed, 7 insertions(+), 4 deletions(-) create mode 100644 pmm_qa/percona-distribution-postgresql/data/postgres-replica.conf diff --git a/pmm_qa/percona-distribution-postgresql/data/postgres-replica.conf b/pmm_qa/percona-distribution-postgresql/data/postgres-replica.conf new file mode 100644 index 00000000..a26d04bd --- /dev/null +++ b/pmm_qa/percona-distribution-postgresql/data/postgres-replica.conf @@ -0,0 +1,2 @@ +shared_preload_libraries = 'pg_stat_statements' +hba_file = '/etc/postgresql/pg_hba.conf' \ No newline at end of file diff --git a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml index facfcd3b..20de3f42 100644 --- a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml @@ -46,6 +46,7 @@ template: src: ./data/pg_hba.conf.j2 dest: "{{ data_dir }}/node1/pg_hba.conf" + loop: "{{ range(1, nodes_count | int + 1) | list }}" - name: Remove old Percona Distribution PostgreSQL primary container community.docker.docker_container: @@ -68,9 +69,9 @@ POSTGRES_PASSWORD: "{{ root_password }}" volumes: - "{{ data_dir }}/node1/data:/var/lib/postgresql/data" - - "./data/postgresql-primary.conf:/etc/postgresql/postgresql-primary.conf:ro" + - "./data/postgresql-primary.conf:/etc/postgresql/postgresql.conf:ro" - "{{ data_dir }}/node1/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro" - command: -c config_file=/etc/postgresql/postgresql-primary.conf + command: -c config_file=/etc/postgresql/postgresql.conf ports: - "{{ pdpgsql_port }}:5432" @@ -109,8 +110,8 @@ POSTGRES_PASSWORD: "{{ root_password }}" volumes: - "{{ data_dir }}/node{{ item }}/data:/var/lib/postgresql/data" - - "./data/postgres-replica.conf:/etc/postgresql/postgresql-primary.conf:ro" - - "{{ data_dir }}/node1/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro" + - "./data/postgres-replica.conf:/etc/postgresql/postgresql.conf:ro" + - "{{ data_dir }}/node{{ item }}/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro" loop: "{{ range(2, nodes_count | int + 1) | list }}" - name: Wipe replica data directory before basebackup From 8242e648cfcb97a3614c88aa14700ebd27d05a20 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 17:37:09 +0200 Subject: [PATCH 180/340] PMM-7: Add pdpgsql --- .../percona-distribution-postgres-setup.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml index 20de3f42..3e3b63e0 100644 --- a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml @@ -159,7 +159,7 @@ user: postgres command: > bash -c ' - PGPASSWORD="{{ root_password }}" psql -h 127.0.0.1 -U postgres -d {{ db_name | default("postgres") }} -c " + PGPASSWORD="{{ root_password }}" psql -U postgres -d {{ db_name | default("postgres") }} -c " DO \$$ BEGIN IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = '\''pmm'\'') THEN From c259d6064b5bdcb01f4ab88d5d8121aec78c8ae8 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 17:42:18 +0200 Subject: [PATCH 181/340] PMM-7: Add pdpgsql --- .../percona-distribution-postgres-setup.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml index 3e3b63e0..c21dc8f8 100644 --- a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml @@ -159,7 +159,7 @@ user: postgres command: > bash -c ' - PGPASSWORD="{{ root_password }}" psql -U postgres -d {{ db_name | default("postgres") }} -c " + PGPASSWORD="{{ root_password }}" psql -h /tmp -U postgres -d {{ db_name | default("postgres") }} -c " DO \$$ BEGIN IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = '\''pmm'\'') THEN From 0613806133db917d4129385fd0b45e97276719ae Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 17:46:06 +0200 Subject: [PATCH 182/340] PMM-7: Add pdpgsql --- .../percona-distribution-postgres-setup.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml index c21dc8f8..459513a1 100644 --- a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml @@ -153,6 +153,10 @@ - "{{ pdpgsql_port + item - 1}}:5432" loop: "{{ range(2, nodes_count | int + 1) | list }}" + - name: Wait 10 for server to restart + pause: + seconds: 10 + - name: Create PostgreSQL user 'pmm' with password community.docker.docker_container_exec: container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" From f62049b9f725ce3ed47ea39ddc1bba9a7e550407 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 17:49:13 +0200 Subject: [PATCH 183/340] PMM-7: Add pdpgsql --- .../percona-distribution-postgres-setup.yml | 8 ++++---- pmm_qa/tasks/install_pmm_client.yml | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml index 459513a1..f437fa55 100644 --- a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml @@ -96,7 +96,7 @@ loop: "{{ range(2, nodes_count | int + 1) | list }}" ignore_errors: yes - - name: Start PostgreSQL replica container + - name: Start Percona Distribution PostgreSQL replica container community.docker.docker_container: name: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" image: "percona/percona-distribution-postgresql:{{ pg_version }}" @@ -134,7 +134,7 @@ " loop: "{{ range(2, nodes_count | int + 1) | list }}" - - name: Restart PostgreSQL container with custom command + - name: Restart Percona Distribution PostgreSQL container with custom command community.docker.docker_container: name: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" image: "percona/percona-distribution-postgresql:{{ pg_version }}" @@ -153,11 +153,11 @@ - "{{ pdpgsql_port + item - 1}}:5432" loop: "{{ range(2, nodes_count | int + 1) | list }}" - - name: Wait 10 for server to restart + - name: Wait 10 seconds for server to restart pause: seconds: 10 - - name: Create PostgreSQL user 'pmm' with password + - name: Create Percona Distribution PostgreSQL user 'pmm' with password community.docker.docker_container_exec: container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" user: postgres diff --git a/pmm_qa/tasks/install_pmm_client.yml b/pmm_qa/tasks/install_pmm_client.yml index 1d19d90c..00ec9bfe 100644 --- a/pmm_qa/tasks/install_pmm_client.yml +++ b/pmm_qa/tasks/install_pmm_client.yml @@ -49,7 +49,7 @@ ) | trim }} -- name: Install wget inside Debian-family container +- name: Install dependencies inside Debian-family container community.docker.docker_container_exec: container: "{{ container_name }}" command: > @@ -60,10 +60,10 @@ user: "root" when: distro_family == "debian" -- name: Install wget on RHEL-family containers +- name: Install dependencies on RHEL-family containers community.docker.docker_container_exec: container: "{{ container_name }}" - command: microdnf install -y wget gnupg2 jq curl + command: microdnf install -y wget gnupg2 jq curl-minimal user: "root" when: distro_family == "rhel" From 1387f04aaff12187003acdc69ddc065469875c8f Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 17:51:28 +0200 Subject: [PATCH 184/340] PMM-7: Add pdpgsql --- pmm_qa/tasks/install_pmm_client.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/tasks/install_pmm_client.yml b/pmm_qa/tasks/install_pmm_client.yml index 00ec9bfe..a7040e5f 100644 --- a/pmm_qa/tasks/install_pmm_client.yml +++ b/pmm_qa/tasks/install_pmm_client.yml @@ -63,7 +63,7 @@ - name: Install dependencies on RHEL-family containers community.docker.docker_container_exec: container: "{{ container_name }}" - command: microdnf install -y wget gnupg2 jq curl-minimal + command: microdnf install -y wget gnupg2 jq curl-minimal lsb-base lsb-release user: "root" when: distro_family == "rhel" From 1f48633e83e3323730d4fd979911f8b3c4a9eb93 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 17:52:43 +0200 Subject: [PATCH 185/340] PMM-7: Add pdpgsql --- pmm_qa/tasks/install_pmm_client.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/tasks/install_pmm_client.yml b/pmm_qa/tasks/install_pmm_client.yml index a7040e5f..c2f2f8ba 100644 --- a/pmm_qa/tasks/install_pmm_client.yml +++ b/pmm_qa/tasks/install_pmm_client.yml @@ -63,7 +63,7 @@ - name: Install dependencies on RHEL-family containers community.docker.docker_container_exec: container: "{{ container_name }}" - command: microdnf install -y wget gnupg2 jq curl-minimal lsb-base lsb-release + command: microdnf install -y wget gnupg2 jq curl-minimal lsb user: "root" when: distro_family == "rhel" From f69fbb7f57188bf139b607d376ebc0c9f83961bd Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 17:54:29 +0200 Subject: [PATCH 186/340] PMM-7: Add pdpgsql --- pmm_qa/tasks/install_pmm_client.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/tasks/install_pmm_client.yml b/pmm_qa/tasks/install_pmm_client.yml index c2f2f8ba..159821fe 100644 --- a/pmm_qa/tasks/install_pmm_client.yml +++ b/pmm_qa/tasks/install_pmm_client.yml @@ -63,7 +63,7 @@ - name: Install dependencies on RHEL-family containers community.docker.docker_container_exec: container: "{{ container_name }}" - command: microdnf install -y wget gnupg2 jq curl-minimal lsb + command: microdnf install -y wget gnupg2 jq curl-minimal redhat-lsb-core user: "root" when: distro_family == "rhel" From 5e8600bdf03dd4e090f344a5a01625c96b918c92 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 18:01:40 +0200 Subject: [PATCH 187/340] PMM-7: Add pdpgsql --- .../percona-distribution-postgres-setup.yml | 2 +- pmm_qa/postgresql/postgresql-setup.yml | 2 +- pmm_qa/tasks/install_pmm_client.yml | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml index f437fa55..27196640 100644 --- a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml @@ -211,5 +211,5 @@ - name: Add service to pmm server community.docker.docker_container_exec: container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" - command: pmm-admin add postgresql --username=pmm --password=pmm pgsql_pmm_{{ pg_version }}_{{ item }}{{ random_service_name_value }} --debug 127.0.0.1:5432 + command: pmm-admin add postgresql --username=pmm --password=pmm --query-source=pgstatmonitor pgsql_pmm_{{ pg_version }}_{{ item }}{{ random_service_name_value }} --debug 127.0.0.1:5432 loop: "{{ range(1, nodes_count | int + 1) | list }}" diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index 7404fcf7..bac483fe 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -205,7 +205,7 @@ - name: Add service to pmm server community.docker.docker_container_exec: container: "pgsql_pmm_{{ pg_version }}_{{ item }}" - command: pmm-admin add postgresql --username=pmm --password=pmm pgsql_pmm_{{ pg_version }}_{{ item }}{{ random_service_name_value }} --debug 127.0.0.1:5432 + command: pmm-admin add postgresql --username=pmm --password=pmm --query-source=pgstatements pgsql_pmm_{{ pg_version }}_{{ item }}{{ random_service_name_value }} --debug 127.0.0.1:5432 loop: "{{ range(1, nodes_count | int + 1) | list }}" - name: Run load on primary node. diff --git a/pmm_qa/tasks/install_pmm_client.yml b/pmm_qa/tasks/install_pmm_client.yml index 159821fe..5c21ecd8 100644 --- a/pmm_qa/tasks/install_pmm_client.yml +++ b/pmm_qa/tasks/install_pmm_client.yml @@ -63,14 +63,14 @@ - name: Install dependencies on RHEL-family containers community.docker.docker_container_exec: container: "{{ container_name }}" - command: microdnf install -y wget gnupg2 jq curl-minimal redhat-lsb-core + command: microdnf install -y wget gnupg2 jq curl-minimal user: "root" when: distro_family == "rhel" - name: Get lsb_release community.docker.docker_container_exec: container: "{{ container_name }}" - command: lsb_release -sc + command: cat /etc/os-release | grep '^ID=' | awk -F'"' '{print $2}' user: "root" register: lsb_release From ed4f4f4e4f63392a39e71f0bbefa5fe13247c103 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 18:05:25 +0200 Subject: [PATCH 188/340] PMM-7: Add pdpgsql --- pmm_qa/tasks/install_pmm_client.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pmm_qa/tasks/install_pmm_client.yml b/pmm_qa/tasks/install_pmm_client.yml index 5c21ecd8..f32846f5 100644 --- a/pmm_qa/tasks/install_pmm_client.yml +++ b/pmm_qa/tasks/install_pmm_client.yml @@ -222,14 +222,14 @@ community.docker.docker_container_exec: container: "{{ container_name }}" user: "root" - command: pmm-agent setup --config-file=/usr/local/percona/pmm/config/pmm-agent.yaml --server-address={{ pmm_server_ip }}:{{ pmm_server_port }} --server-insecure-tls --metrics-mode={{ metrics_mode }} --server-username=admin --server-password={{ admin_password }} + command: pmm-agent setup --config-file=/usr/local/percona/pmm/config/pmm-agent.yaml --server-address={{ pmm_server_ip }}:{{ pmm_server_port }} --server-insecure-tls --metrics-mode={{ metrics_mode }} --node-name={{ container_name }} --server-username=admin --server-password={{ admin_password }} when: metrics_mode | length > 0 - name: Connect pmm client to pmm server using metrics mode community.docker.docker_container_exec: container: "{{ container_name }}" user: "root" - command: pmm-agent setup --config-file=/usr/local/percona/pmm/config/pmm-agent.yaml --server-address={{ pmm_server_ip }}:{{ pmm_server_port }} --server-insecure-tls --server-username=admin --server-password={{ admin_password }} + command: pmm-agent setup --config-file=/usr/local/percona/pmm/config/pmm-agent.yaml --server-address={{ pmm_server_ip }}:{{ pmm_server_port }} --node-name={{ container_name }} --server-insecure-tls --server-username=admin --server-password={{ admin_password }} when: metrics_mode | length == 0 - name: Wait 5 seconds for connection to complete From 470337fbc905d9b2484a17713f4480f6e5d13dc4 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 18:06:24 +0200 Subject: [PATCH 189/340] PMM-7: Add pdpgsql --- pmm_qa/tasks/install_pmm_client.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pmm_qa/tasks/install_pmm_client.yml b/pmm_qa/tasks/install_pmm_client.yml index f32846f5..22772e62 100644 --- a/pmm_qa/tasks/install_pmm_client.yml +++ b/pmm_qa/tasks/install_pmm_client.yml @@ -70,7 +70,8 @@ - name: Get lsb_release community.docker.docker_container_exec: container: "{{ container_name }}" - command: cat /etc/os-release | grep '^ID=' | awk -F'"' '{print $2}' + command: > + bash -c "grep '^ID=' /etc/os-release | cut -d '=' -f2 | tr -d '\"'" user: "root" register: lsb_release From 0863f914a0f681ea47ce2b010e604b2733686842 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 18:08:21 +0200 Subject: [PATCH 190/340] PMM-7: Add pdpgsql --- pmm_qa/tasks/install_pmm_client.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/pmm_qa/tasks/install_pmm_client.yml b/pmm_qa/tasks/install_pmm_client.yml index 22772e62..a9835a44 100644 --- a/pmm_qa/tasks/install_pmm_client.yml +++ b/pmm_qa/tasks/install_pmm_client.yml @@ -106,6 +106,7 @@ ' when: - distro_family == 'rhel' + ignore_errors: true - name: Install pmm client experimental on Debian-family containers community.docker.docker_container_exec: From c0033d083ac56f294a19035034350851248205b9 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 18:11:51 +0200 Subject: [PATCH 191/340] PMM-7: Add pdpgsql --- pmm_qa/tasks/install_pmm_client.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pmm_qa/tasks/install_pmm_client.yml b/pmm_qa/tasks/install_pmm_client.yml index a9835a44..3832b506 100644 --- a/pmm_qa/tasks/install_pmm_client.yml +++ b/pmm_qa/tasks/install_pmm_client.yml @@ -224,14 +224,14 @@ community.docker.docker_container_exec: container: "{{ container_name }}" user: "root" - command: pmm-agent setup --config-file=/usr/local/percona/pmm/config/pmm-agent.yaml --server-address={{ pmm_server_ip }}:{{ pmm_server_port }} --server-insecure-tls --metrics-mode={{ metrics_mode }} --node-name={{ container_name }} --server-username=admin --server-password={{ admin_password }} + command: pmm-agent setup --config-file=/usr/local/percona/pmm/config/pmm-agent.yaml --server-address={{ pmm_server_ip }}:{{ pmm_server_port }} --server-insecure-tls --metrics-mode={{ metrics_mode }} --server-username=admin --server-password={{ admin_password }} {{ container_name }} when: metrics_mode | length > 0 - name: Connect pmm client to pmm server using metrics mode community.docker.docker_container_exec: container: "{{ container_name }}" user: "root" - command: pmm-agent setup --config-file=/usr/local/percona/pmm/config/pmm-agent.yaml --server-address={{ pmm_server_ip }}:{{ pmm_server_port }} --node-name={{ container_name }} --server-insecure-tls --server-username=admin --server-password={{ admin_password }} + command: pmm-agent setup --config-file=/usr/local/percona/pmm/config/pmm-agent.yaml --server-address={{ pmm_server_ip }}:{{ pmm_server_port }} --server-insecure-tls --server-username=admin --server-password={{ admin_password }} {{ container_name }} when: metrics_mode | length == 0 - name: Wait 5 seconds for connection to complete From a75c5ae2fa2f109f6e8d30f26744adcd21189b16 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 18:15:31 +0200 Subject: [PATCH 192/340] PMM-7: Add pdpgsql --- pmm_qa/tasks/install_pmm_client.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/tasks/install_pmm_client.yml b/pmm_qa/tasks/install_pmm_client.yml index 3832b506..0ba0d50d 100644 --- a/pmm_qa/tasks/install_pmm_client.yml +++ b/pmm_qa/tasks/install_pmm_client.yml @@ -71,7 +71,7 @@ community.docker.docker_container_exec: container: "{{ container_name }}" command: > - bash -c "grep '^ID=' /etc/os-release | cut -d '=' -f2 | tr -d '\"'" + bash -c "grep '^VERSION_CODENAME=' /etc/os-release | cut -d '=' -f2 | tr -d '\"'" user: "root" register: lsb_release From 3570db1fc2181907d31a4ab5fb528c6cbc5c5d36 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 18:16:12 +0200 Subject: [PATCH 193/340] PMM-7: Add pdpgsql --- .../percona-distribution-postgres-setup.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml index 27196640..92d6c127 100644 --- a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml @@ -211,5 +211,5 @@ - name: Add service to pmm server community.docker.docker_container_exec: container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" - command: pmm-admin add postgresql --username=pmm --password=pmm --query-source=pgstatmonitor pgsql_pmm_{{ pg_version }}_{{ item }}{{ random_service_name_value }} --debug 127.0.0.1:5432 + command: pmm-admin add postgresql --username=pmm --password=pmm --query-source=pgstatmonitor pdpgsql_pmm_{{ pg_version }}_{{ item }}{{ random_service_name_value }} --debug 127.0.0.1:5432 loop: "{{ range(1, nodes_count | int + 1) | list }}" From 6536c6e10dc25cc7a207cfc534e044196daf4796 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 18:20:17 +0200 Subject: [PATCH 194/340] PMM-7: Add pdpgsql --- .../percona-distribution-postgresql/data/postgres-replica.conf | 2 +- .../data/postgresql-primary.conf | 2 +- .../percona-distribution-postgres-setup.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pmm_qa/percona-distribution-postgresql/data/postgres-replica.conf b/pmm_qa/percona-distribution-postgresql/data/postgres-replica.conf index a26d04bd..e41320a2 100644 --- a/pmm_qa/percona-distribution-postgresql/data/postgres-replica.conf +++ b/pmm_qa/percona-distribution-postgresql/data/postgres-replica.conf @@ -1,2 +1,2 @@ -shared_preload_libraries = 'pg_stat_statements' +shared_preload_libraries = 'pg_stat_monitor' hba_file = '/etc/postgresql/pg_hba.conf' \ No newline at end of file diff --git a/pmm_qa/percona-distribution-postgresql/data/postgresql-primary.conf b/pmm_qa/percona-distribution-postgresql/data/postgresql-primary.conf index 8356b760..6cc8f147 100644 --- a/pmm_qa/percona-distribution-postgresql/data/postgresql-primary.conf +++ b/pmm_qa/percona-distribution-postgresql/data/postgresql-primary.conf @@ -4,4 +4,4 @@ wal_keep_size = 64MB hot_standby = on listen_addresses = '*' hba_file = '/etc/postgresql/pg_hba.conf' -shared_preload_libraries = 'pg_stat_statements' \ No newline at end of file +shared_preload_libraries = 'pg_stat_monitor' \ No newline at end of file diff --git a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml index 92d6c127..f2d3d52f 100644 --- a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml @@ -180,7 +180,7 @@ user: postgres command: > psql -U postgres -d 'postgres' -c " - CREATE EXTENSION IF NOT EXISTS pg_stat_statements; + CREATE EXTENSION IF NOT EXISTS pg_stat_monitor; " - name: Install and add pmm client. From d7c71b663c8d7c465ba6dcaa01f85fb9b51f24f3 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 20:20:39 +0200 Subject: [PATCH 195/340] PMM-7: Add pdpgsql --- pmm_qa/postgresql/postgresql-setup.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index bac483fe..8db42aaf 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -205,7 +205,7 @@ - name: Add service to pmm server community.docker.docker_container_exec: container: "pgsql_pmm_{{ pg_version }}_{{ item }}" - command: pmm-admin add postgresql --username=pmm --password=pmm --query-source=pgstatements pgsql_pmm_{{ pg_version }}_{{ item }}{{ random_service_name_value }} --debug 127.0.0.1:5432 + command: pmm-admin add postgresql --username=root --password={{ root_password }} --query-source=pgstatements pgsql_pmm_{{ pg_version }}_{{ item }}{{ random_service_name_value }} --debug 127.0.0.1:5432 loop: "{{ range(1, nodes_count | int + 1) | list }}" - name: Run load on primary node. From bc5fb3b251a000299cfeef6dcbaf2da9d05003b6 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 20:25:46 +0200 Subject: [PATCH 196/340] PMM-7: Add pgsql --- .../data/postgres-replica.conf | 5 ++++- .../data/postgresql-primary.conf | 5 ++++- pmm_qa/postgresql/postgresql-setup.yml | 12 ++++-------- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/pmm_qa/percona-distribution-postgresql/data/postgres-replica.conf b/pmm_qa/percona-distribution-postgresql/data/postgres-replica.conf index e41320a2..1a43c6ad 100644 --- a/pmm_qa/percona-distribution-postgresql/data/postgres-replica.conf +++ b/pmm_qa/percona-distribution-postgresql/data/postgres-replica.conf @@ -1,2 +1,5 @@ shared_preload_libraries = 'pg_stat_monitor' -hba_file = '/etc/postgresql/pg_hba.conf' \ No newline at end of file +hba_file = '/etc/postgresql/pg_hba.conf' +pg_stat_statements.track=all +track_io_timing=ON +track_activity_query_size=2048 \ No newline at end of file diff --git a/pmm_qa/percona-distribution-postgresql/data/postgresql-primary.conf b/pmm_qa/percona-distribution-postgresql/data/postgresql-primary.conf index 6cc8f147..597de4cd 100644 --- a/pmm_qa/percona-distribution-postgresql/data/postgresql-primary.conf +++ b/pmm_qa/percona-distribution-postgresql/data/postgresql-primary.conf @@ -4,4 +4,7 @@ wal_keep_size = 64MB hot_standby = on listen_addresses = '*' hba_file = '/etc/postgresql/pg_hba.conf' -shared_preload_libraries = 'pg_stat_monitor' \ No newline at end of file +shared_preload_libraries = 'pg_stat_monitor' +pg_stat_statements.track=all +track_io_timing=ON +track_activity_query_size=2048 \ No newline at end of file diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index 8db42aaf..925b3a5f 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -158,13 +158,9 @@ command: > bash -c ' psql -U postgres -d {{ db_name | default("postgres") }} -c " - DO \$$ - BEGIN - IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = '\''pmm'\'') THEN - CREATE ROLE pmm LOGIN PASSWORD '\''pmm'\''; - END IF; - END - \$$;" + CREATE USER pmm WITH PASSWORD 'pmm'; + GRANT pg_monitor TO pmm; + " ' loop: "{{ range(1, nodes_count | int + 1) | list }}" @@ -205,7 +201,7 @@ - name: Add service to pmm server community.docker.docker_container_exec: container: "pgsql_pmm_{{ pg_version }}_{{ item }}" - command: pmm-admin add postgresql --username=root --password={{ root_password }} --query-source=pgstatements pgsql_pmm_{{ pg_version }}_{{ item }}{{ random_service_name_value }} --debug 127.0.0.1:5432 + command: pmm-admin add postgresql --username=pmm --password=pmm --query-source=pgstatements pgsql_pmm_{{ pg_version }}_{{ item }}{{ random_service_name_value }} --debug 127.0.0.1:5432 loop: "{{ range(1, nodes_count | int + 1) | list }}" - name: Run load on primary node. From 48b7a7c16fd2c04d00a51abecfc1b7c7607189ca Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 20:29:26 +0200 Subject: [PATCH 197/340] PMM-7: Add pgsql --- .../data/postgres-replica.conf | 7 ++++--- .../data/postgresql-primary.conf | 6 ++++-- .../percona-distribution-postgres-setup.yml | 11 ++++------- pmm_qa/postgresql/data/postgres-replica.conf | 4 ++++ pmm_qa/postgresql/data/postgresql-primary.conf | 5 ++++- 5 files changed, 20 insertions(+), 13 deletions(-) diff --git a/pmm_qa/percona-distribution-postgresql/data/postgres-replica.conf b/pmm_qa/percona-distribution-postgresql/data/postgres-replica.conf index 1a43c6ad..9e0382a0 100644 --- a/pmm_qa/percona-distribution-postgresql/data/postgres-replica.conf +++ b/pmm_qa/percona-distribution-postgresql/data/postgres-replica.conf @@ -1,5 +1,6 @@ shared_preload_libraries = 'pg_stat_monitor' -hba_file = '/etc/postgresql/pg_hba.conf' -pg_stat_statements.track=all +track_activity_query_size=2048 track_io_timing=ON -track_activity_query_size=2048 \ No newline at end of file +max_connections=1000 +listen_addresses = '*' +pg_stat_monitor.pgsm_enable_query_plan = 'yes' \ No newline at end of file diff --git a/pmm_qa/percona-distribution-postgresql/data/postgresql-primary.conf b/pmm_qa/percona-distribution-postgresql/data/postgresql-primary.conf index 597de4cd..9e8686b5 100644 --- a/pmm_qa/percona-distribution-postgresql/data/postgresql-primary.conf +++ b/pmm_qa/percona-distribution-postgresql/data/postgresql-primary.conf @@ -5,6 +5,8 @@ hot_standby = on listen_addresses = '*' hba_file = '/etc/postgresql/pg_hba.conf' shared_preload_libraries = 'pg_stat_monitor' -pg_stat_statements.track=all +track_activity_query_size=2048 track_io_timing=ON -track_activity_query_size=2048 \ No newline at end of file +max_connections=1000 +listen_addresses = '*' +pg_stat_monitor.pgsm_enable_query_plan = 'yes' \ No newline at end of file diff --git a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml index f2d3d52f..b2f2c4ec 100644 --- a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml @@ -164,13 +164,9 @@ command: > bash -c ' PGPASSWORD="{{ root_password }}" psql -h /tmp -U postgres -d {{ db_name | default("postgres") }} -c " - DO \$$ - BEGIN - IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = '\''pmm'\'') THEN - CREATE ROLE pmm LOGIN PASSWORD '\''pmm'\''; - END IF; - END - \$$;" + CREATE USER pmm WITH SUPERUSER ENCRYPTED PASSWORD 'pmm'; + GRANT pg_monitor TO pmm; + " ' loop: "{{ range(1, nodes_count | int + 1) | list }}" @@ -180,6 +176,7 @@ user: postgres command: > psql -U postgres -d 'postgres' -c " + CREATE DATABASE contrib_regression; CREATE EXTENSION IF NOT EXISTS pg_stat_monitor; " diff --git a/pmm_qa/postgresql/data/postgres-replica.conf b/pmm_qa/postgresql/data/postgres-replica.conf index 13346e28..c97e6dfd 100644 --- a/pmm_qa/postgresql/data/postgres-replica.conf +++ b/pmm_qa/postgresql/data/postgres-replica.conf @@ -1 +1,5 @@ shared_preload_libraries = 'pg_stat_statements' +hba_file = '/etc/postgresql/pg_hba.conf' +pg_stat_statements.track=all +track_io_timing=ON +track_activity_query_size=2048 \ No newline at end of file diff --git a/pmm_qa/postgresql/data/postgresql-primary.conf b/pmm_qa/postgresql/data/postgresql-primary.conf index 8356b760..fb099965 100644 --- a/pmm_qa/postgresql/data/postgresql-primary.conf +++ b/pmm_qa/postgresql/data/postgresql-primary.conf @@ -4,4 +4,7 @@ wal_keep_size = 64MB hot_standby = on listen_addresses = '*' hba_file = '/etc/postgresql/pg_hba.conf' -shared_preload_libraries = 'pg_stat_statements' \ No newline at end of file +shared_preload_libraries = 'pg_stat_statements' +pg_stat_statements.track=all +track_io_timing=ON +track_activity_query_size=2048 \ No newline at end of file From 290402774d0a710ee0d3acc64bf4670bdea4255b Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 20:31:04 +0200 Subject: [PATCH 198/340] PMM-7: Add pgsql --- pmm_qa/postgresql/postgresql-setup.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index 925b3a5f..283000aa 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -158,7 +158,7 @@ command: > bash -c ' psql -U postgres -d {{ db_name | default("postgres") }} -c " - CREATE USER pmm WITH PASSWORD 'pmm'; + CREATE USER pmm WITH PASSWORD '\''pmm'\''; GRANT pg_monitor TO pmm; " ' From 116ee76e6a252a642bc00d4251fd9e9ef57f7191 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 20:33:12 +0200 Subject: [PATCH 199/340] PMM-7: Add pgsql --- .../percona-distribution-postgres-setup.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml index b2f2c4ec..5cf95e21 100644 --- a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml @@ -164,7 +164,7 @@ command: > bash -c ' PGPASSWORD="{{ root_password }}" psql -h /tmp -U postgres -d {{ db_name | default("postgres") }} -c " - CREATE USER pmm WITH SUPERUSER ENCRYPTED PASSWORD 'pmm'; + CREATE USER pmm WITH SUPERUSER ENCRYPTED PASSWORD '\''pmm'\''; GRANT pg_monitor TO pmm; " ' From 4931238bb1226308d4ca37f448daeb848a0780ab Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 20:37:26 +0200 Subject: [PATCH 200/340] PMM-7: Add pgsql --- .../percona-distribution-postgres-setup.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml index 5cf95e21..538ce707 100644 --- a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml @@ -176,7 +176,6 @@ user: postgres command: > psql -U postgres -d 'postgres' -c " - CREATE DATABASE contrib_regression; CREATE EXTENSION IF NOT EXISTS pg_stat_monitor; " From f388423b8de94d2da8032370ffcea5b40750854c Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 20:39:21 +0200 Subject: [PATCH 201/340] PMM-7: Add pgsql --- .../percona-distribution-postgres-setup.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml index 538ce707..2ade1b00 100644 --- a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml @@ -148,7 +148,7 @@ volumes: - "{{ data_dir }}/node{{ item }}/data:/var/lib/postgresql/data" - "./data/postgres-replica.conf:/etc/postgresql/postgresql-primary.conf:ro" - - "{{ data_dir }}/node1/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro" +# - "{{ data_dir }}/node1/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro" ports: - "{{ pdpgsql_port + item - 1}}:5432" loop: "{{ range(2, nodes_count | int + 1) | list }}" From dfaa2225f5b6925b1c216e3ac181a73508e8ac6e Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 20:43:49 +0200 Subject: [PATCH 202/340] PMM-7: Add pgsql --- .../percona-distribution-postgres-setup.yml | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml index 2ade1b00..a2121e76 100644 --- a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml @@ -203,6 +203,37 @@ random_service_name_value: "_{{ 9999 | random + 1 }}" loop: "{{ range(1, nodes_count | int + 1) | list }}" when: "('pgsql_pmm_' ~ pg_version ~ '_' ~ item) in pmm_server_services.stdout" + - name: Set distro family (debian/rhel) + set_fact: + distro_family: >- + {{ + ( + 'debian' if 'debian' in container_os_info.stdout | lower else + 'rhel' if 'rhel' in container_os_info.stdout | lower or 'centos' in container_os_info.stdout | lower or 'fedora' in container_os_info.stdout | lower + else 'unknown' + ) | trim + }} + + - name: Install dependencies inside Debian-family container + community.docker.docker_container_exec: + container: "{{ container_name }}" + command: > + sh -c ' + apt-get update && + apt-get install percona-pg-stat-monitor{{ pg_version }} + ' + user: "root" + when: distro_family == "debian" + + - name: Install dependencies inside Debian-family container + community.docker.docker_container_exec: + container: "{{ container_name }}" + command: > + sh -c ' + microdnf install percona-pg-stat-monitor{{ pg_version }} + ' + user: "root" + when: distro_family == "rhel" - name: Add service to pmm server community.docker.docker_container_exec: From 3c69fdb04d67035a3040bb57c1151f1d1931a367 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 20:45:14 +0200 Subject: [PATCH 203/340] PMM-7: Add pgsql --- pmm_qa/postgresql/postgresql-setup.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index 283000aa..2c7a1a73 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -139,7 +139,7 @@ name: "pgsql_pmm_{{ pg_version }}_{{ item }}" restart: true state: started - command: -c config_file=/etc/postgresql/postgresql-primary.conf + command: -c config_file=/etc/postgresql/postgresql-replica.conf networks: - name: "{{ network_name }}" env: From 50178ea5a0eb0688849712b7f4eb978ad7e0c607 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 20:46:47 +0200 Subject: [PATCH 204/340] PMM-7: Add pgsql --- .../percona-distribution-postgres-setup.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml index a2121e76..b70862fa 100644 --- a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml @@ -216,7 +216,7 @@ - name: Install dependencies inside Debian-family container community.docker.docker_container_exec: - container: "{{ container_name }}" + container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" command: > sh -c ' apt-get update && @@ -224,16 +224,18 @@ ' user: "root" when: distro_family == "debian" + loop: "{{ range(1, nodes_count | int + 1) | list }}" - - name: Install dependencies inside Debian-family container + - name: Install dependencies inside RHEL-family container community.docker.docker_container_exec: - container: "{{ container_name }}" + container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" command: > sh -c ' microdnf install percona-pg-stat-monitor{{ pg_version }} ' user: "root" when: distro_family == "rhel" + loop: "{{ range(1, nodes_count | int + 1) | list }}" - name: Add service to pmm server community.docker.docker_container_exec: From d803857d6922eabd203c437aa3366e59a5aa88fa Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 20:47:24 +0200 Subject: [PATCH 205/340] PMM-7: Add pgsql --- pmm_qa/postgresql/postgresql-setup.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index 2c7a1a73..9ac3b902 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -139,14 +139,14 @@ name: "pgsql_pmm_{{ pg_version }}_{{ item }}" restart: true state: started - command: -c config_file=/etc/postgresql/postgresql-replica.conf + command: -c config_file=/etc/postgresql/postgresql.conf networks: - name: "{{ network_name }}" env: POSTGRES_PASSWORD: "{{ root_password }}" volumes: - "{{ data_dir }}/node{{ item }}/data:/var/lib/postgresql/data" - - "./data/postgres-replica.conf:/etc/postgresql/postgresql-primary.conf:ro" + - "./data/postgres-replica.conf:/etc/postgresql/postgresql.conf:ro" ports: - "{{ pgsql_port + item - 1 }}:5432" loop: "{{ range(2, nodes_count | int + 1) | list }}" From 7219ebcb03c73aca9f2c2118add20bca9defa2bd Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 20:49:26 +0200 Subject: [PATCH 206/340] PMM-7: Add pgsql --- pmm_qa/postgresql/data/postgres-replica.conf | 1 - pmm_qa/postgresql/postgresql-setup.yml | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/pmm_qa/postgresql/data/postgres-replica.conf b/pmm_qa/postgresql/data/postgres-replica.conf index c97e6dfd..c3666295 100644 --- a/pmm_qa/postgresql/data/postgres-replica.conf +++ b/pmm_qa/postgresql/data/postgres-replica.conf @@ -1,5 +1,4 @@ shared_preload_libraries = 'pg_stat_statements' -hba_file = '/etc/postgresql/pg_hba.conf' pg_stat_statements.track=all track_io_timing=ON track_activity_query_size=2048 \ No newline at end of file diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index 9ac3b902..96fc1a45 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -111,7 +111,7 @@ POSTGRES_PASSWORD: "{{ root_password }}" volumes: - "{{ data_dir }}/node{{ item }}/data:/var/lib/postgresql/data" - - "./data/postgresql-replica.conf:/etc/postgresql/postgresql-primary.conf:ro" + - "./data/postgresql-replica.conf:/etc/postgresql/postgresql.conf:ro" loop: "{{ range(2, nodes_count | int + 1) | list }}" - name: Wipe replica data directory before basebackup From d16286713b9bbc8a46cea817223e1e7aafa5dbbe Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 20:54:20 +0200 Subject: [PATCH 207/340] PMM-7: Add pgsql --- pmm_qa/postgresql/postgresql-setup.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index 96fc1a45..0c8c19a3 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -164,6 +164,17 @@ ' loop: "{{ range(1, nodes_count | int + 1) | list }}" + - name: Create PostgreSQL user 'pmm' with password + community.docker.docker_container_exec: + container: "pgsql_pmm_{{ pg_version }}_1" + user: postgres + command: > + bash -c ' + psql -U postgres -d {{ db_name | default("postgres") }} -c " + GRANT pg_monitor TO pmm; + " + ' + - name: Create pg_stat_statements extension community.docker.docker_container_exec: container: "pgsql_pmm_{{ pg_version }}_1" From 077cb4d7099c885289e9d4ba410417a4c2c42d19 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 20:54:32 +0200 Subject: [PATCH 208/340] PMM-7: Add pgsql --- pmm_qa/postgresql/postgresql-setup.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index 0c8c19a3..7c514267 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -159,7 +159,6 @@ bash -c ' psql -U postgres -d {{ db_name | default("postgres") }} -c " CREATE USER pmm WITH PASSWORD '\''pmm'\''; - GRANT pg_monitor TO pmm; " ' loop: "{{ range(1, nodes_count | int + 1) | list }}" From 80f921e1346df7d34cb0534ce9cd58d81ce8e458 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 20:55:55 +0200 Subject: [PATCH 209/340] PMM-7: Add pgsql --- pmm_qa/postgresql/postgresql-setup.yml | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index 7c514267..9dd9fd61 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -151,18 +151,6 @@ - "{{ pgsql_port + item - 1 }}:5432" loop: "{{ range(2, nodes_count | int + 1) | list }}" - - name: Create PostgreSQL user 'pmm' with password - community.docker.docker_container_exec: - container: "pgsql_pmm_{{ pg_version }}_{{ item }}" - user: postgres - command: > - bash -c ' - psql -U postgres -d {{ db_name | default("postgres") }} -c " - CREATE USER pmm WITH PASSWORD '\''pmm'\''; - " - ' - loop: "{{ range(1, nodes_count | int + 1) | list }}" - - name: Create PostgreSQL user 'pmm' with password community.docker.docker_container_exec: container: "pgsql_pmm_{{ pg_version }}_1" @@ -170,6 +158,7 @@ command: > bash -c ' psql -U postgres -d {{ db_name | default("postgres") }} -c " + CREATE USER pmm WITH PASSWORD '\''pmm'\''; GRANT pg_monitor TO pmm; " ' From 9b23a419e15d019daa156181d88cc62549150e41 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 20:59:08 +0200 Subject: [PATCH 210/340] PMM-7: Add pgsql --- .../data/postgres-replica.conf | 5 ++++- .../percona-distribution-postgres-setup.yml | 19 +++++++++++-------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/pmm_qa/percona-distribution-postgresql/data/postgres-replica.conf b/pmm_qa/percona-distribution-postgresql/data/postgres-replica.conf index 9e0382a0..a5308f98 100644 --- a/pmm_qa/percona-distribution-postgresql/data/postgres-replica.conf +++ b/pmm_qa/percona-distribution-postgresql/data/postgres-replica.conf @@ -3,4 +3,7 @@ track_activity_query_size=2048 track_io_timing=ON max_connections=1000 listen_addresses = '*' -pg_stat_monitor.pgsm_enable_query_plan = 'yes' \ No newline at end of file +pg_stat_monitor.pgsm_enable_query_plan = 'yes' +pg_stat_monitor.pgsm_query_max_len = 2048 +pg_stat_monitor.pgsm_normalized_query=1 +pg_stat_monitor.pgsm_enable_query_plan=1 \ No newline at end of file diff --git a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml index b70862fa..e314d335 100644 --- a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml @@ -170,14 +170,7 @@ ' loop: "{{ range(1, nodes_count | int + 1) | list }}" - - name: Create pg_stat_statements extension - community.docker.docker_container_exec: - container: "pdpgsql_pmm_{{ pg_version }}_1" - user: postgres - command: > - psql -U postgres -d 'postgres' -c " - CREATE EXTENSION IF NOT EXISTS pg_stat_monitor; - " + - name: Install and add pmm client. include_tasks: ../tasks/install_pmm_client.yml @@ -237,6 +230,16 @@ when: distro_family == "rhel" loop: "{{ range(1, nodes_count | int + 1) | list }}" + - name: Create pg_stat_statements extension + community.docker.docker_container_exec: + container: "pdpgsql_pmm_{{ pg_version }}_1" + user: postgres + command: > + psql -U postgres -d 'postgres' -c " + CREATE EXTENSION IF NOT EXISTS pg_stat_monitor; + SELECT pg_stat_monitor_version(); + " + - name: Add service to pmm server community.docker.docker_container_exec: container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" From 7c85d02329dce25c9c76d011c181dd7087549e16 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 21:00:14 +0200 Subject: [PATCH 211/340] PMM-7: Add pgsql --- .../percona-distribution-postgres-setup.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml index e314d335..8a4ffbb0 100644 --- a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml @@ -232,13 +232,14 @@ - name: Create pg_stat_statements extension community.docker.docker_container_exec: - container: "pdpgsql_pmm_{{ pg_version }}_1" + container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" user: postgres command: > psql -U postgres -d 'postgres' -c " CREATE EXTENSION IF NOT EXISTS pg_stat_monitor; SELECT pg_stat_monitor_version(); " + loop: "{{ range(1, nodes_count | int + 1) | list }}" - name: Add service to pmm server community.docker.docker_container_exec: From 948d3e3137d3d870883a1e2d62768aee4b6904e8 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 21:03:56 +0200 Subject: [PATCH 212/340] PMM-7: Add pgsql --- .../percona-distribution-postgres-setup.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml index 8a4ffbb0..caae85e3 100644 --- a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml @@ -246,3 +246,17 @@ container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" command: pmm-admin add postgresql --username=pmm --password=pmm --query-source=pgstatmonitor pdpgsql_pmm_{{ pg_version }}_{{ item }}{{ random_service_name_value }} --debug 127.0.0.1:5432 loop: "{{ range(1, nodes_count | int + 1) | list }}" + + - name: Run load on primary node. + include_tasks: ./tasks/run_load_pgsql.yml + vars: + node_name: "pdpgsql_pmm_{{ pg_version }}_1" + pgbench_clients: 10 + pgbench_time: 120 + pgbench_scale: 1000 + + - name: Check replication status on primary + community.docker.docker_container_exec: + container: "pgsql_pmm_{{ pg_version }}_1" + user: postgres + command: psql -c "SELECT * FROM pg_stat_replication;" From 75c4ecf1323654eede2826b8ca015455105673e3 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 21:12:04 +0200 Subject: [PATCH 213/340] PMM-7: Add pgsql --- .../percona-distribution-postgres-setup.yml | 2 +- pmm_qa/postgresql/tasks/run_load_pgsql.yml | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml index caae85e3..1d590b37 100644 --- a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml @@ -248,7 +248,7 @@ loop: "{{ range(1, nodes_count | int + 1) | list }}" - name: Run load on primary node. - include_tasks: ./tasks/run_load_pgsql.yml + include_tasks: ../postgresql/tasks/run_load_pgsql.yml vars: node_name: "pdpgsql_pmm_{{ pg_version }}_1" pgbench_clients: 10 diff --git a/pmm_qa/postgresql/tasks/run_load_pgsql.yml b/pmm_qa/postgresql/tasks/run_load_pgsql.yml index a22cba70..5cc1df92 100644 --- a/pmm_qa/postgresql/tasks/run_load_pgsql.yml +++ b/pmm_qa/postgresql/tasks/run_load_pgsql.yml @@ -65,7 +65,6 @@ user: postgres command: > pgbench -c {{ pgbench_clients }} -T {{ pgbench_time }} -j 4 pgbench -# -f /workload.sql pgbench register: pgbench_result - name: Print pgbench results From fb96bf664ff3d8762d61eec4673231a1bfcd81e6 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 7 May 2025 21:17:28 +0200 Subject: [PATCH 214/340] PMM-7: Add pgsql --- .../percona-distribution-postgres-setup.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml index 1d590b37..4d078708 100644 --- a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml @@ -255,6 +255,17 @@ pgbench_time: 120 pgbench_scale: 1000 + - name: Create pg_stat_statements extension + community.docker.docker_container_exec: + container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" + user: postgres + command: > + psql -U postgres -d 'pgbench' -c " + CREATE EXTENSION IF NOT EXISTS pg_stat_monitor; + SELECT pg_stat_monitor_version(); + " + loop: "{{ range(1, nodes_count | int + 1) | list }}" + - name: Check replication status on primary community.docker.docker_container_exec: container: "pgsql_pmm_{{ pg_version }}_1" From 29972655d6d7ff29bd324d8355dad421b59f2a1b Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Thu, 8 May 2025 16:50:34 +0200 Subject: [PATCH 215/340] PMM-7: Add pgsql --- pmm_qa/postgresql/postgresql-setup.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index 9dd9fd61..4b594c0a 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -129,7 +129,7 @@ bash -c " export PGPASSWORD='{{ replication_password }}' && \ timeout 60s \ - pg_basebackup --pgdata=/var/lib/postgresql/data -R -v -P \ + pg_basebackup --pgdata=/var/lib/postgresql/data -R -v -Fp -Xs -P \ --host=pgsql_pmm_{{ pg_version }}_1 --port=5432 -U {{ replication_user }} " loop: "{{ range(2, nodes_count | int + 1) | list }}" From 4576eaeb5d95398ef1400ba917f7b177f826a530 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Thu, 8 May 2025 17:06:17 +0200 Subject: [PATCH 216/340] PMM-7: Add pgsql --- .../percona-distribution-postgres-setup.yml | 44 +++++++++++++------ pmm_qa/postgresql/tasks/run_load_pgsql.yml | 17 +------ 2 files changed, 32 insertions(+), 29 deletions(-) diff --git a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml index 4d078708..c1d02286 100644 --- a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml @@ -114,6 +114,37 @@ - "{{ data_dir }}/node{{ item }}/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro" loop: "{{ range(2, nodes_count | int + 1) | list }}" + - name: Create custom database for pgbench + community.docker.docker_container_exec: + container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" + user: postgres + command: > + bash -c " + echo \" + CREATE DATABASE pgbench; + \\c pgbench + GRANT CONNECT ON DATABASE pgbench TO pmm; + GRANT USAGE ON SCHEMA public TO pmm; + GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public TO pmm; + ALTER DEFAULT PRIVILEGES IN SCHEMA public + GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO pmm; + \" | psql -U postgres -v ON_ERROR_STOP=1 + " + loop: "{{ range(1, nodes_count | int + 1) | list }}" + + - name: Create Percona Distribution PostgreSQL user 'pmm' with password + community.docker.docker_container_exec: + container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" + user: postgres + command: > + bash -c ' + PGPASSWORD="{{ root_password }}" psql -h /tmp -U postgres -d {{ db_name | default("postgres") }} -c " + CREATE USER pmm WITH SUPERUSER ENCRYPTED PASSWORD '\''pmm'\''; + GRANT pg_monitor TO pmm; + " + ' + loop: "{{ range(1, nodes_count | int + 1) | list }}" + - name: Wipe replica data directory before basebackup community.docker.docker_container_exec: container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" @@ -157,19 +188,6 @@ pause: seconds: 10 - - name: Create Percona Distribution PostgreSQL user 'pmm' with password - community.docker.docker_container_exec: - container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" - user: postgres - command: > - bash -c ' - PGPASSWORD="{{ root_password }}" psql -h /tmp -U postgres -d {{ db_name | default("postgres") }} -c " - CREATE USER pmm WITH SUPERUSER ENCRYPTED PASSWORD '\''pmm'\''; - GRANT pg_monitor TO pmm; - " - ' - loop: "{{ range(1, nodes_count | int + 1) | list }}" - - name: Install and add pmm client. diff --git a/pmm_qa/postgresql/tasks/run_load_pgsql.yml b/pmm_qa/postgresql/tasks/run_load_pgsql.yml index 5cc1df92..d1b46305 100644 --- a/pmm_qa/postgresql/tasks/run_load_pgsql.yml +++ b/pmm_qa/postgresql/tasks/run_load_pgsql.yml @@ -17,22 +17,7 @@ ) | trim }} -- name: Create custom database for pgbench - community.docker.docker_container_exec: - container: "{{ node_name }}" - user: postgres - command: > - bash -c " - echo \" - CREATE DATABASE pgbench; - \\c pgbench - GRANT CONNECT ON DATABASE pgbench TO pmm; - GRANT USAGE ON SCHEMA public TO pmm; - GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public TO pmm; - ALTER DEFAULT PRIVILEGES IN SCHEMA public - GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO pmm; - \" | psql -U postgres -v ON_ERROR_STOP=1 - " + - name: Ensure pgbench is installed (Debian-based container) community.docker.docker_container_exec: From c18e6eba77793e98c2583b7cb52e069bed71232d Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Thu, 8 May 2025 17:17:19 +0200 Subject: [PATCH 217/340] PMM-7: Add pgsql --- .../percona-distribution-postgres-setup.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml index c1d02286..f5e64155 100644 --- a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml @@ -114,6 +114,13 @@ - "{{ data_dir }}/node{{ item }}/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro" loop: "{{ range(2, nodes_count | int + 1) | list }}" + - name: Wipe replica data directory before basebackup + community.docker.docker_container_exec: + container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" + user: root + command: rm -rf /var/lib/postgresql/data/* + loop: "{{ range(2, nodes_count | int + 1) | list }}" + - name: Create custom database for pgbench community.docker.docker_container_exec: container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" @@ -145,13 +152,6 @@ ' loop: "{{ range(1, nodes_count | int + 1) | list }}" - - name: Wipe replica data directory before basebackup - community.docker.docker_container_exec: - container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" - user: root - command: rm -rf /var/lib/postgresql/data/* - loop: "{{ range(2, nodes_count | int + 1) | list }}" - - name: Run pg_basebackup from primary to replica community.docker.docker_container_exec: container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" From 173132a32ec2cf30d4ae13986ca35d8db1c02f95 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Thu, 8 May 2025 17:25:10 +0200 Subject: [PATCH 218/340] PMM-7: Add pgsql --- .../percona-distribution-postgres-setup.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml index f5e64155..e14ff40b 100644 --- a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml @@ -188,8 +188,6 @@ pause: seconds: 10 - - - name: Install and add pmm client. include_tasks: ../tasks/install_pmm_client.yml vars: From 7187264b1b8fb77abfb05441f281b64174330057 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Thu, 8 May 2025 17:29:11 +0200 Subject: [PATCH 219/340] PMM-7: Add pgsql --- pmm_qa/postgresql/postgresql-setup.yml | 43 +++++++++++++++++++------- 1 file changed, 32 insertions(+), 11 deletions(-) diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index 4b594c0a..20e041cf 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -97,6 +97,24 @@ loop: "{{ range(2, nodes_count | int + 1) | list }}" ignore_errors: yes + - name: Create custom database for pgbench + community.docker.docker_container_exec: + container: "pgsql_pmm_{{ pg_version }}_{{ item }}" + user: postgres + command: > + bash -c " + echo \" + CREATE DATABASE pgbench; + \\c pgbench + GRANT CONNECT ON DATABASE pgbench TO pmm; + GRANT USAGE ON SCHEMA public TO pmm; + GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public TO pmm; + ALTER DEFAULT PRIVILEGES IN SCHEMA public + GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO pmm; + \" | psql -U postgres -v ON_ERROR_STOP=1 + " + loop: "{{ range(1, nodes_count | int + 1) | list }}" + - name: Start PostgreSQL replica container community.docker.docker_container: name: "pgsql_pmm_{{ pg_version }}_{{ item }}" @@ -121,6 +139,19 @@ command: rm -rf /var/lib/postgresql/data/* loop: "{{ range(2, nodes_count | int + 1) | list }}" + - name: Create PostgreSQL user 'pmm' with password + community.docker.docker_container_exec: + container: "pgsql_pmm_{{ pg_version }}_{{ item }}" + user: postgres + command: > + bash -c ' + psql -U postgres -d {{ db_name | default("postgres") }} -c " + CREATE USER pmm WITH PASSWORD '\''pmm'\''; + GRANT pg_monitor TO pmm; + " + ' + loop: "{{ range(1, nodes_count | int + 1) | list }}" + - name: Run pg_basebackup from primary to replica community.docker.docker_container_exec: container: "pgsql_pmm_{{ pg_version }}_{{ item }}" @@ -151,17 +182,7 @@ - "{{ pgsql_port + item - 1 }}:5432" loop: "{{ range(2, nodes_count | int + 1) | list }}" - - name: Create PostgreSQL user 'pmm' with password - community.docker.docker_container_exec: - container: "pgsql_pmm_{{ pg_version }}_1" - user: postgres - command: > - bash -c ' - psql -U postgres -d {{ db_name | default("postgres") }} -c " - CREATE USER pmm WITH PASSWORD '\''pmm'\''; - GRANT pg_monitor TO pmm; - " - ' + - name: Create pg_stat_statements extension community.docker.docker_container_exec: From 120157410d65a061b02b5081a3d1f85b489b3363 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Thu, 8 May 2025 17:33:46 +0200 Subject: [PATCH 220/340] PMM-7: Add pgsql --- pmm_qa/postgresql/postgresql-setup.yml | 36 ++++++++++++++------------ 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index 20e041cf..939d0a06 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -97,23 +97,7 @@ loop: "{{ range(2, nodes_count | int + 1) | list }}" ignore_errors: yes - - name: Create custom database for pgbench - community.docker.docker_container_exec: - container: "pgsql_pmm_{{ pg_version }}_{{ item }}" - user: postgres - command: > - bash -c " - echo \" - CREATE DATABASE pgbench; - \\c pgbench - GRANT CONNECT ON DATABASE pgbench TO pmm; - GRANT USAGE ON SCHEMA public TO pmm; - GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public TO pmm; - ALTER DEFAULT PRIVILEGES IN SCHEMA public - GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO pmm; - \" | psql -U postgres -v ON_ERROR_STOP=1 - " - loop: "{{ range(1, nodes_count | int + 1) | list }}" + - name: Start PostgreSQL replica container community.docker.docker_container: @@ -152,6 +136,24 @@ ' loop: "{{ range(1, nodes_count | int + 1) | list }}" + - name: Create custom database for pgbench + community.docker.docker_container_exec: + container: "pgsql_pmm_{{ pg_version }}_{{ item }}" + user: postgres + command: > + bash -c " + echo \" + CREATE DATABASE pgbench; + \\c pgbench + GRANT CONNECT ON DATABASE pgbench TO pmm; + GRANT USAGE ON SCHEMA public TO pmm; + GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public TO pmm; + ALTER DEFAULT PRIVILEGES IN SCHEMA public + GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO pmm; + \" | psql -U postgres -v ON_ERROR_STOP=1 + " + loop: "{{ range(1, nodes_count | int + 1) | list }}" + - name: Run pg_basebackup from primary to replica community.docker.docker_container_exec: container: "pgsql_pmm_{{ pg_version }}_{{ item }}" From 8a529e98c5a1b1a6093740018b36547a2ba999d3 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Thu, 8 May 2025 17:37:57 +0200 Subject: [PATCH 221/340] PMM-7: Add pgsql --- pmm_qa/postgresql/postgresql-setup.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index 939d0a06..7cc12d83 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -125,7 +125,7 @@ - name: Create PostgreSQL user 'pmm' with password community.docker.docker_container_exec: - container: "pgsql_pmm_{{ pg_version }}_{{ item }}" + container: "pgsql_pmm_{{ pg_version }}_1" user: postgres command: > bash -c ' @@ -134,11 +134,10 @@ GRANT pg_monitor TO pmm; " ' - loop: "{{ range(1, nodes_count | int + 1) | list }}" - name: Create custom database for pgbench community.docker.docker_container_exec: - container: "pgsql_pmm_{{ pg_version }}_{{ item }}" + container: "pgsql_pmm_{{ pg_version }}_1" user: postgres command: > bash -c " @@ -152,7 +151,6 @@ GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO pmm; \" | psql -U postgres -v ON_ERROR_STOP=1 " - loop: "{{ range(1, nodes_count | int + 1) | list }}" - name: Run pg_basebackup from primary to replica community.docker.docker_container_exec: From 9b04d9c5c8778feb0e16ba3be32982acb00e9977 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Thu, 8 May 2025 17:44:30 +0200 Subject: [PATCH 222/340] PMM-7: Add pgsql --- pmm_qa/postgresql/postgresql-setup.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index 7cc12d83..a7f82e2e 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -159,7 +159,7 @@ command: > bash -c " export PGPASSWORD='{{ replication_password }}' && \ - timeout 60s \ + timeout 120s \ pg_basebackup --pgdata=/var/lib/postgresql/data -R -v -Fp -Xs -P \ --host=pgsql_pmm_{{ pg_version }}_1 --port=5432 -U {{ replication_user }} " From b954a960e34e4c484ddc87d33999c10dbaa545d8 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Thu, 8 May 2025 20:27:33 +0200 Subject: [PATCH 223/340] PMM-7: Add pgsql --- .../percona-distribution-postgresql/data/postgres-replica.conf | 1 + pmm_qa/postgresql/postgresql-setup.yml | 2 ++ 2 files changed, 3 insertions(+) diff --git a/pmm_qa/percona-distribution-postgresql/data/postgres-replica.conf b/pmm_qa/percona-distribution-postgresql/data/postgres-replica.conf index a5308f98..ab63afb9 100644 --- a/pmm_qa/percona-distribution-postgresql/data/postgres-replica.conf +++ b/pmm_qa/percona-distribution-postgresql/data/postgres-replica.conf @@ -1,3 +1,4 @@ +hba_file = '/etc/postgresql/pg_hba.conf' shared_preload_libraries = 'pg_stat_monitor' track_activity_query_size=2048 track_io_timing=ON diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index a7f82e2e..05e7e13f 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -114,6 +114,7 @@ volumes: - "{{ data_dir }}/node{{ item }}/data:/var/lib/postgresql/data" - "./data/postgresql-replica.conf:/etc/postgresql/postgresql.conf:ro" + - "{{ data_dir }}/node1/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro" loop: "{{ range(2, nodes_count | int + 1) | list }}" - name: Wipe replica data directory before basebackup @@ -178,6 +179,7 @@ volumes: - "{{ data_dir }}/node{{ item }}/data:/var/lib/postgresql/data" - "./data/postgres-replica.conf:/etc/postgresql/postgresql.conf:ro" + - "{{ data_dir }}/node1/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro" ports: - "{{ pgsql_port + item - 1 }}:5432" loop: "{{ range(2, nodes_count | int + 1) | list }}" From 4e25f6a70a0d8fb6af882800e78e56d8fdf6a2b1 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Thu, 8 May 2025 20:39:42 +0200 Subject: [PATCH 224/340] PMM-7: Add pgsql --- pmm_qa/postgresql/data/postgres-replica.conf | 1 + 1 file changed, 1 insertion(+) diff --git a/pmm_qa/postgresql/data/postgres-replica.conf b/pmm_qa/postgresql/data/postgres-replica.conf index c3666295..92b1987f 100644 --- a/pmm_qa/postgresql/data/postgres-replica.conf +++ b/pmm_qa/postgresql/data/postgres-replica.conf @@ -1,3 +1,4 @@ +hba_file = '/etc/postgresql/pg_hba.conf' shared_preload_libraries = 'pg_stat_statements' pg_stat_statements.track=all track_io_timing=ON From 3f4ea3fcaa3a950e8732d9d34472df67200cbb31 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Thu, 8 May 2025 20:50:29 +0200 Subject: [PATCH 225/340] PMM-7: Add pgsql --- pmm_qa/postgresql/postgresql-setup.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index 05e7e13f..24cdd4d3 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -105,7 +105,7 @@ image: "postgres:{{ pg_version }}-bookworm" restart_policy: "no" state: started - command: sleep infinity +# command: sleep infinity networks: - name: "{{ network_name }}" env: @@ -184,8 +184,6 @@ - "{{ pgsql_port + item - 1 }}:5432" loop: "{{ range(2, nodes_count | int + 1) | list }}" - - - name: Create pg_stat_statements extension community.docker.docker_container_exec: container: "pgsql_pmm_{{ pg_version }}_1" From 85bfcc864dc73592715a416c9d6bd2dd1fe1edc7 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Thu, 8 May 2025 20:50:37 +0200 Subject: [PATCH 226/340] PMM-7: Add pgsql --- pmm_qa/postgresql/postgresql-setup.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index 24cdd4d3..b390f512 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -109,7 +109,7 @@ networks: - name: "{{ network_name }}" env: - POSTGRES_INITDB_SKIP: "yes" +# POSTGRES_INITDB_SKIP: "yes" POSTGRES_PASSWORD: "{{ root_password }}" volumes: - "{{ data_dir }}/node{{ item }}/data:/var/lib/postgresql/data" From a68117e83b4454df6ba95567b7e856e02cc6ba5f Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Thu, 8 May 2025 20:51:46 +0200 Subject: [PATCH 227/340] PMM-7: Add pgsql --- pmm_qa/postgresql/postgresql-setup.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index b390f512..d350eed3 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -105,7 +105,7 @@ image: "postgres:{{ pg_version }}-bookworm" restart_policy: "no" state: started -# command: sleep infinity + command: sleep infinity networks: - name: "{{ network_name }}" env: From 6882e19d664fed39e57699679bbced71ae67bc61 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Thu, 8 May 2025 21:01:20 +0200 Subject: [PATCH 228/340] PMM-7: Add pgsql --- pmm_qa/postgresql/postgresql-setup.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index d350eed3..3cc52443 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -97,8 +97,6 @@ loop: "{{ range(2, nodes_count | int + 1) | list }}" ignore_errors: yes - - - name: Start PostgreSQL replica container community.docker.docker_container: name: "pgsql_pmm_{{ pg_version }}_{{ item }}" @@ -109,7 +107,6 @@ networks: - name: "{{ network_name }}" env: -# POSTGRES_INITDB_SKIP: "yes" POSTGRES_PASSWORD: "{{ root_password }}" volumes: - "{{ data_dir }}/node{{ item }}/data:/var/lib/postgresql/data" @@ -169,6 +166,7 @@ - name: Restart PostgreSQL container with custom command community.docker.docker_container: name: "pgsql_pmm_{{ pg_version }}_{{ item }}" + image: "postgres:{{ pg_version }}-bookworm" restart: true state: started command: -c config_file=/etc/postgresql/postgresql.conf From 0e249ccd7b8910f7fce8474647be80ad60e4df18 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Thu, 8 May 2025 21:05:23 +0200 Subject: [PATCH 229/340] PMM-7: Add pgsql --- pmm_qa/postgresql/data/postgres-replica.conf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pmm_qa/postgresql/data/postgres-replica.conf b/pmm_qa/postgresql/data/postgres-replica.conf index 92b1987f..52c86776 100644 --- a/pmm_qa/postgresql/data/postgres-replica.conf +++ b/pmm_qa/postgresql/data/postgres-replica.conf @@ -2,4 +2,5 @@ hba_file = '/etc/postgresql/pg_hba.conf' shared_preload_libraries = 'pg_stat_statements' pg_stat_statements.track=all track_io_timing=ON -track_activity_query_size=2048 \ No newline at end of file +track_activity_query_size=2048 +listen_addresses = '*' \ No newline at end of file From 263eb16ba9f231ba524424c7ff5dc6e3db3b7555 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 12 May 2025 08:41:44 +0200 Subject: [PATCH 230/340] PMM-7: Fix pmm client install --- pmm_qa/tasks/install_pmm_client.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pmm_qa/tasks/install_pmm_client.yml b/pmm_qa/tasks/install_pmm_client.yml index 0ba0d50d..7599c596 100644 --- a/pmm_qa/tasks/install_pmm_client.yml +++ b/pmm_qa/tasks/install_pmm_client.yml @@ -169,7 +169,11 @@ community.docker.docker_container_exec: container: "{{ container_name }}" user: "root" - command: microdnf install -y pmm-client + command: > + sh -c ' + percona-release enable-only pmm3-client release && + microdnf install -y pmm-client + ' when: distro_family == "rhel" and client_version == "pmm3-latest" - name: Install specific PMM client version on Debian-family containers From f0fda84e89f4000afae065bceff0e5e21ff8b514 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 12 May 2025 08:47:54 +0200 Subject: [PATCH 231/340] PMM-7: Fix pmm client install --- pmm_qa/tasks/install_pmm_client.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/pmm_qa/tasks/install_pmm_client.yml b/pmm_qa/tasks/install_pmm_client.yml index 7599c596..de83b510 100644 --- a/pmm_qa/tasks/install_pmm_client.yml +++ b/pmm_qa/tasks/install_pmm_client.yml @@ -160,6 +160,7 @@ user: "root" command: > sh -c ' + percona-release enable-only pmm3-client release && apt-get update && apt-get -y install pmm-client ' From 93c8559dd7f491f9f79779043f5971659b88b605 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Mon, 12 May 2025 09:32:50 +0200 Subject: [PATCH 232/340] PMM-7: New MySQL setup --- pmm_qa/mysql/mysql_setup.yml | 236 ++++++++++++++++++ pmm_qa/mysql/tasks/mysql-setup-single.yml | 31 +++ .../percona_server/percona-server-setup.yml | 2 +- ...up.yml => percona-server-setup-single.yml} | 0 4 files changed, 268 insertions(+), 1 deletion(-) create mode 100644 pmm_qa/mysql/mysql_setup.yml create mode 100644 pmm_qa/mysql/tasks/mysql-setup-single.yml rename pmm_qa/percona_server/tasks/{percona-server-setup.yml => percona-server-setup-single.yml} (100%) diff --git a/pmm_qa/mysql/mysql_setup.yml b/pmm_qa/mysql/mysql_setup.yml new file mode 100644 index 00000000..4523a815 --- /dev/null +++ b/pmm_qa/mysql/mysql_setup.yml @@ -0,0 +1,236 @@ +--- +# MySQL 8.4 and higher single instance and also Cluster with Group Replication +- name: Setup MySQL 8.4 and higher. Cluster with Group Replication in Docker + hosts: localhost + connection: local + gather_facts: yes + vars: + ps_version: "{{ lookup('env', 'PS_VERSION') | default('8.4', true) }}" + cluster_name: "mysql_cluster" + replication_user: "repl_user" + replication_password: "GRgrO9301RuF" + root_password: "GRgrO9301RuF" + mysql_port: 33066 + mysql_listen_port: 3306 + group_seeds_port: 34061 + nodes_count: "{{ (lookup('env', 'NODES_COUNT') | default('3', true)) | int }}" + network_name: "pmm-qa" + data_dir: "{{ lookup('env', 'HOME') }}/mysql_cluster_data" + server_id_start: 1 + pmm_server_ip: "{{ lookup('vars', 'extra_pmm_server_ip', default=lookup('env','PMM_SERVER_IP') | default('127.0.0.1', true) ) }}" + client_version: "{{ lookup('vars', 'extra_client_version', default=lookup('env','CLIENT_VERSION') | default('3-dev-latest', true) ) }}" + admin_password: "{{ lookup('vars', 'extra_admin_password', default=lookup('env','ADMIN_PASSWORD') | default('admin', true) ) }}" + query_source: "{{ lookup('env', 'QUERY_SOURCE') | default('perfschema', true) }}" + metrics_mode: "{{ lookup('env', 'metrics_mode') }}" + setup_type: "{{ lookup('env', 'SETUP_TYPE') }}" + random_service_name_value: "" + + tasks: + - name: Mofidy the node count for group replication + set_fact: + nodes_count: 3 + when: nodes_count | int < 3 and setup_type == "gr" + + - name: Chance to correct nodes count for async replication + set_fact: + nodes_count: 2 + when: nodes_count | int < 2 and setup_type == "replication" + + - name: Create Docker network + community.docker.docker_network: + name: "{{ network_name }}" + state: present + + - name: Remove old data folders + shell: 'rm -fr {{ data_dir }}' + loop: "{{ range(1, nodes_count | int + 1) | list }}" + + - name: Create data directories + file: + path: "{{ data_dir }}/node{{ item }}/data" + state: directory + mode: '0755' + loop: "{{ range(1, nodes_count | int + 1) | list }}" + + - name: Remove old MySQL containers + community.docker.docker_container: + name: "ps_pmm_{{ ps_version }}_{{ item }}" + image: "mysql:{{ ps_version }}" + restart_policy: always + state: absent + loop: "{{ range(1, nodes_count | int + 1) | list }}" + ignore_errors: yes + + - name: Recursively change ownership of a directory + shell: "sudo chown -R 1001:1001 {{ data_dir }}/node{{ item }}/data" + loop: "{{ range(1, nodes_count | int + 1) | list }}" + + - name: Setup MySQL group replication + include_tasks: ./tasks/percona-server-group-replication-setup.yml + when: setup_type == "gr" + + - name: Setup MySQL with async replication + include_tasks: ./tasks/percona-server-async-replication-setup.yml + when: setup_type == "replication" + + - name: Setup MySQL + include_tasks: tasks/mysql-setup-single.yml + when: setup_type != "gr" and setup_type != "replication" + + - name: Create slowlog configuration for mysql nodes + community.docker.docker_container_exec: + container: "ps_pmm_{{ ps_version }}_{{ item }}" + command: > + mysql -uroot -p{{ root_password }} -e " + SET GLOBAL slow_query_log='ON'; + SET GLOBAL long_query_time=0; + SET GLOBAL log_slow_admin_statements=ON; + SET GLOBAL log_slow_slave_statements=ON; + " + loop: "{{ range(1, nodes_count | int + 1) | list }}" + when: query_source == "slowlog" + + - name: Install and add pmm client. + include_tasks: ../tasks/install_pmm_client.yml + vars: + container_name: "ps_pmm_{{ ps_version }}_{{ item }}" + loop: "{{ range(1, nodes_count | int + 1) | list }}" + + - name: Get already connected services to pmm server + community.docker.docker_container_exec: + container: "ps_pmm_{{ ps_version }}_1" + command: > + sh -c 'curl --location --insecure -u"admin:{{ admin_password }}" -s --request GET "http://{{ pmm_server_ip }}:{{ '80' if pmm_server_ip is ansible.utils.ipv4 else '8080' }}/v1/management/services" | jq -r ".services[].service_name"' + register: pmm_server_services + + - name: Display already connected services to pmm server + debug: + msg: "{{ pmm_server_services.stdout | split('\n') }}" + + - name: Find out if service is already connected to pmm server + block: + - name: Loop through MySQL databases + set_fact: + random_service_name_value: "_{{ 9999 | random + 1 }}" + loop: "{{ range(1, nodes_count | int + 1) | list }}" + when: "('ps_pmm_' ~ ps_version ~ '_' ~ item) in pmm_server_services.stdout" + + - name: Add service to pmm server + community.docker.docker_container_exec: + container: "ps_pmm_{{ ps_version }}_{{ item }}" + command: pmm-admin add mysql --query-source={{ query_source }} --username=root --password={{ root_password }} --environment=ps-gr-dev --cluster=ps-gr-dev-cluster --replication-set=ps-gr-replication ps_pmm_{{ ps_version }}_{{ item }}{{ random_service_name_value }} --debug 127.0.0.1:3306 + loop: "{{ range(1, nodes_count | int + 1) | list }}" + when: setup_type == "gr" + + - name: Add service to pmm server + community.docker.docker_container_exec: + container: "ps_pmm_{{ ps_version }}_{{ item }}" + command: pmm-admin add mysql --query-source={{ query_source }} --username=root --password={{ root_password }} --environment=ps-replication-dev --cluster=ps-replication-dev-cluster --replication-set=ps-async-replication ps_pmm_{{ ps_version }}_{{ item }}{{ random_service_name_value }} --debug 127.0.0.1:3306 + loop: "{{ range(1, nodes_count | int + 1) | list }}" + when: setup_type == "replication" + + - name: Add service to pmm server + community.docker.docker_container_exec: + container: "ps_pmm_{{ ps_version }}_{{ item }}" + command: pmm-admin add mysql --query-source={{ query_source }} --username=root --password={{ root_password }} --environment=ps-dev ps_pmm_{{ ps_version }}_{{ item }}{{ random_service_name_value }} --debug 127.0.0.1:3306 + loop: "{{ range(1, nodes_count | int + 1) | list }}" + when: setup_type != "gr" and setup_type != "replication" + + - name: Install sysbench inside of all MySQL nodes + community.docker.docker_container_exec: + container: "ps_pmm_{{ ps_version }}_{{ item }}" + user: "root" + command: > + /bin/sh -c " + wget -O epel-release.rpm --progress=dot:giga https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && + rpm -i epel-release.rpm && + microdnf install -y sysbench + " + loop: "{{ range(1, nodes_count | int + 1) | list }}" + + - name: Prepare sysbench inside of all MySQL nodes + community.docker.docker_container_exec: + container: "ps_pmm_{{ ps_version }}_{{ item }}" + command: > + mysql -uroot -p{{ root_password }} -e " + SET GLOBAL super_read_only = OFF; + SET GLOBAL read_only = OFF; + " + loop: "{{ range(1, nodes_count | int + 1) | list }}" + + - name: Prepare sysbench inside of all MySQL nodes + community.docker.docker_container_exec: + container: "ps_pmm_{{ ps_version }}_{{ item }}" + command: > + mysql -uroot -p{{ root_password }} -e " + CREATE DATABASE sbtest; + CREATE USER 'sbtest'@'localhost' IDENTIFIED BY 'password'; + GRANT ALL PRIVILEGES ON *.* TO 'sbtest'@'localhost'; + CREATE USER 'sbtest'@'127.0.0.1' IDENTIFIED BY 'password'; + GRANT ALL PRIVILEGES ON *.* TO 'sbtest'@'127.0.0.1'; + FLUSH PRIVILEGES; + " + loop: "{{ range(1, nodes_count | int + 1) | list }}" + when: setup_type != "gr" and setup_type != "replication" + + - name: Prepare sysbench inside of all MySQL nodes + community.docker.docker_container_exec: + container: "ps_pmm_{{ ps_version }}_1" + command: > + mysql -uroot -p{{ root_password }} -e " + CREATE DATABASE sbtest; + CREATE USER 'sbtest'@'localhost' IDENTIFIED BY 'password'; + GRANT ALL PRIVILEGES ON *.* TO 'sbtest'@'localhost'; + CREATE USER 'sbtest'@'127.0.0.1' IDENTIFIED BY 'password'; + GRANT ALL PRIVILEGES ON *.* TO 'sbtest'@'127.0.0.1'; + FLUSH PRIVILEGES; + " + when: setup_type == "gr" or setup_type == "replication" + + - name: Prepare data for sysbench inside of all MySQL nodes + community.docker.docker_container_exec: + container: "ps_pmm_{{ ps_version }}_{{ item }}" + command: > + sysbench /usr/share/sysbench/oltp_read_write.lua + --mysql-host=127.0.0.1 + --mysql-port=3306 + --mysql-user=sbtest + --mysql-password=password + --mysql-db=sbtest + --tables=10 + --table-size=100000 + prepare + when: setup_type != "gr" and setup_type != "replication" + loop: "{{ range(1, nodes_count | int + 1) | list }}" + + - name: Prepare data for sysbench inside of first MySQL nodes + community.docker.docker_container_exec: + container: "ps_pmm_{{ ps_version }}_1" + command: > + sysbench /usr/share/sysbench/oltp_read_write.lua + --mysql-host=127.0.0.1 + --mysql-port=3306 + --mysql-user=sbtest + --mysql-password=password + --mysql-db=sbtest + --tables=10 + --table-size=100000 + prepare + when: setup_type == "gr" or setup_type == "replication" + + - name: Run load for sysbench inside of all MySQL nodes + community.docker.docker_container_exec: + container: "ps_pmm_{{ ps_version }}_{{ item }}" + command: > + sysbench /usr/share/sysbench/oltp_read_write.lua + --mysql-host=127.0.0.1 + --mysql-port=3306 + --mysql-user=sbtest + --mysql-password=password + --mysql-db=sbtest + --tables=10 + --table-size=100000 + --threads=16 + --time=60 + run + loop: "{{ range(1, nodes_count | int + 1) | list }}" diff --git a/pmm_qa/mysql/tasks/mysql-setup-single.yml b/pmm_qa/mysql/tasks/mysql-setup-single.yml new file mode 100644 index 00000000..2ca82971 --- /dev/null +++ b/pmm_qa/mysql/tasks/mysql-setup-single.yml @@ -0,0 +1,31 @@ +- name: Generate my.cnf for each node + template: + src: ./data/my.cnf.j2 + dest: "{{ data_dir }}/node{{ item }}/my.cnf" + loop: "{{ range(1, nodes_count | int + 1) | list }}" + +- name: Start MySQL containers + community.docker.docker_container: + name: "ps_pmm_{{ ps_version }}_{{ item }}" + image: "mysql:{{ ps_version }}" + restart_policy: always + state: started + networks: + - name: "{{ network_name }}" + env: + MYSQL_ROOT_PASSWORD: "{{ root_password }}" + ports: + - "{{ mysql_port + item - 1 }}:{{ mysql_listen_port }}" + - "{{ group_seeds_port + item - 1 }}:{{ group_seeds_port }}" + volumes: + - "{{ data_dir }}/node{{ item }}/my.cnf:/etc/mysql/my.cnf" + - "{{ data_dir }}/node{{ item }}/data:/var/lib/mysql" + loop: "{{ range(1, nodes_count | int + 1) | list }}" + +- name: Wait for MySQL to be available + wait_for: + host: localhost + port: "{{ mysql_port + item - 1 }}" + delay: 10 + timeout: 300 + loop: "{{ range(1, nodes_count | int + 1) | list }}" diff --git a/pmm_qa/percona_server/percona-server-setup.yml b/pmm_qa/percona_server/percona-server-setup.yml index a7f4d8c4..d6ac4170 100644 --- a/pmm_qa/percona_server/percona-server-setup.yml +++ b/pmm_qa/percona_server/percona-server-setup.yml @@ -74,7 +74,7 @@ when: setup_type == "replication" - name: Setup Percona Server - include_tasks: ./tasks/percona-server-setup.yml + include_tasks: tasks/percona-server-setup-single.yml when: setup_type != "gr" and setup_type != "replication" - name: Create slowlog configuration for mysql nodes diff --git a/pmm_qa/percona_server/tasks/percona-server-setup.yml b/pmm_qa/percona_server/tasks/percona-server-setup-single.yml similarity index 100% rename from pmm_qa/percona_server/tasks/percona-server-setup.yml rename to pmm_qa/percona_server/tasks/percona-server-setup-single.yml From d4ff70e8799751f91280a322cb992e87f8aa12d6 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 13 May 2025 10:45:01 +0200 Subject: [PATCH 233/340] PMM-7: New MySQL setup --- pmm_qa/pmm-framework.py | 46 +++++++++++++++++++++++++---------------- 1 file changed, 28 insertions(+), 18 deletions(-) diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 68c92d12..22ad0677 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -237,6 +237,7 @@ def setup_mysql(db_type, db_version=None, db_config=None, args=None): # Gather Version details ms_version = os.getenv('MS_VERSION') or db_version or database_configs[db_type]["versions"][-1] + ms_version_int = int(ms_version.replace(".", "")) # Check Setup Types setup_type = '' @@ -249,26 +250,35 @@ def setup_mysql(db_type, db_version=None, db_config=None, args=None): setup_type = '' no_of_nodes = 2 - # Define environment variables for playbook - env_vars = { - 'GROUP_REPLICATION': setup_type, - 'MS_NODES': no_of_nodes, - 'MS_VERSION': ms_version, - 'PMM_SERVER_IP': args.pmm_server_ip or container_name or '127.0.0.1', - 'MS_CONTAINER': 'mysql_pmm_' + str(ms_version), - 'CLIENT_VERSION': get_value('CLIENT_VERSION', db_type, args, db_config), - 'QUERY_SOURCE': get_value('QUERY_SOURCE', db_type, args, db_config), - 'MS_TARBALL': get_value('TARBALL', db_type, args, db_config), - 'ADMIN_PASSWORD': os.getenv('ADMIN_PASSWORD') or args.pmm_server_password or 'admin', - 'PMM_QA_GIT_BRANCH': os.getenv('PMM_QA_GIT_BRANCH') or 'v3' - } - - # Ansible playbook filename - playbook_filename = 'ms_pmm_setup.yml' + if ms_version_int >= 84: + # Define environment variables for playbook + env_vars = { + 'PMM_SERVER_IP': args.pmm_server_ip or container_name or '127.0.0.1', + 'SETUP_TYPE': setup_type_value, + 'NODES_COUNT': get_value('NODES_COUNT', db_type, args, db_config), + 'QUERY_SOURCE': get_value('QUERY_SOURCE', db_type, args, db_config), + 'PS_VERSION': ps_version, + 'CLIENT_VERSION': get_value('CLIENT_VERSION', db_type, args, db_config), + 'ADMIN_PASSWORD': os.getenv('ADMIN_PASSWORD') or args.pmm_server_password or 'admin', + } - # Call the function to run the Ansible playbook - run_ansible_playbook(playbook_filename, env_vars, args) + run_ansible_playbook('mysql/mysql_setup.yml', env_vars, args) + else: + # Define environment variables for playbook + env_vars = { + 'GROUP_REPLICATION': setup_type, + 'MS_NODES': no_of_nodes, + 'MS_VERSION': ms_version, + 'PMM_SERVER_IP': args.pmm_server_ip or container_name or '127.0.0.1', + 'MS_CONTAINER': 'mysql_pmm_' + str(ms_version), + 'CLIENT_VERSION': get_value('CLIENT_VERSION', db_type, args, db_config), + 'QUERY_SOURCE': get_value('QUERY_SOURCE', db_type, args, db_config), + 'MS_TARBALL': get_value('TARBALL', db_type, args, db_config), + 'ADMIN_PASSWORD': os.getenv('ADMIN_PASSWORD') or args.pmm_server_password or 'admin', + 'PMM_QA_GIT_BRANCH': os.getenv('PMM_QA_GIT_BRANCH') or 'v3' + } + run_ansible_playbook('ms_pmm_setup.yml', env_vars, args) def setup_ssl_mysql(db_type, db_version=None, db_config=None, args=None): # Check if PMM server is running From ff1dfb4e5fb834d4179698612ce6351870784f20 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 13 May 2025 10:48:32 +0200 Subject: [PATCH 234/340] PMM-7: New MySQL setup --- pmm_qa/mysql/mysql_setup.yml | 40 ++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/pmm_qa/mysql/mysql_setup.yml b/pmm_qa/mysql/mysql_setup.yml index 4523a815..45f3778b 100644 --- a/pmm_qa/mysql/mysql_setup.yml +++ b/pmm_qa/mysql/mysql_setup.yml @@ -5,7 +5,7 @@ connection: local gather_facts: yes vars: - ps_version: "{{ lookup('env', 'PS_VERSION') | default('8.4', true) }}" + ms_version: "{{ lookup('env', 'ms_vERSION') | default('8.4', true) }}" cluster_name: "mysql_cluster" replication_user: "repl_user" replication_password: "GRgrO9301RuF" @@ -54,8 +54,8 @@ - name: Remove old MySQL containers community.docker.docker_container: - name: "ps_pmm_{{ ps_version }}_{{ item }}" - image: "mysql:{{ ps_version }}" + name: "mysql_pmm_{{ ms_version }}_{{ item }}" + image: "mysql:{{ ms_version }}" restart_policy: always state: absent loop: "{{ range(1, nodes_count | int + 1) | list }}" @@ -79,7 +79,7 @@ - name: Create slowlog configuration for mysql nodes community.docker.docker_container_exec: - container: "ps_pmm_{{ ps_version }}_{{ item }}" + container: "mysql_pmm_{{ ms_version }}_{{ item }}" command: > mysql -uroot -p{{ root_password }} -e " SET GLOBAL slow_query_log='ON'; @@ -93,12 +93,12 @@ - name: Install and add pmm client. include_tasks: ../tasks/install_pmm_client.yml vars: - container_name: "ps_pmm_{{ ps_version }}_{{ item }}" + container_name: "mysql_pmm_{{ ms_version }}_{{ item }}" loop: "{{ range(1, nodes_count | int + 1) | list }}" - name: Get already connected services to pmm server community.docker.docker_container_exec: - container: "ps_pmm_{{ ps_version }}_1" + container: "mysql_pmm_{{ ms_version }}_1" command: > sh -c 'curl --location --insecure -u"admin:{{ admin_password }}" -s --request GET "http://{{ pmm_server_ip }}:{{ '80' if pmm_server_ip is ansible.utils.ipv4 else '8080' }}/v1/management/services" | jq -r ".services[].service_name"' register: pmm_server_services @@ -113,32 +113,32 @@ set_fact: random_service_name_value: "_{{ 9999 | random + 1 }}" loop: "{{ range(1, nodes_count | int + 1) | list }}" - when: "('ps_pmm_' ~ ps_version ~ '_' ~ item) in pmm_server_services.stdout" + when: "('mysql_pmm_' ~ ms_version ~ '_' ~ item) in pmm_server_services.stdout" - name: Add service to pmm server community.docker.docker_container_exec: - container: "ps_pmm_{{ ps_version }}_{{ item }}" - command: pmm-admin add mysql --query-source={{ query_source }} --username=root --password={{ root_password }} --environment=ps-gr-dev --cluster=ps-gr-dev-cluster --replication-set=ps-gr-replication ps_pmm_{{ ps_version }}_{{ item }}{{ random_service_name_value }} --debug 127.0.0.1:3306 + container: "mysql_pmm_{{ ms_version }}_{{ item }}" + command: pmm-admin add mysql --query-source={{ query_source }} --username=root --password={{ root_password }} --environment=ps-gr-dev --cluster=ps-gr-dev-cluster --replication-set=ps-gr-replication mysql_pmm_{{ ms_version }}_{{ item }}{{ random_service_name_value }} --debug 127.0.0.1:3306 loop: "{{ range(1, nodes_count | int + 1) | list }}" when: setup_type == "gr" - name: Add service to pmm server community.docker.docker_container_exec: - container: "ps_pmm_{{ ps_version }}_{{ item }}" - command: pmm-admin add mysql --query-source={{ query_source }} --username=root --password={{ root_password }} --environment=ps-replication-dev --cluster=ps-replication-dev-cluster --replication-set=ps-async-replication ps_pmm_{{ ps_version }}_{{ item }}{{ random_service_name_value }} --debug 127.0.0.1:3306 + container: "mysql_pmm_{{ ms_version }}_{{ item }}" + command: pmm-admin add mysql --query-source={{ query_source }} --username=root --password={{ root_password }} --environment=ps-replication-dev --cluster=ps-replication-dev-cluster --replication-set=ps-async-replication mysql_pmm_{{ ms_version }}_{{ item }}{{ random_service_name_value }} --debug 127.0.0.1:3306 loop: "{{ range(1, nodes_count | int + 1) | list }}" when: setup_type == "replication" - name: Add service to pmm server community.docker.docker_container_exec: - container: "ps_pmm_{{ ps_version }}_{{ item }}" - command: pmm-admin add mysql --query-source={{ query_source }} --username=root --password={{ root_password }} --environment=ps-dev ps_pmm_{{ ps_version }}_{{ item }}{{ random_service_name_value }} --debug 127.0.0.1:3306 + container: "mysql_pmm_{{ ms_version }}_{{ item }}" + command: pmm-admin add mysql --query-source={{ query_source }} --username=root --password={{ root_password }} --environment=ps-dev mysql_pmm_{{ ms_version }}_{{ item }}{{ random_service_name_value }} --debug 127.0.0.1:3306 loop: "{{ range(1, nodes_count | int + 1) | list }}" when: setup_type != "gr" and setup_type != "replication" - name: Install sysbench inside of all MySQL nodes community.docker.docker_container_exec: - container: "ps_pmm_{{ ps_version }}_{{ item }}" + container: "mysql_pmm_{{ ms_version }}_{{ item }}" user: "root" command: > /bin/sh -c " @@ -150,7 +150,7 @@ - name: Prepare sysbench inside of all MySQL nodes community.docker.docker_container_exec: - container: "ps_pmm_{{ ps_version }}_{{ item }}" + container: "mysql_pmm_{{ ms_version }}_{{ item }}" command: > mysql -uroot -p{{ root_password }} -e " SET GLOBAL super_read_only = OFF; @@ -160,7 +160,7 @@ - name: Prepare sysbench inside of all MySQL nodes community.docker.docker_container_exec: - container: "ps_pmm_{{ ps_version }}_{{ item }}" + container: "mysql_pmm_{{ ms_version }}_{{ item }}" command: > mysql -uroot -p{{ root_password }} -e " CREATE DATABASE sbtest; @@ -175,7 +175,7 @@ - name: Prepare sysbench inside of all MySQL nodes community.docker.docker_container_exec: - container: "ps_pmm_{{ ps_version }}_1" + container: "mysql_pmm_{{ ms_version }}_1" command: > mysql -uroot -p{{ root_password }} -e " CREATE DATABASE sbtest; @@ -189,7 +189,7 @@ - name: Prepare data for sysbench inside of all MySQL nodes community.docker.docker_container_exec: - container: "ps_pmm_{{ ps_version }}_{{ item }}" + container: "mysql_pmm_{{ ms_version }}_{{ item }}" command: > sysbench /usr/share/sysbench/oltp_read_write.lua --mysql-host=127.0.0.1 @@ -205,7 +205,7 @@ - name: Prepare data for sysbench inside of first MySQL nodes community.docker.docker_container_exec: - container: "ps_pmm_{{ ps_version }}_1" + container: "mysql_pmm_{{ ms_version }}_1" command: > sysbench /usr/share/sysbench/oltp_read_write.lua --mysql-host=127.0.0.1 @@ -220,7 +220,7 @@ - name: Run load for sysbench inside of all MySQL nodes community.docker.docker_container_exec: - container: "ps_pmm_{{ ps_version }}_{{ item }}" + container: "mysql_pmm_{{ ms_version }}_{{ item }}" command: > sysbench /usr/share/sysbench/oltp_read_write.lua --mysql-host=127.0.0.1 From 3ad1d884340e4085a3f0b4aae6768e1464407156 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 13 May 2025 10:49:22 +0200 Subject: [PATCH 235/340] PMM-7: New MySQL setup --- pmm_qa/pmm-framework.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 22ad0677..88320b23 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -257,7 +257,7 @@ def setup_mysql(db_type, db_version=None, db_config=None, args=None): 'SETUP_TYPE': setup_type_value, 'NODES_COUNT': get_value('NODES_COUNT', db_type, args, db_config), 'QUERY_SOURCE': get_value('QUERY_SOURCE', db_type, args, db_config), - 'PS_VERSION': ps_version, + 'PS_VERSION': ms_version, 'CLIENT_VERSION': get_value('CLIENT_VERSION', db_type, args, db_config), 'ADMIN_PASSWORD': os.getenv('ADMIN_PASSWORD') or args.pmm_server_password or 'admin', } From 57b47d07a3f2caea9ec312f1f847ea729b2a57b7 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 13 May 2025 10:50:17 +0200 Subject: [PATCH 236/340] PMM-7: New MySQL setup --- .../mysql/data/init-async-replication.sql.j2 | 13 ++++++ pmm_qa/mysql/data/my-async-replication.cnf.j2 | 43 +++++++++++++++++++ pmm_qa/mysql/data/my.cnf.j2 | 2 + 3 files changed, 58 insertions(+) create mode 100644 pmm_qa/mysql/data/init-async-replication.sql.j2 create mode 100644 pmm_qa/mysql/data/my-async-replication.cnf.j2 create mode 100644 pmm_qa/mysql/data/my.cnf.j2 diff --git a/pmm_qa/mysql/data/init-async-replication.sql.j2 b/pmm_qa/mysql/data/init-async-replication.sql.j2 new file mode 100644 index 00000000..a96ef137 --- /dev/null +++ b/pmm_qa/mysql/data/init-async-replication.sql.j2 @@ -0,0 +1,13 @@ +-- Create replication user and grant necessary privileges +SET SQL_LOG_BIN=0; +CREATE USER '{{ replication_user }}'@'%' IDENTIFIED WITH 'caching_sha2_password' BY '{{ replication_password }}' REQUIRE NONE; +GRANT REPLICATION SLAVE ON *.* TO '{{ replication_user }}'@'%'; +GRANT CONNECTION_ADMIN ON *.* TO '{{ replication_user }}'@'%'; +GRANT BACKUP_ADMIN ON *.* TO '{{ replication_user }}'@'%'; +FLUSH PRIVILEGES; +SET SQL_LOG_BIN=1; + +{% if item == 1 %} +-- Primary server: enable binary logging for replication +FLUSH BINARY LOGS; +{% endif %} diff --git a/pmm_qa/mysql/data/my-async-replication.cnf.j2 b/pmm_qa/mysql/data/my-async-replication.cnf.j2 new file mode 100644 index 00000000..014d37f7 --- /dev/null +++ b/pmm_qa/mysql/data/my-async-replication.cnf.j2 @@ -0,0 +1,43 @@ +[mysqld] +# General server configuration +server_id={{ item }} +bind-address=0.0.0.0 +port={{ mysql_listen_port }} +userstat=1 + +# Authentication settings for caching_sha2_password +caching_sha2_password_auto_generate_rsa_keys=ON +# The following two parameters tell MySQL where to store the RSA key pair +caching_sha2_password_private_key_path=private_key.pem +caching_sha2_password_public_key_path=public_key.pem + +# Replication settings +gtid_mode=ON +enforce_gtid_consistency=ON +log_bin=binlog +log_replica_updates=ON +sync_binlog=1 +binlog_checksum=NONE +disabled_storage_engines="MyISAM,BLACKHOLE,FEDERATED,ARCHIVE,MEMORY" +# MacOS-specific, where table names are case-sensitive +lower_case_table_names=2 + +# MySQL 8.4 compatibility settings +report_host=ps_pmm_{{ ps_version }}_{{ item }} + +# Replica configuration - applies to all nodes except primary (they'll be able to become replicas) +{% if item != 1 %} +# Replica specific settings +replica_parallel_workers=4 +replica_parallel_type=LOGICAL_CLOCK +replica_preserve_commit_order=1 +{% endif %} + +# Crash-safe replication settings +relay-log=ps_pmm_{{ ps_version }}_{{ item }}-relay-bin +relay_log_recovery=ON +relay_log_purge=ON + +# Performance and connection settings +max_connections=1000 +innodb_buffer_pool_size=256M diff --git a/pmm_qa/mysql/data/my.cnf.j2 b/pmm_qa/mysql/data/my.cnf.j2 new file mode 100644 index 00000000..fd4b27f2 --- /dev/null +++ b/pmm_qa/mysql/data/my.cnf.j2 @@ -0,0 +1,2 @@ +[mysqld] +userstat=1 \ No newline at end of file From d5aa6561deddbf69636484bdc488eb1b2bd10cb7 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 13 May 2025 10:51:07 +0200 Subject: [PATCH 237/340] PMM-7: New MySQL setup --- pmm_qa/mysql/tasks/mysql-setup-single.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pmm_qa/mysql/tasks/mysql-setup-single.yml b/pmm_qa/mysql/tasks/mysql-setup-single.yml index 2ca82971..a7e4a457 100644 --- a/pmm_qa/mysql/tasks/mysql-setup-single.yml +++ b/pmm_qa/mysql/tasks/mysql-setup-single.yml @@ -6,8 +6,8 @@ - name: Start MySQL containers community.docker.docker_container: - name: "ps_pmm_{{ ps_version }}_{{ item }}" - image: "mysql:{{ ps_version }}" + name: "mysql_pmm_{{ ms_version }}_{{ item }}" + image: "mysql:{{ ms_version }}" restart_policy: always state: started networks: From b0f42928c63210896537a5570c805505b3e4ae4c Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 13 May 2025 10:52:36 +0200 Subject: [PATCH 238/340] PMM-7: New MySQL setup --- pmm_qa/mysql/mysql_setup.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pmm_qa/mysql/mysql_setup.yml b/pmm_qa/mysql/mysql_setup.yml index 45f3778b..d7af02f6 100644 --- a/pmm_qa/mysql/mysql_setup.yml +++ b/pmm_qa/mysql/mysql_setup.yml @@ -10,9 +10,9 @@ replication_user: "repl_user" replication_password: "GRgrO9301RuF" root_password: "GRgrO9301RuF" - mysql_port: 33066 + mysql_port: 43066 mysql_listen_port: 3306 - group_seeds_port: 34061 + group_seeds_port: 44061 nodes_count: "{{ (lookup('env', 'NODES_COUNT') | default('3', true)) | int }}" network_name: "pmm-qa" data_dir: "{{ lookup('env', 'HOME') }}/mysql_cluster_data" From 46e1a6a003a514c8be719a0c03919740a2bf7513 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 13 May 2025 10:52:56 +0200 Subject: [PATCH 239/340] PMM-7: New MySQL setup --- pmm_qa/mysql/mysql_setup.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/mysql/mysql_setup.yml b/pmm_qa/mysql/mysql_setup.yml index d7af02f6..61e7d81a 100644 --- a/pmm_qa/mysql/mysql_setup.yml +++ b/pmm_qa/mysql/mysql_setup.yml @@ -10,7 +10,7 @@ replication_user: "repl_user" replication_password: "GRgrO9301RuF" root_password: "GRgrO9301RuF" - mysql_port: 43066 + mysql_port: 4306 mysql_listen_port: 3306 group_seeds_port: 44061 nodes_count: "{{ (lookup('env', 'NODES_COUNT') | default('3', true)) | int }}" From a8dab656fb9b8e1b3e6441024a9f529e0ebff682 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 13 May 2025 10:53:09 +0200 Subject: [PATCH 240/340] PMM-7: New MySQL setup --- pmm_qa/mysql/mysql_setup.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/mysql/mysql_setup.yml b/pmm_qa/mysql/mysql_setup.yml index 61e7d81a..8c3b4bdd 100644 --- a/pmm_qa/mysql/mysql_setup.yml +++ b/pmm_qa/mysql/mysql_setup.yml @@ -5,7 +5,7 @@ connection: local gather_facts: yes vars: - ms_version: "{{ lookup('env', 'ms_vERSION') | default('8.4', true) }}" + ms_version: "{{ lookup('env', 'MS_VERSION') | default('8.4', true) }}" cluster_name: "mysql_cluster" replication_user: "repl_user" replication_password: "GRgrO9301RuF" From b1a8184bd95eb144db99eb5120d41d8c3fcb9b00 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 13 May 2025 10:55:08 +0200 Subject: [PATCH 241/340] PMM-7: New MySQL setup --- pmm_qa/mysql/data/my.cnf.j2 | 2 -- 1 file changed, 2 deletions(-) diff --git a/pmm_qa/mysql/data/my.cnf.j2 b/pmm_qa/mysql/data/my.cnf.j2 index fd4b27f2..e69de29b 100644 --- a/pmm_qa/mysql/data/my.cnf.j2 +++ b/pmm_qa/mysql/data/my.cnf.j2 @@ -1,2 +0,0 @@ -[mysqld] -userstat=1 \ No newline at end of file From 4fbe2f855d140c90bac6bad556b13b8062262e5c Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 13 May 2025 10:57:38 +0200 Subject: [PATCH 242/340] PMM-7: New MySQL setup --- pmm_qa/tasks/install_pmm_client.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/pmm_qa/tasks/install_pmm_client.yml b/pmm_qa/tasks/install_pmm_client.yml index de83b510..a8c08951 100644 --- a/pmm_qa/tasks/install_pmm_client.yml +++ b/pmm_qa/tasks/install_pmm_client.yml @@ -66,6 +66,7 @@ command: microdnf install -y wget gnupg2 jq curl-minimal user: "root" when: distro_family == "rhel" + ignore_errors: true - name: Get lsb_release community.docker.docker_container_exec: From a556fe6deaed6a25e5aabc5a8a5b3d86cd49fa4a Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 13 May 2025 14:11:35 +0200 Subject: [PATCH 243/340] PMM-7: New MySQL setup --- pmm_qa/percona_server/percona-server-setup.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pmm_qa/percona_server/percona-server-setup.yml b/pmm_qa/percona_server/percona-server-setup.yml index d6ac4170..b88b431e 100644 --- a/pmm_qa/percona_server/percona-server-setup.yml +++ b/pmm_qa/percona_server/percona-server-setup.yml @@ -199,6 +199,7 @@ --mysql-db=sbtest --tables=10 --table-size=100000 + --rate=1 prepare when: setup_type != "gr" and setup_type != "replication" loop: "{{ range(1, nodes_count | int + 1) | list }}" @@ -215,6 +216,7 @@ --mysql-db=sbtest --tables=10 --table-size=100000 + --rate=1 prepare when: setup_type == "gr" or setup_type == "replication" @@ -231,6 +233,7 @@ --tables=10 --table-size=100000 --threads=16 - --time=60 + --time=60 + --rate=1 run loop: "{{ range(1, nodes_count | int + 1) | list }}" From 44b1a478bd7765782e8df9f6ae0e6a7ca90e150b Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 13 May 2025 14:30:10 +0200 Subject: [PATCH 244/340] PMM-7: New MySQL setup --- pmm_qa/pmm-framework.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 88320b23..2af4ce5c 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -742,7 +742,7 @@ def mongo_ssl_setup(script_filename, args): ['cp', f'{scripts_path}docker-compose-pmm-psmdb.yml', f'{compose_file_path}']) admin_password = os.getenv('ADMIN_PASSWORD') or args.pmm_server_password or 'admin' subprocess.run(['sed', '-i', f's/PMM_AGENT_SERVER_PASSWORD=admin/PMM_AGENT_SERVER_PASSWORD={admin_password}/g', f'{compose_file_path}']) - subprocess.run(['sed', '-i', '/container_name/a\ networks:\\\n \\- pmm-qa', f'{compose_file_path}']) + # subprocess.run(['sed', '-i', '/container_name/a\ networks:\\\n \\- pmm-qa', f'{compose_file_path}']) subprocess.run(['sed', '-i', '$a\\\nnetworks:\\\n pmm-qa:\\\n name: pmm-qa\\\n external: true', f'{compose_file_path}']) subprocess.run(['sed', '-i', From f11debddcecf9507b74284ffd75f3867e57edc60 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 13 May 2025 14:37:45 +0200 Subject: [PATCH 245/340] PMM-7: New MySQL setup --- pmm_qa/pmm-framework.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 2af4ce5c..b9833a7f 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -742,7 +742,7 @@ def mongo_ssl_setup(script_filename, args): ['cp', f'{scripts_path}docker-compose-pmm-psmdb.yml', f'{compose_file_path}']) admin_password = os.getenv('ADMIN_PASSWORD') or args.pmm_server_password or 'admin' subprocess.run(['sed', '-i', f's/PMM_AGENT_SERVER_PASSWORD=admin/PMM_AGENT_SERVER_PASSWORD={admin_password}/g', f'{compose_file_path}']) - # subprocess.run(['sed', '-i', '/container_name/a\ networks:\\\n \\- pmm-qa', f'{compose_file_path}']) + subprocess.run(['sed', '-i', '/container_name/a\\ networks:\\\\n \\\\- pmm-qa', f'{compose_file_path}']) subprocess.run(['sed', '-i', '$a\\\nnetworks:\\\n pmm-qa:\\\n name: pmm-qa\\\n external: true', f'{compose_file_path}']) subprocess.run(['sed', '-i', From 4862ed8d98fe1b03f785256184a4613ee60f203d Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 13 May 2025 14:42:33 +0200 Subject: [PATCH 246/340] PMM-7: New MySQL setup --- pmm_qa/pmm-framework.py | 45 ++++++++++++++++------------------------- 1 file changed, 17 insertions(+), 28 deletions(-) diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index b9833a7f..8dc35e7c 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -237,7 +237,6 @@ def setup_mysql(db_type, db_version=None, db_config=None, args=None): # Gather Version details ms_version = os.getenv('MS_VERSION') or db_version or database_configs[db_type]["versions"][-1] - ms_version_int = int(ms_version.replace(".", "")) # Check Setup Types setup_type = '' @@ -250,35 +249,25 @@ def setup_mysql(db_type, db_version=None, db_config=None, args=None): setup_type = '' no_of_nodes = 2 - if ms_version_int >= 84: - # Define environment variables for playbook - env_vars = { - 'PMM_SERVER_IP': args.pmm_server_ip or container_name or '127.0.0.1', - 'SETUP_TYPE': setup_type_value, - 'NODES_COUNT': get_value('NODES_COUNT', db_type, args, db_config), - 'QUERY_SOURCE': get_value('QUERY_SOURCE', db_type, args, db_config), - 'PS_VERSION': ms_version, - 'CLIENT_VERSION': get_value('CLIENT_VERSION', db_type, args, db_config), - 'ADMIN_PASSWORD': os.getenv('ADMIN_PASSWORD') or args.pmm_server_password or 'admin', - } + # Define environment variables for playbook + env_vars = { + 'GROUP_REPLICATION': setup_type, + 'MS_NODES': no_of_nodes, + 'MS_VERSION': ms_version, + 'PMM_SERVER_IP': args.pmm_server_ip or container_name or '127.0.0.1', + 'MS_CONTAINER': 'mysql_pmm_' + str(ms_version), + 'CLIENT_VERSION': get_value('CLIENT_VERSION', db_type, args, db_config), + 'QUERY_SOURCE': get_value('QUERY_SOURCE', db_type, args, db_config), + 'MS_TARBALL': get_value('TARBALL', db_type, args, db_config), + 'ADMIN_PASSWORD': os.getenv('ADMIN_PASSWORD') or args.pmm_server_password or 'admin', + 'PMM_QA_GIT_BRANCH': os.getenv('PMM_QA_GIT_BRANCH') or 'v3' + } - run_ansible_playbook('mysql/mysql_setup.yml', env_vars, args) - else: - # Define environment variables for playbook - env_vars = { - 'GROUP_REPLICATION': setup_type, - 'MS_NODES': no_of_nodes, - 'MS_VERSION': ms_version, - 'PMM_SERVER_IP': args.pmm_server_ip or container_name or '127.0.0.1', - 'MS_CONTAINER': 'mysql_pmm_' + str(ms_version), - 'CLIENT_VERSION': get_value('CLIENT_VERSION', db_type, args, db_config), - 'QUERY_SOURCE': get_value('QUERY_SOURCE', db_type, args, db_config), - 'MS_TARBALL': get_value('TARBALL', db_type, args, db_config), - 'ADMIN_PASSWORD': os.getenv('ADMIN_PASSWORD') or args.pmm_server_password or 'admin', - 'PMM_QA_GIT_BRANCH': os.getenv('PMM_QA_GIT_BRANCH') or 'v3' - } + # Ansible playbook filename + playbook_filename = 'ms_pmm_setup.yml' - run_ansible_playbook('ms_pmm_setup.yml', env_vars, args) + # Call the function to run the Ansible playbook + run_ansible_playbook(playbook_filename, env_vars, args) def setup_ssl_mysql(db_type, db_version=None, db_config=None, args=None): # Check if PMM server is running From 6a670ee9c6eeb2e08b3b218e10a09e6a1cdff09f Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 14 May 2025 07:53:12 +0200 Subject: [PATCH 247/340] PMM-7: New MySQL setup --- pmm_qa/mysql/setup_mysql.py | 12 ++++++++++++ pmm_qa/pmm-framework.py | 39 +++++++++++++++++++++---------------- 2 files changed, 34 insertions(+), 17 deletions(-) create mode 100644 pmm_qa/mysql/setup_mysql.py diff --git a/pmm_qa/mysql/setup_mysql.py b/pmm_qa/mysql/setup_mysql.py new file mode 100644 index 00000000..86121ff0 --- /dev/null +++ b/pmm_qa/mysql/setup_mysql.py @@ -0,0 +1,12 @@ +def setup_mysql_docker(db_type, db_config=None, args=None): + env_vars = { + 'SETUP_TYPE': setup_type, + 'MS_VERSION': ms_version, + 'PMM_SERVER_IP': args.pmm_server_ip or container_name or '127.0.0.1', + 'CLIENT_VERSION': get_value('CLIENT_VERSION', db_type, args, db_config), + 'QUERY_SOURCE': get_value('QUERY_SOURCE', db_type, args, db_config), + 'ADMIN_PASSWORD': os.getenv('ADMIN_PASSWORD') or args.pmm_server_password or 'admin', + 'NODES_COUNT': get_value('NODES_COUNT', db_type, args, db_config), + } + + run_ansible_playbook('./mysql_setup.yml', env_vars, args) diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 8dc35e7c..656bda13 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -5,6 +5,7 @@ import ansible_runner import requests import re +from mysql.setup_mysql import setup_mysql_docker # Database configurations database_configs = { @@ -237,6 +238,7 @@ def setup_mysql(db_type, db_version=None, db_config=None, args=None): # Gather Version details ms_version = os.getenv('MS_VERSION') or db_version or database_configs[db_type]["versions"][-1] + ms_version_int = int(ms_version.replace(".", "")) # Check Setup Types setup_type = '' @@ -249,25 +251,28 @@ def setup_mysql(db_type, db_version=None, db_config=None, args=None): setup_type = '' no_of_nodes = 2 - # Define environment variables for playbook - env_vars = { - 'GROUP_REPLICATION': setup_type, - 'MS_NODES': no_of_nodes, - 'MS_VERSION': ms_version, - 'PMM_SERVER_IP': args.pmm_server_ip or container_name or '127.0.0.1', - 'MS_CONTAINER': 'mysql_pmm_' + str(ms_version), - 'CLIENT_VERSION': get_value('CLIENT_VERSION', db_type, args, db_config), - 'QUERY_SOURCE': get_value('QUERY_SOURCE', db_type, args, db_config), - 'MS_TARBALL': get_value('TARBALL', db_type, args, db_config), - 'ADMIN_PASSWORD': os.getenv('ADMIN_PASSWORD') or args.pmm_server_password or 'admin', - 'PMM_QA_GIT_BRANCH': os.getenv('PMM_QA_GIT_BRANCH') or 'v3' - } + if ms_version_int >= 84: + setup_mysql_docker(db_type, db_config, args) + else: + # Define environment variables for playbook + env_vars = { + 'GROUP_REPLICATION': setup_type, + 'MS_NODES': no_of_nodes, + 'MS_VERSION': ms_version, + 'PMM_SERVER_IP': args.pmm_server_ip or container_name or '127.0.0.1', + 'MS_CONTAINER': 'mysql_pmm_' + str(ms_version), + 'CLIENT_VERSION': get_value('CLIENT_VERSION', db_type, args, db_config), + 'QUERY_SOURCE': get_value('QUERY_SOURCE', db_type, args, db_config), + 'MS_TARBALL': get_value('TARBALL', db_type, args, db_config), + 'ADMIN_PASSWORD': os.getenv('ADMIN_PASSWORD') or args.pmm_server_password or 'admin', + 'PMM_QA_GIT_BRANCH': os.getenv('PMM_QA_GIT_BRANCH') or 'v3' + } - # Ansible playbook filename - playbook_filename = 'ms_pmm_setup.yml' + # Ansible playbook filename + playbook_filename = 'ms_pmm_setup.yml' - # Call the function to run the Ansible playbook - run_ansible_playbook(playbook_filename, env_vars, args) + # Call the function to run the Ansible playbook + run_ansible_playbook(playbook_filename, env_vars, args) def setup_ssl_mysql(db_type, db_version=None, db_config=None, args=None): # Check if PMM server is running From 17d2118922c32e69b3da9c1002250817cc7515dd Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 14 May 2025 07:55:26 +0200 Subject: [PATCH 248/340] PMM-7: New MySQL setup --- pmm_qa/mysql/setup_mysql.py | 2 +- pmm_qa/pmm-framework.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pmm_qa/mysql/setup_mysql.py b/pmm_qa/mysql/setup_mysql.py index 86121ff0..ebaf25c6 100644 --- a/pmm_qa/mysql/setup_mysql.py +++ b/pmm_qa/mysql/setup_mysql.py @@ -1,4 +1,4 @@ -def setup_mysql_docker(db_type, db_config=None, args=None): +def setup_mysql_docker(db_type, setup_type, ms_version, db_config=None, args=None): env_vars = { 'SETUP_TYPE': setup_type, 'MS_VERSION': ms_version, diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 656bda13..9ddccd0a 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -252,7 +252,7 @@ def setup_mysql(db_type, db_version=None, db_config=None, args=None): no_of_nodes = 2 if ms_version_int >= 84: - setup_mysql_docker(db_type, db_config, args) + setup_mysql_docker(db_type, setup_type, ms_version, db_config, args) else: # Define environment variables for playbook env_vars = { From 943a98beaca40b9590711ce2cef5085560eb64a3 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 14 May 2025 07:56:08 +0200 Subject: [PATCH 249/340] PMM-7: New MySQL setup --- pmm_qa/mysql/setup_mysql.py | 2 +- pmm_qa/pmm-framework.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pmm_qa/mysql/setup_mysql.py b/pmm_qa/mysql/setup_mysql.py index ebaf25c6..03f74da8 100644 --- a/pmm_qa/mysql/setup_mysql.py +++ b/pmm_qa/mysql/setup_mysql.py @@ -1,4 +1,4 @@ -def setup_mysql_docker(db_type, setup_type, ms_version, db_config=None, args=None): +def setup_mysql_docker(db_type, setup_type, ms_version, container_name, db_config=None, args=None): env_vars = { 'SETUP_TYPE': setup_type, 'MS_VERSION': ms_version, diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 9ddccd0a..0338a1a3 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -252,7 +252,7 @@ def setup_mysql(db_type, db_version=None, db_config=None, args=None): no_of_nodes = 2 if ms_version_int >= 84: - setup_mysql_docker(db_type, setup_type, ms_version, db_config, args) + setup_mysql_docker(db_type, setup_type, ms_version, container_name, db_config, args) else: # Define environment variables for playbook env_vars = { From 8c31fab846d1c81dc710e16e40a58cc8d4b17941 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 14 May 2025 07:58:14 +0200 Subject: [PATCH 250/340] PMM-7: New MySQL setup --- pmm_qa/mysql/setup_mysql.py | 2 ++ pmm_qa/pmm-framework.py | 21 +-------------------- pmm_qa/scripts/get_env_value.py | 17 +++++++++++++++++ 3 files changed, 20 insertions(+), 20 deletions(-) create mode 100644 pmm_qa/scripts/get_env_value.py diff --git a/pmm_qa/mysql/setup_mysql.py b/pmm_qa/mysql/setup_mysql.py index 03f74da8..9eef6136 100644 --- a/pmm_qa/mysql/setup_mysql.py +++ b/pmm_qa/mysql/setup_mysql.py @@ -1,3 +1,5 @@ +from ..scripts.get_env_value import get_value + def setup_mysql_docker(db_type, setup_type, ms_version, container_name, db_config=None, args=None): env_vars = { 'SETUP_TYPE': setup_type, diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 0338a1a3..2697e5bb 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -6,6 +6,7 @@ import requests import re from mysql.setup_mysql import setup_mysql_docker +from scripts.get_env_value import get_value # Database configurations database_configs = { @@ -153,26 +154,6 @@ def get_running_container_name(): return None - -def get_value(key, db_type, args, db_config): - # Check if the variable exists in the environment - env_value = os.environ.get(key) - if env_value is not None: - return env_value - - # Only for client_version we accept global command line argument - if key == "CLIENT_VERSION" and args.client_version is not None: - return args.client_version - - # Check if the variable exists in the args config - config_value = db_config.get(key) - if config_value is not None: - return config_value - - # Fall back to default configs value or empty '' - return database_configs[db_type]["configurations"].get(key, '') - - def setup_ps(db_type, db_version=None, db_config=None, args=None): # Check if PMM server is running container_name = get_running_container_name() diff --git a/pmm_qa/scripts/get_env_value.py b/pmm_qa/scripts/get_env_value.py new file mode 100644 index 00000000..8425b22b --- /dev/null +++ b/pmm_qa/scripts/get_env_value.py @@ -0,0 +1,17 @@ +def get_value(key, db_type, args, db_config): + # Check if the variable exists in the environment + env_value = os.environ.get(key) + if env_value is not None: + return env_value + + # Only for client_version we accept global command line argument + if key == "CLIENT_VERSION" and args.client_version is not None: + return args.client_version + + # Check if the variable exists in the args config + config_value = db_config.get(key) + if config_value is not None: + return config_value + + # Fall back to default configs value or empty '' + return database_configs[db_type]["configurations"].get(key, '') From f5039585dcd92469de76b2ee6e75925a0ba3b2c2 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 14 May 2025 08:03:15 +0200 Subject: [PATCH 251/340] PMM-7: New MySQL setup --- pmm_qa/__init__.py | 0 pmm_qa/mysql/__init__.py | 0 pmm_qa/mysql/setup_mysql.py | 2 +- pmm_qa/pmm-framework.py | 4 ++-- pmm_qa/scripts/__init__.py | 0 5 files changed, 3 insertions(+), 3 deletions(-) create mode 100644 pmm_qa/__init__.py create mode 100644 pmm_qa/mysql/__init__.py create mode 100644 pmm_qa/scripts/__init__.py diff --git a/pmm_qa/__init__.py b/pmm_qa/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pmm_qa/mysql/__init__.py b/pmm_qa/mysql/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pmm_qa/mysql/setup_mysql.py b/pmm_qa/mysql/setup_mysql.py index 9eef6136..00d9c5d9 100644 --- a/pmm_qa/mysql/setup_mysql.py +++ b/pmm_qa/mysql/setup_mysql.py @@ -1,4 +1,4 @@ -from ..scripts.get_env_value import get_value +from pmm_qa.scripts.get_env_value import get_value def setup_mysql_docker(db_type, setup_type, ms_version, container_name, db_config=None, args=None): env_vars = { diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 2697e5bb..2f1a3156 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -5,8 +5,8 @@ import ansible_runner import requests import re -from mysql.setup_mysql import setup_mysql_docker -from scripts.get_env_value import get_value +from pmm_qa.mysql.setup_mysql import setup_mysql_docker +from pmm_qa.scripts.get_env_value import get_value # Database configurations database_configs = { diff --git a/pmm_qa/scripts/__init__.py b/pmm_qa/scripts/__init__.py new file mode 100644 index 00000000..e69de29b From dc26cc7d0497b514faab6cca67e884b7dc95be22 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 14 May 2025 08:07:05 +0200 Subject: [PATCH 252/340] PMM-7: New MySQL setup --- pmm_qa/mysql/setup_mysql.py | 2 +- pmm_qa/pmm-framework.py | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/pmm_qa/mysql/setup_mysql.py b/pmm_qa/mysql/setup_mysql.py index 00d9c5d9..8cd6d373 100644 --- a/pmm_qa/mysql/setup_mysql.py +++ b/pmm_qa/mysql/setup_mysql.py @@ -1,4 +1,4 @@ -from pmm_qa.scripts.get_env_value import get_value +from scripts.get_env_value import get_value def setup_mysql_docker(db_type, setup_type, ms_version, container_name, db_config=None, args=None): env_vars = { diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 2f1a3156..852d4329 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -1,12 +1,12 @@ -import subprocess -import argparse -import os -import sys -import ansible_runner -import requests -import re -from pmm_qa.mysql.setup_mysql import setup_mysql_docker -from pmm_qa.scripts.get_env_value import get_value +# import subprocess +# import argparse +# import os +# import sys +# import ansible_runner +# import requests +# import re +from mysql.setup_mysql import setup_mysql_docker +from scripts.get_env_value import get_value # Database configurations database_configs = { From 9942ecc052bde7dd8d515efec5782522179d85e9 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 14 May 2025 08:12:03 +0200 Subject: [PATCH 253/340] PMM-7: New MySQL setup --- pmm_qa/pmm-framework.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 852d4329..2697e5bb 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -1,10 +1,10 @@ -# import subprocess -# import argparse -# import os -# import sys -# import ansible_runner -# import requests -# import re +import subprocess +import argparse +import os +import sys +import ansible_runner +import requests +import re from mysql.setup_mysql import setup_mysql_docker from scripts.get_env_value import get_value From 95c85507f860bb3414f3f69ff7865f84bcb9c336 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 14 May 2025 08:12:50 +0200 Subject: [PATCH 254/340] PMM-7: New MySQL setup --- pmm_qa/mysql/setup_mysql.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pmm_qa/mysql/setup_mysql.py b/pmm_qa/mysql/setup_mysql.py index 8cd6d373..b654fe97 100644 --- a/pmm_qa/mysql/setup_mysql.py +++ b/pmm_qa/mysql/setup_mysql.py @@ -1,3 +1,4 @@ +import os from scripts.get_env_value import get_value def setup_mysql_docker(db_type, setup_type, ms_version, container_name, db_config=None, args=None): From f15d09c5548bc9448615b1bc876f9143281e8ac3 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 14 May 2025 08:13:44 +0200 Subject: [PATCH 255/340] PMM-7: New MySQL setup --- pmm_qa/scripts/get_env_value.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pmm_qa/scripts/get_env_value.py b/pmm_qa/scripts/get_env_value.py index 8425b22b..5bd039da 100644 --- a/pmm_qa/scripts/get_env_value.py +++ b/pmm_qa/scripts/get_env_value.py @@ -1,3 +1,5 @@ +import os + def get_value(key, db_type, args, db_config): # Check if the variable exists in the environment env_value = os.environ.get(key) From 835347075793a53e121304e2b09ac10e4f8c361a Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 14 May 2025 08:18:21 +0200 Subject: [PATCH 256/340] PMM-7: New MySQL setup --- pmm_qa/pmm-framework.py | 81 +----------------------------- pmm_qa/scripts/database_options.py | 77 ++++++++++++++++++++++++++++ pmm_qa/scripts/get_env_value.py | 3 +- 3 files changed, 80 insertions(+), 81 deletions(-) create mode 100644 pmm_qa/scripts/database_options.py diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 2697e5bb..655c0da3 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -7,86 +7,7 @@ import re from mysql.setup_mysql import setup_mysql_docker from scripts.get_env_value import get_value - -# Database configurations -database_configs = { - "PSMDB": { - "versions": ["4.4", "5.0", "6.0", "7.0", "8.0", "latest"], - "configurations": {"CLIENT_VERSION": "3-dev-latest", "SETUP_TYPE": "pss", "COMPOSE_PROFILES": "classic", - "TARBALL": ""} - }, - "MLAUNCH_PSMDB": { - "versions": ["4.4", "5.0", "6.0", "7.0", "8.0"], - "configurations": {"CLIENT_VERSION": "3-dev-latest", "SETUP_TYPE": "pss", "TARBALL": ""} - }, - "MLAUNCH_MODB": { - "versions": ["4.4", "5.0", "6.0", "7.0", "8.0"], - "configurations": {"CLIENT_VERSION": "3-dev-latest", "SETUP_TYPE": "pss", "TARBALL": ""} - }, - "SSL_MLAUNCH": { - "versions": ["4.4", "5.0", "6.0", "7.0", "8.0"], - "configurations": {"CLIENT_VERSION": "3-dev-latest", "SETUP_TYPE": "pss", "COMPOSE_PROFILES": "classic", - "TARBALL": ""} - }, - "SSL_PSMDB": { - "versions": ["4.4", "5.0", "6.0", "7.0", "8.0", "latest"], - "configurations": {"CLIENT_VERSION": "3-dev-latest", "SETUP_TYPE": "pss", "COMPOSE_PROFILES": "classic", - "TARBALL": ""} - }, - "MYSQL": { - "versions": ["8.4", "8.0"], - "configurations": {"QUERY_SOURCE": "perfschema", "SETUP_TYPE": "", "CLIENT_VERSION": "3-dev-latest", - "TARBALL": ""} - }, - "PS": { - "versions": ["5.7", "8.4", "8.0"], - "configurations": {"QUERY_SOURCE": "perfschema", "SETUP_TYPE": "", "CLIENT_VERSION": "3-dev-latest", - "TARBALL": "", "NODES_COUNT": 1} - }, - "SSL_MYSQL": { - "versions": ["5.7", "8.4", "8.0"], - "configurations": {"QUERY_SOURCE": "perfschema", "SETUP_TYPE": "", "CLIENT_VERSION": "3-dev-latest", - "TARBALL": ""} - }, - "PGSQL": { - "versions": ["11", "12", "13", "14", "15", "16", "17"], - "configurations": {"QUERY_SOURCE": "pgstatements", "CLIENT_VERSION": "3-dev-latest", "USE_SOCKET": "", - "SETUP_TYPE": ""} - }, - "PDPGSQL": { - "versions": ["11", "12", "13", "14", "15", "16", "17"], - "configurations": {"CLIENT_VERSION": "3-dev-latest", "USE_SOCKET": "", "SETUP_TYPE": ""} - }, - "SSL_PDPGSQL": { - "versions": ["11", "12", "13", "14", "15", "16", "17"], - "configurations": {"CLIENT_VERSION": "3-dev-latest", "USE_SOCKET": ""} - }, - "PXC": { - "versions": ["5.7", "8.0"], - "configurations": {"CLIENT_VERSION": "3-dev-latest", "QUERY_SOURCE": "perfschema", "TARBALL": ""} - }, - "PROXYSQL": { - "versions": ["2"], - "configurations": {"PACKAGE": ""} - }, - "HAPROXY": { - "versions": [""], - "configurations": {"CLIENT_VERSION": "3-dev-latest"} - }, - "EXTERNAL": { - "REDIS": { - "versions": ["1.14.0", "1.58.0"], - }, - "NODEPROCESS": { - "versions": ["0.7.5", "0.7.10"], - }, - "configurations": {"CLIENT_VERSION": "3-dev-latest"} - }, - "DOCKERCLIENTS": { - "configurations": {} # Empty dictionary for consistency - }, -} - +from scripts.database_options import database_options as database_configs def run_ansible_playbook(playbook_filename, env_vars, args): # Get Script Dir diff --git a/pmm_qa/scripts/database_options.py b/pmm_qa/scripts/database_options.py new file mode 100644 index 00000000..8faba728 --- /dev/null +++ b/pmm_qa/scripts/database_options.py @@ -0,0 +1,77 @@ +database_options = { + "PSMDB": { + "versions": ["4.4", "5.0", "6.0", "7.0", "8.0", "latest"], + "configurations": {"CLIENT_VERSION": "3-dev-latest", "SETUP_TYPE": "pss", "COMPOSE_PROFILES": "classic", + "TARBALL": ""} + }, + "MLAUNCH_PSMDB": { + "versions": ["4.4", "5.0", "6.0", "7.0", "8.0"], + "configurations": {"CLIENT_VERSION": "3-dev-latest", "SETUP_TYPE": "pss", "TARBALL": ""} + }, + "MLAUNCH_MODB": { + "versions": ["4.4", "5.0", "6.0", "7.0", "8.0"], + "configurations": {"CLIENT_VERSION": "3-dev-latest", "SETUP_TYPE": "pss", "TARBALL": ""} + }, + "SSL_MLAUNCH": { + "versions": ["4.4", "5.0", "6.0", "7.0", "8.0"], + "configurations": {"CLIENT_VERSION": "3-dev-latest", "SETUP_TYPE": "pss", "COMPOSE_PROFILES": "classic", + "TARBALL": ""} + }, + "SSL_PSMDB": { + "versions": ["4.4", "5.0", "6.0", "7.0", "8.0", "latest"], + "configurations": {"CLIENT_VERSION": "3-dev-latest", "SETUP_TYPE": "pss", "COMPOSE_PROFILES": "classic", + "TARBALL": ""} + }, + "MYSQL": { + "versions": ["8.4", "8.0"], + "configurations": {"QUERY_SOURCE": "perfschema", "SETUP_TYPE": "", "CLIENT_VERSION": "3-dev-latest", + "TARBALL": ""} + }, + "PS": { + "versions": ["5.7", "8.4", "8.0"], + "configurations": {"QUERY_SOURCE": "perfschema", "SETUP_TYPE": "", "CLIENT_VERSION": "3-dev-latest", + "TARBALL": "", "NODES_COUNT": 1} + }, + "SSL_MYSQL": { + "versions": ["5.7", "8.4", "8.0"], + "configurations": {"QUERY_SOURCE": "perfschema", "SETUP_TYPE": "", "CLIENT_VERSION": "3-dev-latest", + "TARBALL": ""} + }, + "PGSQL": { + "versions": ["11", "12", "13", "14", "15", "16", "17"], + "configurations": {"QUERY_SOURCE": "pgstatements", "CLIENT_VERSION": "3-dev-latest", "USE_SOCKET": "", + "SETUP_TYPE": ""} + }, + "PDPGSQL": { + "versions": ["11", "12", "13", "14", "15", "16", "17"], + "configurations": {"CLIENT_VERSION": "3-dev-latest", "USE_SOCKET": "", "SETUP_TYPE": ""} + }, + "SSL_PDPGSQL": { + "versions": ["11", "12", "13", "14", "15", "16", "17"], + "configurations": {"CLIENT_VERSION": "3-dev-latest", "USE_SOCKET": ""} + }, + "PXC": { + "versions": ["5.7", "8.0"], + "configurations": {"CLIENT_VERSION": "3-dev-latest", "QUERY_SOURCE": "perfschema", "TARBALL": ""} + }, + "PROXYSQL": { + "versions": ["2"], + "configurations": {"PACKAGE": ""} + }, + "HAPROXY": { + "versions": [""], + "configurations": {"CLIENT_VERSION": "3-dev-latest"} + }, + "EXTERNAL": { + "REDIS": { + "versions": ["1.14.0", "1.58.0"], + }, + "NODEPROCESS": { + "versions": ["0.7.5", "0.7.10"], + }, + "configurations": {"CLIENT_VERSION": "3-dev-latest"} + }, + "DOCKERCLIENTS": { + "configurations": {} # Empty dictionary for consistency + }, +} \ No newline at end of file diff --git a/pmm_qa/scripts/get_env_value.py b/pmm_qa/scripts/get_env_value.py index 5bd039da..130406cd 100644 --- a/pmm_qa/scripts/get_env_value.py +++ b/pmm_qa/scripts/get_env_value.py @@ -1,4 +1,5 @@ import os +from database_options import database_options def get_value(key, db_type, args, db_config): # Check if the variable exists in the environment @@ -16,4 +17,4 @@ def get_value(key, db_type, args, db_config): return config_value # Fall back to default configs value or empty '' - return database_configs[db_type]["configurations"].get(key, '') + return database_options[db_type]["configurations"].get(key, '') From 744b875ed1f5890bc7fd0b80ee9fdce6a99174ad Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 14 May 2025 08:21:46 +0200 Subject: [PATCH 257/340] PMM-7: New MySQL setup --- pmm_qa/scripts/get_env_value.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/scripts/get_env_value.py b/pmm_qa/scripts/get_env_value.py index 130406cd..74d65149 100644 --- a/pmm_qa/scripts/get_env_value.py +++ b/pmm_qa/scripts/get_env_value.py @@ -1,5 +1,5 @@ import os -from database_options import database_options +from .database_options import database_options def get_value(key, db_type, args, db_config): # Check if the variable exists in the environment From 538ccb8bcf1ef96ed7086f2ad94cd591a0ba8f24 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 14 May 2025 08:23:44 +0200 Subject: [PATCH 258/340] PMM-7: New MySQL setup --- pmm_qa/mysql/setup_mysql.py | 1 + pmm_qa/pmm-framework.py | 32 +------------------------- pmm_qa/scripts/run_ansible_playbook.py | 28 ++++++++++++++++++++++ 3 files changed, 30 insertions(+), 31 deletions(-) create mode 100644 pmm_qa/scripts/run_ansible_playbook.py diff --git a/pmm_qa/mysql/setup_mysql.py b/pmm_qa/mysql/setup_mysql.py index b654fe97..5a3f1ff7 100644 --- a/pmm_qa/mysql/setup_mysql.py +++ b/pmm_qa/mysql/setup_mysql.py @@ -1,5 +1,6 @@ import os from scripts.get_env_value import get_value +from scripts.run_ansible_playbook import run_ansible_playbook def setup_mysql_docker(db_type, setup_type, ms_version, container_name, db_config=None, args=None): env_vars = { diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 655c0da3..dc4add92 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -8,37 +8,7 @@ from mysql.setup_mysql import setup_mysql_docker from scripts.get_env_value import get_value from scripts.database_options import database_options as database_configs - -def run_ansible_playbook(playbook_filename, env_vars, args): - # Get Script Dir - script_path = os.path.abspath(sys.argv[0]) - script_dir = os.path.dirname(script_path) - playbook_path = script_dir + "/" + playbook_filename - verbosity_level = 1 - - - - if args.verbosity_level is not None: - verbosity_level = int(args.verbosity_level) - - if args.verbose: - print(f'Options set after considering Defaults: {env_vars}') - - r = ansible_runner.run( - private_data_dir=script_dir, - playbook=playbook_path, - inventory='127.0.0.1', - cmdline='-l localhost, --connection=local', - envvars=env_vars, - suppress_env_files=True, - verbosity=verbosity_level, - ) - - print(f'{playbook_filename} playbook execution {r.status}') - - if r.rc != 0: - exit(1) - +from scripts.run_ansible_playbook import run_ansible_playbook def get_running_container_name(): container_image_name = "pmm-server" diff --git a/pmm_qa/scripts/run_ansible_playbook.py b/pmm_qa/scripts/run_ansible_playbook.py new file mode 100644 index 00000000..b1c47ad2 --- /dev/null +++ b/pmm_qa/scripts/run_ansible_playbook.py @@ -0,0 +1,28 @@ +def run_ansible_playbook(playbook_filename, env_vars, args): + # Get Script Dir + script_path = os.path.abspath(sys.argv[0]) + script_dir = os.path.dirname(script_path) + playbook_path = script_dir + "/" + playbook_filename + verbosity_level = 1 + + if args.verbosity_level is not None: + verbosity_level = int(args.verbosity_level) + + if args.verbose: + print(f'Options set after considering Defaults: {env_vars}') + + r = ansible_runner.run( + private_data_dir=script_dir, + playbook=playbook_path, + inventory='127.0.0.1', + cmdline='-l localhost, --connection=local', + envvars=env_vars, + suppress_env_files=True, + verbosity=verbosity_level, + ) + + print(f'{playbook_filename} playbook execution {r.status}') + + if r.rc != 0: + exit(1) + From 6b315abef8804dc960cfdcada36bc2ba71f12a69 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 14 May 2025 08:24:48 +0200 Subject: [PATCH 259/340] PMM-7: New MySQL setup --- pmm_qa/pmm-framework.py | 1 - pmm_qa/scripts/run_ansible_playbook.py | 3 +++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index dc4add92..71c2dd6f 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -2,7 +2,6 @@ import argparse import os import sys -import ansible_runner import requests import re from mysql.setup_mysql import setup_mysql_docker diff --git a/pmm_qa/scripts/run_ansible_playbook.py b/pmm_qa/scripts/run_ansible_playbook.py index b1c47ad2..e2b51ea6 100644 --- a/pmm_qa/scripts/run_ansible_playbook.py +++ b/pmm_qa/scripts/run_ansible_playbook.py @@ -1,3 +1,6 @@ +import os +import ansible_runner + def run_ansible_playbook(playbook_filename, env_vars, args): # Get Script Dir script_path = os.path.abspath(sys.argv[0]) From d21226371f1efdb9545dfcdfd800a35ecede53bd Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 14 May 2025 08:25:26 +0200 Subject: [PATCH 260/340] PMM-7: New MySQL setup --- pmm_qa/scripts/run_ansible_playbook.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pmm_qa/scripts/run_ansible_playbook.py b/pmm_qa/scripts/run_ansible_playbook.py index e2b51ea6..66d1a334 100644 --- a/pmm_qa/scripts/run_ansible_playbook.py +++ b/pmm_qa/scripts/run_ansible_playbook.py @@ -1,5 +1,6 @@ import os import ansible_runner +import sys def run_ansible_playbook(playbook_filename, env_vars, args): # Get Script Dir From e7caad94305b9d08dff4b29886ec317403fd05a1 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 14 May 2025 08:26:37 +0200 Subject: [PATCH 261/340] PMM-7: New MySQL setup --- pmm_qa/mysql/setup_mysql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/mysql/setup_mysql.py b/pmm_qa/mysql/setup_mysql.py index 5a3f1ff7..cd04ed59 100644 --- a/pmm_qa/mysql/setup_mysql.py +++ b/pmm_qa/mysql/setup_mysql.py @@ -13,4 +13,4 @@ def setup_mysql_docker(db_type, setup_type, ms_version, container_name, db_confi 'NODES_COUNT': get_value('NODES_COUNT', db_type, args, db_config), } - run_ansible_playbook('./mysql_setup.yml', env_vars, args) + run_ansible_playbook('mysql/mysql_setup.yml', env_vars, args) From 6cf6ccc317fec949a5936804da0de8bfc6d7f6c8 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 14 May 2025 08:31:10 +0200 Subject: [PATCH 262/340] PMM-7: New MySQL setup --- pmm_qa/tasks/install_pmm_client.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pmm_qa/tasks/install_pmm_client.yml b/pmm_qa/tasks/install_pmm_client.yml index a8c08951..dda754e8 100644 --- a/pmm_qa/tasks/install_pmm_client.yml +++ b/pmm_qa/tasks/install_pmm_client.yml @@ -63,7 +63,14 @@ - name: Install dependencies on RHEL-family containers community.docker.docker_container_exec: container: "{{ container_name }}" - command: microdnf install -y wget gnupg2 jq curl-minimal + command: microdnf install -y wget gnupg2 jq + user: "root" + when: distro_family == "rhel" + +- name: Install curl on RHEL-family containers + community.docker.docker_container_exec: + container: "{{ container_name }}" + command: microdnf install -y curl-minimal user: "root" when: distro_family == "rhel" ignore_errors: true From 7d5d9e4352e7c73a7e4070fa7128a90b9b057e97 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 14 May 2025 08:40:52 +0200 Subject: [PATCH 263/340] PMM-7: New MySQL setup --- pmm_qa/mysql/mysql_setup.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/mysql/mysql_setup.yml b/pmm_qa/mysql/mysql_setup.yml index 8c3b4bdd..12cb4cf3 100644 --- a/pmm_qa/mysql/mysql_setup.yml +++ b/pmm_qa/mysql/mysql_setup.yml @@ -13,7 +13,7 @@ mysql_port: 4306 mysql_listen_port: 3306 group_seeds_port: 44061 - nodes_count: "{{ (lookup('env', 'NODES_COUNT') | default('3', true)) | int }}" + nodes_count: "{{ (lookup('env', 'NODES_COUNT') | default('1', true)) | int }}" network_name: "pmm-qa" data_dir: "{{ lookup('env', 'HOME') }}/mysql_cluster_data" server_id_start: 1 From 578b3ca7f4446ea9cb6b4a7906fa72865b4c3c17 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 14 May 2025 08:49:30 +0200 Subject: [PATCH 264/340] PMM-7: New MySQL setup --- pmm_qa/mysql/setup_mysql.py | 6 +++--- pmm_qa/pmm-framework.py | 2 +- pmm_qa/scripts/database_options.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pmm_qa/mysql/setup_mysql.py b/pmm_qa/mysql/setup_mysql.py index cd04ed59..c38b5302 100644 --- a/pmm_qa/mysql/setup_mysql.py +++ b/pmm_qa/mysql/setup_mysql.py @@ -2,10 +2,10 @@ from scripts.get_env_value import get_value from scripts.run_ansible_playbook import run_ansible_playbook -def setup_mysql_docker(db_type, setup_type, ms_version, container_name, db_config=None, args=None): +def setup_mysql_docker(db_type, container_name, db_config=None, args=None): env_vars = { - 'SETUP_TYPE': setup_type, - 'MS_VERSION': ms_version, + 'SETUP_TYPE': get_value('SETUP_TYPE', db_type, args, db_config).lower(), + 'MS_VERSION': get_value('MS_VERSION') or db_version or database_configs[db_type]["versions"][-1], 'PMM_SERVER_IP': args.pmm_server_ip or container_name or '127.0.0.1', 'CLIENT_VERSION': get_value('CLIENT_VERSION', db_type, args, db_config), 'QUERY_SOURCE': get_value('QUERY_SOURCE', db_type, args, db_config), diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 71c2dd6f..b9a0cb39 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -123,7 +123,7 @@ def setup_mysql(db_type, db_version=None, db_config=None, args=None): no_of_nodes = 2 if ms_version_int >= 84: - setup_mysql_docker(db_type, setup_type, ms_version, container_name, db_config, args) + setup_mysql_docker(db_type, container_name, db_config, args) else: # Define environment variables for playbook env_vars = { diff --git a/pmm_qa/scripts/database_options.py b/pmm_qa/scripts/database_options.py index 8faba728..7b76bdba 100644 --- a/pmm_qa/scripts/database_options.py +++ b/pmm_qa/scripts/database_options.py @@ -23,7 +23,7 @@ "TARBALL": ""} }, "MYSQL": { - "versions": ["8.4", "8.0"], + "versions": ["8.0", "8.4"], "configurations": {"QUERY_SOURCE": "perfschema", "SETUP_TYPE": "", "CLIENT_VERSION": "3-dev-latest", "TARBALL": ""} }, From 73e449a563ede7a68b5a69e7dcebde83071bc4f8 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 14 May 2025 08:51:24 +0200 Subject: [PATCH 265/340] PMM-7: New MySQL setup --- pmm_qa/mysql/setup_mysql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/mysql/setup_mysql.py b/pmm_qa/mysql/setup_mysql.py index c38b5302..aaa9ab72 100644 --- a/pmm_qa/mysql/setup_mysql.py +++ b/pmm_qa/mysql/setup_mysql.py @@ -5,7 +5,7 @@ def setup_mysql_docker(db_type, container_name, db_config=None, args=None): env_vars = { 'SETUP_TYPE': get_value('SETUP_TYPE', db_type, args, db_config).lower(), - 'MS_VERSION': get_value('MS_VERSION') or db_version or database_configs[db_type]["versions"][-1], + 'MS_VERSION': get_value('MS_VERSION', db_type, args, db_config), 'PMM_SERVER_IP': args.pmm_server_ip or container_name or '127.0.0.1', 'CLIENT_VERSION': get_value('CLIENT_VERSION', db_type, args, db_config), 'QUERY_SOURCE': get_value('QUERY_SOURCE', db_type, args, db_config), From 740c401fe34cf9c64fbc19f68fdb51d5f409a98e Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 14 May 2025 08:54:45 +0200 Subject: [PATCH 266/340] PMM-7: New MySQL setup --- pmm_qa/mysql/mysql_setup.yml | 2 +- .../tasks/mysql-group-replication-setup.yml | 127 ++++++++++++++++++ 2 files changed, 128 insertions(+), 1 deletion(-) create mode 100644 pmm_qa/mysql/tasks/mysql-group-replication-setup.yml diff --git a/pmm_qa/mysql/mysql_setup.yml b/pmm_qa/mysql/mysql_setup.yml index 12cb4cf3..3c311eb9 100644 --- a/pmm_qa/mysql/mysql_setup.yml +++ b/pmm_qa/mysql/mysql_setup.yml @@ -66,7 +66,7 @@ loop: "{{ range(1, nodes_count | int + 1) | list }}" - name: Setup MySQL group replication - include_tasks: ./tasks/percona-server-group-replication-setup.yml + include_tasks: ./tasks/mysql-group-replication-setup.yml when: setup_type == "gr" - name: Setup MySQL with async replication diff --git a/pmm_qa/mysql/tasks/mysql-group-replication-setup.yml b/pmm_qa/mysql/tasks/mysql-group-replication-setup.yml new file mode 100644 index 00000000..8ae4e86b --- /dev/null +++ b/pmm_qa/mysql/tasks/mysql-group-replication-setup.yml @@ -0,0 +1,127 @@ +- name: Generate my.cnf for each node + template: + src: my.cnf.j2 + dest: "{{ data_dir }}/node{{ item }}/my.cnf" + loop: "{{ range(1, nodes_count | int + 1) | list }}" + +- name: Create initialization script for each node + template: + src: init.sql.j2 + dest: "{{ data_dir }}/node{{ item }}/init.sql" + loop: "{{ range(1, nodes_count | int + 1) | list }}" + +- name: Start MySQL containers with group replication + community.docker.docker_container: + name: "ps_pmm_{{ ps_version }}_{{ item }}" + image: "mysql:{{ ps_version }}" + restart_policy: always + state: started + networks: + - name: "{{ network_name }}" + env: + MYSQL_ROOT_PASSWORD: "{{ root_password }}" + ports: + - "{{ mysql_port + item - 1 }}:{{ mysql_listen_port }}" + - "{{ group_seeds_port + item - 1 }}:{{ group_seeds_port }}" + volumes: + - "{{ data_dir }}/node{{ item }}/data:/var/lib/mysql" + - "{{ data_dir }}/node{{ item }}/my.cnf:/etc/mysql/my.cnf" + - "{{ data_dir }}/node{{ item }}/init.sql:/docker-entrypoint-initdb.d/init.sql" + loop: "{{ range(1, nodes_count | int + 1) | list }}" + +- name: Wait for MySQL to be available + wait_for: + host: localhost + port: "{{ mysql_port + item - 1 }}" + delay: 10 + timeout: 300 + loop: "{{ range(1, nodes_count | int + 1) | list }}" + +- name: Reset configuration for all nodes + community.docker.docker_container_exec: + container: "ps_pmm_{{ ps_version }}_{{ item }}" + command: > + mysql -uroot -p{{ root_password }} -e " + RESET BINARY LOGS AND GTIDS; + RESET REPLICA ALL; + SET GLOBAL gtid_purged=''; + " + loop: "{{ range(1, nodes_count | int + 1) | list }}" + +- name: Bootstrap first node in the cluster + community.docker.docker_container_exec: + container: "ps_pmm_{{ ps_version }}_1" + command: > + mysql -uroot -p{{ root_password }} -e " + SET GLOBAL group_replication_bootstrap_group=ON; + START GROUP_REPLICATION; + SET GLOBAL group_replication_bootstrap_group=OFF;" + retries: 5 + delay: 10 + +- name: Wait 5 seconds for bootstrap to complete + pause: + seconds: 5 + +- name: Start group replication on other nodes + community.docker.docker_container_exec: + container: "ps_pmm_{{ ps_version }}_{{ item }}" + command: mysql -uroot -p{{ root_password }} -e "START GROUP_REPLICATION;" + loop: "{{ range(2, nodes_count | int + 1) | list }}" + ignore_errors: yes + +- name: Wait 10 seconds for the other nodes to join + pause: + seconds: 10 + +- name: Create and seed a test database on primary + community.docker.docker_container_exec: + container: "ps_pmm_{{ ps_version }}_1" + command: > + mysql -uroot -p{{ root_password}} -e " + CREATE DATABASE testdb; + USE testdb; + CREATE TABLE testdb (id INT PRIMARY KEY, data VARCHAR(100)); + INSERT INTO testdb VALUES (1, 'Initial data from node mysql1');" + +- name: Check replication status on first node + community.docker.docker_container_exec: + container: "ps_pmm_{{ ps_version }}_1" + command: mysql -uroot -p{{ root_password }} -e "SELECT * FROM performance_schema.replication_group_members;" + register: replication_status + +- name: Display replication status + debug: + var: replication_status.stdout + +- name: Check replication group members count + community.docker.docker_container_exec: + container: "ps_pmm_{{ ps_version }}_1" + command: mysql -uroot -p{{ root_password }} -e "SELECT COUNT(*) AS count FROM performance_schema.replication_group_members;" + register: member_count + +- name: Display member count + debug: + var: member_count.stdout + +- name: Set verification instructions + set_fact: + verification_msg: | + MySQL Cluster setup complete! + + To verify replication is working: + 1. Connect to the first node: + docker exec -it ps_pmm_{{ ps_version }}_1 mysql -uroot -p{{ root_password }} + + 2. Insert data in the test database: + USE testdb; + INSERT INTO testdb VALUES (100, 'Test replication'); + + 3. Connect to other nodes and verify data is replicated: + docker exec -it ps_pmm_{{ ps_version }}_2 mysql -uroot -p{{ root_password }} + USE testdb; + SELECT * FROM testdb; + +- name: Display verification instructions + debug: + msg: "{{ verification_msg | split('\n') }}" From c8fe7b1e70d1704b8ac9ec6f83541e69db68e508 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 14 May 2025 08:58:28 +0200 Subject: [PATCH 267/340] PMM-7: New MySQL setup --- pmm_qa/mysql/data/my.cnf.j2 | 2 ++ pmm_qa/mysql/init.sql.j2 | 15 +++++++++++++ pmm_qa/mysql/my.cnf.j2 | 45 +++++++++++++++++++++++++++++++++++++ 3 files changed, 62 insertions(+) create mode 100644 pmm_qa/mysql/init.sql.j2 create mode 100644 pmm_qa/mysql/my.cnf.j2 diff --git a/pmm_qa/mysql/data/my.cnf.j2 b/pmm_qa/mysql/data/my.cnf.j2 index e69de29b..fd4b27f2 100644 --- a/pmm_qa/mysql/data/my.cnf.j2 +++ b/pmm_qa/mysql/data/my.cnf.j2 @@ -0,0 +1,2 @@ +[mysqld] +userstat=1 \ No newline at end of file diff --git a/pmm_qa/mysql/init.sql.j2 b/pmm_qa/mysql/init.sql.j2 new file mode 100644 index 00000000..19185831 --- /dev/null +++ b/pmm_qa/mysql/init.sql.j2 @@ -0,0 +1,15 @@ +-- Create replication user and grant necessary privileges +SET SQL_LOG_BIN=0; +CREATE USER '{{ replication_user }}'@'%' IDENTIFIED BY '{{ replication_password }}'; +GRANT REPLICATION SLAVE ON *.* TO '{{ replication_user }}'@'%'; +GRANT CONNECTION_ADMIN ON *.* TO '{{ replication_user }}'@'%'; +GRANT BACKUP_ADMIN ON *.* TO '{{ replication_user }}'@'%'; +GRANT GROUP_REPLICATION_STREAM ON *.* TO '{{ replication_user }}'@'%'; +-- GRANT SERVICE_CONNECTION_ADMIN ON *.* TO '{{ replication_user }}'@'%'; +-- GRANT SYSTEM_VARIABLES_ADMIN ON *.* TO '{{ replication_user }}'@'%'; +FLUSH PRIVILEGES; +SET SQL_LOG_BIN=1; + +-- Configure group replication recovery credentials +CHANGE REPLICATION SOURCE TO SOURCE_USER='{{ replication_user }}', SOURCE_PASSWORD='{{ replication_password }}' FOR CHANNEL 'group_replication_recovery'; + diff --git a/pmm_qa/mysql/my.cnf.j2 b/pmm_qa/mysql/my.cnf.j2 new file mode 100644 index 00000000..8fcdbbf6 --- /dev/null +++ b/pmm_qa/mysql/my.cnf.j2 @@ -0,0 +1,45 @@ +[mysqld] +# General server configuration +server_id={{ server_id_start + item - 1 }} +bind-address=0.0.0.0 +port={{ mysql_listen_port }} +userstat=1 + +# General replication settings +gtid_mode=ON +enforce_gtid_consistency=ON +binlog_checksum=NONE +log_bin=binlog +log_replica_updates=ON +disabled_storage_engines="MyISAM,BLACKHOLE,FEDERATED,ARCHIVE,MEMORY" +lower_case_table_names=2 # MacOS-specific, but also good generally + +# MySQL 8.4 compatibility settings +report_host=ps_pmm_{{ps_version}}_{{ item }} + +# Group Replication Settings +plugin_load_add='group_replication.so' +loose-group_replication_group_name='aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee' +loose-group_replication_local_address='ps_pmm_{{ps_version}}_{{ item }}:{{ group_seeds_port }}' +loose-group_replication_group_seeds='{% for i in range(1, nodes_count | int + 1) %}ps_pmm_{{ps_version}}_{{ i }}:{{ group_seeds_port }}{% if not loop.last %},{% endif %}{% endfor %}' +loose-group_replication_communication_stack=XCOM + +# Group replication behavior +loose-group_replication_start_on_boot=OFF +loose-group_replication_bootstrap_group=OFF +loose-group_replication_single_primary_mode=ON +loose-group_replication_enforce_update_everywhere_checks=OFF + +# Recovery settings +loose-group_replication_recovery_get_public_key=ON +loose-group_replication_recovery_retry_count=10 +loose-group_replication_recovery_reconnect_interval=60 + +# Crash-safe replication settings +relay-log=ps_pmm_{{ps_version}}_{{ item }}-relay-bin +relay_log_recovery=ON +relay_log_purge=ON + +# Performance and connection settings +max_connections=1000 +innodb_buffer_pool_size=256M From 4ec5fd90cb1e3a9cd770c315b77fb99834cdee8f Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 14 May 2025 09:00:56 +0200 Subject: [PATCH 268/340] PMM-7: New MySQL setup --- .../tasks/mysql-group-replication-setup.yml | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/pmm_qa/mysql/tasks/mysql-group-replication-setup.yml b/pmm_qa/mysql/tasks/mysql-group-replication-setup.yml index 8ae4e86b..f3a4597c 100644 --- a/pmm_qa/mysql/tasks/mysql-group-replication-setup.yml +++ b/pmm_qa/mysql/tasks/mysql-group-replication-setup.yml @@ -12,8 +12,8 @@ - name: Start MySQL containers with group replication community.docker.docker_container: - name: "ps_pmm_{{ ps_version }}_{{ item }}" - image: "mysql:{{ ps_version }}" + name: "ps_pmm_{{ ms_version }}_{{ item }}" + image: "mysql:{{ ms_version }}" restart_policy: always state: started networks: @@ -39,7 +39,7 @@ - name: Reset configuration for all nodes community.docker.docker_container_exec: - container: "ps_pmm_{{ ps_version }}_{{ item }}" + container: "ps_pmm_{{ ms_version }}_{{ item }}" command: > mysql -uroot -p{{ root_password }} -e " RESET BINARY LOGS AND GTIDS; @@ -50,7 +50,7 @@ - name: Bootstrap first node in the cluster community.docker.docker_container_exec: - container: "ps_pmm_{{ ps_version }}_1" + container: "ps_pmm_{{ ms_version }}_1" command: > mysql -uroot -p{{ root_password }} -e " SET GLOBAL group_replication_bootstrap_group=ON; @@ -65,7 +65,7 @@ - name: Start group replication on other nodes community.docker.docker_container_exec: - container: "ps_pmm_{{ ps_version }}_{{ item }}" + container: "ps_pmm_{{ ms_version }}_{{ item }}" command: mysql -uroot -p{{ root_password }} -e "START GROUP_REPLICATION;" loop: "{{ range(2, nodes_count | int + 1) | list }}" ignore_errors: yes @@ -76,7 +76,7 @@ - name: Create and seed a test database on primary community.docker.docker_container_exec: - container: "ps_pmm_{{ ps_version }}_1" + container: "ps_pmm_{{ ms_version }}_1" command: > mysql -uroot -p{{ root_password}} -e " CREATE DATABASE testdb; @@ -86,7 +86,7 @@ - name: Check replication status on first node community.docker.docker_container_exec: - container: "ps_pmm_{{ ps_version }}_1" + container: "ps_pmm_{{ ms_version }}_1" command: mysql -uroot -p{{ root_password }} -e "SELECT * FROM performance_schema.replication_group_members;" register: replication_status @@ -96,7 +96,7 @@ - name: Check replication group members count community.docker.docker_container_exec: - container: "ps_pmm_{{ ps_version }}_1" + container: "ps_pmm_{{ ms_version }}_1" command: mysql -uroot -p{{ root_password }} -e "SELECT COUNT(*) AS count FROM performance_schema.replication_group_members;" register: member_count @@ -111,14 +111,14 @@ To verify replication is working: 1. Connect to the first node: - docker exec -it ps_pmm_{{ ps_version }}_1 mysql -uroot -p{{ root_password }} + docker exec -it ps_pmm_{{ ms_version }}_1 mysql -uroot -p{{ root_password }} 2. Insert data in the test database: USE testdb; INSERT INTO testdb VALUES (100, 'Test replication'); 3. Connect to other nodes and verify data is replicated: - docker exec -it ps_pmm_{{ ps_version }}_2 mysql -uroot -p{{ root_password }} + docker exec -it ps_pmm_{{ ms_version }}_2 mysql -uroot -p{{ root_password }} USE testdb; SELECT * FROM testdb; From 1cda8437b38f5050c858ff834b37c9373b700701 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 14 May 2025 09:03:16 +0200 Subject: [PATCH 269/340] PMM-7: New MySQL setup --- pmm_qa/mysql/{ => data/group_replication}/init.sql.j2 | 0 pmm_qa/mysql/{ => data/group_replication}/my.cnf.j2 | 0 pmm_qa/mysql/tasks/mysql-group-replication-setup.yml | 4 ++-- 3 files changed, 2 insertions(+), 2 deletions(-) rename pmm_qa/mysql/{ => data/group_replication}/init.sql.j2 (100%) rename pmm_qa/mysql/{ => data/group_replication}/my.cnf.j2 (100%) diff --git a/pmm_qa/mysql/init.sql.j2 b/pmm_qa/mysql/data/group_replication/init.sql.j2 similarity index 100% rename from pmm_qa/mysql/init.sql.j2 rename to pmm_qa/mysql/data/group_replication/init.sql.j2 diff --git a/pmm_qa/mysql/my.cnf.j2 b/pmm_qa/mysql/data/group_replication/my.cnf.j2 similarity index 100% rename from pmm_qa/mysql/my.cnf.j2 rename to pmm_qa/mysql/data/group_replication/my.cnf.j2 diff --git a/pmm_qa/mysql/tasks/mysql-group-replication-setup.yml b/pmm_qa/mysql/tasks/mysql-group-replication-setup.yml index f3a4597c..68b9bfb4 100644 --- a/pmm_qa/mysql/tasks/mysql-group-replication-setup.yml +++ b/pmm_qa/mysql/tasks/mysql-group-replication-setup.yml @@ -1,12 +1,12 @@ - name: Generate my.cnf for each node template: - src: my.cnf.j2 + src: ./data/group_replication/my.cnf.j2 dest: "{{ data_dir }}/node{{ item }}/my.cnf" loop: "{{ range(1, nodes_count | int + 1) | list }}" - name: Create initialization script for each node template: - src: init.sql.j2 + src: ./data/group_replication/init.sql.j2 dest: "{{ data_dir }}/node{{ item }}/init.sql" loop: "{{ range(1, nodes_count | int + 1) | list }}" From f536f7cc70ce31e42a72aa2d7328cce36ff45227 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 14 May 2025 09:04:08 +0200 Subject: [PATCH 270/340] PMM-7: New MySQL setup --- .../tasks/mysql-group-replication-setup.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/pmm_qa/mysql/tasks/mysql-group-replication-setup.yml b/pmm_qa/mysql/tasks/mysql-group-replication-setup.yml index 68b9bfb4..e7d7cf92 100644 --- a/pmm_qa/mysql/tasks/mysql-group-replication-setup.yml +++ b/pmm_qa/mysql/tasks/mysql-group-replication-setup.yml @@ -12,7 +12,7 @@ - name: Start MySQL containers with group replication community.docker.docker_container: - name: "ps_pmm_{{ ms_version }}_{{ item }}" + name: "mysql_pmm_{{ ms_version }}_{{ item }}" image: "mysql:{{ ms_version }}" restart_policy: always state: started @@ -39,7 +39,7 @@ - name: Reset configuration for all nodes community.docker.docker_container_exec: - container: "ps_pmm_{{ ms_version }}_{{ item }}" + container: "mysql_pmm_{{ ms_version }}_{{ item }}" command: > mysql -uroot -p{{ root_password }} -e " RESET BINARY LOGS AND GTIDS; @@ -50,7 +50,7 @@ - name: Bootstrap first node in the cluster community.docker.docker_container_exec: - container: "ps_pmm_{{ ms_version }}_1" + container: "mysql_pmm_{{ ms_version }}_1" command: > mysql -uroot -p{{ root_password }} -e " SET GLOBAL group_replication_bootstrap_group=ON; @@ -65,7 +65,7 @@ - name: Start group replication on other nodes community.docker.docker_container_exec: - container: "ps_pmm_{{ ms_version }}_{{ item }}" + container: "mysql_pmm_{{ ms_version }}_{{ item }}" command: mysql -uroot -p{{ root_password }} -e "START GROUP_REPLICATION;" loop: "{{ range(2, nodes_count | int + 1) | list }}" ignore_errors: yes @@ -76,7 +76,7 @@ - name: Create and seed a test database on primary community.docker.docker_container_exec: - container: "ps_pmm_{{ ms_version }}_1" + container: "mysql_pmm_{{ ms_version }}_1" command: > mysql -uroot -p{{ root_password}} -e " CREATE DATABASE testdb; @@ -86,7 +86,7 @@ - name: Check replication status on first node community.docker.docker_container_exec: - container: "ps_pmm_{{ ms_version }}_1" + container: "mysql_pmm_{{ ms_version }}_1" command: mysql -uroot -p{{ root_password }} -e "SELECT * FROM performance_schema.replication_group_members;" register: replication_status @@ -96,7 +96,7 @@ - name: Check replication group members count community.docker.docker_container_exec: - container: "ps_pmm_{{ ms_version }}_1" + container: "mysql_pmm_{{ ms_version }}_1" command: mysql -uroot -p{{ root_password }} -e "SELECT COUNT(*) AS count FROM performance_schema.replication_group_members;" register: member_count @@ -111,14 +111,14 @@ To verify replication is working: 1. Connect to the first node: - docker exec -it ps_pmm_{{ ms_version }}_1 mysql -uroot -p{{ root_password }} + docker exec -it mysql_pmm_{{ ms_version }}_1 mysql -uroot -p{{ root_password }} 2. Insert data in the test database: USE testdb; INSERT INTO testdb VALUES (100, 'Test replication'); 3. Connect to other nodes and verify data is replicated: - docker exec -it ps_pmm_{{ ms_version }}_2 mysql -uroot -p{{ root_password }} + docker exec -it mysql_pmm_{{ ms_version }}_2 mysql -uroot -p{{ root_password }} USE testdb; SELECT * FROM testdb; From 8e56f67c7274e9c1d66316b0efe89e24e8e6b309 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 14 May 2025 09:06:23 +0200 Subject: [PATCH 271/340] PMM-7: New MySQL setup --- pmm_qa/mysql/data/group_replication/my.cnf.j2 | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pmm_qa/mysql/data/group_replication/my.cnf.j2 b/pmm_qa/mysql/data/group_replication/my.cnf.j2 index 8fcdbbf6..d672e396 100644 --- a/pmm_qa/mysql/data/group_replication/my.cnf.j2 +++ b/pmm_qa/mysql/data/group_replication/my.cnf.j2 @@ -15,13 +15,13 @@ disabled_storage_engines="MyISAM,BLACKHOLE,FEDERATED,ARCHIVE,MEMORY" lower_case_table_names=2 # MacOS-specific, but also good generally # MySQL 8.4 compatibility settings -report_host=ps_pmm_{{ps_version}}_{{ item }} +report_host=mysql_pmm_{{ms_version}}_{{ item }} # Group Replication Settings plugin_load_add='group_replication.so' loose-group_replication_group_name='aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee' -loose-group_replication_local_address='ps_pmm_{{ps_version}}_{{ item }}:{{ group_seeds_port }}' -loose-group_replication_group_seeds='{% for i in range(1, nodes_count | int + 1) %}ps_pmm_{{ps_version}}_{{ i }}:{{ group_seeds_port }}{% if not loop.last %},{% endif %}{% endfor %}' +loose-group_replication_local_address='mysql_pmm_{{ms_version}}_{{ item }}:{{ group_seeds_port }}' +loose-group_replication_group_seeds='{% for i in range(1, nodes_count | int + 1) %}mysql_pmm_{{ms_version}}_{{ i }}:{{ group_seeds_port }}{% if not loop.last %},{% endif %}{% endfor %}' loose-group_replication_communication_stack=XCOM # Group replication behavior @@ -36,7 +36,7 @@ loose-group_replication_recovery_retry_count=10 loose-group_replication_recovery_reconnect_interval=60 # Crash-safe replication settings -relay-log=ps_pmm_{{ps_version}}_{{ item }}-relay-bin +relay-log=mysql_pmm_{{ms_version}}_{{ item }}-relay-bin relay_log_recovery=ON relay_log_purge=ON From 7f87fdb734d586f6e0921726118b709b038c625f Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 14 May 2025 09:09:27 +0200 Subject: [PATCH 272/340] PMM-7: New MySQL setup --- pmm_qa/mysql/data/group_replication/my.cnf.j2 | 1 - 1 file changed, 1 deletion(-) diff --git a/pmm_qa/mysql/data/group_replication/my.cnf.j2 b/pmm_qa/mysql/data/group_replication/my.cnf.j2 index d672e396..a2ec2020 100644 --- a/pmm_qa/mysql/data/group_replication/my.cnf.j2 +++ b/pmm_qa/mysql/data/group_replication/my.cnf.j2 @@ -3,7 +3,6 @@ server_id={{ server_id_start + item - 1 }} bind-address=0.0.0.0 port={{ mysql_listen_port }} -userstat=1 # General replication settings gtid_mode=ON From 7356cd848e3fbc12651373a850da4c3dc36d86df Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 14 May 2025 09:09:52 +0200 Subject: [PATCH 273/340] PMM-7: New MySQL setup --- pmm_qa/mysql/data/my.cnf.j2 | 1 - 1 file changed, 1 deletion(-) diff --git a/pmm_qa/mysql/data/my.cnf.j2 b/pmm_qa/mysql/data/my.cnf.j2 index fd4b27f2..6423dabf 100644 --- a/pmm_qa/mysql/data/my.cnf.j2 +++ b/pmm_qa/mysql/data/my.cnf.j2 @@ -1,2 +1 @@ [mysqld] -userstat=1 \ No newline at end of file From 3a07ebf6c65ac1f186a1f6d1b2d931950a8ad90b Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 14 May 2025 09:15:29 +0200 Subject: [PATCH 274/340] PMM-7: New MySQL setup --- pmm_qa/mysql/data/my-async-replication.cnf.j2 | 4 +- ...percona-server-async-replication-setup.yml | 129 ++++++++++++++++++ 2 files changed, 131 insertions(+), 2 deletions(-) create mode 100644 pmm_qa/mysql/tasks/percona-server-async-replication-setup.yml diff --git a/pmm_qa/mysql/data/my-async-replication.cnf.j2 b/pmm_qa/mysql/data/my-async-replication.cnf.j2 index 014d37f7..f9e38a9c 100644 --- a/pmm_qa/mysql/data/my-async-replication.cnf.j2 +++ b/pmm_qa/mysql/data/my-async-replication.cnf.j2 @@ -23,7 +23,7 @@ disabled_storage_engines="MyISAM,BLACKHOLE,FEDERATED,ARCHIVE,MEMORY" lower_case_table_names=2 # MySQL 8.4 compatibility settings -report_host=ps_pmm_{{ ps_version }}_{{ item }} +report_host=mysql_pmm_{{ ms_version }}_{{ item }} # Replica configuration - applies to all nodes except primary (they'll be able to become replicas) {% if item != 1 %} @@ -34,7 +34,7 @@ replica_preserve_commit_order=1 {% endif %} # Crash-safe replication settings -relay-log=ps_pmm_{{ ps_version }}_{{ item }}-relay-bin +relay-log=mysql_pmm_{{ ms_version }}_{{ item }}-relay-bin relay_log_recovery=ON relay_log_purge=ON diff --git a/pmm_qa/mysql/tasks/percona-server-async-replication-setup.yml b/pmm_qa/mysql/tasks/percona-server-async-replication-setup.yml new file mode 100644 index 00000000..85c045e1 --- /dev/null +++ b/pmm_qa/mysql/tasks/percona-server-async-replication-setup.yml @@ -0,0 +1,129 @@ +- name: Generate my.cnf for each node + template: + src: ./data/my-async-replication.cnf.j2 + dest: "{{ data_dir }}/node{{ item }}/my.cnf" + loop: "{{ range(1, nodes_count | int + 1) | list }}" + +- name: Create initialization script for each node + template: + src: ./data/init-async-replication.sql.j2 + dest: "{{ data_dir }}/node{{ item }}/init.sql" + loop: "{{ range(1, nodes_count | int + 1) | list }}" + +- name: Start Percona Server containers with async replication + community.docker.docker_container: + name: "mysql_pmm_{{ ms_version }}_{{ item }}" + image: "mysql:{{ ms_version }}" + restart_policy: always + state: started + networks: + - name: "{{ network_name }}" + env: + MYSQL_ROOT_PASSWORD: "{{ root_password }}" + ports: + - "{{ mysql_port + item - 1 }}:{{ mysql_listen_port }}" + volumes: + - "{{ data_dir }}/node{{ item }}/data:/var/lib/mysql" + - "{{ data_dir }}/node{{ item }}/my.cnf:/etc/mysql/my.cnf" + - "{{ data_dir }}/node{{ item }}/init.sql:/docker-entrypoint-initdb.d/init.sql" + loop: "{{ range(1, nodes_count | int + 1) | list }}" + +- name: Wait for MySQL to be available + wait_for: + host: localhost + port: "{{ mysql_port + item - 1 }}" + delay: 10 + timeout: 300 + loop: "{{ range(1, nodes_count | int + 1) | list }}" + +- name: Wait 5 seconds for percona server start to complete + pause: + seconds: 5 + +- name: Reset configuration for all nodes + community.docker.docker_container_exec: + container: "mysql_pmm_{{ ms_version }}_{{ item }}" + command: > + mysql -uroot -p{{ root_password }} -e " + RESET BINARY LOGS AND GTIDS; + RESET REPLICA ALL; + " + loop: "{{ range(1, nodes_count | int + 1) | list }}" + ignore_errors: yes + +- name: Get primary mysql_pmm_{{ ms_version }}_1 binary log status + community.docker.docker_container_exec: + container: "mysql_pmm_{{ ms_version }}_1" + command: > + mysql -uroot -p{{ root_password }} -e " + SHOW BINARY LOG STATUS\G + " + register: primary_status + changed_when: false + + +- name: Display binary log status for primary + debug: + msg: "{{ primary_status.stdout | split('\n') }}" + +- name: Configure replica servers (container2-containerN) + community.docker.docker_container_exec: + container: "mysql_pmm_{{ ms_version }}_{{ item }}" + command: > + mysql -uroot -p{{ root_password }} -e " + CHANGE REPLICATION SOURCE TO + SOURCE_HOST='mysql_pmm_{{ ms_version }}_1', + SOURCE_PORT={{ mysql_listen_port }}, + SOURCE_USER='{{ replication_user }}', + SOURCE_PASSWORD='{{ replication_password }}', + SOURCE_AUTO_POSITION=1, + SOURCE_PUBLIC_KEY_PATH='', + GET_SOURCE_PUBLIC_KEY=1; + START REPLICA; + " + loop: "{{ range(2, nodes_count | int + 1) | list }}" + +- name: Create and seed a test database on primary + community.docker.docker_container_exec: + container: "mysql_pmm_{{ ms_version }}_1" + command: > + mysql -uroot -p{{ root_password}} -e " + CREATE DATABASE testdb; + USE testdb; + CREATE TABLE testdb (id INT PRIMARY KEY, data VARCHAR(100)); + INSERT INTO testdb VALUES (1, 'Initial data from node mysql1');" + +- name: Check replication status on replica nodes + community.docker.docker_container_exec: + container: "mysql_pmm_{{ ms_version }}_{{ item }}" + command: mysql -uroot -p{{ root_password }} -e "SHOW REPLICA STATUS\G" + register: replication_status + loop: "{{ range(2, nodes_count | int + 1) | list }}" + changed_when: false + +- name: Display replication status for each replica + debug: + msg: "{{ replication_status.results[item - 2].stdout_lines }}" + loop: "{{ range(2, nodes_count | int + 1) | list }}" + +- name: Set verification instructions + set_fact: + verification_msg: | + MySQL Cluster setup complete with asynchronous replication! + + To verify replication is working: + 1. Connect to the primary (mysql_pmm_{{ ms_version }}_1): + docker exec -it mysql_pmm_{{ ms_version }}_1 mysql -uroot -p{{ root_password }} + + 2. Insert data in the test database: + USE testdb; + INSERT INTO testdb VALUES (100, 'Test replication'); + + 3. Connect to replicas and verify data is replicated: + docker exec -it mysql_pmm_{{ ms_version }}_2 mysql -uroot -p{{ root_password }} + USE testdb; + SELECT * FROM testdb; + +- name: Display verification instructions + debug: + msg: "{{ verification_msg | split('\n') }}" From 939c21810803b272e096b5877ab1ae5c42e1f29d Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 14 May 2025 09:20:23 +0200 Subject: [PATCH 275/340] PMM-7: New MySQL setup --- pmm_qa/mysql/mysql_setup.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pmm_qa/mysql/mysql_setup.yml b/pmm_qa/mysql/mysql_setup.yml index 3c311eb9..24d29239 100644 --- a/pmm_qa/mysql/mysql_setup.yml +++ b/pmm_qa/mysql/mysql_setup.yml @@ -118,21 +118,21 @@ - name: Add service to pmm server community.docker.docker_container_exec: container: "mysql_pmm_{{ ms_version }}_{{ item }}" - command: pmm-admin add mysql --query-source={{ query_source }} --username=root --password={{ root_password }} --environment=ps-gr-dev --cluster=ps-gr-dev-cluster --replication-set=ps-gr-replication mysql_pmm_{{ ms_version }}_{{ item }}{{ random_service_name_value }} --debug 127.0.0.1:3306 + command: pmm-admin add mysql --query-source={{ query_source }} --username=root --password={{ root_password }} --environment=ms-gr-dev --cluster=ms-gr-dev-cluster --replication-set=ms-gr-replication mysql_pmm_{{ ms_version }}_{{ item }}{{ random_service_name_value }} --debug 127.0.0.1:3306 loop: "{{ range(1, nodes_count | int + 1) | list }}" when: setup_type == "gr" - name: Add service to pmm server community.docker.docker_container_exec: container: "mysql_pmm_{{ ms_version }}_{{ item }}" - command: pmm-admin add mysql --query-source={{ query_source }} --username=root --password={{ root_password }} --environment=ps-replication-dev --cluster=ps-replication-dev-cluster --replication-set=ps-async-replication mysql_pmm_{{ ms_version }}_{{ item }}{{ random_service_name_value }} --debug 127.0.0.1:3306 + command: pmm-admin add mysql --query-source={{ query_source }} --username=root --password={{ root_password }} --environment=ms-replication-dev --cluster=ms-replication-dev-cluster --replication-set=ms-async-replication mysql_pmm_{{ ms_version }}_{{ item }}{{ random_service_name_value }} --debug 127.0.0.1:3306 loop: "{{ range(1, nodes_count | int + 1) | list }}" when: setup_type == "replication" - name: Add service to pmm server community.docker.docker_container_exec: container: "mysql_pmm_{{ ms_version }}_{{ item }}" - command: pmm-admin add mysql --query-source={{ query_source }} --username=root --password={{ root_password }} --environment=ps-dev mysql_pmm_{{ ms_version }}_{{ item }}{{ random_service_name_value }} --debug 127.0.0.1:3306 + command: pmm-admin add mysql --query-source={{ query_source }} --username=root --password={{ root_password }} --environment=ms-dev mysql_pmm_{{ ms_version }}_{{ item }}{{ random_service_name_value }} --debug 127.0.0.1:3306 loop: "{{ range(1, nodes_count | int + 1) | list }}" when: setup_type != "gr" and setup_type != "replication" From 3413d851c5d1a09caafadb40a8587c867175f2fb Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 14 May 2025 10:32:29 +0200 Subject: [PATCH 276/340] PMM-7: New MySQL setup --- pmm_qa/mysql/data/my-async-replication.cnf.j2 | 1 - 1 file changed, 1 deletion(-) diff --git a/pmm_qa/mysql/data/my-async-replication.cnf.j2 b/pmm_qa/mysql/data/my-async-replication.cnf.j2 index f9e38a9c..091b7f75 100644 --- a/pmm_qa/mysql/data/my-async-replication.cnf.j2 +++ b/pmm_qa/mysql/data/my-async-replication.cnf.j2 @@ -3,7 +3,6 @@ server_id={{ item }} bind-address=0.0.0.0 port={{ mysql_listen_port }} -userstat=1 # Authentication settings for caching_sha2_password caching_sha2_password_auto_generate_rsa_keys=ON From ccef85ea933bb38c9e7755ea823a97d2e40d3080 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 3 Jun 2025 14:38:56 +0200 Subject: [PATCH 277/340] PMM-7: PDPGSQL --- .../percona-distribution-postgres-setup.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml index e14ff40b..2b0edb47 100644 --- a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml @@ -81,6 +81,7 @@ port: "{{ pdpgsql_port }}" delay: 10 timeout: 300 + - name: Create replication user community.docker.docker_container_exec: container: "pdpgsql_pmm_{{ pg_version }}_1" @@ -112,6 +113,8 @@ - "{{ data_dir }}/node{{ item }}/data:/var/lib/postgresql/data" - "./data/postgres-replica.conf:/etc/postgresql/postgresql.conf:ro" - "{{ data_dir }}/node{{ item }}/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro" + ports: + - "{{ pdpgsql_port + item - 1}}:5432" loop: "{{ range(2, nodes_count | int + 1) | list }}" - name: Wipe replica data directory before basebackup @@ -186,7 +189,7 @@ - name: Wait 10 seconds for server to restart pause: - seconds: 10 + seconds: 30 - name: Install and add pmm client. include_tasks: ../tasks/install_pmm_client.yml From dd725e93fdcf2ae34cc677dac8e1919ecf560efd Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 3 Jun 2025 14:50:40 +0200 Subject: [PATCH 278/340] PMM-7: PDPGSQL --- .../percona-distribution-postgres-setup.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml index 2b0edb47..aef62a64 100644 --- a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml @@ -131,6 +131,14 @@ command: > bash -c " echo \" + DO \$\$ + BEGIN + IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = 'pmm') THEN + CREATE ROLE pmm LOGIN PASSWORD 'securepassword'; + END IF; + END + \$\$; + CREATE DATABASE pgbench; \\c pgbench GRANT CONNECT ON DATABASE pgbench TO pmm; From d8de015c64dcbdbbd656b175a306f9cf43e894e8 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 3 Jun 2025 14:54:32 +0200 Subject: [PATCH 279/340] PMM-7: PDPGSQL --- .../percona-distribution-postgres-setup.yml | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml index aef62a64..f1951690 100644 --- a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml @@ -134,7 +134,7 @@ DO \$\$ BEGIN IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = 'pmm') THEN - CREATE ROLE pmm LOGIN PASSWORD 'securepassword'; + CREATE ROLE pmm LOGIN PASSWORD 'pmm'; END IF; END \$\$; @@ -156,9 +156,15 @@ user: postgres command: > bash -c ' - PGPASSWORD="{{ root_password }}" psql -h /tmp -U postgres -d {{ db_name | default("postgres") }} -c " - CREATE USER pmm WITH SUPERUSER ENCRYPTED PASSWORD '\''pmm'\''; - GRANT pg_monitor TO pmm; + PGPASSWORD="{{ root_password }}" psql -h localhost -U postgres -d {{ db_name | default("postgres") }} -c " + DO \$\$ + BEGIN + IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = '\''pmm'\'') THEN + CREATE USER pmm WITH SUPERUSER ENCRYPTED PASSWORD '\''pmm'\''; + GRANT pg_monitor TO pmm; + END IF; + END + \$\$; " ' loop: "{{ range(1, nodes_count | int + 1) | list }}" From a75de6d6f9069ea1853b4be85378047890a4774d Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 3 Jun 2025 14:58:01 +0200 Subject: [PATCH 280/340] PMM-7: PDPGSQL --- .../percona-distribution-postgres-setup.yml | 23 ++----------------- 1 file changed, 2 insertions(+), 21 deletions(-) diff --git a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml index f1951690..ef916a64 100644 --- a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml @@ -126,7 +126,7 @@ - name: Create custom database for pgbench community.docker.docker_container_exec: - container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" + container: "pdpgsql_pmm_{{ pg_version }}_1" user: postgres command: > bash -c " @@ -148,26 +148,7 @@ GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO pmm; \" | psql -U postgres -v ON_ERROR_STOP=1 " - loop: "{{ range(1, nodes_count | int + 1) | list }}" - - - name: Create Percona Distribution PostgreSQL user 'pmm' with password - community.docker.docker_container_exec: - container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" - user: postgres - command: > - bash -c ' - PGPASSWORD="{{ root_password }}" psql -h localhost -U postgres -d {{ db_name | default("postgres") }} -c " - DO \$\$ - BEGIN - IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = '\''pmm'\'') THEN - CREATE USER pmm WITH SUPERUSER ENCRYPTED PASSWORD '\''pmm'\''; - GRANT pg_monitor TO pmm; - END IF; - END - \$\$; - " - ' - loop: "{{ range(1, nodes_count | int + 1) | list }}" +# loop: "{{ range(1, nodes_count | int + 1) | list }}" - name: Run pg_basebackup from primary to replica community.docker.docker_container_exec: From c33535cf8b57f45def259b7871892ad3c6682659 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 3 Jun 2025 15:10:29 +0200 Subject: [PATCH 281/340] PMM-7: PDPGSQL --- .../percona-distribution-postgres-setup.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml index ef916a64..51cf430b 100644 --- a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml @@ -157,7 +157,7 @@ command: > bash -c " export PGPASSWORD='{{ replication_password }}' && \ - timeout 60s \ + timeout 180s \ pg_basebackup --pgdata=/var/lib/postgresql/data -R -v -P \ --host=pdpgsql_pmm_{{ pg_version }}_1 --port=5432 -U {{ replication_user }} " From 7037d8d65a614c7b459a3710adf8e9834d513c64 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 3 Jun 2025 15:18:38 +0200 Subject: [PATCH 282/340] PMM-7: PDPGSQL --- .../data/pg_hba_replica.conf.j2 | 2 ++ .../percona-distribution-postgres-setup.yml | 9 +++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) create mode 100644 pmm_qa/percona-distribution-postgresql/data/pg_hba_replica.conf.j2 diff --git a/pmm_qa/percona-distribution-postgresql/data/pg_hba_replica.conf.j2 b/pmm_qa/percona-distribution-postgresql/data/pg_hba_replica.conf.j2 new file mode 100644 index 00000000..1d9565c1 --- /dev/null +++ b/pmm_qa/percona-distribution-postgresql/data/pg_hba_replica.conf.j2 @@ -0,0 +1,2 @@ +# Allow replication connections from replicas +host replication {{ replication_user }} 0.0.0.0/0 md5 \ No newline at end of file diff --git a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml index 51cf430b..4755d1f9 100644 --- a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml @@ -46,7 +46,12 @@ template: src: ./data/pg_hba.conf.j2 dest: "{{ data_dir }}/node1/pg_hba.conf" - loop: "{{ range(1, nodes_count | int + 1) | list }}" + + - name: Generate pg_hba.conf for replication node + template: + src: ./data/pg_hba_replica.conf.j2 + dest: "{{ data_dir }}/node{{ item }}/pg_hba.conf" + loop: "{{ range(2, nodes_count | int + 1) | list }}" - name: Remove old Percona Distribution PostgreSQL primary container community.docker.docker_container: @@ -177,7 +182,7 @@ volumes: - "{{ data_dir }}/node{{ item }}/data:/var/lib/postgresql/data" - "./data/postgres-replica.conf:/etc/postgresql/postgresql-primary.conf:ro" -# - "{{ data_dir }}/node1/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro" + - "{{ data_dir }}/node{{ item }}/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro" ports: - "{{ pdpgsql_port + item - 1}}:5432" loop: "{{ range(2, nodes_count | int + 1) | list }}" From fdf64ef197742878ffcfa2cf8c21a07bd6e3b6e3 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 3 Jun 2025 15:22:44 +0200 Subject: [PATCH 283/340] PMM-7: PDPGSQL --- .../data/pg_hba_replica.conf.j2 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pmm_qa/percona-distribution-postgresql/data/pg_hba_replica.conf.j2 b/pmm_qa/percona-distribution-postgresql/data/pg_hba_replica.conf.j2 index 1d9565c1..3d944a4c 100644 --- a/pmm_qa/percona-distribution-postgresql/data/pg_hba_replica.conf.j2 +++ b/pmm_qa/percona-distribution-postgresql/data/pg_hba_replica.conf.j2 @@ -1,2 +1,3 @@ # Allow replication connections from replicas -host replication {{ replication_user }} 0.0.0.0/0 md5 \ No newline at end of file +host replication {{ replication_user }} 0.0.0.0/0 md5 +host postgres postgres 0.0.0.0/0 md5 From 219b79371eed6dbdcb9f2e218c9138d1e2d20ba1 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 3 Jun 2025 15:23:35 +0200 Subject: [PATCH 284/340] PMM-7: PDPGSQL --- pmm_qa/mysql/__init__.py | 0 .../mysql/data/group_replication/init.sql.j2 | 15 -- pmm_qa/mysql/data/group_replication/my.cnf.j2 | 44 ---- .../mysql/data/init-async-replication.sql.j2 | 13 - pmm_qa/mysql/data/my-async-replication.cnf.j2 | 42 ---- pmm_qa/mysql/data/my.cnf.j2 | 1 - pmm_qa/mysql/mysql_setup.yml | 236 ------------------ pmm_qa/mysql/setup_mysql.py | 16 -- .../tasks/mysql-group-replication-setup.yml | 127 ---------- pmm_qa/mysql/tasks/mysql-setup-single.yml | 31 --- ...percona-server-async-replication-setup.yml | 129 ---------- 11 files changed, 654 deletions(-) delete mode 100644 pmm_qa/mysql/__init__.py delete mode 100644 pmm_qa/mysql/data/group_replication/init.sql.j2 delete mode 100644 pmm_qa/mysql/data/group_replication/my.cnf.j2 delete mode 100644 pmm_qa/mysql/data/init-async-replication.sql.j2 delete mode 100644 pmm_qa/mysql/data/my-async-replication.cnf.j2 delete mode 100644 pmm_qa/mysql/data/my.cnf.j2 delete mode 100644 pmm_qa/mysql/mysql_setup.yml delete mode 100644 pmm_qa/mysql/setup_mysql.py delete mode 100644 pmm_qa/mysql/tasks/mysql-group-replication-setup.yml delete mode 100644 pmm_qa/mysql/tasks/mysql-setup-single.yml delete mode 100644 pmm_qa/mysql/tasks/percona-server-async-replication-setup.yml diff --git a/pmm_qa/mysql/__init__.py b/pmm_qa/mysql/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/pmm_qa/mysql/data/group_replication/init.sql.j2 b/pmm_qa/mysql/data/group_replication/init.sql.j2 deleted file mode 100644 index 19185831..00000000 --- a/pmm_qa/mysql/data/group_replication/init.sql.j2 +++ /dev/null @@ -1,15 +0,0 @@ --- Create replication user and grant necessary privileges -SET SQL_LOG_BIN=0; -CREATE USER '{{ replication_user }}'@'%' IDENTIFIED BY '{{ replication_password }}'; -GRANT REPLICATION SLAVE ON *.* TO '{{ replication_user }}'@'%'; -GRANT CONNECTION_ADMIN ON *.* TO '{{ replication_user }}'@'%'; -GRANT BACKUP_ADMIN ON *.* TO '{{ replication_user }}'@'%'; -GRANT GROUP_REPLICATION_STREAM ON *.* TO '{{ replication_user }}'@'%'; --- GRANT SERVICE_CONNECTION_ADMIN ON *.* TO '{{ replication_user }}'@'%'; --- GRANT SYSTEM_VARIABLES_ADMIN ON *.* TO '{{ replication_user }}'@'%'; -FLUSH PRIVILEGES; -SET SQL_LOG_BIN=1; - --- Configure group replication recovery credentials -CHANGE REPLICATION SOURCE TO SOURCE_USER='{{ replication_user }}', SOURCE_PASSWORD='{{ replication_password }}' FOR CHANNEL 'group_replication_recovery'; - diff --git a/pmm_qa/mysql/data/group_replication/my.cnf.j2 b/pmm_qa/mysql/data/group_replication/my.cnf.j2 deleted file mode 100644 index a2ec2020..00000000 --- a/pmm_qa/mysql/data/group_replication/my.cnf.j2 +++ /dev/null @@ -1,44 +0,0 @@ -[mysqld] -# General server configuration -server_id={{ server_id_start + item - 1 }} -bind-address=0.0.0.0 -port={{ mysql_listen_port }} - -# General replication settings -gtid_mode=ON -enforce_gtid_consistency=ON -binlog_checksum=NONE -log_bin=binlog -log_replica_updates=ON -disabled_storage_engines="MyISAM,BLACKHOLE,FEDERATED,ARCHIVE,MEMORY" -lower_case_table_names=2 # MacOS-specific, but also good generally - -# MySQL 8.4 compatibility settings -report_host=mysql_pmm_{{ms_version}}_{{ item }} - -# Group Replication Settings -plugin_load_add='group_replication.so' -loose-group_replication_group_name='aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee' -loose-group_replication_local_address='mysql_pmm_{{ms_version}}_{{ item }}:{{ group_seeds_port }}' -loose-group_replication_group_seeds='{% for i in range(1, nodes_count | int + 1) %}mysql_pmm_{{ms_version}}_{{ i }}:{{ group_seeds_port }}{% if not loop.last %},{% endif %}{% endfor %}' -loose-group_replication_communication_stack=XCOM - -# Group replication behavior -loose-group_replication_start_on_boot=OFF -loose-group_replication_bootstrap_group=OFF -loose-group_replication_single_primary_mode=ON -loose-group_replication_enforce_update_everywhere_checks=OFF - -# Recovery settings -loose-group_replication_recovery_get_public_key=ON -loose-group_replication_recovery_retry_count=10 -loose-group_replication_recovery_reconnect_interval=60 - -# Crash-safe replication settings -relay-log=mysql_pmm_{{ms_version}}_{{ item }}-relay-bin -relay_log_recovery=ON -relay_log_purge=ON - -# Performance and connection settings -max_connections=1000 -innodb_buffer_pool_size=256M diff --git a/pmm_qa/mysql/data/init-async-replication.sql.j2 b/pmm_qa/mysql/data/init-async-replication.sql.j2 deleted file mode 100644 index a96ef137..00000000 --- a/pmm_qa/mysql/data/init-async-replication.sql.j2 +++ /dev/null @@ -1,13 +0,0 @@ --- Create replication user and grant necessary privileges -SET SQL_LOG_BIN=0; -CREATE USER '{{ replication_user }}'@'%' IDENTIFIED WITH 'caching_sha2_password' BY '{{ replication_password }}' REQUIRE NONE; -GRANT REPLICATION SLAVE ON *.* TO '{{ replication_user }}'@'%'; -GRANT CONNECTION_ADMIN ON *.* TO '{{ replication_user }}'@'%'; -GRANT BACKUP_ADMIN ON *.* TO '{{ replication_user }}'@'%'; -FLUSH PRIVILEGES; -SET SQL_LOG_BIN=1; - -{% if item == 1 %} --- Primary server: enable binary logging for replication -FLUSH BINARY LOGS; -{% endif %} diff --git a/pmm_qa/mysql/data/my-async-replication.cnf.j2 b/pmm_qa/mysql/data/my-async-replication.cnf.j2 deleted file mode 100644 index 091b7f75..00000000 --- a/pmm_qa/mysql/data/my-async-replication.cnf.j2 +++ /dev/null @@ -1,42 +0,0 @@ -[mysqld] -# General server configuration -server_id={{ item }} -bind-address=0.0.0.0 -port={{ mysql_listen_port }} - -# Authentication settings for caching_sha2_password -caching_sha2_password_auto_generate_rsa_keys=ON -# The following two parameters tell MySQL where to store the RSA key pair -caching_sha2_password_private_key_path=private_key.pem -caching_sha2_password_public_key_path=public_key.pem - -# Replication settings -gtid_mode=ON -enforce_gtid_consistency=ON -log_bin=binlog -log_replica_updates=ON -sync_binlog=1 -binlog_checksum=NONE -disabled_storage_engines="MyISAM,BLACKHOLE,FEDERATED,ARCHIVE,MEMORY" -# MacOS-specific, where table names are case-sensitive -lower_case_table_names=2 - -# MySQL 8.4 compatibility settings -report_host=mysql_pmm_{{ ms_version }}_{{ item }} - -# Replica configuration - applies to all nodes except primary (they'll be able to become replicas) -{% if item != 1 %} -# Replica specific settings -replica_parallel_workers=4 -replica_parallel_type=LOGICAL_CLOCK -replica_preserve_commit_order=1 -{% endif %} - -# Crash-safe replication settings -relay-log=mysql_pmm_{{ ms_version }}_{{ item }}-relay-bin -relay_log_recovery=ON -relay_log_purge=ON - -# Performance and connection settings -max_connections=1000 -innodb_buffer_pool_size=256M diff --git a/pmm_qa/mysql/data/my.cnf.j2 b/pmm_qa/mysql/data/my.cnf.j2 deleted file mode 100644 index 6423dabf..00000000 --- a/pmm_qa/mysql/data/my.cnf.j2 +++ /dev/null @@ -1 +0,0 @@ -[mysqld] diff --git a/pmm_qa/mysql/mysql_setup.yml b/pmm_qa/mysql/mysql_setup.yml deleted file mode 100644 index 24d29239..00000000 --- a/pmm_qa/mysql/mysql_setup.yml +++ /dev/null @@ -1,236 +0,0 @@ ---- -# MySQL 8.4 and higher single instance and also Cluster with Group Replication -- name: Setup MySQL 8.4 and higher. Cluster with Group Replication in Docker - hosts: localhost - connection: local - gather_facts: yes - vars: - ms_version: "{{ lookup('env', 'MS_VERSION') | default('8.4', true) }}" - cluster_name: "mysql_cluster" - replication_user: "repl_user" - replication_password: "GRgrO9301RuF" - root_password: "GRgrO9301RuF" - mysql_port: 4306 - mysql_listen_port: 3306 - group_seeds_port: 44061 - nodes_count: "{{ (lookup('env', 'NODES_COUNT') | default('1', true)) | int }}" - network_name: "pmm-qa" - data_dir: "{{ lookup('env', 'HOME') }}/mysql_cluster_data" - server_id_start: 1 - pmm_server_ip: "{{ lookup('vars', 'extra_pmm_server_ip', default=lookup('env','PMM_SERVER_IP') | default('127.0.0.1', true) ) }}" - client_version: "{{ lookup('vars', 'extra_client_version', default=lookup('env','CLIENT_VERSION') | default('3-dev-latest', true) ) }}" - admin_password: "{{ lookup('vars', 'extra_admin_password', default=lookup('env','ADMIN_PASSWORD') | default('admin', true) ) }}" - query_source: "{{ lookup('env', 'QUERY_SOURCE') | default('perfschema', true) }}" - metrics_mode: "{{ lookup('env', 'metrics_mode') }}" - setup_type: "{{ lookup('env', 'SETUP_TYPE') }}" - random_service_name_value: "" - - tasks: - - name: Mofidy the node count for group replication - set_fact: - nodes_count: 3 - when: nodes_count | int < 3 and setup_type == "gr" - - - name: Chance to correct nodes count for async replication - set_fact: - nodes_count: 2 - when: nodes_count | int < 2 and setup_type == "replication" - - - name: Create Docker network - community.docker.docker_network: - name: "{{ network_name }}" - state: present - - - name: Remove old data folders - shell: 'rm -fr {{ data_dir }}' - loop: "{{ range(1, nodes_count | int + 1) | list }}" - - - name: Create data directories - file: - path: "{{ data_dir }}/node{{ item }}/data" - state: directory - mode: '0755' - loop: "{{ range(1, nodes_count | int + 1) | list }}" - - - name: Remove old MySQL containers - community.docker.docker_container: - name: "mysql_pmm_{{ ms_version }}_{{ item }}" - image: "mysql:{{ ms_version }}" - restart_policy: always - state: absent - loop: "{{ range(1, nodes_count | int + 1) | list }}" - ignore_errors: yes - - - name: Recursively change ownership of a directory - shell: "sudo chown -R 1001:1001 {{ data_dir }}/node{{ item }}/data" - loop: "{{ range(1, nodes_count | int + 1) | list }}" - - - name: Setup MySQL group replication - include_tasks: ./tasks/mysql-group-replication-setup.yml - when: setup_type == "gr" - - - name: Setup MySQL with async replication - include_tasks: ./tasks/percona-server-async-replication-setup.yml - when: setup_type == "replication" - - - name: Setup MySQL - include_tasks: tasks/mysql-setup-single.yml - when: setup_type != "gr" and setup_type != "replication" - - - name: Create slowlog configuration for mysql nodes - community.docker.docker_container_exec: - container: "mysql_pmm_{{ ms_version }}_{{ item }}" - command: > - mysql -uroot -p{{ root_password }} -e " - SET GLOBAL slow_query_log='ON'; - SET GLOBAL long_query_time=0; - SET GLOBAL log_slow_admin_statements=ON; - SET GLOBAL log_slow_slave_statements=ON; - " - loop: "{{ range(1, nodes_count | int + 1) | list }}" - when: query_source == "slowlog" - - - name: Install and add pmm client. - include_tasks: ../tasks/install_pmm_client.yml - vars: - container_name: "mysql_pmm_{{ ms_version }}_{{ item }}" - loop: "{{ range(1, nodes_count | int + 1) | list }}" - - - name: Get already connected services to pmm server - community.docker.docker_container_exec: - container: "mysql_pmm_{{ ms_version }}_1" - command: > - sh -c 'curl --location --insecure -u"admin:{{ admin_password }}" -s --request GET "http://{{ pmm_server_ip }}:{{ '80' if pmm_server_ip is ansible.utils.ipv4 else '8080' }}/v1/management/services" | jq -r ".services[].service_name"' - register: pmm_server_services - - - name: Display already connected services to pmm server - debug: - msg: "{{ pmm_server_services.stdout | split('\n') }}" - - - name: Find out if service is already connected to pmm server - block: - - name: Loop through MySQL databases - set_fact: - random_service_name_value: "_{{ 9999 | random + 1 }}" - loop: "{{ range(1, nodes_count | int + 1) | list }}" - when: "('mysql_pmm_' ~ ms_version ~ '_' ~ item) in pmm_server_services.stdout" - - - name: Add service to pmm server - community.docker.docker_container_exec: - container: "mysql_pmm_{{ ms_version }}_{{ item }}" - command: pmm-admin add mysql --query-source={{ query_source }} --username=root --password={{ root_password }} --environment=ms-gr-dev --cluster=ms-gr-dev-cluster --replication-set=ms-gr-replication mysql_pmm_{{ ms_version }}_{{ item }}{{ random_service_name_value }} --debug 127.0.0.1:3306 - loop: "{{ range(1, nodes_count | int + 1) | list }}" - when: setup_type == "gr" - - - name: Add service to pmm server - community.docker.docker_container_exec: - container: "mysql_pmm_{{ ms_version }}_{{ item }}" - command: pmm-admin add mysql --query-source={{ query_source }} --username=root --password={{ root_password }} --environment=ms-replication-dev --cluster=ms-replication-dev-cluster --replication-set=ms-async-replication mysql_pmm_{{ ms_version }}_{{ item }}{{ random_service_name_value }} --debug 127.0.0.1:3306 - loop: "{{ range(1, nodes_count | int + 1) | list }}" - when: setup_type == "replication" - - - name: Add service to pmm server - community.docker.docker_container_exec: - container: "mysql_pmm_{{ ms_version }}_{{ item }}" - command: pmm-admin add mysql --query-source={{ query_source }} --username=root --password={{ root_password }} --environment=ms-dev mysql_pmm_{{ ms_version }}_{{ item }}{{ random_service_name_value }} --debug 127.0.0.1:3306 - loop: "{{ range(1, nodes_count | int + 1) | list }}" - when: setup_type != "gr" and setup_type != "replication" - - - name: Install sysbench inside of all MySQL nodes - community.docker.docker_container_exec: - container: "mysql_pmm_{{ ms_version }}_{{ item }}" - user: "root" - command: > - /bin/sh -c " - wget -O epel-release.rpm --progress=dot:giga https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && - rpm -i epel-release.rpm && - microdnf install -y sysbench - " - loop: "{{ range(1, nodes_count | int + 1) | list }}" - - - name: Prepare sysbench inside of all MySQL nodes - community.docker.docker_container_exec: - container: "mysql_pmm_{{ ms_version }}_{{ item }}" - command: > - mysql -uroot -p{{ root_password }} -e " - SET GLOBAL super_read_only = OFF; - SET GLOBAL read_only = OFF; - " - loop: "{{ range(1, nodes_count | int + 1) | list }}" - - - name: Prepare sysbench inside of all MySQL nodes - community.docker.docker_container_exec: - container: "mysql_pmm_{{ ms_version }}_{{ item }}" - command: > - mysql -uroot -p{{ root_password }} -e " - CREATE DATABASE sbtest; - CREATE USER 'sbtest'@'localhost' IDENTIFIED BY 'password'; - GRANT ALL PRIVILEGES ON *.* TO 'sbtest'@'localhost'; - CREATE USER 'sbtest'@'127.0.0.1' IDENTIFIED BY 'password'; - GRANT ALL PRIVILEGES ON *.* TO 'sbtest'@'127.0.0.1'; - FLUSH PRIVILEGES; - " - loop: "{{ range(1, nodes_count | int + 1) | list }}" - when: setup_type != "gr" and setup_type != "replication" - - - name: Prepare sysbench inside of all MySQL nodes - community.docker.docker_container_exec: - container: "mysql_pmm_{{ ms_version }}_1" - command: > - mysql -uroot -p{{ root_password }} -e " - CREATE DATABASE sbtest; - CREATE USER 'sbtest'@'localhost' IDENTIFIED BY 'password'; - GRANT ALL PRIVILEGES ON *.* TO 'sbtest'@'localhost'; - CREATE USER 'sbtest'@'127.0.0.1' IDENTIFIED BY 'password'; - GRANT ALL PRIVILEGES ON *.* TO 'sbtest'@'127.0.0.1'; - FLUSH PRIVILEGES; - " - when: setup_type == "gr" or setup_type == "replication" - - - name: Prepare data for sysbench inside of all MySQL nodes - community.docker.docker_container_exec: - container: "mysql_pmm_{{ ms_version }}_{{ item }}" - command: > - sysbench /usr/share/sysbench/oltp_read_write.lua - --mysql-host=127.0.0.1 - --mysql-port=3306 - --mysql-user=sbtest - --mysql-password=password - --mysql-db=sbtest - --tables=10 - --table-size=100000 - prepare - when: setup_type != "gr" and setup_type != "replication" - loop: "{{ range(1, nodes_count | int + 1) | list }}" - - - name: Prepare data for sysbench inside of first MySQL nodes - community.docker.docker_container_exec: - container: "mysql_pmm_{{ ms_version }}_1" - command: > - sysbench /usr/share/sysbench/oltp_read_write.lua - --mysql-host=127.0.0.1 - --mysql-port=3306 - --mysql-user=sbtest - --mysql-password=password - --mysql-db=sbtest - --tables=10 - --table-size=100000 - prepare - when: setup_type == "gr" or setup_type == "replication" - - - name: Run load for sysbench inside of all MySQL nodes - community.docker.docker_container_exec: - container: "mysql_pmm_{{ ms_version }}_{{ item }}" - command: > - sysbench /usr/share/sysbench/oltp_read_write.lua - --mysql-host=127.0.0.1 - --mysql-port=3306 - --mysql-user=sbtest - --mysql-password=password - --mysql-db=sbtest - --tables=10 - --table-size=100000 - --threads=16 - --time=60 - run - loop: "{{ range(1, nodes_count | int + 1) | list }}" diff --git a/pmm_qa/mysql/setup_mysql.py b/pmm_qa/mysql/setup_mysql.py deleted file mode 100644 index aaa9ab72..00000000 --- a/pmm_qa/mysql/setup_mysql.py +++ /dev/null @@ -1,16 +0,0 @@ -import os -from scripts.get_env_value import get_value -from scripts.run_ansible_playbook import run_ansible_playbook - -def setup_mysql_docker(db_type, container_name, db_config=None, args=None): - env_vars = { - 'SETUP_TYPE': get_value('SETUP_TYPE', db_type, args, db_config).lower(), - 'MS_VERSION': get_value('MS_VERSION', db_type, args, db_config), - 'PMM_SERVER_IP': args.pmm_server_ip or container_name or '127.0.0.1', - 'CLIENT_VERSION': get_value('CLIENT_VERSION', db_type, args, db_config), - 'QUERY_SOURCE': get_value('QUERY_SOURCE', db_type, args, db_config), - 'ADMIN_PASSWORD': os.getenv('ADMIN_PASSWORD') or args.pmm_server_password or 'admin', - 'NODES_COUNT': get_value('NODES_COUNT', db_type, args, db_config), - } - - run_ansible_playbook('mysql/mysql_setup.yml', env_vars, args) diff --git a/pmm_qa/mysql/tasks/mysql-group-replication-setup.yml b/pmm_qa/mysql/tasks/mysql-group-replication-setup.yml deleted file mode 100644 index e7d7cf92..00000000 --- a/pmm_qa/mysql/tasks/mysql-group-replication-setup.yml +++ /dev/null @@ -1,127 +0,0 @@ -- name: Generate my.cnf for each node - template: - src: ./data/group_replication/my.cnf.j2 - dest: "{{ data_dir }}/node{{ item }}/my.cnf" - loop: "{{ range(1, nodes_count | int + 1) | list }}" - -- name: Create initialization script for each node - template: - src: ./data/group_replication/init.sql.j2 - dest: "{{ data_dir }}/node{{ item }}/init.sql" - loop: "{{ range(1, nodes_count | int + 1) | list }}" - -- name: Start MySQL containers with group replication - community.docker.docker_container: - name: "mysql_pmm_{{ ms_version }}_{{ item }}" - image: "mysql:{{ ms_version }}" - restart_policy: always - state: started - networks: - - name: "{{ network_name }}" - env: - MYSQL_ROOT_PASSWORD: "{{ root_password }}" - ports: - - "{{ mysql_port + item - 1 }}:{{ mysql_listen_port }}" - - "{{ group_seeds_port + item - 1 }}:{{ group_seeds_port }}" - volumes: - - "{{ data_dir }}/node{{ item }}/data:/var/lib/mysql" - - "{{ data_dir }}/node{{ item }}/my.cnf:/etc/mysql/my.cnf" - - "{{ data_dir }}/node{{ item }}/init.sql:/docker-entrypoint-initdb.d/init.sql" - loop: "{{ range(1, nodes_count | int + 1) | list }}" - -- name: Wait for MySQL to be available - wait_for: - host: localhost - port: "{{ mysql_port + item - 1 }}" - delay: 10 - timeout: 300 - loop: "{{ range(1, nodes_count | int + 1) | list }}" - -- name: Reset configuration for all nodes - community.docker.docker_container_exec: - container: "mysql_pmm_{{ ms_version }}_{{ item }}" - command: > - mysql -uroot -p{{ root_password }} -e " - RESET BINARY LOGS AND GTIDS; - RESET REPLICA ALL; - SET GLOBAL gtid_purged=''; - " - loop: "{{ range(1, nodes_count | int + 1) | list }}" - -- name: Bootstrap first node in the cluster - community.docker.docker_container_exec: - container: "mysql_pmm_{{ ms_version }}_1" - command: > - mysql -uroot -p{{ root_password }} -e " - SET GLOBAL group_replication_bootstrap_group=ON; - START GROUP_REPLICATION; - SET GLOBAL group_replication_bootstrap_group=OFF;" - retries: 5 - delay: 10 - -- name: Wait 5 seconds for bootstrap to complete - pause: - seconds: 5 - -- name: Start group replication on other nodes - community.docker.docker_container_exec: - container: "mysql_pmm_{{ ms_version }}_{{ item }}" - command: mysql -uroot -p{{ root_password }} -e "START GROUP_REPLICATION;" - loop: "{{ range(2, nodes_count | int + 1) | list }}" - ignore_errors: yes - -- name: Wait 10 seconds for the other nodes to join - pause: - seconds: 10 - -- name: Create and seed a test database on primary - community.docker.docker_container_exec: - container: "mysql_pmm_{{ ms_version }}_1" - command: > - mysql -uroot -p{{ root_password}} -e " - CREATE DATABASE testdb; - USE testdb; - CREATE TABLE testdb (id INT PRIMARY KEY, data VARCHAR(100)); - INSERT INTO testdb VALUES (1, 'Initial data from node mysql1');" - -- name: Check replication status on first node - community.docker.docker_container_exec: - container: "mysql_pmm_{{ ms_version }}_1" - command: mysql -uroot -p{{ root_password }} -e "SELECT * FROM performance_schema.replication_group_members;" - register: replication_status - -- name: Display replication status - debug: - var: replication_status.stdout - -- name: Check replication group members count - community.docker.docker_container_exec: - container: "mysql_pmm_{{ ms_version }}_1" - command: mysql -uroot -p{{ root_password }} -e "SELECT COUNT(*) AS count FROM performance_schema.replication_group_members;" - register: member_count - -- name: Display member count - debug: - var: member_count.stdout - -- name: Set verification instructions - set_fact: - verification_msg: | - MySQL Cluster setup complete! - - To verify replication is working: - 1. Connect to the first node: - docker exec -it mysql_pmm_{{ ms_version }}_1 mysql -uroot -p{{ root_password }} - - 2. Insert data in the test database: - USE testdb; - INSERT INTO testdb VALUES (100, 'Test replication'); - - 3. Connect to other nodes and verify data is replicated: - docker exec -it mysql_pmm_{{ ms_version }}_2 mysql -uroot -p{{ root_password }} - USE testdb; - SELECT * FROM testdb; - -- name: Display verification instructions - debug: - msg: "{{ verification_msg | split('\n') }}" diff --git a/pmm_qa/mysql/tasks/mysql-setup-single.yml b/pmm_qa/mysql/tasks/mysql-setup-single.yml deleted file mode 100644 index a7e4a457..00000000 --- a/pmm_qa/mysql/tasks/mysql-setup-single.yml +++ /dev/null @@ -1,31 +0,0 @@ -- name: Generate my.cnf for each node - template: - src: ./data/my.cnf.j2 - dest: "{{ data_dir }}/node{{ item }}/my.cnf" - loop: "{{ range(1, nodes_count | int + 1) | list }}" - -- name: Start MySQL containers - community.docker.docker_container: - name: "mysql_pmm_{{ ms_version }}_{{ item }}" - image: "mysql:{{ ms_version }}" - restart_policy: always - state: started - networks: - - name: "{{ network_name }}" - env: - MYSQL_ROOT_PASSWORD: "{{ root_password }}" - ports: - - "{{ mysql_port + item - 1 }}:{{ mysql_listen_port }}" - - "{{ group_seeds_port + item - 1 }}:{{ group_seeds_port }}" - volumes: - - "{{ data_dir }}/node{{ item }}/my.cnf:/etc/mysql/my.cnf" - - "{{ data_dir }}/node{{ item }}/data:/var/lib/mysql" - loop: "{{ range(1, nodes_count | int + 1) | list }}" - -- name: Wait for MySQL to be available - wait_for: - host: localhost - port: "{{ mysql_port + item - 1 }}" - delay: 10 - timeout: 300 - loop: "{{ range(1, nodes_count | int + 1) | list }}" diff --git a/pmm_qa/mysql/tasks/percona-server-async-replication-setup.yml b/pmm_qa/mysql/tasks/percona-server-async-replication-setup.yml deleted file mode 100644 index 85c045e1..00000000 --- a/pmm_qa/mysql/tasks/percona-server-async-replication-setup.yml +++ /dev/null @@ -1,129 +0,0 @@ -- name: Generate my.cnf for each node - template: - src: ./data/my-async-replication.cnf.j2 - dest: "{{ data_dir }}/node{{ item }}/my.cnf" - loop: "{{ range(1, nodes_count | int + 1) | list }}" - -- name: Create initialization script for each node - template: - src: ./data/init-async-replication.sql.j2 - dest: "{{ data_dir }}/node{{ item }}/init.sql" - loop: "{{ range(1, nodes_count | int + 1) | list }}" - -- name: Start Percona Server containers with async replication - community.docker.docker_container: - name: "mysql_pmm_{{ ms_version }}_{{ item }}" - image: "mysql:{{ ms_version }}" - restart_policy: always - state: started - networks: - - name: "{{ network_name }}" - env: - MYSQL_ROOT_PASSWORD: "{{ root_password }}" - ports: - - "{{ mysql_port + item - 1 }}:{{ mysql_listen_port }}" - volumes: - - "{{ data_dir }}/node{{ item }}/data:/var/lib/mysql" - - "{{ data_dir }}/node{{ item }}/my.cnf:/etc/mysql/my.cnf" - - "{{ data_dir }}/node{{ item }}/init.sql:/docker-entrypoint-initdb.d/init.sql" - loop: "{{ range(1, nodes_count | int + 1) | list }}" - -- name: Wait for MySQL to be available - wait_for: - host: localhost - port: "{{ mysql_port + item - 1 }}" - delay: 10 - timeout: 300 - loop: "{{ range(1, nodes_count | int + 1) | list }}" - -- name: Wait 5 seconds for percona server start to complete - pause: - seconds: 5 - -- name: Reset configuration for all nodes - community.docker.docker_container_exec: - container: "mysql_pmm_{{ ms_version }}_{{ item }}" - command: > - mysql -uroot -p{{ root_password }} -e " - RESET BINARY LOGS AND GTIDS; - RESET REPLICA ALL; - " - loop: "{{ range(1, nodes_count | int + 1) | list }}" - ignore_errors: yes - -- name: Get primary mysql_pmm_{{ ms_version }}_1 binary log status - community.docker.docker_container_exec: - container: "mysql_pmm_{{ ms_version }}_1" - command: > - mysql -uroot -p{{ root_password }} -e " - SHOW BINARY LOG STATUS\G - " - register: primary_status - changed_when: false - - -- name: Display binary log status for primary - debug: - msg: "{{ primary_status.stdout | split('\n') }}" - -- name: Configure replica servers (container2-containerN) - community.docker.docker_container_exec: - container: "mysql_pmm_{{ ms_version }}_{{ item }}" - command: > - mysql -uroot -p{{ root_password }} -e " - CHANGE REPLICATION SOURCE TO - SOURCE_HOST='mysql_pmm_{{ ms_version }}_1', - SOURCE_PORT={{ mysql_listen_port }}, - SOURCE_USER='{{ replication_user }}', - SOURCE_PASSWORD='{{ replication_password }}', - SOURCE_AUTO_POSITION=1, - SOURCE_PUBLIC_KEY_PATH='', - GET_SOURCE_PUBLIC_KEY=1; - START REPLICA; - " - loop: "{{ range(2, nodes_count | int + 1) | list }}" - -- name: Create and seed a test database on primary - community.docker.docker_container_exec: - container: "mysql_pmm_{{ ms_version }}_1" - command: > - mysql -uroot -p{{ root_password}} -e " - CREATE DATABASE testdb; - USE testdb; - CREATE TABLE testdb (id INT PRIMARY KEY, data VARCHAR(100)); - INSERT INTO testdb VALUES (1, 'Initial data from node mysql1');" - -- name: Check replication status on replica nodes - community.docker.docker_container_exec: - container: "mysql_pmm_{{ ms_version }}_{{ item }}" - command: mysql -uroot -p{{ root_password }} -e "SHOW REPLICA STATUS\G" - register: replication_status - loop: "{{ range(2, nodes_count | int + 1) | list }}" - changed_when: false - -- name: Display replication status for each replica - debug: - msg: "{{ replication_status.results[item - 2].stdout_lines }}" - loop: "{{ range(2, nodes_count | int + 1) | list }}" - -- name: Set verification instructions - set_fact: - verification_msg: | - MySQL Cluster setup complete with asynchronous replication! - - To verify replication is working: - 1. Connect to the primary (mysql_pmm_{{ ms_version }}_1): - docker exec -it mysql_pmm_{{ ms_version }}_1 mysql -uroot -p{{ root_password }} - - 2. Insert data in the test database: - USE testdb; - INSERT INTO testdb VALUES (100, 'Test replication'); - - 3. Connect to replicas and verify data is replicated: - docker exec -it mysql_pmm_{{ ms_version }}_2 mysql -uroot -p{{ root_password }} - USE testdb; - SELECT * FROM testdb; - -- name: Display verification instructions - debug: - msg: "{{ verification_msg | split('\n') }}" From 4be5b515fc494f2035f5766a7b12060aba095234 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 3 Jun 2025 15:27:48 +0200 Subject: [PATCH 285/340] PMM-7: PDPGSQL --- pmm_qa/percona-distribution-postgresql/data/pg_hba.conf.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/percona-distribution-postgresql/data/pg_hba.conf.j2 b/pmm_qa/percona-distribution-postgresql/data/pg_hba.conf.j2 index 7d58f0c8..c52427c8 100644 --- a/pmm_qa/percona-distribution-postgresql/data/pg_hba.conf.j2 +++ b/pmm_qa/percona-distribution-postgresql/data/pg_hba.conf.j2 @@ -1,3 +1,3 @@ host replication {{ replication_user }} 0.0.0.0/0 md5 +local all postgres 0.0.0.0/0 trust host all all 0.0.0.0/0 md5 -local all postgres trust \ No newline at end of file From e6e806ab2078b9c4db03cc390ed6a87cfe48cba7 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 3 Jun 2025 15:31:11 +0200 Subject: [PATCH 286/340] PMM-7: PDPGSQL --- pmm_qa/pmm-framework.py | 38 +++++++++++++++++--------------------- 1 file changed, 17 insertions(+), 21 deletions(-) diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index b9a0cb39..495d299a 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -4,7 +4,6 @@ import sys import requests import re -from mysql.setup_mysql import setup_mysql_docker from scripts.get_env_value import get_value from scripts.database_options import database_options as database_configs from scripts.run_ansible_playbook import run_ansible_playbook @@ -122,28 +121,25 @@ def setup_mysql(db_type, db_version=None, db_config=None, args=None): setup_type = '' no_of_nodes = 2 - if ms_version_int >= 84: - setup_mysql_docker(db_type, container_name, db_config, args) - else: - # Define environment variables for playbook - env_vars = { - 'GROUP_REPLICATION': setup_type, - 'MS_NODES': no_of_nodes, - 'MS_VERSION': ms_version, - 'PMM_SERVER_IP': args.pmm_server_ip or container_name or '127.0.0.1', - 'MS_CONTAINER': 'mysql_pmm_' + str(ms_version), - 'CLIENT_VERSION': get_value('CLIENT_VERSION', db_type, args, db_config), - 'QUERY_SOURCE': get_value('QUERY_SOURCE', db_type, args, db_config), - 'MS_TARBALL': get_value('TARBALL', db_type, args, db_config), - 'ADMIN_PASSWORD': os.getenv('ADMIN_PASSWORD') or args.pmm_server_password or 'admin', - 'PMM_QA_GIT_BRANCH': os.getenv('PMM_QA_GIT_BRANCH') or 'v3' - } + # Define environment variables for playbook + env_vars = { + 'GROUP_REPLICATION': setup_type, + 'MS_NODES': no_of_nodes, + 'MS_VERSION': ms_version, + 'PMM_SERVER_IP': args.pmm_server_ip or container_name or '127.0.0.1', + 'MS_CONTAINER': 'mysql_pmm_' + str(ms_version), + 'CLIENT_VERSION': get_value('CLIENT_VERSION', db_type, args, db_config), + 'QUERY_SOURCE': get_value('QUERY_SOURCE', db_type, args, db_config), + 'MS_TARBALL': get_value('TARBALL', db_type, args, db_config), + 'ADMIN_PASSWORD': os.getenv('ADMIN_PASSWORD') or args.pmm_server_password or 'admin', + 'PMM_QA_GIT_BRANCH': os.getenv('PMM_QA_GIT_BRANCH') or 'v3' + } - # Ansible playbook filename - playbook_filename = 'ms_pmm_setup.yml' + # Ansible playbook filename + playbook_filename = 'ms_pmm_setup.yml' - # Call the function to run the Ansible playbook - run_ansible_playbook(playbook_filename, env_vars, args) + # Call the function to run the Ansible playbook + run_ansible_playbook(playbook_filename, env_vars, args) def setup_ssl_mysql(db_type, db_version=None, db_config=None, args=None): # Check if PMM server is running From 21b16f5dd22855b7ee60afe13b3def8232d70a5f Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 3 Jun 2025 15:32:39 +0200 Subject: [PATCH 287/340] PMM-7: PDPGSQL --- pmm_qa/percona-distribution-postgresql/data/pg_hba.conf.j2 | 2 +- .../percona-distribution-postgresql/data/pg_hba_replica.conf.j2 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pmm_qa/percona-distribution-postgresql/data/pg_hba.conf.j2 b/pmm_qa/percona-distribution-postgresql/data/pg_hba.conf.j2 index c52427c8..7d58f0c8 100644 --- a/pmm_qa/percona-distribution-postgresql/data/pg_hba.conf.j2 +++ b/pmm_qa/percona-distribution-postgresql/data/pg_hba.conf.j2 @@ -1,3 +1,3 @@ host replication {{ replication_user }} 0.0.0.0/0 md5 -local all postgres 0.0.0.0/0 trust host all all 0.0.0.0/0 md5 +local all postgres trust \ No newline at end of file diff --git a/pmm_qa/percona-distribution-postgresql/data/pg_hba_replica.conf.j2 b/pmm_qa/percona-distribution-postgresql/data/pg_hba_replica.conf.j2 index 3d944a4c..b28bd898 100644 --- a/pmm_qa/percona-distribution-postgresql/data/pg_hba_replica.conf.j2 +++ b/pmm_qa/percona-distribution-postgresql/data/pg_hba_replica.conf.j2 @@ -1,3 +1,3 @@ # Allow replication connections from replicas host replication {{ replication_user }} 0.0.0.0/0 md5 -host postgres postgres 0.0.0.0/0 md5 +local all postgres trust \ No newline at end of file From f840033118bed91933deea7436c82e1215decee5 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 3 Jun 2025 15:37:40 +0200 Subject: [PATCH 288/340] PMM-7: PDPGSQL --- .../data/pg_hba_replica.conf.j2 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pmm_qa/percona-distribution-postgresql/data/pg_hba_replica.conf.j2 b/pmm_qa/percona-distribution-postgresql/data/pg_hba_replica.conf.j2 index b28bd898..63e15207 100644 --- a/pmm_qa/percona-distribution-postgresql/data/pg_hba_replica.conf.j2 +++ b/pmm_qa/percona-distribution-postgresql/data/pg_hba_replica.conf.j2 @@ -1,3 +1,4 @@ # Allow replication connections from replicas host replication {{ replication_user }} 0.0.0.0/0 md5 -local all postgres trust \ No newline at end of file +local all postgres trust +host all all 0.0.0.0/0 md5 \ No newline at end of file From efc12a83d61e5b3b932f5402d00f6fbc8f564e53 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 3 Jun 2025 15:43:40 +0200 Subject: [PATCH 289/340] PMM-7: PDPGSQL --- .../percona-distribution-postgres-setup.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml index 4755d1f9..e774f5fb 100644 --- a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml @@ -129,7 +129,7 @@ command: rm -rf /var/lib/postgresql/data/* loop: "{{ range(2, nodes_count | int + 1) | list }}" - - name: Create custom database for pgbench + - name: Create pmm user on all services community.docker.docker_container_exec: container: "pdpgsql_pmm_{{ pg_version }}_1" user: postgres @@ -143,7 +143,17 @@ END IF; END \$\$; + " + loop: "{{ range(1, nodes_count | int + 1) | list }}" + + - name: Create custom database for pgbench + community.docker.docker_container_exec: + container: "pdpgsql_pmm_{{ pg_version }}_1" + user: postgres + command: > + bash -c " + echo \" CREATE DATABASE pgbench; \\c pgbench GRANT CONNECT ON DATABASE pgbench TO pmm; From cfea1b352ae30604c16a97e9157bd09e5bbf7678 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 3 Jun 2025 15:46:43 +0200 Subject: [PATCH 290/340] PMM-7: PDPGSQL --- .../percona-distribution-postgres-setup.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml index e774f5fb..5ca0591d 100644 --- a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml @@ -131,18 +131,17 @@ - name: Create pmm user on all services community.docker.docker_container_exec: - container: "pdpgsql_pmm_{{ pg_version }}_1" + container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" user: postgres command: > - bash -c " - echo \" - DO \$\$ + psql -U postgres -v ON_ERROR_STOP=1 -c " + DO $$ BEGIN IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = 'pmm') THEN CREATE ROLE pmm LOGIN PASSWORD 'pmm'; END IF; END - \$\$; + $$; " loop: "{{ range(1, nodes_count | int + 1) | list }}" From 1bb7208f1e3b81a195f274ed928188484b7bda7a Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 3 Jun 2025 15:48:14 +0200 Subject: [PATCH 291/340] PMM-7: PDPGSQL --- .../percona-distribution-postgres-setup.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml index 5ca0591d..835f9f31 100644 --- a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml @@ -134,7 +134,7 @@ container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" user: postgres command: > - psql -U postgres -v ON_ERROR_STOP=1 -c " + psql -h localhost -U postgres -v ON_ERROR_STOP=1 -c " DO $$ BEGIN IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = 'pmm') THEN From 43b81d97f429cf04b577f43c1307884292e45fa3 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 3 Jun 2025 15:52:24 +0200 Subject: [PATCH 292/340] PMM-7: PDPGSQL --- .../percona-distribution-postgres-setup.yml | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml index 835f9f31..7d836698 100644 --- a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml @@ -134,14 +134,16 @@ container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" user: postgres command: > - psql -h localhost -U postgres -v ON_ERROR_STOP=1 -c " - DO $$ - BEGIN - IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = 'pmm') THEN - CREATE ROLE pmm LOGIN PASSWORD 'pmm'; - END IF; - END - $$; + bash -c " + echo \" + DO $$ + BEGIN + IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = 'pmm') THEN + CREATE ROLE pmm LOGIN PASSWORD 'pmm'; + END IF; + END + $$; + \" | psql -U postgres -v ON_ERROR_STOP=1 " loop: "{{ range(1, nodes_count | int + 1) | list }}" From aa07775b3217353d8e28ecaab75b6ffa77e656ff Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 3 Jun 2025 15:56:12 +0200 Subject: [PATCH 293/340] PMM-7: PDPGSQL --- .../percona-distribution-postgres-setup.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml index 7d836698..0cc1721d 100644 --- a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml @@ -143,7 +143,7 @@ END IF; END $$; - \" | psql -U postgres -v ON_ERROR_STOP=1 + \" | psql -h localhost -p 5432 -U postgres -v ON_ERROR_STOP=1 " loop: "{{ range(1, nodes_count | int + 1) | list }}" From 5f6cbe067df68bdf9b5615942dd00b3b50617d96 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 3 Jun 2025 15:59:13 +0200 Subject: [PATCH 294/340] PMM-7: PDPGSQL --- pmm_qa/percona-distribution-postgresql/data/pg_hba.conf.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pmm_qa/percona-distribution-postgresql/data/pg_hba.conf.j2 b/pmm_qa/percona-distribution-postgresql/data/pg_hba.conf.j2 index 7d58f0c8..f8831219 100644 --- a/pmm_qa/percona-distribution-postgresql/data/pg_hba.conf.j2 +++ b/pmm_qa/percona-distribution-postgresql/data/pg_hba.conf.j2 @@ -1,3 +1,3 @@ host replication {{ replication_user }} 0.0.0.0/0 md5 -host all all 0.0.0.0/0 md5 -local all postgres trust \ No newline at end of file +host all all 0.0.0.0/0 md5 +local all postgres trust \ No newline at end of file From 95b83f0a76a705acd3841af7da72df7a0a30b3a1 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 3 Jun 2025 16:00:45 +0200 Subject: [PATCH 295/340] PMM-7: PDPGSQL --- pmm_qa/postgresql/postgresql-setup.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index 3cc52443..e4317dfa 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -108,12 +108,22 @@ - name: "{{ network_name }}" env: POSTGRES_PASSWORD: "{{ root_password }}" + ports: + - "{{ pgsql_port + item - 1 }}:5432" volumes: - "{{ data_dir }}/node{{ item }}/data:/var/lib/postgresql/data" - "./data/postgresql-replica.conf:/etc/postgresql/postgresql.conf:ro" - "{{ data_dir }}/node1/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro" loop: "{{ range(2, nodes_count | int + 1) | list }}" + - name: Wait for PgSQL to be available + wait_for: + host: localhost + port: "{{ pgsql_port }}" + delay: 10 + timeout: 300 + loop: "{{ range(2, nodes_count | int + 1) | list }}" + - name: Wipe replica data directory before basebackup community.docker.docker_container_exec: container: "pgsql_pmm_{{ pg_version }}_{{ item }}" From cf4b6fa015a1fe64d65da0e99505d509bef2888d Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 3 Jun 2025 16:02:40 +0200 Subject: [PATCH 296/340] PMM-7: PDPGSQL --- pmm_qa/postgresql/postgresql-setup.yml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index e4317dfa..3cc52443 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -108,22 +108,12 @@ - name: "{{ network_name }}" env: POSTGRES_PASSWORD: "{{ root_password }}" - ports: - - "{{ pgsql_port + item - 1 }}:5432" volumes: - "{{ data_dir }}/node{{ item }}/data:/var/lib/postgresql/data" - "./data/postgresql-replica.conf:/etc/postgresql/postgresql.conf:ro" - "{{ data_dir }}/node1/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro" loop: "{{ range(2, nodes_count | int + 1) | list }}" - - name: Wait for PgSQL to be available - wait_for: - host: localhost - port: "{{ pgsql_port }}" - delay: 10 - timeout: 300 - loop: "{{ range(2, nodes_count | int + 1) | list }}" - - name: Wipe replica data directory before basebackup community.docker.docker_container_exec: container: "pgsql_pmm_{{ pg_version }}_{{ item }}" From 54c4b3d013bba3cc26956c4f5a71cb7891996368 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 3 Jun 2025 16:06:21 +0200 Subject: [PATCH 297/340] PMM-7: PDPGSQL --- pmm_qa/postgresql/tasks/run_load_pgsql.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pmm_qa/postgresql/tasks/run_load_pgsql.yml b/pmm_qa/postgresql/tasks/run_load_pgsql.yml index d1b46305..500eea05 100644 --- a/pmm_qa/postgresql/tasks/run_load_pgsql.yml +++ b/pmm_qa/postgresql/tasks/run_load_pgsql.yml @@ -42,14 +42,14 @@ container: "{{ node_name }}" user: postgres command: > - pgbench -i -s {{ pgbench_scale }} pgbench + pgbench -i -s {{ pgbench_scale }} postgres - name: Run pgbench benchmark community.docker.docker_container_exec: container: "{{ node_name }}" user: postgres command: > - pgbench -c {{ pgbench_clients }} -T {{ pgbench_time }} -j 4 pgbench + pgbench -c {{ pgbench_clients }} -T {{ pgbench_time }} -j 4 postgres register: pgbench_result - name: Print pgbench results From 63c3f17a19fa363f5cee7ca0b0f3c8c3b22a9550 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 3 Jun 2025 16:09:20 +0200 Subject: [PATCH 298/340] PMM-7: PDPGSQL --- pmm_qa/postgresql/postgresql-setup.yml | 4 ---- pmm_qa/postgresql/tasks/run_load_pgsql.yml | 4 ++-- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/pmm_qa/postgresql/postgresql-setup.yml b/pmm_qa/postgresql/postgresql-setup.yml index 3cc52443..d451bd52 100644 --- a/pmm_qa/postgresql/postgresql-setup.yml +++ b/pmm_qa/postgresql/postgresql-setup.yml @@ -6,17 +6,13 @@ gather_facts: yes vars: pg_version: "{{ lookup('env', 'PGSQL_VERSION') | default('17', true) }}" -# cluster_name: "mysql_cluster" replication_user: "repl_user" replication_password: "GRgrO9301RuF" root_password: "GRgrO9301RuF" pgsql_port: 6432 -# mysql_listen_port: 3306 -# group_seeds_port: 34061 nodes_count: "{{ (lookup('env', 'NODES_COUNT') | default('2', true)) | int }}" network_name: "pmm-qa" data_dir: "{{ lookup('env', 'HOME') }}/pgsql_cluster_data" -# server_id_start: 1 pmm_server_ip: "{{ lookup('vars', 'extra_pmm_server_ip', default=lookup('env','PMM_SERVER_IP') | default('127.0.0.1', true) ) }}" client_version: "{{ lookup('vars', 'extra_client_version', default=lookup('env','CLIENT_VERSION') | default('3-dev-latest', true) ) }}" admin_password: "{{ lookup('vars', 'extra_admin_password', default=lookup('env','ADMIN_PASSWORD') | default('admin', true) ) }}" diff --git a/pmm_qa/postgresql/tasks/run_load_pgsql.yml b/pmm_qa/postgresql/tasks/run_load_pgsql.yml index 500eea05..d1b46305 100644 --- a/pmm_qa/postgresql/tasks/run_load_pgsql.yml +++ b/pmm_qa/postgresql/tasks/run_load_pgsql.yml @@ -42,14 +42,14 @@ container: "{{ node_name }}" user: postgres command: > - pgbench -i -s {{ pgbench_scale }} postgres + pgbench -i -s {{ pgbench_scale }} pgbench - name: Run pgbench benchmark community.docker.docker_container_exec: container: "{{ node_name }}" user: postgres command: > - pgbench -c {{ pgbench_clients }} -T {{ pgbench_time }} -j 4 postgres + pgbench -c {{ pgbench_clients }} -T {{ pgbench_time }} -j 4 pgbench register: pgbench_result - name: Print pgbench results From 48f07d45a0298f94495495e346d12f3db126da02 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 3 Jun 2025 16:10:39 +0200 Subject: [PATCH 299/340] PMM-7: PDPGSQL --- .../percona-distribution-postgres-setup.yml | 500 ++++++++++-------- 1 file changed, 292 insertions(+), 208 deletions(-) diff --git a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml index 0cc1721d..56bda9a1 100644 --- a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml @@ -6,17 +6,13 @@ gather_facts: yes vars: pg_version: "{{ lookup('env', 'PGSQL_VERSION') | default('17', true) }}" -# cluster_name: "mysql_cluster" replication_user: "repl_user" replication_password: "GRgrO9301RuF" root_password: "GRgrO9301RuF" pdpgsql_port: 5432 -# mysql_listen_port: 3306 -# group_seeds_port: 34061 nodes_count: "{{ (lookup('env', 'NODES_COUNT') | default('2', true)) | int }}" network_name: "pmm-qa" data_dir: "{{ lookup('env', 'HOME') }}/pgsql_cluster_data" -# server_id_start: 1 pmm_server_ip: "{{ lookup('vars', 'extra_pmm_server_ip', default=lookup('env','PMM_SERVER_IP') | default('127.0.0.1', true) ) }}" client_version: "{{ lookup('vars', 'extra_client_version', default=lookup('env','CLIENT_VERSION') | default('3-dev-latest', true) ) }}" admin_password: "{{ lookup('vars', 'extra_admin_password', default=lookup('env','ADMIN_PASSWORD') | default('admin', true) ) }}" @@ -44,27 +40,21 @@ - name: Generate pg_hba.conf for primary node template: - src: ./data/pg_hba.conf.j2 + src: data/pg_hba.conf.j2 dest: "{{ data_dir }}/node1/pg_hba.conf" - - name: Generate pg_hba.conf for replication node - template: - src: ./data/pg_hba_replica.conf.j2 - dest: "{{ data_dir }}/node{{ item }}/pg_hba.conf" - loop: "{{ range(2, nodes_count | int + 1) | list }}" - - - name: Remove old Percona Distribution PostgreSQL primary container + - name: Remove old PostgreSQL primary container community.docker.docker_container: - name: "pdpgsql_pmm_{{ pg_version }}_1" - image: "percona/percona-distribution-postgresql:{{ pg_version }}" + name: "pgsql_pmm_{{ pg_version }}_1" + image: "postgres:{{ pg_version }}-bookworm" restart_policy: always state: absent ignore_errors: yes - - name: Start Percona Distribution PostgreSQL primary container + - name: Start PostgreSQL primary container community.docker.docker_container: - name: "pdpgsql_pmm_{{ pg_version }}_1" - image: "percona/percona-distribution-postgresql:{{ pg_version }}" + name: "pgsql_pmm_{{ pg_version }}_1" + image: "postgres:{{ pg_version }}-bookworm" restart_policy: always state: started recreate: true @@ -78,226 +68,320 @@ - "{{ data_dir }}/node1/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro" command: -c config_file=/etc/postgresql/postgresql.conf ports: - - "{{ pdpgsql_port }}:5432" + - "{{ pgsql_port }}:5432" - - name: Wait for Percona Distribution PgSQL to be available + - name: Wait for PgSQL to be available wait_for: host: localhost - port: "{{ pdpgsql_port }}" + port: "{{ pgsql_port }}" delay: 10 timeout: 300 - name: Create replication user community.docker.docker_container_exec: - container: "pdpgsql_pmm_{{ pg_version }}_1" + container: "pgsql_pmm_{{ pg_version }}_1" user: postgres command: > psql -c " CREATE ROLE {{ replication_user }} WITH REPLICATION LOGIN ENCRYPTED PASSWORD '{{ replication_password }}'; " + - name: Stop and remove replica if exists community.docker.docker_container: - name: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" + name: "pgsql_pmm_{{ pg_version }}_{{ item }}" state: absent loop: "{{ range(2, nodes_count | int + 1) | list }}" ignore_errors: yes - - name: Start Percona Distribution PostgreSQL replica container + - name: Start PostgreSQL replica container community.docker.docker_container: - name: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" - image: "percona/percona-distribution-postgresql:{{ pg_version }}" + name: "pgsql_pmm_{{ pg_version }}_{{ item }}" + image: "postgres:{{ pg_version }}-bookworm" restart_policy: "no" state: started command: sleep infinity networks: - name: "{{ network_name }}" env: - POSTGRES_INITDB_SKIP: "yes" POSTGRES_PASSWORD: "{{ root_password }}" volumes: - "{{ data_dir }}/node{{ item }}/data:/var/lib/postgresql/data" - - "./data/postgres-replica.conf:/etc/postgresql/postgresql.conf:ro" - - "{{ data_dir }}/node{{ item }}/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro" - ports: - - "{{ pdpgsql_port + item - 1}}:5432" - loop: "{{ range(2, nodes_count | int + 1) | list }}" - - - name: Wipe replica data directory before basebackup - community.docker.docker_container_exec: - container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" - user: root - command: rm -rf /var/lib/postgresql/data/* + - "./data/postgresql-replica.conf:/etc/postgresql/postgresql.conf:ro" + - "{{ data_dir }}/node1/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro" loop: "{{ range(2, nodes_count | int + 1) | list }}" - - name: Create pmm user on all services - community.docker.docker_container_exec: - container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" - user: postgres - command: > - bash -c " - echo \" - DO $$ - BEGIN - IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = 'pmm') THEN - CREATE ROLE pmm LOGIN PASSWORD 'pmm'; - END IF; - END - $$; - \" | psql -h localhost -p 5432 -U postgres -v ON_ERROR_STOP=1 - " - loop: "{{ range(1, nodes_count | int + 1) | list }}" - - - - name: Create custom database for pgbench - community.docker.docker_container_exec: - container: "pdpgsql_pmm_{{ pg_version }}_1" - user: postgres - command: > - bash -c " - echo \" - CREATE DATABASE pgbench; - \\c pgbench - GRANT CONNECT ON DATABASE pgbench TO pmm; - GRANT USAGE ON SCHEMA public TO pmm; - GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public TO pmm; - ALTER DEFAULT PRIVILEGES IN SCHEMA public - GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO pmm; - \" | psql -U postgres -v ON_ERROR_STOP=1 - " +# - name: Create Docker network +# community.docker.docker_network: +# name: "{{ network_name }}" +# state: present +# ignore_errors: yes +# +# - name: Remove old data folders +# shell: 'rm -fr {{ data_dir }}' +# +# - name: Create data directories +# file: +# path: "{{ data_dir }}/node{{ item }}/data" +# state: directory +# mode: '0755' # loop: "{{ range(1, nodes_count | int + 1) | list }}" - - - name: Run pg_basebackup from primary to replica - community.docker.docker_container_exec: - container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" - user: root - command: > - bash -c " - export PGPASSWORD='{{ replication_password }}' && \ - timeout 180s \ - pg_basebackup --pgdata=/var/lib/postgresql/data -R -v -P \ - --host=pdpgsql_pmm_{{ pg_version }}_1 --port=5432 -U {{ replication_user }} - " - loop: "{{ range(2, nodes_count | int + 1) | list }}" - - - name: Restart Percona Distribution PostgreSQL container with custom command - community.docker.docker_container: - name: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" - image: "percona/percona-distribution-postgresql:{{ pg_version }}" - restart: true - state: started - command: -c config_file=/etc/postgresql/postgresql-primary.conf - networks: - - name: "{{ network_name }}" - env: - POSTGRES_PASSWORD: "{{ root_password }}" - volumes: - - "{{ data_dir }}/node{{ item }}/data:/var/lib/postgresql/data" - - "./data/postgres-replica.conf:/etc/postgresql/postgresql-primary.conf:ro" - - "{{ data_dir }}/node{{ item }}/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro" - ports: - - "{{ pdpgsql_port + item - 1}}:5432" - loop: "{{ range(2, nodes_count | int + 1) | list }}" - - - name: Wait 10 seconds for server to restart - pause: - seconds: 30 - - - name: Install and add pmm client. - include_tasks: ../tasks/install_pmm_client.yml - vars: - container_name: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" - loop: "{{ range(1, nodes_count | int + 1) | list }}" - - - name: Get already connected services to pmm server - community.docker.docker_container_exec: - container: "pdpgsql_pmm_{{ pg_version }}_1" - command: > - sh -c 'curl --location --insecure -u"admin:{{ admin_password }}" -s --request GET "http://{{ pmm_server_ip }}:{{ '80' if pmm_server_ip is ansible.utils.ipv4 else '8080' }}/v1/management/services" | jq -r ".services[].service_name"' - register: pmm_server_services - - - name: Display already connected services to pmm server - debug: - msg: "{{ pmm_server_services.stdout | split('\n') }}" - - - name: Find out if service is already connected to pmm server - block: - - name: Loop through percona servers - set_fact: - random_service_name_value: "_{{ 9999 | random + 1 }}" - loop: "{{ range(1, nodes_count | int + 1) | list }}" - when: "('pgsql_pmm_' ~ pg_version ~ '_' ~ item) in pmm_server_services.stdout" - - name: Set distro family (debian/rhel) - set_fact: - distro_family: >- - {{ - ( - 'debian' if 'debian' in container_os_info.stdout | lower else - 'rhel' if 'rhel' in container_os_info.stdout | lower or 'centos' in container_os_info.stdout | lower or 'fedora' in container_os_info.stdout | lower - else 'unknown' - ) | trim - }} - - - name: Install dependencies inside Debian-family container - community.docker.docker_container_exec: - container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" - command: > - sh -c ' - apt-get update && - apt-get install percona-pg-stat-monitor{{ pg_version }} - ' - user: "root" - when: distro_family == "debian" - loop: "{{ range(1, nodes_count | int + 1) | list }}" - - - name: Install dependencies inside RHEL-family container - community.docker.docker_container_exec: - container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" - command: > - sh -c ' - microdnf install percona-pg-stat-monitor{{ pg_version }} - ' - user: "root" - when: distro_family == "rhel" - loop: "{{ range(1, nodes_count | int + 1) | list }}" - - - name: Create pg_stat_statements extension - community.docker.docker_container_exec: - container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" - user: postgres - command: > - psql -U postgres -d 'postgres' -c " - CREATE EXTENSION IF NOT EXISTS pg_stat_monitor; - SELECT pg_stat_monitor_version(); - " - loop: "{{ range(1, nodes_count | int + 1) | list }}" - - - name: Add service to pmm server - community.docker.docker_container_exec: - container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" - command: pmm-admin add postgresql --username=pmm --password=pmm --query-source=pgstatmonitor pdpgsql_pmm_{{ pg_version }}_{{ item }}{{ random_service_name_value }} --debug 127.0.0.1:5432 - loop: "{{ range(1, nodes_count | int + 1) | list }}" - - - name: Run load on primary node. - include_tasks: ../postgresql/tasks/run_load_pgsql.yml - vars: - node_name: "pdpgsql_pmm_{{ pg_version }}_1" - pgbench_clients: 10 - pgbench_time: 120 - pgbench_scale: 1000 - - - name: Create pg_stat_statements extension - community.docker.docker_container_exec: - container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" - user: postgres - command: > - psql -U postgres -d 'pgbench' -c " - CREATE EXTENSION IF NOT EXISTS pg_stat_monitor; - SELECT pg_stat_monitor_version(); - " - loop: "{{ range(1, nodes_count | int + 1) | list }}" - - - name: Check replication status on primary - community.docker.docker_container_exec: - container: "pgsql_pmm_{{ pg_version }}_1" - user: postgres - command: psql -c "SELECT * FROM pg_stat_replication;" +# +# - name: Generate pg_hba.conf for primary node +# template: +# src: ./data/pg_hba.conf.j2 +# dest: "{{ data_dir }}/node1/pg_hba.conf" +# +# - name: Generate pg_hba.conf for replication node +# template: +# src: ./data/pg_hba_replica.conf.j2 +# dest: "{{ data_dir }}/node{{ item }}/pg_hba.conf" +# loop: "{{ range(2, nodes_count | int + 1) | list }}" +# +# - name: Remove old Percona Distribution PostgreSQL primary container +# community.docker.docker_container: +# name: "pdpgsql_pmm_{{ pg_version }}_1" +# image: "percona/percona-distribution-postgresql:{{ pg_version }}" +# restart_policy: always +# state: absent +# ignore_errors: yes +# +# - name: Start Percona Distribution PostgreSQL primary container +# community.docker.docker_container: +# name: "pdpgsql_pmm_{{ pg_version }}_1" +# image: "percona/percona-distribution-postgresql:{{ pg_version }}" +# restart_policy: always +# state: started +# recreate: true +# networks: +# - name: "{{ network_name }}" +# env: +# POSTGRES_PASSWORD: "{{ root_password }}" +# volumes: +# - "{{ data_dir }}/node1/data:/var/lib/postgresql/data" +# - "./data/postgresql-primary.conf:/etc/postgresql/postgresql.conf:ro" +# - "{{ data_dir }}/node1/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro" +# command: -c config_file=/etc/postgresql/postgresql.conf +# ports: +# - "{{ pdpgsql_port }}:5432" +# +# - name: Wait for Percona Distribution PgSQL to be available +# wait_for: +# host: localhost +# port: "{{ pdpgsql_port }}" +# delay: 10 +# timeout: 300 +# +# - name: Create replication user +# community.docker.docker_container_exec: +# container: "pdpgsql_pmm_{{ pg_version }}_1" +# user: postgres +# command: > +# psql -c " +# CREATE ROLE {{ replication_user }} WITH REPLICATION LOGIN ENCRYPTED PASSWORD '{{ replication_password }}'; +# " +# - name: Stop and remove replica if exists +# community.docker.docker_container: +# name: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" +# state: absent +# loop: "{{ range(2, nodes_count | int + 1) | list }}" +# ignore_errors: yes +# +# - name: Start Percona Distribution PostgreSQL replica container +# community.docker.docker_container: +# name: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" +# image: "percona/percona-distribution-postgresql:{{ pg_version }}" +# restart_policy: "no" +# state: started +# command: sleep infinity +# networks: +# - name: "{{ network_name }}" +# env: +# POSTGRES_INITDB_SKIP: "yes" +# POSTGRES_PASSWORD: "{{ root_password }}" +# volumes: +# - "{{ data_dir }}/node{{ item }}/data:/var/lib/postgresql/data" +# - "./data/postgres-replica.conf:/etc/postgresql/postgresql.conf:ro" +# - "{{ data_dir }}/node{{ item }}/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro" +# ports: +# - "{{ pdpgsql_port + item - 1}}:5432" +# loop: "{{ range(2, nodes_count | int + 1) | list }}" +# +# - name: Wipe replica data directory before basebackup +# community.docker.docker_container_exec: +# container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" +# user: root +# command: rm -rf /var/lib/postgresql/data/* +# loop: "{{ range(2, nodes_count | int + 1) | list }}" +# +# - name: Create pmm user on all services +# community.docker.docker_container_exec: +# container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" +# user: postgres +# command: > +# bash -c " +# echo \" +# DO $$ +# BEGIN +# IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = 'pmm') THEN +# CREATE ROLE pmm LOGIN PASSWORD 'pmm'; +# END IF; +# END +# $$; +# \" | psql -h localhost -p 5432 -U postgres -v ON_ERROR_STOP=1 +# " +# loop: "{{ range(1, nodes_count | int + 1) | list }}" +# +# +# - name: Create custom database for pgbench +# community.docker.docker_container_exec: +# container: "pdpgsql_pmm_{{ pg_version }}_1" +# user: postgres +# command: > +# bash -c " +# echo \" +# CREATE DATABASE pgbench; +# \\c pgbench +# GRANT CONNECT ON DATABASE pgbench TO pmm; +# GRANT USAGE ON SCHEMA public TO pmm; +# GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public TO pmm; +# ALTER DEFAULT PRIVILEGES IN SCHEMA public +# GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO pmm; +# \" | psql -U postgres -v ON_ERROR_STOP=1 +# " +## loop: "{{ range(1, nodes_count | int + 1) | list }}" +# +# - name: Run pg_basebackup from primary to replica +# community.docker.docker_container_exec: +# container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" +# user: root +# command: > +# bash -c " +# export PGPASSWORD='{{ replication_password }}' && \ +# timeout 180s \ +# pg_basebackup --pgdata=/var/lib/postgresql/data -R -v -P \ +# --host=pdpgsql_pmm_{{ pg_version }}_1 --port=5432 -U {{ replication_user }} +# " +# loop: "{{ range(2, nodes_count | int + 1) | list }}" +# +# - name: Restart Percona Distribution PostgreSQL container with custom command +# community.docker.docker_container: +# name: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" +# image: "percona/percona-distribution-postgresql:{{ pg_version }}" +# restart: true +# state: started +# command: -c config_file=/etc/postgresql/postgresql-primary.conf +# networks: +# - name: "{{ network_name }}" +# env: +# POSTGRES_PASSWORD: "{{ root_password }}" +# volumes: +# - "{{ data_dir }}/node{{ item }}/data:/var/lib/postgresql/data" +# - "./data/postgres-replica.conf:/etc/postgresql/postgresql-primary.conf:ro" +# - "{{ data_dir }}/node{{ item }}/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro" +# ports: +# - "{{ pdpgsql_port + item - 1}}:5432" +# loop: "{{ range(2, nodes_count | int + 1) | list }}" +# +# - name: Wait 10 seconds for server to restart +# pause: +# seconds: 30 +# +# - name: Install and add pmm client. +# include_tasks: ../tasks/install_pmm_client.yml +# vars: +# container_name: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" +# loop: "{{ range(1, nodes_count | int + 1) | list }}" +# +# - name: Get already connected services to pmm server +# community.docker.docker_container_exec: +# container: "pdpgsql_pmm_{{ pg_version }}_1" +# command: > +# sh -c 'curl --location --insecure -u"admin:{{ admin_password }}" -s --request GET "http://{{ pmm_server_ip }}:{{ '80' if pmm_server_ip is ansible.utils.ipv4 else '8080' }}/v1/management/services" | jq -r ".services[].service_name"' +# register: pmm_server_services +# +# - name: Display already connected services to pmm server +# debug: +# msg: "{{ pmm_server_services.stdout | split('\n') }}" +# +# - name: Find out if service is already connected to pmm server +# block: +# - name: Loop through percona servers +# set_fact: +# random_service_name_value: "_{{ 9999 | random + 1 }}" +# loop: "{{ range(1, nodes_count | int + 1) | list }}" +# when: "('pgsql_pmm_' ~ pg_version ~ '_' ~ item) in pmm_server_services.stdout" +# - name: Set distro family (debian/rhel) +# set_fact: +# distro_family: >- +# {{ +# ( +# 'debian' if 'debian' in container_os_info.stdout | lower else +# 'rhel' if 'rhel' in container_os_info.stdout | lower or 'centos' in container_os_info.stdout | lower or 'fedora' in container_os_info.stdout | lower +# else 'unknown' +# ) | trim +# }} +# +# - name: Install dependencies inside Debian-family container +# community.docker.docker_container_exec: +# container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" +# command: > +# sh -c ' +# apt-get update && +# apt-get install percona-pg-stat-monitor{{ pg_version }} +# ' +# user: "root" +# when: distro_family == "debian" +# loop: "{{ range(1, nodes_count | int + 1) | list }}" +# +# - name: Install dependencies inside RHEL-family container +# community.docker.docker_container_exec: +# container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" +# command: > +# sh -c ' +# microdnf install percona-pg-stat-monitor{{ pg_version }} +# ' +# user: "root" +# when: distro_family == "rhel" +# loop: "{{ range(1, nodes_count | int + 1) | list }}" +# +# - name: Create pg_stat_statements extension +# community.docker.docker_container_exec: +# container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" +# user: postgres +# command: > +# psql -U postgres -d 'postgres' -c " +# CREATE EXTENSION IF NOT EXISTS pg_stat_monitor; +# SELECT pg_stat_monitor_version(); +# " +# loop: "{{ range(1, nodes_count | int + 1) | list }}" +# +# - name: Add service to pmm server +# community.docker.docker_container_exec: +# container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" +# command: pmm-admin add postgresql --username=pmm --password=pmm --query-source=pgstatmonitor pdpgsql_pmm_{{ pg_version }}_{{ item }}{{ random_service_name_value }} --debug 127.0.0.1:5432 +# loop: "{{ range(1, nodes_count | int + 1) | list }}" +# +# - name: Run load on primary node. +# include_tasks: ../postgresql/tasks/run_load_pgsql.yml +# vars: +# node_name: "pdpgsql_pmm_{{ pg_version }}_1" +# pgbench_clients: 10 +# pgbench_time: 120 +# pgbench_scale: 1000 +# +# - name: Create pg_stat_statements extension +# community.docker.docker_container_exec: +# container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" +# user: postgres +# command: > +# psql -U postgres -d 'pgbench' -c " +# CREATE EXTENSION IF NOT EXISTS pg_stat_monitor; +# SELECT pg_stat_monitor_version(); +# " +# loop: "{{ range(1, nodes_count | int + 1) | list }}" +# +# - name: Check replication status on primary +# community.docker.docker_container_exec: +# container: "pgsql_pmm_{{ pg_version }}_1" +# user: postgres +# command: psql -c "SELECT * FROM pg_stat_replication;" From e8f774ffae6ce90f51ef7382e992a67adea54af6 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 3 Jun 2025 16:11:56 +0200 Subject: [PATCH 300/340] PMM-7: PDPGSQL --- .../percona-distribution-postgres-setup.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml index 56bda9a1..228564a8 100644 --- a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml @@ -45,7 +45,7 @@ - name: Remove old PostgreSQL primary container community.docker.docker_container: - name: "pgsql_pmm_{{ pg_version }}_1" + name: "pdpgsql_pmm_{{ pg_version }}_1" image: "postgres:{{ pg_version }}-bookworm" restart_policy: always state: absent @@ -53,8 +53,8 @@ - name: Start PostgreSQL primary container community.docker.docker_container: - name: "pgsql_pmm_{{ pg_version }}_1" - image: "postgres:{{ pg_version }}-bookworm" + name: "pdpgsql_pmm_{{ pg_version }}_1" + image: "percona/percona-distribution-postgresql:{{ pg_version }}" restart_policy: always state: started recreate: true @@ -79,7 +79,7 @@ - name: Create replication user community.docker.docker_container_exec: - container: "pgsql_pmm_{{ pg_version }}_1" + container: "pdpgsql_pmm_{{ pg_version }}_1" user: postgres command: > psql -c " @@ -88,15 +88,15 @@ - name: Stop and remove replica if exists community.docker.docker_container: - name: "pgsql_pmm_{{ pg_version }}_{{ item }}" + name: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" state: absent loop: "{{ range(2, nodes_count | int + 1) | list }}" ignore_errors: yes - name: Start PostgreSQL replica container community.docker.docker_container: - name: "pgsql_pmm_{{ pg_version }}_{{ item }}" - image: "postgres:{{ pg_version }}-bookworm" + name: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" + image: "percona/percona-distribution-postgresql:{{ pg_version }}" restart_policy: "no" state: started command: sleep infinity @@ -382,6 +382,6 @@ # # - name: Check replication status on primary # community.docker.docker_container_exec: -# container: "pgsql_pmm_{{ pg_version }}_1" +# container: "pdpgsql_pmm_{{ pg_version }}_1" # user: postgres # command: psql -c "SELECT * FROM pg_stat_replication;" From 734cd720606cedc88f4691e4c1dbc8dc02d2472f Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 3 Jun 2025 16:13:39 +0200 Subject: [PATCH 301/340] PMM-7: PDPGSQL --- .../percona-distribution-postgres-setup.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml index 228564a8..43f17101 100644 --- a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml @@ -20,6 +20,7 @@ metrics_mode: "auto" setup_type: "{{ lookup('env', 'SETUP_TYPE') }}" random_service_name_value: "" + docker_repo: "percona/percona-distribution-postgresql" tasks: - name: Create Docker network @@ -46,7 +47,7 @@ - name: Remove old PostgreSQL primary container community.docker.docker_container: name: "pdpgsql_pmm_{{ pg_version }}_1" - image: "postgres:{{ pg_version }}-bookworm" + image: "{{ docker_repo }}:{{ pg_version }}" restart_policy: always state: absent ignore_errors: yes @@ -54,7 +55,7 @@ - name: Start PostgreSQL primary container community.docker.docker_container: name: "pdpgsql_pmm_{{ pg_version }}_1" - image: "percona/percona-distribution-postgresql:{{ pg_version }}" + image: "{{ docker_repo }}:{{ pg_version }}" restart_policy: always state: started recreate: true @@ -96,7 +97,7 @@ - name: Start PostgreSQL replica container community.docker.docker_container: name: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" - image: "percona/percona-distribution-postgresql:{{ pg_version }}" + image: "{{ docker_repo }}:{{ pg_version }}" restart_policy: "no" state: started command: sleep infinity From 61e60f0d0f219724c582927766c1e4727b25ba40 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 3 Jun 2025 16:14:05 +0200 Subject: [PATCH 302/340] PMM-7: PDPGSQL --- .../percona-distribution-postgres-setup.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml index 43f17101..08aad612 100644 --- a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml @@ -69,12 +69,12 @@ - "{{ data_dir }}/node1/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro" command: -c config_file=/etc/postgresql/postgresql.conf ports: - - "{{ pgsql_port }}:5432" + - "{{ pdpgsql_port }}:5432" - name: Wait for PgSQL to be available wait_for: host: localhost - port: "{{ pgsql_port }}" + port: "{{ pdpgsql_port }}" delay: 10 timeout: 300 From 2efd110d1fbb2a893b4241cad27b51f33a90e511 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 3 Jun 2025 16:15:28 +0200 Subject: [PATCH 303/340] PMM-7: PDPGSQL --- .../percona-distribution-postgres-setup.yml | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml index 08aad612..32ca919c 100644 --- a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml @@ -111,6 +111,26 @@ - "{{ data_dir }}/node1/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro" loop: "{{ range(2, nodes_count | int + 1) | list }}" + + - name: Wipe replica data directory before basebackup + community.docker.docker_container_exec: + container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" + user: root + command: rm -rf /var/lib/postgresql/data/* + loop: "{{ range(2, nodes_count | int + 1) | list }}" + + - name: Create PostgreSQL user 'pmm' with password + community.docker.docker_container_exec: + container: "pdpgsql_pmm_{{ pg_version }}_1" + user: postgres + command: > + bash -c ' + psql -U postgres -d {{ db_name | default("postgres") }} -c " + CREATE USER pmm WITH PASSWORD '\''pmm'\''; + GRANT pg_monitor TO pmm; + " + ' + # - name: Create Docker network # community.docker.docker_network: # name: "{{ network_name }}" From ce84f66c2caa7d3077dd5896dc3cc8cdf7521bd7 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 3 Jun 2025 16:18:35 +0200 Subject: [PATCH 304/340] PMM-7: PDPGSQL --- .../percona-distribution-postgres-setup.yml | 72 +++++++++++++------ 1 file changed, 50 insertions(+), 22 deletions(-) diff --git a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml index 32ca919c..aa089ba6 100644 --- a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml @@ -111,7 +111,6 @@ - "{{ data_dir }}/node1/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro" loop: "{{ range(2, nodes_count | int + 1) | list }}" - - name: Wipe replica data directory before basebackup community.docker.docker_container_exec: container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" @@ -131,27 +130,56 @@ " ' -# - name: Create Docker network -# community.docker.docker_network: -# name: "{{ network_name }}" -# state: present -# ignore_errors: yes -# -# - name: Remove old data folders -# shell: 'rm -fr {{ data_dir }}' -# -# - name: Create data directories -# file: -# path: "{{ data_dir }}/node{{ item }}/data" -# state: directory -# mode: '0755' -# loop: "{{ range(1, nodes_count | int + 1) | list }}" -# -# - name: Generate pg_hba.conf for primary node -# template: -# src: ./data/pg_hba.conf.j2 -# dest: "{{ data_dir }}/node1/pg_hba.conf" -# + - name: Create custom database for pgbench + community.docker.docker_container_exec: + container: "pdpgsql_pmm_{{ pg_version }}_1" + user: postgres + command: > + bash -c " + echo \" + CREATE DATABASE pgbench; + \\c pgbench + GRANT CONNECT ON DATABASE pgbench TO pmm; + GRANT USAGE ON SCHEMA public TO pmm; + GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public TO pmm; + ALTER DEFAULT PRIVILEGES IN SCHEMA public + GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO pmm; + \" | psql -U postgres -v ON_ERROR_STOP=1 + " + - name: Run pg_basebackup from primary to replica + community.docker.docker_container_exec: + container: "pgsql_pmm_{{ pg_version }}_{{ item }}" + user: root + command: > + bash -c " + export PGPASSWORD='{{ replication_password }}' && \ + timeout 120s \ + pg_basebackup --pgdata=/var/lib/postgresql/data -R -v -Fp -Xs -P \ + --host=pgsql_pmm_{{ pg_version }}_1 --port=5432 -U {{ replication_user }} + " + loop: "{{ range(2, nodes_count | int + 1) | list }}" + + - name: Restart PostgreSQL container with custom command + community.docker.docker_container: + name: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" + image: "{{ docker_repo }}:{{ pg_version }}" + restart: true + state: started + command: -c config_file=/etc/postgresql/postgresql.conf + networks: + - name: "{{ network_name }}" + env: + POSTGRES_PASSWORD: "{{ root_password }}" + volumes: + - "{{ data_dir }}/node{{ item }}/data:/var/lib/postgresql/data" + - "./data/postgres-replica.conf:/etc/postgresql/postgresql.conf:ro" + - "{{ data_dir }}/node1/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro" + ports: + - "{{ pgsql_port + item - 1 }}:5432" + loop: "{{ range(2, nodes_count | int + 1) | list }}" + + + # - name: Generate pg_hba.conf for replication node # template: # src: ./data/pg_hba_replica.conf.j2 From 3e9c7131e073f1ac013531db30e7262e4c74c3a3 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 3 Jun 2025 16:20:36 +0200 Subject: [PATCH 305/340] PMM-7: PDPGSQL --- .../percona-distribution-postgres-setup.yml | 2 +- .../tasks/install_pg_stat-monitor.yml | 44 +++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) create mode 100644 pmm_qa/percona-distribution-postgresql/tasks/install_pg_stat-monitor.yml diff --git a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml index aa089ba6..6eb5db09 100644 --- a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml @@ -148,7 +148,7 @@ " - name: Run pg_basebackup from primary to replica community.docker.docker_container_exec: - container: "pgsql_pmm_{{ pg_version }}_{{ item }}" + container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" user: root command: > bash -c " diff --git a/pmm_qa/percona-distribution-postgresql/tasks/install_pg_stat-monitor.yml b/pmm_qa/percona-distribution-postgresql/tasks/install_pg_stat-monitor.yml new file mode 100644 index 00000000..b94af9d0 --- /dev/null +++ b/pmm_qa/percona-distribution-postgresql/tasks/install_pg_stat-monitor.yml @@ -0,0 +1,44 @@ +- name: Set distro family (debian/rhel) + set_fact: + distro_family: >- + {{ + ( + 'debian' if 'debian' in container_os_info.stdout | lower else + 'rhel' if 'rhel' in container_os_info.stdout | lower or 'centos' in container_os_info.stdout | lower or 'fedora' in container_os_info.stdout | lower + else 'unknown' + ) | trim + }} + +# - name: Install dependencies inside Debian-family container +# community.docker.docker_container_exec: +# container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" +# command: > +# sh -c ' +# apt-get update && +# apt-get install percona-pg-stat-monitor{{ pg_version }} +# ' +# user: "root" +# when: distro_family == "debian" +# loop: "{{ range(1, nodes_count | int + 1) | list }}" +# +# - name: Install dependencies inside RHEL-family container +# community.docker.docker_container_exec: +# container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" +# command: > +# sh -c ' +# microdnf install percona-pg-stat-monitor{{ pg_version }} +# ' +# user: "root" +# when: distro_family == "rhel" +# loop: "{{ range(1, nodes_count | int + 1) | list }}" +# +# - name: Create pg_stat_statements extension +# community.docker.docker_container_exec: +# container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" +# user: postgres +# command: > +# psql -U postgres -d 'postgres' -c " +# CREATE EXTENSION IF NOT EXISTS pg_stat_monitor; +# SELECT pg_stat_monitor_version(); +# " +# loop: "{{ range(1, nodes_count | int + 1) | list }}" From 2d67cd997cf828f67f46cf13b46971f2ae83df4f Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 3 Jun 2025 16:22:16 +0200 Subject: [PATCH 306/340] PMM-7: PDPGSQL --- .../percona-distribution-postgres-setup.yml | 2 +- .../tasks/install_pg_stat-monitor.yml | 32 +++++++++---------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml index 6eb5db09..295fb9d9 100644 --- a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml @@ -155,7 +155,7 @@ export PGPASSWORD='{{ replication_password }}' && \ timeout 120s \ pg_basebackup --pgdata=/var/lib/postgresql/data -R -v -Fp -Xs -P \ - --host=pgsql_pmm_{{ pg_version }}_1 --port=5432 -U {{ replication_user }} + --host=pdpgsql_pmm_{{ pg_version }}_1 --port=5432 -U {{ replication_user }} " loop: "{{ range(2, nodes_count | int + 1) | list }}" diff --git a/pmm_qa/percona-distribution-postgresql/tasks/install_pg_stat-monitor.yml b/pmm_qa/percona-distribution-postgresql/tasks/install_pg_stat-monitor.yml index b94af9d0..d88ab198 100644 --- a/pmm_qa/percona-distribution-postgresql/tasks/install_pg_stat-monitor.yml +++ b/pmm_qa/percona-distribution-postgresql/tasks/install_pg_stat-monitor.yml @@ -9,22 +9,22 @@ ) | trim }} -# - name: Install dependencies inside Debian-family container -# community.docker.docker_container_exec: -# container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" -# command: > -# sh -c ' -# apt-get update && -# apt-get install percona-pg-stat-monitor{{ pg_version }} -# ' -# user: "root" -# when: distro_family == "debian" -# loop: "{{ range(1, nodes_count | int + 1) | list }}" -# -# - name: Install dependencies inside RHEL-family container -# community.docker.docker_container_exec: -# container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" -# command: > +- name: Install dependencies inside Debian-family container + community.docker.docker_container_exec: + container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" + command: > + sh -c ' + apt-get update && + apt-get install percona-pg-stat-monitor{{ pg_version }} + ' + user: "root" + when: distro_family == "debian" + loop: "{{ range(1, nodes_count | int + 1) | list }}" + +- name: Install dependencies inside RHEL-family container + community.docker.docker_container_exec: + container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" + command: > # sh -c ' # microdnf install percona-pg-stat-monitor{{ pg_version }} # ' From 2be096453dca7902a1dadbdd8d1abaa0d83cf90d Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 3 Jun 2025 16:24:01 +0200 Subject: [PATCH 307/340] PMM-7: PDPGSQL --- .../percona-distribution-postgres-setup.yml | 3 +- .../tasks/install_pg_stat-monitor.yml | 34 +++++++++---------- 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml index 295fb9d9..c9db5809 100644 --- a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml @@ -178,7 +178,8 @@ - "{{ pgsql_port + item - 1 }}:5432" loop: "{{ range(2, nodes_count | int + 1) | list }}" - + - name: Install pg stat monitor. + include_tasks: ./tasks/install_pg_stat-monitor.yml # - name: Generate pg_hba.conf for replication node # template: diff --git a/pmm_qa/percona-distribution-postgresql/tasks/install_pg_stat-monitor.yml b/pmm_qa/percona-distribution-postgresql/tasks/install_pg_stat-monitor.yml index d88ab198..111df5dd 100644 --- a/pmm_qa/percona-distribution-postgresql/tasks/install_pg_stat-monitor.yml +++ b/pmm_qa/percona-distribution-postgresql/tasks/install_pg_stat-monitor.yml @@ -25,20 +25,20 @@ community.docker.docker_container_exec: container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" command: > -# sh -c ' -# microdnf install percona-pg-stat-monitor{{ pg_version }} -# ' -# user: "root" -# when: distro_family == "rhel" -# loop: "{{ range(1, nodes_count | int + 1) | list }}" -# -# - name: Create pg_stat_statements extension -# community.docker.docker_container_exec: -# container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" -# user: postgres -# command: > -# psql -U postgres -d 'postgres' -c " -# CREATE EXTENSION IF NOT EXISTS pg_stat_monitor; -# SELECT pg_stat_monitor_version(); -# " -# loop: "{{ range(1, nodes_count | int + 1) | list }}" + sh -c ' + microdnf install percona-pg-stat-monitor{{ pg_version }} + ' + user: "root" + when: distro_family == "rhel" + loop: "{{ range(1, nodes_count | int + 1) | list }}" + +- name: Create pg_stat_statements extension + community.docker.docker_container_exec: + container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" + user: postgres + command: > + psql -U postgres -d 'postgres' -c " + CREATE EXTENSION IF NOT EXISTS pg_stat_monitor; + SELECT pg_stat_monitor_version(); + " + loop: "{{ range(1, nodes_count | int + 1) | list }}" From d51e145b7b6bc1338edd818b55cd0bfabb222d91 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 3 Jun 2025 16:25:51 +0200 Subject: [PATCH 308/340] PMM-7: PDPGSQL --- .../percona-distribution-postgres-setup.yml | 261 +++--------------- 1 file changed, 31 insertions(+), 230 deletions(-) diff --git a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml index c9db5809..082f5937 100644 --- a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml @@ -175,241 +175,42 @@ - "./data/postgres-replica.conf:/etc/postgresql/postgresql.conf:ro" - "{{ data_dir }}/node1/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro" ports: - - "{{ pgsql_port + item - 1 }}:5432" + - "{{ pdpgsql_port + item - 1 }}:5432" loop: "{{ range(2, nodes_count | int + 1) | list }}" - name: Install pg stat monitor. include_tasks: ./tasks/install_pg_stat-monitor.yml -# - name: Generate pg_hba.conf for replication node -# template: -# src: ./data/pg_hba_replica.conf.j2 -# dest: "{{ data_dir }}/node{{ item }}/pg_hba.conf" -# loop: "{{ range(2, nodes_count | int + 1) | list }}" -# -# - name: Remove old Percona Distribution PostgreSQL primary container -# community.docker.docker_container: -# name: "pdpgsql_pmm_{{ pg_version }}_1" -# image: "percona/percona-distribution-postgresql:{{ pg_version }}" -# restart_policy: always -# state: absent -# ignore_errors: yes -# -# - name: Start Percona Distribution PostgreSQL primary container -# community.docker.docker_container: -# name: "pdpgsql_pmm_{{ pg_version }}_1" -# image: "percona/percona-distribution-postgresql:{{ pg_version }}" -# restart_policy: always -# state: started -# recreate: true -# networks: -# - name: "{{ network_name }}" -# env: -# POSTGRES_PASSWORD: "{{ root_password }}" -# volumes: -# - "{{ data_dir }}/node1/data:/var/lib/postgresql/data" -# - "./data/postgresql-primary.conf:/etc/postgresql/postgresql.conf:ro" -# - "{{ data_dir }}/node1/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro" -# command: -c config_file=/etc/postgresql/postgresql.conf -# ports: -# - "{{ pdpgsql_port }}:5432" -# -# - name: Wait for Percona Distribution PgSQL to be available -# wait_for: -# host: localhost -# port: "{{ pdpgsql_port }}" -# delay: 10 -# timeout: 300 -# -# - name: Create replication user -# community.docker.docker_container_exec: -# container: "pdpgsql_pmm_{{ pg_version }}_1" -# user: postgres -# command: > -# psql -c " -# CREATE ROLE {{ replication_user }} WITH REPLICATION LOGIN ENCRYPTED PASSWORD '{{ replication_password }}'; -# " -# - name: Stop and remove replica if exists -# community.docker.docker_container: -# name: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" -# state: absent -# loop: "{{ range(2, nodes_count | int + 1) | list }}" -# ignore_errors: yes -# -# - name: Start Percona Distribution PostgreSQL replica container -# community.docker.docker_container: -# name: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" -# image: "percona/percona-distribution-postgresql:{{ pg_version }}" -# restart_policy: "no" -# state: started -# command: sleep infinity -# networks: -# - name: "{{ network_name }}" -# env: -# POSTGRES_INITDB_SKIP: "yes" -# POSTGRES_PASSWORD: "{{ root_password }}" -# volumes: -# - "{{ data_dir }}/node{{ item }}/data:/var/lib/postgresql/data" -# - "./data/postgres-replica.conf:/etc/postgresql/postgresql.conf:ro" -# - "{{ data_dir }}/node{{ item }}/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro" -# ports: -# - "{{ pdpgsql_port + item - 1}}:5432" -# loop: "{{ range(2, nodes_count | int + 1) | list }}" -# -# - name: Wipe replica data directory before basebackup -# community.docker.docker_container_exec: -# container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" -# user: root -# command: rm -rf /var/lib/postgresql/data/* -# loop: "{{ range(2, nodes_count | int + 1) | list }}" -# -# - name: Create pmm user on all services -# community.docker.docker_container_exec: -# container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" -# user: postgres -# command: > -# bash -c " -# echo \" -# DO $$ -# BEGIN -# IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = 'pmm') THEN -# CREATE ROLE pmm LOGIN PASSWORD 'pmm'; -# END IF; -# END -# $$; -# \" | psql -h localhost -p 5432 -U postgres -v ON_ERROR_STOP=1 -# " -# loop: "{{ range(1, nodes_count | int + 1) | list }}" -# -# -# - name: Create custom database for pgbench -# community.docker.docker_container_exec: -# container: "pdpgsql_pmm_{{ pg_version }}_1" -# user: postgres -# command: > -# bash -c " -# echo \" -# CREATE DATABASE pgbench; -# \\c pgbench -# GRANT CONNECT ON DATABASE pgbench TO pmm; -# GRANT USAGE ON SCHEMA public TO pmm; -# GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public TO pmm; -# ALTER DEFAULT PRIVILEGES IN SCHEMA public -# GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO pmm; -# \" | psql -U postgres -v ON_ERROR_STOP=1 -# " -## loop: "{{ range(1, nodes_count | int + 1) | list }}" -# -# - name: Run pg_basebackup from primary to replica -# community.docker.docker_container_exec: -# container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" -# user: root -# command: > -# bash -c " -# export PGPASSWORD='{{ replication_password }}' && \ -# timeout 180s \ -# pg_basebackup --pgdata=/var/lib/postgresql/data -R -v -P \ -# --host=pdpgsql_pmm_{{ pg_version }}_1 --port=5432 -U {{ replication_user }} -# " -# loop: "{{ range(2, nodes_count | int + 1) | list }}" -# -# - name: Restart Percona Distribution PostgreSQL container with custom command -# community.docker.docker_container: -# name: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" -# image: "percona/percona-distribution-postgresql:{{ pg_version }}" -# restart: true -# state: started -# command: -c config_file=/etc/postgresql/postgresql-primary.conf -# networks: -# - name: "{{ network_name }}" -# env: -# POSTGRES_PASSWORD: "{{ root_password }}" -# volumes: -# - "{{ data_dir }}/node{{ item }}/data:/var/lib/postgresql/data" -# - "./data/postgres-replica.conf:/etc/postgresql/postgresql-primary.conf:ro" -# - "{{ data_dir }}/node{{ item }}/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro" -# ports: -# - "{{ pdpgsql_port + item - 1}}:5432" -# loop: "{{ range(2, nodes_count | int + 1) | list }}" -# -# - name: Wait 10 seconds for server to restart -# pause: -# seconds: 30 -# -# - name: Install and add pmm client. -# include_tasks: ../tasks/install_pmm_client.yml -# vars: -# container_name: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" -# loop: "{{ range(1, nodes_count | int + 1) | list }}" -# -# - name: Get already connected services to pmm server -# community.docker.docker_container_exec: -# container: "pdpgsql_pmm_{{ pg_version }}_1" -# command: > -# sh -c 'curl --location --insecure -u"admin:{{ admin_password }}" -s --request GET "http://{{ pmm_server_ip }}:{{ '80' if pmm_server_ip is ansible.utils.ipv4 else '8080' }}/v1/management/services" | jq -r ".services[].service_name"' -# register: pmm_server_services -# -# - name: Display already connected services to pmm server -# debug: -# msg: "{{ pmm_server_services.stdout | split('\n') }}" -# -# - name: Find out if service is already connected to pmm server -# block: -# - name: Loop through percona servers -# set_fact: -# random_service_name_value: "_{{ 9999 | random + 1 }}" -# loop: "{{ range(1, nodes_count | int + 1) | list }}" -# when: "('pgsql_pmm_' ~ pg_version ~ '_' ~ item) in pmm_server_services.stdout" -# - name: Set distro family (debian/rhel) -# set_fact: -# distro_family: >- -# {{ -# ( -# 'debian' if 'debian' in container_os_info.stdout | lower else -# 'rhel' if 'rhel' in container_os_info.stdout | lower or 'centos' in container_os_info.stdout | lower or 'fedora' in container_os_info.stdout | lower -# else 'unknown' -# ) | trim -# }} -# -# - name: Install dependencies inside Debian-family container -# community.docker.docker_container_exec: -# container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" -# command: > -# sh -c ' -# apt-get update && -# apt-get install percona-pg-stat-monitor{{ pg_version }} -# ' -# user: "root" -# when: distro_family == "debian" -# loop: "{{ range(1, nodes_count | int + 1) | list }}" -# -# - name: Install dependencies inside RHEL-family container -# community.docker.docker_container_exec: -# container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" -# command: > -# sh -c ' -# microdnf install percona-pg-stat-monitor{{ pg_version }} -# ' -# user: "root" -# when: distro_family == "rhel" -# loop: "{{ range(1, nodes_count | int + 1) | list }}" -# -# - name: Create pg_stat_statements extension -# community.docker.docker_container_exec: -# container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" -# user: postgres -# command: > -# psql -U postgres -d 'postgres' -c " -# CREATE EXTENSION IF NOT EXISTS pg_stat_monitor; -# SELECT pg_stat_monitor_version(); -# " -# loop: "{{ range(1, nodes_count | int + 1) | list }}" -# -# - name: Add service to pmm server -# community.docker.docker_container_exec: -# container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" -# command: pmm-admin add postgresql --username=pmm --password=pmm --query-source=pgstatmonitor pdpgsql_pmm_{{ pg_version }}_{{ item }}{{ random_service_name_value }} --debug 127.0.0.1:5432 -# loop: "{{ range(1, nodes_count | int + 1) | list }}" + - name: Install and add pmm client. + include_tasks: ../tasks/install_pmm_client.yml + vars: + container_name: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" + loop: "{{ range(1, nodes_count | int + 1) | list }}" + + - name: Get already connected services to pmm server + community.docker.docker_container_exec: + container: "pdpgsql_pmm_{{ pg_version }}_1" + command: > + sh -c 'curl --location --insecure -u"admin:{{ admin_password }}" -s --request GET "http://{{ pmm_server_ip }}:{{ '80' if pmm_server_ip is ansible.utils.ipv4 else '8080' }}/v1/management/services" | jq -r ".services[].service_name"' + register: pmm_server_services + + - name: Display already connected services to pmm server + debug: + msg: "{{ pmm_server_services.stdout | split('\n') }}" + + - name: Find out if service is already connected to pmm server + block: + - name: Loop through percona servers + set_fact: + random_service_name_value: "_{{ 9999 | random + 1 }}" + loop: "{{ range(1, nodes_count | int + 1) | list }}" + when: "('pdpgsql_pmm_' ~ pg_version ~ '_' ~ item) in pmm_server_services.stdout" + + - name: Add service to pmm server + community.docker.docker_container_exec: + container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" + command: pmm-admin add postgresql --username=pmm --password=pmm --query-source=pgstatmonitor pdpgsql_pmm_{{ pg_version }}_{{ item }}{{ random_service_name_value }} --debug 127.0.0.1:5432 + loop: "{{ range(1, nodes_count | int + 1) | list }}" # # - name: Run load on primary node. # include_tasks: ../postgresql/tasks/run_load_pgsql.yml From ddc33606834364b21c4796eac255e58a23414149 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 3 Jun 2025 16:26:48 +0200 Subject: [PATCH 309/340] PMM-7: PDPGSQL --- .../percona-distribution-postgres-setup.yml | 39 +++++++------------ 1 file changed, 14 insertions(+), 25 deletions(-) diff --git a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml index 082f5937..68a9279b 100644 --- a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml @@ -211,28 +211,17 @@ container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" command: pmm-admin add postgresql --username=pmm --password=pmm --query-source=pgstatmonitor pdpgsql_pmm_{{ pg_version }}_{{ item }}{{ random_service_name_value }} --debug 127.0.0.1:5432 loop: "{{ range(1, nodes_count | int + 1) | list }}" -# -# - name: Run load on primary node. -# include_tasks: ../postgresql/tasks/run_load_pgsql.yml -# vars: -# node_name: "pdpgsql_pmm_{{ pg_version }}_1" -# pgbench_clients: 10 -# pgbench_time: 120 -# pgbench_scale: 1000 -# -# - name: Create pg_stat_statements extension -# community.docker.docker_container_exec: -# container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" -# user: postgres -# command: > -# psql -U postgres -d 'pgbench' -c " -# CREATE EXTENSION IF NOT EXISTS pg_stat_monitor; -# SELECT pg_stat_monitor_version(); -# " -# loop: "{{ range(1, nodes_count | int + 1) | list }}" -# -# - name: Check replication status on primary -# community.docker.docker_container_exec: -# container: "pdpgsql_pmm_{{ pg_version }}_1" -# user: postgres -# command: psql -c "SELECT * FROM pg_stat_replication;" + + - name: Run load on primary node. + include_tasks: ../postgresql/tasks/run_load_pgsql.yml + vars: + node_name: "pdpgsql_pmm_{{ pg_version }}_1" + pgbench_clients: 10 + pgbench_time: 120 + pgbench_scale: 1000 + + - name: Check replication status on primary + community.docker.docker_container_exec: + container: "pgsql_pmm_{{ pg_version }}_1" + user: postgres + command: psql -c "SELECT * FROM pg_stat_replication;" From 8e55e2e6d3b99a985ce28d78fb1866343f7d6c65 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 3 Jun 2025 16:29:17 +0200 Subject: [PATCH 310/340] PMM-7: PDPGSQL --- .../tasks/install_pg_stat-monitor.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pmm_qa/percona-distribution-postgresql/tasks/install_pg_stat-monitor.yml b/pmm_qa/percona-distribution-postgresql/tasks/install_pg_stat-monitor.yml index 111df5dd..4284c409 100644 --- a/pmm_qa/percona-distribution-postgresql/tasks/install_pg_stat-monitor.yml +++ b/pmm_qa/percona-distribution-postgresql/tasks/install_pg_stat-monitor.yml @@ -17,7 +17,7 @@ apt-get update && apt-get install percona-pg-stat-monitor{{ pg_version }} ' - user: "root" + user: "root" when: distro_family == "debian" loop: "{{ range(1, nodes_count | int + 1) | list }}" @@ -28,7 +28,7 @@ sh -c ' microdnf install percona-pg-stat-monitor{{ pg_version }} ' - user: "root" + user: "root" when: distro_family == "rhel" loop: "{{ range(1, nodes_count | int + 1) | list }}" From 4ba95e52f057b7b5ba441b5ca4cf1873a6368334 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 3 Jun 2025 16:30:36 +0200 Subject: [PATCH 311/340] PMM-7: PDPGSQL --- pmm_qa/percona-distribution-postgresql/data/pg_hba.conf.j2 | 2 +- .../percona-distribution-postgresql/data/pg_hba_replica.conf.j2 | 2 +- .../percona-distribution-postgresql/data/postgres-replica.conf | 2 +- .../data/postgresql-primary.conf | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pmm_qa/percona-distribution-postgresql/data/pg_hba.conf.j2 b/pmm_qa/percona-distribution-postgresql/data/pg_hba.conf.j2 index f8831219..8488d8cc 100644 --- a/pmm_qa/percona-distribution-postgresql/data/pg_hba.conf.j2 +++ b/pmm_qa/percona-distribution-postgresql/data/pg_hba.conf.j2 @@ -1,3 +1,3 @@ host replication {{ replication_user }} 0.0.0.0/0 md5 host all all 0.0.0.0/0 md5 -local all postgres trust \ No newline at end of file +local all postgres trust diff --git a/pmm_qa/percona-distribution-postgresql/data/pg_hba_replica.conf.j2 b/pmm_qa/percona-distribution-postgresql/data/pg_hba_replica.conf.j2 index 63e15207..4a3a14e6 100644 --- a/pmm_qa/percona-distribution-postgresql/data/pg_hba_replica.conf.j2 +++ b/pmm_qa/percona-distribution-postgresql/data/pg_hba_replica.conf.j2 @@ -1,4 +1,4 @@ # Allow replication connections from replicas host replication {{ replication_user }} 0.0.0.0/0 md5 local all postgres trust -host all all 0.0.0.0/0 md5 \ No newline at end of file +host all all 0.0.0.0/0 md5 diff --git a/pmm_qa/percona-distribution-postgresql/data/postgres-replica.conf b/pmm_qa/percona-distribution-postgresql/data/postgres-replica.conf index ab63afb9..af6750d6 100644 --- a/pmm_qa/percona-distribution-postgresql/data/postgres-replica.conf +++ b/pmm_qa/percona-distribution-postgresql/data/postgres-replica.conf @@ -7,4 +7,4 @@ listen_addresses = '*' pg_stat_monitor.pgsm_enable_query_plan = 'yes' pg_stat_monitor.pgsm_query_max_len = 2048 pg_stat_monitor.pgsm_normalized_query=1 -pg_stat_monitor.pgsm_enable_query_plan=1 \ No newline at end of file +pg_stat_monitor.pgsm_enable_query_plan=1 diff --git a/pmm_qa/percona-distribution-postgresql/data/postgresql-primary.conf b/pmm_qa/percona-distribution-postgresql/data/postgresql-primary.conf index 9e8686b5..bcd4a3c2 100644 --- a/pmm_qa/percona-distribution-postgresql/data/postgresql-primary.conf +++ b/pmm_qa/percona-distribution-postgresql/data/postgresql-primary.conf @@ -9,4 +9,4 @@ track_activity_query_size=2048 track_io_timing=ON max_connections=1000 listen_addresses = '*' -pg_stat_monitor.pgsm_enable_query_plan = 'yes' \ No newline at end of file +pg_stat_monitor.pgsm_enable_query_plan = 'yes' From 9d261b14a6ea94ab867857acf2aded0b6d518f0a Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 3 Jun 2025 16:33:16 +0200 Subject: [PATCH 312/340] PMM-7: PDPGSQL --- .../tasks/install_pg_stat-monitor.yml | 6 ++++++ pmm_qa/percona_server/percona-server-setup.yml | 3 --- pmm_qa/pg_stat_monitor_setup.sh | 2 +- pmm_qa/pg_stat_statements_setup.sh | 2 +- 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/pmm_qa/percona-distribution-postgresql/tasks/install_pg_stat-monitor.yml b/pmm_qa/percona-distribution-postgresql/tasks/install_pg_stat-monitor.yml index 4284c409..3c8710b2 100644 --- a/pmm_qa/percona-distribution-postgresql/tasks/install_pg_stat-monitor.yml +++ b/pmm_qa/percona-distribution-postgresql/tasks/install_pg_stat-monitor.yml @@ -1,3 +1,9 @@ +- name: Detect OS inside the container + community.docker.docker_container_exec: + container: "pdpgsql_pmm_{{ pg_version }}_1" + command: cat /etc/os-release + register: container_os_info + - name: Set distro family (debian/rhel) set_fact: distro_family: >- diff --git a/pmm_qa/percona_server/percona-server-setup.yml b/pmm_qa/percona_server/percona-server-setup.yml index b88b431e..4c450f1d 100644 --- a/pmm_qa/percona_server/percona-server-setup.yml +++ b/pmm_qa/percona_server/percona-server-setup.yml @@ -199,7 +199,6 @@ --mysql-db=sbtest --tables=10 --table-size=100000 - --rate=1 prepare when: setup_type != "gr" and setup_type != "replication" loop: "{{ range(1, nodes_count | int + 1) | list }}" @@ -216,7 +215,6 @@ --mysql-db=sbtest --tables=10 --table-size=100000 - --rate=1 prepare when: setup_type == "gr" or setup_type == "replication" @@ -234,6 +232,5 @@ --table-size=100000 --threads=16 --time=60 - --rate=1 run loop: "{{ range(1, nodes_count | int + 1) | list }}" diff --git a/pmm_qa/pg_stat_monitor_setup.sh b/pmm_qa/pg_stat_monitor_setup.sh index 010f7838..cc6fb54e 100644 --- a/pmm_qa/pg_stat_monitor_setup.sh +++ b/pmm_qa/pg_stat_monitor_setup.sh @@ -95,7 +95,7 @@ make USE_PGXS=1 # Install built PGSM library into server make USE_PGXS=1 install -# Stop server and edit postgresql-primary.conf to load PGSM library with required configurations +# Stop server and edit postgresql.conf to load PGSM library with required configurations service postgresql stop echo "shared_preload_libraries = 'pg_stat_monitor'" >> /etc/postgresql/${pgsql_version}/main/postgresql.conf echo "track_activity_query_size=2048" >> /etc/postgresql/${pgsql_version}/main/postgresql.conf diff --git a/pmm_qa/pg_stat_statements_setup.sh b/pmm_qa/pg_stat_statements_setup.sh index c04f2493..550d8131 100644 --- a/pmm_qa/pg_stat_statements_setup.sh +++ b/pmm_qa/pg_stat_statements_setup.sh @@ -71,7 +71,7 @@ export PATH="/usr/lib/postgresql/${pgsql_version}/bin:$PATH" echo $PATH cp /usr/lib/postgresql/${pgsql_version}/bin/pg_config /usr/bin -# Stop server and edit postgresql-primary.conf to load pg_stat_sstatement library with required configurations +# Stop server and edit postgresql.conf to load pg_stat_sstatement library with required configurations service postgresql stop echo "shared_preload_libraries = 'pg_stat_statements'" >> /etc/postgresql/${pgsql_version}/main/postgresql.conf echo "track_activity_query_size=2048" >> /etc/postgresql/${pgsql_version}/main/postgresql.conf From c6439ada2d98e5ea3cab9d9494e68cdd5602c43f Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 3 Jun 2025 16:33:54 +0200 Subject: [PATCH 313/340] PMM-7: PDPGSQL --- pmm_qa/postgresql/data/pg_hba.conf.j2 | 2 +- pmm_qa/postgresql/data/postgres-replica.conf | 2 +- pmm_qa/postgresql/data/postgresql-primary.conf | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pmm_qa/postgresql/data/pg_hba.conf.j2 b/pmm_qa/postgresql/data/pg_hba.conf.j2 index 7d58f0c8..43a9816a 100644 --- a/pmm_qa/postgresql/data/pg_hba.conf.j2 +++ b/pmm_qa/postgresql/data/pg_hba.conf.j2 @@ -1,3 +1,3 @@ host replication {{ replication_user }} 0.0.0.0/0 md5 host all all 0.0.0.0/0 md5 -local all postgres trust \ No newline at end of file +local all postgres trust diff --git a/pmm_qa/postgresql/data/postgres-replica.conf b/pmm_qa/postgresql/data/postgres-replica.conf index 52c86776..9005fb12 100644 --- a/pmm_qa/postgresql/data/postgres-replica.conf +++ b/pmm_qa/postgresql/data/postgres-replica.conf @@ -3,4 +3,4 @@ shared_preload_libraries = 'pg_stat_statements' pg_stat_statements.track=all track_io_timing=ON track_activity_query_size=2048 -listen_addresses = '*' \ No newline at end of file +listen_addresses = '*' diff --git a/pmm_qa/postgresql/data/postgresql-primary.conf b/pmm_qa/postgresql/data/postgresql-primary.conf index fb099965..4ca543a8 100644 --- a/pmm_qa/postgresql/data/postgresql-primary.conf +++ b/pmm_qa/postgresql/data/postgresql-primary.conf @@ -7,4 +7,4 @@ hba_file = '/etc/postgresql/pg_hba.conf' shared_preload_libraries = 'pg_stat_statements' pg_stat_statements.track=all track_io_timing=ON -track_activity_query_size=2048 \ No newline at end of file +track_activity_query_size=2048 From fad3bc83d7f705618bc5c6ae182eec58a4a8a7bf Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 3 Jun 2025 16:38:23 +0200 Subject: [PATCH 314/340] PMM-7: PDPGSQL --- .../percona-distribution-postgres-setup.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml index 68a9279b..19e7e571 100644 --- a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml @@ -146,6 +146,7 @@ GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO pmm; \" | psql -U postgres -v ON_ERROR_STOP=1 " + - name: Run pg_basebackup from primary to replica community.docker.docker_container_exec: container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" From ab82c00a91c3b69beb66437c5d0bebb279bebdda Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 3 Jun 2025 16:38:50 +0200 Subject: [PATCH 315/340] PMM-7: PDPGSQL --- .../percona-distribution-postgres-setup.yml | 132 +++++++++--------- 1 file changed, 66 insertions(+), 66 deletions(-) diff --git a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml index 19e7e571..6616b14c 100644 --- a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml @@ -160,69 +160,69 @@ " loop: "{{ range(2, nodes_count | int + 1) | list }}" - - name: Restart PostgreSQL container with custom command - community.docker.docker_container: - name: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" - image: "{{ docker_repo }}:{{ pg_version }}" - restart: true - state: started - command: -c config_file=/etc/postgresql/postgresql.conf - networks: - - name: "{{ network_name }}" - env: - POSTGRES_PASSWORD: "{{ root_password }}" - volumes: - - "{{ data_dir }}/node{{ item }}/data:/var/lib/postgresql/data" - - "./data/postgres-replica.conf:/etc/postgresql/postgresql.conf:ro" - - "{{ data_dir }}/node1/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro" - ports: - - "{{ pdpgsql_port + item - 1 }}:5432" - loop: "{{ range(2, nodes_count | int + 1) | list }}" - - - name: Install pg stat monitor. - include_tasks: ./tasks/install_pg_stat-monitor.yml - - - name: Install and add pmm client. - include_tasks: ../tasks/install_pmm_client.yml - vars: - container_name: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" - loop: "{{ range(1, nodes_count | int + 1) | list }}" - - - name: Get already connected services to pmm server - community.docker.docker_container_exec: - container: "pdpgsql_pmm_{{ pg_version }}_1" - command: > - sh -c 'curl --location --insecure -u"admin:{{ admin_password }}" -s --request GET "http://{{ pmm_server_ip }}:{{ '80' if pmm_server_ip is ansible.utils.ipv4 else '8080' }}/v1/management/services" | jq -r ".services[].service_name"' - register: pmm_server_services - - - name: Display already connected services to pmm server - debug: - msg: "{{ pmm_server_services.stdout | split('\n') }}" - - - name: Find out if service is already connected to pmm server - block: - - name: Loop through percona servers - set_fact: - random_service_name_value: "_{{ 9999 | random + 1 }}" - loop: "{{ range(1, nodes_count | int + 1) | list }}" - when: "('pdpgsql_pmm_' ~ pg_version ~ '_' ~ item) in pmm_server_services.stdout" - - - name: Add service to pmm server - community.docker.docker_container_exec: - container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" - command: pmm-admin add postgresql --username=pmm --password=pmm --query-source=pgstatmonitor pdpgsql_pmm_{{ pg_version }}_{{ item }}{{ random_service_name_value }} --debug 127.0.0.1:5432 - loop: "{{ range(1, nodes_count | int + 1) | list }}" - - - name: Run load on primary node. - include_tasks: ../postgresql/tasks/run_load_pgsql.yml - vars: - node_name: "pdpgsql_pmm_{{ pg_version }}_1" - pgbench_clients: 10 - pgbench_time: 120 - pgbench_scale: 1000 - - - name: Check replication status on primary - community.docker.docker_container_exec: - container: "pgsql_pmm_{{ pg_version }}_1" - user: postgres - command: psql -c "SELECT * FROM pg_stat_replication;" +# - name: Restart PostgreSQL container with custom command +# community.docker.docker_container: +# name: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" +# image: "{{ docker_repo }}:{{ pg_version }}" +# restart: true +# state: started +# command: -c config_file=/etc/postgresql/postgresql.conf +# networks: +# - name: "{{ network_name }}" +# env: +# POSTGRES_PASSWORD: "{{ root_password }}" +# volumes: +# - "{{ data_dir }}/node{{ item }}/data:/var/lib/postgresql/data" +# - "./data/postgres-replica.conf:/etc/postgresql/postgresql.conf:ro" +# - "{{ data_dir }}/node1/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro" +# ports: +# - "{{ pdpgsql_port + item - 1 }}:5432" +# loop: "{{ range(2, nodes_count | int + 1) | list }}" +# +# - name: Install pg stat monitor. +# include_tasks: ./tasks/install_pg_stat-monitor.yml +# +# - name: Install and add pmm client. +# include_tasks: ../tasks/install_pmm_client.yml +# vars: +# container_name: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" +# loop: "{{ range(1, nodes_count | int + 1) | list }}" +# +# - name: Get already connected services to pmm server +# community.docker.docker_container_exec: +# container: "pdpgsql_pmm_{{ pg_version }}_1" +# command: > +# sh -c 'curl --location --insecure -u"admin:{{ admin_password }}" -s --request GET "http://{{ pmm_server_ip }}:{{ '80' if pmm_server_ip is ansible.utils.ipv4 else '8080' }}/v1/management/services" | jq -r ".services[].service_name"' +# register: pmm_server_services +# +# - name: Display already connected services to pmm server +# debug: +# msg: "{{ pmm_server_services.stdout | split('\n') }}" +# +# - name: Find out if service is already connected to pmm server +# block: +# - name: Loop through percona servers +# set_fact: +# random_service_name_value: "_{{ 9999 | random + 1 }}" +# loop: "{{ range(1, nodes_count | int + 1) | list }}" +# when: "('pdpgsql_pmm_' ~ pg_version ~ '_' ~ item) in pmm_server_services.stdout" +# +# - name: Add service to pmm server +# community.docker.docker_container_exec: +# container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" +# command: pmm-admin add postgresql --username=pmm --password=pmm --query-source=pgstatmonitor pdpgsql_pmm_{{ pg_version }}_{{ item }}{{ random_service_name_value }} --debug 127.0.0.1:5432 +# loop: "{{ range(1, nodes_count | int + 1) | list }}" +# +# - name: Run load on primary node. +# include_tasks: ../postgresql/tasks/run_load_pgsql.yml +# vars: +# node_name: "pdpgsql_pmm_{{ pg_version }}_1" +# pgbench_clients: 10 +# pgbench_time: 120 +# pgbench_scale: 1000 +# +# - name: Check replication status on primary +# community.docker.docker_container_exec: +# container: "pgsql_pmm_{{ pg_version }}_1" +# user: postgres +# command: psql -c "SELECT * FROM pg_stat_replication;" From 5adc502eab80c37b05bb49f303f08af48ad78c7d Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 3 Jun 2025 16:38:59 +0200 Subject: [PATCH 316/340] PMM-7: PDPGSQL --- .../percona-distribution-postgres-setup.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml index 6616b14c..a0daab95 100644 --- a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml @@ -221,8 +221,8 @@ # pgbench_time: 120 # pgbench_scale: 1000 # -# - name: Check replication status on primary -# community.docker.docker_container_exec: -# container: "pgsql_pmm_{{ pg_version }}_1" -# user: postgres -# command: psql -c "SELECT * FROM pg_stat_replication;" + - name: Check replication status on primary + community.docker.docker_container_exec: + container: "pgsql_pmm_{{ pg_version }}_1" + user: postgres + command: psql -c "SELECT * FROM pg_stat_replication;" From 6e83772b64bac55d146f01d01f82509472fea1d6 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 3 Jun 2025 16:42:35 +0200 Subject: [PATCH 317/340] PMM-7: PDPGSQL --- .../percona-distribution-postgres-setup.yml | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml index a0daab95..0607c75b 100644 --- a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml @@ -160,25 +160,25 @@ " loop: "{{ range(2, nodes_count | int + 1) | list }}" -# - name: Restart PostgreSQL container with custom command -# community.docker.docker_container: -# name: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" -# image: "{{ docker_repo }}:{{ pg_version }}" -# restart: true -# state: started -# command: -c config_file=/etc/postgresql/postgresql.conf -# networks: -# - name: "{{ network_name }}" -# env: -# POSTGRES_PASSWORD: "{{ root_password }}" -# volumes: -# - "{{ data_dir }}/node{{ item }}/data:/var/lib/postgresql/data" -# - "./data/postgres-replica.conf:/etc/postgresql/postgresql.conf:ro" -# - "{{ data_dir }}/node1/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro" -# ports: -# - "{{ pdpgsql_port + item - 1 }}:5432" -# loop: "{{ range(2, nodes_count | int + 1) | list }}" -# + - name: Restart PostgreSQL container with custom command + community.docker.docker_container: + name: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" + image: "{{ docker_repo }}:{{ pg_version }}" + restart: true + state: started + command: -c config_file=/etc/postgresql/postgresql.conf + networks: + - name: "{{ network_name }}" + env: + POSTGRES_PASSWORD: "{{ root_password }}" + volumes: + - "{{ data_dir }}/node{{ item }}/data:/var/lib/postgresql/data" + - "./data/postgres-replica.conf:/etc/postgresql/postgresql.conf:ro" + - "{{ data_dir }}/node1/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro" + ports: + - "{{ pdpgsql_port + item - 1 }}:5432" + loop: "{{ range(2, nodes_count | int + 1) | list }}" + # - name: Install pg stat monitor. # include_tasks: ./tasks/install_pg_stat-monitor.yml # @@ -223,6 +223,6 @@ # - name: Check replication status on primary community.docker.docker_container_exec: - container: "pgsql_pmm_{{ pg_version }}_1" + container: "pdpgsql_pmm_{{ pg_version }}_1" user: postgres command: psql -c "SELECT * FROM pg_stat_replication;" From e21de7405b799bb7a04d2685d2191ddeb9f2dcd9 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 3 Jun 2025 16:45:52 +0200 Subject: [PATCH 318/340] PMM-7: PDPGSQL --- .../percona-distribution-postgres-setup.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml index 0607c75b..3a6b6673 100644 --- a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml @@ -155,7 +155,7 @@ bash -c " export PGPASSWORD='{{ replication_password }}' && \ timeout 120s \ - pg_basebackup --pgdata=/var/lib/postgresql/data -R -v -Fp -Xs -P \ + pg_basebackup --pgdata=/var/lib/postgresql/data -R -v -P \ --host=pdpgsql_pmm_{{ pg_version }}_1 --port=5432 -U {{ replication_user }} " loop: "{{ range(2, nodes_count | int + 1) | list }}" From 65058bbed1f3577901b2503c46fe9f88e7af8e5f Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 3 Jun 2025 16:52:19 +0200 Subject: [PATCH 319/340] PMM-7: PDPGSQL --- .../percona-distribution-postgres-setup.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml index 3a6b6673..7fd52f1a 100644 --- a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml @@ -111,6 +111,11 @@ - "{{ data_dir }}/node1/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro" loop: "{{ range(2, nodes_count | int + 1) | list }}" + + - name: Wait 10 seconds for the nodes to start + pause: + seconds: 30 + - name: Wipe replica data directory before basebackup community.docker.docker_container_exec: container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" From 84e326554ffcce599513cd4c48fb1a80636a18a4 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 3 Jun 2025 17:11:35 +0200 Subject: [PATCH 320/340] PMM-7: PDPGSQL --- .../percona-distribution-postgres-setup.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml index 7fd52f1a..55006abf 100644 --- a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml @@ -94,7 +94,7 @@ loop: "{{ range(2, nodes_count | int + 1) | list }}" ignore_errors: yes - - name: Start PostgreSQL replica container + - name: Start Percona Distribution PostgreSQL replica container community.docker.docker_container: name: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" image: "{{ docker_repo }}:{{ pg_version }}" @@ -104,6 +104,7 @@ networks: - name: "{{ network_name }}" env: + POSTGRES_INITDB_SKIP: "yes" POSTGRES_PASSWORD: "{{ root_password }}" volumes: - "{{ data_dir }}/node{{ item }}/data:/var/lib/postgresql/data" @@ -111,8 +112,7 @@ - "{{ data_dir }}/node1/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro" loop: "{{ range(2, nodes_count | int + 1) | list }}" - - - name: Wait 10 seconds for the nodes to start + - name: Wait 30 seconds for the nodes to start pause: seconds: 30 @@ -160,7 +160,7 @@ bash -c " export PGPASSWORD='{{ replication_password }}' && \ timeout 120s \ - pg_basebackup --pgdata=/var/lib/postgresql/data -R -v -P \ + pg_basebackup --pgdata=/var/lib/postgresql/data -R -v -Fp -Xs -P \ --host=pdpgsql_pmm_{{ pg_version }}_1 --port=5432 -U {{ replication_user }} " loop: "{{ range(2, nodes_count | int + 1) | list }}" From 2f12317b91c8967421221759f1c2d58b9bc76a89 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 3 Jun 2025 17:19:02 +0200 Subject: [PATCH 321/340] PMM-7: PDPGSQL --- .../percona-distribution-postgres-setup.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml index 55006abf..85b31d90 100644 --- a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml @@ -97,14 +97,13 @@ - name: Start Percona Distribution PostgreSQL replica container community.docker.docker_container: name: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" - image: "{{ docker_repo }}:{{ pg_version }}" + image: "postgres:{{ pg_version }}" restart_policy: "no" state: started command: sleep infinity networks: - name: "{{ network_name }}" env: - POSTGRES_INITDB_SKIP: "yes" POSTGRES_PASSWORD: "{{ root_password }}" volumes: - "{{ data_dir }}/node{{ item }}/data:/var/lib/postgresql/data" From caee574dd087f481a96967df5b809b2c40aef593 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 3 Jun 2025 17:26:47 +0200 Subject: [PATCH 322/340] PMM-7: PDPGSQL --- .../percona-distribution-postgres-setup.yml | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml index 85b31d90..c3bbdd05 100644 --- a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml @@ -97,24 +97,21 @@ - name: Start Percona Distribution PostgreSQL replica container community.docker.docker_container: name: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" - image: "postgres:{{ pg_version }}" + image: "{{ docker_repo }}:{{ pg_version }}" restart_policy: "no" state: started command: sleep infinity networks: - name: "{{ network_name }}" env: + POSTGRES_INITDB_SKIP: "yes" POSTGRES_PASSWORD: "{{ root_password }}" volumes: - "{{ data_dir }}/node{{ item }}/data:/var/lib/postgresql/data" - - "./data/postgresql-replica.conf:/etc/postgresql/postgresql.conf:ro" - - "{{ data_dir }}/node1/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro" + - "./data/postgres-replica.conf:/etc/postgresql/postgresql.conf:ro" + - "{{ data_dir }}/node{{ item }}/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro" loop: "{{ range(2, nodes_count | int + 1) | list }}" - - name: Wait 30 seconds for the nodes to start - pause: - seconds: 30 - - name: Wipe replica data directory before basebackup community.docker.docker_container_exec: container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" @@ -164,13 +161,13 @@ " loop: "{{ range(2, nodes_count | int + 1) | list }}" - - name: Restart PostgreSQL container with custom command + - name: Restart Percona Distribution PostgreSQL container with custom command community.docker.docker_container: name: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" image: "{{ docker_repo }}:{{ pg_version }}" restart: true state: started - command: -c config_file=/etc/postgresql/postgresql.conf + command: -c config_file=/etc/postgresql/postgresql-replica.conf networks: - name: "{{ network_name }}" env: @@ -178,9 +175,9 @@ volumes: - "{{ data_dir }}/node{{ item }}/data:/var/lib/postgresql/data" - "./data/postgres-replica.conf:/etc/postgresql/postgresql.conf:ro" - - "{{ data_dir }}/node1/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro" +# - "{{ data_dir }}/node1/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro" ports: - - "{{ pdpgsql_port + item - 1 }}:5432" + - "{{ pdpgsql_port + item - 1}}:5432" loop: "{{ range(2, nodes_count | int + 1) | list }}" # - name: Install pg stat monitor. From 9e7b74618e5a8f4251945f93915a1fc1bc0fa259 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 3 Jun 2025 17:30:08 +0200 Subject: [PATCH 323/340] PMM-7: PDPGSQL --- .../percona-distribution-postgres-setup.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml index c3bbdd05..2c072b6c 100644 --- a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml @@ -167,7 +167,7 @@ image: "{{ docker_repo }}:{{ pg_version }}" restart: true state: started - command: -c config_file=/etc/postgresql/postgresql-replica.conf + command: -c config_file=/etc/postgresql/postgresql.conf networks: - name: "{{ network_name }}" env: From fafdef0be51b97c8996ea6c52e4aa42b4ca95324 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 3 Jun 2025 17:40:09 +0200 Subject: [PATCH 324/340] PMM-7: PDPGSQL --- .../percona-distribution-postgres-setup.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml index 2c072b6c..773eb677 100644 --- a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml @@ -175,7 +175,7 @@ volumes: - "{{ data_dir }}/node{{ item }}/data:/var/lib/postgresql/data" - "./data/postgres-replica.conf:/etc/postgresql/postgresql.conf:ro" -# - "{{ data_dir }}/node1/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro" + - "{{ data_dir }}/node1/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro" ports: - "{{ pdpgsql_port + item - 1}}:5432" loop: "{{ range(2, nodes_count | int + 1) | list }}" From b43bfc60e6dd1ced5a0ab132e1527b6760656cfe Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 3 Jun 2025 17:45:40 +0200 Subject: [PATCH 325/340] PMM-7: PDPGSQL --- .../percona-distribution-postgres-setup.yml | 62 +++++++++---------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml index 773eb677..7ace3ea5 100644 --- a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml @@ -148,37 +148,37 @@ \" | psql -U postgres -v ON_ERROR_STOP=1 " - - name: Run pg_basebackup from primary to replica - community.docker.docker_container_exec: - container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" - user: root - command: > - bash -c " - export PGPASSWORD='{{ replication_password }}' && \ - timeout 120s \ - pg_basebackup --pgdata=/var/lib/postgresql/data -R -v -Fp -Xs -P \ - --host=pdpgsql_pmm_{{ pg_version }}_1 --port=5432 -U {{ replication_user }} - " - loop: "{{ range(2, nodes_count | int + 1) | list }}" - - - name: Restart Percona Distribution PostgreSQL container with custom command - community.docker.docker_container: - name: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" - image: "{{ docker_repo }}:{{ pg_version }}" - restart: true - state: started - command: -c config_file=/etc/postgresql/postgresql.conf - networks: - - name: "{{ network_name }}" - env: - POSTGRES_PASSWORD: "{{ root_password }}" - volumes: - - "{{ data_dir }}/node{{ item }}/data:/var/lib/postgresql/data" - - "./data/postgres-replica.conf:/etc/postgresql/postgresql.conf:ro" - - "{{ data_dir }}/node1/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro" - ports: - - "{{ pdpgsql_port + item - 1}}:5432" - loop: "{{ range(2, nodes_count | int + 1) | list }}" +# - name: Run pg_basebackup from primary to replica +# community.docker.docker_container_exec: +# container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" +# user: root +# command: > +# bash -c " +# export PGPASSWORD='{{ replication_password }}' && \ +# timeout 120s \ +# pg_basebackup --pgdata=/var/lib/postgresql/data -R -v -Fp -Xs -P \ +# --host=pdpgsql_pmm_{{ pg_version }}_1 --port=5432 -U {{ replication_user }} +# " +# loop: "{{ range(2, nodes_count | int + 1) | list }}" +# +# - name: Restart Percona Distribution PostgreSQL container with custom command +# community.docker.docker_container: +# name: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" +# image: "{{ docker_repo }}:{{ pg_version }}" +# restart: true +# state: started +# command: -c config_file=/etc/postgresql/postgresql.conf +# networks: +# - name: "{{ network_name }}" +# env: +# POSTGRES_PASSWORD: "{{ root_password }}" +# volumes: +# - "{{ data_dir }}/node{{ item }}/data:/var/lib/postgresql/data" +# - "./data/postgres-replica.conf:/etc/postgresql/postgresql.conf:ro" +# - "{{ data_dir }}/node1/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro" +# ports: +# - "{{ pdpgsql_port + item - 1}}:5432" +# loop: "{{ range(2, nodes_count | int + 1) | list }}" # - name: Install pg stat monitor. # include_tasks: ./tasks/install_pg_stat-monitor.yml From b17514865100747f60ccbc86f4199d0dd201e746 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 3 Jun 2025 17:52:50 +0200 Subject: [PATCH 326/340] PMM-7: PDPGSQL --- .../percona-distribution-postgres-setup.yml | 62 +++++++++---------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml index 7ace3ea5..db019cd7 100644 --- a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml @@ -148,37 +148,37 @@ \" | psql -U postgres -v ON_ERROR_STOP=1 " -# - name: Run pg_basebackup from primary to replica -# community.docker.docker_container_exec: -# container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" -# user: root -# command: > -# bash -c " -# export PGPASSWORD='{{ replication_password }}' && \ -# timeout 120s \ -# pg_basebackup --pgdata=/var/lib/postgresql/data -R -v -Fp -Xs -P \ -# --host=pdpgsql_pmm_{{ pg_version }}_1 --port=5432 -U {{ replication_user }} -# " -# loop: "{{ range(2, nodes_count | int + 1) | list }}" -# -# - name: Restart Percona Distribution PostgreSQL container with custom command -# community.docker.docker_container: -# name: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" -# image: "{{ docker_repo }}:{{ pg_version }}" -# restart: true -# state: started -# command: -c config_file=/etc/postgresql/postgresql.conf -# networks: -# - name: "{{ network_name }}" -# env: -# POSTGRES_PASSWORD: "{{ root_password }}" -# volumes: -# - "{{ data_dir }}/node{{ item }}/data:/var/lib/postgresql/data" -# - "./data/postgres-replica.conf:/etc/postgresql/postgresql.conf:ro" -# - "{{ data_dir }}/node1/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro" -# ports: -# - "{{ pdpgsql_port + item - 1}}:5432" -# loop: "{{ range(2, nodes_count | int + 1) | list }}" + - name: Run pg_basebackup from primary to replica + community.docker.docker_container_exec: + container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" + user: root + command: > + bash -c " + export PGPASSWORD='{{ replication_password }}' && \ + timeout 120s \ + pg_basebackup --pgdata=/var/lib/postgresql/data -R -v -Fp -Xs -P \ + --host=pdpgsql_pmm_{{ pg_version }}_1 --port=5432 -U {{ replication_user }} > /tmp/pg_basebackup.log 2>&1 + " + loop: "{{ range(2, nodes_count | int + 1) | list }}" + + - name: Restart Percona Distribution PostgreSQL container with custom command + community.docker.docker_container: + name: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" + image: "{{ docker_repo }}:{{ pg_version }}" + restart: true + state: started + command: -c config_file=/etc/postgresql/postgresql.conf + networks: + - name: "{{ network_name }}" + env: + POSTGRES_PASSWORD: "{{ root_password }}" + volumes: + - "{{ data_dir }}/node{{ item }}/data:/var/lib/postgresql/data" + - "./data/postgres-replica.conf:/etc/postgresql/postgresql.conf:ro" + - "{{ data_dir }}/node1/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro" + ports: + - "{{ pdpgsql_port + item - 1}}:5432" + loop: "{{ range(2, nodes_count | int + 1) | list }}" # - name: Install pg stat monitor. # include_tasks: ./tasks/install_pg_stat-monitor.yml From 2dd5cef7510cee3862152f17f90ef99da1513583 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Tue, 3 Jun 2025 17:54:41 +0200 Subject: [PATCH 327/340] PMM-7: PDPGSQL --- .../data/postgresql-primary.conf | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pmm_qa/percona-distribution-postgresql/data/postgresql-primary.conf b/pmm_qa/percona-distribution-postgresql/data/postgresql-primary.conf index bcd4a3c2..65b7a68f 100644 --- a/pmm_qa/percona-distribution-postgresql/data/postgresql-primary.conf +++ b/pmm_qa/percona-distribution-postgresql/data/postgresql-primary.conf @@ -10,3 +10,7 @@ track_io_timing=ON max_connections=1000 listen_addresses = '*' pg_stat_monitor.pgsm_enable_query_plan = 'yes' +log_connections = on +log_disconnections = on +log_replication_commands = on +log_statement = 'all' \ No newline at end of file From 4cd38931de381ec55f6c795f40a7bc147e000805 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 4 Jun 2025 07:08:06 +0200 Subject: [PATCH 328/340] PMM-7: PDPGSQL --- .../percona-distribution-postgres-setup.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml index db019cd7..9859ce2b 100644 --- a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml @@ -161,6 +161,12 @@ " loop: "{{ range(2, nodes_count | int + 1) | list }}" + - name: Remove temporary backup container + docker_container: + name: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" + state: absent + loop: "{{ range(2, nodes_count | int + 1) | list }}" + - name: Restart Percona Distribution PostgreSQL container with custom command community.docker.docker_container: name: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" From 8111a1f31e30f36dc8f4b9a4dbacec591d6105d5 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 4 Jun 2025 07:15:33 +0200 Subject: [PATCH 329/340] PMM-7: PDPGSQL --- .../percona-distribution-postgres-setup.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml index 9859ce2b..98e226b8 100644 --- a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml @@ -44,6 +44,12 @@ src: data/pg_hba.conf.j2 dest: "{{ data_dir }}/node1/pg_hba.conf" + - name: Generate pg_hba.conf for replica node + template: + src: data/pg_hba-replica.conf.j2 + dest: "{{ data_dir }}/node{{ item }}/pg_hba.conf" + loop: "{{ range(2, nodes_count | int + 1) | list }}" + - name: Remove old PostgreSQL primary container community.docker.docker_container: name: "pdpgsql_pmm_{{ pg_version }}_1" From 2ebead29707c33f1ec8faaeb911cf0452a8e2c75 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 4 Jun 2025 07:17:00 +0200 Subject: [PATCH 330/340] PMM-7: PDPGSQL --- .../percona-distribution-postgres-setup.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml index 98e226b8..b9c205e5 100644 --- a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml @@ -46,7 +46,7 @@ - name: Generate pg_hba.conf for replica node template: - src: data/pg_hba-replica.conf.j2 + src: data/pg_hba_replica.conf.j2 dest: "{{ data_dir }}/node{{ item }}/pg_hba.conf" loop: "{{ range(2, nodes_count | int + 1) | list }}" From 18e178363b06fc08fe3323c3f1c50e70399d8ea4 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 4 Jun 2025 07:24:42 +0200 Subject: [PATCH 331/340] PMM-7: PDPGSQL --- .../data/pg_hba_replica.conf.j2 | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/pmm_qa/percona-distribution-postgresql/data/pg_hba_replica.conf.j2 b/pmm_qa/percona-distribution-postgresql/data/pg_hba_replica.conf.j2 index 4a3a14e6..8488d8cc 100644 --- a/pmm_qa/percona-distribution-postgresql/data/pg_hba_replica.conf.j2 +++ b/pmm_qa/percona-distribution-postgresql/data/pg_hba_replica.conf.j2 @@ -1,4 +1,3 @@ -# Allow replication connections from replicas -host replication {{ replication_user }} 0.0.0.0/0 md5 -local all postgres trust -host all all 0.0.0.0/0 md5 +host replication {{ replication_user }} 0.0.0.0/0 md5 +host all all 0.0.0.0/0 md5 +local all postgres trust From d9b5b75ea54849ea0cf475cd68f9350d546293c4 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 4 Jun 2025 07:28:34 +0200 Subject: [PATCH 332/340] PMM-7: PDPGSQL --- .../data/postgres-replica.conf | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pmm_qa/percona-distribution-postgresql/data/postgres-replica.conf b/pmm_qa/percona-distribution-postgresql/data/postgres-replica.conf index af6750d6..b6a3d6ba 100644 --- a/pmm_qa/percona-distribution-postgresql/data/postgres-replica.conf +++ b/pmm_qa/percona-distribution-postgresql/data/postgres-replica.conf @@ -8,3 +8,7 @@ pg_stat_monitor.pgsm_enable_query_plan = 'yes' pg_stat_monitor.pgsm_query_max_len = 2048 pg_stat_monitor.pgsm_normalized_query=1 pg_stat_monitor.pgsm_enable_query_plan=1 +log_connections = on +log_disconnections = on +log_replication_commands = on +log_statement = 'all' \ No newline at end of file From 52ff3c9fd709db325f08df8bc3351f049ce8d652 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 4 Jun 2025 07:49:33 +0200 Subject: [PATCH 333/340] PMM-7: PDPGSQL --- .../percona-distribution-postgres-setup.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml index b9c205e5..77b230e5 100644 --- a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml @@ -70,7 +70,7 @@ env: POSTGRES_PASSWORD: "{{ root_password }}" volumes: - - "{{ data_dir }}/node1/data:/var/lib/postgresql/data" + - "{{ data_dir }}/node1/data:/data/db" - "./data/postgresql-primary.conf:/etc/postgresql/postgresql.conf:ro" - "{{ data_dir }}/node1/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro" command: -c config_file=/etc/postgresql/postgresql.conf @@ -113,7 +113,7 @@ POSTGRES_INITDB_SKIP: "yes" POSTGRES_PASSWORD: "{{ root_password }}" volumes: - - "{{ data_dir }}/node{{ item }}/data:/var/lib/postgresql/data" + - "{{ data_dir }}/node{{ item }}/data:/data/db" - "./data/postgres-replica.conf:/etc/postgresql/postgresql.conf:ro" - "{{ data_dir }}/node{{ item }}/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro" loop: "{{ range(2, nodes_count | int + 1) | list }}" @@ -122,7 +122,7 @@ community.docker.docker_container_exec: container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" user: root - command: rm -rf /var/lib/postgresql/data/* + command: rm -rf /data/db loop: "{{ range(2, nodes_count | int + 1) | list }}" - name: Create PostgreSQL user 'pmm' with password @@ -162,7 +162,7 @@ bash -c " export PGPASSWORD='{{ replication_password }}' && \ timeout 120s \ - pg_basebackup --pgdata=/var/lib/postgresql/data -R -v -Fp -Xs -P \ + pg_basebackup --pgdata=/data/db -R -v -Fp -Xs -P \ --host=pdpgsql_pmm_{{ pg_version }}_1 --port=5432 -U {{ replication_user }} > /tmp/pg_basebackup.log 2>&1 " loop: "{{ range(2, nodes_count | int + 1) | list }}" @@ -185,7 +185,7 @@ env: POSTGRES_PASSWORD: "{{ root_password }}" volumes: - - "{{ data_dir }}/node{{ item }}/data:/var/lib/postgresql/data" + - "{{ data_dir }}/node{{ item }}/data:/data/db" - "./data/postgres-replica.conf:/etc/postgresql/postgresql.conf:ro" - "{{ data_dir }}/node1/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro" ports: From 7fa26b5c762525650b0b2d25ddb1657c84d2021f Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 4 Jun 2025 07:56:07 +0200 Subject: [PATCH 334/340] PMM-7: PDPGSQL --- .../percona-distribution-postgres-setup.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml index 77b230e5..c138d227 100644 --- a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml @@ -50,6 +50,13 @@ dest: "{{ data_dir }}/node{{ item }}/pg_hba.conf" loop: "{{ range(2, nodes_count | int + 1) | list }}" + - name: Fix permissions on data directory + file: + path: "{{ data_dir }}/node1/data" + owner: 999 + group: 999 + recurse: yes + - name: Remove old PostgreSQL primary container community.docker.docker_container: name: "pdpgsql_pmm_{{ pg_version }}_1" From 8f29f1d03c41d40a20d17a83ab3934530af41779 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 4 Jun 2025 07:57:17 +0200 Subject: [PATCH 335/340] PMM-7: PDPGSQL --- .../percona-distribution-postgres-setup.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml index c138d227..eb1430e8 100644 --- a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml @@ -53,8 +53,8 @@ - name: Fix permissions on data directory file: path: "{{ data_dir }}/node1/data" - owner: 999 - group: 999 + owner: 1001 + group: 1001 recurse: yes - name: Remove old PostgreSQL primary container From 8289c87f94a705db3f87fba517630923d51e91f3 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 4 Jun 2025 07:58:53 +0200 Subject: [PATCH 336/340] PMM-7: PDPGSQL --- .../percona-distribution-postgres-setup.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml index eb1430e8..4b5aff54 100644 --- a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml @@ -52,10 +52,11 @@ - name: Fix permissions on data directory file: - path: "{{ data_dir }}/node1/data" + path: "{{ data_dir }}/node{{ item }}/data" owner: 1001 group: 1001 recurse: yes + loop: "{{ range(1, nodes_count | int + 1) | list }}" - name: Remove old PostgreSQL primary container community.docker.docker_container: From ecc6d61e4131487d8b08ae1cedf1f8e1e43ee181 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 4 Jun 2025 08:00:47 +0200 Subject: [PATCH 337/340] PMM-7: PDPGSQL --- .../percona-distribution-postgres-setup.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml index 4b5aff54..f3ef0bd4 100644 --- a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml @@ -130,7 +130,7 @@ community.docker.docker_container_exec: container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" user: root - command: rm -rf /data/db + command: rm -rf /data/db/* loop: "{{ range(2, nodes_count | int + 1) | list }}" - name: Create PostgreSQL user 'pmm' with password From 677497b2eb100a91ef56922921c753c4dde03877 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 4 Jun 2025 08:07:18 +0200 Subject: [PATCH 338/340] PMM-7: PDPGSQL --- .../percona-distribution-postgres-setup.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml index f3ef0bd4..1b952f8d 100644 --- a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml @@ -181,6 +181,14 @@ state: absent loop: "{{ range(2, nodes_count | int + 1) | list }}" + - name: Fix permissions on data directory + file: + path: "{{ data_dir }}/node{{ item }}/data" + owner: 1001 + group: 1001 + recurse: yes + loop: "{{ range(2, nodes_count | int + 1) | list }}" + - name: Restart Percona Distribution PostgreSQL container with custom command community.docker.docker_container: name: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" From da7d02c07ee4a1726907c9fb199cffb69c663518 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 4 Jun 2025 08:10:35 +0200 Subject: [PATCH 339/340] PMM-7: PDPGSQL --- .../percona-distribution-postgres-setup.yml | 84 +++++++++---------- 1 file changed, 42 insertions(+), 42 deletions(-) diff --git a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml index 1b952f8d..279013fa 100644 --- a/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml +++ b/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml @@ -208,48 +208,48 @@ - "{{ pdpgsql_port + item - 1}}:5432" loop: "{{ range(2, nodes_count | int + 1) | list }}" -# - name: Install pg stat monitor. -# include_tasks: ./tasks/install_pg_stat-monitor.yml -# -# - name: Install and add pmm client. -# include_tasks: ../tasks/install_pmm_client.yml -# vars: -# container_name: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" -# loop: "{{ range(1, nodes_count | int + 1) | list }}" -# -# - name: Get already connected services to pmm server -# community.docker.docker_container_exec: -# container: "pdpgsql_pmm_{{ pg_version }}_1" -# command: > -# sh -c 'curl --location --insecure -u"admin:{{ admin_password }}" -s --request GET "http://{{ pmm_server_ip }}:{{ '80' if pmm_server_ip is ansible.utils.ipv4 else '8080' }}/v1/management/services" | jq -r ".services[].service_name"' -# register: pmm_server_services -# -# - name: Display already connected services to pmm server -# debug: -# msg: "{{ pmm_server_services.stdout | split('\n') }}" -# -# - name: Find out if service is already connected to pmm server -# block: -# - name: Loop through percona servers -# set_fact: -# random_service_name_value: "_{{ 9999 | random + 1 }}" -# loop: "{{ range(1, nodes_count | int + 1) | list }}" -# when: "('pdpgsql_pmm_' ~ pg_version ~ '_' ~ item) in pmm_server_services.stdout" -# -# - name: Add service to pmm server -# community.docker.docker_container_exec: -# container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" -# command: pmm-admin add postgresql --username=pmm --password=pmm --query-source=pgstatmonitor pdpgsql_pmm_{{ pg_version }}_{{ item }}{{ random_service_name_value }} --debug 127.0.0.1:5432 -# loop: "{{ range(1, nodes_count | int + 1) | list }}" -# -# - name: Run load on primary node. -# include_tasks: ../postgresql/tasks/run_load_pgsql.yml -# vars: -# node_name: "pdpgsql_pmm_{{ pg_version }}_1" -# pgbench_clients: 10 -# pgbench_time: 120 -# pgbench_scale: 1000 -# + - name: Install pg stat monitor. + include_tasks: ./tasks/install_pg_stat-monitor.yml + + - name: Install and add pmm client. + include_tasks: ../tasks/install_pmm_client.yml + vars: + container_name: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" + loop: "{{ range(1, nodes_count | int + 1) | list }}" + + - name: Get already connected services to pmm server + community.docker.docker_container_exec: + container: "pdpgsql_pmm_{{ pg_version }}_1" + command: > + sh -c 'curl --location --insecure -u"admin:{{ admin_password }}" -s --request GET "http://{{ pmm_server_ip }}:{{ '80' if pmm_server_ip is ansible.utils.ipv4 else '8080' }}/v1/management/services" | jq -r ".services[].service_name"' + register: pmm_server_services + + - name: Display already connected services to pmm server + debug: + msg: "{{ pmm_server_services.stdout | split('\n') }}" + + - name: Find out if service is already connected to pmm server + block: + - name: Loop through percona servers + set_fact: + random_service_name_value: "_{{ 9999 | random + 1 }}" + loop: "{{ range(1, nodes_count | int + 1) | list }}" + when: "('pdpgsql_pmm_' ~ pg_version ~ '_' ~ item) in pmm_server_services.stdout" + + - name: Add service to pmm server + community.docker.docker_container_exec: + container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" + command: pmm-admin add postgresql --username=pmm --password=pmm --query-source=pgstatmonitor pdpgsql_pmm_{{ pg_version }}_{{ item }}{{ random_service_name_value }} --debug 127.0.0.1:5432 + loop: "{{ range(1, nodes_count | int + 1) | list }}" + + - name: Run load on primary node. + include_tasks: ../postgresql/tasks/run_load_pgsql.yml + vars: + node_name: "pdpgsql_pmm_{{ pg_version }}_1" + pgbench_clients: 10 + pgbench_time: 120 + pgbench_scale: 1000 + - name: Check replication status on primary community.docker.docker_container_exec: container: "pdpgsql_pmm_{{ pg_version }}_1" From 876b56816b12475d8e3d9ec6c47fcd7942e7f977 Mon Sep 17 00:00:00 2001 From: Peter Sirotnak Date: Wed, 4 Jun 2025 08:17:37 +0200 Subject: [PATCH 340/340] PMM-7: PDPGSQL --- .../tasks/install_pg_stat-monitor.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pmm_qa/percona-distribution-postgresql/tasks/install_pg_stat-monitor.yml b/pmm_qa/percona-distribution-postgresql/tasks/install_pg_stat-monitor.yml index 3c8710b2..85a6d57f 100644 --- a/pmm_qa/percona-distribution-postgresql/tasks/install_pg_stat-monitor.yml +++ b/pmm_qa/percona-distribution-postgresql/tasks/install_pg_stat-monitor.yml @@ -40,11 +40,10 @@ - name: Create pg_stat_statements extension community.docker.docker_container_exec: - container: "pdpgsql_pmm_{{ pg_version }}_{{ item }}" + container: "pdpgsql_pmm_{{ pg_version }}_1" user: postgres command: > psql -U postgres -d 'postgres' -c " CREATE EXTENSION IF NOT EXISTS pg_stat_monitor; SELECT pg_stat_monitor_version(); " - loop: "{{ range(1, nodes_count | int + 1) | list }}"