Skip to content

Commit dcbec76

Browse files
maths22maths22
committed
Support replicating index creation (and use parsed queries for transformation)
1 parent f175f52 commit dcbec76

45 files changed

Lines changed: 20434 additions & 1293 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/test.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ jobs:
88
fail-fast: false
99
matrix:
1010
pg_version: ["11", "12", "13", "14", "15", "16", "17", "18"]
11-
from_version: ["2.2", "2.3"]
11+
from_version: ["2.3", "2.4"]
1212
steps:
1313
- uses: actions/checkout@v6
1414
- name: Remove default PostgreSQL

‎Makefile‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,11 @@ DATA = pgl_ddl_deploy--1.0.sql pgl_ddl_deploy--1.0--1.1.sql \
1010
pgl_ddl_deploy--2.0.sql pgl_ddl_deploy--2.0--2.1.sql \
1111
pgl_ddl_deploy--2.1.sql pgl_ddl_deploy--2.1--2.2.sql \
1212
pgl_ddl_deploy--2.2.sql pgl_ddl_deploy--2.2--2.3.sql \
13-
pgl_ddl_deploy--2.3.sql
14-
MODULES = pgl_ddl_deploy ddl_deparse
13+
pgl_ddl_deploy--2.3.sql pgl_ddl_deploy--2.3--2.4.sql \
14+
pgl_ddl_deploy--2.4.sql
15+
MODULES = ddl_deparse
16+
MODULE_big = pgl_ddl_deploy
17+
OBJS = postgres_deparse.o pgl_ddl_deploy.o
1518

1619
REGRESS := 01_create_ext 02_setup 03_add_configs 04_deploy 04_deploy_update \
1720
05_allowed 06_multi 07_edges 08_ignored \
@@ -52,7 +55,8 @@ REGRESS := 01_create_ext 02_setup 03_add_configs 04_deploy 04_deploy_update \
5255
54_new_setup \
5356
55_raise_message \
5457
56_1_6_features \
55-
57_native_features
58+
57_2_4_features \
59+
99_native_features
5660
PG_CONFIG = pg_config
5761
PGXS := $(shell $(PG_CONFIG) --pgxs)
5862
include $(PGXS)

‎README.md‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,9 +39,9 @@ https://innovation.enova.com/pursuing-postgres-ddl-replication/
3939

4040
# <a name="release_notes"></a>Release Notes
4141

42-
### Release 2.3
42+
### Release 2.4
4343
Summary of changes:
44-
* Support for Postgres 17 & 18
44+
* Support for replicating index DDL
4545

4646
### Release 2.2
4747
Summary of changes:
@@ -356,6 +356,11 @@ SQL statement with a single node `parsetree`) will be eligible for propagation.
356356
be maintained by DDL replication. Thus only `ALTER TABLE`
357357
statements are permitted here. This option is incompatible with
358358
`include_schema_regex`.
359+
- `include_indexes`: if true, will replicate `CREATE INDEX`/`ALTER INDEX`/`DROP INDEX`
360+
statements. This can be undesirable if the replica is intended to have different
361+
configuration from the primary. Also, indexes cannot be created with `CREATE INDEX CONCURRENTLY`
362+
and will therefore potentially block operations on the table, which can be problematic
363+
if the replica is in active read usage.
359364
- `queue_subscriber_failures`: if true, DDL will be allowed to fail on subscriber
360365
without breaking replication, and queued for retry using function
361366
`pgl_ddl_deploy.retry_all_subscriber_logs()`. This is useful for example if you

‎expected/05_allowed.out‎

Lines changed: 252 additions & 252 deletions
Large diffs are not rendered by default.

‎expected/06_multi_1.out‎

Lines changed: 22 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -70,12 +70,12 @@ COMMIT
7070
SELECT set_name, ddl_sql_raw, ddl_sql_sent FROM pgl_ddl_deploy.events ORDER BY id DESC LIMIT 10;
7171
set_name | ddl_sql_raw | ddl_sql_sent
7272
----------+-------------------------------------------------------------+------------------------------------------------
73-
test7 | BEGIN; CREATE TABLE foobar.foo(id int primary key); COMMIT; | CREATE TABLE foobar.foo(id int primary key);
74-
test5 | BEGIN; CREATE TABLE foobar.foo(id int primary key); COMMIT; | CREATE TABLE foobar.foo(id int primary key);
75-
test3 | BEGIN; CREATE TABLE foobar.foo(id int primary key); COMMIT; | CREATE TABLE foobar.foo(id int primary key);
76-
test1 | BEGIN; CREATE TABLE foobar.foo(id int primary key); COMMIT; | CREATE TABLE foobar.foo(id int primary key);
77-
test3 | BEGIN; CREATE TABLE foo(id int primary key); COMMIT; | CREATE TABLE foo(id int primary key);
78-
test1 | BEGIN; CREATE TABLE foo(id int primary key); COMMIT; | CREATE TABLE foo(id int primary key);
73+
test7 | BEGIN; CREATE TABLE foobar.foo(id int primary key); COMMIT; | CREATE TABLE foobar.foo (id int PRIMARY KEY);
74+
test5 | BEGIN; CREATE TABLE foobar.foo(id int primary key); COMMIT; | CREATE TABLE foobar.foo (id int PRIMARY KEY);
75+
test3 | BEGIN; CREATE TABLE foobar.foo(id int primary key); COMMIT; | CREATE TABLE foobar.foo (id int PRIMARY KEY);
76+
test1 | BEGIN; CREATE TABLE foobar.foo(id int primary key); COMMIT; | CREATE TABLE foobar.foo (id int PRIMARY KEY);
77+
test3 | BEGIN; CREATE TABLE foo(id int primary key); COMMIT; | CREATE TABLE foo (id int PRIMARY KEY);
78+
test1 | BEGIN; CREATE TABLE foo(id int primary key); COMMIT; | CREATE TABLE foo (id int PRIMARY KEY);
7979
test8 | CREATE SCHEMA foobar; | CREATE SCHEMA foobar;
8080
test7 | CREATE SCHEMA foobar; | CREATE SCHEMA foobar;
8181
test6 | CREATE SCHEMA foobar; | CREATE SCHEMA foobar;
@@ -116,14 +116,14 @@ WARNING: Unhandled deployment logged in pgl_ddl_deploy.unhandled
116116
CREATE TABLE
117117
DROP TABLE
118118
SELECT set_name, ddl_sql_raw, ddl_sql_sent FROM pgl_ddl_deploy.events ORDER BY id DESC LIMIT 10;
119-
set_name | ddl_sql_raw | ddl_sql_sent
120-
----------+-----------------------------------------------------------------------------+-----------------------------------------------------------------------------
121-
test7 | CREATE TABLE foobar.foo(id int primary key); DROP TABLE foobar.foo CASCADE; | CREATE TABLE foobar.foo(id int primary key); DROP TABLE foobar.foo CASCADE;
122-
test5 | CREATE TABLE foobar.foo(id int primary key); DROP TABLE foobar.foo CASCADE; | CREATE TABLE foobar.foo(id int primary key); DROP TABLE foobar.foo CASCADE;
123-
test3 | CREATE TABLE foobar.foo(id int primary key); DROP TABLE foobar.foo CASCADE; | CREATE TABLE foobar.foo(id int primary key); DROP TABLE foobar.foo CASCADE;
124-
test1 | CREATE TABLE foobar.foo(id int primary key); DROP TABLE foobar.foo CASCADE; | CREATE TABLE foobar.foo(id int primary key); DROP TABLE foobar.foo CASCADE;
125-
test3 | CREATE TABLE foo(id int primary key); DROP TABLE foo CASCADE; | CREATE TABLE foo(id int primary key); DROP TABLE foo CASCADE;
126-
test1 | CREATE TABLE foo(id int primary key); DROP TABLE foo CASCADE; | CREATE TABLE foo(id int primary key); DROP TABLE foo CASCADE;
119+
set_name | ddl_sql_raw | ddl_sql_sent
120+
----------+-----------------------------------------------------------------------------+------------------------------------------------------------------------------
121+
test7 | CREATE TABLE foobar.foo(id int primary key); DROP TABLE foobar.foo CASCADE; | CREATE TABLE foobar.foo (id int PRIMARY KEY); DROP TABLE foobar.foo CASCADE;
122+
test5 | CREATE TABLE foobar.foo(id int primary key); DROP TABLE foobar.foo CASCADE; | CREATE TABLE foobar.foo (id int PRIMARY KEY); DROP TABLE foobar.foo CASCADE;
123+
test3 | CREATE TABLE foobar.foo(id int primary key); DROP TABLE foobar.foo CASCADE; | CREATE TABLE foobar.foo (id int PRIMARY KEY); DROP TABLE foobar.foo CASCADE;
124+
test1 | CREATE TABLE foobar.foo(id int primary key); DROP TABLE foobar.foo CASCADE; | CREATE TABLE foobar.foo (id int PRIMARY KEY); DROP TABLE foobar.foo CASCADE;
125+
test3 | CREATE TABLE foo(id int primary key); DROP TABLE foo CASCADE; | CREATE TABLE foo (id int PRIMARY KEY); DROP TABLE foo CASCADE;
126+
test1 | CREATE TABLE foo(id int primary key); DROP TABLE foo CASCADE; | CREATE TABLE foo (id int PRIMARY KEY); DROP TABLE foo CASCADE;
127127
test8 | DROP TABLE foobar.foo CASCADE; | DROP TABLE foobar.foo CASCADE;
128128
test7 | DROP TABLE foobar.foo CASCADE; | DROP TABLE foobar.foo CASCADE;
129129
test6 | DROP TABLE foobar.foo CASCADE; | DROP TABLE foobar.foo CASCADE;
@@ -157,18 +157,18 @@ WARNING: Unhandled deployment logged in pgl_ddl_deploy.unhandled
157157
WARNING: Unhandled deployment logged in pgl_ddl_deploy.unhandled
158158
DROP TABLE
159159
SELECT set_name, ddl_sql_raw, ddl_sql_sent FROM pgl_ddl_deploy.events ORDER BY id DESC LIMIT 10;
160-
set_name | ddl_sql_raw | ddl_sql_sent
161-
----------+----------------------------------------------+----------------------------------------------
160+
set_name | ddl_sql_raw | ddl_sql_sent
161+
----------+----------------------------------------------+-----------------------------------------------
162162
test4 | DROP TABLE foo, foobar.foo CASCADE; | DROP TABLE foo, foobar.foo CASCADE;
163163
test3 | DROP TABLE foo, foobar.foo CASCADE; | DROP TABLE foo, foobar.foo CASCADE;
164164
test2 | DROP TABLE foo, foobar.foo CASCADE; | DROP TABLE foo, foobar.foo CASCADE;
165165
test1 | DROP TABLE foo, foobar.foo CASCADE; | DROP TABLE foo, foobar.foo CASCADE;
166-
test8 | CREATE TABLE foobar.foo(id int primary key); | CREATE TABLE foobar.foo(id int primary key);
167-
test7 | CREATE TABLE foobar.foo(id int primary key); | CREATE TABLE foobar.foo(id int primary key);
168-
test6 | CREATE TABLE foobar.foo(id int primary key); | CREATE TABLE foobar.foo(id int primary key);
169-
test5 | CREATE TABLE foobar.foo(id int primary key); | CREATE TABLE foobar.foo(id int primary key);
170-
test4 | CREATE TABLE foobar.foo(id int primary key); | CREATE TABLE foobar.foo(id int primary key);
171-
test3 | CREATE TABLE foobar.foo(id int primary key); | CREATE TABLE foobar.foo(id int primary key);
166+
test8 | CREATE TABLE foobar.foo(id int primary key); | CREATE TABLE foobar.foo (id int PRIMARY KEY);
167+
test7 | CREATE TABLE foobar.foo(id int primary key); | CREATE TABLE foobar.foo (id int PRIMARY KEY);
168+
test6 | CREATE TABLE foobar.foo(id int primary key); | CREATE TABLE foobar.foo (id int PRIMARY KEY);
169+
test5 | CREATE TABLE foobar.foo(id int primary key); | CREATE TABLE foobar.foo (id int PRIMARY KEY);
170+
test4 | CREATE TABLE foobar.foo(id int primary key); | CREATE TABLE foobar.foo (id int PRIMARY KEY);
171+
test3 | CREATE TABLE foobar.foo(id int primary key); | CREATE TABLE foobar.foo (id int PRIMARY KEY);
172172
(10 rows)
173173

174174
SELECT set_name, ddl_sql_raw, command_tag, reason FROM pgl_ddl_deploy.unhandled ORDER BY id DESC LIMIT 10;

‎expected/07_edges.out‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -5,18 +5,18 @@ CREATE TABLE foo (id SERIAL PRIMARY KEY);
55
--This is an edge case that currently can't be dealt with well with filtered replication.
66
ALTER TABLE foobar.foo ADD COLUMN foo_id INT REFERENCES foo(id);
77
SELECT set_name, ddl_sql_raw, ddl_sql_sent FROM pgl_ddl_deploy.events ORDER BY id DESC LIMIT 10;
8-
set_name | ddl_sql_raw | ddl_sql_sent
9-
----------+------------------------------------------------------------------+------------------------------------------------------------------
10-
test8 | ALTER TABLE foobar.foo ADD COLUMN foo_id INT REFERENCES foo(id); | ALTER TABLE foobar.foo ADD COLUMN foo_id INT REFERENCES foo(id);
11-
test7 | ALTER TABLE foobar.foo ADD COLUMN foo_id INT REFERENCES foo(id); | ALTER TABLE foobar.foo ADD COLUMN foo_id INT REFERENCES foo(id);
12-
test6 | ALTER TABLE foobar.foo ADD COLUMN foo_id INT REFERENCES foo(id); | ALTER TABLE foobar.foo ADD COLUMN foo_id INT REFERENCES foo(id);
13-
test5 | ALTER TABLE foobar.foo ADD COLUMN foo_id INT REFERENCES foo(id); | ALTER TABLE foobar.foo ADD COLUMN foo_id INT REFERENCES foo(id);
14-
test4 | ALTER TABLE foobar.foo ADD COLUMN foo_id INT REFERENCES foo(id); | ALTER TABLE foobar.foo ADD COLUMN foo_id INT REFERENCES foo(id);
15-
test3 | ALTER TABLE foobar.foo ADD COLUMN foo_id INT REFERENCES foo(id); | ALTER TABLE foobar.foo ADD COLUMN foo_id INT REFERENCES foo(id);
16-
test2 | ALTER TABLE foobar.foo ADD COLUMN foo_id INT REFERENCES foo(id); | ALTER TABLE foobar.foo ADD COLUMN foo_id INT REFERENCES foo(id);
17-
test1 | ALTER TABLE foobar.foo ADD COLUMN foo_id INT REFERENCES foo(id); | ALTER TABLE foobar.foo ADD COLUMN foo_id INT REFERENCES foo(id);
18-
test4 | CREATE TABLE foo (id SERIAL PRIMARY KEY); | CREATE TABLE foo (id SERIAL PRIMARY KEY);
19-
test3 | CREATE TABLE foo (id SERIAL PRIMARY KEY); | CREATE TABLE foo (id SERIAL PRIMARY KEY);
8+
set_name | ddl_sql_raw | ddl_sql_sent
9+
----------+------------------------------------------------------------------+-------------------------------------------------------------------
10+
test8 | ALTER TABLE foobar.foo ADD COLUMN foo_id INT REFERENCES foo(id); | ALTER TABLE foobar.foo ADD COLUMN foo_id int REFERENCES foo (id);
11+
test7 | ALTER TABLE foobar.foo ADD COLUMN foo_id INT REFERENCES foo(id); | ALTER TABLE foobar.foo ADD COLUMN foo_id int REFERENCES foo (id);
12+
test6 | ALTER TABLE foobar.foo ADD COLUMN foo_id INT REFERENCES foo(id); | ALTER TABLE foobar.foo ADD COLUMN foo_id int REFERENCES foo (id);
13+
test5 | ALTER TABLE foobar.foo ADD COLUMN foo_id INT REFERENCES foo(id); | ALTER TABLE foobar.foo ADD COLUMN foo_id int REFERENCES foo (id);
14+
test4 | ALTER TABLE foobar.foo ADD COLUMN foo_id INT REFERENCES foo(id); | ALTER TABLE foobar.foo ADD COLUMN foo_id int REFERENCES foo (id);
15+
test3 | ALTER TABLE foobar.foo ADD COLUMN foo_id INT REFERENCES foo(id); | ALTER TABLE foobar.foo ADD COLUMN foo_id int REFERENCES foo (id);
16+
test2 | ALTER TABLE foobar.foo ADD COLUMN foo_id INT REFERENCES foo(id); | ALTER TABLE foobar.foo ADD COLUMN foo_id int REFERENCES foo (id);
17+
test1 | ALTER TABLE foobar.foo ADD COLUMN foo_id INT REFERENCES foo(id); | ALTER TABLE foobar.foo ADD COLUMN foo_id int REFERENCES foo (id);
18+
test4 | CREATE TABLE foo (id SERIAL PRIMARY KEY); | CREATE TABLE foo (id serial PRIMARY KEY);
19+
test3 | CREATE TABLE foo (id SERIAL PRIMARY KEY); | CREATE TABLE foo (id serial PRIMARY KEY);
2020
(10 rows)
2121

2222
DROP TABLE foobar.foo CASCADE;

0 commit comments

Comments
 (0)