diff --git a/autonomous_cloud_import/README.md b/autonomous_cloud_import/README.md index f55da97..f775bac 100644 --- a/autonomous_cloud_import/README.md +++ b/autonomous_cloud_import/README.md @@ -2,8 +2,6 @@ This repository contains prerequisite SQL objects (views) to support **optimal parallel processing** and **reliable restart capability** when migrating from **non-Oracle (heterogeneous) source databases**. -You can place the SQL statements provided below into separate files (recommended structure included). - --- ## Overview @@ -30,19 +28,14 @@ If the required views are **not** created on the source database, the migration ### Purpose To enable parallel processing and reliable restart capability on a PostgreSQL source database, create the required views. -### Files (recommended) -Place the SQL into files such as: - -- `postgres/ALL_TAB_PARTITIONS.sql` -- `postgres/ALL_PART_KEY_COLUMNS.sql` -- `postgres/ALL_PART_TABLES.sql` +### Files +- `postgres/views.sql` ### Required Views - `ALL_TAB_PARTITIONS` - `ALL_PART_KEY_COLUMNS` - `ALL_PART_TABLES` -> Add the PostgreSQL SQL definitions to the files above. --- @@ -56,12 +49,8 @@ Place the SQL into files such as: ### Purpose To enable parallel processing and reliable restart capability on a MySQL source database, create the required views. -### Files (recommended) -Place the SQL into files such as: - -- `mysql/ALL_PART_TABLES.sql` -- `mysql/ALL_PART_KEY_COLUMNS.sql` -- `mysql/ALL_TAB_PARTITIONS.sql` +### Files +- `mysql/views.sql` ### Required Views - `ALL_PART_TABLES` diff --git a/autonomous_cloud_import/mysql/ALL_PART_KEY_COLUMNS.sql b/autonomous_cloud_import/mysql/ALL_PART_KEY_COLUMNS.sql deleted file mode 100644 index fe6a755..0000000 --- a/autonomous_cloud_import/mysql/ALL_PART_KEY_COLUMNS.sql +++ /dev/null @@ -1,50 +0,0 @@ -CREATE OR REPLACE VIEW ALL_PART_KEY_COLUMNS AS -WITH part_tables AS ( - -- one row per partitioned table - SELECT DISTINCT - TABLE_SCHEMA, - TABLE_NAME, - PARTITION_EXPRESSION - FROM information_schema.PARTITIONS - WHERE PARTITION_NAME IS NOT NULL - AND PARTITION_EXPRESSION IS NOT NULL -), -key_list AS ( - SELECT - TABLE_SCHEMA AS OWNER, - TABLE_NAME AS NAME, - - -- extract text inside parentheses - TRIM( - BOTH ')' - FROM TRIM( - BOTH '(' - FROM SUBSTRING_INDEX(PARTITION_EXPRESSION, '(', -1) - ) - ) AS key_list - FROM part_tables -) -SELECT - k.OWNER, - k.NAME, - 'TABLE' AS OBJECT_TYPE, - TRIM(BOTH '`' FROM - SUBSTRING_INDEX( - SUBSTRING_INDEX(k.key_list, ',', n.n), - ',', -1 - ) - ) AS COLUMN_NAME, - n.n AS COLUMN_POSITION -FROM key_list k -JOIN ( - SELECT 1 n UNION ALL SELECT 2 UNION ALL SELECT 3 UNION ALL - SELECT 4 UNION ALL SELECT 5 UNION ALL SELECT 6 UNION ALL - SELECT 7 UNION ALL SELECT 8 -) n - ON n.n <= 1 + LENGTH(k.key_list) - LENGTH(REPLACE(k.key_list, ',', '')) --- exclude expression-based partitions -WHERE k.key_list NOT REGEXP '[()]' -ORDER BY - k.OWNER, - k.NAME, - COLUMN_POSITION; \ No newline at end of file diff --git a/autonomous_cloud_import/mysql/ALL_PART_TABLES.sql b/autonomous_cloud_import/mysql/ALL_PART_TABLES.sql deleted file mode 100644 index 777fd27..0000000 --- a/autonomous_cloud_import/mysql/ALL_PART_TABLES.sql +++ /dev/null @@ -1,14 +0,0 @@ -CREATE VIEW ALL_PART_TABLES AS -SELECT - TABLE_SCHEMA AS OWNER, - TABLE_NAME, - PARTITION_METHOD AS PARTITIONING_TYPE, - COUNT(DISTINCT PARTITION_NAME) AS PARTITION_COUNT, - LENGTH(PARTITION_EXPRESSION) - LENGTH(REPLACE(PARTITION_EXPRESSION, ',', '')) + 1 AS PARTITIONING_KEY_COUNT -FROM - information_schema.partitions -GROUP BY - TABLE_SCHEMA, - TABLE_NAME, - PARTITION_METHOD, - PARTITION_EXPRESSION; \ No newline at end of file diff --git a/autonomous_cloud_import/mysql/ALL_TAB_PARTITIONS.sql b/autonomous_cloud_import/mysql/ALL_TAB_PARTITIONS.sql deleted file mode 100644 index 2215e7d..0000000 --- a/autonomous_cloud_import/mysql/ALL_TAB_PARTITIONS.sql +++ /dev/null @@ -1,32 +0,0 @@ -CREATE OR REPLACE VIEW ALL_TAB_PARTITIONS AS -SELECT - TABLE_SCHEMA AS TABLE_OWNER, - TABLE_NAME, - PARTITION_NAME, - - MIN(PARTITION_ORDINAL_POSITION) AS PARTITION_POSITION, - - CASE - WHEN MAX(PARTITION_DESCRIPTION) = 'MAXVALUE' - THEN 'MAXVALUE' - ELSE TRIM(BOTH '''' FROM MAX(PARTITION_DESCRIPTION)) - END AS HIGH_VALUE, - - LENGTH( - CASE - WHEN MAX(PARTITION_DESCRIPTION) = 'MAXVALUE' - THEN 'MAXVALUE' - ELSE TRIM(BOTH '''' FROM MAX(PARTITION_DESCRIPTION)) - END - ) AS HIGH_VALUE_LENGTH, - - MAX(PARTITION_METHOD) AS PARTITIONING_TYPE, - - 1 AS PARTITION_KEY_COUNT - -FROM information_schema.PARTITIONS -WHERE PARTITION_NAME IS NOT NULL -GROUP BY - TABLE_SCHEMA, - TABLE_NAME, - PARTITION_NAME; \ No newline at end of file diff --git a/autonomous_cloud_import/mysql/views.sql b/autonomous_cloud_import/mysql/views.sql new file mode 100644 index 0000000..4cf350b --- /dev/null +++ b/autonomous_cloud_import/mysql/views.sql @@ -0,0 +1,90 @@ +-- MySQL view creation script. +-- Execute with: mysql < views.sql + +CREATE OR REPLACE VIEW ALL_PART_KEY_COLUMNS AS +WITH part_tables AS ( + SELECT DISTINCT + TABLE_SCHEMA, + TABLE_NAME, + PARTITION_EXPRESSION + FROM information_schema.PARTITIONS + WHERE PARTITION_NAME IS NOT NULL + AND PARTITION_EXPRESSION IS NOT NULL +), +key_list AS ( + SELECT + TABLE_SCHEMA AS OWNER, + TABLE_NAME AS NAME, + TRIM( + BOTH ')' + FROM TRIM( + BOTH '(' + FROM SUBSTRING_INDEX(PARTITION_EXPRESSION, '(', -1) + ) + ) AS key_list + FROM part_tables +) +SELECT + k.OWNER, + k.NAME, + 'TABLE' AS OBJECT_TYPE, + TRIM(BOTH '`' FROM + SUBSTRING_INDEX( + SUBSTRING_INDEX(k.key_list, ',', n.n), + ',', -1 + ) + ) AS COLUMN_NAME, + n.n AS COLUMN_POSITION +FROM key_list k +JOIN ( + SELECT 1 n UNION ALL SELECT 2 UNION ALL SELECT 3 UNION ALL + SELECT 4 UNION ALL SELECT 5 UNION ALL SELECT 6 UNION ALL + SELECT 7 UNION ALL SELECT 8 +) n + ON n.n <= 1 + LENGTH(k.key_list) - LENGTH(REPLACE(k.key_list, ',', '')) +WHERE k.key_list NOT REGEXP '[()]' +ORDER BY + k.OWNER, + k.NAME, + COLUMN_POSITION; + +CREATE OR REPLACE VIEW ALL_PART_TABLES AS +SELECT + TABLE_SCHEMA AS OWNER, + TABLE_NAME, + PARTITION_METHOD AS PARTITIONING_TYPE, + COUNT(DISTINCT PARTITION_NAME) AS PARTITION_COUNT, + LENGTH(PARTITION_EXPRESSION) - LENGTH(REPLACE(PARTITION_EXPRESSION, ',', '')) + 1 AS PARTITIONING_KEY_COUNT +FROM information_schema.partitions +GROUP BY + TABLE_SCHEMA, + TABLE_NAME, + PARTITION_METHOD, + PARTITION_EXPRESSION; + +CREATE OR REPLACE VIEW ALL_TAB_PARTITIONS AS +SELECT + TABLE_SCHEMA AS TABLE_OWNER, + TABLE_NAME, + PARTITION_NAME, + MIN(PARTITION_ORDINAL_POSITION) AS PARTITION_POSITION, + CASE + WHEN MAX(PARTITION_DESCRIPTION) = 'MAXVALUE' + THEN 'MAXVALUE' + ELSE TRIM(BOTH '''' FROM MAX(PARTITION_DESCRIPTION)) + END AS HIGH_VALUE, + LENGTH( + CASE + WHEN MAX(PARTITION_DESCRIPTION) = 'MAXVALUE' + THEN 'MAXVALUE' + ELSE TRIM(BOTH '''' FROM MAX(PARTITION_DESCRIPTION)) + END + ) AS HIGH_VALUE_LENGTH, + MAX(PARTITION_METHOD) AS PARTITIONING_TYPE, + 1 AS PARTITION_KEY_COUNT +FROM information_schema.PARTITIONS +WHERE PARTITION_NAME IS NOT NULL +GROUP BY + TABLE_SCHEMA, + TABLE_NAME, + PARTITION_NAME; diff --git a/autonomous_cloud_import/postgres/ALL_PART_KEY_COLUMNS.sql b/autonomous_cloud_import/postgres/ALL_PART_KEY_COLUMNS.sql deleted file mode 100644 index 4a9cefc..0000000 --- a/autonomous_cloud_import/postgres/ALL_PART_KEY_COLUMNS.sql +++ /dev/null @@ -1,27 +0,0 @@ -CREATE OR REPLACE VIEW "ALL_PART_KEY_COLUMNS" AS -SELECT - ns.nspname AS "OWNER", - c.relname AS "NAME", - 'TABLE' AS "OBJECT_TYPE", - a.attname AS "COLUMN_NAME", - ord.ordinality AS "COLUMN_POSITION" - -FROM pg_partitioned_table pt -JOIN pg_class c - ON c.oid = pt.partrelid -JOIN pg_namespace ns - ON ns.oid = c.relnamespace - --- explode partition key column numbers -JOIN LATERAL unnest(pt.partattrs) - WITH ORDINALITY AS ord(attnum, ordinality) - ON true - -JOIN pg_attribute a - ON a.attrelid = c.oid - AND a.attnum = ord.attnum - -ORDER BY - "OWNER", - "NAME", - "COLUMN_POSITION"; \ No newline at end of file diff --git a/autonomous_cloud_import/postgres/ALL_PART_TABLES.sql b/autonomous_cloud_import/postgres/ALL_PART_TABLES.sql deleted file mode 100644 index 4d62c72..0000000 --- a/autonomous_cloud_import/postgres/ALL_PART_TABLES.sql +++ /dev/null @@ -1,47 +0,0 @@ -CREATE OR REPLACE VIEW "ALL_PART_TABLES" AS -WITH "PARTITION_DETAILS" AS ( - SELECT - parent_ns.nspname AS "OWNER", - parent.relname AS "TABLE_NAME", - parent.oid AS "PARENT_OID", - count(child.oid) AS "PARTITION_COUNT", -- Count partitions - pg_partitioned_table.partstrat AS "PARTSTRAT", -- Partition strategy (RANGE, LIST, HASH) - pg_partitioned_table.partattrs AS "PARTITION_COLUMNS" -- Column numbers involved in partitioning - FROM - pg_inherits i - JOIN pg_class child ON child.oid = i.inhrelid - JOIN pg_class parent ON parent.oid = i.inhparent - JOIN pg_namespace parent_ns ON parent.relnamespace = parent_ns.oid - JOIN pg_partitioned_table ON parent.oid = pg_partitioned_table.partrelid - WHERE - parent.relkind = 'p' -- Only partitioned tables - GROUP BY - parent_ns.nspname, parent.relname, parent.oid, pg_partitioned_table.partstrat, pg_partitioned_table.partattrs -), -"PARTITIONING_KEYS" AS ( - SELECT - ppt.partrelid AS "PARENT_OID", - -- Count the number of partitioning columns by counting entries in partattrs - array_length(ppt.partattrs, 1) AS "PARTITIONING_KEY_COUNT" - FROM - pg_partitioned_table ppt - WHERE - ppt.partattrs IS NOT NULL -- Only count if there are partition columns -) -SELECT - pd."OWNER", - pd."TABLE_NAME", - pd."PARTITION_COUNT", - pk."PARTITIONING_KEY_COUNT", - CASE - WHEN pd."PARTSTRAT" = 'r' THEN 'RANGE' - WHEN pd."PARTSTRAT" = 'l' THEN 'LIST' - WHEN pd."PARTSTRAT" = 'h' THEN 'HASH' - ELSE 'UNKNOWN' - END AS "PARTITIONING_TYPE" -FROM - "PARTITION_DETAILS" pd -JOIN - "PARTITIONING_KEYS" pk ON pd."PARENT_OID" = pk."PARENT_OID" -ORDER BY - pd."OWNER", pd."TABLE_NAME"; \ No newline at end of file diff --git a/autonomous_cloud_import/postgres/ALL_TAB_PARTITIONS.sql b/autonomous_cloud_import/postgres/ALL_TAB_PARTITIONS.sql deleted file mode 100644 index 9a5c801..0000000 --- a/autonomous_cloud_import/postgres/ALL_TAB_PARTITIONS.sql +++ /dev/null @@ -1,62 +0,0 @@ -CREATE OR REPLACE VIEW "ALL_TAB_PARTITIONS" AS -SELECT - ns.nspname AS "TABLE_OWNER", - parent.relname AS "TABLE_NAME", - child.relname AS "PARTITION_NAME", - - -- Oracle-style partition position - ROW_NUMBER() OVER ( - PARTITION BY parent.oid - ORDER BY child.oid - ) AS "PARTITION_POSITION", - - -- Oracle-style HIGH_VALUE (upper bound only, no quotes) - CASE - WHEN pg_get_expr(child.relpartbound, child.oid) ~ 'TO \(MAXVALUE\)' - THEN 'MAXVALUE' - ELSE trim( - BOTH '''' - FROM regexp_replace( - pg_get_expr(child.relpartbound, child.oid), - '.*TO \((.*)\)$', - '\1' - ) - ) - END AS "HIGH_VALUE", - - -- Length of HIGH_VALUE (Oracle compatibility) - LENGTH( - CASE - WHEN pg_get_expr(child.relpartbound, child.oid) ~ 'TO \(MAXVALUE\)' - THEN 'MAXVALUE' - ELSE trim( - BOTH '''' - FROM regexp_replace( - pg_get_expr(child.relpartbound, child.oid), - '.*TO \((.*)\)$', - '\1' - ) - ) - END - ) AS "HIGH_VALUE_LENGTH", - - -- Partitioning type (Oracle naming) - CASE pt.partstrat - WHEN 'r' THEN 'RANGE' - WHEN 'l' THEN 'LIST' - WHEN 'h' THEN 'HASH' - END AS "PARTITIONING_TYPE", - - -- Number of partition key columns - pt.partnatts AS "PARTITION_KEY_COUNT" - -FROM pg_inherits i -JOIN pg_class parent - ON parent.oid = i.inhparent -JOIN pg_class child - ON child.oid = i.inhrelid -JOIN pg_namespace ns - ON ns.oid = parent.relnamespace -JOIN pg_partitioned_table pt - ON pt.partrelid = parent.oid -WHERE parent.relkind = 'p'; \ No newline at end of file diff --git a/autonomous_cloud_import/postgres/views.sql b/autonomous_cloud_import/postgres/views.sql new file mode 100644 index 0000000..d1a5746 --- /dev/null +++ b/autonomous_cloud_import/postgres/views.sql @@ -0,0 +1,128 @@ +-- PostgreSQL view creation script. +-- Execute with: psql -f views.sql + +CREATE OR REPLACE VIEW "ALL_PART_KEY_COLUMNS" AS +SELECT + ns.nspname AS "OWNER", + c.relname AS "NAME", + 'TABLE' AS "OBJECT_TYPE", + a.attname AS "COLUMN_NAME", + ord.ordinality AS "COLUMN_POSITION" +FROM pg_partitioned_table pt +JOIN pg_class c + ON c.oid = pt.partrelid +JOIN pg_namespace ns + ON ns.oid = c.relnamespace +JOIN LATERAL unnest(pt.partattrs) + WITH ORDINALITY AS ord(attnum, ordinality) + ON true +JOIN pg_attribute a + ON a.attrelid = c.oid + AND a.attnum = ord.attnum +ORDER BY + "OWNER", + "NAME", + "COLUMN_POSITION"; + +CREATE OR REPLACE VIEW "ALL_PART_TABLES" AS +WITH "PARTITION_DETAILS" AS ( + SELECT + parent_ns.nspname AS "OWNER", + parent.relname AS "TABLE_NAME", + parent.oid AS "PARENT_OID", + count(child.oid) AS "PARTITION_COUNT", + pg_partitioned_table.partstrat AS "PARTSTRAT", + pg_partitioned_table.partattrs AS "PARTITION_COLUMNS" + FROM pg_inherits i + JOIN pg_class child + ON child.oid = i.inhrelid + JOIN pg_class parent + ON parent.oid = i.inhparent + JOIN pg_namespace parent_ns + ON parent.relnamespace = parent_ns.oid + JOIN pg_partitioned_table + ON parent.oid = pg_partitioned_table.partrelid + WHERE parent.relkind = 'p' + GROUP BY + parent_ns.nspname, + parent.relname, + parent.oid, + pg_partitioned_table.partstrat, + pg_partitioned_table.partattrs +), +"PARTITIONING_KEYS" AS ( + SELECT + ppt.partrelid AS "PARENT_OID", + array_length(ppt.partattrs, 1) AS "PARTITIONING_KEY_COUNT" + FROM pg_partitioned_table ppt + WHERE ppt.partattrs IS NOT NULL +) +SELECT + pd."OWNER", + pd."TABLE_NAME", + pd."PARTITION_COUNT", + pk."PARTITIONING_KEY_COUNT", + CASE + WHEN pd."PARTSTRAT" = 'r' THEN 'RANGE' + WHEN pd."PARTSTRAT" = 'l' THEN 'LIST' + WHEN pd."PARTSTRAT" = 'h' THEN 'HASH' + ELSE 'UNKNOWN' + END AS "PARTITIONING_TYPE" +FROM "PARTITION_DETAILS" pd +JOIN "PARTITIONING_KEYS" pk + ON pd."PARENT_OID" = pk."PARENT_OID" +ORDER BY + pd."OWNER", + pd."TABLE_NAME"; + +CREATE OR REPLACE VIEW "ALL_TAB_PARTITIONS" AS +SELECT + ns.nspname AS "TABLE_OWNER", + parent.relname AS "TABLE_NAME", + child.relname AS "PARTITION_NAME", + ROW_NUMBER() OVER ( + PARTITION BY parent.oid + ORDER BY child.oid + ) AS "PARTITION_POSITION", + CASE + WHEN pg_get_expr(child.relpartbound, child.oid) ~ 'TO \(MAXVALUE\)' + THEN 'MAXVALUE' + ELSE trim( + BOTH '''' + FROM regexp_replace( + pg_get_expr(child.relpartbound, child.oid), + '.*TO \((.*)\)$', + '\1' + ) + ) + END AS "HIGH_VALUE", + LENGTH( + CASE + WHEN pg_get_expr(child.relpartbound, child.oid) ~ 'TO \(MAXVALUE\)' + THEN 'MAXVALUE' + ELSE trim( + BOTH '''' + FROM regexp_replace( + pg_get_expr(child.relpartbound, child.oid), + '.*TO \((.*)\)$', + '\1' + ) + ) + END + ) AS "HIGH_VALUE_LENGTH", + CASE pt.partstrat + WHEN 'r' THEN 'RANGE' + WHEN 'l' THEN 'LIST' + WHEN 'h' THEN 'HASH' + END AS "PARTITIONING_TYPE", + pt.partnatts AS "PARTITION_KEY_COUNT" +FROM pg_inherits i +JOIN pg_class parent + ON parent.oid = i.inhparent +JOIN pg_class child + ON child.oid = i.inhrelid +JOIN pg_namespace ns + ON ns.oid = parent.relnamespace +JOIN pg_partitioned_table pt + ON pt.partrelid = parent.oid +WHERE parent.relkind = 'p';