diff --git a/autonomous_cloud_import/README.md b/autonomous_cloud_import/README.md new file mode 100644 index 0000000..f55da97 --- /dev/null +++ b/autonomous_cloud_import/README.md @@ -0,0 +1,69 @@ +# Heterogeneous Database Migration Prerequisites (PostgreSQL & MySQL) + +This repository contains prerequisite SQL objects (views) to support **optimal parallel processing** and **reliable restart capability** when migrating from **non-Oracle (heterogeneous) source databases**. + +You can place the SQL statements provided below into separate files (recommended structure included). + +--- + +## Overview + +For heterogeneous source databases, certain prerequisites must be met to enable: +- **Parallel processing** during migration, and +- **Reliable restart capability** (resume in-progress work without restarting from the beginning) + +If the required views are **not** created on the source database, the migration may **fall back to CTAS**, and any in-progress table migration may **restart from the beginning**. + +> Note: API parameters are similar to **database link parameters**. For details, refer to **Database Link Parameters** in the public documentation. + +--- + +## General Prerequisites (All Heterogeneous Sources) + +- Ensure the supplied user credentials have the required privileges to access the schema being migrated. +- The database specified by `service_name` must provide access to the target schema. + +--- + +## PostgreSQL + +### Purpose +To enable parallel processing and reliable restart capability on a PostgreSQL source database, create the required views. + +### Files (recommended) +Place the SQL into files such as: + +- `postgres/ALL_TAB_PARTITIONS.sql` +- `postgres/ALL_PART_KEY_COLUMNS.sql` +- `postgres/ALL_PART_TABLES.sql` + +### Required Views +- `ALL_TAB_PARTITIONS` +- `ALL_PART_KEY_COLUMNS` +- `ALL_PART_TABLES` + +> Add the PostgreSQL SQL definitions to the files above. + +--- + +## MySQL + +### Parameter Requirements +- For a MySQL source database, set `schema_list` to an empty array (`[]`). +- The value provided in `service_name` is used as the schema name, because MySQL does not support schemas in the same way as other databases. +- For `table_list`, specify tables using the `service_name` value as the schema name, along with the corresponding table name. + +### Purpose +To enable parallel processing and reliable restart capability on a MySQL source database, create the required views. + +### Files (recommended) +Place the SQL into files such as: + +- `mysql/ALL_PART_TABLES.sql` +- `mysql/ALL_PART_KEY_COLUMNS.sql` +- `mysql/ALL_TAB_PARTITIONS.sql` + +### Required Views +- `ALL_PART_TABLES` +- `ALL_PART_KEY_COLUMNS` +- `ALL_TAB_PARTITIONS` diff --git a/autonomous_cloud_import/mysql/ALL_PART_KEY_COLUMNS.sql b/autonomous_cloud_import/mysql/ALL_PART_KEY_COLUMNS.sql new file mode 100644 index 0000000..fe6a755 --- /dev/null +++ b/autonomous_cloud_import/mysql/ALL_PART_KEY_COLUMNS.sql @@ -0,0 +1,50 @@ +CREATE OR REPLACE VIEW ALL_PART_KEY_COLUMNS AS +WITH part_tables AS ( + -- one row per partitioned table + SELECT DISTINCT + TABLE_SCHEMA, + TABLE_NAME, + PARTITION_EXPRESSION + FROM information_schema.PARTITIONS + WHERE PARTITION_NAME IS NOT NULL + AND PARTITION_EXPRESSION IS NOT NULL +), +key_list AS ( + SELECT + TABLE_SCHEMA AS OWNER, + TABLE_NAME AS NAME, + + -- extract text inside parentheses + TRIM( + BOTH ')' + FROM TRIM( + BOTH '(' + FROM SUBSTRING_INDEX(PARTITION_EXPRESSION, '(', -1) + ) + ) AS key_list + FROM part_tables +) +SELECT + k.OWNER, + k.NAME, + 'TABLE' AS OBJECT_TYPE, + TRIM(BOTH '`' FROM + SUBSTRING_INDEX( + SUBSTRING_INDEX(k.key_list, ',', n.n), + ',', -1 + ) + ) AS COLUMN_NAME, + n.n AS COLUMN_POSITION +FROM key_list k +JOIN ( + SELECT 1 n UNION ALL SELECT 2 UNION ALL SELECT 3 UNION ALL + SELECT 4 UNION ALL SELECT 5 UNION ALL SELECT 6 UNION ALL + SELECT 7 UNION ALL SELECT 8 +) n + ON n.n <= 1 + LENGTH(k.key_list) - LENGTH(REPLACE(k.key_list, ',', '')) +-- exclude expression-based partitions +WHERE k.key_list NOT REGEXP '[()]' +ORDER BY + k.OWNER, + k.NAME, + COLUMN_POSITION; \ No newline at end of file diff --git a/autonomous_cloud_import/mysql/ALL_PART_TABLES.sql b/autonomous_cloud_import/mysql/ALL_PART_TABLES.sql new file mode 100644 index 0000000..777fd27 --- /dev/null +++ b/autonomous_cloud_import/mysql/ALL_PART_TABLES.sql @@ -0,0 +1,14 @@ +CREATE VIEW ALL_PART_TABLES AS +SELECT + TABLE_SCHEMA AS OWNER, + TABLE_NAME, + PARTITION_METHOD AS PARTITIONING_TYPE, + COUNT(DISTINCT PARTITION_NAME) AS PARTITION_COUNT, + LENGTH(PARTITION_EXPRESSION) - LENGTH(REPLACE(PARTITION_EXPRESSION, ',', '')) + 1 AS PARTITIONING_KEY_COUNT +FROM + information_schema.partitions +GROUP BY + TABLE_SCHEMA, + TABLE_NAME, + PARTITION_METHOD, + PARTITION_EXPRESSION; \ No newline at end of file diff --git a/autonomous_cloud_import/mysql/ALL_TAB_PARTITIONS.sql b/autonomous_cloud_import/mysql/ALL_TAB_PARTITIONS.sql new file mode 100644 index 0000000..2215e7d --- /dev/null +++ b/autonomous_cloud_import/mysql/ALL_TAB_PARTITIONS.sql @@ -0,0 +1,32 @@ +CREATE OR REPLACE VIEW ALL_TAB_PARTITIONS AS +SELECT + TABLE_SCHEMA AS TABLE_OWNER, + TABLE_NAME, + PARTITION_NAME, + + MIN(PARTITION_ORDINAL_POSITION) AS PARTITION_POSITION, + + CASE + WHEN MAX(PARTITION_DESCRIPTION) = 'MAXVALUE' + THEN 'MAXVALUE' + ELSE TRIM(BOTH '''' FROM MAX(PARTITION_DESCRIPTION)) + END AS HIGH_VALUE, + + LENGTH( + CASE + WHEN MAX(PARTITION_DESCRIPTION) = 'MAXVALUE' + THEN 'MAXVALUE' + ELSE TRIM(BOTH '''' FROM MAX(PARTITION_DESCRIPTION)) + END + ) AS HIGH_VALUE_LENGTH, + + MAX(PARTITION_METHOD) AS PARTITIONING_TYPE, + + 1 AS PARTITION_KEY_COUNT + +FROM information_schema.PARTITIONS +WHERE PARTITION_NAME IS NOT NULL +GROUP BY + TABLE_SCHEMA, + TABLE_NAME, + PARTITION_NAME; \ No newline at end of file diff --git a/autonomous_cloud_import/postgres/ALL_PART_KEY_COLUMNS.sql b/autonomous_cloud_import/postgres/ALL_PART_KEY_COLUMNS.sql new file mode 100644 index 0000000..4a9cefc --- /dev/null +++ b/autonomous_cloud_import/postgres/ALL_PART_KEY_COLUMNS.sql @@ -0,0 +1,27 @@ +CREATE OR REPLACE VIEW "ALL_PART_KEY_COLUMNS" AS +SELECT + ns.nspname AS "OWNER", + c.relname AS "NAME", + 'TABLE' AS "OBJECT_TYPE", + a.attname AS "COLUMN_NAME", + ord.ordinality AS "COLUMN_POSITION" + +FROM pg_partitioned_table pt +JOIN pg_class c + ON c.oid = pt.partrelid +JOIN pg_namespace ns + ON ns.oid = c.relnamespace + +-- explode partition key column numbers +JOIN LATERAL unnest(pt.partattrs) + WITH ORDINALITY AS ord(attnum, ordinality) + ON true + +JOIN pg_attribute a + ON a.attrelid = c.oid + AND a.attnum = ord.attnum + +ORDER BY + "OWNER", + "NAME", + "COLUMN_POSITION"; \ No newline at end of file diff --git a/autonomous_cloud_import/postgres/ALL_PART_TABLES.sql b/autonomous_cloud_import/postgres/ALL_PART_TABLES.sql new file mode 100644 index 0000000..4d62c72 --- /dev/null +++ b/autonomous_cloud_import/postgres/ALL_PART_TABLES.sql @@ -0,0 +1,47 @@ +CREATE OR REPLACE VIEW "ALL_PART_TABLES" AS +WITH "PARTITION_DETAILS" AS ( + SELECT + parent_ns.nspname AS "OWNER", + parent.relname AS "TABLE_NAME", + parent.oid AS "PARENT_OID", + count(child.oid) AS "PARTITION_COUNT", -- Count partitions + pg_partitioned_table.partstrat AS "PARTSTRAT", -- Partition strategy (RANGE, LIST, HASH) + pg_partitioned_table.partattrs AS "PARTITION_COLUMNS" -- Column numbers involved in partitioning + FROM + pg_inherits i + JOIN pg_class child ON child.oid = i.inhrelid + JOIN pg_class parent ON parent.oid = i.inhparent + JOIN pg_namespace parent_ns ON parent.relnamespace = parent_ns.oid + JOIN pg_partitioned_table ON parent.oid = pg_partitioned_table.partrelid + WHERE + parent.relkind = 'p' -- Only partitioned tables + GROUP BY + parent_ns.nspname, parent.relname, parent.oid, pg_partitioned_table.partstrat, pg_partitioned_table.partattrs +), +"PARTITIONING_KEYS" AS ( + SELECT + ppt.partrelid AS "PARENT_OID", + -- Count the number of partitioning columns by counting entries in partattrs + array_length(ppt.partattrs, 1) AS "PARTITIONING_KEY_COUNT" + FROM + pg_partitioned_table ppt + WHERE + ppt.partattrs IS NOT NULL -- Only count if there are partition columns +) +SELECT + pd."OWNER", + pd."TABLE_NAME", + pd."PARTITION_COUNT", + pk."PARTITIONING_KEY_COUNT", + CASE + WHEN pd."PARTSTRAT" = 'r' THEN 'RANGE' + WHEN pd."PARTSTRAT" = 'l' THEN 'LIST' + WHEN pd."PARTSTRAT" = 'h' THEN 'HASH' + ELSE 'UNKNOWN' + END AS "PARTITIONING_TYPE" +FROM + "PARTITION_DETAILS" pd +JOIN + "PARTITIONING_KEYS" pk ON pd."PARENT_OID" = pk."PARENT_OID" +ORDER BY + pd."OWNER", pd."TABLE_NAME"; \ No newline at end of file diff --git a/autonomous_cloud_import/postgres/ALL_TAB_PARTITIONS.sql b/autonomous_cloud_import/postgres/ALL_TAB_PARTITIONS.sql new file mode 100644 index 0000000..9a5c801 --- /dev/null +++ b/autonomous_cloud_import/postgres/ALL_TAB_PARTITIONS.sql @@ -0,0 +1,62 @@ +CREATE OR REPLACE VIEW "ALL_TAB_PARTITIONS" AS +SELECT + ns.nspname AS "TABLE_OWNER", + parent.relname AS "TABLE_NAME", + child.relname AS "PARTITION_NAME", + + -- Oracle-style partition position + ROW_NUMBER() OVER ( + PARTITION BY parent.oid + ORDER BY child.oid + ) AS "PARTITION_POSITION", + + -- Oracle-style HIGH_VALUE (upper bound only, no quotes) + CASE + WHEN pg_get_expr(child.relpartbound, child.oid) ~ 'TO \(MAXVALUE\)' + THEN 'MAXVALUE' + ELSE trim( + BOTH '''' + FROM regexp_replace( + pg_get_expr(child.relpartbound, child.oid), + '.*TO \((.*)\)$', + '\1' + ) + ) + END AS "HIGH_VALUE", + + -- Length of HIGH_VALUE (Oracle compatibility) + LENGTH( + CASE + WHEN pg_get_expr(child.relpartbound, child.oid) ~ 'TO \(MAXVALUE\)' + THEN 'MAXVALUE' + ELSE trim( + BOTH '''' + FROM regexp_replace( + pg_get_expr(child.relpartbound, child.oid), + '.*TO \((.*)\)$', + '\1' + ) + ) + END + ) AS "HIGH_VALUE_LENGTH", + + -- Partitioning type (Oracle naming) + CASE pt.partstrat + WHEN 'r' THEN 'RANGE' + WHEN 'l' THEN 'LIST' + WHEN 'h' THEN 'HASH' + END AS "PARTITIONING_TYPE", + + -- Number of partition key columns + pt.partnatts AS "PARTITION_KEY_COUNT" + +FROM pg_inherits i +JOIN pg_class parent + ON parent.oid = i.inhparent +JOIN pg_class child + ON child.oid = i.inhrelid +JOIN pg_namespace ns + ON ns.oid = parent.relnamespace +JOIN pg_partitioned_table pt + ON pt.partrelid = parent.oid +WHERE parent.relkind = 'p'; \ No newline at end of file