Merge pull request #2407 from flow-php/migrations-exclusion-policy

feature: postgresql bundle - migrations exclusion policy

Author: norberttech

documentation/components/bridges/symfony-postgresql-bundle.md

@@ -163,6 +163,69 @@ flow_postgresql:
 
 Multiple catalog providers can be combined. They are merged via `ChainCatalogProvider` into a single catalog.
 
+### Exclusions
+
+By default `flow:migrations:diff` compares **every** object found in the database against the catalog, so any
+object that is not described by a catalog provider is reported as a difference to drop. That is a problem when some
+objects are created outside of migrations — for example tables generated dynamically from user-uploaded content, or
+a whole schema owned by another system. List those objects under `migrations.exclude` and they are skipped while the
+diff is generated (the migrations tracking table is always excluded automatically).
+
+```yaml
+flow_postgresql:
+  migrations:
+    enabled: true
+    exclude:
+      - { schema: tenant_data }                  # exclude an entire schema and everything in it
+      - { table: legacy_audit }                  # exclude a table by exact name
+      - { starts_with: user_upload_ }            # exclude objects whose name starts with a prefix
+      - { ends_with: _tmp, type: view }          # ...ending with a suffix, narrowed to views
+      - { pattern: '/^cache_\d+$/', type: sequence } # ...matching a PCRE pattern, narrowed to sequences
+      - { exact: scratch, for_schema: staging }  # exact name, narrowed to the "staging" schema
+      - { policy_id: app.my_exclusion_policy }   # delegate to a custom ExclusionPolicy service
+```
+
+Each entry defines **exactly one** matcher:
+
+| Key           | Excludes                                                            |
+|---------------|--------------------------------------------------------------------|
+| `schema`      | The named schema and every object inside it (tables, views, sequences, functions, …) |
+| `table`       | A table by exact name (shorthand for `exact` narrowed to tables)   |
+| `exact`       | Any object whose name matches exactly                              |
+| `starts_with` | Any object whose name starts with the given prefix                 |
+| `ends_with`   | Any object whose name ends with the given suffix                   |
+| `pattern`     | Any object whose name matches the given PCRE pattern (with delimiters) |
+| `policy_id`   | Delegates to a service implementing `Flow\PostgreSql\Schema\Exclusion\ExclusionPolicy` |
+
+The `exact`, `starts_with`, `ends_with`, `pattern` and `policy_id` matchers can be narrowed with optional scopes:
+
+- `type` — limit to a single object type: `table`, `view`, `materialized_view`, `sequence`, `function`,
+  `procedure`, `domain` or `extension`. When omitted, the matcher applies to every type.
+- `for_schema` — limit to a single schema. When omitted, the matcher applies to every schema.
+
+Tables and whole schemas are filtered before they are introspected, so excluding dynamically created tables also
+avoids the cost of reading their columns, indexes and constraints on every diff.
+
+A custom `policy_id` service implements the same interface and receives every candidate object:
+
+```php
+<?php
+
+namespace App\Database;
+
+use Flow\PostgreSql\Schema\Exclusion\ExclusionPolicy;
+use Flow\PostgreSql\Schema\Exclusion\SchemaObject;
+
+final class MyExclusionPolicy implements ExclusionPolicy
+{
+    public function exclude(SchemaObject $object): bool
+    {
+        // $object->type, $object->schema and $object->name are available
+        return str_contains($object->name ?? '', '__generated__');
+    }
+}
+```
+
 ## Console Commands
 
 ### Database Commands

src/bridge/symfony/postgresql-bundle/src/Flow/Bridge/Symfony/PostgreSqlBundle/FlowPostgreSqlBundle.php

@@ -38,6 +38,14 @@
 use Flow\PostgreSql\Migrations\Store\MigrationStore;
 use Flow\PostgreSql\Migrations\VersionGenerator\TimestampVersionGenerator;
 use Flow\PostgreSql\Migrations\VersionResolver;
+use Flow\PostgreSql\Schema\Exclusion\AnyExclusionPolicy;
+use Flow\PostgreSql\Schema\Exclusion\EndsWithExclusionPolicy;
+use Flow\PostgreSql\Schema\Exclusion\ExactMatchExclusionPolicy;
+use Flow\PostgreSql\Schema\Exclusion\PatternExclusionPolicy;
+use Flow\PostgreSql\Schema\Exclusion\ScopedExclusionPolicy;
+use Flow\PostgreSql\Schema\Exclusion\SchemaObjectType;
+use Flow\PostgreSql\Schema\Exclusion\StartsWithExclusionPolicy;
+use Flow\PostgreSql\Schema\Exclusion\WholeSchemaExclusionPolicy;
 use Flow\Telemetry\Provider\Clock\SystemClock;
 use LogicException;
 use Override;
@@ -324,6 +332,50 @@ public function configure(DefinitionConfigurator $definition): void
             ->defaultTrue()
             ->info('Generate rollback files when creating migrations (default: true)')
             ->end()
+            ->arrayNode('exclude')
+            ->info('Schema objects excluded from migration diffing (e.g. tables created dynamically at runtime). Each entry defines exactly one matcher: schema, table, exact, starts_with, ends_with, pattern or policy_id.')
+            ->arrayPrototype()
+            ->children()
+            ->scalarNode('schema')
+            ->defaultNull()
+            ->info('Exclude an entire schema with all of its objects.')
+            ->end()
+            ->scalarNode('table')
+            ->defaultNull()
+            ->info('Exclude a table by exact name (shorthand for exact scoped to tables).')
+            ->end()
+            ->scalarNode('exact')
+            ->defaultNull()
+            ->info('Exclude any object whose name matches exactly.')
+            ->end()
+            ->scalarNode('starts_with')
+            ->defaultNull()
+            ->info('Exclude objects whose name starts with this prefix.')
+            ->end()
+            ->scalarNode('ends_with')
+            ->defaultNull()
+            ->info('Exclude objects whose name ends with this suffix.')
+            ->end()
+            ->scalarNode('pattern')
+            ->defaultNull()
+            ->info('Exclude objects whose name matches this PCRE pattern (with delimiters).')
+            ->end()
+            ->scalarNode('policy_id')
+            ->defaultNull()
+            ->info('Service ID of a custom Flow\\PostgreSql\\Schema\\Exclusion\\ExclusionPolicy.')
+            ->end()
+            ->enumNode('type')
+            ->values(['table', 'view', 'materialized_view', 'sequence', 'function', 'procedure', 'domain', 'extension'])
+            ->defaultNull()
+            ->info('Narrow the matcher to a single object type (ignored when using "schema" or "table").')
+            ->end()
+            ->scalarNode('for_schema')
+            ->defaultNull()
+            ->info('Narrow the matcher to a single schema.')
+            ->end()
+            ->end()
+            ->end()
+            ->end()
             ->end()
             ->end()
             ->arrayNode('catalog_providers')
@@ -347,7 +399,7 @@ public function configure(DefinitionConfigurator $definition): void
     }
 
     /**
-     * @param array{connections: array<string, array{dsn: string, test_transaction_rollback: bool, context?: array<string, mixed>, telemetry?: array{service_id: string, clock_service_id: ?string, trace_queries: bool, trace_transactions: bool, collect_metrics: bool, log_queries: bool, max_query_length: int, include_parameters: bool, max_parameters: int, max_parameter_length: int}}>, messenger: array{enabled: bool, table_name: string, schema: string}, cache: array{pools?: array<string, array{connection: ?string, table_name: string, schema: string, id_col: string, data_col: string, lifetime_col: string, time_col: string, namespace: string, default_lifetime: int, marshaller_service_id: ?string, share_connection: bool}>}, session: array{enabled: bool, connection: ?string, table_name: string, schema: string, id_col: string, data_col: string, lifetime_col: string, time_col: string, lock_mode: string, ttl: ?int, share_connection: bool}, migrations: array{enabled: bool, directory: string, namespace: string, table_name: string, table_schema: string, migration_file_name: string, rollback_file_name: string, all_or_nothing: bool, generate_rollback: bool}, catalog_providers: list<array{catalog_provider_id: ?string, catalog: ?array<string, mixed>}>} $config
+     * @param array{connections: array<string, array{dsn: string, test_transaction_rollback: bool, context?: array<string, mixed>, telemetry?: array{service_id: string, clock_service_id: ?string, trace_queries: bool, trace_transactions: bool, collect_metrics: bool, log_queries: bool, max_query_length: int, include_parameters: bool, max_parameters: int, max_parameter_length: int}}>, messenger: array{enabled: bool, table_name: string, schema: string}, cache: array{pools?: array<string, array{connection: ?string, table_name: string, schema: string, id_col: string, data_col: string, lifetime_col: string, time_col: string, namespace: string, default_lifetime: int, marshaller_service_id: ?string, share_connection: bool}>}, session: array{enabled: bool, connection: ?string, table_name: string, schema: string, id_col: string, data_col: string, lifetime_col: string, time_col: string, lock_mode: string, ttl: ?int, share_connection: bool}, migrations: array{enabled: bool, directory: string, namespace: string, table_name: string, table_schema: string, migration_file_name: string, rollback_file_name: string, all_or_nothing: bool, generate_rollback: bool, exclude?: list<array{schema: ?string, table: ?string, exact: ?string, starts_with: ?string, ends_with: ?string, pattern: ?string, policy_id: ?string, type: ?string, for_schema: ?string}>}, catalog_providers: list<array{catalog_provider_id: ?string, catalog: ?array<string, mixed>}>} $config
      */
     #[Override]
     public function loadExtension(array $config, ContainerConfigurator $configurator, ContainerBuilder $container): void
@@ -578,7 +630,7 @@ private function registerMessenger(
     }
 
     /**
-     * @param array{enabled: bool, directory: string, namespace: string, table_name: string, table_schema: string, migration_file_name: string, rollback_file_name: string, all_or_nothing: bool, generate_rollback: bool} $mc
+     * @param array{enabled: bool, directory: string, namespace: string, table_name: string, table_schema: string, migration_file_name: string, rollback_file_name: string, all_or_nothing: bool, generate_rollback: bool, exclude?: list<array{schema: ?string, table: ?string, exact: ?string, starts_with: ?string, ends_with: ?string, pattern: ?string, policy_id: ?string, type: ?string, for_schema: ?string}>} $mc
      */
     private function registerMigrations(string $name, array $mc, ContainerBuilder $container, bool $isFirst): void
     {
@@ -595,6 +647,7 @@ private function registerMigrations(string $name, array $mc, ContainerBuilder $c
             $mc['rollback_file_name'],
             $mc['all_or_nothing'],
             $mc['generate_rollback'],
+            $this->buildExclusionPolicy($mc['exclude'] ?? []),
         ]);
         $configDef->setPublic(true);
         $container->setDefinition("flow.postgresql.{$name}.migrations.configuration", $configDef);
@@ -684,6 +737,61 @@ private function registerMigrations(string $name, array $mc, ContainerBuilder $c
         }
     }
 
+    /**
+     * @param list<array{schema: ?string, table: ?string, exact: ?string, starts_with: ?string, ends_with: ?string, pattern: ?string, policy_id: ?string, type: ?string, for_schema: ?string}> $exclude
+     */
+    private function buildExclusionPolicy(array $exclude): ?Definition
+    {
+        if ($exclude === []) {
+            return null;
+        }
+
+        $policies = [];
+
+        foreach ($exclude as $entry) {
+            $policies[] = $this->buildExclusionPolicyEntry($entry);
+        }
+
+        return new Definition(AnyExclusionPolicy::class, $policies);
+    }
+
+    /**
+     * @param array{schema: ?string, table: ?string, exact: ?string, starts_with: ?string, ends_with: ?string, pattern: ?string, policy_id: ?string, type: ?string, for_schema: ?string} $entry
+     */
+    private function buildExclusionPolicyEntry(array $entry): Definition|Reference
+    {
+        if ($entry['schema'] !== null) {
+            return new Definition(WholeSchemaExclusionPolicy::class, [$entry['schema']]);
+        }
+
+        $impliedType = null;
+
+        if ($entry['table'] !== null) {
+            $matcher = new Definition(ExactMatchExclusionPolicy::class, [$entry['table']]);
+            $impliedType = SchemaObjectType::TABLE;
+        } elseif ($entry['exact'] !== null) {
+            $matcher = new Definition(ExactMatchExclusionPolicy::class, [$entry['exact']]);
+        } elseif ($entry['starts_with'] !== null) {
+            $matcher = new Definition(StartsWithExclusionPolicy::class, [$entry['starts_with']]);
+        } elseif ($entry['ends_with'] !== null) {
+            $matcher = new Definition(EndsWithExclusionPolicy::class, [$entry['ends_with']]);
+        } elseif ($entry['pattern'] !== null) {
+            $matcher = new Definition(PatternExclusionPolicy::class, [$entry['pattern']]);
+        } elseif ($entry['policy_id'] !== null) {
+            $matcher = new Reference($entry['policy_id']);
+        } else {
+            throw new LogicException('Each "flow_postgresql.migrations.exclude" entry must define exactly one of: schema, table, exact, starts_with, ends_with, pattern, policy_id.');
+        }
+
+        $type = $entry['type'] !== null ? SchemaObjectType::from($entry['type']) : $impliedType;
+
+        if ($type === null && $entry['for_schema'] === null) {
+            return $matcher;
+        }
+
+        return new Definition(ScopedExclusionPolicy::class, [$matcher, $type, $entry['for_schema']]);
+    }
+
     /**
      * @param array{enabled?: bool, connection?: ?string, table_name?: string, schema?: string, id_col?: string, data_col?: string, lifetime_col?: string, time_col?: string, lock_mode?: string, ttl?: ?int, share_connection?: bool} $sessionConfig
      * @param list<string> $connectionNames

src/bridge/symfony/postgresql-bundle/tests/Flow/Bridge/Symfony/PostgreSqlBundle/Tests/Integration/FlowPostgreSqlExtensionTest.php

@@ -37,6 +37,9 @@
 use Flow\PostgreSql\Migrations\VersionResolver;
 use Flow\PostgreSql\Schema\Catalog;
 use Flow\PostgreSql\Schema\ChainCatalogProvider;
+use Flow\PostgreSql\Schema\Exclusion\AnyExclusionPolicy;
+use Flow\PostgreSql\Schema\Exclusion\SchemaObject;
+use Flow\PostgreSql\Schema\Exclusion\SchemaObjectType;
 use Flow\Telemetry\Provider\Clock\SystemClock;
 use Flow\Telemetry\Telemetry;
 use LogicException;
@@ -450,6 +453,88 @@ public function test_class_aliases_for_migration_services(): void
         static::assertTrue($this->getContainer()->has(DiffMigrationGenerator::class));
     }
 
+    public function test_migrations_exclude_config_builds_exclusion_policy(): void
+    {
+        $this->bootKernel([
+            'config' => static function (TestKernel $kernel): void {
+                $kernel->addTestContainerConfigurator(static function (ContainerBuilder $container): void {
+                    $container->register('flow.postgresql.default.client', SpyClient::class)->setPublic(true);
+                });
+                $kernel->addTestExtensionConfig('flow_postgresql', [
+                    'connections' => [
+                        'default' => [
+                            'dsn' => 'postgresql://postgres:postgres@localhost:5432/postgres',
+                        ],
+                    ],
+                    'migrations' => [
+                        'enabled' => true,
+                        'directory' => '/tmp/test_migrations',
+                        'namespace' => 'App\\Migrations',
+                        'exclude' => [
+                            ['schema' => 'tenant_data'],
+                            ['starts_with' => 'user_upload_'],
+                            ['ends_with' => '_tmp', 'type' => 'view'],
+                            ['pattern' => '/^cache_\d+$/', 'type' => 'sequence'],
+                            ['table' => 'legacy_audit'],
+                        ],
+                    ],
+                    'catalog_providers' => [
+                        ['catalog' => ['schemas' => []]],
+                    ],
+                ]);
+            },
+        ]);
+
+        $configuration = $this->getContainer()->get('flow.postgresql.default.migrations.configuration');
+        static::assertInstanceOf(MigrationsConfiguration::class, $configuration);
+
+        $policy = $configuration->exclusionPolicy;
+        static::assertInstanceOf(AnyExclusionPolicy::class, $policy);
+
+        static::assertTrue($policy->exclude(new SchemaObject(SchemaObjectType::TABLE, 'tenant_data', 'uploads')));
+        static::assertTrue($policy->exclude(new SchemaObject(SchemaObjectType::TABLE, 'public', 'user_upload_9')));
+
+        static::assertTrue($policy->exclude(new SchemaObject(SchemaObjectType::VIEW, 'public', 'report_tmp')));
+        static::assertFalse($policy->exclude(new SchemaObject(SchemaObjectType::TABLE, 'public', 'report_tmp')));
+
+        static::assertTrue($policy->exclude(new SchemaObject(SchemaObjectType::SEQUENCE, 'public', 'cache_42')));
+        static::assertFalse($policy->exclude(new SchemaObject(SchemaObjectType::TABLE, 'public', 'cache_42')));
+
+        static::assertTrue($policy->exclude(new SchemaObject(SchemaObjectType::TABLE, 'public', 'legacy_audit')));
+        static::assertFalse($policy->exclude(new SchemaObject(SchemaObjectType::SEQUENCE, 'public', 'legacy_audit')));
+
+        static::assertFalse($policy->exclude(new SchemaObject(SchemaObjectType::TABLE, 'public', 'orders')));
+    }
+
+    public function test_migrations_exclude_entry_without_matcher_throws(): void
+    {
+        $this->expectException(LogicException::class);
+        $this->expectExceptionMessage('must define exactly one of');
+
+        $this->bootKernel([
+            'config' => static function (TestKernel $kernel): void {
+                $kernel->addTestContainerConfigurator(static function (ContainerBuilder $container): void {
+                    $container->register('flow.postgresql.default.client', SpyClient::class)->setPublic(true);
+                });
+                $kernel->addTestExtensionConfig('flow_postgresql', [
+                    'connections' => [
+                        'default' => [
+                            'dsn' => 'postgresql://postgres:postgres@localhost:5432/postgres',
+                        ],
+                    ],
+                    'migrations' => [
+                        'enabled' => true,
+                        'directory' => '/tmp/test_migrations',
+                        'namespace' => 'App\\Migrations',
+                        'exclude' => [
+                            ['for_schema' => 'public'],
+                        ],
+                    ],
+                ]);
+            },
+        ]);
+    }
+
     public function test_client_with_telemetry_creates_default_clock_when_not_specified(): void
     {
         $this->bootKernel([