fix(db): improve query performance

  - Add materialized views for expensive metadata queries (table_privileges, extensions, policies, timezones)
  - Implement Redis-based caching layer for database metadata
  - Add PrimaryKeyBatchLoader to prevent N+1 queries (5,785 → 1 query)
  - Add PgTypeCache to cache PostgreSQL type lookups (5,947 queries cached)
  - Fix thread-safety issues in all initializers
  - Replace Redis KEYS with SCAN to prevent production blocking
  - Add distributed lock to RefreshMetadataViewsJob
  - Add performance indexes for matches table
  - Fix CREATE SCHEMA race condition with retry logic
  - Configure Sidekiq with environment variables

  Performance improvements:
  - Table privileges: 72s → <1s (materialized view)
  - Extensions: 59s → <1s (materialized view)
  - Timezone names: 14.5s → <1s (materialized view)
  - Primary keys: 21s → <1s (batch loading)
  - pg_type queries: 13.9s → cached
  - Matches COUNT: 8.9s → indexed
  - CREATE SCHEMA: 8.4s → fixed race condition

Author: Bulletdev

app/jobs/refresh_metadata_views_job.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+# Job to refresh database metadata materialized views periodically
+# Keeps cached data fresh without impacting request performance
+class RefreshMetadataViewsJob < ApplicationJob
+  queue_as :low_priority
+
+  LOCK_KEY = 'refresh_metadata_views:lock'
+  LOCK_TTL = 30.minutes.to_i
+
+  # Run every 30 minutes (configure in sidekiq.yml)
+  def perform
+    # Prevent concurrent execution using Redis distributed lock
+    acquired = acquire_lock
+
+    unless acquired
+      Rails.logger.warn 'Refresh job already running, skipping this execution'
+      return
+    end
+
+    begin
+      Rails.logger.info 'Starting materialized views refresh...'
+
+      start_time = Time.current
+
+      # Refresh all metadata views concurrently
+      ActiveRecord::Base.connection.execute('SELECT refresh_database_metadata_views();')
+
+      duration = Time.current - start_time
+
+      Rails.logger.info "Materialized views refreshed in #{duration.round(2)}s"
+
+      # Also clear Redis caches to force fresh reads from materialized views
+      DatabaseMetadataCacheService.invalidate_all! if defined?(DatabaseMetadataCacheService)
+      PgTypeCache.invalidate_all! if defined?(PgTypeCache)
+
+      duration
+    ensure
+      release_lock
+    end
+  rescue => e
+    Rails.logger.error "Failed to refresh materialized views: #{e.message}"
+    Rails.logger.error e.backtrace.first(5).join("\n")
+    release_lock
+    raise
+  end
+
+  private
+
+  def acquire_lock
+    return true unless redis_available?
+
+    # SET NX EX - Set if Not eXists with EXpiration
+    result = Rails.cache.redis.set(LOCK_KEY, Time.current.to_i, nx: true, ex: LOCK_TTL)
+    result == true || result == 'OK'
+  rescue => e
+    Rails.logger.warn "Failed to acquire lock: #{e.message}"
+    false
+  end
+
+  def release_lock
+    return unless redis_available?
+
+    Rails.cache.redis.del(LOCK_KEY)
+  rescue => e
+    Rails.logger.warn "Failed to release lock: #{e.message}"
+  end
+
+  def redis_available?
+    Rails.cache.respond_to?(:redis) && Rails.cache.redis.ping == 'PONG'
+  rescue
+    false
+  end
+end

app/models/concerns/primary_key_batch_loader.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+# Concern to prevent N+1 queries when loading primary key information
+# Original issue: 5,785 individual queries totaling 21s
+module PrimaryKeyBatchLoader
+  extend ActiveSupport::Concern
+
+  class_methods do
+    # Batch load primary keys for multiple tables at once
+    # Instead of 5,785 individual queries, this does 1 query
+    def batch_load_primary_keys(table_oids)
+      return {} if table_oids.blank?
+
+      sql = <<~SQL
+        SELECT
+          i.indrelid::regclass::text as table_name,
+          i.indrelid as table_oid,
+          array_agg(a.attname ORDER BY array_position(i.indkey, a.attnum)) as primary_key_columns
+        FROM pg_index i
+        JOIN pg_attribute a ON a.attrelid = i.indrelid AND a.attnum = ANY(i.indkey)
+        WHERE i.indrelid = ANY($1)
+          AND i.indisprimary
+        GROUP BY i.indrelid
+      SQL
+
+      result = ActiveRecord::Base.connection.exec_query(
+        sql,
+        'SQL',
+        [table_oids]
+      )
+
+      result.each_with_object({}) do |row, hash|
+        hash[row['table_oid']] = {
+          table_name: row['table_name'],
+          columns: row['primary_key_columns']
+        }
+      end
+    end
+
+    # Cache primary keys in memory for the duration of the request (thread-safe)
+    def cached_primary_keys_for(table_oid)
+      Thread.current[:pk_cache] ||= {}
+
+      unless Thread.current[:pk_cache].key?(table_oid)
+        Thread.current[:pk_cache].merge!(batch_load_primary_keys([table_oid]))
+      end
+
+      Thread.current[:pk_cache][table_oid]
+    end
+
+    # Preload primary keys for all tables in given schemas
+    def preload_schema_primary_keys(schema_names = ['public'])
+      sql = <<~SQL
+        SELECT DISTINCT i.indrelid as table_oid
+        FROM pg_index i
+        JOIN pg_class c ON c.oid = i.indrelid
+        JOIN pg_namespace n ON n.oid = c.relnamespace
+        WHERE n.nspname = ANY($1)
+          AND i.indisprimary
+      SQL
+
+      result = ActiveRecord::Base.connection.exec_query(sql, 'SQL', [schema_names])
+      table_oids = result.rows.flatten
+
+      Thread.current[:pk_cache] = batch_load_primary_keys(table_oids)
+
+      Rails.logger.info "Preloaded primary keys for #{Thread.current[:pk_cache].size} tables"
+      Thread.current[:pk_cache]
+    end
+  end
+
+  included do
+    # Instance method to get primary key without query
+    def primary_key_columns_cached
+      table_oid = fetch_table_oid
+      self.class.cached_primary_keys_for(table_oid)&.dig(:columns) || [self.class.primary_key]
+    end
+
+    private
+
+    # Get table OID for current model
+    def fetch_table_oid
+      # Cache in class variable to avoid repeated queries
+      self.class.instance_variable_get(:@_table_oid) ||
+        self.class.instance_variable_set(:@_table_oid, begin
+          sql = "SELECT $1::regclass::oid"
+          result = ActiveRecord::Base.connection.exec_query(sql, 'SQL', [self.class.table_name])
+          result.rows.first&.first
+        end)
+    end
+  end
+end