Decouple validation from import

Gemfile

@@ -3,7 +3,7 @@ source "https://rubygems.org"
 ruby "3.4.4"
 
 # Bundle edge Rails instead: gem "rails", github: "rails/rails", branch: "main"
-gem "rails", "~> 7.1.2"
+gem "rails", "8.0.2.1"
 
 # Use postgresql as the database for Active Record
 
@@ -98,3 +98,7 @@ gem 'ostruct'
 gem "oj", "~> 3.16"
 
 gem "retriable", "~> 3.1"
+
+# Database and job processing
+gem "sqlite3", ">= 2.1"
+gem "solid_queue"

app/jobs/gap_detection_job.rb

@@ -0,0 +1,112 @@
+class GapDetectionJob < ApplicationJob
+  queue_as :gap_detection
+
+  def perform
+    return unless validation_enabled?
+
+    Rails.logger.info "GapDetectionJob: Starting gap detection scan"
+
+    # Get current import range
+    import_range = get_import_range
+    return unless import_range
+
+    start_block, end_block = import_range
+
+    # Find validation gaps
+    gaps = ValidationResult.validation_gaps(start_block, end_block)
+
+    if gaps.empty?
+      Rails.logger.debug "GapDetectionJob: No validation gaps found in range #{start_block}..#{end_block}"
+      return
+    end
+
+    Rails.logger.info "GapDetectionJob: Found #{gaps.length} validation gaps: #{gaps.first(5).join(', ')}#{gaps.length > 5 ? '...' : ''}"
+
+    # Enqueue validation jobs for gaps
+    gaps.each do |block_number|
+      begin
+        # Get L2 block data for this L1 block
+        l2_blocks = get_l2_blocks_for_l1_block(block_number)
+
+        if l2_blocks.any?
+          l2_block_hashes = l2_blocks.map { |block| block[:hash] }
+          ValidationJob.perform_later(block_number, l2_block_hashes, nil)
+          Rails.logger.debug "GapDetectionJob: Enqueued validation for block #{block_number}"
+        else
+          Rails.logger.warn "GapDetectionJob: No L2 blocks found for L1 block #{block_number}"
+        end
+      rescue => e
+        Rails.logger.error "GapDetectionJob: Failed to enqueue validation for block #{block_number}: #{e.message}"
+      end
+    end
+
+    Rails.logger.info "GapDetectionJob: Enqueued #{gaps.length} validation jobs for gaps"
+  end
+
+  private
+
+  def validation_enabled?
+    ENV.fetch('VALIDATION_ENABLED', 'false').casecmp?('true')
+  end
+
+  def get_import_range
+    begin
+      # Get the range of blocks we should have validation for
+      # Use the current L2 blockchain state to determine what's been imported
+      latest_l2_block = GethDriver.client.call("eth_getBlockByNumber", ["latest", false])
+      return nil if latest_l2_block.nil?
+
+      latest_l2_block_number = latest_l2_block['number'].to_i(16)
+      return nil if latest_l2_block_number == 0
+
+      # Get L1 attributes to find the corresponding L1 block
+      l1_attributes = GethDriver.client.get_l1_attributes(latest_l2_block_number)
+      current_l1_block = l1_attributes[:number]
+
+      # Check the last validated block
+      last_validated = ValidationResult.last_validated_block || 0
+
+      # We should validate from the oldest reasonable point to current
+      # Don't go back more than 1000 blocks to avoid overwhelming the system
+      start_block = [last_validated - 100, current_l1_block - 1000].max
+
+      [start_block, current_l1_block]
+    rescue => e
+      Rails.logger.error "GapDetectionJob: Failed to determine import range: #{e.message}"
+      nil
+    end
+  end
+
+  def get_l2_blocks_for_l1_block(l1_block_number)
+    # Query Geth to find L2 blocks that were created from this L1 block
+    # This is complex - we need to scan L2 blocks and check their L1 attributes
+    begin
+      l2_blocks = []
+
+      # Get current L2 tip to know the range to search
+      latest_l2_block = GethDriver.client.call("eth_getBlockByNumber", ["latest", false])
+      return [] if latest_l2_block.nil?
+
+      latest_l2_block_number = latest_l2_block['number'].to_i(16)
+
+      # Search backwards from current L2 tip to find blocks from this L1 block
+      # This is expensive but necessary for gap filling
+      (0..latest_l2_block_number).reverse_each do |l2_block_num|
+        l1_attributes = GethDriver.client.get_l1_attributes(l2_block_num)
+
+        if l1_attributes[:number] == l1_block_number
+          l2_block = GethDriver.client.call("eth_getBlockByNumber", ["0x#{l2_block_num.to_s(16)}", false])
+          l2_blocks << { number: l2_block_num, hash: l2_block['hash'] }
+        elsif l1_attributes[:number] < l1_block_number
+          # We've gone too far back
+          break
+        end
+      end
+
+      l2_blocks
+    rescue => e
+      Rails.logger.error "GapDetectionJob: Failed to get L2 blocks for L1 block #{l1_block_number}: #{e.message}"
+      []
+    end
+  end
+end

app/jobs/validation_job.rb

@@ -0,0 +1,26 @@
+class ValidationJob < ApplicationJob
+  queue_as :validation
+
+  # Retry up to 3 times with fixed delays to avoid the exponentially_longer issue
+  retry_on StandardError, wait: 5.seconds, attempts: 3
+
+  def perform(l1_block_number, l2_block_hashes)
+    start_time = Time.current
+
+    begin
+      # Use the unified ValidationResult model to validate and save
+      validation_result = ValidationResult.validate_and_save(l1_block_number, l2_block_hashes)
+
+      elapsed_time = Time.current - start_time
+      Rails.logger.info "ValidationJob: Block #{l1_block_number} validation completed in #{elapsed_time.round(3)}s"
+
+    rescue => e
+      Rails.logger.error "ValidationJob: Validation failed for block #{l1_block_number}: #{e.message}"
+
+      # Validation failure will be detected by import process via database query
+
+      # Re-raise to trigger retry mechanism
+      raise e
+    end
+  end
+end

app/models/application_record.rb

@@ -0,0 +1,3 @@
+class ApplicationRecord < ActiveRecord::Base
+  primary_abstract_class
+end

app/models/validation_result.rb

@@ -0,0 +1,123 @@
+class ValidationResult < ApplicationRecord
+  self.primary_key = 'l1_block'
+
+  scope :successful, -> { where(success: true) }
+  scope :failed, -> { where(success: false) }
+  scope :recent, -> { order(validated_at: :desc) }
+  scope :in_range, ->(start_block, end_block) { where(l1_block: start_block..end_block) }
+
+  # Class methods for validation management
+  def self.last_validated_block
+    maximum(:l1_block)
+  end
+
+  def self.validation_gaps(start_block, end_block)
+    # Use SQL recursive CTE to find gaps efficiently
+    sql = <<~SQL
+      WITH RECURSIVE expected(n) AS (
+        SELECT ? AS n
+        UNION ALL
+        SELECT n + 1 FROM expected WHERE n < ?
+      )
+      SELECT n AS missing_block
+      FROM expected
+      LEFT JOIN validation_results vr ON vr.l1_block = expected.n
+      WHERE vr.l1_block IS NULL
+      ORDER BY n
+    SQL
+
+    connection.execute(sql, [start_block, end_block]).map { |row| row['missing_block'] }
+  end
+
+  def self.validation_stats(since: 1.hour.ago)
+    results = where('validated_at >= ?', since)
+    total = results.count
+    passed = results.successful.count
+    failed = results.failed.count
+
+    {
+      total: total,
+      passed: passed,
+      failed: failed,
+      pass_rate: total > 0 ? (passed.to_f / total * 100).round(2) : 0
+    }
+  end
+
+  def self.recent_failures(limit: 10)
+    failed.recent.limit(limit)
+  end
+
+  # Class method to perform validation and save result
+  def self.validate_and_save(l1_block_number, l2_block_hashes)
+    Rails.logger.info "ValidationResult: Validating L1 block #{l1_block_number}"
+
+    begin
+      # Create validator and validate (validator fetches its own API data)
+      validator = BlockValidator.new
+      block_result = validator.validate_l1_block(l1_block_number, l2_block_hashes)
+
+      # Store validation result using create_or_find_by for concurrency safety
+      validation_result = create_or_find_by(l1_block: l1_block_number) do |vr|
+        vr.success = block_result.success
+        vr.error_details = block_result.errors.to_json
+        vr.validation_stats = block_result.stats.to_json
+        vr.validated_at = Time.current
+      end
+
+      # Log the result
+      validation_result.log_summary
+
+      validation_result
+    rescue => e
+      Rails.logger.error "ValidationResult: Exception validating block #{l1_block_number}: #{e.message}"
+
+      # Record the validation error
+      error_result = create_or_find_by(l1_block: l1_block_number) do |vr|
+        vr.success = false
+        vr.error_details = [e.message].to_json
+        vr.validation_stats = { exception: true, exception_class: e.class.name }.to_json
+        vr.validated_at = Time.current
+      end
+
+      error_result.log_summary
+      raise e
+    end
+  end
+
+  # Instance methods
+  def parsed_errors
+    return [] unless error_details.present?
+    JSON.parse(error_details) rescue []
+  end
+
+  def parsed_stats
+    return {} unless validation_stats.present?
+    JSON.parse(validation_stats) rescue {}
+  end
+
+  def failure_summary
+    return nil if success?
+
+    error_list = parsed_errors
+    return "No error details" if error_list.empty?
+
+    # Return first few errors for summary
+    error_list.first(3).join('; ')
+  end
+
+  def log_summary(logger = Rails.logger)
+    if success?
+      stats_data = parsed_stats
+      if stats_data['actual_creations'].to_i > 0 || stats_data['actual_transfers'].to_i > 0 || stats_data['storage_checks'].to_i > 0
+        logger.info "✅ Block #{l1_block} validated successfully: " \
+                    "#{stats_data['actual_creations']} creations, " \
+                    "#{stats_data['actual_transfers']} transfers, " \
+                    "#{stats_data['storage_checks']} storage checks"
+      end
+    else
+      logger.error "❌ Block #{l1_block} validation failed with #{parsed_errors.size} errors:"
+      parsed_errors.first(5).each { |e| logger.error "  - #{e}" }
+      logger.error "  ... and #{parsed_errors.size - 5} more errors" if parsed_errors.size > 5
+    end
+  end
+end