Run compatible specs in strict and strict2

Assisted-By: devx/a8210bca-b84d-4916-bd2f-849956155755

Author: tobi

CLAUDE.md

@@ -143,6 +143,7 @@ ruby -Ilib scripts/verifiers/spec_name_collisions.rb # advisory: no two specs sh
 ruby -Ilib scripts/verifiers/filesystem_extensions.rb # advisory: filesystem keys should have valid extensions
 ruby -Ilib scripts/verifiers/minimum_complexity.rb  # specs with advanced features sit above the beginner ramp
 ruby -Ilib scripts/verifiers/sequential_complexity.rb # opted-in curated files advance exactly one level per spec
+ruby -Ilib scripts/verifiers/strict_mode_compatibility.rb # strict/strict2-only specs must need their single-mode restriction
 ```
 
 **Blocking verifiers** (must be green before pushing):
@@ -165,6 +166,9 @@ ruby -Ilib scripts/verifiers/sequential_complexity.rb # opted-in curated files a
   `_metadata.sequential_complexity: true` must advance exactly one complexity
   level per spec. This prevents one-lesson ramps from collapsing back into
   round-number buckets.
+- `strict_mode_compatibility` — a strict- or strict2-only spec must actually
+  differ in the other mode; otherwise it declares `[strict2, strict]` and runs
+  for adapters supporting either strictness.
 - `jsonrpc_portability` — specs must not send non-primitive Ruby types
   (non-string keys, symbols, unregistered objects) over JSON-RPC, which would
   trigger `_rpc_drop` markers. Time/Date/DateTime are whitelisted (sent as

lib/liquid/spec/cli/matrix.rb

@@ -211,15 +211,16 @@ def run_comparison(specs, adapters, reference_name, options)
             matched = 0
             skipped = 0
             checked = 0
+            variants = expand_error_mode_variants(specs, adapters)
 
             max_failures = options[:max_failures]
             verbose = options[:verbose]
 
-            print("Running #{specs.size} specs: ")
+            print("Running #{variants.size} specs: ")
             $stdout.flush
 
             TimeFreezer.freeze_spec_time do
-              specs.each do |spec|
+              variants.each do |spec|
                 # Run on ALL adapters that can run this spec
                 outputs = {}
                 adapters.each do |name, adapter|
@@ -296,11 +297,41 @@ def run_comparison(specs, adapters, reference_name, options)
 
             # Print results (limited by max_failures for display only)
             print_results_v2(differences, adapters, options, max_failures)
-            print_summary(matched, differences.size, skipped, checked, specs.size, adapters)
+            print_summary(matched, differences.size, skipped, checked, variants.size, adapters)
 
             exit(1) if differences.any?
           end
 
+          # Matrix executes each explicit compatible parse mode independently.
+          # Ordinary specs use strict2, the harness-wide default, so adapters
+          # that only implement strict/lax are not compared against a different
+          # parser contract by accident.
+          def expand_error_mode_variants(specs, adapters)
+            supported = LiquidSpec::PARSE_ERROR_MODES.select do |mode|
+              feature = :"#{mode}_parsing"
+              adapters.values.any? { |adapter| !adapter.missing_features.include?(feature) }
+            end
+
+            specs.flat_map do |spec|
+              declared = declared_error_modes(spec)
+              modes = if declared.empty?
+                supported.include?(:strict2) ? [:strict2] : [supported.first].compact
+              else
+                LiquidSpec::PARSE_ERROR_MODES.select { |mode| declared.include?(mode) && supported.include?(mode) }
+              end
+
+              modes.map { |mode| spec.with_error_mode(mode, label: declared.length > 1) }
+            end
+          end
+
+          def declared_error_modes(spec)
+            return spec.error_modes unless spec.error_modes.empty?
+
+            LiquidSpec::PARSE_ERROR_MODES.select do |mode|
+              spec.features.include?(:"#{mode}_parsing")
+            end
+          end
+
 
 
 

lib/liquid/spec/cli/runner.rb

@@ -2166,7 +2166,7 @@ def expand_error_mode_variants(specs)
             supported = config.error_modes
 
             specs.flat_map do |spec|
-              declared = spec.respond_to?(:error_modes) ? spec.error_modes : []
+              declared = declared_error_modes(spec)
               modes = if declared.nil? || declared.empty?
                 [supported.first]
               else
@@ -2180,6 +2180,19 @@ def expand_error_mode_variants(specs)
             end
           end
 
+          # Older generated specs can carry a parse-mode feature tag without an
+          # explicit error_mode field. Treat that tag as the declaration while
+          # migrating the corpus; otherwise expansion would silently replace a
+          # lax requirement with the strict2 default.
+          def declared_error_modes(spec)
+            declared = spec.respond_to?(:error_modes) ? spec.error_modes : []
+            return declared unless declared.nil? || declared.empty?
+
+            LiquidSpec::PARSE_ERROR_MODES.select do |mode|
+              spec.features.include?(:"#{mode}_parsing")
+            end
+          end
+
           def count_unrunnable_specs(specs, missing_features)
             missing_set = Set.new(missing_features.map(&:to_sym))
             specs.count do |spec|

lib/liquid/spec/verifiers.rb

@@ -23,6 +23,7 @@ module Verifiers
         "sequential_complexity" => "SequentialComplexityVerifier",
         "spec_name_collisions" => "SpecNameCollisionVerifier",
         "spec_schema" => "SpecSchemaVerifier",
+        "strict_mode_compatibility" => "StrictModeCompatibilityVerifier",
       }.freeze
 
       Result = Struct.new(:name, :exit_code, :advisory, keyword_init: true)

scripts/verifiers/strict_mode_compatibility.rb

@@ -0,0 +1,64 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# Lint: a spec restricted to strict or strict2 must actually need that
+# restriction. If liquid-ruby produces the same status and output in both
+# modes, declare error_mode: [strict2, strict] so adapters supporting either
+# strictness exercise it.
+
+require "liquid/spec/spec_loader"
+require "liquid/spec/adapter_runner"
+require "liquid/spec/time_freezer"
+
+module StrictModeCompatibilityVerifier
+  class << self
+    def run
+      adapter = Liquid::Spec::AdapterRunner.new(name: "strict-mode-verifier")
+        .load_dsl(reference_adapter)
+      adapter.ensure_setup!
+
+      offenders = []
+      Liquid::Spec::TimeFreezer.freeze_spec_time do
+        Liquid::Spec::SpecLoader.load_all.each do |spec|
+          next unless spec.error_modes.length == 1
+          next unless [:strict, :strict2].include?(spec.error_mode)
+
+          strict2 = result_for(adapter, spec, :strict2)
+          strict = result_for(adapter, spec, :strict)
+          next unless strict2 == strict
+
+          offenders << spec
+        end
+      end
+
+      if offenders.empty?
+        puts "OK: all strict/strict2-only specs require their single-mode declaration."
+        return 0
+      end
+
+      puts "Found #{offenders.size} strict/strict2 spec(s) compatible with both modes:\n\n"
+      offenders.each do |spec|
+        puts "  #{relative_path(spec.source_file)}:#{spec.line_number}  #{spec.name}"
+      end
+      puts "\nDeclare error_mode: [strict2, strict] for each compatible spec."
+      1
+    end
+
+    private
+
+    def result_for(adapter, spec, mode)
+      result = adapter.run_single(spec.with_error_mode(mode))
+      [result.status, result.output.to_s]
+    end
+
+    def reference_adapter
+      File.expand_path("../../examples/liquid_ruby.rb", __dir__)
+    end
+
+    def relative_path(path)
+      path.sub(%r{\A#{Regexp.escape(Dir.pwd)}/}, "")
+    end
+  end
+end
+
+exit StrictModeCompatibilityVerifier.run if $PROGRAM_NAME == __FILE__

specs/basics/drops.yml

@@ -351,7 +351,7 @@ specs:
     render_error:
       - ErrorDrop
   complexity: 200
-  error_mode: strict
+  error_mode: [strict2, strict]
   hint: |
     ErrorDrop raises on any property access. The error should propagate
     as a render error.

specs/basics/dynamic-partials.yml

@@ -492,7 +492,7 @@ specs:
     render_error:
       - Illegal template name
   complexity: 550
-  error_mode: strict
+  error_mode: [strict2, strict]
   hint: |
     When the variable resolving the partial name is nil/undefined,
     an "Illegal template name" error should be raised as an exception.
@@ -507,7 +507,7 @@ specs:
   render_errors: true
   features: [inline_errors]
   complexity: 550
-  error_mode: strict
+  error_mode: [strict2, strict]
   hint: |
     Same as dynamic_include_nil_name but with inline error rendering.
     When render_errors is true, the error message is rendered into the
@@ -519,7 +519,7 @@ specs:
     render_error:
       - Could not find
   complexity: 550
-  error_mode: strict
+  error_mode: [strict2, strict]
   hint: |
     An empty string resolves to a file lookup for "" which doesn't exist.
     This produces a "Could not find asset" exception.
@@ -532,13 +532,13 @@ specs:
   render_errors: true
   features: [inline_errors]
   complexity: 550
-  error_mode: strict
+  error_mode: [strict2, strict]
   hint: |
     Same as dynamic_include_empty_string_name but with inline error rendering.
 
 - name: dynamic_include_missing_partial
   template: "{% assign tpl = 'nonexistent' %}{% include tpl %}"
-  error_mode: strict
+  error_mode: [strict2, strict]
   errors:
     render_error:
       - Could not find
@@ -555,7 +555,7 @@ specs:
   render_errors: true
   features: [inline_errors]
   complexity: 550
-  error_mode: strict
+  error_mode: [strict2, strict]
   hint: |
     Same as dynamic_include_missing_partial but with inline error rendering.
 

specs/basics/dynamic_partials.yml

@@ -40,7 +40,7 @@ specs:
 - name: dynamic_include_missing_partial_error
   complexity: 1000
   template: "{% include tpl %}"
-  error_mode: strict
+  error_mode: [strict2, strict]
   filesystem: {}
   environment:
     tpl: "missing"
@@ -51,7 +51,7 @@ specs:
 - name: dynamic_include_missing_partial_inline_error
   complexity: 1000
   template: "{% include tpl %}"
-  error_mode: strict
+  error_mode: [strict2, strict]
   filesystem: {}
   environment:
     tpl: "missing"
@@ -64,7 +64,7 @@ specs:
 - name: dynamic_include_illegal_name_error
   complexity: 1000
   template: "{% include tpl %}"
-  error_mode: strict
+  error_mode: [strict2, strict]
   filesystem:
     x: "X"
   environment:
@@ -76,7 +76,7 @@ specs:
 - name: dynamic_include_illegal_name_inline_error
   complexity: 1000
   template: "{% include tpl %}"
-  error_mode: strict
+  error_mode: [strict2, strict]
   filesystem:
     x: "X"
   environment:

specs/basics/error-handling.yml

@@ -47,7 +47,7 @@
   environment: {}
   expected: ''
   complexity: 230
-  error_mode: strict
+  error_mode: [strict2, strict]
   hint: |
     In liquid-ruby, strict mode only affects parsing, not runtime variable
     lookup. Undefined variables still render as empty strings. Other
@@ -69,7 +69,7 @@
   environment: {}
   expected: ''
   complexity: 245
-  error_mode: strict
+  error_mode: [strict2, strict]
   hint: |
     In liquid-ruby, strict mode only affects parsing. Property access on
     undefined variables still returns nil (empty output). Other implementations
@@ -115,7 +115,7 @@
   template: "{{ 'hello' | nonexistent_filter }}"
   expected: hello
   complexity: 270
-  error_mode: strict
+  error_mode: [strict2, strict]
   hint: |
     In liquid-ruby, unknown filters are ignored even in strict mode.
     The filter passes through its input unchanged, like lax mode.
@@ -158,7 +158,7 @@
 - name: unknown_tag_strict_parse_error
   template: "{% nonexistent_tag %}content{% endnonexistent_tag %}"
   complexity: 300
-  error_mode: strict
+  error_mode: [strict2, strict]
   errors:
     parse_error:
     - Unknown tag
@@ -169,7 +169,7 @@
 - name: unknown_tag_strict2_parse_error
   template: "{% nonexistent_tag %}content{% endnonexistent_tag %}"
   complexity: 300
-  error_mode: strict2
+  error_mode: [strict2, strict]
   errors:
     parse_error:
     - Unknown tag
@@ -300,7 +300,7 @@
       profile:
   expected: Anonymous
   complexity: 295
-  error_mode: strict
+  error_mode: [strict2, strict]
   hint: |
     Even in strict mode, nil-safe property access should work when the value
     is explicitly nil (not undefined). Here 'profile' exists but is nil, so
@@ -484,7 +484,7 @@
     c
     d
     {{ foo | }}
-  error_mode: :strict
+  error_mode: [strict2, strict]
   errors:
     parse_error:
       - "Expected id"
@@ -500,7 +500,7 @@
     line 1
     line 2
     {{ 'oops }}
-  error_mode: :strict
+  error_mode: [strict2, strict]
   errors:
     parse_error:
       - "Unexpected character"

specs/basics/filter-edge-cases.yml

@@ -191,7 +191,7 @@
     output:
       - "cannot sort"
   complexity: 100
-  error_mode: strict
+  error_mode: [strict2, strict]
   hint: |
     Sorting arrays with mixed types (numbers, strings, booleans) raises
     an error because comparison between different types is undefined.