define a small emissions_1990_validation dataset list in config.yml and abstract complex logic out of spec

Author: louispt1

lib/graph_data_validation/config.yml

@@ -1,4 +1,10 @@
 dataset_collections:
+  emissions_1990_validation:
+    - nl2023
+    - nl2019
+    - DE_germany
+    - UKNI01_northern_ireland
+
   countries:
     - AT_austria
     - BEGM11002_antwerpen

lib/graph_data_validation/lib/datasets.rb

@@ -10,6 +10,7 @@ def initialize(*area_codes, env: :dev)
       if env == :dev
         Settings.etsource_lazy_load_dataset = true
         Atlas.data_dir = Etsource::Base.clean_path(File.expand_path("../etsource"))
+        Etsource::Loader.instance.clear!(:gqueries)
       end
 
       # Silently reject unexisting areas

lib/graph_data_validation/lib/emissions_csv_reconciler.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module GraphDataValidation
+  class EmissionsCsvReconciler
+    def initialize(dataset)
+      @emissions = Atlas::Dataset.find(dataset.scenario.area_code).emissions
+    end
+
+    # Returns the 1990 inventory total in Mt for the given GHG types,
+    # excluding bunkers rows and negating LULUCF removals (stored as positive
+    # values in the CSV but representing a carbon sink).
+    def total_mt(ghgs:)
+      @emissions.table.sum do |row|
+        next 0.0 unless row[:year] == 1990 && ghgs.include?(row[:ghg])
+        next 0.0 if row[:etm_sector].to_s.casecmp?('bunkers')
+
+        value = row[:value].to_f
+        removal = row[:etm_sector].to_s.casecmp?('lulucf') &&
+          row[:etm_subsector].to_s.casecmp?('removals')
+        removal ? -value : value
+      end / 1000.0
+    end
+  end
+end

lib/graph_data_validation/spec/dataset_spec.rb

@@ -9,3 +9,4 @@
     end
   end
 end
+

lib/graph_data_validation/spec/emissions_1990_spec.rb

@@ -0,0 +1,12 @@
+require_relative 'spec_helper'
+require_relative '../lib/datasets'
+require_relative '../lib/emissions_csv_reconciler'
+require_relative 'validation/emissions'
+
+describe 'Validating 1990 emissions inventory' do
+  GraphDataValidation::Datasets.from_collection(:emissions_1990_validation).each do |dataset|
+    context "with area_code #{dataset.scenario.area_code}" do
+      it_behaves_like 'emissions_1990_reconciliation', dataset
+    end
+  end
+end

lib/graph_data_validation/spec/validation/emissions.rb

@@ -18,40 +18,23 @@
       end
     end
   end
+end
 
-  # Reconcile the 1990 direct_emissions total queries against the dataset's
-  # emissions.csv (read from the local ETSource).
-  #
-  # Bunkers is excluded on both sides: its query scales the inventory by the
-  # bunkers_allocated_percentage_* inputs (default 0), so it cannot equal the raw
-  # CSV. Subtracting the bunkers query and the Bunkers CSV rows makes the check
-  # allocation-independent.
-  #
-  # LULUCF removals are stored as positive values in the CSV but represent a
-  # carbon sink; the queries subtract them (NEG), so the CSV sum negates them too.
-  emissions = Atlas::Dataset.find(dataset.scenario.area_code).emissions
+RSpec.shared_examples('emissions_1990_reconciliation') do |dataset|
+  reconciler = GraphDataValidation::EmissionsCsvReconciler.new(dataset)
 
   [
     { total: 'direct_emissions_co2_1990',       bunkers: 'direct_emissions_bunkers_co2_1990',       ghgs: %w[co2] },
     { total: 'direct_emissions_other_ghg_1990', bunkers: 'direct_emissions_bunkers_other_ghg_1990', ghgs: %w[other_ghg] },
     { total: 'direct_emissions_total_ghg_1990', bunkers: 'direct_emissions_bunkers_total_ghg_1990', ghgs: %w[co2 other_ghg] }
   ].each do |check|
-    # Queries report Mt; the inventory CSV is in kton CO2-eq. Exclude bunkers.
     queried_mt = dataset.query("present:Q(#{check[:total]})") -
                  dataset.query("present:Q(#{check[:bunkers]})")
-    emissions_csv_mt = emissions.table.sum do |row|
-      next 0.0 unless row[:year] == 1990 && check[:ghgs].include?(row[:ghg])
-      next 0.0 if row[:etm_sector].to_s.casecmp?('bunkers')
-
-      value = row[:value].to_f
-      removal = row[:etm_sector].to_s.casecmp?('lulucf') &&
-                row[:etm_subsector].to_s.casecmp?('removals')
-      removal ? -value : value
-    end / 1000.0
+    expected_mt = reconciler.total_mt(ghgs: check[:ghgs])
 
-    context "1990 inventory reconciliation for #{check[:total]}" do
+    context "1990 emissions results for #{check[:total]}" do
       it 'matches emissions.csv (excl. bunkers)' do
-        expect(queried_mt).to be_within(0.001 * emissions_csv_mt + 1e-6).of(emissions_csv_mt)
+        expect(queried_mt).to be_within(0.001 * expected_mt + 1e-6).of(expected_mt)
       end
     end
   end