Merge pull request #1342 from ElixirTeSS/multi-provider-ingestion

fbacall · web-flow · commit 8148581f8d8d · 2026-07-27T13:24:42.000+01:00
Store origin URI when exchanging
diff --git a/app/assets/stylesheets/application.scss b/app/assets/stylesheets/application.scss
@@ -690,6 +690,11 @@ sup {
   margin-left: 5px;
 }
 
+.exchange-icon {
+  color: $brand-primary;
+  margin-left: 5px;
+}
+
 .days_ago_text {
   color: grey;
   font-style: italic;
diff --git a/app/controllers/events_controller.rb b/app/controllers/events_controller.rb
@@ -243,7 +243,8 @@ def event_params
                                   :timezone, :content_provider_id, { collection_ids: [] }, { node_ids: [] },
                                   { node_names: [] }, { target_audience: [] }, { eligibility: [] }, :visible,
                                   { host_institutions: [] }, :capacity, :contact, :recognition, :learning_objectives,
-                                  :prerequisites, :tech_requirements, :cost_basis, :cost_value, :cost_currency, :language, :presence,
+                                  :prerequisites, :tech_requirements, :cost_basis, :cost_value, :cost_currency, :language,
+                                  :presence, :origin_uri,
                                   external_resources_attributes: %i[id url title _destroy],
                                   external_resources: %i[url title], material_ids: [],
                                   llm_interaction_attributes: %i[id scrape_or_process model prompt input output needs_processing _destroy],
diff --git a/app/controllers/materials_controller.rb b/app/controllers/materials_controller.rb
@@ -170,7 +170,7 @@ def material_params
                                      :last_scraped, :scraper_record, :remote_created_date, :remote_updated_date,
                                      :content_provider_id, :difficulty_level, :version, :status,
                                      :date_created, :date_modified, :date_published, :other_types,
-                                     :prerequisites, :syllabus, :visible, :learning_objectives, { subsets: [] },
+                                     :prerequisites, :syllabus, :visible, :learning_objectives, :origin_uri, { subsets: [] },
                                      { target_audience: [] },
                                      { collection_ids: [] }, { keywords: [] }, { resource_type: [] },
                                      { scientific_topic_names: [] }, { scientific_topic_uris: [] },
diff --git a/app/helpers/application_helper.rb b/app/helpers/application_helper.rb
@@ -29,7 +29,8 @@ module ApplicationHelper
     private: { icon: 'fa-eye-slash', message: 'This resource is private' },
     missing: { icon: 'fa-chain-broken', message: 'This resource has been offline for over three days' },
     check: { icon: 'fa-check', message: 'This resource is enabled' },
-    cross: { icon: 'fa-times', message: 'This resource has been disabled' }
+    cross: { icon: 'fa-times', message: 'This resource has been disabled' },
+    exchanged: { icon: 'fa-exchange', message: 'This resource originated from another TeSS registry' },
   }.freeze
 
   # Countries that have priority in the country selection menu. Using ISO 3166-1 Alpha2 code.
@@ -93,6 +94,11 @@ def event_status_icon(event, size = nil)
     end
   end
 
+  def exchange_icon(resource, size = nil)
+    return unless resource.origin_uri.present?
+    "<span class='exchange-icon pull-right'>#{icon_for(:exchanged, size)}</span>".html_safe
+  end
+
   def icon_for(type, size = nil, options = {})
     options[:class] ||= "info-icon#{'-' + size.to_s if size}"
     "<i class=\"fa #{ICONS[type][:icon]} has-tooltip #{options[:class]}\"
diff --git a/app/models/application_record.rb b/app/models/application_record.rb
@@ -6,4 +6,7 @@ class ApplicationRecord < ActiveRecord::Base
   include AutocompleteManager
   include ArrayFieldCleaner
 
+  def oai_identifier
+    "#{self.class.model_name.route_key}/#{id}"
+  end
 end
diff --git a/app/models/event.rb b/app/models/event.rb
@@ -157,6 +157,8 @@ class Event < ApplicationRecord
   validates :presence, inclusion: { in: presences.keys, allow_blank: true }
   validates :keywords, length: { maximum: 20 }
   validate :allowed_url
+  validates :origin_uri, url: { allow_blank: true }
+
   clean_array_fields(:keywords, :fields, :event_types, :target_audience,
                      :eligibility, :host_institutions, :sponsors)
   update_suggestions(:keywords, :target_audience, :host_institutions)
diff --git a/app/models/material.rb b/app/models/material.rb
@@ -123,6 +123,7 @@ class Material < ApplicationRecord
   validates :url, url: true
   validates :other_types, presence: true, if: proc { |m| m.resource_type.include?('other') }
   validates :keywords, length: { maximum: 20 }
+  validates :origin_uri, url: { allow_blank: true }
 
   clean_array_fields(:keywords, :fields,
                      :target_audience, :resource_type, :subsets)
diff --git a/app/views/common/_origin_info.erb b/app/views/common/_origin_info.erb
@@ -0,0 +1,12 @@
+<% unless resource.origin_uri.blank? %>
+  <h4 class="nav-heading"><%= t('exchange.title') %></h4>
+  <div class="nav-block" id="origin-info">
+    <p>
+      <%= t('exchange.info', resource_type: resource.model_name.human) %>
+    </p>
+
+    <p>
+      <%= external_link_button t('exchange.link'), resource.origin_uri %>
+    </p>
+  </div>
+<% end %>
diff --git a/app/views/events/_event.html.erb b/app/views/events/_event.html.erb
@@ -19,6 +19,7 @@
             <% end %>
 
             <%= event_status_icon(event) %>
+            <%= exchange_icon(event) %>
           </div>
           <% if event.event_types.any? %>
             <% event.event_types.each do |t| %>
diff --git a/app/views/events/show.html.erb b/app/views/events/show.html.erb
@@ -1,6 +1,7 @@
 <div class="wrapper collapsing-wrapper">
   <%# SIDEBAR %>
   <div class="collapsing-sidebar" id="sidebar">
+    <%= render partial: 'common/origin_info', locals: { resource: @event } %>
     <%= render partial: 'content_providers/partials/content_provider_info', locals: { content_provider: @event.content_provider } %>
     <%= render partial: "nodes/partials/associated_node_info", locals: { associated_nodes: @event.associated_nodes } %>
     <%= render(partial: 'users/partials/user_info', locals: { user: @event.user }) if current_user.try(:is_admin?) %>
diff --git a/app/views/materials/_material.html.erb b/app/views/materials/_material.html.erb
@@ -17,6 +17,7 @@
           <%= scrape_status_icon(material) %>
           <%= suggestion_icon(material) %>
         <% end %>
+        <%= exchange_icon(material) %>
       </div>
       <div class="sub-heading">
         <%= display_attribute_no_label(material, :resource_type) { |values| values.join(', ') } %>
diff --git a/app/views/materials/show.html.erb b/app/views/materials/show.html.erb
@@ -3,6 +3,7 @@
     <%= render partial: 'learning_paths/partials/navigation',
                locals: { topic_link: @learning_path_topic_link,
                          topic_item: @learning_path_topic_item } if @learning_path_topic_link && @learning_path_topic_item %>
+    <%= render partial: 'common/origin_info', locals: { resource: @material } %>
     <%= render partial: 'content_providers/partials/content_provider_info', locals: { content_provider: @material.content_provider } %>
     <%= render partial: 'nodes/partials/associated_node_info', locals: { associated_nodes: @material.associated_nodes } %>
     <%= render(partial: 'users/partials/user_info', locals: { user: @material.user }) if current_user.try(:is_admin?) %>
diff --git a/config/initializers/oai_provider.rb b/config/initializers/oai_provider.rb
@@ -12,12 +12,27 @@ def initialize
   end
 end
 
+module OAI::Provider::Response
+  class RecordResponse < Base
+    private
+
+    # Allow for a custom identifier to be used in the record without affecting the query (which happens with `identifier_field`)
+    def identifier_for(record)
+      "#{provider.prefix}:#{record.respond_to?(:oai_identifier) ? record.oai_identifier : record.send(provider.model.identifier_field)}"
+    end
+  end
+end
+
 class TrainingProvider < OAI::Provider::Base
   repository_name TeSS::Config.site['title']
   repository_url "#{TeSS::Config.base_url}/oai-pmh"
   record_prefix "oai:#{URI(TeSS::Config.base_url).host}"
   admin_email TeSS::Config.contact_email
-  sample_id '142' # so that example id is oai:domain:142
+  sample_id 'materials/142' # so that example id is oai:domain:materials/142
+  extra_description %(
+     <description>
+       <tess-instance xmlns="http://tesshub.org/xmlns" />
+     </description>)
 
   register_format(OAIRDF.instance)
 end
diff --git a/config/locales/en.yml b/config/locales/en.yml
@@ -1188,4 +1188,8 @@ en:
             any: Any time
             week: Within the last week
             month: Within the last month
-            year: Within the last year
+            year: Within the last year
+  exchange:
+    title: Original entry
+    info: "This %{resource_type} originated from another TeSS registry. For complete, up-to-date information, please visit the original entry:"
+    link: Go to original entry
diff --git a/db/migrate/20260710154329_add_origin_uri_to_tables.rb b/db/migrate/20260710154329_add_origin_uri_to_tables.rb
@@ -0,0 +1,6 @@
+class AddOriginUriToTables < ActiveRecord::Migration[8.1]
+  def change
+    add_column :materials, :origin_uri, :string
+    add_column :events, :origin_uri, :string
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[8.1].define(version: 2026_04_21_144919) do
+ActiveRecord::Schema[8.1].define(version: 2026_07_10_154329) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "pg_catalog.plpgsql"
 
@@ -212,6 +212,7 @@
     t.text "notes"
     t.string "open_science", default: [], array: true
     t.string "organizer"
+    t.string "origin_uri"
     t.string "postcode"
     t.text "prerequisites"
     t.integer "presence", default: 0
@@ -371,6 +372,7 @@
     t.date "last_scraped"
     t.text "learning_objectives"
     t.string "licence", default: "notspecified"
+    t.string "origin_uri"
     t.string "other_types"
     t.text "prerequisites"
     t.date "remote_created_date"
diff --git a/lib/ingestors/event_ingestion.rb b/lib/ingestors/event_ingestion.rb
@@ -6,7 +6,6 @@ def add_event(event)
       if event.is_a?(Hash)
         c = EventsController.new
         c.params = { event: event }
-        c.send(:event_params)
         event = OpenStruct.new(c.send(:event_params))
       end
       event = handle_auto_parsing(event)
diff --git a/lib/ingestors/material_ingestion.rb b/lib/ingestors/material_ingestion.rb
@@ -6,7 +6,6 @@ def add_material(material)
       if material.is_a?(Hash)
         c = MaterialsController.new
         c.params = { material: material }
-        c.send(:material_params)
         material = OpenStruct.new(c.send(:material_params))
       end
       material = handle_auto_parsing(material)
diff --git a/lib/ingestors/oai_pmh_ingestor.rb b/lib/ingestors/oai_pmh_ingestor.rb
@@ -22,8 +22,11 @@ def initialize
 
     def read(source_url)
       client = OAI::Client.new source_url, headers: { 'From' => config[:mail], 'User-Agent' => config[:user_agent] }
+
+      tess_instance = client.identify.descriptions.any? { |d| d.get_elements('//tess-instance').any? }
+
       found_bioschemas = begin
-        read_oai_rdf(client)
+        read_oai_rdf(client, tess_instance: tess_instance)
       rescue OAI::ArgumentException
         false
       end
@@ -75,7 +78,7 @@ def read_oai_dublin_core(client)
       @messages << "found #{count} records"
     end
 
-    def read_oai_rdf(client)
+    def read_oai_rdf(client, tess_instance: false)
       provider_events = []
       provider_materials = []
       totals = Hash.new(0)
@@ -86,6 +89,17 @@ def read_oai_rdf(client)
         output = parse_bioschemas(bioschemas_xml)
         next unless output
 
+        extra_metadata = {}
+        if tess_instance
+          origin_uri = metadata_tag.at_xpath('//rdf:RDF/*/@rdf:about', 'rdf' => 'http://www.w3.org/1999/02/22-rdf-syntax-ns#')&.value
+          extra_metadata[:origin_uri] = origin_uri
+        end
+
+        extra_metadata.each do |key, value|
+          output[:resources][:events].each { |r| r[key] ||= value }
+          output[:resources][:materials].each { |r| r[key] ||= value }
+        end
+
         provider_events += output[:resources][:events]
         provider_materials += output[:resources][:materials]
         output[:totals].each do |key, value|
diff --git a/test/controllers/events_controller_test.rb b/test/controllers/events_controller_test.rb
@@ -1704,4 +1704,19 @@ class EventsControllerTest < ActionController::TestCase
       end
     end
   end
+
+  test 'should link back to original entry if origin URI set' do
+    get :show, params: { id: @event }
+    assert_response :success
+
+    assert_select '#origin-info', count: 0
+
+    uri = 'https://some-tess-instance.org/events/123'
+    @event.update(origin_uri: uri)
+
+    get :show, params: { id: @event }
+    assert_response :success
+
+    assert_select '#origin-info a[href=?]', uri
+  end
 end
diff --git a/test/controllers/materials_controller_test.rb b/test/controllers/materials_controller_test.rb
@@ -1706,4 +1706,19 @@ class MaterialsControllerTest < ActionController::TestCase
     assert_select 'p.scientific_topics a[href="http://edamontology.org/topic_0622"]', text: 'Genomics'
     assert_select 'p.operations a[href="http://edamontology.org/operation_0292"]', text: 'Sequence alignment'
   end
+
+  test 'should link back to original entry if origin URI set' do
+    get :show, params: { id: @material }
+    assert_response :success
+
+    assert_select '#origin-info', count: 0
+
+    uri = 'https://some-tess-instance.org/materials/123'
+    @material.update(origin_uri: uri)
+
+    get :show, params: { id: @material }
+    assert_response :success
+
+    assert_select '#origin-info a[href=?]', uri
+  end
 end
diff --git a/test/models/material_test.rb b/test/models/material_test.rb
@@ -751,4 +751,21 @@ class MaterialTest < ActiveSupport::TestCase
     bioschemas = m.to_bioschemas.first.generate
     refute bioschemas.key?(:license)
   end
+
+  test 'validates origin_uri' do
+    @material.origin_uri = nil
+    assert @material.valid?
+
+    @material.origin_uri = ''
+    assert @material.valid?
+
+    @material.origin_uri = 'https://tess-instance.org/materials/123'
+    assert @material.valid?
+
+    @material.origin_uri = 'hamster'
+    refute @material.valid?
+
+    @material.origin_uri = 'ftp://tess-instance.org/materials/123'
+    refute @material.valid?
+  end
 end
diff --git a/test/unit/ingestors/oai_pmh_ingestor_test.rb b/test/unit/ingestors/oai_pmh_ingestor_test.rb
diff --git a/test/vcr_cassettes/ingestors/tess_oai_pmh_identify.yml b/test/vcr_cassettes/ingestors/tess_oai_pmh_identify.yml
diff --git a/test/vcr_cassettes/ingestors/tess_oai_pmh_listrecords.yml b/test/vcr_cassettes/ingestors/tess_oai_pmh_listrecords.yml
