Skip to content

Commit 9f60ed3

Browse files
fbacallfbacall
authored
Merge pull request #1180 from pan-training/oai_pmh_ingestors
Add OAI-PMH Ingestors
2 parents f73687a + 390d612 commit 9f60ed3

4 files changed

Lines changed: 411 additions & 1 deletion

File tree

‎app/views/layouts/application.html.erb‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33

44
<%= render 'layouts/head' %>
55

6-
<body class=<%= 'sticky-navbar-enabled' if TeSS::Config.feature['sticky_navbar'] %> <%= 'header-notice-present' if TeSS::Config.header_notice&.strip.present? %>>
6+
<body class="<%= 'sticky-navbar-enabled' if TeSS::Config.feature['sticky_navbar'] %> <%= 'header-notice-present' if TeSS::Config.header_notice&.strip.present? %>">
77
<%= render partial: 'layouts/header' %>
88

99
<div id="main-container" class="<%= @container_class || 'container-fluid' %>">

‎lib/ingestors/ingestor_factory.rb‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ def self.ingestors
1111
Ingestors::MaterialCsvIngestor,
1212
Ingestors::TessEventIngestor,
1313
Ingestors::ZenodoIngestor,
14+
Ingestors::OaiPmhIngestor,
1415
Ingestors::GithubIngestor,
1516
] + taxila_ingestors + llm_ingestors
1617
end

‎lib/ingestors/oai_pmh_ingestor.rb‎

Lines changed: 207 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,207 @@
1+
require 'tess_rdf_extractors'
2+
3+
module Ingestors
4+
class OaiPmhIngestor < Ingestor
5+
def self.config
6+
{
7+
key: 'oai_pmh',
8+
title: 'OAI-PMH',
9+
user_agent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:102.0) Gecko/20100101 Firefox/102.0',
10+
mail: Rails.configuration.tess['contact_email']
11+
}
12+
end
13+
14+
def initialize
15+
super
16+
17+
# to use some helper functions that are instance level methods of BioschemasIngestor
18+
@bioschemas_manager = BioschemasIngestor.new
19+
end
20+
21+
def read(source_url)
22+
client = OAI::Client.new source_url, headers: { 'From' => config[:mail], 'User-Agent' => config[:user_agent] }
23+
found_bioschemas = begin
24+
read_oai_rdf(client)
25+
rescue OAI::ArgumentException
26+
false
27+
end
28+
29+
read_oai_dublin_core(client) unless found_bioschemas
30+
end
31+
32+
def ns
33+
{
34+
'dc' => 'http://purl.org/dc/elements/1.1/',
35+
'oai_dc' => 'http://www.openarchives.org/OAI/2.0/oai_dc/'
36+
}
37+
end
38+
39+
def read_oai_dublin_core(client)
40+
count = 0
41+
client.list_records(metadata_prefix: 'oai_dc').full.each do |record|
42+
xml_string = record.metadata.to_s
43+
doc = Nokogiri::XML(xml_string)
44+
45+
types = doc.xpath('//dc:type', ns).map(&:text)
46+
# this event detection heuristic captures in particular
47+
# - http://purl.org/dc/dcmitype/Event (the standard way of typing an event in dublin core)
48+
# - https://schema.org/Event
49+
if types.any? { |t| t.downcase.include? 'event' }
50+
read_dublin_core_event(doc)
51+
else
52+
read_dublin_core_material(doc)
53+
end
54+
55+
count += 1
56+
end
57+
@messages << "found #{count} records"
58+
end
59+
60+
def read_dublin_core_material(xml_doc)
61+
material = OpenStruct.new
62+
material.title = xml_doc.at_xpath('//dc:title', ns)&.text
63+
material.description = convert_description(xml_doc.at_xpath('//dc:description', ns)&.text)
64+
material.authors = xml_doc.xpath('//dc:creator', ns).map(&:text)
65+
material.contributors = xml_doc.xpath('//dc:contributor', ns).map(&:text)
66+
67+
rights = xml_doc.xpath('//dc:rights', ns).map { |n| n.text&.strip }.reject(&:empty?)
68+
material.licence = rights.find { |r| r.start_with?('http://', 'https://') } || rights.first || 'notspecified'
69+
70+
dates = xml_doc.xpath('//dc:date', ns).map(&:text)
71+
parsed_dates = dates.map do |d|
72+
Date.parse(d)
73+
rescue StandardError
74+
nil
75+
end.compact
76+
material.date_created = parsed_dates.first
77+
material.date_modified = parsed_dates.last if parsed_dates.size > 1
78+
79+
identifiers = xml_doc.xpath('//dc:identifier', ns).map(&:text)
80+
doi = identifiers.find { |id| id.start_with?('10.') || id.start_with?('https://doi.org/') || id.start_with?('http://doi.org/') }
81+
if doi
82+
doi = doi&.sub(%r{https?://doi\.org/}, '')
83+
material.doi = "https://doi.org/#{doi}"
84+
end
85+
material.url = identifiers.find { |id| id.start_with?('http://', 'https://') }
86+
87+
material.keywords = xml_doc.xpath('//dc:subject', ns).map(&:text)
88+
material.resource_type = xml_doc.xpath('//dc:type', ns).map(&:text)
89+
material.contact = xml_doc.at_xpath('//dc:publisher', ns)&.text
90+
91+
add_material material
92+
end
93+
94+
def read_dublin_core_event(xml_doc)
95+
event = OpenStruct.new
96+
97+
event.title = xml_doc.at_xpath('//dc:title', ns)&.text
98+
event.description = convert_description(xml_doc.at_xpath('//dc:description', ns)&.text)
99+
event.url = xml_doc.xpath('//dc:identifier', ns).map(&:text).find { |id| id.start_with?('http://', 'https://') }
100+
event.contact = xml_doc.at_xpath('//dc:publisher', ns)&.text
101+
event.organizer = xml_doc.at_xpath('//dc:creator', ns)&.text
102+
event.keywords = xml_doc.xpath('//dc:subject', ns).map(&:text)
103+
event.event_types = xml_doc.xpath('//dc:type', ns).map(&:text)
104+
105+
dates = xml_doc.xpath('//dc:date', ns).map(&:text)
106+
parsed_dates = dates.map do |d|
107+
Date.parse(d)
108+
rescue StandardError
109+
nil
110+
end.compact
111+
event.start = parsed_dates.first
112+
event.end = parsed_dates.last
113+
114+
add_event event
115+
end
116+
117+
def read_oai_rdf(client)
118+
provider_events = []
119+
provider_materials = []
120+
totals = Hash.new(0)
121+
122+
client.list_records(metadata_prefix: 'rdf').full.each do |record|
123+
metadata_tag = Nokogiri::XML(record.metadata.to_s)
124+
bioschemas_xml = metadata_tag.at_xpath('metadata/rdf:RDF', 'rdf' => 'http://www.w3.org/1999/02/22-rdf-syntax-ns#')&.to_s
125+
output = parse_bioschemas(bioschemas_xml)
126+
next unless output
127+
128+
provider_events += output[:resources][:events]
129+
provider_materials += output[:resources][:materials]
130+
output[:totals].each do |key, value|
131+
totals[key] += value
132+
end
133+
end
134+
135+
if totals.keys.any?
136+
bioschemas_summary = "Bioschemas summary:\n"
137+
totals.each do |type, count|
138+
bioschemas_summary << "\n - #{type}: #{count}"
139+
end
140+
@messages << bioschemas_summary
141+
end
142+
143+
@bioschemas_manager.deduplicate(provider_events).each do |event_params|
144+
add_event(event_params)
145+
end
146+
147+
@bioschemas_manager.deduplicate(provider_materials).each do |material_params|
148+
add_material(material_params)
149+
end
150+
151+
provider_events.any? || provider_materials.any?
152+
end
153+
154+
def parse_bioschemas(content)
155+
output = {
156+
resources: {
157+
events: [],
158+
materials: []
159+
},
160+
totals: Hash.new(0)
161+
}
162+
163+
return output unless content
164+
165+
begin
166+
events = Tess::Rdf::EventExtractor.new(content, :rdfxml).extract do |p|
167+
@bioschemas_manager.convert_params(p)
168+
end
169+
courses = Tess::Rdf::CourseExtractor.new(content, :rdfxml).extract do |p|
170+
@bioschemas_manager.convert_params(p)
171+
end
172+
course_instances = Tess::Rdf::CourseInstanceExtractor.new(content, :rdfxml).extract do |p|
173+
@bioschemas_manager.convert_params(p)
174+
end
175+
learning_resources = Tess::Rdf::LearningResourceExtractor.new(content, :rdfxml).extract do |p|
176+
@bioschemas_manager.convert_params(p)
177+
end
178+
output[:totals]['Events'] += events.count
179+
output[:totals]['Courses'] += courses.count
180+
output[:totals]['CourseInstances'] += course_instances.count
181+
output[:totals]['LearningResources'] += learning_resources.count
182+
183+
@bioschemas_manager.deduplicate(events + courses + course_instances).each do |event|
184+
output[:resources][:events] << event
185+
end
186+
187+
@bioschemas_manager.deduplicate(learning_resources).each do |material|
188+
output[:resources][:materials] << material
189+
end
190+
rescue StandardError => e
191+
Rails.logger.error("#{e.class}: #{e.message}")
192+
Rails.logger.error(e.backtrace.join("\n")) if e.backtrace&.any?
193+
error = 'An error'
194+
comment = nil
195+
if e.is_a?(RDF::ReaderError)
196+
error = 'A parsing error'
197+
comment = 'Please check your page contains valid RDF/XML.'
198+
end
199+
message = "#{error} occurred while reading the source."
200+
message << " #{comment}" if comment
201+
@messages << message
202+
end
203+
204+
output
205+
end
206+
end
207+
end

0 commit comments

Comments
 (0)