diff --git a/lib/ingestors/github_ingestor.rb b/lib/ingestors/github_ingestor.rb new file mode 100644 index 000000000..79872f9a1 --- /dev/null +++ b/lib/ingestors/github_ingestor.rb @@ -0,0 +1,243 @@ +# frozen_string_literal: true + +require 'open-uri' +require 'json' +require 'httparty' +require 'nokogiri' + +module Ingestors + # GithubIngestor fetches repository information from GitHub to populate the materials' metadata. + # API requests counter: + # 1. Get the repo's general metadata #{GITHUB_API_BASE}/#{full_name} + # and keys: name, full_name, owner.login, html_url, description, + # homepage, topics, license.{key, spdx}, archived, + # created_at, pushed_at, updated_at, contributors_url + # 2. Get the doi #{GITHUB_API_BASE}/#{full_name}/contents/README.md + # and key: content + # 3. Get the version/release #{GITHUB_API_BASE}/#{full_name}/releases + # and key: tag_name (first) + # 4. Get the contributors' list #{GITHUB_API_BASE}/#{full_name}/contributors + # and key: login (from all entries) + class GithubIngestor < Ingestor # rubocop:disable Metrics/ClassLength + include Ingestors::Concerns::SitemapHelpers + + GITHUB_API_BASE = 'https://api.github.com/repos' + CACHE_PREFIX = 'github_ingestor_' + TTL = 1.week # cache expiration time (time to live before cache expires) + + def self.config + { + key: 'github', + title: 'GitHub Repository or Page', + category: :materials, + user_agent: 'TeSS Github ingestor' + } + end + + # Reads from direct GitHub URLs, .xml sitemaps, and .txt sitemaps. + # Fetches repository metadata, contributors, releases, and DOIs (from CITATION.cff). + # It handles automatically GitHub Pages URLs (github.io) and standard github.com URLs. + # It caches API requests to avoid repeated calls. + def read(source_url) + @verbose = false + # Returns either a map of unique URL entries, either the URL itself + sources = parse_sitemap(source_url) + + sources.each do |url| + # Reads each source, if github.{com|io}, gets the repo's api, if not, next + repo_api_url = to_github_api(url) + next unless repo_api_url + + # Gets the cached repo data or reads and sets it + key = "#{CACHE_PREFIX}#{repo_api_url.gsub(%r{https?://}, '').gsub('/', '_')}" + repo_data = cache_fetch(key, repo_api_url) + next unless repo_data + + # Add to material + add_material to_material(repo_data) + end + rescue StandardError => e + Rails.logger.error("#{e.class}: read() failed, #{e.message}") + end + + private + + # Takes a github.{com|io} url and returns its api.github.com url + def to_github_api(url) + uri = URI(url) + parts = uri.path.split('/') # 'example.com/foo/bar' will have path == '/foo/bar', so three parts + + # http(s)://github.com// is the strict way to pass + if uri.host&.downcase == 'github.com' && (uri.host.count('.') == 1) && parts.size == 3 + github_api_from_com(parts) + # http(s)://.github.io/ is the strict way to pass + elsif uri.host&.downcase&.end_with?('.github.io') && (uri.host.count('.') == 2) && parts.size >= 2 + github_api_from_io(uri, parts) + end + end + + def github_api_from_com(parts) + "#{GITHUB_API_BASE}/#{parts[1]}/#{parts[2]}" + end + + def github_api_from_io(uri, parts) + repo = parts[1] + owner = uri.host.split('.').first + "#{GITHUB_API_BASE}/#{owner}/#{repo}" + end + + # Fetch cached data or opens webpage/api and cache it + # I chose to cache because GitHub limits up to 60 requests per hour for unauthenticated user + # https://docs.github.com/en/rest/using-the-rest-api/rate-limits-for-the-rest-api?apiVersion=2022-11-28#primary-rate-limit-for-unauthenticated-users + # One GitHub URL equals to 4 GitHub API requests. + # key: string key for the cache + # url: url to open + # ttl: time-to-live in seconds (default 7 days) + def cache_fetch(key, url) + Rails.cache.fetch(key, expires_in: TTL, skip_nil: true) do + JSON.parse(open_url(url).read) + end + end + + # Sets material hash keys and values and add them to material + def to_material(repo_data) # rubocop:disable Metrics/AbcSize + github_io_homepage = github_io_homepage? repo_data['homepage'] + url = github_io_homepage ? repo_data['homepage'] : repo_data['html_url'] + redirected_url = get_redirected_url(url) + html = get_html(redirected_url) + + material = OpenStruct.new + material.title = repo_data['name'].titleize + material.url = url + material.description = github_io_homepage ? fetch_definition(html, redirected_url) : repo_data['description'] + material.keywords = repo_data['topics'] + material.licence = fetch_licence(repo_data['license']) + material.status = repo_data['archived'] ? 'Archived' : 'Active' + material.doi = fetch_doi(repo_data['full_name']) + material.version = fetch_latest_release(repo_data['full_name']) + material.date_created = repo_data['created_at'] + material.date_published = repo_data['pushed_at'] + material.date_modified = repo_data['updated_at'] + material.contributors = fetch_contributors(repo_data['contributors_url'], repo_data['full_name']) + material.resource_type = github_io_homepage ? ['Github Page'] : ['Github Repository'] + material.prerequisites = fetch_prerequisites(html) + material + end + + def github_io_homepage?(homepage) + return false if homepage.nil? || homepage.empty? + + url = URI(homepage) + url.host&.downcase&.end_with?('.github.io') + end + + def get_html(url) + response = HTTParty.get(url, follow_redirects: true, headers: { 'User-Agent' => config[:user_agent] }) + Nokogiri::HTML(response.body) + end + + # DEFINITION – Opens the GitHub homepage, fetches the 3 first >50 char

tags'text + # and joins them with a 'Read more...' link at the end of the description + # Some of the first

tags were not descriptive, thus skipping them + def fetch_definition(html, url) + desc = '' + round = 3 + html.css('p').each do |p| + p_txt = p&.text&.strip&.gsub(/\s+/, ' ') || '' + next if p_txt.length < 50 || round.zero? + + desc = "#{desc}\n#{p_txt}" + round -= 1 + end + "#{desc}\n(...) [Read more...](#{url})" + end + + # LICENCE – Get proper licence + # the licence must match the format of config/dictionaries/licences.yml + def fetch_licence(licence) + return 'notspecified' if licence.nil? || licence == 'null' + return 'other-at' if licence['key'] == 'other' + + licence['spdx_id'] + end + + # DOI – Fetches DOI from various sources in a repo + # I chose to only read the `README.md` as it seems to have the DOI badge almost everytime. + # Whereas enabling the fetching of CITATION.cff or CITATION.md would result in increasing + # the number of api request. + def fetch_doi(full_name) + filename = 'README.md' + url = "#{GITHUB_API_BASE}/#{full_name}/contents/#{filename}" + data = cache_fetch("#{CACHE_PREFIX}doi_#{full_name.gsub('/', '_')}_#{filename.downcase}", url) + return nil unless data && data['content'] + + decoded = Base64.decode64(data['content']) + doi_match = decoded.match(%r{doi.org/\s*([^\s,)]+)}i) + doi_match ? "https://doi.org/#{doi_match[1]}" : nil + end + + # RELEASE – Opens releases API address and returns last release + def fetch_latest_release(full_name) + url = "#{GITHUB_API_BASE}/#{full_name}/releases" + releases = cache_fetch("#{CACHE_PREFIX}releases_#{full_name.gsub('/', '_')}", url) + releases.is_a?(Array) && releases.first ? releases.first['tag_name'] : nil + end + + # CONTRIBUTORS – Opens contributors API address and returns list of contributors + def fetch_contributors(contributors_url, full_name) + contributors = cache_fetch("#{CACHE_PREFIX}contributors_#{full_name.gsub('/', '_')}", contributors_url) + return [] unless contributors + + contributors.map { |c| (c['login']) } + end + + # PREREQUISITES – From the homepage HTML, looks for

tags which are children of ... + def fetch_prerequisites(html) + prereq_paragraphs = [] + + # ... any heading tag (h1–h6) or span tag with text "prereq" (EN) or "prerreq" (ES) + prereq_paragraphs = fetch_prerequisites_from_h(html, prereq_paragraphs) + + # ... any tag with id containing "prereq" (EN) or "prerreq" (ES) + prereq_paragraphs = fetch_prerequisites_from_id_or_class(html, prereq_paragraphs) if prereq_paragraphs.empty? + + prereq_paragraphs&.join("\n")&.gsub(/\n\n+/, "\n")&.strip || '' + end + + def fetch_prerequisites_from_h(html, prereq_paragraphs) + html.xpath('//h1|//h2|//h3|//h4|//h5|//h6|//span').each do |h| + next unless h.text =~ /prereq|prerreq/i # if prereq in text + + paragraph = h.xpath('following-sibling::*') + .take_while { |sib| %w[p ul ol].include?(sib.name) } # take either p, ul or ol + prereq_paragraphs.concat(paragraph) if paragraph + end + prereq_paragraphs + end + + def fetch_prerequisites_from_id_or_class(html, prereq_paragraphs) + html.xpath('//*[@id]').each do |node| + next unless prereq_node?(node) + + extract_following_paragraphs(node, prereq_paragraphs) + extract_nested_paragraphs(node, prereq_paragraphs) if prereq_paragraphs.empty? + end + prereq_paragraphs + end + + def prereq_node?(node) + [node['id'], node['class']].compact.any? { |attr| attr =~ /prereq|prerreq/i } + end + + def extract_following_paragraphs(node, prereq_paragraphs) + paragraphs = node.xpath('following-sibling::*') + .take_while { |sib| %w[p ul ol].include?(sib.name) } + prereq_paragraphs.concat(paragraphs) if paragraphs + end + + def extract_nested_paragraphs(node, prereq_paragraphs) + paragraphs = node.xpath('.//p | .//ul | .//ol') + prereq_paragraphs.concat(paragraphs) if paragraphs.any? + end + end +end diff --git a/lib/ingestors/ingestor.rb b/lib/ingestors/ingestor.rb index a662985cf..1fa4fa7be 100644 --- a/lib/ingestors/ingestor.rb +++ b/lib/ingestors/ingestor.rb @@ -72,6 +72,34 @@ def open_url(url, raise: false, token: nil) end end + # Some URLs automatically redirects the user to another webpage + # This method gets a URL and returns the last redirected URL (as shown by a 30X response or a `meta[http-equiv="Refresh"]` tag) + def get_redirected_url(url, limit = 5) # rubocop:disable Metrics/AbcSize + raise 'Too many redirects' if limit.zero? + + https_url = to_https(url) # some `homepage` were http + response = HTTParty.get(https_url, follow_redirects: true, headers: { 'User-Agent' => config[:user_agent] || 'TeSS Bot' }) + return https_url unless response.headers['content-type']&.include?('html') + + doc = Nokogiri::HTML(response.body) + meta = doc.at('meta[http-equiv="Refresh"]') + if meta && meta.to_s =~ /url=(.+)/i + content = meta['content'] + relative_path = content[/url=(.+)/i, 1] + base = https_url.end_with?('/') ? https_url : "#{https_url}/" + escaped_path = URI::DEFAULT_PARSER.escape(relative_path).to_s + new_url = "#{base}#{escaped_path}" + return get_redirected_url(new_url, limit - 1) + end + https_url + end + + def to_https(url) + uri = URI.parse(url) + uri.scheme = 'https' + uri.to_s + end + def convert_description(input) return input if input.nil? diff --git a/lib/ingestors/ingestor_factory.rb b/lib/ingestors/ingestor_factory.rb index 1da61a790..11d5a2151 100644 --- a/lib/ingestors/ingestor_factory.rb +++ b/lib/ingestors/ingestor_factory.rb @@ -11,6 +11,7 @@ def self.ingestors Ingestors::MaterialCsvIngestor, Ingestors::TessEventIngestor, Ingestors::ZenodoIngestor, + Ingestors::GithubIngestor, ] + taxila_ingestors + llm_ingestors end diff --git a/test/fixtures/files/ingestion/github/api-github-com.json b/test/fixtures/files/ingestion/github/api-github-com.json new file mode 100644 index 000000000..1c10c1115 --- /dev/null +++ b/test/fixtures/files/ingestion/github/api-github-com.json @@ -0,0 +1,25 @@ +{ + "name": "cpluspluscourse", + "full_name": "hsf-training/cpluspluscourse", + "owner": { + "login": "hsf-training" + }, + "html_url": "https://github.com/hsf-training/cpluspluscourse", + "description": "C++ Course Taught at CERN", + "homepage": "", + "topics": [ + "those", + "are", + "keywords" + ], + "license": { + "key": "apache-2.0", + "name": "Apache License 2.0", + "spdx_id": "Apache-2.0" + }, + "archived": true, + "created_at": "2025-09-29T14:38:38Z", + "updated_at": "2025-09-30T14:38:38Z", + "pushed_at": "2025-09-28T14:38:38Z", + "contributors_url": "https://api.github.com/repos/hsf-training/cpluspluscourse/contributors" +} \ No newline at end of file diff --git a/test/fixtures/files/ingestion/github/api-github-io-01.json b/test/fixtures/files/ingestion/github/api-github-io-01.json new file mode 100644 index 000000000..3fad889d9 --- /dev/null +++ b/test/fixtures/files/ingestion/github/api-github-io-01.json @@ -0,0 +1,25 @@ +{ + "name": "python-novice-inflammation", + "full_name": "swcarpentry/python-novice-inflammation", + "owner": { + "login": "swcarpentry" + }, + "html_url": "https://github.com/swcarpentry/python-novice-inflammation", + "description": "This is not going to be read", + "homepage": "https://swcarpentry.github.io/python-novice-inflammation/", + "topics": [ + "key", + "words", + "in topics" + ], + "license": { + "key": "apache-2.0", + "name": "Apache License 2.0", + "spdx_id": "Apache-2.0" + }, + "archived": false, + "created_at": "2025-09-29T14:38:38Z", + "updated_at": "2025-09-30T14:38:38Z", + "pushed_at": "2025-09-28T14:38:38Z", + "contributors_url": "https://api.github.com/repos/swcarpentry/python-novice-inflammation/contributors" +} \ No newline at end of file diff --git a/test/fixtures/files/ingestion/github/api-github-io-02.json b/test/fixtures/files/ingestion/github/api-github-io-02.json new file mode 100644 index 000000000..361090e8d --- /dev/null +++ b/test/fixtures/files/ingestion/github/api-github-io-02.json @@ -0,0 +1,25 @@ +{ + "name": "hsf-training-scikit-hep-webpage", + "full_name": "hsf-training/hsf-training-scikit-hep-webpage", + "owner": { + "login": "hsf-training" + }, + "html_url": "https://github.com/hsf-training/hsf-training-scikit-hep-webpage", + "description": null, + "homepage": "https://hsf-training.github.io/hsf-training-scikit-hep-webpage/", + "topics": [ + "hacktoberfest", + "hey", + "test" + ], + "license": { + "key": "other", + "name": "Other", + "spdx_id": "NOASSERTION" + }, + "archived": false, + "created_at": "2022-03-23T17:00:05Z", + "updated_at": "2025-09-29T06:14:55Z", + "pushed_at": "2025-09-23T20:09:10Z", + "contributors_url": "https://api.github.com/repos/hsf-training/hsf-training-scikit-hep-webpage/contributors" +} \ No newline at end of file diff --git a/test/fixtures/files/ingestion/github/api-modified.json b/test/fixtures/files/ingestion/github/api-modified.json new file mode 100644 index 000000000..131978bc6 --- /dev/null +++ b/test/fixtures/files/ingestion/github/api-modified.json @@ -0,0 +1,11 @@ +{ + "name": "bigchange", + "full_name": "hsf-training/cpluspluscourse", + "html_url": "https://github.com/hsf-training/cpluspluscourse", + "topics": [ + "those", + "are", + "NOT" + ], + "contributors_url": "https://api.github.com/repos/hsf-training/cpluspluscourse/contributors" +} \ No newline at end of file diff --git a/test/fixtures/files/ingestion/github/contributors.json b/test/fixtures/files/ingestion/github/contributors.json new file mode 100644 index 000000000..19474429b --- /dev/null +++ b/test/fixtures/files/ingestion/github/contributors.json @@ -0,0 +1,8 @@ +[ + { + "login": "jane" + }, + { + "login": "doe" + } +] diff --git a/test/fixtures/files/ingestion/github/mock.html b/test/fixtures/files/ingestion/github/mock.html new file mode 100644 index 000000000..4e3b9a165 --- /dev/null +++ b/test/fixtures/files/ingestion/github/mock.html @@ -0,0 +1,12 @@ + + + + The answer to 42 + +

This will not be fetched

+

This is the second p tag of the page fetched because it has more than 50 chars

+ +

Prerequisites

+

1. Be kind

+ + \ No newline at end of file diff --git a/test/fixtures/files/ingestion/github/readme.json b/test/fixtures/files/ingestion/github/readme.json new file mode 100644 index 000000000..9a476ba77 --- /dev/null +++ b/test/fixtures/files/ingestion/github/readme.json @@ -0,0 +1,4 @@ +{ + "name": "README.md", + "content": "IyBgcHloZmAgVHV0b3JpYWwKCioqVGhlIHR1dG9yaWFsIGlzIGJhc2VkIG9m\nZiBvZiBbYHB5aGZgIGB2MC43LjZgXShodHRwczovL3B5cGkub3JnL3Byb2pl\nY3QvcHloZi8wLjcuNi8pKioKClshW0JpbmRlcl0oaHR0cHM6Ly9teWJpbmRl\nci5vcmcvYmFkZ2VfbG9nby5zdmcpXShodHRwczovL215YmluZGVyLm9yZy92\nMi9naC9weWhmL3B5aGYtdHV0b3JpYWwvbWFpbj91cmxwYXRoPWxhYikKWyFb\nSnVweXRlckxpdGVdKGh0dHBzOi8vanVweXRlcmxpdGUucnRmZC5pby9lbi9s\nYXRlc3QvX3N0YXRpYy9iYWRnZS5zdmcpXShodHRwczovL3B5aGYuZ2l0aHVi\nLmlvL3B5aGYtdHV0b3JpYWwvbGl2ZS9sYWIvaW5kZXguaHRtbD9wYXRoPWp1\ncHl0ZXJsaXRlLmlweW5iKQpbIVtET0ldKGh0dHBzOi8vemVub2RvLm9yZy9i\nYWRnZS9ET0kvMTAuNTI4MS96ZW5vZG8uNDY3MDMyMS5zdmcpXShodHRwczov\nL2RvaS5vcmcvMTAuNTI4MS96ZW5vZG8uNDY3MDMyMSkKClshW0RlcGxveSBK\ndXB5dGVyIEJvb2tdKGh0dHBzOi8vZ2l0aHViLmNvbS9weWhmL3B5aGYtdHV0\nb3JpYWwvYWN0aW9ucy93b3JrZmxvd3MvZGVwbG95LWp1cHl0ZXItYm9vay55\nbWwvYmFkZ2Uuc3ZnKV0oaHR0cHM6Ly9weWhmLmdpdGh1Yi5pby9weWhmLXR1\ndG9yaWFsLykKWyFbcHJlLWNvbW1pdC5jaSBzdGF0dXNdKGh0dHBzOi8vcmVz\ndWx0cy5wcmUtY29tbWl0LmNpL2JhZGdlL2dpdGh1Yi9weWhmL3B5aGYtdHV0\nb3JpYWwvbWFpbi5zdmcpXShodHRwczovL3Jlc3VsdHMucHJlLWNvbW1pdC5j\naS9sYXRlc3QvZ2l0aHViL3B5aGYvcHloZi10dXRvcmlhbC9tYWluKQoKIyMg\nU2V0dXAKCiMjIyBVc2luZyBgcGl4aWAgKHJlY29tbWVuZGVkKQoKT24gYW55\nIGB4ODZgIExpbnV4IG1hY2hpbmUgb3IgYW55IG1hY09TIG1hY2hpbmUgZmly\nc3QgaW5zdGFsbCBbYHBpeGlgXShodHRwczovL3BpeGkuc2gvKSBhbmQgdGhl\nbiBmcm9tIHRoZSB0b3AgbGV2ZWwgb2YgdGhlIHJlcG9zaXRvcnkgcnVuCgpg\nYGAKcGl4aSBpbnN0YWxsIC0tZW52aXJvbm1lbnQgYm9vawpgYGAKCiMjIyBV\nc2luZyBhIG1hbnVhbGx5IGNvbnRyb2xsZWQgdmlydHVhbCBlbnZpcm9ubWVu\ndAoKSW4gYSBQeXRob24gdmlydHVhbCBlbnZpcm9ubWVudCBydW4gdGhlIGZv\nbGxvd2luZwoKYGBgCnB5dGhvbiAtbSBwaXAgaW5zdGFsbCAtLXJlcXVpcmUt\naGFzaGVzIC0tcmVxdWlyZW1lbnQgYm9vay9yZXF1aXJlbWVudHMubG9jawpg\nYGAKCiMjIEJ1aWxkCgpUbyBidWlsZCB0aGUgYm9vayBhZnRlciBzZXR1cCBz\naW1wbHkgcnVuCgojIyMgVXNpbmcgYHBpeGlgCgpgYGAKcGl4aSBydW4gYnVp\nbGQKYGBgCgojIyMgTG9jYWwgdmlydHVhbCBlbnZpcm9ubWVudAoKYGBgCm1h\na2UgYnVpbGQKYGBgCgojIyBCdWlsZCBsb2NrIGZpbGUKClRvIGJ1aWxkIGEg\nYHV2IHBpcCBjb21waWxlYCBsb2NrIGZpbGUgZm9yIGxvY2FsIHVzZSBgbm94\nYAoKYGBgCm5veApgYGAKClRvIGJ1aWxkIGEgbG9jayBmaWxlIGZvciBkZXBs\nb3ltZW50IHVzZSBEb2NrZXIgdG8gYXZvaWQgZGlmZmVyZW5jZXMgYmV0d2Vl\nbiBvcGVyYXRpbmcgc3lzdGVtcyB3aXRoCgpgYGAKYmFzaCBsb2NrLnNoCmBg\nYAoKb3IKCmBgYApub3ggLS1zZXNzaW9uIGRvY2tlcgpgYGAKCiMjIFBhc3Qg\ndHV0b3JpYWxzCgoqIFtDb21wdXRhdGlvbmFsIEhFUCBUcmFpbmVlc2hpcCBT\ndW1tZXIgU2Nob29sIDIwMjNdKGh0dHBzOi8vaW5kaWNvLmNlcm4uY2gvZXZl\nbnQvMTI5MzMxMy8pICgyMDIzLTA3LTI2KQoqIFtEQU5DRS9Db0RhUyBXb3Jr\nc2hvcCAyMDIyXShodHRwczovL2luZGljby5jZXJuLmNoL2V2ZW50LzExNTEz\nMjkvKSAoMjAyMi0wNy0yMikKKiBbUHlIRVAgVG9waWNhbCBNZWV0aW5nIEFw\ncmlsIDIwMjFdKGh0dHBzOi8vaW5kaWNvLmNlcm4uY2gvZXZlbnQvOTg1NDI1\nLykgKDIwMjEtMDQtMDcpCiAgIC0gWyFbWmVuZG8gRE9JXShodHRwczovL3pl\nbm9kby5vcmcvYmFkZ2UvRE9JLzEwLjUyODEvemVub2RvLjQ2NzAzMjIuc3Zn\nKV0oaHR0cHM6Ly9kb2kub3JnLzEwLjUyODEvemVub2RvLjQ2NzAzMjIpCiog\nW0xIQyBSZWludGVycHJldGF0aW9uIEZvcnVtIFdvcmtzaG9wIDIwMjFdKGh0\ndHBzOi8vaW5kaWNvLmNlcm4uY2gvZXZlbnQvOTgyNTUzL2NvbnRyaWJ1dGlv\nbnMvNDIxOTQ4Ny8pICgyMDIxLTAyLTE4KQogICAtIFshW1plbm9kbyBET0ld\nKGh0dHBzOi8vemVub2RvLm9yZy9iYWRnZS9ET0kvMTAuNTI4MS96ZW5vZG8u\nNDU0OTQyOC5zdmcpXShodHRwczovL2RvaS5vcmcvMTAuNTI4MS96ZW5vZG8u\nNDU0OTQyOCkKKiBbQVRMQVMgRXhvdGljcyArIFNVU1kgV29ya3Nob3AgMjAy\nMF0oaHR0cHM6Ly9pbmRpY28uY2Vybi5jaC9ldmVudC84OTg5NjUvc2Vzc2lv\nbnMvMzU1ODA2LykgKDIwMjAtMDktMjUpCiogW1B5SEVQIDIwMjBdKGh0dHBz\nOi8vaW5kaWNvLmNlcm4uY2gvZXZlbnQvODgyODI0L2NvbnRyaWJ1dGlvbnMv\nMzkzMTI5Mi8pICgyMDIwLTA3LTE2KQogICAtIFshW1plbm9kbyBET0ldKGh0\ndHBzOi8vemVub2RvLm9yZy9iYWRnZS9ET0kvMTAuNTI4MS96ZW5vZG8uNDE1\nMjkxNi5zdmcpXShodHRwczovL2RvaS5vcmcvMTAuNTI4MS96ZW5vZG8uNDE1\nMjkxNikK\n" +} \ No newline at end of file diff --git a/test/fixtures/files/ingestion/github/releases.json b/test/fixtures/files/ingestion/github/releases.json new file mode 100644 index 000000000..a2ce3d637 --- /dev/null +++ b/test/fixtures/files/ingestion/github/releases.json @@ -0,0 +1,5 @@ +[ + { + "tag_name": "1.0.0" + } +] \ No newline at end of file diff --git a/test/fixtures/files/ingestion/github/sitemap.txt b/test/fixtures/files/ingestion/github/sitemap.txt new file mode 100644 index 000000000..15c538107 --- /dev/null +++ b/test/fixtures/files/ingestion/github/sitemap.txt @@ -0,0 +1,4 @@ +https://github.com/hsf-training/cpluspluscourse +http://swcarpentry.github.io/python-novice-inflammation/ +https://hsf-training.github.io/hsf-training-scikit-hep-webpage/ +https://testwebsite.blabla diff --git a/test/helpers/sitemap_helper_test.rb b/test/helpers/sitemap_helper_test.rb new file mode 100644 index 000000000..d7e3fa9af --- /dev/null +++ b/test/helpers/sitemap_helper_test.rb @@ -0,0 +1,53 @@ +require 'test_helper' + +class SitemapHelperTest < ActionView::TestCase + setup do + @ingestor = DummyIngestor.new + @messages = [] + end + + test 'parse_sitemap with url only returns url only' do + url = 'https://test.com' + assert_equal @ingestor.send(:parse_sitemap, url), [url] + end + + test 'parse_sitemap returns parsed URLs from sitemap.xml' do + sitemap_xml = <<~XML + + + + https://app.com/events/123 + + + https://app.com/events/456 + + + XML + + stub_request(:get, 'https://app.com/events/sitemap.xml') + .to_return(status: 200, body: sitemap_xml, headers: { 'Content-Type' => 'application/xml' }) + + urls = @ingestor.send(:parse_sitemap, 'https://app.com/events/sitemap.xml') + assert_equal ['https://app.com/events/123', 'https://app.com/events/456'], urls + end + + test 'parse_sitemap returns parsed URLs from sitemap.txt' do + sitemap_txt = <<~TXT + https://app.com/events/123 + https://app.com/events/456 + TXT + + stub_request(:get, 'https://app.com/events/sitemap.txt') + .to_return(status: 200, body: sitemap_txt, headers: { 'Content-Type' => 'txt' }) + + urls = @ingestor.send(:parse_sitemap, 'https://app.com/events/sitemap.txt') + assert_equal ['https://app.com/events/123', 'https://app.com/events/456'], urls + end + + private + + def webmock(url, filename) + file = Rails.root.join('test', 'fixtures', 'files', 'ingestion', filename) + WebMock.stub_request(:get, url).to_return(status: 200, headers: {}, body: file.read) + end +end diff --git a/test/test_helper.rb b/test/test_helper.rb index 50e2cd221..ac56d9b0d 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -400,6 +400,21 @@ def mock_facets end end end + + class DummyIngestor < Ingestors::Ingestor + include Ingestors::Concerns::SitemapHelpers + + def self.config + { + key: 'dummy' + } + end + + def read(url) + parse_sitemap(url) + open_url(url) + end + end end # Minitest's `stub` method but ignores any blocks diff --git a/test/unit/ingestors/github_ingestor_test.rb b/test/unit/ingestors/github_ingestor_test.rb new file mode 100644 index 000000000..83935c31d --- /dev/null +++ b/test/unit/ingestors/github_ingestor_test.rb @@ -0,0 +1,407 @@ +# frozen_string_literal: true + +require 'test_helper' + +class GithubIngestorTest < ActiveSupport::TestCase # rubocop:disable Metrics/ClassLength + include ActiveSupport::Testing::TimeHelpers + + class GithubIngestorDecorator + attr_reader :open_url_counter + + def initialize(ingestor) + @ingestor = ingestor + @open_url_counter = 0 + + # Capture decorator reference inside the ingestor’s singleton method + decorator_ref = self + + @ingestor.define_singleton_method(:open_url) do |url, **kwargs| + decorator_ref.instance_variable_set( + :@open_url_counter, + decorator_ref.open_url_counter + 1 + ) + + super(url, **kwargs) + end + end + + def read(url) + @ingestor.read(url) + end + + def materials + @ingestor.materials + end + end + + setup do + @ingestor = Ingestors::GithubIngestor.new + @ingestor_decorated = GithubIngestorDecorator.new(@ingestor) + @user = users(:regular_user) + @content_provider = content_providers(:portal_provider) + mock_timezone # System time zone should not affect test result + + # Sitemap + webmock('https://hsf-training.org/training-center/sitemap.txt', 'github/sitemap.txt') + + # Pages' html + webmock('https://github.com/hsf-training/cpluspluscourse', 'github/mock.html') + webmock('https://swcarpentry.github.io/python-novice-inflammation/', 'github/mock.html') + webmock('https://hsf-training.github.io/hsf-training-scikit-hep-webpage/', 'github/mock.html') + webmock('https://testwebsite.blabla', 'github/mock.html') + + # API + webmock('https://api.github.com/repos/hsf-training/cpluspluscourse', 'github/api-github-com.json') + webmock('https://api.github.com/repos/swcarpentry/python-novice-inflammation', 'github/api-github-io-01.json') + webmock('https://api.github.com/repos/hsf-training/hsf-training-scikit-hep-webpage', 'github/api-github-io-02.json') + + # Contributors + webmock('https://api.github.com/repos/hsf-training/cpluspluscourse/contributors', 'github/contributors.json') + webmock('https://api.github.com/repos/swcarpentry/python-novice-inflammation/contributors', 'github/contributors.json') + webmock('https://api.github.com/repos/hsf-training/hsf-training-scikit-hep-webpage/contributors', 'github/contributors.json') + + # Readme + webmock('https://api.github.com/repos/hsf-training/cpluspluscourse/contents/README.md', 'github/readme.json') + webmock('https://api.github.com/repos/swcarpentry/python-novice-inflammation/contents/README.md', 'github/readme.json') + webmock('https://api.github.com/repos/hsf-training/hsf-training-scikit-hep-webpage/contents/README.md', 'github/readme.json') + + # Releases + webmock('https://api.github.com/repos/hsf-training/cpluspluscourse/releases', 'github/releases.json') + webmock('https://api.github.com/repos/swcarpentry/python-novice-inflammation/releases', 'github/releases.json') + webmock('https://api.github.com/repos/hsf-training/hsf-training-scikit-hep-webpage/releases', 'github/releases.json') + + # HTML + html = File.read(Rails.root.join('test/fixtures/files/ingestion/github/mock.html')) + @doc = Nokogiri::HTML(html) + + # config cache working here + @old_cache_store = Rails.cache + Rails.cache = ActiveSupport::Cache::MemoryStore.new + end + + teardown do + reset_timezone + Rails.cache = @old_cache_store + end + + test 'valid https github.com URL' do + url = 'https://github.com/hsf-training/cpluspluscourse' + expected = 'https://api.github.com/repos/hsf-training/cpluspluscourse' + assert_equal expected, @ingestor.send(:to_github_api, url) + end + + test 'valid http github.com URL' do + url = 'http://github.com/hsf-training/cpluspluscourse' + expected = 'https://api.github.com/repos/hsf-training/cpluspluscourse' + assert_equal expected, @ingestor.send(:to_github_api, url) + end + + test 'invalid github.com URL missing reponame' do + url = 'https://github.com/hsf-training/' + assert_nil @ingestor.send(:to_github_api, url) + end + + test 'invalid github.com URL too deep path' do + url = 'https://github.com/hsf-training/cpluspluscourse/extra' + assert_nil @ingestor.send(:to_github_api, url) + end + + test 'valid github.io URL' do + url = 'https://hsf-training.github.io/cpluspluscourse/' + expected = 'https://api.github.com/repos/hsf-training/cpluspluscourse' + assert_equal expected, @ingestor.send(:to_github_api, url) + end + + test 'invalid github.io with multiple subdomains' do + url = 'https://bad.url.github.io/repo' + assert_nil @ingestor.send(:to_github_api, url) + end + + test 'invalid github.io without reponame' do + url = 'https://hsf-training.github.io/' + assert_nil @ingestor.send(:to_github_api, url) + end + + test 'invalid non-github host' do + url = 'https://gitlab.com/hsf-training/cpluspluscourse' + assert_nil @ingestor.send(:to_github_api, url) + end + + test 'should read sitemap.txt composed of github.com and github.io but avoid keeping the non-github URLs' do + # Read sitemap + @ingestor.read('https://hsf-training.org/training-center/sitemap.txt') + + # There should be 3 github.{com|io} urls among 4 urls + assert_equal 3, @ingestor.materials.count + messages = @ingestor.messages.join("\n") + assert_includes messages, 'Parsing .txt sitemap:' + assert_includes messages, "\n - 4 URLs found" + + assert_difference('Material.count', 3) do + @ingestor.write(@user, @content_provider) + end + + # It thus adds 3 materials + assert_equal 3, @ingestor.stats[:materials][:added] + assert_equal 0, @ingestor.stats[:materials][:updated] + assert_equal 0, @ingestor.stats[:materials][:rejected] + end + + test 'open uri safety' do + ingestor = Ingestors::GithubIngestor.new + dir = Rails.root.join('tmp') + file = dir.join("test#{SecureRandom.urlsafe_base64(20)}").to_s + refute File.exist?(file) + begin + ingestor.open_url("| touch #{file}") + rescue StandardError + end + Dir.children(dir) # This is needed or the `exist?` check below seems to return a stale result + refute File.exist?(file) + end + + test 'should read github.com source' do + @ingestor.read('https://github.com/hsf-training/cpluspluscourse') + @ingestor.write(@user, @content_provider) + + sample = @ingestor.materials.detect { |e| e.title == 'Cpluspluscourse' } + assert sample.persisted? + + assert_equal sample.url, 'https://github.com/hsf-training/cpluspluscourse' + assert_equal sample.description, 'C++ Course Taught at CERN' # taken from api.description + assert_equal sample.keywords, %w[those are keywords] + assert_equal sample.licence, 'Apache-2.0' + assert_equal sample.status, 'Archived' + assert_equal sample.doi, 'https://doi.org/10.5281/zenodo.4670321' + assert_equal sample.version, '1.0.0' + assert_equal sample.date_created, Date.new(2025, 9, 29) + assert_equal sample.date_published, Date.new(2025, 9, 28) + assert_equal sample.date_modified, Date.new(2025, 9, 30) + assert_equal sample.contributors, %w[jane doe] + assert_equal sample.resource_type, ['Github Repository'] # when no gh page + assert_equal sample.prerequisites, '1. Be kind' + end + + test 'should read github.io source' do + @ingestor.read('https://swcarpentry.github.io/python-novice-inflammation/') + @ingestor.write(@user, @content_provider) + + sample = @ingestor.materials.detect { |e| e.title == 'Python Novice Inflammation' } + assert sample.persisted? + + assert_equal sample.url, 'https://swcarpentry.github.io/python-novice-inflammation/' + assert_equal sample.description, "This is the second p tag of the page fetched because it has more than 50 chars\n(...) [Read more...](https://swcarpentry.github.io/python-novice-inflammation/)" + assert_equal sample.status, 'Active' # changed to false + assert_equal sample.resource_type, ['Github Page'] # when gh page + end + + test 'should cache github repo metadata' do + cache = 'github_ingestor_api.github.com_repos_hsf-training_cpluspluscourse' + + @ingestor.read('https://github.com/hsf-training/cpluspluscourse') + + # Assert cache now exists + assert Rails.cache.exist?(cache), "Expected Rails cache #{cache} to exist" + + # Assert cache value + cache_read = Rails.cache.read(cache) + assert_equal cache_read['name'], 'cpluspluscourse' + + # Assert material value matching the cache value + sample = @ingestor.materials.detect { |e| e.title == 'Cpluspluscourse' } + assert_equal sample.title, cache_read['name'].titleize + end + + test 'should write cache of github repo metadata when first read and read cache only when second read' do + cache = 'github_ingestor_api.github.com_repos_hsf-training_cpluspluscourse' + Rails.cache.delete(cache) + @ingestor_decorated.read('https://github.com/hsf-training/cpluspluscourse') + assert_equal @ingestor_decorated.open_url_counter, 4 # we do 4 Github API request per new read + + # Set current cache `name` to something else to assure ourselves we keep this one + cache_modified = Rails.cache.read(cache) + cache_modified['name'] = 'manualchange' + Rails.cache.write(cache, cache_modified) + assert_equal cache_modified['name'], 'manualchange' + + # Reading a second time (within a specific time period) should GET cache and not SET cache + @ingestor_decorated.read('https://github.com/hsf-training/cpluspluscourse') + assert_equal @ingestor_decorated.open_url_counter, 4 # still have 4 Github API request because we used the cached data + cache_second = Rails.cache.read(cache) + + # Assert cache STILL exists + assert Rails.cache.exist?(cache), "Expected Rails cache #{cache} to exist" + + # Assert `name` cache is still the changed one in the second-read cache + assert_equal cache_second['name'], 'manualchange' + + # Material value is independent from this manual cache modification + sample = @ingestor_decorated.materials.detect { |e| e.title == 'Cpluspluscourse' } + assert_equal sample.title, 'Cpluspluscourse' + end + + test 'should set cache and test cache and material before ttl' do + cache = 'github_ingestor_api.github.com_repos_hsf-training_cpluspluscourse' + Rails.cache.delete(cache) + @ingestor_decorated.read('https://github.com/hsf-training/cpluspluscourse') + assert_equal @ingestor_decorated.open_url_counter, 4 # we do 4 Github API request per new read + sample = @ingestor_decorated.materials.detect { |e| e.title == 'Cpluspluscourse' } + assert_equal sample.title, 'Cpluspluscourse' + expires_at7 = Rails.cache.send(:read_entry, cache)&.expires_at + assert_equal (expires_at7 - Time.now.to_f).round, 7 * 24 * 60 * 60 # time to live is 7 days by default + + # Let's change material and cache to be sure it is this one we are using + sample.title = 'Manualchange' + cache_modified = Rails.cache.read(cache) + cache_modified['name'] = 'manualchange' + Rails.cache.write(cache, cache_modified, expires_in: 3.seconds) # set new cache ttl to 3 seconds + assert_equal cache_modified['name'], 'manualchange' + + # Before ttl, content is changed, cache is not changed nor material + # Assert it is really 3 sec + expires_at3 = Rails.cache.send(:read_entry, cache)&.expires_at + assert_equal (expires_at3 - Time.now.to_f).round, 3 + sleep(1) + # Assert ttl is now 2 sec + expires_at2 = Rails.cache.send(:read_entry, cache)&.expires_at + assert_equal (expires_at2 - Time.now.to_f).round, 2 + + # After 1 second, even if the content changes, the previous material and cache are still used + webmock('https://api.github.com/repos/hsf-training/cpluspluscourse', 'github/api-modified.json') + @ingestor_decorated.read('https://github.com/hsf-training/cpluspluscourse') + assert_equal @ingestor_decorated.open_url_counter, 4 # we still have 4 GitHub APIs requests, we use cached data even if content changes, but we are still < TTL + sample_second = @ingestor_decorated.materials.detect { |e| e.title == 'Manualchange' } + cache_after = Rails.cache.read(cache) + # manualchange instead of bigchange + assert_equal sample_second.title, 'Manualchange' + assert_equal cache_after['name'], 'manualchange' + # Assert ttl is still 2 sec + expires_at2again = Rails.cache.send(:read_entry, cache)&.expires_at + assert_equal (expires_at2again - Time.now.to_f).round, 2 + end + + test 'should set cache then test cache and material after ttl' do + cache = 'github_ingestor_api.github.com_repos_hsf-training_cpluspluscourse' + Rails.cache.delete(cache) + @ingestor_decorated.read('https://github.com/hsf-training/cpluspluscourse') + assert_equal @ingestor_decorated.open_url_counter, 4 # one new read, 4 GitHub API requests + sample = @ingestor_decorated.materials.detect { |e| e.title == 'Cpluspluscourse' } + sample.title = 'Manualchange' + cache_modified = Rails.cache.read(cache) + cache_modified['name'] = 'manualchange' + Rails.cache.write(cache, cache_modified, expires_in: 2.seconds) + assert_equal Rails.cache.read(cache)['name'], 'manualchange' + + # After ttl, if content – and thus cache – are changed, then the material should be partially changed + expires_at2 = Rails.cache.send(:read_entry, cache)&.expires_at + assert_equal (expires_at2 - Time.now.to_f).round, 2 + # After 2 seconds, if the content changes, the cache expires and we fetch the newest content + sleep(2) + # We do not have cache anymore + assert_nil Rails.cache.read(cache) + # But the material is still here with the manual modification + sample_after_ttl = @ingestor_decorated.materials.detect { |e| e.title == cache_modified['name'].titleize } + assert_equal sample_after_ttl.title, cache_modified['name'].titleize + + # Now if content has changed and we read it + webmock('https://api.github.com/repos/hsf-training/cpluspluscourse', 'github/api-modified.json') + @ingestor_decorated.read('https://github.com/hsf-training/cpluspluscourse') + assert_equal @ingestor_decorated.open_url_counter, 5 # one new read after TTL, here it is 1 added new GitHub API request because we only changed this one in the test + + # We should have our cache changed as well as the ENTIRE material + cache_after_ttl = Rails.cache.read(cache) + assert_equal cache_after_ttl['name'], 'bigchange' + sample_after_ttl_and_change = @ingestor_decorated.materials.detect { |e| e.title == cache_after_ttl['name'].titleize } + # The material title has changed – because api-modified.json only have 'name' + assert_equal sample_after_ttl_and_change.title, cache_after_ttl['name'].titleize + # The material keywords is the new one + assert_equal sample_after_ttl_and_change.keywords, %w[those are NOT] + # The material description is the new one: nil + assert_nil sample_after_ttl_and_change.description + end + + test 'std errors when exception is raised' do + @ingestor.stub(:parse_sitemap, ->(*) { raise StandardError, 'test failure' }) do + mock_logger = Minitest::Mock.new + mock_logger.expect(:error, nil, ['StandardError: read() failed, test failure']) + + Rails.stub(:logger, mock_logger) do + @ingestor.send(:read, 'https://github.com/example') + end + mock_logger.verify + end + + assert true, 'Mock verification passed' + end + + test 'prereq_node? returns true when id includes prereq' do + node = Nokogiri::XML::Node.new('div', @doc) + node['id'] = 'prerequisites' + assert @ingestor.send(:prereq_node?, node) + end + + test 'prereq_node? returns true when class includes prereq' do + node = Nokogiri::XML::Node.new('div', @doc) + node['class'] = 'has_prereq_section' + assert @ingestor.send(:prereq_node?, node) + end + + test 'prereq_node? returns false for unrelated id/class' do + node = Nokogiri::XML::Node.new('div', @doc) + node['id'] = 'requirements' + refute @ingestor.send(:prereq_node?, node) + end + + test 'extract_following_paragraphs finds immediate sibling paragraphs' do + doc = Nokogiri::HTML(<<~HTML) +
+

First paragraph

+
  • List item
+

Stop here

+ HTML + + node = doc.at_xpath('//*[@id="prereq"]') + paragraphs = [] + @ingestor.send(:extract_following_paragraphs, node, paragraphs) + + assert_equal 2, paragraphs.size + assert_equal %w[p ul], paragraphs.map(&:name) + end + + test 'extract_nested_paragraphs finds paragraphs inside the node' do + doc = Nokogiri::HTML(<<~HTML) +
+

Nested paragraph

+
  • Nested list
+
+ HTML + + node = doc.at_xpath('//*[@id="prereq"]') + paragraphs = [] + @ingestor.send(:extract_nested_paragraphs, node, paragraphs) + + assert_equal 2, paragraphs.size + assert_equal %w[p ul], paragraphs.map(&:name) + end + + test 'fetch_prerequisites_from_id_or_class collects prerequisites correctly' do + doc = Nokogiri::HTML(<<~HTML) +
+

Be kind

+

Have Ruby installed

+ HTML + + paragraphs = [] + result = @ingestor.send(:fetch_prerequisites_from_id_or_class, doc, paragraphs) + + assert_equal 2, result.size + assert_equal 'Be kind', result.first.text + end + + private + + def webmock(url, filename) + file = Rails.root.join('test', 'fixtures', 'files', 'ingestion', filename) + WebMock.stub_request(:get, url).to_return(status: 200, headers: {}, body: file.read) + end +end \ No newline at end of file diff --git a/test/unit/ingestors/ingestor_test.rb b/test/unit/ingestors/ingestor_test.rb index bbb7feb69..28ad68760 100644 --- a/test/unit/ingestors/ingestor_test.rb +++ b/test/unit/ingestors/ingestor_test.rb @@ -113,4 +113,78 @@ class IngestorTest < ActiveSupport::TestCase assert_nil(event.language) end + test 'open_url returns content when URL is valid' do + ingestor = DummyIngestor.new + stub_request(:get, 'https://example.com').to_return(body: 'ok', status: 200) + + result = ingestor.open_url('https://example.com') + assert_equal 'ok', result.read + end + + test 'open_url raises HTTPRedirect after too many retries' do + ingestor = DummyIngestor.new + fake_uri = URI('https://example.com/') + + URI.stub(:parse, fake_uri) do + fake_uri.define_singleton_method(:open) do |*_args| + raise OpenURI::HTTPRedirect.new('Redirect', 1, URI('https://redirected.com')) + end + + assert_raises(OpenURI::HTTPRedirect) do + ingestor.open_url('https://example.com/') + end + end + end + + test 'open_url raises HTTPError' do + ingestor = DummyIngestor.new + stub_request(:get, 'https://bad.com') + .to_raise(OpenURI::HTTPError.new('404 not found', StringIO.new)) + + result = ingestor.open_url('https://bad.com') + assert_nil result + assert_includes ingestor.instance_variable_get(:@messages).last, + "Couldn't open URL https://bad.com: 404 not found" + end + + test 'get_redirected_url follows meta refresh redirect' do + ingestor = DummyIngestor.new + html_with_meta = '' + redirected_html = 'FinalDone' + + HTTParty.stub(:get, lambda { |url, **| + case url + when 'https://example.com/' + OpenStruct.new(headers: { 'content-type' => 'text/html' }, body: html_with_meta) + when 'https://example.com//redirected' + OpenStruct.new(headers: { 'content-type' => 'text/html' }, body: redirected_html) + else + raise "Unexpected URL: #{url}" + end + }) do + result = ingestor.get_redirected_url('http://example.com/') + assert_equal 'https://example.com//redirected', result + end + end + + test 'get_redirected_url returns original url when no meta redirect' do + ingestor = DummyIngestor.new + html_no_meta = 'No redirect' + + HTTParty.stub(:get, ->(_url, **) { OpenStruct.new(headers: { 'content-type' => 'text/html' }, body: html_no_meta) }) do + result = ingestor.get_redirected_url('http://example.com/') + assert_equal 'https://example.com/', result + end + end + + test 'get_redirected_url raises when too many redirects' do + ingestor = DummyIngestor.new + html_with_meta = '' + + HTTParty.stub(:get, ->(_url, **) { OpenStruct.new(headers: { 'content-type' => 'text/html' }, body: html_with_meta) }) do + assert_raises(RuntimeError, 'Too many redirects') do + ingestor.get_redirected_url('http://example.com/', 0) + end + end + end end