Skip to content

Commit ceff245

Browse files
kennethriojakennethrioja
committed
test(github-ingestor): added
1 parent ec6bbe3 commit ceff245

11 files changed

Lines changed: 373 additions & 47 deletions

File tree

‎lib/ingestors/github_ingestor.rb‎

Lines changed: 41 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -7,14 +7,20 @@
77
module Ingestors
88
# GithubIngestor fetches repository information from GitHub to populate the materials' metadata.
99
# API requests counter:
10-
# 1. Get the repo general metadata (json) from #{GITHUB_API_BASE}/#{full_name}
11-
# 2. Get the doi using '#{GITHUB_API_BASE}/#{full_name}/contents/README.md'
12-
# 3. Get the version using '#{GITHUB_API_BASE}/#{full_name}/releases'
13-
# 4. Get the list of contributors using '#{GITHUB_API_BASE}/#{full_name}/contributors'
10+
# 1. Get the repo's general metadata #{GITHUB_API_BASE}/#{full_name}
11+
# 2. Get the doi #{GITHUB_API_BASE}/#{full_name}/contents/README.md
12+
# 3. Get the version/release #{GITHUB_API_BASE}/#{full_name}/releases
13+
# 4. Get the list of contributors #{GITHUB_API_BASE}/#{full_name}/contributors
14+
# Searched keys:
15+
# api -> name, full_name, owner.login, html_url, description, homepage, topics, license.{key, spdx}, archived, created_at, pushed_at, updated_at, contributors_url,
16+
# doi -> content
17+
# version -> tag_name (first)
18+
# contributors -> login (from all entries)
1419
class GithubIngestor < Ingestor
1520
GITHUB_API_BASE = 'https://api.github.com/repos'.freeze
1621
GITHUB_COM_BASE = 'https://github.com'.freeze
1722
REDIS = Redis.new(url: TeSS::Config.redis_url)
23+
TTL_SEC = 30 * 24 * 60 * 60 # time to live in second after the cache is deleted
1824

1925
def self.config
2026
{
@@ -61,27 +67,30 @@ def read(source_url)
6167
end
6268
next unless repo_data
6369

64-
# From the data, pass it to materials
65-
to_material(repo_data)
70+
# From the data, translate it to materials
71+
material = to_material(repo_data)
72+
73+
# Add to material
74+
add_material material
6675
rescue StandardError => e
6776
@messages << "#{self.class.name} failed for #{url}, #{e.message}"
6877
end
6978
end
7079

71-
# private
80+
private
7281

7382
# Fetch cached data or opens webpage/api and cache it
7483
# I chose to cache because GitHub limmits up to 60 requests per hour for unauthenticated user
7584
# https://docs.github.com/en/rest/using-the-rest-api/rate-limits-for-the-rest-api?apiVersion=2022-11-28#primary-rate-limit-for-unauthenticated-users
7685
# key: string key for the cache
7786
# ttl: time-to-live in seconds (default 30 days)
78-
def cache_or_set(key, ttl: 30 * 24 * 60 * 60)
87+
def cache_or_set(key, ttl: TTL_SEC)
7988
Rails.logger.info "[Github Cache] GET cache #{key}"
8089
cached = REDIS.get(key)
8190
return JSON.parse(cached) if cached
8291

8392
data = yield
84-
Rails.logger.info "[GithubCache] SET cache #{key}\nData: #{data.to_json}"
93+
Rails.logger.info "[Github Cache] SET cache #{key}"
8594
REDIS.set(key, data.to_json, ex: ttl) if data
8695
data
8796
rescue StandardError => e
@@ -109,14 +118,15 @@ def to_github_api(url)
109118

110119
# Sets material hash keys and values and add them to material
111120
def to_material(repo_data)
112-
homepage_url = get_redirected_url(repo_data['homepage'])
113-
response = HTTParty.get(homepage_url, follow_redirects: true, headers: { 'User-Agent' => config[:user_agent] })
121+
homepage_nil_or_empty = repo_data['homepage'].nil? || repo_data['homepage'].empty?
122+
url = homepage_nil_or_empty ? repo_data['html_url'] : get_redirected_url(repo_data['homepage']) # if no page, put github.com repo
123+
response = HTTParty.get(url, follow_redirects: true, headers: { 'User-Agent' => config[:user_agent] })
114124
doc = Nokogiri::HTML(response.body)
115125

116126
material = OpenStruct.new
117-
material.title = "GitHub #{repo_data['homepage'].to_s.strip.empty? ? 'repo' : 'page'}: #{repo_data['name'].titleize} (#{repo_data['owner']['login']})"
118-
material.url = repo_data['homepage'].to_s.strip.empty? ? repo_data['html_url'] : homepage_url
119-
material.description = repo_data['homepage'].to_s.strip.empty? ? repo_data['description'] : fetch_definition(doc, homepage_url)
127+
material.title = repo_data['name'].titleize
128+
material.url = url
129+
material.description = homepage_nil_or_empty ? repo_data['description'] : fetch_definition(doc, url)
120130
material.keywords = repo_data['topics']
121131
material.licence = fetch_licence(repo_data['license'])
122132
material.status = repo_data['archived'] ? 'Archived' : 'Active'
@@ -127,31 +137,38 @@ def to_material(repo_data)
127137
material.date_published = repo_data['pushed_at']
128138
material.date_modified = repo_data['updated_at']
129139
material.contributors = fetch_contributors(repo_data['contributors_url'])
130-
material.resource_type = 'GitHub repository'
140+
material.resource_type = homepage_nil_or_empty ? ['Github Repository'] : ['Github Page']
131141
material.prerequisites = fetch_prerequisites(doc)
132142

133-
add_material material
143+
material
134144
end
135145

136146
# URL – Some github homepages automatically redirects the user to another webpage
137147
# This method will get the last redirected URL (as shown by a 30X response or a `meta[http-equiv="Refresh"]` tag)
138148
def get_redirected_url(url, limit = 5)
139149
raise 'Too many redirects' if limit.zero?
140150

141-
response = HTTParty.get(url, follow_redirects: true, headers: { 'User-Agent' => config[:user_agent] })
142-
return url unless response.headers['content-type']&.include?('html')
151+
https_url = to_https(url) # some `homepage` were http
152+
response = HTTParty.get(https_url, follow_redirects: true, headers: { 'User-Agent' => config[:user_agent] })
153+
return https_url unless response.headers['content-type']&.include?('html')
143154

144155
doc = Nokogiri::HTML(response.body)
145156
meta = doc.at('meta[http-equiv="Refresh"]')
146157
if meta && meta.to_s =~ /url=(.+)/i
147158
content = meta['content']
148159
relative_path = content[/url=(.+)/i, 1]
149-
base = url.end_with?('/') ? url : "#{url}/"
160+
base = https_url.end_with?('/') ? https_url : "#{https_url}/"
150161
escaped_path = URI::DEFAULT_PARSER.escape(relative_path).to_s
151162
new_url = "#{base}#{escaped_path}"
152163
return get_redirected_url(new_url, limit - 1)
153164
end
154-
url
165+
https_url
166+
end
167+
168+
def to_https(url)
169+
uri = URI.parse(url)
170+
uri.scheme = 'https'
171+
uri.to_s
155172
end
156173

157174
# DEFINITION – Opens the GitHub homepage, fetches the 3 first >25 char <p> tags'text
@@ -164,11 +181,10 @@ def fetch_definition(doc, url)
164181
p_txt = p&.text&.strip&.gsub(/\s+/, ' ')
165182
next if (p_txt.length < 25) || round.zero?
166183

167-
desc = "#{desc}#{p_txt}
168-
"
184+
desc = "#{desc}\n#{p_txt}"
169185
round -= 1
170186
end
171-
"#{desc}(...) [Read more...](#{url})"
187+
"#{desc}\n(...) [Read more...](#{url})"
172188
end
173189

174190
# LICENCE – Get proper licence
@@ -177,7 +193,7 @@ def fetch_licence(licence)
177193
return 'notspecified' if licence.nil? || licence == 'null'
178194
return 'other-at' if licence['key'] == 'other'
179195

180-
licence['key'].upcase_first
196+
licence['spdx_id']
181197
end
182198

183199
# DOI – Fetches DOI from various sources in a repo
@@ -231,7 +247,7 @@ def fetch_contributors(contributors_url)
231247
content = open_url(contributors_url)
232248
content ? JSON.parse(content.read) : nil
233249
end
234-
contributors.map { |c| c['login'] }
250+
contributors.map { |c| (c['login']) }
235251
rescue StandardError
236252
nil
237253
end

‎test/fixtures/files/ingestion/github/api-github-com.json‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
{
2+
"name": "cpluspluscourse",
3+
"full_name": "hsf-training/cpluspluscourse",
4+
"owner": {
5+
"login": "hsf-training"
6+
},
7+
"html_url": "https://github.com/hsf-training/cpluspluscourse",
8+
"description": "C++ Course Taught at CERN",
9+
"homepage": "",
10+
"topics": [
11+
"those",
12+
"are",
13+
"keywords"
14+
],
15+
"license": {
16+
"key": "apache-2.0",
17+
"name": "Apache License 2.0",
18+
"spdx_id": "Apache-2.0"
19+
},
20+
"archived": true,
21+
"created_at": "2025-09-29T14:38:38Z",
22+
"updated_at": "2025-09-30T14:38:38Z",
23+
"pushed_at": "2025-09-28T14:38:38Z"
24+
}

‎test/fixtures/files/ingestion/github/api-github-io-01.json‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
{
2+
"name": "python-novice-inflammation",
3+
"full_name": "swcarpentry/python-novice-inflammation",
4+
"owner": {
5+
"login": "swcarpentry"
6+
},
7+
"html_url": "https://github.com/swcarpentry/python-novice-inflammation",
8+
"description": "This is not going to be read",
9+
"homepage": "https://swcarpentry.github.io/python-novice-inflammation/",
10+
"topics": [
11+
"key",
12+
"words",
13+
"in topics"
14+
],
15+
"license": {
16+
"key": "apache-2.0",
17+
"name": "Apache License 2.0",
18+
"spdx_id": "Apache-2.0"
19+
},
20+
"archived": false,
21+
"created_at": "2025-09-29T14:38:38Z",
22+
"updated_at": "2025-09-30T14:38:38Z",
23+
"pushed_at": "2025-09-28T14:38:38Z"
24+
}

‎test/fixtures/files/ingestion/github/api-github-io-02.json‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
{
2+
"name": "hsf-training-scikit-hep-webpage",
3+
"full_name": "hsf-training/hsf-training-scikit-hep-webpage",
4+
"owner": {
5+
"login": "hsf-training"
6+
},
7+
"html_url": "https://github.com/hsf-training/hsf-training-scikit-hep-webpage",
8+
"description": null,
9+
"homepage": "https://hsf-training.github.io/hsf-training-scikit-hep-webpage/",
10+
"topics": [
11+
"hacktoberfest",
12+
"hey",
13+
"test"
14+
],
15+
"license": {
16+
"key": "other",
17+
"name": "Other",
18+
"spdx_id": "NOASSERTION"
19+
},
20+
"archived": false,
21+
"created_at": "2022-03-23T17:00:05Z",
22+
"updated_at": "2025-09-29T06:14:55Z",
23+
"pushed_at": "2025-09-23T20:09:10Z",
24+
"contributors_url": "https://api.github.com/repos/hsf-training/hsf-training-scikit-hep-webpage/contributors"
25+
}

‎test/fixtures/files/ingestion/github/api-modified.json‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
{
2+
"name": "bigchange",
3+
"html_url": "https://github.com/hsf-training/cpluspluscourse",
4+
"topics": [
5+
"those",
6+
"are",
7+
"NOT"
8+
]
9+
}

‎test/fixtures/files/ingestion/github/contributors.json‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
[
2+
{
3+
"login": "jane"
4+
},
5+
{
6+
"login": "doe"
7+
}
8+
]

‎test/fixtures/files/ingestion/github/mock.html‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
<!DOCTYPE html>
2+
<html lang="en">
3+
<head>
4+
<title>The answer to 42</title>
5+
</head>
6+
<p>This is the first p tag of the page</p>
7+
<p>This is the second p tag of the page</p>
8+
<body>
9+
<h1>Prerequisites</h1>
10+
<p>1. Be kind</p>
11+
</body>
12+
</html>

‎test/fixtures/files/ingestion/github/readme.json‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
{
2+
"name": "README.md",
3+
"content": "IyBgcHloZmAgVHV0b3JpYWwKCioqVGhlIHR1dG9yaWFsIGlzIGJhc2VkIG9m\nZiBvZiBbYHB5aGZgIGB2MC43LjZgXShodHRwczovL3B5cGkub3JnL3Byb2pl\nY3QvcHloZi8wLjcuNi8pKioKClshW0JpbmRlcl0oaHR0cHM6Ly9teWJpbmRl\nci5vcmcvYmFkZ2VfbG9nby5zdmcpXShodHRwczovL215YmluZGVyLm9yZy92\nMi9naC9weWhmL3B5aGYtdHV0b3JpYWwvbWFpbj91cmxwYXRoPWxhYikKWyFb\nSnVweXRlckxpdGVdKGh0dHBzOi8vanVweXRlcmxpdGUucnRmZC5pby9lbi9s\nYXRlc3QvX3N0YXRpYy9iYWRnZS5zdmcpXShodHRwczovL3B5aGYuZ2l0aHVi\nLmlvL3B5aGYtdHV0b3JpYWwvbGl2ZS9sYWIvaW5kZXguaHRtbD9wYXRoPWp1\ncHl0ZXJsaXRlLmlweW5iKQpbIVtET0ldKGh0dHBzOi8vemVub2RvLm9yZy9i\nYWRnZS9ET0kvMTAuNTI4MS96ZW5vZG8uNDY3MDMyMS5zdmcpXShodHRwczov\nL2RvaS5vcmcvMTAuNTI4MS96ZW5vZG8uNDY3MDMyMSkKClshW0RlcGxveSBK\ndXB5dGVyIEJvb2tdKGh0dHBzOi8vZ2l0aHViLmNvbS9weWhmL3B5aGYtdHV0\nb3JpYWwvYWN0aW9ucy93b3JrZmxvd3MvZGVwbG95LWp1cHl0ZXItYm9vay55\nbWwvYmFkZ2Uuc3ZnKV0oaHR0cHM6Ly9weWhmLmdpdGh1Yi5pby9weWhmLXR1\ndG9yaWFsLykKWyFbcHJlLWNvbW1pdC5jaSBzdGF0dXNdKGh0dHBzOi8vcmVz\ndWx0cy5wcmUtY29tbWl0LmNpL2JhZGdlL2dpdGh1Yi9weWhmL3B5aGYtdHV0\nb3JpYWwvbWFpbi5zdmcpXShodHRwczovL3Jlc3VsdHMucHJlLWNvbW1pdC5j\naS9sYXRlc3QvZ2l0aHViL3B5aGYvcHloZi10dXRvcmlhbC9tYWluKQoKIyMg\nU2V0dXAKCiMjIyBVc2luZyBgcGl4aWAgKHJlY29tbWVuZGVkKQoKT24gYW55\nIGB4ODZgIExpbnV4IG1hY2hpbmUgb3IgYW55IG1hY09TIG1hY2hpbmUgZmly\nc3QgaW5zdGFsbCBbYHBpeGlgXShodHRwczovL3BpeGkuc2gvKSBhbmQgdGhl\nbiBmcm9tIHRoZSB0b3AgbGV2ZWwgb2YgdGhlIHJlcG9zaXRvcnkgcnVuCgpg\nYGAKcGl4aSBpbnN0YWxsIC0tZW52aXJvbm1lbnQgYm9vawpgYGAKCiMjIyBV\nc2luZyBhIG1hbnVhbGx5IGNvbnRyb2xsZWQgdmlydHVhbCBlbnZpcm9ubWVu\ndAoKSW4gYSBQeXRob24gdmlydHVhbCBlbnZpcm9ubWVudCBydW4gdGhlIGZv\nbGxvd2luZwoKYGBgCnB5dGhvbiAtbSBwaXAgaW5zdGFsbCAtLXJlcXVpcmUt\naGFzaGVzIC0tcmVxdWlyZW1lbnQgYm9vay9yZXF1aXJlbWVudHMubG9jawpg\nYGAKCiMjIEJ1aWxkCgpUbyBidWlsZCB0aGUgYm9vayBhZnRlciBzZXR1cCBz\naW1wbHkgcnVuCgojIyMgVXNpbmcgYHBpeGlgCgpgYGAKcGl4aSBydW4gYnVp\nbGQKYGBgCgojIyMgTG9jYWwgdmlydHVhbCBlbnZpcm9ubWVudAoKYGBgCm1h\na2UgYnVpbGQKYGBgCgojIyBCdWlsZCBsb2NrIGZpbGUKClRvIGJ1aWxkIGEg\nYHV2IHBpcCBjb21waWxlYCBsb2NrIGZpbGUgZm9yIGxvY2FsIHVzZSBgbm94\nYAoKYGBgCm5veApgYGAKClRvIGJ1aWxkIGEgbG9jayBmaWxlIGZvciBkZXBs\nb3ltZW50IHVzZSBEb2NrZXIgdG8gYXZvaWQgZGlmZmVyZW5jZXMgYmV0d2Vl\nbiBvcGVyYXRpbmcgc3lzdGVtcyB3aXRoCgpgYGAKYmFzaCBsb2NrLnNoCmBg\nYAoKb3IKCmBgYApub3ggLS1zZXNzaW9uIGRvY2tlcgpgYGAKCiMjIFBhc3Qg\ndHV0b3JpYWxzCgoqIFtDb21wdXRhdGlvbmFsIEhFUCBUcmFpbmVlc2hpcCBT\ndW1tZXIgU2Nob29sIDIwMjNdKGh0dHBzOi8vaW5kaWNvLmNlcm4uY2gvZXZl\nbnQvMTI5MzMxMy8pICgyMDIzLTA3LTI2KQoqIFtEQU5DRS9Db0RhUyBXb3Jr\nc2hvcCAyMDIyXShodHRwczovL2luZGljby5jZXJuLmNoL2V2ZW50LzExNTEz\nMjkvKSAoMjAyMi0wNy0yMikKKiBbUHlIRVAgVG9waWNhbCBNZWV0aW5nIEFw\ncmlsIDIwMjFdKGh0dHBzOi8vaW5kaWNvLmNlcm4uY2gvZXZlbnQvOTg1NDI1\nLykgKDIwMjEtMDQtMDcpCiAgIC0gWyFbWmVuZG8gRE9JXShodHRwczovL3pl\nbm9kby5vcmcvYmFkZ2UvRE9JLzEwLjUyODEvemVub2RvLjQ2NzAzMjIuc3Zn\nKV0oaHR0cHM6Ly9kb2kub3JnLzEwLjUyODEvemVub2RvLjQ2NzAzMjIpCiog\nW0xIQyBSZWludGVycHJldGF0aW9uIEZvcnVtIFdvcmtzaG9wIDIwMjFdKGh0\ndHBzOi8vaW5kaWNvLmNlcm4uY2gvZXZlbnQvOTgyNTUzL2NvbnRyaWJ1dGlv\nbnMvNDIxOTQ4Ny8pICgyMDIxLTAyLTE4KQogICAtIFshW1plbm9kbyBET0ld\nKGh0dHBzOi8vemVub2RvLm9yZy9iYWRnZS9ET0kvMTAuNTI4MS96ZW5vZG8u\nNDU0OTQyOC5zdmcpXShodHRwczovL2RvaS5vcmcvMTAuNTI4MS96ZW5vZG8u\nNDU0OTQyOCkKKiBbQVRMQVMgRXhvdGljcyArIFNVU1kgV29ya3Nob3AgMjAy\nMF0oaHR0cHM6Ly9pbmRpY28uY2Vybi5jaC9ldmVudC84OTg5NjUvc2Vzc2lv\nbnMvMzU1ODA2LykgKDIwMjAtMDktMjUpCiogW1B5SEVQIDIwMjBdKGh0dHBz\nOi8vaW5kaWNvLmNlcm4uY2gvZXZlbnQvODgyODI0L2NvbnRyaWJ1dGlvbnMv\nMzkzMTI5Mi8pICgyMDIwLTA3LTE2KQogICAtIFshW1plbm9kbyBET0ldKGh0\ndHBzOi8vemVub2RvLm9yZy9iYWRnZS9ET0kvMTAuNTI4MS96ZW5vZG8uNDE1\nMjkxNi5zdmcpXShodHRwczovL2RvaS5vcmcvMTAuNTI4MS96ZW5vZG8uNDE1\nMjkxNikK\n"
4+
}

‎test/fixtures/files/ingestion/github/releases.json‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
[
2+
{
3+
"tag_name": "1.0.0"
4+
}
5+
]

‎test/fixtures/files/ingestion/github/sitemap.txt‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
https://github.com/hsf-training/cpluspluscourse
2+
http://swcarpentry.github.io/python-novice-inflammation/
3+
https://hsf-training.github.io/hsf-training-scikit-hep-webpage/
4+
https://testwebsite.blabla

0 commit comments

Comments
 (0)