Skip to content

Commit 17528d9

Browse files
jmerlejmerle
committed
rxjs: finish scraper and filters
1 parent 2043885 commit 17528d9

File tree

6 files changed

+117
-73
lines changed

6 files changed

+117
-73
lines changed

‎assets/javascripts/templates/pages/about_tmpl.coffee‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -211,8 +211,7 @@ credits = [
211211
'2017 Cypress.io',
212212
'MIT',
213213
'https://raw.githubusercontent.com/cypress-io/cypress-documentation/develop/LICENSE.md'
214-
],
215-
[
214+
], [
216215
'D',
217216
'1999-2018 The D Language Foundation',
218217
'Boost',
@@ -572,8 +571,7 @@ credits = [
572571
'2016-2018, The Pony Developers & 2014-2015, Causality Ltd.',
573572
'BSD',
574573
'https://raw.githubusercontent.com/ponylang/ponyc/master/LICENSE'
575-
],
576-
[
574+
], [
577575
'PostgreSQL',
578576
'1996-2018 The PostgreSQL Global Development Group<br>&copy; 1994 The Regents of the University of California',
579577
'PostgreSQL',
@@ -648,13 +646,17 @@ credits = [
648646
'2010 The Rust Project Developers',
649647
'MIT',
650648
'https://raw.githubusercontent.com/rust-lang/rust/master/LICENSE-MIT'
649+
], [
650+
'RxJS',
651+
'2015-2018 Google, Inc., Netflix, Inc., Microsoft Corp. and contributors',
652+
'Apache',
653+
'https://raw.githubusercontent.com/ReactiveX/rxjs/master/LICENSE.txt'
651654
], [
652655
'Salt Stack',
653656
'2019 SaltStack',
654657
'Apache',
655658
'https://raw.githubusercontent.com/saltstack/salt/develop/LICENSE'
656-
],
657-
[
659+
], [
658660
'Sass',
659661
'2006-2016 Hampton Catlin, Nathan Weizenbaum, and Chris Eppstein',
660662
'MIT',
@@ -664,8 +666,7 @@ credits = [
664666
'2002-2019 EPFL, with contributions from Lightbend',
665667
'Apache',
666668
'https://raw.githubusercontent.com/scala/scala-lang/master/license.md'
667-
],
668-
[
669+
], [
669670
'scikit-image',
670671
'2011 the scikit-image team',
671672
'BSD',
@@ -765,8 +766,7 @@ credits = [
765766
'2003-2019 WordPress Foundation',
766767
'GPLv2+',
767768
'https://wordpress.org/about/license/'
768-
],
769-
[
769+
], [
770770
'Yarn',
771771
'2016-present Yarn Contributors',
772772
'BSD',

‎lib/docs/filters/rxjs/clean_html.rb‎

Lines changed: 46 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,11 @@ def call
77
at_css('h1').content = 'RxJS Documentation'
88
end
99

10+
if at_css('h1').nil?
11+
title = subpath.rpartition('/').last.titleize
12+
doc.prepend_child("<h1>#{title}</h1>")
13+
end
14+
1015
css('br', 'hr', '.material-icons', '.header-link', '.breadcrumb').remove
1116

1217
css('.content', 'article', '.api-header', 'section', '.instance-member').each do |node|
@@ -65,6 +70,16 @@ def call
6570

6671
if node['class'] && node['class'].include?('api-heading')
6772
node.name = 'h3'
73+
74+
unless node.ancestors('.instance-method').empty?
75+
matches = node.inner_html.scan(/([^(& ]+)[(&]/)
76+
77+
unless matches.empty? || matches[0][0] == 'constructor'
78+
node['name'] = matches[0][0]
79+
node['id'] = node['name'].downcase + '-'
80+
end
81+
end
82+
6883
node.inner_html = "<code>#{node.inner_html}</code>"
6984
end
7085

@@ -77,25 +92,48 @@ def call
7792
node.remove_attribute('class')
7893
end
7994

80-
css('h1[class]').remove_attr('class')
81-
css('table[class]').remove_attr('class')
82-
css('table[width]').remove_attr('width')
83-
css('tr[style]').remove_attr('style')
95+
css('td > .overloads').each do |node|
96+
node.replace node.at_css('.detail-contents')
97+
end
98+
99+
css('td.short-description p').each do |node|
100+
signature = node.parent.parent.next_element.at_css('h3[id]')
101+
signature.after(node) unless signature.nil?
102+
end
84103

85-
if at_css('.api-type-label.module')
86-
at_css('h1').content = subpath.remove('api/')
104+
css('.method-table').each do |node|
105+
node.replace node.at_css('tbody')
87106
end
88107

89-
css('th h3').each do |node|
90-
node.name = 'span'
108+
css('.api-body > table > caption').each do |node|
109+
node.name = 'center'
110+
lift_out_of_table node
91111
end
92112

113+
css('.api-body > table > tbody > tr:not([class]) > td > *').each do |node|
114+
lift_out_of_table node
115+
end
116+
117+
css('.api-body > table').each do |node|
118+
node.remove if node.content.strip.blank?
119+
end
120+
121+
css('h1[class]').remove_attr('class')
122+
css('table[class]').remove_attr('class')
123+
css('table[width]').remove_attr('width')
124+
css('tr[style]').remove_attr('style')
125+
93126
css('code code').each do |node|
94127
node.before(node.children).remove
95128
end
96129

97130
doc
98131
end
132+
133+
def lift_out_of_table(node)
134+
table = node.ancestors('table').first
135+
table.previous_element.after(node)
136+
end
99137
end
100138
end
101139
end

‎lib/docs/filters/rxjs/entries.rb‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,22 +2,28 @@ module Docs
22
class Rxjs
33
class EntriesFilter < Docs::EntriesFilter
44
def get_name
5-
name = at_css('h1').content
5+
title = at_css('h1')
6+
name = title.nil? ? subpath.rpartition('/').last.titleize : title.content
67
name.prepend "#{$1}. " if subpath =~ /\-pt(\d+)/
8+
name += '()' unless at_css('.api-type-label.function').nil?
79
name
810
end
911

1012
def get_type
1113
if slug.start_with?('guide')
1214
'Guide'
13-
elsif at_css('.api-type-label.module')
14-
name.split('/').first
1515
elsif slug.start_with?('api/')
1616
slug.split('/').second
1717
else
1818
'Miscellaneous'
1919
end
2020
end
21+
22+
def additional_entries
23+
css('h3[id]').map do |node|
24+
["#{name}.#{node['name']}()", node['id']]
25+
end
26+
end
2127
end
2228
end
2329
end

‎lib/docs/scrapers/rxjs.rb‎

Lines changed: 51 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -4,81 +4,81 @@ module Docs
44
class Rxjs < UrlScraper
55
self.name = 'RxJS'
66
self.type = 'rxjs'
7+
self.release = '6.5.2'
8+
self.base_url = 'https://rxjs.dev/'
9+
self.root_path = 'guide/overview'
710
self.links = {
811
home: 'https://rxjs.dev/',
912
code: 'https://github.com/ReactiveX/rxjs'
1013
}
1114

15+
html_filters.push 'rxjs/clean_html', 'rxjs/entries'
16+
17+
options[:follow_links] = false
18+
options[:only_patterns] = [/guide\//, /api\//]
19+
options[:skip_patterns] = [/api\/([^\/]+)\.json/]
20+
options[:fix_urls_before_parse] = ->(url) do
21+
url.sub! %r{\Aguide/}, '/guide/'
22+
url.sub! %r{\Aapi/}, '/api/'
23+
url.sub! %r{\Agenerated/}, '/generated/'
24+
url
25+
end
26+
1227
options[:max_image_size] = 256_000
1328

1429
options[:attribution] = <<-HTML
1530
&copy; 2015&ndash;2018 Google, Inc., Netflix, Inc., Microsoft Corp. and contributors.<br>
1631
Code licensed under an Apache-2.0 License. Documentation licensed under CC BY 4.0.
1732
HTML
1833

19-
module Common
20-
private
34+
def get_latest_version(opts)
35+
json = fetch_json('https://rxjs.dev/generated/navigation.json', opts)
36+
json['__versionInfo']['raw']
37+
end
2138

22-
def initial_urls
23-
initial_urls = []
39+
private
2440

25-
Request.run "#{self.class.base_url}generated/navigation.json" do |response|
26-
data = JSON.parse(response.body)
27-
dig = ->(entry) do
28-
initial_urls << url_for("generated/docs/#{entry['url']}.json") if entry['url'] && entry['url'] != 'api'
29-
entry['children'].each(&dig) if entry['children']
30-
end
31-
data['SideNav'].each(&dig)
32-
end
41+
def initial_urls
42+
initial_urls = []
3343

34-
Request.run "#{self.class.base_url}generated/docs/api/api-list.json" do |response|
35-
data = JSON.parse(response.body)
36-
dig = ->(entry) do
37-
initial_urls << url_for("generated/docs/#{entry['path']}.json") if entry['path']
38-
initial_urls << url_for("generated/docs/api/#{entry['name']}.json") if entry['name'] && !entry['path']
39-
entry['items'].each(&dig) if entry['items']
40-
end
41-
data.each(&dig)
44+
Request.run "#{self.class.base_url}generated/navigation.json" do |response|
45+
data = JSON.parse(response.body)
46+
dig = ->(entry) do
47+
initial_urls << url_for("generated/docs/#{entry['url']}.json") if entry['url'] && entry['url'] != 'api'
48+
entry['children'].each(&dig) if entry['children']
4249
end
43-
44-
initial_urls
50+
data['SideNav'].each(&dig)
4551
end
4652

47-
def handle_response(response)
48-
if response.mime_type.include?('json')
49-
begin
50-
response.options[:response_body] = JSON.parse(response.body)['contents']
51-
rescue JSON::ParserError
52-
response.options[:response_body] = ''
53-
end
54-
response.headers['Content-Type'] = 'text/html'
55-
response.url.path = response.url.path.sub('/generated/docs/', '/').remove('.json')
56-
response.effective_url.path = response.effective_url.path.sub('/generated/docs/', '/').remove('.json')
53+
Request.run "#{self.class.base_url}generated/docs/api/api-list.json" do |response|
54+
data = JSON.parse(response.body)
55+
dig = ->(entry) do
56+
initial_urls << url_for("generated/docs/#{entry['path']}.json") if entry['path']
57+
initial_urls << url_for("generated/docs/api/#{entry['name']}.json") if entry['name'] && !entry['path']
58+
entry['items'].each(&dig) if entry['items']
5759
end
58-
super
60+
data.each(&dig)
5961
end
60-
end
6162

62-
version do
63-
self.release = '6.3.3'
64-
self.base_url = 'https://rxjs.dev/'
65-
self.root_path = 'guide/overview'
66-
67-
html_filters.push 'rxjs/clean_html', 'rxjs/entries'
68-
69-
options[:follow_links] = false
70-
options[:only_patterns] = [/\Aguide/, /\Aapi/]
71-
options[:fix_urls_before_parse] = ->(url) do
72-
url.sub! %r{\Aguide/}, '/guide/'
73-
url.sub! %r{\Aapi/}, '/api/'
74-
url.sub! %r{\Agenerated/}, '/generated/'
75-
url
63+
initial_urls.select do |url|
64+
options[:only_patterns].any? { |pattern| url =~ pattern } &&
65+
options[:skip_patterns].none? { |pattern| url =~ pattern }
7666
end
77-
78-
include Docs::Rxjs::Common
7967
end
8068

81-
private
69+
def handle_response(response)
70+
if response.mime_type.include?('json')
71+
begin
72+
response.options[:response_body] = JSON.parse(response.body)['contents']
73+
rescue JSON::ParserError
74+
response.options[:response_body] = ''
75+
end
76+
response.headers['Content-Type'] = 'text/html'
77+
response.url.path = response.url.path.sub('/generated/docs/', '/').remove('.json')
78+
response.effective_url.path = response.effective_url.path.sub('/generated/docs/', '/').remove('.json')
79+
end
80+
super
81+
end
8282

8383
def parse(response)
8484
response.body.gsub! '<code-example', '<pre'

‎public/icons/docs/rxjs/16.png‎

-3.75 KB
Loading

‎public/icons/docs/rxjs/SOURCE‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
http://reactivex.io/
1+
https://github.com/ReactiveX/reactivex.github.io/blob/develop/favicon.ico

0 commit comments

Comments
 (0)