Skip to content

Commit e526bf1

Browse files
TerrtiaTerrtia
committed
chg: [forum] crawler add max account crawler config + forum icons
1 parent f8b00e0 commit e526bf1

12 files changed

Lines changed: 112 additions & 28 deletions

‎bin/lib/crawlers.py‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2223,6 +2223,39 @@ def get_nb_running_forum_crawler_accounts():
22232223
return r_crawler.zcard('forum:crawl:running')
22242224

22252225

2226+
def get_forum_crawler_max_accounts():
2227+
nb_accounts = r_cache.hget('crawler:lacus', 'forum_nb_accounts')
2228+
if not nb_accounts:
2229+
nb_accounts = r_db.hget('crawler:lacus', 'forum_nb_accounts')
2230+
if not nb_accounts:
2231+
nb_accounts = 0
2232+
save_forum_crawler_max_accounts(nb_accounts)
2233+
else:
2234+
r_cache.hset('crawler:lacus', 'forum_nb_accounts', int(nb_accounts))
2235+
return int(nb_accounts)
2236+
2237+
2238+
def save_forum_crawler_max_accounts(nb_accounts):
2239+
r_db.hset('crawler:lacus', 'forum_nb_accounts', int(nb_accounts))
2240+
r_cache.hset('crawler:lacus', 'forum_nb_accounts', int(nb_accounts))
2241+
2242+
2243+
def api_set_forum_crawler_max_accounts(data):
2244+
nb_accounts = data.get('nb', 0)
2245+
try:
2246+
nb_accounts = int(nb_accounts)
2247+
if nb_accounts < 0:
2248+
nb_accounts = 0
2249+
except (TypeError, ValueError):
2250+
return {'error': 'Invalid number of forum account crawler tasks to launch'}, 400
2251+
save_forum_crawler_max_accounts(nb_accounts)
2252+
return nb_accounts, 200
2253+
2254+
2255+
def can_launch_forum_crawler_account():
2256+
return get_nb_running_forum_crawler_accounts() < get_forum_crawler_max_accounts()
2257+
2258+
22262259
#### CRAWLER CAPTURE ####
22272260

22282261
def get_nb_crawler_captures():

‎var/www/blueprints/crawler_splash.py‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1099,6 +1099,7 @@ def crawler_settings():
10991099
lacus_url = crawlers.get_lacus_url()
11001100
api_key = crawlers.get_hidden_lacus_api_key()
11011101
nb_captures = crawlers.get_crawler_max_captures()
1102+
nb_forum_accounts = crawlers.get_forum_crawler_max_accounts()
11021103

11031104
is_manager_connected = crawlers.get_lacus_connection_metadata(force_ping=True)
11041105
is_crawler_working = crawlers.is_test_ail_crawlers_successful()
@@ -1118,6 +1119,7 @@ def crawler_settings():
11181119
is_manager_connected=is_manager_connected,
11191120
lacus_url=lacus_url, api_key=api_key,
11201121
nb_captures=nb_captures,
1122+
nb_forum_accounts=nb_forum_accounts,
11211123
# all_proxies=all_proxies,
11221124
is_crawler_working=is_crawler_working,
11231125
crawler_test_metadata=crawler_test_metadata,
@@ -1153,15 +1155,21 @@ def crawler_lacus_settings_crawler_manager():
11531155
def crawler_settings_crawlers_to_launch():
11541156
if request.method == 'POST':
11551157
nb_captures = request.form.get('nb_captures')
1158+
nb_forum_accounts = request.form.get('nb_forum_accounts')
11561159
res = crawlers.api_set_crawler_max_captures({'nb': nb_captures})
1160+
if res[1] != 200:
1161+
return create_json_response(res[0], res[1])
1162+
res = crawlers.api_set_forum_crawler_max_accounts({'nb': nb_forum_accounts})
11571163
if res[1] != 200:
11581164
return create_json_response(res[0], res[1])
11591165
else:
11601166
return redirect(url_for('crawler_splash.crawler_settings'))
11611167
else:
11621168
nb_captures = crawlers.get_crawler_max_captures()
1169+
nb_forum_accounts = crawlers.get_forum_crawler_max_accounts()
11631170
return render_template("settings_edit_crawlers_to_launch.html",
1164-
nb_captures=nb_captures)
1171+
nb_captures=nb_captures,
1172+
nb_forum_accounts=nb_forum_accounts)
11651173

11661174

11671175
@crawler_splash.route('/crawler/settings/crawler/test', methods=['GET'])

‎var/www/templates/crawler/crawler_splash/settings_crawler.html‎

Lines changed: 20 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -133,16 +133,25 @@ <h4 class="mb-0">AIL Lacus Crawler</h4>
133133
</div>
134134

135135
<div class="card border-secondary my-4">
136-
<div class="card-body d-flex flex-wrap justify-content-between align-items-center">
137-
<div>
138-
<h5 class="card-title mb-1">Concurrent Crawlers</h5>
139-
<p class="text-muted mb-0">Number of crawler workers launched in parallel.</p>
140-
</div>
141-
<div class="text-right mt-3 mt-md-0">
142-
<div class="display-4 text-primary mb-2">{{ nb_captures }}</div>
143-
<a href="{{ url_for('crawler_splash.crawler_settings_crawlers_to_launch') }}" class="btn btn-info">
144-
Update Value <i class="fas fa-pencil-alt ml-1"></i>
145-
</a>
136+
<div class="card-body">
137+
<div class="row align-items-center">
138+
<div class="col-md-5 mb-3 mb-md-0">
139+
<h5 class="card-title mb-1">Concurrent Crawlers</h5>
140+
<p class="text-muted mb-0">Number of Lacus tasks launched in parallel.</p>
141+
</div>
142+
<div class="col-md-3 text-md-center mb-3 mb-md-0">
143+
<div class="metric-label">Crawler tasks</div>
144+
<div class="display-4 text-primary mb-0">{{ nb_captures }}</div>
145+
</div>
146+
<div class="col-md-3 text-md-center mb-3 mb-md-0">
147+
<div class="metric-label">Forum account tasks</div>
148+
<div class="display-4 text-primary mb-0">{{ nb_forum_accounts }}</div>
149+
</div>
150+
<div class="col-md-1 text-md-right">
151+
<a href="{{ url_for('crawler_splash.crawler_settings_crawlers_to_launch') }}" class="btn btn-info">
152+
Edit <i class="fas fa-pencil-alt ml-1"></i>
153+
</a>
154+
</div>
146155
</div>
147156
</div>
148157
</div>
@@ -355,4 +364,4 @@ <h5 class="card-title mb-1">Recent crawler logs</h5>
355364
$("#nav_settings").addClass("active");
356365
});
357366

358-
</script>
367+
</script>

‎var/www/templates/crawler/crawler_splash/settings_edit_crawlers_to_launch.html‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,9 +30,13 @@
3030
<div class="card-header bg-dark text-white">
3131

3232
<form action="{{ url_for('crawler_splash.crawler_settings_crawlers_to_launch') }}" method="post" enctype="multipart/form-data">
33-
<h3 class="card-title">Number of Concurrent Crawlers to Launch:</h3>
33+
<h3 class="card-title">Concurrent Lacus Tasks:</h3>
34+
<label for="nb_captures" class="mt-2">Number of concurrent crawler tasks to launch</label>
3435
<input class="form-control" type="number" id="nb_captures" value="{{ nb_captures }}" min="1" name="nb_captures" required>
3536

37+
<label for="nb_forum_accounts" class="mt-3">Number of concurrent forum account Lacus tasks to launch</label>
38+
<input class="form-control" type="number" id="nb_forum_accounts" value="{{ nb_forum_accounts }}" min="0" name="nb_forum_accounts" required>
39+
3640
<button type="submit" class="btn btn-primary my-2">Edit <i class="fas fa-pencil-alt"></i></button>
3741
</form>
3842
</div>

‎var/www/templates/forums_explorer/block_forum_post_search_result.html‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
<div class="card mb-3">
22
<div class="card-header d-flex justify-content-between align-items-center">
33
<div>
4-
<svg xmlns="http://www.w3.org/2000/svg" width="1em" height="1em" viewBox="0 0 24 24">
5-
<path d="M0 0h24v24H0z" fill="none" />
6-
<path fill="currentColor" d="M20 2H4a2 2 0 0 0-2 2v18l4-4h14a2 2 0 0 0 2-2V4a2 2 0 0 0-2-2M6 9h12v2H6m8 3H6v-2h8m4-4H6V6h12" />
7-
</svg> Forum post
4+
<svg xmlns="http://www.w3.org/2000/svg" width="1em" height="1em" viewBox="0 0 24 24" aria-hidden="true" focusable="false">
5+
<path d="M0 0h24v24H0z" fill="none" />
6+
<path fill="currentColor" d="M20 2H4a2 2 0 0 0-2 2v18l4-4h14a2 2 0 0 0 2-2V4a2 2 0 0 0-2-2M6 9h12v2H6m8 3H6v-2h8m4-4H6V6h12" />
7+
</svg> Forum post
88
</div>
99
{% if obj.get('link') %}<a class="btn btn-sm btn-outline-secondary" href="{{ obj['link'] }}"><i class="fas fa-project-diagram"></i> Correlation</a>{% endif %}
1010
</div>

‎var/www/templates/forums_explorer/block_forum_thread_search_result.html‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
<div class="card mb-3">
22
<div class="card-header d-flex justify-content-between align-items-center">
33
<div>
4-
<svg xmlns="http://www.w3.org/2000/svg" width="1em" height="1em" viewBox="0 0 24 24">
4+
<svg xmlns="http://www.w3.org/2000/svg" width="1em" height="1em" viewBox="0 0 24 24" aria-hidden="true" focusable="false">
55
<path d="M0 0h24v24H0z" fill="none" />
66
<path fill="currentColor" d="M2 16.59L5.59 13H15a2 2 0 0 0 2-2V6a2 2 0 0 0-2-2H4a2 2 0 0 0-2 2zM2 18H1V6a3 3 0 0 1 3-3h11a3 3 0 0 1 3 3v5a3 3 0 0 1-3 3H6zm19 2.59V10a2 2 0 0 0-2-2V7a3 3 0 0 1 3 3v12h-1l-4-4H8c-1.24 0-2.3-.75-2.76-1.82l.8-.8C6.21 16.3 7 17 8 17h9.41z" />
77
</svg> Forum thread
@@ -13,7 +13,10 @@
1313
</div>
1414
<div class="card-body">
1515
{% if obj.get('nb_posts') is not none %}
16-
<div class="text-muted small mb-2"><i class="fas fa-comments"></i> {{ obj.get('nb_posts') }} posts</div>
16+
<div class="text-muted small mb-2"><svg xmlns="http://www.w3.org/2000/svg" width="1em" height="1em" viewBox="0 0 24 24" aria-hidden="true" focusable="false">
17+
<path d="M0 0h24v24H0z" fill="none" />
18+
<path fill="currentColor" d="M20 2H4a2 2 0 0 0-2 2v18l4-4h14a2 2 0 0 0 2-2V4a2 2 0 0 0-2-2M6 9h12v2H6m8 3H6v-2h8m4-4H6V6h12" />
19+
</svg> {{ obj.get('nb_posts') }} posts</div>
1720
{% endif %}
1821
<div>{{ obj.get('result') }}</div>
1922
</div>

‎var/www/templates/forums_explorer/forums_explorer_forum.html‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,10 @@
3333
</nav>
3434

3535
<div class="forum-title-card p-4 mb-4">
36-
<h1 class="mb-2"><i class="fas fa-comments"></i> {{ forum.get('name') or forum['id'] }}</h1>
36+
<h1 class="mb-2"><svg xmlns="http://www.w3.org/2000/svg" width="1em" height="1em" viewBox="0 0 24 24" aria-hidden="true" focusable="false">
37+
<path d="M0 0h24v24H0z" fill="none" />
38+
<path fill="currentColor" d="M17 12V3a1 1 0 0 0-1-1H3a1 1 0 0 0-1 1v14l4-4h10a1 1 0 0 0 1-1m4-6h-2v9H6v2a1 1 0 0 0 1 1h11l4 4V7a1 1 0 0 0-1-1" />
39+
</svg> {{ forum.get('name') or forum['id'] }}</h1>
3740
<div>
3841
<span class="mr-3"><i class="fas fa-fingerprint"></i> {{ forum['id'] }}</span>
3942
<span class="mr-3"><i class="fas fa-layer-group"></i> {{ forum.get('forum_type') or forum.get('subtype') or '' }}</span>

‎var/www/templates/forums_explorer/forums_explorer_index.html‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,10 @@
2525
{% include 'sidebars/sidebar_objects.html' %}
2626
<div class="col-12 col-lg-10" id="core_content">
2727
<div class="forum-hero my-3 p-4">
28-
<h1 class="mb-1"><i class="fas fa-comments"></i> Forums Explorer</h1>
28+
<h1 class="mb-1"><svg xmlns="http://www.w3.org/2000/svg" width="1em" height="1em" viewBox="0 0 24 24" aria-hidden="true" focusable="false">
29+
<path d="M0 0h24v24H0z" fill="none" />
30+
<path fill="currentColor" d="M17 12V3a1 1 0 0 0-1-1H3a1 1 0 0 0-1 1v14l4-4h10a1 1 0 0 0 1-1m4-6h-2v9H6v2a1 1 0 0 0 1 1h11l4 4V7a1 1 0 0 0-1-1" />
31+
</svg> Forums Explorer</h1>
2932
<p class="mb-0">Browse imported forum archives like a live forum: forums, subforums, threads, and posts.</p>
3033
<a class="btn btn-sm btn-outline-light mt-3" href="{{ url_for('forums_explorer.forum_explorer_crawler_status') }}"><i class="fas fa-spider"></i> Forum crawler status</a>
3134
</div>
@@ -60,7 +63,10 @@ <h1 class="mb-1"><i class="fas fa-comments"></i> Forums Explorer</h1>
6063
<div class="forum-board-header">Imported forums</div>
6164
{% for forum in forums %}
6265
<a class="forum-row" href="{{ url_for('forums_explorer.forum_explorer_forums') }}?id={{ forum['id'] }}">
63-
<div class="forum-icon"><i class="fas fa-comments"></i></div>
66+
<div class="forum-icon"><svg xmlns="http://www.w3.org/2000/svg" width="1em" height="1em" viewBox="0 0 24 24" aria-hidden="true" focusable="false">
67+
<path d="M0 0h24v24H0z" fill="none" />
68+
<path fill="currentColor" d="M17 12V3a1 1 0 0 0-1-1H3a1 1 0 0 0-1 1v14l4-4h10a1 1 0 0 0 1-1m4-6h-2v9H6v2a1 1 0 0 0 1 1h11l4 4V7a1 1 0 0 0-1-1" />
69+
</svg></div>
6470
<div class="flex-grow-1">
6571
<h5 class="mb-1">{{ forum.get('name') or forum['id'] }}</h5>
6672
<div class="text-muted">

‎var/www/templates/forums_explorer/forums_explorer_subforum.html‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,10 @@
4141
</nav>
4242

4343
<div class="forum-title-card p-4 mb-4">
44-
<h1 class="mb-2"><i class="far fa-comments"></i> {{ subforum.get('name') or subforum['id'] }}</h1>
44+
<h1 class="mb-2"><svg xmlns="http://www.w3.org/2000/svg" width="1em" height="1em" viewBox="0 0 24 24" aria-hidden="true" focusable="false">
45+
<path d="M0 0h24v24H0z" fill="none" />
46+
<path fill="currentColor" d="M15 4v7H5.17L4 12.17V4zm1-2H3a1 1 0 0 0-1 1v14l4-4h10a1 1 0 0 0 1-1V3a1 1 0 0 0-1-1m5 4h-2v9H6v2a1 1 0 0 0 1 1h11l4 4V7a1 1 0 0 0-1-1" />
47+
</svg> {{ subforum.get('name') or subforum['id'] }}</h1>
4548
<div>
4649
<span class="mr-3"><i class="fas fa-fingerprint"></i> {{ subforum['id'] }}</span>
4750
<span class="mr-3"><i class="fas fa-layer-group"></i> {{ subforum['subtype'] }}</span>
@@ -59,7 +62,10 @@ <h1 class="mb-2"><i class="far fa-comments"></i> {{ subforum.get('name') or subf
5962
<div class="forum-board-header">Threads</div>
6063
{% for thread in meta['threads'] %}
6164
<a class="forum-row thread-row" href="{{ url_for('forums_explorer.forum_explorer_thread') }}?subtype={{ thread['subtype'] }}&id={{ thread['id'] }}">
62-
<div class="forum-icon"><i class="fas fa-comment-dots"></i></div>
65+
<div class="forum-icon"><svg xmlns="http://www.w3.org/2000/svg" width="1em" height="1em" viewBox="0 0 24 24" aria-hidden="true" focusable="false">
66+
<path d="M0 0h24v24H0z" fill="none" />
67+
<path fill="currentColor" d="M2 16.59L5.59 13H15a2 2 0 0 0 2-2V6a2 2 0 0 0-2-2H4a2 2 0 0 0-2 2zM2 18H1V6a3 3 0 0 1 3-3h11a3 3 0 0 1 3 3v5a3 3 0 0 1-3 3H6zm19 2.59V10a2 2 0 0 0-2-2V7a3 3 0 0 1 3 3v12h-1l-4-4H8c-1.24 0-2.3-.75-2.76-1.82l.8-.8C6.21 16.3 7 17 8 17h9.41z" />
68+
</svg></div>
6369
<div class="flex-grow-1">
6470
<h5 class="mb-1">{{ thread.get('name') or thread['id'] }}</h5>
6571
<div class="text-muted small">

‎var/www/templates/forums_explorer/forums_explorer_thread.html‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -43,11 +43,17 @@
4343
</nav>
4444

4545
<div class="thread-title-card p-4 mb-4">
46-
<h1 class="mb-2"><i class="fas fa-comment-dots"></i> {{ thread.get('name') or thread['id'] }}</h1>
46+
<h1 class="mb-2"><svg xmlns="http://www.w3.org/2000/svg" width="1em" height="1em" viewBox="0 0 24 24" aria-hidden="true" focusable="false">
47+
<path d="M0 0h24v24H0z" fill="none" />
48+
<path fill="currentColor" d="M2 16.59L5.59 13H15a2 2 0 0 0 2-2V6a2 2 0 0 0-2-2H4a2 2 0 0 0-2 2zM2 18H1V6a3 3 0 0 1 3-3h11a3 3 0 0 1 3 3v5a3 3 0 0 1-3 3H6zm19 2.59V10a2 2 0 0 0-2-2V7a3 3 0 0 1 3 3v12h-1l-4-4H8c-1.24 0-2.3-.75-2.76-1.82l.8-.8C6.21 16.3 7 17 8 17h9.41z" />
49+
</svg> {{ thread.get('name') or thread['id'] }}</h1>
4750
<div>
4851
<span class="mr-3"><i class="fas fa-fingerprint"></i> {{ thread['id'] }}</span>
4952
<span class="mr-3"><i class="fas fa-layer-group"></i> {{ thread['subtype'] }}</span>
50-
<span class="mr-3"><i class="fas fa-comments"></i> {{ thread.get('nb_posts') or 0 }} posts</span>
53+
<span class="mr-3"><svg xmlns="http://www.w3.org/2000/svg" width="1em" height="1em" viewBox="0 0 24 24" aria-hidden="true" focusable="false">
54+
<path d="M0 0h24v24H0z" fill="none" />
55+
<path fill="currentColor" d="M20 2H4a2 2 0 0 0-2 2v18l4-4h14a2 2 0 0 0 2-2V4a2 2 0 0 0-2-2M6 9h12v2H6m8 3H6v-2h8m4-4H6V6h12" />
56+
</svg> {{ thread.get('nb_posts') or 0 }} posts</span>
5157
{% if thread.get('url') %}<span class="text-white"><i class="fas fa-link"></i> {{ thread['url'] }}</span>{% endif %}
5258
</div>
5359
{% if thread.get('info') %}<div class="mt-3 bg-light text-dark rounded p-3">{{ thread['info'] }}</div>{% endif %}

0 commit comments

Comments
 (0)