chg: [crawler settings] display recent crawler logs in settings page

Author: Terrtia

bin/lib/crawlers.py

@@ -16,6 +16,7 @@
 import sys
 import time
 import uuid
+from collections import deque
 
 from multiprocessing import Process as Proc
 
@@ -188,6 +189,23 @@ def get_date_crawled_items_source(date):
 def get_har_dir():
     return HAR_DIR
 
+
+def get_last_crawler_logs(lines=100):
+    log_path = os.path.join(os.environ['AIL_HOME'], 'logs', 'crawlers.log')
+    if not os.path.exists(log_path):
+        return ['No crawler logs available.']
+    if os.path.getsize(log_path) == 0:
+        return ['Crawler log file is empty.']
+    try:
+        with open(log_path, 'r', encoding='utf-8', errors='replace') as f:
+            last_lines = deque(f, maxlen=lines)
+    except OSError:
+        return ['No crawler logs available.']
+
+    if not last_lines:
+        return ['Crawler log file is empty.']
+    return [line.rstrip('\n') for line in last_lines]
+
 def is_valid_onion_v3_domain(domain):
     if len(domain) == 62:  # v3 address
         return domain[:56].isalnum()

var/www/blueprints/crawler_splash.py

@@ -61,6 +61,7 @@ def create_json_response(data, status_code):
     return Response(json.dumps(data, indent=2, sort_keys=True), mimetype='application/json'), status_code
 
 
+
 # ============= ROUTES ==============
 @crawler_splash.route("/crawlers/dashboard", methods=['GET'])
 @login_required
@@ -1062,6 +1063,7 @@ def crawler_settings():
 
     is_onion_filter_enabled = crawlers.is_onion_filter_enabled(cache=False)
     is_onion_filter_unknown = crawlers.is_onion_filter_unknown(cache=False)
+    crawler_logs = crawlers.get_last_crawler_logs(lines=100)
 
     # TODO REGISTER PROXY
     # all_proxies = crawlers.get_all_proxies_metadata()
@@ -1076,7 +1078,8 @@ def crawler_settings():
                            is_crawler_working=is_crawler_working,
                            crawler_error_mess=crawler_error_mess,
                            is_onion_filter_enabled=is_onion_filter_enabled,
-                           is_onion_filter_unknown=is_onion_filter_unknown
+                           is_onion_filter_unknown=is_onion_filter_unknown,
+                           crawler_logs=crawler_logs
                            )
 
 

var/www/templates/crawler/crawler_splash/settings_crawler.html

@@ -56,6 +56,12 @@
 			max-height: 260px;
 			overflow-y: auto;
 		}
+		.crawler-logs-output {
+			border-radius: .5rem;
+			max-height: 360px;
+			overflow-y: auto;
+			white-space: pre;
+		}
 	</style>
 </head>
 
@@ -225,6 +231,16 @@ <h5 class="card-title d-flex justify-content-between align-items-center flex-wra
 							</div>
 						</div>
 
+
+						<div class="card border-secondary my-4">
+							<div class="card-body text-dark">
+								<h5 class="card-title mb-1">Recent crawler logs</h5>
+								<p class="text-muted mb-3">Recent crawler errors and status messages from <code>logs/crawlers.log</code> (last 100 lines).</p>
+								<pre class="bg-dark text-white p-3 crawler-logs-output">{% for line in crawler_logs %}{{ line }}
+{% endfor %}</pre>
+							</div>
+						</div>
+
 						<a href="{{ url_for('crawler_splash.crawler_blacklist') }}" class="btn btn-outline-danger">
 							Blacklisted domains
 						</a>