Skip to content

Commit 92a432d

Browse files
michei69kairoreed-bot
michei69
and
kairoreed-bot
committed
perf: HTTP connection pooling, pre-compiled RCC regex, batched data transfer, cached thumbnail icon lookup, version-first regex routing
Co-authored-by: Kairo <kairo.r33d@gmail.com>
1 parent 9ab77a1 commit 92a432d

5 files changed

Lines changed: 91 additions & 83 deletions

File tree

‎Source/assets/extractor.py‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@
77
import os
88
import re
99

10+
_HTTP_POOL = urllib3.PoolManager()
11+
1012

1113
def _get_cookie_from_system() -> str | None:
1214
'''
@@ -99,8 +101,7 @@ def download_item(url: str, cookie: str | None = None) -> bytes | None:
99101
headers["Roblox-Browser-Asset-Request"] = "false"
100102

101103
try:
102-
http = urllib3.PoolManager()
103-
response = http.request('GET', url, headers=headers)
104+
response = _HTTP_POOL.request('GET', url, headers=headers)
104105
if response.status != 200:
105106
return None
106107
return response.data

‎Source/assets/thumbnail.py‎

Lines changed: 24 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1,44 +1,45 @@
11
from . import const, extractor
22
import json
3+
import functools
34
import game_config
5+
from config_type.types import wrappers
6+
7+
8+
@functools.cache
9+
def _get_icon_map() -> dict[int, 'wrappers.uri_obj']:
10+
config = game_config.get_cached_config()
11+
result = {}
12+
for gp in config.remote_data.gamepasses.values():
13+
if gp.icon is not None:
14+
result[gp.id_num] = gp.icon
15+
for dp in config.remote_data.devproducts.values():
16+
if dp.icon is not None:
17+
result[dp.id_num] = dp.icon
18+
return result
19+
420

521
def transform_to_id_num(asset_id: str) -> int:
6-
asset_sub = asset_id[len(const.THUMB_PREFIX):-4] # strip .png
22+
asset_sub = asset_id[len(const.THUMB_PREFIX):-4]
723

824
try:
925
return int(asset_sub)
1026
except:
1127
return 0
1228

29+
1330
def check(asset_id: str):
1431
return asset_id.startswith(const.THUMB_PREFIX)
1532

33+
1634
def load_asset(asset_id: str) -> bytes | None:
1735
id = transform_to_id_num(asset_id)
1836
if id == 0:
1937
return None
20-
21-
config = game_config.get_cached_config()
22-
passes = config.remote_data.gamepasses
23-
for gamepass in passes.values():
24-
if gamepass.id_num == id:
25-
if gamepass.icon is None:
26-
# technically we could fetch it from the thumbnails api,
27-
# but gamepass.id_num is usually the product_id, not
28-
# the gamepass_id (which is different)
29-
return None
30-
return gamepass.icon.extract()
31-
32-
products = config.remote_data.devproducts
33-
for product in products.values():
34-
if product.id_num == id:
35-
# same case as gamepasses
36-
if product.icon is None:
37-
return None
38-
return product.icon.extract()
39-
40-
# technically we could also download the decal and parse its contents
41-
# to extract the thumbnail, but that might be overkill
38+
39+
icon = _get_icon_map().get(id)
40+
if icon is not None:
41+
return icon.extract()
42+
4243
raw = extractor.download_item(
4344
"https://thumbnails.roblox.com/v1/assets?assetids=%s&size=700x700&format=Png&isCircular=false" %
4445
(id,)

‎Source/data_transfer/transferer.py‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,15 @@ def extract(self) -> dict[str, dict[str, Any]]:
5454
val = dataclasses.asdict(item)
5555
result[item.guid] = val
5656
except queue.Empty:
57-
pass
57+
return result
58+
59+
while True:
60+
try:
61+
item = self.input_queue.get_nowait()
62+
except queue.Empty:
63+
break
64+
val = dataclasses.asdict(item)
65+
result[item.guid] = val
5866

5967
return result
6068

‎Source/logger/rcc.py‎

Lines changed: 52 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -2,127 +2,124 @@
22
import re
33

44
patterns = [
5-
# 2021E
6-
(
7-
br'^'
5+
re.compile(b''.join([
6+
# 2021E
7+
br'^',
88

99
# Current timestamp.
10-
br'(?P<timestamp>[\d\.]+),'
10+
br'(?P<timestamp>[\d\.]+),',
1111

1212
# Changes randomly between RCC sessions.
13-
br'[0-9a-f]+,'
13+
br'[0-9a-f]+,',
1414

1515
# RCC's own log level.
16-
br'(?P<rcc_log_num>\d+),'
16+
br'(?P<rcc_log_num>\d+),',
1717

1818
# Constant across all RCC logs.
19-
br'GameServer,'
19+
br'GameServer,',
2020

2121
# Defaults to 1818.
22-
br'[^\s,]+,'
22+
br'[^\s,]+,',
2323

2424
# Defaults to 13058.
25-
br'[^\s,]+,'
25+
br'[^\s,]+,',
2626

2727
# Defaults to "https://localhost:2005/.127.0.0.1".
28-
br'[^\s,]+,'
28+
br'[^\s,]+,',
2929

3030
# Constant across all RCC logs.
31-
br'Test,'
31+
br'Test,',
3232

3333
# Defaults to "https://localhost:2005".
34-
br'[^\s,]+,'
34+
br'[^\s,]+,',
3535

3636
# Constant across all RCC logs.
37-
br'unknown,'
37+
br'unknown,',
3838

3939
# Constant across all RCC logs.
40-
br'Test'
40+
br'Test',
4141

4242
# Gets FLog type.
43-
br' \[(?P<log_name>.*?)\]'
43+
br' \[(?P<log_name>.*?)\]',
4444

4545
# Captures the rest of the line.
46-
br' (?P<rest_of_line>.+)$'
47-
),
48-
49-
# 2021E
50-
(
51-
br'^'
46+
br' (?P<rest_of_line>.+)$',
47+
])),
48+
re.compile(b''.join([
49+
# 2021E
50+
br'^',
5251

5352
# Current timestamp.
54-
br'(?P<timestamp>[\d\.]+),'
53+
br'(?P<timestamp>[\d\.]+),',
5554

5655
# Changes randomly between RCC sessions.
57-
br'[0-9a-f]+,'
56+
br'[0-9a-f]+,',
5857

5958
# RCC's own log level.
60-
br'(?P<rcc_log_num>\d+)'
59+
br'(?P<rcc_log_num>\d+)',
6160

6261
# Gets FLog type.
63-
br' \[(?P<log_name>.*?)\]'
62+
br' \[(?P<log_name>.*?)\]',
6463

6564
# Captures the rest of the line.
66-
br' (?P<rest_of_line>.+)$'
67-
),
68-
69-
# 2018M
70-
(
71-
br'^'
65+
br' (?P<rest_of_line>.+)$',
66+
])),
67+
re.compile(b''.join([
68+
# 2018M
69+
br'^',
7270

7371
# Current timestamp.
74-
br'(?P<timestamp>[\d\.]+),'
72+
br'(?P<timestamp>[\d\.]+),',
7573

7674
# Changes randomly.
77-
br'[0-9a-f]+,'
75+
br'[0-9a-f]+,',
7876

7977
# RCC's own log level.
80-
br'(?P<rcc_log_num>\d+)'
78+
br'(?P<rcc_log_num>\d+)',
8179

8280
# Captures the rest of the line.
83-
br' (?P<rest_of_line>.+)$'
84-
),
85-
86-
# 2018M
87-
(
88-
br'^'
81+
br' (?P<rest_of_line>.+)$',
82+
])),
83+
re.compile(b''.join([
84+
# 2018M
85+
br'^',
8986

9087
# Current timestamp.
91-
br'(?P<timestamp>[\d\.]+),'
88+
br'(?P<timestamp>[\d\.]+),',
9289

9390
# Changes randomly.
94-
br'[0-9a-f]+,'
91+
br'[0-9a-f]+,',
9592

9693
# RCC's own log level.
97-
br'(?P<rcc_log_num>\d+),'
94+
br'(?P<rcc_log_num>\d+),',
9895

9996
# Constant across all RCC logs.
100-
br'GameServer,'
97+
br'GameServer,',
10198

10299
# Defaults to 1818.
103-
br'[^\s,]+,'
100+
br'[^\s,]+,',
104101

105102
# Defaults to 13058.
106-
br'[^\s,]+,'
103+
br'[^\s,]+,',
107104

108105
# Defaults to "https://localhost:2005/.127.0.0.1".
109-
br'[^\s,]+,'
106+
br'[^\s,]+,',
110107

111108
# Constant across all RCC logs.
112-
br'Test,'
109+
br'Test,',
113110

114111
# Defaults to "https://localhost:2005".
115-
br'[^\s,]+,'
112+
br'[^\s,]+,',
116113

117114
# Constant across all RCC logs.
118-
br'unknown,'
115+
br'unknown,',
119116

120117
# Constant across all RCC logs.
121-
br'Test'
118+
br'Test',
122119

123120
# Captures the rest of the line.
124-
br' (?P<rest_of_line>.+)$'
125-
),
121+
br' (?P<rest_of_line>.+)$',
122+
])),
126123
]
127124

128125

@@ -139,9 +136,9 @@ def get_message(log_filter: filter.filter_type_bin, bcolors: bc.bcolors, text: b
139136

140137
match = next(
141138
(
142-
re.match(pattern, text)
139+
pattern.match(text)
143140
for pattern in patterns
144-
if (match := re.match(pattern, text)) is not None
141+
if (match := pattern.match(text)) is not None
145142
), None,
146143
)
147144

‎Source/web_server/_logic.py‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -295,14 +295,15 @@ def __open_from_static(self) -> bool:
295295
return func(self)
296296

297297
def __open_from_regex(self) -> bool:
298+
version = self.game_config.game_setup.roblox_version
298299
for key, func in SERVER_FUNCS.items():
299300
if key.mode != func_mode.REGEX:
300301
continue
302+
if key.version != version:
303+
continue
301304
match = re.fullmatch(key.path, self.url_split.path)
302305
if match is None:
303306
continue
304-
if key.version != self.game_config.game_setup.roblox_version:
305-
continue
306307

307308
try:
308309
return func(self, match)

0 commit comments

Comments
 (0)