Skip to content

Commit 8eea6f6

Browse files
bogadisabogadisa
authored
Merge pull request #827 from Path-of-Modifiers/824-optimize-downtime-between-data-retrieval-cycles
824 Fix currency and league issues
2 parents 29104e6 + 0e260a7 commit 8eea6f6

121 files changed

Lines changed: 4004 additions & 1741 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎src/.env‎

Lines changed: 4 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ DOCKER_IMAGE_FRONTEND=pom_frontend
1818

1919
# Backend
2020
PRIVATIZE_API=
21-
TESTING=True
21+
TESTING=
2222
RATE_LIMIT=
2323
BACKEND_CORS_ORIGINS="http://localhost,http://localhost:5173,http://localhost:5174,https://localhost,https://localhost:5173,https://localhost:5174"
2424
PROJECT_NAME="pathofmodifiers"
@@ -35,20 +35,15 @@ SMTP_USER=noreply@pathofmodifiers.com
3535
SMTP_PASSWORD=changethis
3636
TURNSTILE_SECRET_KEY=1x0000000000000000000000000000000AA
3737

38-
# League
39-
CURRENT_SOFTCORE_LEAGUE="Mirage"
40-
ALL_SOFTCORE_LEAGUES="Mirage|Keepers|Mercenaries|Phrecia"
41-
42-
LEAGUE_LAUNCH_TIME=2026-03-06T19:00:00Z # ISO 8601 format. Round backwards to whole hour number
4338

4439
# Data retrieval
4540
MANUAL_NEXT_CHANGE_ID=True
46-
# Change ID Retrieved 02.09.2024:
47-
NEXT_CHANGE_ID=2621291179-2592930469-2516326916-2790039644-2710714910
41+
# Change ID Retrieved 28.06.2026:
42+
NEXT_CHANGE_ID=3176140940-3103957281-3029294684-3370770619-3261964453
4843
POE_PUBLIC_STASHES_AUTH_TOKEN=changethis
4944
# Enables the public stashes creation script in data_ret/app/tests/scripts.
5045
# Set value "True" to activate:
51-
DATA_RET_TEST_PUB_STASH_SIM_DATA_DEPOSIT_ENABLED=True
46+
DATA_RET_TEST_PUB_STASH_SIM_DATA_DEPOSIT_ENABLED=
5247

5348
# Postgres
5449
POSTGRES_PORT=5432

‎src/backend_api/app/alembic/versions/965e766db0a0_added_league_table.py‎

Lines changed: 267 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,267 @@
1+
"""Added league table
2+
3+
Revision ID: 965e766db0a0
4+
Revises: 17daa1c96438
5+
Create Date: 2026-06-30 15:49:45.173827
6+
7+
"""
8+
9+
from typing import Sequence, Union
10+
11+
from alembic import op
12+
import sqlalchemy as sa
13+
14+
from app.alembic.replaceable_objects.main import ReplaceableTrigger
15+
16+
# revision identifiers, used by Alembic.
17+
revision: str = "965e766db0a0"
18+
down_revision: Union[str, None] = "17daa1c96438"
19+
branch_labels: Union[str, Sequence[str], None] = None
20+
depends_on: Union[str, Sequence[str], None] = None
21+
22+
unidentified_aggregation_trigger_old = ReplaceableTrigger(
23+
"aggregate_unidentified",
24+
"unidentified_item",
25+
"""
26+
RETURNS TRIGGER AS ${name}$
27+
DECLARE
28+
current_hour INT;
29+
divine_id INT;
30+
divine_value FLOAT;
31+
BEGIN
32+
IF EXISTS (SELECT 1 FROM unidentified_item AS unid WHERE unid."createdHoursSinceLaunch" = NEW."createdHoursSinceLaunch") THEN
33+
RETURN NEW;
34+
END IF;
35+
36+
current_hour := (SELECT MAX(unid."createdHoursSinceLaunch") FROM unidentified_item AS unid);
37+
divine_id := (SELECT MAX(cur."currencyId") FROM currency AS cur WHERE cur."tradeName"='divine');
38+
divine_value := (SELECT cur."valueInChaos" FROM currency AS cur WHERE cur."currencyId"=divine_id);
39+
40+
41+
WITH aggregates AS (
42+
SELECT name, unid."itemBaseTypeId", unid."createdHoursSinceLaunch", league, ilvl, stddev(unid."currencyAmount" * cur."valueInChaos") AS std, AVG(unid."currencyAmount" * cur."valueInChaos") AS calc_avg, COUNT(unid."itemId") AS calc_count
43+
FROM unidentified_item AS unid
44+
NATURAL JOIN currency as cur
45+
WHERE unid."createdHoursSinceLaunch" = current_hour
46+
AND NOT aggregated
47+
GROUP BY name, unid."itemBaseTypeId", unid."createdHoursSinceLaunch", league, ilvl
48+
), affected_item_ids AS (
49+
SELECT unid."itemId" FROM unidentified_item AS unid WHERE NOT aggregated AND unid."createdHoursSinceLaunch"=current_hour
50+
)
51+
52+
INSERT INTO unidentified_item (name, "itemBaseTypeId", "createdHoursSinceLaunch", league, "currencyId", ilvl, "currencyAmount", "nItems", identified, rarity, aggregated)
53+
SELECT name, unid."itemBaseTypeId", unid."createdHoursSinceLaunch", league, divine_id, ilvl, AVG(unid."currencyAmount" * cur."valueInChaos") / divine_value, calc_count, identified, rarity, TRUE
54+
FROM unidentified_item AS unid
55+
NATURAL JOIN aggregates
56+
NATURAL JOIN currency AS cur
57+
WHERE unid."currencyAmount" * cur."valueInChaos" <= calc_avg + 1.97 * std
58+
AND unid."currencyAmount" * cur."valueInChaos" >= calc_avg - 1.97 * std
59+
GROUP BY name, unid."itemBaseTypeId", unid."createdHoursSinceLaunch", league, ilvl, identified, rarity, calc_count;
60+
61+
DELETE FROM unidentified_item as unid
62+
WHERE NOT aggregated AND unid."createdHoursSinceLaunch"=current_hour;
63+
64+
RETURN NEW;
65+
END;
66+
${name}$ LANGUAGE plpgsql;
67+
""",
68+
"""
69+
BEFORE INSERT ON {table}
70+
FOR EACH ROW
71+
EXECUTE FUNCTION {name}();
72+
""",
73+
)
74+
75+
76+
def upgrade() -> None:
77+
# ### commands auto generated by Alembic - please adjust! ###
78+
op.create_table(
79+
"league",
80+
sa.Column(
81+
"leagueId", sa.SmallInteger(), sa.Identity(always=False), nullable=False
82+
),
83+
sa.Column("name", sa.Text(), nullable=False),
84+
sa.Column("validFrom", sa.DateTime(timezone=True), nullable=False),
85+
sa.Column("validTo", sa.DateTime(timezone=True), nullable=True),
86+
sa.Column("version", sa.Float(), nullable=False),
87+
sa.PrimaryKeyConstraint("leagueId"),
88+
)
89+
op.execute("""
90+
INSERT INTO league (name, "validFrom", version)
91+
SELECT DISTINCT
92+
league,
93+
TIMESTAMP '1970-01-01 00:00:00',
94+
0.0
95+
FROM item
96+
WHERE league IS NOT NULL
97+
""")
98+
99+
# ---- Item table ----
100+
op.add_column("item", sa.Column("leagueId", sa.SmallInteger()))
101+
op.execute("""
102+
UPDATE item
103+
SET "leagueId" = l."leagueId"
104+
FROM league l
105+
WHERE item.league = l.name
106+
""")
107+
op.alter_column("item", "leagueId", nullable=False)
108+
op.create_foreign_key(
109+
"fk_item_league",
110+
"item",
111+
"league",
112+
["leagueId"],
113+
["leagueId"],
114+
ondelete="RESTRICT",
115+
onupdate="CASCADE",
116+
)
117+
op.drop_column("item", "league")
118+
119+
# ---- Unidentified item table ----
120+
op.add_column("unidentified_item", sa.Column("leagueId", sa.SmallInteger()))
121+
op.execute("""
122+
UPDATE unidentified_item
123+
SET "leagueId" = l."leagueId"
124+
FROM league l
125+
WHERE league = l.name
126+
""")
127+
op.alter_column("unidentified_item", "leagueId", nullable=False)
128+
op.create_foreign_key(
129+
"fk_unidentified_item_league",
130+
"unidentified_item",
131+
"league",
132+
["leagueId"],
133+
["leagueId"],
134+
ondelete="RESTRICT",
135+
onupdate="CASCADE",
136+
)
137+
op.drop_column("unidentified_item", "league")
138+
139+
# ---- Currency table ----
140+
op.add_column("currency", sa.Column("leagueId", sa.SmallInteger()))
141+
# Deletes all currency ids that are not used
142+
op.execute("""
143+
DELETE FROM currency c
144+
WHERE NOT EXISTS (
145+
SELECT 1
146+
FROM item i
147+
WHERE i."currencyId" = c."currencyId"
148+
)
149+
""")
150+
op.execute("""
151+
UPDATE currency c
152+
SET "leagueId" = i.leagueId
153+
FROM (
154+
SELECT item."currencyId", MIN(item."leagueId") AS leagueId
155+
FROM item
156+
GROUP BY item."currencyId"
157+
) i
158+
WHERE c."currencyId" = i."currencyId"
159+
""")
160+
op.alter_column("currency", "leagueId", nullable=False)
161+
op.create_foreign_key(
162+
"fk_currency_league",
163+
"currency",
164+
"league",
165+
["leagueId"],
166+
["leagueId"],
167+
ondelete="RESTRICT",
168+
onupdate="CASCADE",
169+
)
170+
171+
# ---- Indexes ----
172+
# Manual SQL to drop query instead of `op.drop_index` is needed due to hypertables
173+
op.execute("""
174+
DROP INDEX IF EXISTS ix_item_name_itemBaseTypeId_createdHoursSinceLaunch_league
175+
""")
176+
op.create_index(
177+
"ix_item_name_itemBaseTypeId_createdHoursSinceLaunch_leagueId",
178+
"item",
179+
[
180+
"name",
181+
"itemBaseTypeId",
182+
"createdHoursSinceLaunch",
183+
"leagueId",
184+
],
185+
unique=False,
186+
)
187+
188+
op.execute("""
189+
DROP INDEX IF EXISTS ix_unid_item_name_itemBaseTypeId_createdHoursSinceLaunch_league
190+
""")
191+
op.create_index(
192+
"ix_unid_name_itemBaseTypeId_createdHoursSinceLaunch_leagueId",
193+
"unidentified_item",
194+
[
195+
"name",
196+
"itemBaseTypeId",
197+
"createdHoursSinceLaunch",
198+
"leagueId",
199+
],
200+
unique=False,
201+
)
202+
203+
# ---- triggers ----
204+
op.drop_trigger(unidentified_aggregation_trigger_old)
205+
# ### end Alembic commands ###
206+
207+
208+
def downgrade() -> None:
209+
# ### commands auto generated by Alembic - please adjust! ###
210+
# ---- Item table ----
211+
op.add_column(
212+
"item",
213+
sa.Column("league", sa.TEXT(), autoincrement=False),
214+
)
215+
op.execute("""
216+
UPDATE item i
217+
SET league = l.name
218+
FROM league l
219+
WHERE i."leagueId" = l."leagueId"
220+
""")
221+
op.alter_column("item", "league", nullable=False)
222+
op.drop_constraint("fk_item_league", "item", type_="foreignkey")
223+
op.execute("""
224+
DROP INDEX IF EXISTS ix_item_name_itemBaseTypeId_createdHoursSinceLaunch_leagueId
225+
""")
226+
op.create_index(
227+
"ix_item_name_itemBaseTypeId_createdHoursSinceLaunch_league",
228+
"item",
229+
["name", "itemBaseTypeId", "createdHoursSinceLaunch", "league"],
230+
unique=False,
231+
)
232+
op.drop_column("item", "leagueId")
233+
234+
# ---- Unidentified item table ----
235+
op.add_column(
236+
"unidentified_item",
237+
sa.Column("league", sa.TEXT(), autoincrement=False),
238+
)
239+
op.execute("""
240+
UPDATE unidentified_item ui
241+
SET league = l.name
242+
FROM league l
243+
WHERE ui."leagueId" = l."leagueId"
244+
""")
245+
op.alter_column("unidentified_item", "league", nullable=False)
246+
op.drop_constraint(
247+
"fk_unidentified_item_league", "unidentified_item", type_="foreignkey"
248+
)
249+
op.execute("""
250+
DROP INDEX IF EXISTS ix_unid_name_itemBaseTypeId_createdHoursSinceLaunch_leagueId
251+
""")
252+
op.create_index(
253+
"ix_unid_item_name_itemBaseTypeId_createdHoursSinceLaunch_league",
254+
"unidentified_item",
255+
["name", "itemBaseTypeId", "createdHoursSinceLaunch", "league"],
256+
unique=False,
257+
)
258+
op.drop_column("unidentified_item", "leagueId")
259+
260+
# ---- Currency table ----
261+
op.drop_column("currency", "leagueId")
262+
263+
op.drop_table("league")
264+
265+
# ---- triggers ----
266+
op.create_trigger(unidentified_aggregation_trigger_old)
267+
# ### end Alembic commands ###

0 commit comments

Comments
 (0)