#824 Removed league .env backend and fixed different hours per league

Author: bogadisa

src/.env

@@ -18,7 +18,7 @@ DOCKER_IMAGE_FRONTEND=pom_frontend
 
 # Backend
 PRIVATIZE_API=
-TESTING=True
+TESTING=
 RATE_LIMIT=
 BACKEND_CORS_ORIGINS="http://localhost,http://localhost:5173,http://localhost:5174,https://localhost,https://localhost:5173,https://localhost:5174"
 PROJECT_NAME="pathofmodifiers"
@@ -35,20 +35,15 @@ SMTP_USER=noreply@pathofmodifiers.com
 SMTP_PASSWORD=changethis
 TURNSTILE_SECRET_KEY=1x0000000000000000000000000000000AA
 
-# League
-CURRENT_SOFTCORE_LEAGUE="Mirage"
-ALL_SOFTCORE_LEAGUES="Mirage|Keepers|Mercenaries|Phrecia"
-
-LEAGUE_LAUNCH_TIME=2026-03-06T19:00:00Z # ISO 8601 format. Round backwards to whole hour number
 
 # Data retrieval
 MANUAL_NEXT_CHANGE_ID=True
-# Change ID Retrieved 02.09.2024:
-NEXT_CHANGE_ID=2621291179-2592930469-2516326916-2790039644-2710714910
+# Change ID Retrieved 28.06.2026:
+NEXT_CHANGE_ID=3176140940-3103957281-3029294684-3370770619-3261964453
 POE_PUBLIC_STASHES_AUTH_TOKEN=changethis
 # Enables the public stashes creation script in data_ret/app/tests/scripts.
 # Set value "True" to activate:
-DATA_RET_TEST_PUB_STASH_SIM_DATA_DEPOSIT_ENABLED=True
+DATA_RET_TEST_PUB_STASH_SIM_DATA_DEPOSIT_ENABLED=
 
 # Postgres
 POSTGRES_PORT=5432

src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/config.py

@@ -1,5 +1,3 @@
-from datetime import datetime
-
 from pydantic import (
     HttpUrl,
     computed_field,
@@ -27,12 +25,6 @@ def BACKEND_BASE_URL(self) -> HttpUrl:
     OATH_ACC_TOKEN_CONTACT_EMAIL: str
     FIRST_SUPERUSER: str
     FIRST_SUPERUSER_PASSWORD: str
-    CURRENT_SOFTCORE_LEAGUE: str
-
-    @computed_field  # type: ignore[prop-decorator]
-    @property
-    def CURRENT_HARDCORE_LEAGUE(self) -> str:
-        return f"Hardcore {self.CURRENT_SOFTCORE_LEAGUE}"
 
     POE_PUBLIC_STASHES_AUTH_TOKEN: str
     OAUTH_CLIENT_ID: str
@@ -43,12 +35,5 @@ def CURRENT_HARDCORE_LEAGUE(self) -> str:
 
     MAX_TIME_PER_MINI_BATCH: int = 3 * 60
 
-    LEAGUE_LAUNCH_TIME: str
-
-    @computed_field  # type: ignore[prop-decorator]
-    @property
-    def LEAGUE_LAUNCH_DATETIME_OBJECT(self) -> datetime:
-        return datetime.fromisoformat(self.LEAGUE_LAUNCH_TIME)
-
 
 settings = Settings()  # type: ignore

src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/poe_api_handler.py

@@ -38,8 +38,8 @@ def __init__(
         self,
         url: str,
         auth_token: str,
-        leagues: list[dict[str, Any]],
         *,
+        leagues: list[dict[str, Any]],
         n_wanted_items: int = 100,
         n_unique_wanted_items: int = 5,
         item_detectors: list[UniqueDetector] | None = None,

src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/detectors/base.py

@@ -64,7 +64,6 @@ def _general_filter(self, df: pd.DataFrame) -> pd.DataFrame:
             return pd.DataFrame(columns=df.columns)
 
         df = df.loc[df["league"].isin(self.leagues)]
-        df["leagueId"] = df["league"].map(self.league_to_id)
 
         return df
 

src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/main.py

@@ -38,7 +38,6 @@
 
 class ContinuousDataRetrieval:
     auth_token = settings.POE_PUBLIC_STASHES_AUTH_TOKEN
-    current_league = settings.CURRENT_SOFTCORE_LEAGUE
     stash_tab_url = "https://api.pathofexile.com/public-stash-tabs"
 
     backend_base_url = settings.BACKEND_BASE_URL
@@ -53,13 +52,12 @@ def __init__(
         items_per_batch: int,
         data_transformers: dict[str, PoEAPIDataTransformerBase],
     ):
+        self.leagues = self._get_leagues()
         self.data_transformers: dict[str, PoEAPIDataTransformerBase] = {
-            key: data_transformer()
+            key: data_transformer(self.leagues)
             for key, data_transformer in data_transformers.items()
         }
 
-        self.leagues = self._get_leagues()
-
         self.poe_api_handler = PoEAPIHandler(
             url=self.stash_tab_url,
             auth_token=self.auth_token,
@@ -188,7 +186,9 @@ def _initialize_data_stream_threads(
 
     def _follow_data_dump_stream(self):
         try:
-            current_hour = find_hours_since_launch()
+            current_hours = find_hours_since_launch(self.leagues)
+            # Only need to refer to one league to see when a new hour starts
+            current_hour = current_hours[self.leagues[0]["leagueId"]]
             next_hour = current_hour + 1
             logger.info("Retrieving modifiers from db.")
             modifier_dfs = self._get_modifiers()
@@ -204,9 +204,10 @@ def _follow_data_dump_stream(self):
                         modifier_df=modifier_dfs[data_transformer_type],
                         currency_df=currency_df.copy(deep=True),
                         item_base_types=item_base_types,
-                        current_hour=current_hour,
+                        current_hours=current_hours,
                     )
-                current_hour = find_hours_since_launch()
+                current_hours = find_hours_since_launch(self.leagues)
+                current_hour = current_hours[self.leagues[0]["leagueId"]]
             for data_transformer_type in self.data_transformers:
                 self.data_transformers[data_transformer_type].end_of_hour_cleanup()
 

src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/transforming_data/transform_currency_api_data.py

@@ -1,27 +1,11 @@
 import pandas as pd
 
 from data_retrieval_app.external_data_retrieval.config import settings
-from data_retrieval_app.external_data_retrieval.data_retrieval.currency_api_handler import (
-    CurrencyAPIHandler,
-)
 from data_retrieval_app.logs.logger import transform_logger as logger
 from data_retrieval_app.pom_api_authentication import get_superuser_token_headers
 from data_retrieval_app.utils import get_data_safe, insert_data
 
 
-def load_currency_data():
-    """
-    Loads data from the poe.ninja currency API.
-    """
-    poe_ninja_currency_api_handler = CurrencyAPIHandler(
-        url=f"https://poe.ninja/api/data/currencyoverview?league={settings.CURRENT_SOFTCORE_LEAGUE}&type=Currency"
-    )
-
-    currencies_df = poe_ninja_currency_api_handler.make_request()
-
-    return currencies_df
-
-
 class TransformCurrencyAPIData:
     def __init__(self) -> None:
         logger.debug("Initializing TransformCurrencyAPIData.")

src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/transforming_data/transform_poe_api_data.py

@@ -1,3 +1,5 @@
+from typing import Any
+
 import pandas as pd
 from requests.exceptions import HTTPError
 
@@ -14,24 +16,29 @@
 
 
 class PoEAPIDataTransformerBase:
-    def __init__(self) -> None:
+    def __init__(
+        self,
+        leagues: list[dict[str, Any]],
+    ) -> None:
         logger.debug("Initializing PoEAPIDataTransformer")
 
         self.base_url = settings.BACKEND_BASE_URL
         self.pom_auth_headers = get_superuser_token_headers(self.base_url)
         self.roll_processor = RollProcessor()
 
+        self.league_to_id = {league["name"]: league["leagueId"] for league in leagues}
+
         logger.debug("Initializing PoEAPIDataTransformer done.")
 
-    def _create_item_table(self, df: pd.DataFrame, current_hour: int) -> pd.DataFrame:
+    def _create_item_table(self, df: pd.DataFrame) -> pd.DataFrame:
         """
         Creates the basis of the `item` table.
         """
         self.item_columns = [
             "itemId",
             "id",
             "name",
-            "leagueId",
+            "league",
             "baseType",
             "typeLine",
             "ilvl",
@@ -61,11 +68,12 @@ def _create_item_table(self, df: pd.DataFrame, current_hour: int) -> pd.DataFram
             :, [column for column in self.item_columns if column in df.columns]
         ]  # Can't guarantee all columns are present
 
-        item_df["createdHoursSinceLaunch"] = current_hour
         return item_df
 
     def _find_not_too_highly_priced_item_mask(
-        self, currency_df: pd.DataFrame, item_currency_merged_df: pd.DataFrame
+        self,
+        currency_df: pd.DataFrame,
+        item_currency_merged_df: pd.DataFrame,
     ) -> pd.Series:
         """
         Some items are too highly priced to be legitimate. A boundary of the price equivelant to
@@ -91,6 +99,7 @@ def _transform_item_table(
         item_df: pd.DataFrame,
         currency_df: pd.DataFrame,
         item_base_types: dict[str, int],
+        current_hours: dict[int, int],
     ) -> pd.DataFrame:
         """
         The `item` table requires a foreign key to the `currency` table.
@@ -126,6 +135,9 @@ def transform_influences(row: pd.DataFrame, influence_columns: list[str]):
                         ] = True
                 return influence_dict
 
+        item_df["leagueId"] = item_df["league"].map(self.league_to_id)
+        item_df["createdHoursSinceLaunch"] = item_df["leagueId"].map(current_hours)
+
         base_type_series = item_df["baseType"]
         item_df["itemBaseTypeId"] = base_type_series.apply(transform_base_types)
 
@@ -259,13 +271,15 @@ def _process_item_table(
         df: pd.DataFrame,
         currency_df: pd.DataFrame,
         item_base_types: dict[str, int],
-        current_hour: int,
+        current_hours: dict[int, int],
     ) -> pd.Series:
         """
         Needs to return item ids, as it is used to connect the item modifiers
         """
-        item_df = self._create_item_table(df, current_hour=current_hour)
-        item_df = self._transform_item_table(item_df, currency_df, item_base_types)
+        item_df = self._create_item_table(df)
+        item_df = self._transform_item_table(
+            item_df, currency_df, item_base_types, current_hours=current_hours
+        )
         item_df = self._clean_item_table(item_df)
         insert_data(
             item_df,
@@ -284,11 +298,14 @@ def _transform_unidentified_item_table(
         item_df: pd.DataFrame,
         currency_df: pd.DataFrame,
         item_base_types: dict[str, int],
+        current_hours: dict[int, int],
     ) -> pd.DataFrame:
         """
         For convenience, all unid items are stored in divine prices
         """
-        item_df = self._transform_item_table(item_df, currency_df, item_base_types)
+        item_df = self._transform_item_table(
+            item_df, currency_df, item_base_types, current_hours=current_hours
+        )
 
         item_df["chaos_value"] = (
             item_df["currencyAmount"].astype(float) * item_df["valueInChaos"]
@@ -401,11 +418,11 @@ def _process_unidentified_item_table(
         df: pd.DataFrame,
         currency_df: pd.DataFrame,
         item_base_types: dict[str, int],
-        current_hour: int,
+        current_hours: dict[int, int],
     ) -> None:
-        item_df = self._create_item_table(df, current_hour=current_hour)
+        item_df = self._create_item_table(df)
         item_df = self._transform_unidentified_item_table(
-            item_df, currency_df, item_base_types
+            item_df, currency_df, item_base_types, current_hours=current_hours
         )
         item_df = self._clean_unidentified_item_table(item_df)
         insert_data(
@@ -417,7 +434,7 @@ def _process_unidentified_item_table(
         )
 
     def _create_item_modifier_table(
-        self, df: pd.DataFrame, *, item_id: pd.Series, current_hour: int
+        self, df: pd.DataFrame, *, item_id: pd.Series
     ) -> pd.DataFrame:
         """
         The `item_modifier` table heavily relies on what type of item the modifiers
@@ -428,6 +445,7 @@ def _create_item_modifier_table(
     def _transform_item_modifier_table(
         self,
         item_modifier_df: pd.DataFrame,
+        current_hours: dict[int, int],
     ) -> pd.DataFrame:
         """
         The `item_modifier` table heavily relies on what type of item the modifiers
@@ -450,14 +468,13 @@ def _process_item_modifier_table(
         self,
         df: pd.DataFrame,
         item_id: pd.Series,
-        current_hour: int,
+        current_hours: dict[int, int],
     ) -> None:
         item_modifier_df = self._create_item_modifier_table(
-            df, item_id=item_id, current_hour=current_hour
+            df, item_id=item_id, current_hours=current_hours
         )
         item_modifier_df = self._transform_item_modifier_table(item_modifier_df)
         item_modifier_df = self._clean_item_modifier_table(item_modifier_df)
-
         insert_data(
             item_modifier_df,
             url=self.base_url,
@@ -472,7 +489,7 @@ def transform_into_tables(
         modifier_df: pd.DataFrame,
         currency_df: pd.DataFrame,
         item_base_types: dict[str, int],
-        current_hour: int,
+        current_hours: dict[int, int],
     ) -> None:
         self.roll_processor.add_modifier_df(modifier_df)
         try:
@@ -482,18 +499,18 @@ def transform_into_tables(
                 df.copy(deep=True),
                 currency_df=currency_df,
                 item_base_types=item_base_types,
-                current_hour=current_hour,
+                current_hours=current_hours,
             )
             self._process_unidentified_item_table(
                 df.copy(deep=True),
                 currency_df=currency_df,
                 item_base_types=item_base_types,
-                current_hour=current_hour,
+                current_hours=current_hours,
             )
             self._process_item_modifier_table(
                 df.copy(deep=True),
                 item_id=item_id,
-                current_hour=current_hour,
+                current_hours=current_hours,
             )
             logger.debug("Successfully transformed data into tables.")
 
@@ -508,13 +525,17 @@ def end_of_hour_cleanup(self):
 class UniquePoEAPIDataTransformer(PoEAPIDataTransformerBase):
     @sync_timing_tracker
     def _create_item_modifier_table(
-        self, df: pd.DataFrame, *, item_id: pd.Series, current_hour: int
+        self,
+        df: pd.DataFrame,
+        *,
+        item_id: pd.Series,
+        current_hours: dict[int, int],
     ) -> pd.DataFrame:
         """
         A similiar process to creating the item table, only this time the
         relevant column contains a list and not a JSON-object
         """
-        item_modifier_columns = ["name", "explicitMods"]
+        item_modifier_columns = ["name", "explicitMods", "league"]
         item_modifier_df = df.loc[
             self.price_found_mask,
             item_modifier_columns,
@@ -525,19 +546,23 @@ def _create_item_modifier_table(
         ].reset_index()
 
         item_modifier_df["itemId"] = item_id
+        item_modifier_df["leagueId"] = item_modifier_df["league"].map(self.league_to_id)
+        item_modifier_df["createdHoursSinceLaunch"] = item_modifier_df["leagueId"].map(
+            current_hours
+        )
         item_modifier_df = item_modifier_df.explode("explicitMods", ignore_index=True)
 
         item_modifier_df.rename({"explicitMods": "modifier"}, axis=1, inplace=True)
 
-        item_modifier_df["createdHoursSinceLaunch"] = current_hour
-
         return item_modifier_df
 
     @sync_timing_tracker
     def _transform_item_modifier_table(
-        self, item_modifier_df: pd.DataFrame
+        self,
+        item_modifier_df: pd.DataFrame,
     ) -> pd.DataFrame:
         item_modifier_df = self.roll_processor.add_rolls(df=item_modifier_df)
+
         return item_modifier_df
 
     @property