Merge pull request #814 from Path-of-Modifiers/813-fix-poe-ninja-api-endpoint-change

#813 Switched from poe ninja to poe watch

Author: bogadisa

…rieval/poe_ninja_currency_api_handler.py …l/data_retrieval/currency_api_handler.pysrc/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/poe_ninja_currency_api_handler.py renamed to src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/currency_api_handler.py src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/poe_ninja_currency_api_handler.py renamed to src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/data_retrieval/currency_api_handler.py

@@ -4,27 +4,10 @@
 from data_retrieval_app.logs.logger import external_data_retrieval_logger as logger
 
 
-class PoENinjaCurrencyAPIHandler:
+class CurrencyAPIHandler:
     def __init__(self, url: str) -> None:
         self.url = url
 
-    def _combine_currency_data(self, currencies: list, currency_details: list) -> list:
-        """
-        Combines the currency data.
-        """
-        currencies_df = self._json_to_df(currencies)
-        currency_details_df = self._json_to_df(currency_details)
-
-        combined_currency_data_df = currencies_df.merge(
-            currency_details_df,
-            how="left",
-            # left_on="pay.pay_currency_id",
-            # right_on="id",
-            left_on="currencyTypeName",
-            right_on="name",
-        )
-        return combined_currency_data_df
-
     def _json_to_df(self, currencies: list) -> pd.DataFrame:
         df = pd.json_normalize(currencies)
 
@@ -44,19 +27,16 @@ def make_request(self) -> pd.DataFrame:
             raise e
         response_json = response.json()
 
-        currencies = response_json["lines"]
-        currency_details = response_json["currencyDetails"]
-
-        combined_currency_data_df = self._combine_currency_data(
-            currencies, currency_details
-        )
+        # items_df = pd.DataFrame(response_json["items"])
+        items_df = pd.json_normalize(response_json["items"])
+        currency_df = items_df[items_df["category"] == "currency"]
 
-        return combined_currency_data_df
+        return currency_df
 
     def store_data_to_csv(self, path: str) -> None:
         """
         Stores the data in a CSV. Only to be used for testing purposes.
         """
         currencies_df = self.make_request()
 
-        currencies_df.to_csv(path + "/poe_ninja_currencies.csv", index=False)
+        currencies_df.to_csv(path + "/currencies.csv", index=False)

src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/main.py

@@ -11,19 +11,19 @@
 import requests
 
 from data_retrieval_app.external_data_retrieval.config import settings
+from data_retrieval_app.external_data_retrieval.data_retrieval.currency_api_handler import (
+    CurrencyAPIHandler,
+)
 from data_retrieval_app.external_data_retrieval.data_retrieval.poe_api_handler import (
     PoEAPIHandler,
 )
-from data_retrieval_app.external_data_retrieval.data_retrieval.poe_ninja_currency_api_handler import (
-    PoENinjaCurrencyAPIHandler,
+from data_retrieval_app.external_data_retrieval.transforming_data.transform_currency_api_data import (
+    TransformCurrencyAPIData,
 )
 from data_retrieval_app.external_data_retrieval.transforming_data.transform_poe_api_data import (
     PoEAPIDataTransformerBase,
     UniquePoEAPIDataTransformer,
 )
-from data_retrieval_app.external_data_retrieval.transforming_data.transform_poe_ninja_currency_api_data import (
-    TransformPoENinjaCurrencyAPIData,
-)
 from data_retrieval_app.external_data_retrieval.utils import (
     ProgramRunTooLongException,
     ProgramTooSlowException,
@@ -62,10 +62,10 @@ def __init__(
             n_unique_wanted_items=10,
         )
 
-        self.poe_ninja_currency_api_handler = PoENinjaCurrencyAPIHandler(
-            url=f"https://poe.ninja/api/data/currencyoverview?league={self.current_league}&type=Currency"
+        self.currency_api_handler = CurrencyAPIHandler(
+            url=f"https://api.poe.watch/exchange/ratios?league={self.current_league}&game=poe1"
         )
-        self.poe_ninja_transformer = TransformPoENinjaCurrencyAPIData()
+        self.currency_transformer = TransformCurrencyAPIData()
 
     def _get_modifiers(self) -> dict[str, pd.DataFrame]:
         response = requests.get(self.modifier_url, headers=self.pom_auth_headers)
@@ -147,8 +147,8 @@ def _categorize_new_items(self, df: pd.DataFrame) -> dict[str, pd.DataFrame]:
         return split_dfs
 
     def _get_new_currency_data(self) -> pd.DataFrame:
-        currency_df = self.poe_ninja_currency_api_handler.make_request()
-        currency_df = self.poe_ninja_transformer.transform_into_tables(currency_df)
+        currency_df = self.currency_api_handler.make_request()
+        currency_df = self.currency_transformer.transform_into_tables(currency_df)
         return currency_df
 
     def _initialize_data_stream_threads(
@@ -233,6 +233,8 @@ def retrieve_data(self):
                             )
                             futures[follow_future] = "data_processing"
                     elif future_job == "listener":
+                        logger.exception(crashed_future.exception().with_traceback())
+                        raise crashed_future.exception()
                         new_future = self._initialize_data_stream_threads(
                             executor,
                             listeners=1,

…transform_poe_ninja_currency_api_data.py …ming_data/transform_currency_api_data.pysrc/backend_data_retrieval/data_retrieval_app/external_data_retrieval/transforming_data/transform_poe_ninja_currency_api_data.py renamed to src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/transforming_data/transform_currency_api_data.py src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/transforming_data/transform_poe_ninja_currency_api_data.py renamed to src/backend_data_retrieval/data_retrieval_app/external_data_retrieval/transforming_data/transform_currency_api_data.py

@@ -2,8 +2,8 @@
 import requests
 
 from data_retrieval_app.external_data_retrieval.config import settings
-from data_retrieval_app.external_data_retrieval.data_retrieval.poe_ninja_currency_api_handler import (
-    PoENinjaCurrencyAPIHandler,
+from data_retrieval_app.external_data_retrieval.data_retrieval.currency_api_handler import (
+    CurrencyAPIHandler,
 )
 from data_retrieval_app.logs.logger import transform_logger as logger
 from data_retrieval_app.pom_api_authentication import get_superuser_token_headers
@@ -14,7 +14,7 @@ def load_currency_data():
     """
     Loads data from the poe.ninja currency API.
     """
-    poe_ninja_currency_api_handler = PoENinjaCurrencyAPIHandler(
+    poe_ninja_currency_api_handler = CurrencyAPIHandler(
         url=f"https://poe.ninja/api/data/currencyoverview?league={settings.CURRENT_SOFTCORE_LEAGUE}&type=Currency"
     )
 
@@ -23,57 +23,89 @@ def load_currency_data():
     return currencies_df
 
 
-class TransformPoENinjaCurrencyAPIData:
+class TransformCurrencyAPIData:
     def __init__(self) -> None:
-        logger.debug("Initializing TransformPoENinjaCurrencyAPIData.")
+        logger.debug("Initializing TransformCurrencyAPIData.")
         self.base_url = settings.BACKEND_BASE_URL
         logger.debug(f"Url set to: {self.base_url}")
         self.pom_api_headers = get_superuser_token_headers(self.base_url)
         logger.debug("Headers set to: " + str(self.pom_api_headers))
-        logger.debug("Initializing TransformPoENinjaCurrencyAPIData done.")
+        logger.debug("Initializing TransformCurrencyAPIData done.")
 
-    def _create_currency_table(self, currency_df: pd.DataFrame) -> pd.DataFrame:
+        self.name_to_trade_name = self._get_name_to_trade_name_dict()
+
+    def _get_name_to_trade_name_dict(self) -> dict:
         """
-        Creates the currency table.
+        Retrieves a map for "fancy" currency names, as used in the API, to their trade names, which we need.
         """
-        currency_df.rename(
-            columns={
-                "tradeId": "tradeName",
-                "chaosEquivalent": "valueInChaos",
-            },
-            inplace=True,
-        )
-        return currency_df
+        headers = {
+            "User-Agent": f"OAuth pathofmodifiers/0.1.0 (contact: {settings.OATH_ACC_TOKEN_CONTACT_EMAIL}) StrictMode"
+        }
+        try:
+            response = requests.get(
+                "https://www.pathofexile.com/api/trade/data/static", headers=headers
+            )
+            response.raise_for_status()
+        except Exception as e:
+            logger.error(
+                f"The following error occurred while making request _get_latest_change_id: {e}"
+            )
+            raise e
+
+        response_json = response.json()
+        result = response_json["result"]
+        currencies = {}
+        for category in result:
+            if category["id"] == "Currency":
+                for entry in category["entries"]:
+                    name = entry["text"]
+                    trade_name = entry["id"]
+                    currencies[name] = trade_name
+
+        return currencies
 
     def _transform_currency_table(
         self, currency_df: pd.DataFrame, hours_since_launch: int
     ) -> pd.DataFrame:
         """
         Since a chaos orb is always worth one chaos orb, ninja does not include it in its price api.
         """
+
+        currency_df["chaos.chaosValue"] = currency_df["chaos.chaosValue"].where(
+            (currency_df["chaos.chaosValue"] == 0)
+            | (currency_df["chaos.chaosValue"].isna()),
+            currency_df["divine.chaosValue"],
+        )
+
         chaos_dict = {
-            "tradeName": ["chaos"],
-            "valueInChaos": [1],
+            "name": ["Chaos Orb"],
+            "chaos.chaosValue": [1],
         }
         chaos_df = pd.DataFrame.from_dict(chaos_dict)
         currency_df = pd.concat((currency_df, chaos_df), ignore_index=True)
 
+        currency_df["tradeName"] = currency_df["name"].map(
+            lambda name: self.name_to_trade_name.get(name, pd.NA)
+        )
+
         currency_df["createdHoursSinceLaunch"] = hours_since_launch
         return currency_df
 
     def _clean_currency_table(self, currency_df: pd.DataFrame) -> pd.DataFrame:
         """
         Cleans the currency table of unnecessary columns.
         """
+        currency_df = currency_df.rename(columns={"chaos.chaosValue": "valueInChaos"})
 
-        currency_df.drop(
+        currency_df = currency_df.drop(
             currency_df.columns.difference(
                 ["tradeName", "valueInChaos", "createdHoursSinceLaunch"]
             ),
             axis=1,
-            inplace=True,
         )
-        currency_df = currency_df.loc[~currency_df["tradeName"].isna()].reset_index()
+        currency_df = currency_df.loc[~currency_df["tradeName"].isna()].reset_index(
+            drop=True
+        )
         return currency_df
 
     def _get_latest_currency_id_series(self, currency_df: pd.DataFrame) -> pd.Series:
@@ -102,7 +134,6 @@ def transform_into_tables(self, currency_df: pd.DataFrame) -> pd.DataFrame:
         """
         hours_since_launch = find_hours_since_launch()
         logger.debug("Transforming data into tables.")
-        currency_df = self._create_currency_table(currency_df)
         currency_df = self._transform_currency_table(currency_df, hours_since_launch)
         logger.debug("Successfully transformed data into tables.")
 

src/backend_data_retrieval/data_retrieval_app/tests/external_data_retrieval/data_retrieval/test_poe_ninja_currency_api_handler.py

@@ -1,4 +1,4 @@
-class TestPoENinjaCurrencyAPIHandler:
+class TestCurrencyAPIHandler:
     """
     settings.POE_PUBLIC_STASHES_AUTH_TOKEN and settings.OATH_ACC_TOKEN_CONTACT_EMAIL
     needs to be set to for this test to work

src/backend_data_retrieval/data_retrieval_app/tests/external_data_retrieval/transforming_data/transformation_models/test_transform_poe_ninja_currency_api_data.py

@@ -1,3 +1,3 @@
-class TestTransformPoeNinjaCurrencyAPIData:
+class TestTransformCurrencyAPIData:
     # TODO: Add tests if needed
     pass