Feat : Refactoring 2

Author: NE7K

.gitignore

@@ -191,6 +191,4 @@ LogFile/*
 Docker/*
 
 # test python
-backup.py
-
-백업/*upstock-sentiment.py
+backup.py

main.py

@@ -1,11 +1,14 @@
-# INFO compatibility issue import
+"""
+Main entry point
+"""
+# compatibility issue import
 # from keras.preprocessing.text import Tokenizer
 # from keras.callbacks import EarlyStopping
 # from keras.preprocessing.sequence import pad_sequences
 # from keras.callbacks import TensorBoard
 
 # pip github connect | pip freeze > piplist.txt
-from upstock.storage.model_downloader import download_model_file
+from upstock.storage.downloader import download_model_file
 from upstock.builders.pipeline import run_pipeline
 
 if __name__ == '__main__':

upstock/builders/pipeline.py

@@ -1,13 +1,19 @@
+"""
+training or predict
+"""
+
 import os
-from upstock.config import model_path, tokenizer_path
-from upstock.nodes.predict_node import run_predict
-from upstock.nodes.train_node import run_trian
+from upstock.config import paths
+from upstock.nodes.predict import run_predict
+from upstock.nodes.train import run_train
 
-# save model exists => predict
-# save model not exists => DeepLearning
 def run_pipeline():
-    if os.path.exists(model_path) and os.path.exists(tokenizer_path):
+    """
+    save model exists => predict
+    save model not exists => DeepLearning
+    """
+    if os.path.exists(paths.model) and os.path.exists(paths.tokenizer):
         run_predict()
     else:
         print('Sentiment Model and Tokenizer is not exists, Start DeepLearning')
-        run_trian()
+        run_train()

upstock/config.py

@@ -6,67 +6,39 @@
 """
 
 import os
+import logging
+import sys
+
 from dotenv import load_dotenv
 from supabase import create_client, Client
 from dataclasses import dataclass   # create data class
 
-@dataclass(frozen=True) # immutable
-class Config:
-    # online
-    supabase_url: str
-    supabase_key: str
-    bucket: str = 'sentiemnt_file'  # supabase bucket name
-    tbl_sentiment: str = 'news_sentiment'   # table name
-    source: str = 'finviz'
-    tz_target: str = 'Asia/Seoul'
-    
-    # local path
-    sentiment_path: str = 'DataSets/upstock-sentiment-data.csv' # sentiment data
-    tokenizer_path: str = 'SaveModel/upstock_sentiment_tokenizer.pickle'
-    model_path: str = 'SaveModel/upstock_sentiment_model.keras'
-    model_path_h5: str = 'SaveModel/upstock_sentiment_model.h5' # compatibility issue .h5
-
-    # 논문 근거
-    model_pkl_path: str = 'SaveModel/upstock_sentiment_pkl.pkl' # import matplotlib.pyplot as plt
-
-def load_config() -> Config:
-    load_dotenv()   # env load
-    url = os.getenv('SUPABASE_URL')
-    key = os.getenv('SUPABASE_KEY')
-    if not url or not key:
-        raise RuntimeError('Supabase URL, KEY Error')
-    return Config(supabase_url=url, supabase_key=key)
+# env load
+load_dotenv()
 
-# TEST
-# model.summary()
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s [%(levelname)s] %(name)s - %(message)s',
+    stream=sys.stdout
+)
 
-# 데이터셋 null 개수 출력
-# print(news_data.isnull().sum())
-# RESULT
-# Text         0
-# Sentiment    0
-# dtype: int64
-
-# 길이 열 추가해서 카운트하고 싶으면 Added lenght column
-# sentiment_data['lenght'] = sentiment_data['Text'].str.len()
-# print(sentiment_data['Text'].str.len().max())
-# RESULT 154
-
-# 데이터셋 길이 통계 요약 출력
-# print(sentiment_data['Text'].str.len().describe())
-# RESULT
-# count    5791.000000
-# mean       78.507857
-# std        37.409135
-# min         6.000000
-# 25%        48.000000
-# 50%        79.000000
-# 75%       106.000000
-# max       154.000000
-# Name: Text, dtype: float64
-
-# RESULT 데이터셋 maxlen 95
-# lengths = sentiment_data['Text'].str.len()
-# print(lengths.quantile(0.90)) # 133.0
-# print(lengths.quantile(0.95)) # 141.0
-# exit()
+@dataclass(frozen=True) # 불변
+class SupabaseConfig:
+    url: str = os.getenv('SUPABASE_URL')
+    key: str = os.getenv('SUPABASE_KEY')
+    
+    @property
+    def client(self) -> Client:
+        return create_client(self.url, self.key)
+
+@dataclass(frozen=True)
+class PathConfig:
+    sentiment_data: str = 'DataSets/upstock-sentiment-data.csv' # sentiment data
+    tokenizer: str = 'SaveModel/upstock_sentiment_tokenizer.pickle'
+    model: str = 'SaveModel/upstock_sentiment_model.keras'
+    model_h5: str = 'SaveModel/upstock_sentiment_model.h5'
+    history: str = 'SaveModel/upstock_sentiment_pkl.pkl' # import matplotlib.pyplot as plt
+
+# export
+supabase = SupabaseConfig().client
+paths = PathConfig()

upstock/integrations/supabase_client.py

@@ -1,48 +1,57 @@
+"""
+loading models, tokenizer, dataset
+
+*using logging
+"""
+
 import pickle
 import os
 import pandas as pd
+import logging
 
 from tensorflow.keras.models import load_model
 
-# load file < predict task에는 필요없음
-def load_file(path, description):
+logger = logging.getLogger(__name__)
+
+def load_csv(path: str, description: str = 'CSV'):
     if os.path.exists(path):
         try:
-            print(f'{description} load complete')
+            logger.info(f'{description} loaded complete')
             return pd.read_csv(path)
         except Exception as e:
-            print(f'{description} exists but, load fail {e}')
+            logger.error(f'{description} exists but failed to load {e}')
             return None
     else:
-        print(f'{description} not exists')
+        logger.warning(f'{description} not exists {path}')
         return None
 
 # load save model : https://www.tensorflow.org/guide/keras/save_and_serialize?hl=ko#savedmodel_%ED%98%95%EC%8B%9D
 # model load 지침
-def check_all_model(path, description):
+def load_model_safe(path: str, description: str = 'Model'):
     if os.path.exists(path):
         try:
             model = load_model(path)
-            print(f'{description} load complete')
+            logger.info(f'{description} loaded complete')
             return model
         except Exception as e:
-            print(f'{description} load fail : {e}')
+            logger.error(f'{description} exists but failed to load {e}')
             return None
     else:
-        print(f'{description} not exists')
+        logger.warning(f'{description} not exists {path}')
         return None
 
 # load tokenizer => TextVectorization로 변경 가능성 유의
-def load_pickle(path, description):
+def load_pickle(path: str, description: str = 'Pickle'):
     if os.path.exists(path):
         try:
             with open(path, 'rb') as f:
                 tokenizer = pickle.load(f)
-                print(f'{description} load complete')
+                logger.info(f'{description} loaded complete')
                 return tokenizer
         except Exception as e:
-            print(f'{description} load fail : {e}')
+            logger.error(f'{description} exsists but failed to load {e}')
             return None
     else:
-        print(f'{description} not exists')
+        logger.warning(f'{description} not exists {path}')
         return None
+

upstock/models/artifacts.py

@@ -1,16 +1,31 @@
+"""
+Predict node
+
+fetches news, runs sentiment model, uploads results to supabase
+"""
+
 import hashlib # hash
 import datetime
 import pandas as pd
+import logging
+
 from tensorflow.keras.preprocessing.sequence import pad_sequences
 from finvizfinance.news import News
+from zoneinfo import ZoneInfo
 
-from upstock.config import model_path, tokenizer_path, supabase
-from upstock.models.artifacts import check_all_model, load_pickle
+from upstock.config import paths, supabase
+from upstock.models.artifacts import load_model_safe, load_pickle
+
+logger = logging.getLogger(__name__)
 
 def run_predict():
 
-    model = check_all_model(model_path, 'Sentiment Model .keras version')
-    tokenizer = load_pickle(tokenizer_path, 'Tokenizer')
+    model = load_model_safe(paths.model, 'Sentiment Model (.keras) version')
+    tokenizer = load_pickle(paths.tokenizer, 'Tokenizer')
+    
+    if model is None or tokenizer is None:
+        logger.error('model or tokenizer not available')    # flow task check
+        return
 
     # def load_news():
     #     fnews = News()
@@ -26,14 +41,18 @@ def run_predict():
         all_news = fnews.get_news()
         news_df = all_news['news'] 
     except Exception as e:
-        print(f'finviz news parse fail : {e}')
-
-    # string to date time
-    news_df['parsed_date'] = pd.to_datetime(news_df['Date'], errors='coerce') # BUG format 지정
+        logger.error(f'finviz news parse failed : {e}')
+        return
 
+    # print(news_df['Date'].head(10))
     today = datetime.date.today() # today
-    today_news = news_df[news_df['parsed_date'].dt.date == today] # today == parse data date
 
+    # string to date time
+    news_df['parsed_date'] = pd.to_datetime(
+        news_df['Date'],
+        errors='coerce'
+    )
+    today_news = news_df[news_df['parsed_date'].dt.date == today] # today == parse data date
     predict_texts = today_news['Title'].tolist() # insert pare data
 
     # past predict data
@@ -47,7 +66,6 @@ def run_predict():
 
     predict_data = tokenizer.texts_to_sequences(predict_texts)
     predict_data = pad_sequences(predict_data, maxlen=141) # str.len result 95% 141
-
     prediction = model.predict(predict_data)
     # print(prediction)
 
@@ -65,19 +83,19 @@ def run_predict():
 
     for text, percent in zip(predict_texts, prediction):
         # 강한 긍정과 강한 부정만 끌어다가 쓰기
-        score = float(percent[0])
+        score = float(percent[0])   # float 필수
         if score >= 0.8:
             label = "positive"
         elif score <= 0.3:
             label = "negative"
         else:
             continue
 
-        print(f"[{label}] {text}\n : {score:.2f}\n") # :.2f
+        logger.info(f"[{label}] {text} : {score:.2f}\n") # :.2f
 
         sb_result.append({
             'text': text,
-            'percent': score, # BUG type error float32
+            'percent': score,
             'label': label,
             'source': 'finviz',
             'run_at': datetime.datetime.now(datetime.timezone.utc).isoformat(), # utc time
@@ -91,11 +109,11 @@ def run_predict():
                 .upsert(sb_result, on_conflict='hash')
                 .execute()
             )
-            print(f'supbase upload complete : {len(response.data)}')
+            logger.info(f'supbase upload complete : {len(response.data)} rows')
 
         except Exception as e:
-            print(f'supabase upload fail : {e}')
+            logger.error(f'supabase upload failed : {e}')
 
     else:
-        print('upload data not exist')
+        logger.warning('upload data not exist')