doker upd

Author: EEnota

.dockerignore

@@ -0,0 +1,11 @@
+.git
+.gitignore
+.venv
+__pycache__
+*.pyc
+*.pyo
+*.pyd
+.env
+data/raw
+data/clean
+notebooks

.env.example

@@ -1,7 +1,7 @@
-DB_HOST=localhost
+DB_HOST=db
 DB_PORT=5432
-DB_USER=postgres
-DB_PASSWORD=1234
+DB_USER=news_user
+DB_PASSWORD=CHANGE_ME_STRONG_PASSWORD
 DB_ADMIN_DB=postgres
-DB_NEWS=db_news
-NEWSAPI_KEY =your_key
+DB_NEWS=news_db
+NEWSAPI_KEY=YOUR_REAL_NEWSAPI_KEY

DockerFile

@@ -0,0 +1,13 @@
+FROM python:3.12-slim
+
+WORKDIR /app
+
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV PYTHONUNBUFFERED=1
+
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY . .
+
+CMD ["python", "main.py"]

main.py

@@ -64,6 +64,11 @@ def parse_args():
         type=int,
         default=1
     )
+    parser.add_argument(
+        "--language",
+        type=str,
+        default="ru"
+    )
     return parser.parse_args()
 
 def init_all_tables(debug: bool) -> None:

src/extract.py

@@ -23,10 +23,10 @@ def import_to_raw_json(data:dict[str, Any], key_word: str, page: int) -> str:
 
 
 
-def make_extract_debug( key_word: str, page: int = 1, page_size: int = 100) -> tuple[str, int]:
+def make_extract_debug( key_word: str, page: int = 1, page_size: int = 100, language: str = "ru") -> tuple[str, int]:
     params = {
     "apiKey": settings.KEY_API,
-    "language":settings.langueage,
+    "language":language,
     "q": key_word,
     "pageSize" : page_size,
     "page" : page,
@@ -38,7 +38,7 @@ def make_extract_debug( key_word: str, page: int = 1, page_size: int = 100) -> t
         payload = data.json()
         logger.info(f"raise of status: {data.status_code}")
         payload["fetched_at"] = datetime.now().isoformat()
-        payload["language"] = settings.langueage
+        payload["language"] = language
         payload["key_word"] = key_word
         articles_count = len(payload.get("articles", []))
         if articles_count == 0:
@@ -58,10 +58,10 @@ def make_extract_debug( key_word: str, page: int = 1, page_size: int = 100) -> t
         logger.error("Error: sorry we can't parse JSON")
         raise
 
-def make_extract_web( key_word: str, page: int = 1, page_size: int = 100) -> tuple[dict,int]:
+def make_extract_web( key_word: str, page: int = 1, page_size: int = 100, language:str = "ru") -> tuple[dict,int]:
     params = {
     "apiKey": settings.KEY_API,
-    "language":settings.langueage,
+    "language":language,
     "q": key_word,
     "pageSize" : page_size,
     "page" : page,
@@ -73,7 +73,7 @@ def make_extract_web( key_word: str, page: int = 1, page_size: int = 100) -> tup
         payload = data.json()
         logger.info(f"raise of status: {data.status_code}")
         payload["fetched_at"] = datetime.now().isoformat()
-        payload["language"] = settings.langueage
+        payload["language"] = language
         payload["key_word"] = key_word
         articles_count = len(payload.get("articles", []))
         if articles_count == 0:

src/pipeline.py

@@ -41,12 +41,12 @@ def merge_stats(stats: dict, page_stats: dict)-> None:
     for k,v in page_stats["prime_reason"].items():
         stats["prime_reason"][k] += v
 
-def run_pipeline_for_web_user(user_id: int, search_request_id: int, key_word: str, limit: int, page_size: int) -> int:
+def run_pipeline_for_web_user(user_id: int, search_request_id: int, key_word: str, limit: int, page_size: int, language:str = "ru") -> int:
     statistic = make_empty_stats()
     num_of_news = 0
     page = 1
     while num_of_news < limit:
-        payload, raw_articles_count = make_extract_web(key_word, page, page_size)
+        payload, raw_articles_count = make_extract_web(key_word, page, page_size, language)
         if raw_articles_count == 0:
             logger.warning("there is no more artical")
             break
@@ -59,12 +59,12 @@ def run_pipeline_for_web_user(user_id: int, search_request_id: int, key_word: st
     logger.info(f"{num_of_news} news on key word {key_word} already aploaded")
     return num_of_news
 
-def run_debug_pipeline(keyword: str, limit:int, page_size: int) -> int:
+def run_debug_pipeline(keyword: str, limit:int, page_size: int, language: str = "ru") -> int:
     num_of_news = 0
     page = 1
     while num_of_news < limit:
         max_rows = limit - num_of_news
-        new_file_name, raw_articles_count = make_extract_debug(keyword, page, page_size)
+        new_file_name, raw_articles_count = make_extract_debug(keyword, page, page_size, language)
         if raw_articles_count == 0:
             logger.warning("there is no more artical")
             break

src/worker.py

@@ -63,11 +63,12 @@ def one_request() -> bool:
     keyword = request_row["keyword"]
     limit_count = request_row["limit_count"]
     page_size = request_row["page_size"]
+    language = request_row["language"]
     logger.info(f"Pipeline starts for {user_id} on {search_request_id} by keyword {keyword}")
     mark_as_running(search_request_id)
 
     try:
-        amount_of_articles = run_pipeline_for_web_user(user_id, search_request_id, keyword, limit_count, page_size)
+        amount_of_articles = run_pipeline_for_web_user(user_id, search_request_id, keyword, limit_count, page_size, language=language)
 
         mark_as_success(search_request_id)
         logger.info(f"Pipeline finished successfully for {user_id} on {search_request_id} by {keyword}")