fix: mypy checks

Author: Helias

module/scraping/notice.py

@@ -53,25 +53,25 @@ def from_url(cls, label: str, url: str) -> "Notice | None":
                         table_content += "\t".join(cols_text) + "\n"
                     table.decompose()  # remove table from content
 
-            title = cls.__get_title(cls, soup)
+            title = cls.__get_title(soup)
             content = soup.find("div", attrs={"class": "field-item even"})
 
-            prof = cls.__get_prof(cls, soup)
+            prof = cls.__get_prof(soup)
 
             if title is not None and content is not None:
-                title = title.get_text()
-                content = content.get_text()
+                title_text = title.get_text()
+                content_text = content.get_text()
 
-                content = f"{content.strip()}\n{table_content}"
+                content = f"{content_text.strip()}\n{table_content}"
                 if prof is not None:
-                    title = f"[{prof}]\n{title}"
+                    title_text = f"[{prof}]\n{title_text}"
 
             else:
                 return None
 
-            title = f"\n{title}"
+            title_text = f"\n{title_text}"
 
-            return cls(label, title, content, url)
+            return cls(label, title_text, content, url)
         except (
             requests.Timeout,
             requests.ConnectionError,
@@ -83,7 +83,8 @@ def from_url(cls, label: str, url: str) -> "Notice | None":
 
             return None
 
-    def __get_prof(self, soup: bs4.BeautifulSoup) -> str | None:
+    @staticmethod
+    def __get_prof(soup: bs4.BeautifulSoup) -> str | None:
         """Returns the prof of the notice
         Args:
             soup: BeautifulSoup object of the page
@@ -92,17 +93,23 @@ def __get_prof(self, soup: bs4.BeautifulSoup) -> str | None:
         """
         goto_prof_text = "Vai alla scheda del prof. "
         prof = soup.find("a", text=lambda text: text and goto_prof_text in text)
-        return prof and prof.get_text().replace(goto_prof_text, "")
+        if isinstance(prof, bs4.Tag):
+            return prof.get_text().replace(goto_prof_text, "")
+        return None
 
-    def __get_title(self, soup: bs4.BeautifulSoup) -> bs4.BeautifulSoup | None:
+    @staticmethod
+    def __get_title(soup: bs4.BeautifulSoup) -> bs4.Tag | None:
         """Returns the title of the notice
         Args:
             soup: BeautifulSoup object of the page
         Returns:
             the soup of the title
         """
         title = soup.find("h1", attrs={"class": "page-title"})
-        return title if title else soup.select_one("section#content h1")
+        if isinstance(title, bs4.Tag):
+            return title
+        result = soup.select_one("section#content h1")
+        return result if isinstance(result, bs4.Tag) else None
 
     @property
     def formatted_url(self) -> str:

module/scraping/scraper_group.py

@@ -1,10 +1,14 @@
 """Scrape groups"""
+
 import logging
 import os
 import shutil
+
 import yaml
 from telegram.ext import CallbackContext
-from module.data import NoticeData, GroupConfig, DEFAULT_NOTICES_DATA, config_map
+
+from module.data import DEFAULT_NOTICES_DATA, GroupConfig, NoticeData, config_map
+
 from .notice import Notice
 from .scraper_links import get_links
 
@@ -24,7 +28,9 @@ def scrape_group(context: CallbackContext, group_key: str, group: GroupConfig) -
         logging.info("-- Page '%s'", page_key)
 
         # Generate page folder's path and subpaths
-        base_page_path = f"data/avvisi/{group_key.replace(' ', '_')}/{page_key.replace(' ', '_')}"
+        base_page_path = (
+            f"data/avvisi/{group_key.replace(' ', '_')}/{page_key.replace(' ', '_')}"
+        )
         data_file_path = f"{base_page_path}/notices_data.yaml"
 
         # Initialize folder and data file (if it doesn't exist)
@@ -52,21 +58,28 @@ def scrape_group(context: CallbackContext, group_key: str, group: GroupConfig) -
 
                 # If link has already been scraped
                 # (implying that's invalid page or already posted notice), skip it
-                if link in notices_data["scraped_links"]:
+                if not isinstance(link, str) or link in notices_data["scraped_links"]:
                     logging.info("Link is already present in the list")
                     continue
 
+                # Type guard for mypy to ensure link is str
+                assert isinstance(link, str)
+
                 notice = Notice.from_url(page["label"], group["base_url"] + link)
 
                 # If the notice is valid,
                 # enqueue it to be sent in the channel or in an approval group
                 if notice is not None:
-                    logging.info("Link is valid and seems to contain a notice, spamming")
+                    logging.info(
+                        "Link is valid and seems to contain a notice, spamming"
+                    )
                     notice.send(context, page["channels"])
                 else:
                     logging.info("Link doesn't contain a valid notice")
-                    context.bot.sendMessage(chat_id=config_map["log_group_chatid"],
-                                            text=f"Link doesn't contain a valid notice: {link}")
+                    context.bot.sendMessage(
+                        chat_id=config_map["log_group_chatid"],
+                        text=f"Link doesn't contain a valid notice: {link}",
+                    )
 
                 # Appends current link to scraped ones
                 notices_data["scraped_links"].append(link)

module/scraping/scraper_links.py

@@ -10,7 +10,7 @@
 from module.data import config_map
 
 
-def get_links(url: str) -> "list[str] | None":
+def get_links(url: str) -> list[str | list[str] | None] | None:
     """Generates a list of links to the notices scraped from the page indicated by the url.
 
     Args:

requirements.txt

@@ -8,7 +8,7 @@ chardet==4.0.0
 click==8.1.3
 cryptography==3.4.7
 decorator==5.0.5
-dill==0.3.6
+dill
 idna==2.10
 lazy-object-proxy==1.8.0
 mccabe==0.7.0

requirements_dev.txt

@@ -4,4 +4,5 @@ mypy-extensions==0.4.3
 pylint==4.0.4
 pytest
 flake8
-mypy
+mypy
+typed-ast