Add parameter to print parsing errors to file

Author: Galarzaa90

tibiawikisql/__main__.py

@@ -1,3 +1,5 @@
+"""Command line interface for tibiawiki-sql."""
+
 import sqlite3
 
 import click
@@ -7,21 +9,26 @@
 from tibiawikisql.utils import timed
 
 DATABASE_FILE = "tibiawiki.db"
+PARSING_ERRORS_FILE = "parsing-errors.log"
 
 colorama.init()
 
 
-@click.group(context_settings={'help_option_names': ['-h', '--help']})
-@click.version_option(__version__, '-V', '--version')
-def cli():
-    # Empty command group to disable default command.
-    pass
+@click.group(context_settings={"help_option_names": ["-h", "--help"]})
+@click.version_option(__version__, "-V", "--version")
+def cli() -> None:
+    """Run tibiawiki-sql commands."""
 
 
 @cli.command(name="generate")
-@click.option('-i', '--skip-images', help="Skip fetching and loading images to the database.", is_flag=True)
-@click.option('-o', '--db-name', help="Name for the database file.", default=DATABASE_FILE)
-@click.option('-d', '--skip-deprecated', help="Skips fetching deprecated articles and their images.", is_flag=True)
+@click.option("-i", "--skip-images", help="Skip fetching and loading images to the database.", is_flag=True)
+@click.option("-o", "--db-name", help="Name for the database file.", default=DATABASE_FILE)
+@click.option("-d", "--skip-deprecated", help="Skips fetching deprecated articles and their images.", is_flag=True)
+@click.option(
+    "--log-parsing-errors",
+    help=f"Write every parsing error to {PARSING_ERRORS_FILE}.",
+    is_flag=True,
+)
 @click.option(
     "-c",
     "--skip-category",
@@ -32,16 +39,22 @@ def cli():
         "Skip specific categories. Can be repeated."
     ),
 )
-def generate(skip_images, db_name, skip_deprecated, skip_categories):
+def generate(
+    skip_images: bool,
+    db_name: str,
+    skip_deprecated: bool,
+    log_parsing_errors: bool,
+    skip_categories: tuple[str, ...],
+) -> None:
     """Generates a database file."""
-    with timed() as t:
-        with sqlite3.connect(db_name) as conn:
-            generation.generate(
-                conn,
-                skip_images=skip_images,
-                skip_deprecated=skip_deprecated,
-                skip_categories=skip_categories,
-            )
+    with timed() as t, sqlite3.connect(db_name) as conn:
+        generation.generate(
+            conn,
+            skip_images=skip_images,
+            skip_deprecated=skip_deprecated,
+            skip_categories=skip_categories,
+            parsing_errors_file=PARSING_ERRORS_FILE if log_parsing_errors else None,
+        )
     click.echo(f"Command finished in {t.elapsed:.2f} seconds.")
 
 

tibiawikisql/generation.py

@@ -3,7 +3,9 @@
 
 import datetime
 import platform
+import traceback
 from dataclasses import dataclass
+from pathlib import Path
 from typing import TYPE_CHECKING, Any, TypeVar
 
 import click
@@ -25,6 +27,7 @@
     import sqlite3
     from collections.abc import Callable, Iterable
     from click._termui_impl import ProgressBar
+    from typing import TextIO
 
 V = TypeVar("V")
 
@@ -220,6 +223,67 @@ def _run_images(conn: sqlite3.Connection, _data_store: dict[str, Any], enabled_c
     PostTask("images", _run_images),
 )
 
+PARSING_ERROR_SEPARATOR = "-" * 80
+
+
+def write_parsing_error(
+    file: TextIO,
+    *,
+    category: str,
+    article: Article,
+    error: ArticleParsingError,
+) -> None:
+    """Write a parsing error entry to a log file."""
+    file.write(f"{PARSING_ERROR_SEPARATOR}\n")
+    file.write(f"Category: {category}\n")
+    file.write(f"Article: {article.title}\n")
+    file.write(f"URL: {article.url}\n")
+    file.write("Traceback:\n")
+    file.writelines(traceback.format_exception(type(error), error, error.__traceback__))
+    file.write("\n")
+
+
+def parse_articles(
+    conn: sqlite3.Connection,
+    data_store: dict[str, Any],
+    enabled_categories: set[str],
+    parsing_errors_log: TextIO | None = None,
+) -> int:
+    """Parse category articles into the database."""
+    click.echo("Parsing articles...")
+    parsing_errors_count = 0
+    for key, category in CATEGORIES.items():
+        if key not in enabled_categories:
+            continue
+
+        titles = [entry.title for entry in data_store[key]]
+        parser = category.parser
+        if category.generate_map:
+            data_store[f"{key}_map"] = {}
+        unparsed = []
+        generator = wiki_client.get_articles(titles)
+        with (
+            timed() as t,
+            conn,
+            progress_bar(generator, len(titles), f"Parsing {key}", item_show_func=article_label) as bar,
+        ):
+            for article in bar:
+                try:
+                    entry = parser.from_article(article)
+                    entry.insert(conn)
+                    if category.generate_map:
+                        data_store[f"{key}_map"][entry.title.lower()] = entry.article_id
+                except ArticleParsingError as e:
+                    unparsed.append(article.title)
+                    parsing_errors_count += 1
+                    if parsing_errors_log:
+                        write_parsing_error(parsing_errors_log, category=key, article=article, error=e)
+        if unparsed:
+            click.echo(f"{Fore.RED}Could not parse {len(unparsed):,} articles.{Style.RESET_ALL}")
+            click.echo(f"\t-> {Fore.RED}{f'{Style.RESET_ALL},{Fore.RED}'.join(unparsed)}{Style.RESET_ALL}")
+        click.echo(f"\t{Fore.GREEN}Parsed articles in {t.elapsed:.2f} seconds.{Style.RESET_ALL}")
+    return parsing_errors_count
+
 
 def resolve_enabled_categories(skip_categories: set[str]) -> tuple[set[str], dict[str, set[str]]]:
     """Resolve enabled categories including dependency-based auto-skips."""
@@ -278,6 +342,7 @@ def generate(
     skip_images: bool = False,
     skip_deprecated: bool = False,
     skip_categories: tuple[str, ...] = (),
+    parsing_errors_file: str | None = None,
 ) -> None:
     """Generate a complete TibiaWiki SQLite database."""
     normalized_skip_categories = {category.casefold() for category in skip_categories}
@@ -306,34 +371,18 @@ def generate(
         excluded_titles = deprecated if not category.include_deprecated else None
         data_store[key] = fetch_category_entries(category.name, excluded_titles)
 
-    click.echo("Parsing articles...")
-    for key, category in CATEGORIES.items():
-        if key not in enabled_categories:
-            continue
-
-        titles = [entry.title for entry in data_store[key]]
-        parser = category.parser
-        if category.generate_map:
-            data_store[f"{key}_map"] = {}
-        unparsed = []
-        generator = wiki_client.get_articles(titles)
-        with (
-            timed() as t,
-            conn,
-            progress_bar(generator, len(titles), f"Parsing {key}", item_show_func=article_label) as bar,
-        ):
-            for article in bar:
-                try:
-                    entry = parser.from_article(article)
-                    entry.insert(conn)
-                    if category.generate_map:
-                        data_store[f"{key}_map"][entry.title.lower()] = entry.article_id
-                except ArticleParsingError:
-                    unparsed.append(article.title)
-        if unparsed:
-            click.echo(f"{Fore.RED}Could not parse {len(unparsed):,} articles.{Style.RESET_ALL}")
-            click.echo(f"\t-> {Fore.RED}{f'{Style.RESET_ALL},{Fore.RED}'.join(unparsed)}{Style.RESET_ALL}")
-        click.echo(f"\t{Fore.GREEN}Parsed articles in {t.elapsed:.2f} seconds.{Style.RESET_ALL}")
+    parsing_errors_path = Path(parsing_errors_file) if parsing_errors_file else None
+    if parsing_errors_path:
+        with parsing_errors_path.open("w", encoding="utf-8") as parsing_errors_log:
+            gen_time = datetime.datetime.now(datetime.timezone.utc)
+            parsing_errors_log.write(f"TibiaWikiSQL parsing errors - {gen_time.isoformat()}\n\n")
+            parsing_errors_count = parse_articles(conn, data_store, enabled_categories, parsing_errors_log)
+        click.echo(
+            f"{Fore.YELLOW}Wrote {parsing_errors_count:,} parsing errors to "
+            f"{parsing_errors_path}.{Style.RESET_ALL}",
+        )
+    else:
+        parse_articles(conn, data_store, enabled_categories)
 
     for position in rashid_positions:
         RashidPositionTable.insert(conn, **position.model_dump())

tibiawikisql/parsers/base.py

@@ -209,7 +209,12 @@ def from_article(cls, article: Article) -> M:
             An inherited model object for the current article.
 
         """
-        row = cls.parse_attributes(article)
+        try:
+            row = cls.parse_attributes(article)
+        except ArticleParsingError:
+            raise
+        except Exception as e:
+            raise ArticleParsingError(article, cause=e) from e
         try:
             return cls.model.model_validate(row)
         except ValidationError as e: