OD-2139: Fix skipping tags on damaged imports

Previously, the system would skip importing the entire class of tags
when a malformed tag was found in any story.  This patch changes it to
error on malformed tags, but allow the non-standard (mostly) all string
tag format.

In addition, we now warn when importing stories that have tags not in
the tag table.

Author: hlieberman

efiction/metadata.py

@@ -153,11 +153,13 @@ def _convert_story_tag_table(self, table_name, old_tags):
         if self.tag_table_is_nonstandard[table_name]:
             # Tag table identified by name rather than id.
             original_tagid = "original_tag"
-        return [
-            c["id"]
-            for c in self.tag_tables[table_name]
-            if str(c[original_tagid]) in old_tags[table_name]
-        ]
+
+        valid_tag_ids = {str(c[original_tagid]): c["id"] for c in self.tag_tables[table_name]}
+
+        dropped_tags = [tag for tag in old_tags[table_name] if tag and tag not in valid_tag_ids]
+        if dropped_tags:
+            self.logger.warning(f"Found tags in stories but not in tag table: {dropped_tags}")
+        return [valid_tag_ids[tag] for tag in old_tags[table_name] if tag in valid_tag_ids]
 
     def _convert_story_tags(self, old_story):
         old_tags = {

efiction/tag_converter.py

@@ -3,10 +3,9 @@
 
 from typing import Dict, List
 
-from opendoors.mysql import SqlDb
+from opendoors.mysql import OperationalError, ProgrammingError, SqlDb
 from opendoors.utils import print_progress
 
-
 class TagConverter:
     def __init__(self, config: ConfigParser, logger: Logger, sql: SqlDb):
         self.logger = logger
@@ -56,20 +55,31 @@ def check_for_nonstandard_tag_tables(self) -> bool:
 
                 try:
                     query = f"SELECT {id_name} FROM stories;"
-                    tags = self.sql.execute_and_fetchall(self.working_original, query)
-                    try:
-                        tags = list(
-                            map(
-                                lambda story_tags: story_tags[id_name].replace(",", ""),
-                                tags,
-                            )
-                        )
-                        int("".join(tags))
+                    tag_rows = self.sql.execute_and_fetchall(self.working_original, query)
+                    story_tags = [tag_row[id_name] for tag_row in tag_rows]
+                    # Get the number of story tags which contain something other
+                    # than a comma-separated list of digits
+                    is_numeric = [not tag or tag.isdigit() for tags in story_tags for tag in tags.split(",")]
+
+                    # There are three possibilities for the way tags have been
+                    # put into stories. The most common, by far, is that they're
+                    # all comma-separated lists of integers. If this is not the
+                    # case, then they should be almost all -- though not
+                    # necessarily all -- comma-separated strings. (This is
+                    # because a tag could potentially be all-numeric, and have
+                    # at least one fic only tagged with that all-numeric tag).
+                    if sum(is_numeric) == len(is_numeric):
+                        self.logger.debug(f"Standard story tag syntax in {tag_table_name}")
                         tag_tables[tag_table_name] = False
-                    except Exception:
-                        # Non-integer in identifier
+
+                    elif (sum(is_numeric) / len(is_numeric)) < .1:
+                        self.logger.info(f"Non-standard story tag syntax in {tag_table_name}")
                         tag_tables[tag_table_name] = True
-                except Exception as e:
+
+                    # Finally, the fields could be completely corrupt -- in which case we want to break.
+                    else:
+                        raise Exception(f"Broken story tag syntax in {tag_table_name}")
+                except (OperationalError, ProgrammingError) as e:
                     self.logger.info(e)
                     self.logger.info("No such table?")
                     tag_tables[tag_table_name] = None

opendoors/mysql.py

@@ -6,10 +6,10 @@
 import pymysql
 import sqlparse
 from pymysql.cursors import DictCursor
+from pymysql.err import ProgrammingError, OperationalError # noqa: F401
 
 from opendoors.utils import get_full_path
 
-
 class SqlDb:
     """
     Wrapper and helper methods for MySQL commands