Make bg3 plugin more robust (#204)

* fix plugin icons not showing up when building with mob by embedding svgs instead
* gate yaml import in bg3_file_mapper to ensure its only imported if user is using the functionality
* bail out of mapping function early if modlist is empty
* ensure spacing is consistent in generated modsettings.xml by creating a function to share the creating of the xml strings for ModuleShortDesc
* dont do diffing if file doesnt exist or debug logging is not enabled
* ensure dialog boxes are exec'd so the user can see them
* mv log.txt files to logs dir, only make one logging statement for file moving
* remove empty folders left behind by plugin in documents dir when executable finishes, log folders removed
* add conversion of yamls in bin folder that wont get mapped to the documents directory
* ensure all directories are created when they are needed
* add catch for oserror in pak parsing function
* add a small sleep after creation of modsettings.xml to ensure the executable does not launch too fast to see it
* create directories that should be sent to overwrite folder from the documents folder if they dont exist in the mapping function, log all directories mapped if debug is enabled

Author: daescha

games/baldursgate3/bg3_file_mapper.py

@@ -1,11 +1,9 @@
 import functools
-import json
 import os
 from pathlib import Path
 from typing import Callable, Optional
 
-import yaml
-from PyQt6.QtCore import QDir, qDebug, qInfo, qWarning
+from PyQt6.QtCore import QDir, QLoggingCategory, qDebug, qInfo, qWarning
 from PyQt6.QtWidgets import QApplication
 
 import mobase
@@ -29,29 +27,49 @@ def mappings(self) -> list[mobase.Mapping]:
         qInfo("creating custom bg3 mappings")
         self.current_mappings.clear()
         active_mods = self._utils.active_mods()
-        doc_dir = Path(self.doc_dir().path())
+        if not active_mods:
+            return []
         progress = self._utils.create_progress_window(
             "Mapping files to documents folder", len(active_mods) + 1
         )
-        docs_path_mods = doc_dir / "Mods"
-        docs_path_se = doc_dir / "Script Extender"
+        docs_path_mods = self.doc_path / "Mods"
+        docs_path_se = self.doc_path / "Script Extender"
         for mod in active_mods:
             modpath = Path(mod.absolutePath())
             self.map_files(modpath, dest=docs_path_mods, pattern="*.pak", rel=False)
             self.map_files(modpath / "Script Extender", dest=docs_path_se)
+            if self._utils.convert_yamls_to_json:
+                self.map_files(modpath / "bin", only_convert=True)
             progress.setValue(progress.value() + 1)
             QApplication.processEvents()
             if progress.wasCanceled():
                 qWarning("mapping canceled by user")
                 return self.current_mappings
+        (self._utils.overwrite_path / "Script Extender").mkdir(
+            parents=True, exist_ok=True
+        )
+        (self._utils.overwrite_path / "Stats").mkdir(parents=True, exist_ok=True)
+        (self._utils.overwrite_path / "Temp").mkdir(parents=True, exist_ok=True)
+        (self._utils.overwrite_path / "LevelCache").mkdir(parents=True, exist_ok=True)
+        (self._utils.overwrite_path / "Stats").mkdir(parents=True, exist_ok=True)
+        (self._utils.overwrite_path / "Mods").mkdir(parents=True, exist_ok=True)
+        (self._utils.overwrite_path / "GMCampaigns").mkdir(parents=True, exist_ok=True)
         self.map_files(self._utils.overwrite_path)
         self.create_mapping(
             self._utils.modsettings_path,
-            doc_dir / "PlayerProfiles" / "Public" / self._utils.modsettings_path.name,
+            self.doc_path
+            / "PlayerProfiles"
+            / "Public"
+            / self._utils.modsettings_path.name,
         )
         progress.setValue(len(active_mods) + 1)
         QApplication.processEvents()
         progress.close()
+        cat = QLoggingCategory.defaultCategory()
+        if cat is not None and cat.isDebugEnabled():
+            qDebug(
+                f"resolved mappings: { {m.source: m.destination for m in self.current_mappings} }"
+            )
         return self.current_mappings
 
     def map_files(
@@ -60,6 +78,7 @@ def map_files(
         dest: Optional[Path] = None,
         pattern: str = "*",
         rel: bool = True,
+        only_convert: bool = False,
     ):
         dest = dest if dest else self.doc_path
         dest_func: Callable[[Path], str] = (
@@ -77,6 +96,10 @@ def map_files(
                     if not converted_path.exists() or os.path.getmtime(
                         file
                     ) > os.path.getmtime(converted_path):
+                        import json
+
+                        import yaml
+
                         with open(file, "r") as yaml_file:
                             with open(converted_path, "w") as json_file:
                                 json.dump(
@@ -88,12 +111,16 @@ def map_files(
                     qWarning(f"Error accessing file {converted_path}: {e}")
             elif file.name.endswith(".json"):
                 found_jsons.add(file)
-            else:
+            elif not only_convert:
                 self.create_mapping(file, dest / dest_func(file))
+        if only_convert:
+            return
         for file in found_jsons:
             self.create_mapping(file, dest / dest_func(file))
 
     def create_mapping(self, file: Path, dest: Path):
+        bg3_utils.create_dir_if_needed(dest)
+
         self.current_mappings.append(
             mobase.Mapping(
                 source=str(file),

games/baldursgate3/bg3_utils.py

@@ -2,6 +2,7 @@
 import shutil
 import typing
 from pathlib import Path
+from time import sleep
 
 from PyQt6.QtCore import (
     QCoreApplication,
@@ -27,30 +28,47 @@
 }
 
 
+def get_node_string(
+    folder: str = "",
+    md5: str = "",
+    name: str = "",
+    publish_handle: str = "0",
+    uuid: str = "",
+    version64: str = "0",
+) -> str:
+    return f"""
+                        <node id="ModuleShortDesc">
+                            <attribute id="Folder" type="LSString" value="{folder}"/>
+                            <attribute id="MD5" type="LSString" value="{md5}"/>
+                            <attribute id="Name" type="LSString" value="{name}"/>
+                            <attribute id="PublishHandle" type="uint64" value="{publish_handle}"/>
+                            <attribute id="UUID" type="guid" value="{uuid}"/>
+                            <attribute id="Version64" type="int64" value="{version64}"/>
+                        </node>"""
+
+
 class BG3Utils:
-    _mod_settings_xml_start = """<?xml version="1.0" encoding="UTF-8"?>
-    <save>
-        <version major="4" minor="8" revision="0" build="200"/>
-        <region id="ModuleSettings">
-            <node id="root">
-                <children>
-                    <node id="Mods">
-                        <children>
-                            <node id="ModuleShortDesc">
-                                <attribute id="Folder" type="LSString" value="GustavX"/>
-                                <attribute id="MD5" type="LSString" value=""/>
-                                <attribute id="Name" type="LSString" value="GustavX"/>
-                                <attribute id="PublishHandle" type="uint64" value="0"/>
-                                <attribute id="UUID" type="guid" value="cb555efe-2d9e-131f-8195-a89329d218ea"/>
-                                <attribute id="Version64" type="int64" value="36028797018963968"/>
-                            </node>"""
+    _mod_settings_xml_start = """\
+<?xml version="1.0" encoding="UTF-8"?>
+<save>
+    <version major="4" minor="8" revision="0" build="500"/>
+    <region id="ModuleSettings">
+        <node id="root">
+            <children>
+                <node id="Mods">
+                    <children>""" + get_node_string(
+        folder="GustavX",
+        name="GustavX",
+        uuid="cb555efe-2d9e-131f-8195-a89329d218ea",
+        version64="36028797018963968",
+    )
     _mod_settings_xml_end = """
-                        </children>
-                    </node>
-                </children>
-            </node>
-        </region>
-    </save>"""
+                    </children>
+                </node>
+            </children>
+        </node>
+    </region>
+</save>"""
 
     def __init__(self, name: str):
         self.main_window = None
@@ -89,28 +107,32 @@ def convert_yamls_to_json(self):
 
     @functools.cached_property
     def log_dir(self):
-        return Path(self._organizer.basePath()) / "logs"
+        return create_dir_if_needed(Path(self._organizer.basePath()) / "logs")
 
     @functools.cached_property
     def modsettings_backup(self):
-        return self.plugin_data_path / "temp" / "modsettings.lsx"
+        return create_dir_if_needed(self.plugin_data_path / "temp" / "modsettings.lsx")
 
     @functools.cached_property
     def modsettings_path(self):
-        return Path(self._organizer.profilePath()) / "modsettings.lsx"
+        return create_dir_if_needed(
+            Path(self._organizer.profilePath()) / "modsettings.lsx"
+        )
 
     @functools.cached_property
     def plugin_data_path(self) -> Path:
         """Gets the path to the data folder for the current plugin."""
-        return Path(self._organizer.pluginDataPath(), self._name).absolute()
+        return create_dir_if_needed(
+            Path(self._organizer.pluginDataPath(), self._name).absolute()
+        )
 
     @functools.cached_property
     def tools_dir(self):
-        return self.plugin_data_path / "tools"
+        return create_dir_if_needed(self.plugin_data_path / "tools")
 
     @functools.cached_property
     def overwrite_path(self):
-        return Path(self._organizer.overwritePath())
+        return create_dir_if_needed(Path(self._organizer.overwritePath()))
 
     def active_mods(self) -> list[mobase.IModInterface]:
         modlist = self._organizer.modList()
@@ -234,9 +256,19 @@ def retrieve_mod_metadata_in_new_thread(mod: mobase.IModInterface):
             f"backing up generated file {self.modsettings_path} to {self.modsettings_backup}, "
             f"check the backup after the executable runs for differences with the file used by the game if you encounter issues"
         )
+        self.modsettings_backup.parent.mkdir(parents=True, exist_ok=True)
         shutil.copy(self.modsettings_path, self.modsettings_backup)
+        sleep(0.5)
         return True
 
     def on_mod_installed(self, mod: mobase.IModInterface) -> None:
         if self.lslib_retriever.download_lslib_if_missing():
             self._pak_parser.get_metadata_for_files_in_mod(mod, True)
+
+
+def create_dir_if_needed(path: Path) -> Path:
+    if "." not in path.name[1:]:
+        path.mkdir(parents=True, exist_ok=True)
+    else:
+        path.parent.mkdir(parents=True, exist_ok=True)
+    return path

games/baldursgate3/lslib_retriever.py

@@ -86,6 +86,7 @@ def reporthook(block_num: int, block_size: int, total_size: int) -> None:
                             err.setText(
                                 "LSLib tools are required for the proper generation of the modsettings.xml file, file will not be generated"
                             )
+                            err.exec()
                             return False
                     else:
                         progress = self._utils.create_progress_window(
@@ -106,6 +107,7 @@ def reporthook(block_num: int, block_size: int, total_size: int) -> None:
                     new_msg.setText(
                         self._utils.tr("Latest version of LSLib already downloaded!")
                     )
+                    new_msg.exec()
 
         except Exception as e:
             qDebug(f"Download failed: {e}")

games/baldursgate3/pak_parser.py

@@ -161,17 +161,21 @@ def _get_metadata_for_file(
                 )
                 build_pak = True
                 if pak_path.exists():
-                    pak_creation_time = os.path.getmtime(pak_path)
-                    for root, _, files in os.walk(file):
-                        for f in files:
-                            file_path = os.path.join(root, f)
-                            try:
-                                if os.path.getmtime(file_path) > pak_creation_time:
+                    try:
+                        pak_creation_time = os.path.getmtime(pak_path)
+                        for root, _, files in file.walk():
+                            for f in files:
+                                file_path = root.joinpath(f)
+                                try:
+                                    if os.path.getmtime(file_path) > pak_creation_time:
+                                        break
+                                except OSError as e:
+                                    qDebug(f"Error accessing file {file_path}: {e}")
                                     break
-                            except OSError as e:
-                                qDebug(f"Error accessing file {file_path}: {e}")
-                                break
-                    else:
+                        else:
+                            build_pak = False
+                    except OSError as e:
+                        qDebug(f"Error accessing file {pak_path}: {e}")
                         build_pak = False
                 if build_pak:
                     pak_path.unlink(missing_ok=True)
@@ -274,18 +278,18 @@ def metadata_to_ini(
 
 
 def get_module_short_desc(config: configparser.ConfigParser, file: Path) -> str:
+    if not config.has_section(file.name):
+        return ""
+    section: configparser.SectionProxy = config[file.name]
     return (
         ""
-        if not config.has_section(file.name)
-        or "override" in config[file.name].keys()
-        or "Name" not in config[file.name].keys()
-        else f"""
-                <node id="ModuleShortDesc">
-                    <attribute id="Folder" type="LSString" value="{config[file.name]["Folder"]}"/>
-                    <attribute id="MD5" type="LSString" value="{config[file.name]["MD5"]}"/>
-                    <attribute id="Name" type="LSString" value="{config[file.name]["Name"]}"/>
-                    <attribute id="PublishHandle" type="uint64" value="{config[file.name]["PublishHandle"]}"/>
-                    <attribute id="UUID" type="guid" value="{config[file.name]["UUID"]}"/>
-                    <attribute id="Version64" type="int64" value="{config[file.name]["Version64"]}"/>
-                </node>"""
+        if "override" in section.keys() or "Name" not in section.keys()
+        else bg3_utils.get_node_string(
+            folder=section["Folder"],
+            md5=section["MD5"],
+            name=section["Name"],
+            publish_handle=section["PublishHandle"],
+            uuid=section["UUID"],
+            version64=section["Version64"],
+        )
     )

games/baldursgate3/plugins/bg3_tool_plugin.py

@@ -1,19 +1,21 @@
-from pathlib import Path
-
 from PyQt6.QtCore import QCoreApplication
-from PyQt6.QtGui import QIcon
+from PyQt6.QtGui import QIcon, QPixmap
 
 import mobase
 
 
 class BG3ToolPlugin(mobase.IPluginTool, mobase.IPlugin):
-    icon_file = desc = sub_name = ""
+    desc = sub_name = ""
+    icon_bytes: bytes
 
     def __init__(self):
         mobase.IPluginTool.__init__(self)
         mobase.IPlugin.__init__(self)
         self._pluginName = self._displayName = "BG3 Tools"
         self._pluginVersion = mobase.VersionInfo(1, 0, 0)
+        pixmap = QPixmap()
+        pixmap.loadFromData(self.icon_bytes, "SVG")
+        self.qicon = QIcon(pixmap)
 
     def init(self, organizer: mobase.IOrganizer) -> bool:
         self._organizer = organizer
@@ -41,7 +43,7 @@ def settings(self) -> list[mobase.PluginSetting]:
         return []
 
     def icon(self) -> QIcon:
-        return QIcon(str(Path(__file__).parent / "icons" / self.icon_file))
+        return self.qicon
 
     def description(self) -> str:
         return QCoreApplication.translate(self._pluginName, self.desc)

games/baldursgate3/plugins/check_for_lslib_updates_plugin.py

@@ -1,8 +1,9 @@
 from .bg3_tool_plugin import BG3ToolPlugin
+from .icons import download
 
 
 class BG3ToolCheckForLsLibUpdates(BG3ToolPlugin):
-    icon_file = "ui-update.ico"
+    icon_bytes = download
     sub_name = "Check For LsLib Updates"
     desc = "Check to see if there has been a new release of LSLib and create download dialog if so."
 

games/baldursgate3/plugins/convert_jsons_to_yaml_plugin.py

@@ -6,10 +6,11 @@
 from PyQt6.QtWidgets import QApplication
 
 from .bg3_tool_plugin import BG3ToolPlugin
+from .icons import exchange
 
 
 class BG3ToolConvertJsonsToYaml(BG3ToolPlugin):
-    icon_file = "ui-next.ico"
+    icon_bytes = exchange
     sub_name = "Convert JSONS to YAML"
     desc = "Convert all jsons in active mods to yaml immediately."
 
@@ -39,14 +40,14 @@ def display(self):
 
 
 def _convert_jsons_in_dir_to_yaml(path: Path):
-    import yaml
-
     for file in list(path.rglob("*.json")):
         converted_path = file.parent / file.name.replace(".json", ".yaml")
         try:
             if not converted_path.exists() or os.path.getmtime(file) > os.path.getmtime(
                 converted_path
             ):
+                import yaml
+
                 with open(file, "r") as json_file:
                     with open(converted_path, "w") as yaml_file:
                         yaml.dump(