Add --simulate dry-run mode to ResourcesBot

Run resourcesbot.py directly for wiki previews without login, writes,
git push, or local exports. Centralize page saves in pywikibot_io.
Remove outdated pbw.py and user-config.py references

Author: holybiber

.github/workflows/main.yml

@@ -33,7 +33,7 @@ jobs:
 
       - name: Generate coverage report
         run: |
-          coverage run --source=pywikitools/,pywikitools/correctbot/ --omit=pywikitools/user-config.py -m unittest discover -s pywikitools/test
+          coverage run --source=pywikitools/,pywikitools/correctbot/ -m unittest discover -s pywikitools/test
           coverage xml
 
       - name: Upload coverage to Codecov

Makefile

@@ -42,7 +42,7 @@ format: ## auto-format code with ruff
 	$(RUFF) format .
 
 coverage: ## run tests with coverage report and open HTML in browser
-	$(COVERAGE) run --source=pywikitools/,pywikitools/correctbot/ --omit=pywikitools/user-config.py -m unittest discover -s pywikitools/test
+	$(COVERAGE) run --source=pywikitools/,pywikitools/correctbot/ -m unittest discover -s pywikitools/test
 	$(COVERAGE) report -m
 	$(COVERAGE) html
 	$(BROWSER) htmlcov/index.html

README.md

@@ -87,6 +87,9 @@ set the logging level to INFO (default is WARN) by adding `-l info`.
 $ python path/to/script args
 ```
 
+Bots use `config.ini` for site and username settings and run directly with Python,
+not via pywikibot's `pwb.py` wrapper.
+
 If you're not yet logged in, `pywikibot` will ask you for the password for the
 user you defined in `config.ini`. After successful login, the login cookie is
 stored in `pywikibot-[UserName].lwp` so you don't have to log in every time.

pyproject.toml

@@ -2,8 +2,6 @@
 line-length = 88
 exclude = [
     "env",
-    "pywikitools/user-config.py",
-    "pywikitools/correctbot/user-config.py",
 ]
 
 [tool.ruff.format]

pywikitools/correctbot/bot.py

@@ -244,8 +244,7 @@ def save_to_mediawiki(self, results: List[CorrectionResult]) -> bool:
         """
         Write corrections back to mediawiki
 
-        You should disable pywikibot throttling to avoid CorrectBot runs to take quite long:
-        `put_throttle = 0` in user-config.py
+        Throttling is disabled in __init__ via site.throttle.set_delays(...).
 
         Returns:
             bool: Did we save any corrections to the mediawiki system?

pywikitools/family.py

@@ -1,32 +1,22 @@
 """Family file for connecting to 4training.net with pywikibot
 
-This is intended to be used without using the traditional pywikibot approach
-of using user-config.py for configuration.
-Instead we use pywikibot short site creation (requires pywikibot >= 7.3)
-
-However, if you still want to use the traditional approach, you could just take
-this file, rename it to 4training_family.py and put in the right folder
-(pywikibot families folder or some other folder that you specify with
-user_families_paths in user-config.py).
-
-Then use a user-config.py with the following configuration to connect to test.4training.net:
-family = '4training'
-mylang = 'test'
-usernames['4training']['test'] = 'YourUserName'
+Bots connect via pywikibot short site creation (requires pywikibot >= 7.3) and
+config.ini for site/username settings — no user-config.py needed:
+
+    family = Family()
+    self.site = pywikibot.Site(code='test', fam=family, user='username')
+
+code can be
+'4training' => www.4training.net
+'test' => test.4training.net
+'local' => localhost:8082
 """
 
 import pywikibot
 
 
 class Family(pywikibot.family.SubdomainFamily):
-    """
-    The family class for our 4training.net website
-    for connecting to it via pywikibot short site creation (new feature of pywikibot 7.3):
-    No cumbersome user-config.py is necessary anymore, instead we can write
-    family = Family()
-    self.site = pywikibot.Site(code='test', fam=family, user='username')
-    code can be either '4training' or 'test' to connect to www.4training.net / test.4training.net
-    """
+    """The family class for our 4training.net website."""
 
     name = "4training"
 

pywikitools/pywikibot_io.py

@@ -0,0 +1,18 @@
+"""Shared pywikibot helpers for pywikitools bots."""
+
+import pywikibot
+
+
+def save_page(
+    page: pywikibot.page.BasePage,
+    new_text: str,
+    summary: str,
+    *,
+    simulate: bool = False,
+) -> None:
+    """Assign new page text and save; in simulate mode show a pywikibot diff only."""
+    if simulate:
+        pywikibot.showDiff(page.text, new_text)
+        return
+    page.text = new_text
+    page.save(summary)

pywikitools/resourcesbot/bot.py

@@ -20,6 +20,7 @@
     WorksheetInfo,
     json_decode,
 )
+from pywikitools.pywikibot_io import save_page
 from pywikitools.resourcesbot.modules.post_processing import LanguagePostProcessor
 from pywikitools.resourcesbot.modules.write_summary import WriteSummary
 
@@ -33,6 +34,10 @@
     "write_sidebar_messages",
 ]
 
+SIMULATE_SKIP_MODULES: Final[frozenset[str]] = frozenset(
+    {"export_html", "export_pdf", "export_repository"}
+)
+
 
 def load_module(module_name: str) -> Callable:
     """Load the post-processing module from modules/ and return it
@@ -62,6 +67,7 @@ def __init__(
         limit_to_lang: Optional[str] = None,
         modules: list[str] = AVAILABLE_MODULES,
         rewrite: Optional[str] = None,
+        simulate: bool = False,
     ):
         """
         Args:
@@ -72,8 +78,13 @@ def __init__(
                 limit processing to one language (string with a language code)
             modules:
                 specify which post-processing modules should be executed
+            simulate:
+                dry-run: show pywikibot diffs, skip login, writes, and export modules
         """
+        self._simulate = simulate
         self.modules = modules
+        if simulate:
+            self.modules = [m for m in self.modules if m not in SIMULATE_SKIP_MODULES]
         # read-only list of download file types
         self._file_types: Final[List[str]] = ["pdf", "odt", "odg", "printPdf"]
         self._config = config
@@ -128,6 +139,18 @@ def __init__(
         self._changelog: Dict[str, ChangeLog] = {}
 
     def run(self):
+        previous_simulate = pywikibot.config.simulate
+        if self._simulate:
+            pywikibot.config.simulate = True
+            self.logger.info(
+                "Simulate mode: no MediaWiki writes, skipping export modules"
+            )
+        try:
+            self._run()
+        finally:
+            pywikibot.config.simulate = previous_simulate
+
+    def _run(self):
         if self._read_from_cache:
             try:
                 # List of languages to be read from cache
@@ -175,7 +198,9 @@ def run(self):
         # failed with a WARNING from pywikibot:
         # "No user is logged in on site 4training:en"-> better check and try
         # to log in if necessary
-        if not self.site.logged_in():
+        if self._simulate:
+            self.logger.info("Simulate mode: skipping login")
+        elif not self.site.logged_in():
             self.logger.info("We're not logged in. Trying to log in...")
             self.site.login()
             if not self.site.logged_in():
@@ -208,7 +233,10 @@ def run(self):
 
         for selected_module in self.modules:
             module = load_module(selected_module)(
-                self.fortraininglib, self._config, self.site
+                self.fortraininglib,
+                self._config,
+                self.site,
+                simulate=self._simulate,
             )
             for lang in self._result:
                 module.run(
@@ -222,7 +250,7 @@ def run(self):
 
         # Now run all GlobalPostProcessors
         if not self._limit_to_lang:
-            write_summary = WriteSummary(self.site)
+            write_summary = WriteSummary(self.site, simulate=self._simulate)
             write_summary.run(
                 self._result,
                 self._changelog,
@@ -457,8 +485,12 @@ def _sync_and_compare(self, language_info: LanguageInfo) -> ChangeLog:
             self.logger.warning(
                 f"{page.full_url()} doesn't seem to exist yet. Creating..."
             )
-            page.text = encoded_json
-            page.save("Created JSON data structure")
+            save_page(
+                page,
+                encoded_json,
+                "Created JSON data structure",
+                simulate=self._simulate,
+            )
             rewrite_json = False
         else:
             # Load "old" data structure of this language (from previous resourcesbot run)
@@ -481,8 +513,12 @@ def _sync_and_compare(self, language_info: LanguageInfo) -> ChangeLog:
 
         if rewrite_json:
             # Write the updated JSON structure
-            page.text = encoded_json
-            page.save("Updated JSON data structure")
+            save_page(
+                page,
+                encoded_json,
+                "Updated JSON data structure",
+                simulate=self._simulate,
+            )
             self.logger.info(f"Updated 4training:{lang}.json")
 
         return changes
@@ -510,8 +546,9 @@ def _save_languages_list(self):
         # TODO compare language_list and json.loads(previous_json) to find out if a new
         #  language was added
         if previous_json != encoded_json:
-            page.text = encoded_json
-            page.save("Updated list of languages")
+            save_page(
+                page, encoded_json, "Updated list of languages", simulate=self._simulate
+            )
             self.logger.info("Updated 4training:languages.json")
 
     def _save_number_of_languages(self):
@@ -546,8 +583,12 @@ def _save_number_of_languages(self):
 
         if previous_number_of_languages != number_of_languages:
             try:
-                page.text = number_of_languages
-                page.save("Updated number of languages")
+                save_page(
+                    page,
+                    str(number_of_languages),
+                    "Updated number of languages",
+                    simulate=self._simulate,
+                )
                 self.logger.info(
                     f"Updated MediaWiki:Numberoflanguages to {number_of_languages}"
                 )

pywikitools/resourcesbot/modules/consistency_checks.py

@@ -44,8 +44,10 @@ def __init__(
         fortraininglib: ForTrainingLib,
         config: ConfigParser = None,
         site: pywikibot.site.APISite = None,
+        *,
+        simulate: bool = False,
     ):
-        super().__init__(fortraininglib, config, site)
+        super().__init__(fortraininglib, config, site, simulate=simulate)
         self.logger = logging.getLogger(
             "pywikitools.resourcesbot.modules.consistency_checks"
         )

pywikitools/resourcesbot/modules/export_html.py

@@ -61,8 +61,10 @@ def __init__(
         fortraininglib: ForTrainingLib,
         config: ConfigParser,
         site: pywikibot.site.APISite,
+        *,
+        simulate: bool = False,
     ):
-        super().__init__(fortraininglib, config, site)
+        super().__init__(fortraininglib, config, site, simulate=simulate)
         self._base_folder: str = self._config.get("Paths", "htmlexport", fallback="")
         self.logger: Final[logging.Logger] = logging.getLogger(
             "pywikitools.resourcesbot.modules.export_html"