Automatic generation of needed assets (#447)

* add methods for hashing assets, saving and loading

* add automatic generation of missing assets

* add --no-generate option, improve logs

Author: oscarlevin

pretext/cli.py

@@ -326,6 +326,13 @@ def init(refresh):
     type=click.Choice(ASSETS, case_sensitive=False),
     help="Generates assets for target.  -g [asset] will generate the specific assets given.",
 )
+@click.option(
+    "-q",
+    "--no-generate",
+    is_flag=True,
+    default=False,
+    help="Do not generate assets for target, even if their source has changed since last build.",
+)
 @click.option(
     "-x", "--xmlid", type=click.STRING, help="xml:id of element to be generated."
 )
@@ -340,16 +347,14 @@ def build(
     target,
     clean,
     generate,
+    no_generate,
     xmlid: t.Optional[str],
     project_ptx_override: t.Tuple[str, str],
 ):
     """
     Build [TARGET] according to settings specified by project.ptx.
 
-    If using certain elements (webwork, latex-image, etc.) then
-    using `--generate` may be necessary for a successful build. Generated
-    assets are cached so they need not be regenerated in subsequent builds unless
-    they are changed.
+    If using elements that require separate generation of assets (e.g., webwork, latex-image, etc.) then these will be generated automatically if their source has changed since the last build.  You can suppress this with the `--no-generate` flag, or force a regeneration with the `--generate` flag.
 
     Certain builds may require installations not included with the CLI, or internet
     access to external servers. Command-line paths
@@ -375,23 +380,60 @@ def build(
     target = project.target(name=target_name)
     if target_name is None:
         log.info(
-            f"Since no build target was supplied, the first target of the project.ptx manifest ({target.name()}) will be built."
+            f"Since no build target was supplied, the first target of the project.ptx manifest ({target.name()}) will be built.\n"
         )
         target_name = target.name()
     if target is None:
         utils.show_target_hints(target_name, project, task="build")
         log.critical("Exiting without completing build.")
         return
+    # Automatically generate any assets that have changed.
+    if not no_generate:
+        asset_table = target.load_asset_table()
+        asset_hash_dict = target.asset_hash()
+        if asset_table == asset_hash_dict:
+            log.info(
+                "No change in assets requiring generating detected.  To force regeneration of assets, use `-g` flag.\n"
+            )
+        else:
+            for asset in set(asset[0] for asset in asset_hash_dict.keys()):
+                if asset in ["webwork"]:
+                    if (asset, "") not in asset_table or asset_hash_dict[
+                        (asset, "")
+                    ] != asset_table[(asset, "")]:
+                        project.generate(target.name(), asset_list=[asset])
+                elif (asset, "") not in asset_table or asset_hash_dict[
+                    (asset, "")
+                ] != asset_table[(asset, "")]:
+                    project.generate(target.name(), asset_list=[asset])
+                else:
+                    for id in set(
+                        key[1] for key in asset_hash_dict.keys() if key[0] == asset
+                    ):
+                        if (asset, id) not in asset_table or asset_hash_dict[
+                            (asset, id)
+                        ] != asset_table[(asset, id)]:
+                            log.info(
+                                f"\nIt appears the source has changed of an asset that needs to be generated.  Now generating asset: {asset} with xmlid: {id}."
+                            )
+                            project.generate(
+                                target.name(), asset_list=[asset], xmlid=id
+                            )
+            target.save_asset_table(target.asset_hash())
+    else:
+        log.info("Skipping asset generation as requested.")
     if generate == "ALL":
-        log.info("Generating all assets in default formats.")
+        log.info("Generating all assets in default formats as requested.")
+        log.info(
+            "Note: PreTeXt will automatically generate assets that have been changed since your last build, so this option is no longer necessary unless something isn't happening as expected."
+        )
         project.generate(target.name())
     elif generate is not None:
-        log.warning(f"Generating only {generate} assets.")
+        log.info(f"Generating {generate} assets as requested.")
+        log.info(
+            "Note: PreTeXt will automatically generate assets that have been changed since your last build, so this option is no longer necessary unless something isn't happening as expected."
+        )
         project.generate(target.name(), asset_list=[generate])
-    else:
-        log.warning("Assets like latex-images will not be regenerated for this build")
-        log.warning("(previously generated assets will be used if they exist).")
-        log.warning("To generate these assets before building, run `pretext build -g`.")
     project.build(target.name(), clean)
 
 

pretext/project.py

@@ -1,3 +1,4 @@
+import pickle
 from lxml import etree as ET
 from lxml.etree import Element
 import os
@@ -6,10 +7,12 @@
 import tempfile
 from . import utils, generate, core
 from . import build as builder
+from . import ASSETS
 from pathlib import Path
 import sys
 from .config.xml_overlay import ShadowXmlDocument
 import typing as t
+import hashlib
 
 log = logging.getLogger("ptxlogger")
 
@@ -125,6 +128,59 @@ def xmlid_root(self):
         else:
             return ele.text.strip()
 
+    def asset_hash(self):
+        asset_hash_dict = {}
+        for asset in ASSETS:
+            if asset == "webwork":
+                # WeBWorK must be regenerated every time *any* of the ww exercises change.
+                if len(self.source_xml().xpath(".//webwork[@*|*]")) == 0:
+                    # Only generate a hash if there are actually ww exercises in the source
+                    continue
+                h = hashlib.sha256()
+                for node in self.source_xml().xpath(".//webwork[@*|*]"):
+                    h.update(ET.tostring(node))
+                asset_hash_dict[(asset, "")] = h.digest()
+            elif asset != "ALL":
+                # everything else can be updated individually, if it has an xml:id
+                if len(self.source_xml().xpath(f".//{asset}")) == 0:
+                    # Only generate a hash if there are actually assets of this type in the source
+                    continue
+                h_no_id = hashlib.sha256()
+                for node in self.source_xml().xpath(f".//{asset}"):
+                    # First see if the node has an xml:id, or if it is a child of a node with an xml:id (but we haven't already made this key)
+                    if (
+                        id := node.xpath("@xml:id") or node.xpath("parent::*/@xml:id")
+                    ) and (asset, id[0]) not in asset_hash_dict:
+                        asset_hash_dict[(asset, id[0])] = hashlib.sha256(
+                            ET.tostring(node)
+                        ).digest()
+                    # otherwise collect all non-id'd nodes into a single hash
+                    else:
+                        h_no_id.update(ET.tostring(node))
+                asset_hash_dict[(asset, "")] = h_no_id.digest()
+        return asset_hash_dict
+
+    def save_asset_table(self, asset_table: dict):
+        """
+        Saves the asset_table to a pickle file in the generated assets directory based on the target name.
+        """
+        with open(
+            self.generated_dir().joinpath(f".{self.name()}_assets.pkl"), "wb"
+        ) as f:
+            pickle.dump(asset_table, f)
+
+    def load_asset_table(self) -> dict:
+        """
+        Loads the asset_table from a pickle file in the generated assets directory based on the target name.
+        """
+        try:
+            with open(
+                self.generated_dir().joinpath(f".{self.name()}_assets.pkl"), "rb"
+            ) as f:
+                return pickle.load(f)
+        except Exception:
+            return {}
+
 
 class Project:
     def __init__(self, project_path=None):