Updates

Author: ascholerChemeketa

pretext/pretext

@@ -740,6 +740,36 @@ def main():
                 dest_dir,
                 None
             )
+        elif args.format == "html-incremental":
+            import time
+            start = time.time()
+            # provide needed extra stringparams
+            stringparams["html.quick-dirty"] = "yes"
+            stringparams["publisher"] = publication_file
+
+            # attempt to reuse RS services
+            rs_js, rs_css, rs_cdn_url, rs_version = ptx.query_existing_runestone_services(
+                dest_dir=dest_dir,
+                stringparams=stringparams
+            )
+            ptx._set_runestone_stringparams(stringparams, rs_js, rs_css, rs_version)
+
+            # get pub_vars from the publication file
+            pub_vars = ptx.get_publisher_variable_report(xml_source, publication_file, stringparams)
+
+            print("----Time to set up incremental build:", time.time() - start)
+            start = time.time()
+
+            ptx.html_incremental(
+                xml=xml_source,
+                stringparams=stringparams,
+                xmlid_root=args.xmlid,
+                extra_xsl=extra_stylesheet,
+                dest_dir=dest_dir,
+                pub_vars=pub_vars,
+            )
+            print("----Time for incremental build:", time.time() - start)
+            start = time.time()
         elif args.format == "html-zip":
             # no "subtree root" build is possible
             ptx.html(

pretext/pretext.py

@@ -3485,7 +3485,6 @@ def _set_runestone_stringparams(stringparams, rs_js, rs_css, rs_version):
 
 # A helper function to query the latest Runestone
 # Services file, while failing gracefully
-
 def _runestone_services(stringparams, ext_rs_methods):
     """Query the very latest Runestone Services file from the RS CDN"""
 
@@ -3670,6 +3669,21 @@ def query_runestone_services(services_url):
     return services_response.text
 
 
+def query_existing_runestone_services(dest_dir, stringparams):
+    '''Attempt to get Runestone service data from existing
+       Runestone Services file in _static directory.
+       Returns a tuple of the JS, CSS, CDN URL and version or None'''
+    services_record_files = os.path.join(dest_dir, "_static", "_runestone-services.xml")
+    
+    if os.path.exists(services_record_files):
+        with open(services_record_files, 'r') as f:
+            services_xml = f.read()
+        services = ET.fromstring(services_xml)
+        return _parse_runestone_services(services)
+    else:
+        msg = "query_existing_runestone_services failed: no _runestone-services.xml file found in _static directory"
+        raise RuntimeError(msg)
+
 def _place_runestone_services(tmp_dir, stringparams, ext_rs_methods):
     '''Obtain Runestone Services and place in _static directory of build'''
 
@@ -3840,6 +3854,7 @@ def _build_custom_theme(xml, theme_name, theme_opts, tmp_dir):
 # Temporary helper to move style file for custom ol markers into _static/pretext/css
 def move_ol_marker_css(tmp_dir):
     css_dest = os.path.join(tmp_dir, "_static", "pretext", "css")
+    os.makedirs(css_dest, exist_ok=True)
     src = os.path.join(tmp_dir, "ol-markers.css")
     dest = os.path.join(get_ptx_path(), os.path.join(css_dest, "ol-markers.css"))
     if os.path.exists(src):
@@ -3855,6 +3870,7 @@ def check_color_contrast(color1, color2):
         log.warning("The coloraide module is not available and is necessary for checking color contrast. Install it with 'pip install coloraide' or by using the requirements.txt file.")
 
 def build_or_copy_theme(xml, pub_var_dict, tmp_dir):
+    return
     theme_name = get_publisher_variable(pub_var_dict, 'html-theme-name')
     theme_opts_json = get_publisher_variable(pub_var_dict, 'html-theme-options')
     import json
@@ -3934,19 +3950,46 @@ def download_file(url, dest_filename):
     except Exception as e:
         raise Exception("Failed to save download", dest_filename)
 
-def quick_dirty_check_file(path):
-    """Check for existence of a file, returning True/False
-       logs a message if file was found and will be used
-       instead of downloading/rebuilding"""
-    filename = os.path.basename(path)
-    if os.path.exists(path):
-        log.info("Quick and dirty HTML is using existing {}. Delete {} from output to update.".format(filename, path))
-        return True
-    else:
+def html_incremental(xml, stringparams, xmlid_root, extra_xsl, dest_dir, pub_vars):
+    """Update an HTML incrementally in place.
+       Depends on _static and generated files already being in the destination directory.
+       Caller must supply:
+       * dict of pub_vars containing the results of get_publisher_variable_report
+       * stringparams supplemented with:
+          * rs-js, rs-css, and rs-version (can use _set_runestone_stringparams to set)
+          * publisher: path to publisher file for use by xsltproc
+       """
+    if not pub_vars:
+        log.error("Incremental build lacks required pub_vars dict.  Unable to complete build.")
+        return False
+    if not "rs-js" in stringparams or not "publisher" in stringparams:
+        log.error("Incremental build missing needed stringparam(s).  Unable to complete build.")
         return False
 
+    # to ensure provided stringparams aren't mutated unintentionally
+    stringparams = stringparams.copy()
+
+    # support subtree argument
+    if xmlid_root:
+        stringparams["subtree"] = xmlid_root
+
+    # Optional extra XSL could be None, or sanitized full filename
+    if extra_xsl:
+        extraction_xslt = extra_xsl
+    else:
+        extraction_xslt = os.path.join(get_ptx_xsl_path(), "pretext-html.xsl")
+
+    log.info("incremental convertsion of {} to HTML in {}".format(xml, dest_dir))
+    xsltproc(extraction_xslt, xml, None, dest_dir, stringparams)
+
+
 def html(xml, pub_file, stringparams, xmlid_root, file_format, extra_xsl, out_file, dest_dir, ext_rs_methods):
     """Convert XML source to HTML files, in destination directory or as zip file"""
+    import time
+    start = time.time()
+    start2 = start
+    print("====Starting html()", time.time() - start2)
+    start2 = time.time()
 
     # to ensure provided stringparams aren't mutated unintentionally
     stringparams = stringparams.copy()
@@ -3961,33 +4004,24 @@ def html(xml, pub_file, stringparams, xmlid_root, file_format, extra_xsl, out_fi
     # 1. Grab latest version and store into _static
     #    quick and dirty build will with option 1 will try to use exising _static files
     # 2. Rely on links to CDN copies
-
     pub_vars = get_publisher_variable_report(xml, pub_file, stringparams)
     include_static_files = get_publisher_variable(pub_vars, 'portable-html') != "yes"
 
+    print("====Post pub var checks", time.time() - start2)
+    start2 = time.time()
+
     if include_static_files:
-        # If quick and dirty, check for existing _static files
-        add_runestone_services = True
-        if stringparams.get("html.quick-dirty", "no") == "yes":
-            services_record_files = os.path.join(dest_dir, "_static", "_runestone-services.xml")
-            if quick_dirty_check_file(services_record_files):
-                with open(services_record_files, 'r') as f:
-                    services_xml = f.read()
-                services = ET.fromstring(services_xml)
-                rs_js, rs_css, rs_cdn_url, rs_version = _parse_runestone_services(services)
-                _set_runestone_stringparams(stringparams, rs_js, rs_css, rs_version)
-                add_runestone_services = False
-
-        if add_runestone_services:
-            # interrogate Runestone server (or debugging switches) and populate
-            # NB: stringparams is augmented with Runestone Services information
-            _place_runestone_services(tmp_dir, stringparams, ext_rs_methods)
+        # interrogate Runestone server (or debugging switches) and populate
+        # NB: stringparams is augmented with Runestone Services information
+        _place_runestone_services(tmp_dir, stringparams, ext_rs_methods)
 
     else:  # Options 2 - CDN RS services
         # even if we don't need static files, we need to set stringparams for
         # Runestone Services information.
         _cdn_runestone_services(stringparams, ext_rs_methods)
 
+    print("====Runestone placed", time.time() - start2)
+    start2 = time.time()
     # support publisher file, and subtree argument
     if pub_file:
         stringparams["publisher"] = pub_file
@@ -4000,33 +4034,32 @@ def html(xml, pub_file, stringparams, xmlid_root, file_format, extra_xsl, out_fi
         extraction_xslt = os.path.join(get_ptx_xsl_path(), "pretext-html.xsl")
 
     copy_managed_directories(tmp_dir, external_abs=external_abs, generated_abs=generated_abs)
+    print("====Copy managed done", time.time() - start2)
+    start2 = time.time()
 
     if include_static_files:
         # Copy js and css, but only if not building portable html
         # place JS in scratch directory
         copy_html_js(tmp_dir)
+        build_or_copy_theme(xml, pub_vars, tmp_dir)
 
-        need_theme_css = True
-        # quick and dirty reuses existing theme.css if possible
-        if stringparams.get("html.quick-dirty", "no") == "yes":
-            theme_css = os.path.join(dest_dir, "_static", "pretext", "css", "theme.css")
-            if quick_dirty_check_file(theme_css):
-                need_theme_css = False
-
-        if need_theme_css:
-            build_or_copy_theme(xml, pub_vars, tmp_dir)
-
+    print("====static files included", time.time() - start2)
+    start2 = time.time()
     # Write output into temporary directory
     log.info("converting {} to HTML in {}".format(xml, tmp_dir))
     xsltproc(extraction_xslt, xml, None, tmp_dir, stringparams)
 
+    print("====xsltproc complete", time.time() - start2)
+    start2 = time.time()
     if include_static_files:
         # extra css for custom ol markers
         move_ol_marker_css(tmp_dir)
     if not(include_static_files):
         # remove latex-image generated directories for portable builds
         shutil.rmtree(os.path.join(tmp_dir, "generated", "latex-image"), ignore_errors=True)
 
+    print("====move_ol_marker_css moved", time.time() - start2)
+    start2 = time.time()
     if file_format  == "html":
         # with multiple files, we need to copy a tree
         # see comments at  copy_build_directory()
@@ -4050,6 +4083,10 @@ def html(xml, pub_file, stringparams, xmlid_root, file_format, extra_xsl, out_fi
             log.info("zip file of HTML output deposited as {}".format(derivedname))
     else:
         raise ValueError("PTX:BUG: HTML file format not recognized")
+    
+    print("====copy_build_directory complete", time.time() - start2)
+    start2 = time.time()
+    print("====html() complete", time.time() - start)
 
 
 def revealjs(
@@ -4957,6 +4994,8 @@ def copy_build_directory(build_dir, dest_dir):
 
     for filename in os.listdir(build_dir):
         src = os.path.join(build_dir, filename)
+        import time
+        start = time.time()
         if os.path.isfile(src):
             shutil.copy2(src, dest_dir)
         elif os.path.isdir(src):
@@ -4967,6 +5006,8 @@ def copy_build_directory(build_dir, dest_dir):
         else:
             msg = "the build directory {} contained an unexpected object, {}"
             log.debug(msg.format(build_dir, src))
+        end = time.time()
+        #print("copying {} took {} seconds".format(src, end - start))
 
 
 def targz(output, source_dir):
@@ -5053,6 +5094,9 @@ def get_publisher_variable_report(xml_source, pub_file, params):
                 # value could have spaces, so rejoin other parts
                 variables[parts[0]] = " ".join(parts[1:])
 
+    import pprint
+    pprint.pprint(variables)
+
     return variables
 
 

xsl/pretext-assembly.xsl

@@ -300,11 +300,19 @@ along with PreTeXt.  If not, see <http://www.gnu.org/licenses/>.
 <!-- Pruning templates generate a subtree towards $target-node -->
 <!-- that prunes as many other branches as possible while      -->
 <!-- retaining halfway decent output of the desire tree.       -->
-<xsl:template match="node()|@*" mode="pruning">
+<xsl:template match="@*" mode="pruning">
+    <xsl:apply-templates select="." mode="pruning-keep"/>
+</xsl:template>
+<xsl:template match="node()" mode="pruning">
     <xsl:param name="target-node"/>
+    <!-- <xsl:message>==here at "<xsl:value-of select="name()"/>".</xsl:message>
+    <xsl:message>==target count "<xsl:value-of select="count($target-node)"/>".</xsl:message>
+    <xsl:message>==desc count "<xsl:value-of select="count(descendant::*)"/>".</xsl:message>
+    <xsl:message>==combined count "<xsl:value-of select="count(descendant::*|$target-node)"/>".</xsl:message> -->
     <xsl:choose>
         <!-- This node contains the target node, copy and keep going -->
         <xsl:when test="count(descendant::*|$target-node) = count(descendant::*)">
+            <!-- <xsl:message>==continue at "<xsl:value-of select="name()"/>".</xsl:message> -->
             <xsl:copy>
                 <xsl:apply-templates select="node()|@*" mode="pruning">
                     <xsl:with-param name="target-node" select="$target-node"/>
@@ -313,10 +321,12 @@ along with PreTeXt.  If not, see <http://www.gnu.org/licenses/>.
         </xsl:when>
         <!-- This node is target node, keep it all -->
         <xsl:when test="count(.|$target-node) = 1">
+            <!-- <xsl:message>==keep at "<xsl:value-of select="name()"/>".</xsl:message> -->
             <xsl:apply-templates select="." mode="pruning-keep"/>
         </xsl:when>
         <!-- Shift to aggressive pruning -->
         <xsl:otherwise>
+            <!-- <xsl:message>==aggressive at "<xsl:value-of select="name()"/>".</xsl:message> -->
             <xsl:apply-templates select="." mode="pruning-prune"/>
         </xsl:otherwise>
     </xsl:choose>
@@ -387,14 +397,22 @@ along with PreTeXt.  If not, see <http://www.gnu.org/licenses/>.
 <xsl:variable name="version-docinfo" select="$version-root/docinfo"/>
 <xsl:variable name="version-document-root" select="$version-root/*[not(self::docinfo)]"/>
 
+<xsl:variable name="display1">
+<xsl:message>Number of nodes in pre-pruned: <xsl:value-of select="count($version//node())"/></xsl:message>
+</xsl:variable>
+
 <!-- see pretext-html for full version of these params             -->
 <xsl:param name="subtree" select="''"/>
 <xsl:param name="html.quick-dirty" select="''"/>
 <xsl:variable name="pruning-tree-rtf">
     <xsl:choose>
-        <xsl:when test="$subtree != '' and $html.quick-dirty != ''">
+        <xsl:when test="$subtree != '' and $html.quick-dirty = 'yes'">
+            <xsl:variable name="target-node" select="$version//*[@xml:id = $subtree]"/>
+
+            <!-- <xsl:message>==target "<xsl:value-of select="local-name($target-node)"/>".</xsl:message>
+            <xsl:message>==count "<xsl:value-of select="count($target-node//*)"/>".</xsl:message> -->
             <xsl:apply-templates select="$version" mode="pruning">
-                <xsl:with-param name="target-node" select="//*[@id=$subtree]"/>
+                <xsl:with-param name="target-node" select="$version//*[@xml:id = $subtree]"/>
             </xsl:apply-templates>
         </xsl:when>
         <xsl:otherwise>
@@ -404,6 +422,10 @@ along with PreTeXt.  If not, see <http://www.gnu.org/licenses/>.
 </xsl:variable>
 <xsl:variable name="pruning-tree" select="exsl:node-set($pruning-tree-rtf)"/>
 
+<xsl:variable name="display2">
+<xsl:message>Number of nodes in pruned: <xsl:value-of select="count($pruning-tree//node())"/></xsl:message>
+</xsl:variable>
+
 <!-- This pass adds 100% internal identification for elements before   -->
 <!-- anything has been added or subtracted. The tree it builds is used -->
 <!-- for constructing "View Source" knowls in HTML output as a form of -->