chore: refactor "installing" resources to ~/.ptx (#758)

* rename zip_templates to bundle_resources

* wip

* format

* import bundle_resources tweak

* remove deprecated templates module

* formatting

* lint

* remove tests for deprecated templates module

* clean up templates imports

* fix set_ptx_path in core init

* fix templates install

* fix cli new

* format

* fix devscript alias

* fix devscript

* fix stale core.resources references

* formatting

* add sample article test

* formatting

* more accurate sample article checker

* update core commit to get working sample article

Author: StevenClontz

.devcontainer/postCreateCommand.sh

@@ -8,7 +8,7 @@ echo 'export PATH="/root/.local/bin:$PATH"' > ~/.bashrc
 sudo `which poetry` config virtualenvs.create false
 sudo `which poetry` install --with dev
 python scripts/fetch_core.py
-python scripts/zip_templates.py
+python scripts/bundle_resources.py
 playwright install-deps
 playwright install
 # Run mypy once so that it will install any needed type stubs. After this, the VSCode extension will run it automatically.

.github/workflows/tests.yml

@@ -20,7 +20,7 @@ jobs:
         run: |
           python -m pip install -e .
           python scripts/fetch_core.py
-          python scripts/zip_templates.py
+          python scripts/bundle_resources.py
       - name: Check formatting with black
         run: python -m black --check --diff $(git ls-files "*.py")
       - name: Check for lint
@@ -75,7 +75,7 @@ jobs:
         run: |
           python -m poetry install
           python -m poetry run python scripts/fetch_core.py
-          python -m poetry run python scripts/zip_templates.py
+          python -m poetry run python scripts/bundle_resources.py
 
       - name: Test with pytest
         run: |

.gitignore

@@ -138,11 +138,8 @@ cython_debug/
 
 #pretext-core
 pretext/core/pretext.py
-pretext/core/resources.zip
-#shipped templates
-pretext/templates/resources
-#old "static" stuff (deprecated
-pretext/static
+#zipped resources
+pretext/resources/*.zip
 
 #default new pretext project
 new-pretext-project

pretext/__init__.py

@@ -21,7 +21,7 @@
 
 VERSION = get_version("pretext", Path(__file__).parent.parent)
 
-CORE_COMMIT = "ed4e9d94a75b5b728aca2eb06dfde3f1595ab783"
+CORE_COMMIT = "ac1ca3ca67c9512059afd6fd37a714a7fc988a5d"
 
 
 def activate() -> None:

pretext/cli.py

@@ -25,7 +25,7 @@
 
 from . import (
     utils,
-    templates,
+    resources,
     core,
     constants,
     plastex,
@@ -248,7 +248,8 @@ def devscript(args: List[str]) -> None:
     """
     PY_CMD = sys.executable
     subprocess.run(
-        [PY_CMD, str(core.resources.path("pretext", "pretext"))] + list(args)
+        [PY_CMD, str(resources.resource_base_path() / "core" / "pretext" / "pretext")]
+        + list(args)
     )
 
 
@@ -284,47 +285,35 @@ def new(template: str, directory: Path, url_template: str) -> None:
     """
     directory_fullpath = Path(directory).resolve()
     if utils.project_path(directory_fullpath) is not None:
-        log.warning(
+        log.error(
             f"A project already exists in `{utils.project_path(directory_fullpath)}`."
         )
-        log.warning("No new project will be generated.")
+        log.error("No new project will be generated.")
         return
-    log.info(
-        f"Generating new PreTeXt project in `{directory_fullpath}` using `{template}` template."
-    )
+    log.info(f"Generating new PreTeXt project in `{directory_fullpath}`")
+    directory_fullpath.mkdir(exist_ok=True)
     if url_template is not None:
+        log.info(f"Using template at `{url_template}`")
+        # get project and extract to directory
         r = requests.get(url_template)
         archive = zipfile.ZipFile(io.BytesIO(r.content))
+        with tempfile.TemporaryDirectory(prefix="pretext_") as tmpdirname:
+            archive.extractall(tmpdirname)
+            content_path = [Path(tmpdirname) / i for i in os.listdir(tmpdirname)][0]
+            shutil.copytree(content_path, directory_fullpath, dirs_exist_ok=True)
     else:
-        with templates.resource_path(f"{template}.zip") as template_path:
-            archive = zipfile.ZipFile(template_path)
-    # find (first) project.ptx to use as root of template
-    filenames = [Path(filepath).name for filepath in archive.namelist()]
-    project_ptx_index = filenames.index("project.ptx")
-    project_ptx_path = Path(archive.namelist()[project_ptx_index])
-    project_dir_path = project_ptx_path.parent
-    with tempfile.TemporaryDirectory(prefix="pretext_") as tmpdirname:
-        temp_path = Path(tmpdirname) / "new-project"
-        temp_path.mkdir()
-        for filepath in [
-            filepath
-            for filepath in archive.namelist()
-            if project_dir_path in Path(filepath).parents
-        ]:
-            archive.extract(filepath, path=temp_path)
-        tmpsubdirname = temp_path / project_dir_path
-        shutil.copytree(tmpsubdirname, directory_fullpath, dirs_exist_ok=True)
-    # generate remaining boilerplate like requirements.txt
-    project = Project.parse(directory_fullpath)
-    project.generate_boilerplate(update_requirements=True)
-    if len(project.targets) == 0:
-        log.warning("The generated project has no targets!")
-    else:
-        target = project.targets[0]
-        log.info(f"Success! Open `{target.source_abspath()}` to edit your document")
-    log.info(
-        f"Then try to `pretext build` and `pretext view` from within `{directory_fullpath}`."
-    )
+        log.info(f"Using `{template}` template.")
+        # copy project from installed resources
+        with resources.resource_base_path() / "templates" / f"{template}" as template_path:
+            shutil.copytree(template_path, directory_fullpath, dirs_exist_ok=True)
+        # generate missing boilerplate
+        with utils.working_directory(directory_fullpath):
+            project_path = utils.project_path()
+            if project_path is None:
+                project = Project()
+            else:
+                project = Project.parse(project_path)
+            project.generate_boilerplate(update_requirements=True)
 
 
 # pretext init

pretext/core/__init__.py

@@ -7,10 +7,10 @@
         "Run `scripts/fetch_core.py` to grab a copy of pretex core.\n"
         "The original error message is: " + e.msg
     )
-from . import resources
+from .. import resources
 from .. import CORE_COMMIT, VERSION
 
-set_ptx_path(resources.path())
+set_ptx_path(resources.resource_base_path() / "core")
 
 
 def cli_build_message() -> str:

pretext/core/resources.py

@@ -24,7 +24,7 @@
 from .. import codechat
 from .. import utils
 from .. import types as pt  # PreTeXt types
-from .. import templates
+from ..resources import resource_base_path
 from .. import VERSION
 
 
@@ -286,7 +286,7 @@ def post_validate(self) -> None:
             if not self.publication_abspath().exists():
                 # ... then use the CLI's built-in template file.
                 # TODO: this is wrong, since the returned path is only valid inside the context manager. Instead, need to enter the context here, then exit it when this class is deleted (also problematic).
-                with templates.resource_path("publication.ptx") as self.publication:
+                with resource_base_path() / "templates" / "publication.ptx" as self.publication:
                     pass
         # Otherwise, verify that the provided publication file exists. TODO: It is silly to check that all publication files exist.  We warn when they don't.  If the target we are calling has a non-existent publication file, then that error will be caught anyway.
         else:
@@ -1455,7 +1455,7 @@ def generate_boilerplate(
                         f"Your existing {resource} file has been backed up at {backup_resource_path}."
                     )
             if resource != "requirements.txt":
-                with templates.resource_path(resource) as resource_path:
+                with resource_base_path() / "templates" / resource as resource_path:
                     if (
                         not project_resource_path.exists()
                         or resource_path.read_text()

pretext/project/__init__.py

@@ -0,0 +1,43 @@
+import importlib.resources
+import logging
+from pathlib import Path
+import shutil
+import zipfile
+
+from .. import VERSION, CORE_COMMIT
+
+log = logging.getLogger("ptxlogger")
+
+_RESOURCE_BASE_PATH = Path.home() / ".ptx" / VERSION
+
+
+def install(reinstall: bool = False) -> None:
+    if _RESOURCE_BASE_PATH.exists():
+        if reinstall:
+            log.info(f"Deleting existing resources at {_RESOURCE_BASE_PATH}")
+            shutil.rmtree(_RESOURCE_BASE_PATH)
+        else:
+            log.warning(f"Resources are already installed at {_RESOURCE_BASE_PATH}")
+            return
+    _RESOURCE_BASE_PATH.mkdir(parents=True)
+
+    log.info("Installing core resources")
+    with importlib.resources.path("pretext.resources", "core.zip") as static_zip:
+        with zipfile.ZipFile(static_zip, "r") as zip:
+            zip.extractall(path=_RESOURCE_BASE_PATH)
+        (_RESOURCE_BASE_PATH / f"pretext-{CORE_COMMIT}").rename(
+            _RESOURCE_BASE_PATH / "core"
+        )
+
+    log.info("Installing templates")
+    (_RESOURCE_BASE_PATH / "templates").mkdir()
+    with importlib.resources.path("pretext.resources", "templates.zip") as static_zip:
+        with zipfile.ZipFile(static_zip, "r") as zip:
+            zip.extractall(path=_RESOURCE_BASE_PATH / "templates")
+
+
+def resource_base_path() -> Path:
+    if not _RESOURCE_BASE_PATH.exists():
+        log.info(f"Installing resources to {_RESOURCE_BASE_PATH}")
+        install()
+    return _RESOURCE_BASE_PATH