Switch to a submodule for building Hugo (#272)

* Add more architecture mappings

* Move to an `src/` format package

* Add `hugo` as a submodule

* Prune the sdist and make it smaller

* Check out repository with submodules too

* Update Hugo submodule tag for release update PRs

* Fail if sdist size exceeds 10 MiB

* Update instructions for building from sources

* Drop Hugo git-clone step from setup

* Ensure sdist builds can get an accurate build time

* Git-ignore the build timestamp file

* Some more fixes...

* Claude can try too

* More fixes from Claude

Author: agriyakhetarpal

.github/workflows/cd.yml

@@ -26,6 +26,7 @@ jobs:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:
           persist-credentials: false
+          submodules: recursive
 
       - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
         with:
@@ -34,6 +35,18 @@ jobs:
       - name: Build source distribution
         run: pipx run build --sdist --outdir dist/
 
+      - name: Check sdist size, fail if it exceeds 10 MiB
+        run: |
+          MAX_SIZE_MB=10
+          SDIST=$(ls dist/*.tar.gz)
+          SIZE=$(stat --format=%s "$SDIST")
+          SIZE_MB=$(awk "BEGIN {printf \"%.2f\", $SIZE / 1048576}")
+          echo "sdist size: ${SIZE_MB} MB"
+          if [ "$SIZE" -gt $((MAX_SIZE_MB * 1048576)) ]; then
+            echo "::error::sdist is ${SIZE_MB} MB, exceeding the ${MAX_SIZE_MB} MB limit"
+            exit 1
+          fi
+
       - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
         with:
           name: source_distribution
@@ -49,6 +62,7 @@ jobs:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:
           persist-credentials: false
+          submodules: recursive
 
       - uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0
         with:
@@ -86,6 +100,7 @@ jobs:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:
           persist-credentials: false
+          submodules: recursive
 
       - uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0
         with:
@@ -127,6 +142,7 @@ jobs:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:
           persist-credentials: false
+          submodules: recursive
 
       - uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0
         with:
@@ -169,6 +185,7 @@ jobs:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:
           persist-credentials: false
+          submodules: recursive
 
       - uses: pypa/cibuildwheel@298ed2fb2c105540f5ed055e8a6ad78d82dd3a7e # v3.3.1
         with:
@@ -205,6 +222,7 @@ jobs:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:
           persist-credentials: false
+          submodules: recursive
 
       - uses: pypa/cibuildwheel@298ed2fb2c105540f5ed055e8a6ad78d82dd3a7e # v3.3.1
         with:
@@ -241,6 +259,7 @@ jobs:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:
           persist-credentials: false
+          submodules: recursive
 
       - uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3.7.0
         with:
@@ -286,6 +305,7 @@ jobs:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:
           persist-credentials: false
+          submodules: recursive
 
       - uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3.7.0
         with:
@@ -331,6 +351,7 @@ jobs:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:
           persist-credentials: false
+          submodules: recursive
 
       - uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0
         with:
@@ -364,6 +385,7 @@ jobs:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:
           persist-credentials: false
+          submodules: recursive
 
       - uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0
         with:

.github/workflows/ci.yml

@@ -28,6 +28,7 @@ jobs:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:
           persist-credentials: false
+          submodules: recursive
 
       - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
         with:
@@ -58,6 +59,7 @@ jobs:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:
           persist-credentials: false
+          submodules: recursive
 
       - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
         with:
@@ -118,6 +120,7 @@ jobs:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:
           persist-credentials: false
+          submodules: recursive
 
       - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
         with:
@@ -207,6 +210,7 @@ jobs:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:
           persist-credentials: false
+          submodules: recursive
 
       - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
         with:
@@ -217,6 +221,18 @@ jobs:
           pipx run build --sdist --wheel . --outdir dist/
           pipx run twine check dist/* --strict
 
+      - name: Check sdist size, fail if it exceeds 10 MiB
+        run: |
+          MAX_SIZE_MB=10
+          SDIST=$(ls dist/*.tar.gz)
+          SIZE=$(stat --format=%s "$SDIST")
+          SIZE_MB=$(awk "BEGIN {printf \"%.2f\", $SIZE / 1048576}")
+          echo "sdist size: ${SIZE_MB} MB"
+          if [ "$SIZE" -gt $((MAX_SIZE_MB * 1048576)) ]; then
+            echo "::error::sdist is ${SIZE_MB} MB, exceeding the ${MAX_SIZE_MB} MB limit"
+            exit 1
+          fi
+
       - name: Build, inspect, and display contents of distributions
         shell: bash
         run: |

.github/workflows/update-hugo.yml

@@ -19,6 +19,7 @@ jobs:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:
           persist-credentials: false
+          submodules: recursive
       - name: Check for new Hugo release, and apply updates
         id: check-hugo-release
         env:
@@ -60,7 +61,13 @@ jobs:
           fi
 
           sed -i "s/HUGO_VERSION = \"$CURRENT_VERSION\"/HUGO_VERSION = \"$LATEST_VERSION\"/" setup.py
-          sed -i "s/HUGO_VERSION = \"$CURRENT_VERSION\"/HUGO_VERSION = \"$LATEST_VERSION\"/" hugo/cli.py
+          sed -i "s/HUGO_VERSION = \"$CURRENT_VERSION\"/HUGO_VERSION = \"$LATEST_VERSION\"/" src/hugo/cli.py
+
+          # Update the Hugo submodule to the new version tag
+          cd hugo
+          git fetch origin tag "v$LATEST_VERSION" --no-tags
+          git checkout "v$LATEST_VERSION"
+          cd ..
 
           echo "updated=true" >> "$GITHUB_OUTPUT"
           echo "latest_version=$LATEST_VERSION" >> "$GITHUB_OUTPUT"

.gitignore

@@ -174,10 +174,13 @@ go.work
 .DS_Store
 
 # Binaries
-hugo/binaries/*
+src/hugo/binaries/*
 
 # ruff cache
 .ruff_cache/
 
 # Hugo builder cache
 hugo_cache/
+
+# Generated stamp file for sdist builds
+hugo/.hugo_commit_date

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "hugo"]
+	path = hugo
+	url = https://github.com/gohugoio/hugo.git

MANIFEST.in

@@ -1,6 +1,13 @@
+graft src/hugo
 graft hugo
 
 include LICENSE LICENSE-hugo.txt README.md CODE_OF_CONDUCT.md SECURITY.md pyproject.toml setup.py setup.cfg
-exclude hugo/binaries/hugo-*
+exclude src/hugo/binaries/hugo-*
 
-global-exclude __pycache__ *.py[cod] *.so *.dylib .DS_Store .venv
+# Exclude some files from the Hugo submodule that are too large that they
+# inflate the sdist. They are not needed for building.
+prune hugo/docs
+prune hugo/testscripts
+recursive-exclude hugo testdata/*
+
+global-exclude __pycache__ *.py[cod] *.so *.dylib .DS_Store .venv .git

README.md

@@ -163,10 +163,10 @@ Windows users can use the [Chocolatey package manager](https://chocolatey.org/)
 choco install mingw
 ```
 
-Then, clone the repository and run the build script:
+Then, clone the repository (with submodules) and run the build script:
 
 ```bash
-git clone https://github.com/agriyakhetarpal/hugo-python-distributions@main
+git clone --recurse-submodules https://github.com/agriyakhetarpal/hugo-python-distributions@main
 python -m venv venv
 source venv/bin/activate      # on Unix-based systems
 venv\Scripts\activate.bat     # on Windows

hugo

@@ -51,13 +51,14 @@ Issues = "https://github.com/agriyakhetarpal/hugo-python-distributions/issues"
 Changelog = "https://github.com/agriyakhetarpal/hugo-python-distributions/releases"
 
 [tool.setuptools.packages.find]
+where = ["src"]
 include = ["hugo", "hugo.*"]
 
 [project.scripts]
 hugo = "hugo.cli:__call"
 
 [tool.ruff]
-src = ["hugo"]
+src = ["src"]
 lint.extend-select = [
   "B",        # flake8-bugbear
   "I",        # isort