ci: enable output generation for tutorial

Author: alecbcs

.github/workflows/ci.yml

@@ -50,15 +50,17 @@ jobs:
     needs: changes
     uses: ./.github/workflows/container.yml
 
-  # outputs:
-  #   if: |
-  #     !cancelled()
-  #     && contains(needs.*.result, 'success')
-  #     && !contains(needs.*.result, 'failure')
-  #     && (needs.changes.outputs.outputs == 'true'
-  #     || startsWith(github.ref, 'refs/tags/'))
-  #   needs: [ changes, container ]
-  #   uses: ./.github/workflows/outputs.yml
+  outputs:
+    if: |
+      !cancelled()
+      && contains(needs.*.result, 'success')
+      && !contains(needs.*.result, 'failure')
+      && (needs.changes.outputs.outputs == 'true'
+      || startsWith(github.ref, 'refs/tags/'))
+    needs: [ changes, container ]
+    uses: ./.github/workflows/outputs.yml
+    with:
+      with_pr_container: ${{ needs.container.result == 'success' }}
 
   style:
     if: ${{ needs.changes.outputs.style == 'true' }}

.github/workflows/container.yml

@@ -10,21 +10,6 @@ jobs:
       packages: write
 
     steps:
-      - name: Remove pre-installed apps to make space for build
-        run: |
-          sudo apt-get remove -y '^llvm-.*'
-          sudo apt-get remove -y '^dotnet-.*'
-          sudo apt-get remove -y 'php.*'
-          sudo apt-get remove -y '^mongodb-.*'
-          sudo apt-get remove -y '^mysql-.*'
-          sudo apt-get autoremove -y
-          sudo apt-get clean
-          sudo rm -rf /usr/share/dotnet/
-          sudo rm -rf /usr/local/share/powershell
-          sudo rm -rf /usr/local/share/chromium
-          sudo rm -rf /usr/local/lib/android
-          sudo rm -rf /usr/local/lib/node_modules
-
       - name: Set up QEMU
         uses: docker/setup-qemu-action@49b3bc8e6bdd4a60e6116a5414239cba5943d3cf
 

.github/workflows/outputs.yml

@@ -1,6 +1,10 @@
 name: Generate outputs
 on:
   workflow_call:
+    inputs:
+      with_pr_container:
+        required: true
+        type: boolean
 
 jobs:
   re-generate:
@@ -10,22 +14,6 @@ jobs:
       contents: write
 
     steps:
-    - name: Remove pre-installed apps to make space for build
-      run: |
-        sudo apt-get remove -y '^llvm-.*'
-        sudo apt-get remove -y '^dotnet-.*'
-        sudo apt-get remove -y 'php.*'
-        sudo apt-get remove -y '^mongodb-.*'
-        sudo apt-get remove -y '^mysql-.*'
-        sudo apt-get autoremove -y
-        sudo apt-get clean
-        sudo rm -rf /usr/share/dotnet/
-        sudo rm -rf /usr/local/share/powershell
-        sudo rm -rf /usr/local/share/chromium
-        sudo rm -rf /usr/local/lib/android
-        sudo rm -rf /usr/local/lib/node_modules
-        sudo docker system prune -af
-
     - name: Checkout (Push)
       if: github.event_name != 'pull_request'
       uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
@@ -40,32 +28,34 @@ jobs:
         repository: ${{ github.event.pull_request.head.repo.full_name }}
 
     - name: Download container from artifact if PR
-      if: github.event_name == 'pull_request' && needs.container.result == 'success'
+      if: github.event_name == 'pull_request' && inputs.with_pr_container
       uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16
       with:
         name: tutorial-container
         path: /tmp
 
     - name: Import container from PR artifact
-      if: github.event_name == 'pull_request' && needs.container.result == 'success'
+      if: github.event_name == 'pull_request' && inputs.with_pr_container
       run: |
         docker load --input /tmp/tutorial-container.tar
         docker image ls -a
 
     - name: Re-generate outputs from tutorial scripts
       run: |
         chmod -R a+w outputs
-        make -C outputs -j $(nproc) DOCKER_RUN_OPTS="-e COLUMNS=94 -e COLIFY_SIZE=24x94" container=ghcr.io/spack/tutorial
+        mkdir spack && sudo chown -R 1000:1000 spack
+        DOCKER_RUN_OPTS="-e COLUMNS=94 -e COLIFY_SIZE=24x94 -v $(pwd)/spack:/home/spack/spack"
+        make -C outputs -j $(nproc) DOCKER_RUN_OPTS="$DOCKER_RUN_OPTS" container=ghcr.io/spack/tutorial
 
     - name: Show diff of modified files
       run: |
-        git diff HEAD .
+        git diff HEAD outputs/
 
     - name: Push modified files to branch
       if: ( github.event_name == 'pull_request' && ! github.event.pull_request.head.repo.fork )
       run: |
         git config user.name "github-actions[bot]"
         git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
-        git add .
+        git add outputs/
         git commit -m "Update generated outputs on behalf of [@${{ github.actor }}]"
         git push

container/Dockerfile

@@ -5,19 +5,20 @@ ENV DEBIAN_FRONTEND=noninteractive
 # Install AWS cli
 RUN apt-get update -y && \
     apt-get install -y --no-install-recommends \
-    awscli \
     bash-completion \
+    ca-certificates \
     emacs \
     jq \
     less \
-    vim
+    vim \
+    rclone
 
 # Remove GCC 12 from container
 RUN apt-get remove -y gcc-12
 
 # Download the buildcache
 RUN mkdir /mirror
-RUN aws --region us-east-1 --no-sign-request s3 sync --only-show-errors s3://spack-binaries/releases/v0.23/tutorial /mirror
+RUN rclone copy :s3:spack-binaries/v0.23.1/tutorial /mirror
 
 RUN chmod -R go+r /mirror
 

outputs/Makefile

@@ -25,7 +25,7 @@ sections := environments
 run_targets := $(addprefix run-,$(sections))
 
 # Entrypoint
-update-outputs: #run
+update-outputs: run
 	echo "Filtering raw outputs though col"
 	for raw in raw/*/*.out; do \
 		out=$$(echo $$raw | sed 's.raw/..'); \
@@ -34,17 +34,17 @@ update-outputs: #run
 
 run: run-scripting
 
-run-scripting: run-cache scripting.sh
-run-cache: run-dev cache.sh
+run-scripting: run-dev scripting.sh
+# run-cache: run-dev cache.sh
 run-dev: run-packaging dev.sh
 run-packaging: run-stacks packaging.sh
 run-stacks: run-environments stacks.sh
 run-environments: run-basics environments.sh
 run-basics: basics.sh init_spack.sh defs.sh
 
 local: local-scripting
-local-scripting: local-cache scripting.sh
-local-cache: local-dev cache.sh
+local-scripting: local-dev scripting.sh
+# local-cache: local-dev cache.sh
 local-dev: local-packaging dev.sh
 local-packaging: local-stacks packaging.sh
 local-stacks: local-environments stacks.sh

outputs/packaging.sh

@@ -20,8 +20,8 @@ example packaging/checksum-mpileaks-1  "spack checksum tutorial-mpileaks 1.0"
 
 example --expect-error packaging/install-mpileaks-1  "spack install tutorial-mpileaks"
 
-cp "$PROJECT/package-py-files/1.package.py" "$mpileaks_package_py"
-example packaging/info-mpileaks       "spack info tutorial-mpileaks"
+#cp "$PROJECT/package-py-files/1.package.py" "$mpileaks_package_py"
+#example packaging/info-mpileaks       "spack info tutorial-mpileaks"
 
 cp "$PROJECT/package-py-files/2.package.py" "$mpileaks_package_py"
 example --expect-error packaging/install-mpileaks-2  "spack install tutorial-mpileaks"