updates

Author: ndbroadbent

.cigen/workflows/ci/jobs/test.yml

@@ -4,10 +4,6 @@ source_files:
   - "@tests"
   - "@examples"
 
-requires:
-  - fmt
-  - clippy
-
 steps:
   - run:
       name: Run tests

.github/workflows/ci.yml

@@ -43,8 +43,8 @@ jobs:
           cp target/release/cigen .cigen/bin/cigen
       - uses: actions/upload-artifact@v4
         with:
-          path: .cigen/bin/cigen
           name: cigen-bin
+          path: .cigen/bin/cigen
   clippy:
     runs-on: ubuntu-latest
     container:
@@ -64,8 +64,13 @@ jobs:
         if: runner.os == 'Linux'
         run: |
           set -e
-          sudo apt-get update
-          sudo apt-get install -y protobuf-compiler
+          if command -v sudo >/dev/null 2>&1; then
+            sudo apt-get update
+            sudo apt-get install -y protobuf-compiler
+          else
+            apt-get update
+            apt-get install -y protobuf-compiler
+          fi
       - name: Checkout repository
         uses: actions/checkout@v4
       - name: Download cigen binary
@@ -140,8 +145,13 @@ jobs:
         if: runner.os == 'Linux'
         run: |
           set -e
-          sudo apt-get update
-          sudo apt-get install -y protobuf-compiler
+          if command -v sudo >/dev/null 2>&1; then
+            sudo apt-get update
+            sudo apt-get install -y protobuf-compiler
+          else
+            apt-get update
+            apt-get install -y protobuf-compiler
+          fi
       - name: Checkout repository
         uses: actions/checkout@v4
       - name: Download cigen binary
@@ -202,8 +212,6 @@ jobs:
     container:
       image: rust:latest
     needs:
-      - fmt
-      - clippy
       - build_cigen
     steps:
       - name: Prepare Node runtime for actions
@@ -218,8 +226,13 @@ jobs:
         if: runner.os == 'Linux'
         run: |
           set -e
-          sudo apt-get update
-          sudo apt-get install -y protobuf-compiler
+          if command -v sudo >/dev/null 2>&1; then
+            sudo apt-get update
+            sudo apt-get install -y protobuf-compiler
+          else
+            apt-get update
+            apt-get install -y protobuf-compiler
+          fi
       - name: Checkout repository
         uses: actions/checkout@v4
       - name: Download cigen binary

.github/workflows/docs.yml

@@ -35,8 +35,13 @@ jobs:
         if: runner.os == 'Linux'
         run: |
           set -e
-          sudo apt-get update
-          sudo apt-get install -y protobuf-compiler
+          if command -v sudo >/dev/null 2>&1; then
+            sudo apt-get update
+            sudo apt-get install -y protobuf-compiler
+          else
+            apt-get update
+            apt-get install -y protobuf-compiler
+          fi
       - name: Checkout repository
         uses: actions/checkout@v4
       - name: Download cigen binary
@@ -149,8 +154,13 @@ jobs:
         if: runner.os == 'Linux'
         run: |
           set -e
-          sudo apt-get update
-          sudo apt-get install -y protobuf-compiler
+          if command -v sudo >/dev/null 2>&1; then
+            sudo apt-get update
+            sudo apt-get install -y protobuf-compiler
+          else
+            apt-get update
+            apt-get install -y protobuf-compiler
+          fi
       - name: Checkout repository
         uses: actions/checkout@v4
       - name: Download cigen binary
@@ -226,8 +236,13 @@ jobs:
         if: runner.os == 'Linux'
         run: |
           set -e
-          sudo apt-get update
-          sudo apt-get install -y protobuf-compiler
+          if command -v sudo >/dev/null 2>&1; then
+            sudo apt-get update
+            sudo apt-get install -y protobuf-compiler
+          else
+            apt-get update
+            apt-get install -y protobuf-compiler
+          fi
       - name: Checkout repository
         uses: actions/checkout@v4
       - name: Download cigen binary

.github/workflows/release.yml

@@ -33,8 +33,13 @@ jobs:
         if: runner.os == 'Linux'
         run: |
           set -e
-          sudo apt-get update
-          sudo apt-get install -y protobuf-compiler
+          if command -v sudo >/dev/null 2>&1; then
+            sudo apt-get update
+            sudo apt-get install -y protobuf-compiler
+          else
+            apt-get update
+            apt-get install -y protobuf-compiler
+          fi
       - name: Checkout repository
         uses: actions/checkout@v4
         with:
@@ -144,13 +149,18 @@ jobs:
         if: runner.os == 'Linux'
         run: |
           set -e
-          sudo apt-get update
-          sudo apt-get install -y protobuf-compiler
+          if command -v sudo >/dev/null 2>&1; then
+            sudo apt-get update
+            sudo apt-get install -y protobuf-compiler
+          else
+            apt-get update
+            apt-get install -y protobuf-compiler
+          fi
       - name: Checkout repository
         uses: actions/checkout@v4
         with:
-          fetch-depth: 0
           fetch-tags: true
+          fetch-depth: 0
       - name: Download cigen binary
         uses: actions/download-artifact@v4
         with:
@@ -255,8 +265,13 @@ jobs:
         if: runner.os == 'Linux'
         run: |
           set -e
-          sudo apt-get update
-          sudo apt-get install -y protobuf-compiler
+          if command -v sudo >/dev/null 2>&1; then
+            sudo apt-get update
+            sudo apt-get install -y protobuf-compiler
+          else
+            apt-get update
+            apt-get install -y protobuf-compiler
+          fi
       - name: Checkout repository
         uses: actions/checkout@v4
         with:
@@ -350,11 +365,11 @@ jobs:
         if: steps.job_skip_cache.outputs.cache-hit != 'true'
       - uses: softprops/action-gh-release@v2
         with:
-          prerelease: ${{ contains(env.RELEASE_TAG, '-') }}
+          draft: false
           body_path: changelog.md
+          prerelease: ${{ contains(env.RELEASE_TAG, '-') }}
           name: CIGen v${{ env.RELEASE_VERSION }}
           tag_name: ${{ env.RELEASE_TAG }}
-          draft: false
           files: |
             artifacts/**/*.tar.gz
             artifacts/**/*.sha256

.gitignore

@@ -10,6 +10,7 @@ integration_tests/**/build/
 
 /.circleci/
 /build/
+tmp/
 
 # Nx
 .nx/

CLAUDE.md

@@ -97,7 +97,7 @@ Skipping or reordering these steps is not acceptable—they keep our feedback lo
 
 ## Project Overview
 
-`cigen` is a Rust CLI tool that generates CI pipeline configurations from templates. It integrates with Nx monorepo tooling and supports multiple CI providers starting with CircleCI.
+`cigen` is a Rust CLI tool that generates CI pipeline configurations from templates. It is migrating to a plugin-based architecture that currently targets GitHub Actions for our self-hosting work, with CircleCI support being rebuilt on top of the same system.
 
 See `PRD.txt` for the complete product requirements and specifications.
 
@@ -352,7 +352,7 @@ You can also reference `/Users/ndbroadbent/code/docspring/lib/tools/generate_cir
 
 ## Use Our Own Tool
 
-The goal is to eventually become 'self-hosting' for our own CI pipeline on GitHub Actions. We must have `nx.json` and `project.json` file in the root of the repository, a `.cigen/` directory, and a `.cigen/config.yml` file.
+The goal is to eventually become 'self-hosting' for our own CI pipeline on GitHub Actions. We already keep `turbo.json` and other workspace metadata in the repo alongside `.cigen/` so that the CLI can evolve toward Turborepo-aware features.
 
 We will start by hand-writing our own GitHub Actions workflow files, but eventually migrate to using `cigen` to generate our CI configuration.
 
@@ -395,6 +395,6 @@ We MUST use split config for large configuration sections:
 
 ## Key Concepts
 
-- The tool reads Nx `project.json` files to understand project dependencies and file groups
+- The tool no longer relies on the old workspace-specific metadata that was tied to previous experiments. Future monorepo integrations will be driven by the plugin system (e.g., Turborepo) once the orchestration layer is complete.
 - Templates and configuration live in the `.cigen/` directory
 - The tool supports plugin-based cache backends and CI provider emitters

Cargo.toml

@@ -25,7 +25,7 @@ description = "A CLI tool that generates CI pipeline configurations from templat
 license = "MIT"
 repository = "https://github.com/DocSpring/cigen"
 readme = "README.md"
-keywords = ["cli", "ci", "nx", "circleci", "devops"]
+keywords = ["cli", "ci", "github-actions", "circleci", "devops"]
 categories = ["command-line-utilities", "development-tools"]
 
 [dependencies]

README.md

@@ -1,28 +1,25 @@
 # CIGen
 
-CIGen generates CI configuration from a small set of reusable, provider-agnostic files. It currently targets CircleCI and focuses on correctness, validation, and convention-over-configuration.
+CIGen generates CI configuration from reusable, provider-agnostic files. The project is undergoing a major migration to a Terraform-style plugin architecture: the new Rust core spawns provider binaries over a length-prefixed stdio transport, and those plugins emit provider-specific YAML.
 
-The tool can generate both static CircleCI configs and setup/dynamic configs. The CLI includes utilities for validation, graph visualization, and template-based multi-output generation.
+Today the GitHub Actions provider plugin powers our own CI (`cargo run -- generate --file .cigen`), while the legacy CircleCI emitter is being rebuilt on top of the same plugin system. The CLI still provides the templating, schema validation, and hashing utilities that powered the earlier monolithic implementation.
 
 ## Features (Current)
 
-- CircleCI provider support (config generation and validation)
-- Rust implementation for reliability and performance
-- Template-based multi-output generation (Jinja-like via MiniJinja)
-- Built-in defaults for package/version detection (extensible via YAML)
-- Package installation step generation and job deduplication
-- Basic automatic cache step injection when `job.cache` is declared
-- Architecture variants per job (e.g., `build_amd64`, `build_arm64`)
-- Advanced git checkout: shallow clone by default with configurable clone/fetch options, host key scanning, and path overrides
-- Descriptive error messages and schema/data validation ([miette], JSON Schema)
-- Opt-in Docker builds with a single BASE_HASH and image DAG
+- Plugin manager (`src/plugin/`) with framing helpers, handshake workflow, and async request batching
+- GitHub Actions provider plugin (`plugins/provider-github`) that generates the workflows under `.github/workflows/`
+- Source file grouping and job-hash based skipping backed by `actions/cache`
+- Template-based multi-output generation (MiniJinja) for additional artefacts
+- Rich schema validation using JSON Schema + miette diagnostics
+- Command-line utilities: `cigen generate`, `cigen hash`, and split-config loader (`.cigen/`)
 
 ## Not Yet Implemented / In Progress
 
-- GitHub Actions and other providers
-- Persistent job-status cache for skipping jobs across runs
-- OR-dependencies and approval-status patching via APIs
-- Self-hosted runner-specific optimizations and resource-class mapping
+- Reintroducing CircleCI (and additional providers) as standalone plugins
+- Detect/plan phases that share facts across plugins (currently stubbed)
+- Persistent work-signature storage feeding a real preflight hook
+- Registry/lockfile support for third-party plugins
+- Turborepo-aware project graph ingestion (replacing the previous workspace-specific focus)
 
 ## Why did we build this?
 
@@ -36,9 +33,9 @@ We had built our own internal CI config generation system in Ruby, but it had st
 
 `cigen` simplifies CI/CD configuration management by:
 
-- Generating CI pipeline configurations from reusable templates
+- Generating pipelines from reusable templates and structured config (`.cigen/` split config)
 - Validating configuration (schema + data-level) with helpful spans
-- Emitting native CircleCI 2.1 YAML, including workflows, jobs, and commands
+- Emitting provider-native YAML through pluggable emitters (GitHub Actions today, CircleCI next)
 
 ## Philosophy
 
@@ -50,17 +47,13 @@ We automatically add a highly-optimized git checkout step to the beginning of ea
 
 #### Job skipping
 
-The codebase contains an initial scaffold for job skipping using a source-file hash and a local marker file. This is experimental and not yet wired to a persistent cache backend, so it does not skip across separate runs. A robust, persistent job-status cache is planned.
+Jobs that declare `source_files` automatically receive a skip cache flow. On GitHub Actions this uses `cigen hash` plus `actions/cache`; jobs exit early when nothing changed and record a marker when they succeed. The preflight hook and remote signature storage are still planned so that other providers can share the same mechanism.
 
 #### Cross-platform CI config
 
 CI providers often solve the same problem in different ways. e.g. to avoid duplication in your config, GitHub actions has "reusable workflows" while CircleCI supports "commands".
 
-`cigen` takes the best ideas from each provider and supports our own set of core features. You write your config once, then we compile it to use your chosen CI provider's native features. This avoids vendor lock-in and makes it easier to migrate to other CI providers.
-
-#### You can still use native CI provider features
-
-You can still use native CircleCI features (e.g., orbs). Other providers will come later; unsupported features are validated with clear errors.
+`cigen` takes the best ideas from each provider and supports our own set of core features. You write your config once, then we compile it to use your chosen CI provider's native features. As more providers move onto the plugin system, migrations become a matter of switching emitters instead of rewriting YAML.
 
 ---
 

examples/README.md

@@ -40,7 +40,7 @@ Perfect if you have:
 
 Perfect if you have:
 
-- Nx or Turborepo setup
+- Turborepo-style or large pnpm workspace
 - 10+ apps/libraries
 - Need selective job execution
 - Want 10x faster CI
@@ -77,7 +77,6 @@ Perfect if you want to:
 | Skip logic           | ✅ (auto) | ✅        | ✅ (advanced) | ✅             |
 | Docker builds        | ❌        | ✅        | ❌            | ✅             |
 | Deployments          | ❌        | ✅        | ✅            | ❌             |
-| Nx integration       | ❌        | ❌        | ✅            | ❌             |
 | Multiple providers   | ❌        | ❌        | ❌            | ✅             |
 
 ## Feature Showcase