stage-graph.json

{
  "version": "2025-09-28",
  "pipelines": [
    {
      "id": "full-refresh-parallel",
      "description": "Full catalogue refresh using the parallel worker pool.",
      "stages": [
        "collect-metadata",
        "parallel-processing",
        "aggregate-catalogue",
        "generate-result-markdown"
      ]
    }
  ],
  "artifacts": [
    {
      "id": "github-api-cache",
      "type": "file",
      "path": "website/data/gitHubData.json",
      "format": "json",
      "description": "Cached repository metadata fetched from GitHub/GitLab/Bitbucket/Codeberg APIs."
    },
    {
      "id": "cloned-modules",
      "type": "directory",
      "path": "modules/",
      "description": "Working tree containing shallow clones of each module repository."
    },
    {
      "id": "modules-temp",
      "type": "directory",
      "path": "modules_temp/",
      "description": "Cache of module clones from the previous run; recycled by the pipeline to avoid full re-clones."
    },
    {
      "id": "images-catalogue",
      "type": "directory",
      "path": "website/images/",
      "description": "Resized module screenshots exported for the public catalogue."
    },
    {
      "id": "modules-json",
      "type": "file",
      "path": "website/data/modules.json",
      "format": "json",
      "description": "Final module catalogue powering downstream consumers.",
      "schema": "dist/schemas/modules.final.schema.json"
    },
    {
      "id": "modules-json-min",
      "type": "file",
      "path": "website/data/modules.min.json",
      "format": "json",
      "description": "Minified variant of the module catalogue.",
      "schema": "dist/schemas/modules.min.schema.json"
    },
    {
      "id": "stats-json",
      "type": "file",
      "path": "website/data/stats.json",
      "format": "json",
      "description": "Aggregated metrics collected during analysis.",
      "schema": "dist/schemas/stats.schema.json"
    },
    {
      "id": "skipped-modules-json",
      "type": "file",
      "path": "website/data/skipped_modules.json",
      "format": "json",
      "description": "Modules that failed processing (e.g. inaccessible repositories), displayed separately on the website."
    },
    {
      "id": "result-markdown",
      "type": "file",
      "path": "website/result.md",
      "format": "markdown",
      "description": "Human-readable issue breakdown generated from the latest in-memory analysis payload and published stats."
    }
  ],
  "stages": [
    {
      "id": "collect-metadata",
      "name": "Collect Metadata",
      "description": "Unified stage: Fetch module list from Wiki and enrich with repository metadata.",
      "command": {
        "executable": "node",
        "args": ["scripts/collect-metadata/index.ts"]
      },
      "dependsOn": [],
      "inputs": [
        {
          "type": "external",
          "kind": "http",
          "uri": "https://raw.githubusercontent.com/wiki/MagicMirrorOrg/MagicMirror/3rd-Party-Modules.md",
          "description": "Authoritative list of community modules maintained in the MagicMirror wiki."
        },
        {
          "artifact": "github-api-cache",
          "mode": "read",
          "optional": true
        }
      ],
      "outputs": [
        {
          "artifact": "github-api-cache",
          "mode": "write"
        }
      ],
      "sideEffects": [
        {
          "type": "network",
          "description": "Fetches Wiki and makes API calls to GitHub/GitLab/etc."
        }
      ],
      "environment": [
        {
          "name": "GITHUB_TOKEN",
          "required": false,
          "description": "Personal access token used to increase API rate limits."
        }
      ]
    },
    {
      "id": "parallel-processing",
      "name": "Parallel module processing",
      "description": "Clone, enrich, and analyze modules in parallel using the worker pool.",
      "command": {
        "executable": "node",
        "args": ["scripts/parallel-processing.ts"]
      },
      "dependsOn": ["collect-metadata"],
      "inputs": [],
      "outputs": [
        {
          "artifact": "cloned-modules",
          "mode": "write"
        },
        {
          "artifact": "images-catalogue",
          "mode": "write"
        },
        {
          "artifact": "skipped-modules-json",
          "mode": "write"
        }
      ],
      "sideEffects": [
        {
          "type": "filesystem",
          "description": "Creates/updates module clones in modules/ and modules_temp/."
        },
        {
          "type": "filesystem",
          "description": "Generates resized module screenshots in website/images/."
        },
        {
          "type": "process",
          "description": "Spawns worker processes for parallel module processing."
        },
        {
          "type": "network",
          "description": "Git operations and npm commands may access external resources."
        }
      ]
    },
    {
      "id": "aggregate-catalogue",
      "name": "Aggregate catalogue outputs",
      "description": "Build final published catalogue outputs from the in-memory analysis payload.",
      "command": {
        "executable": "node",
        "args": ["scripts/aggregate-catalogue.ts"]
      },
      "dependsOn": ["parallel-processing"],
      "inputs": [],
      "outputs": [
        {
          "artifact": "modules-json",
          "mode": "write"
        },
        {
          "artifact": "modules-json-min",
          "mode": "write"
        },
        {
          "artifact": "stats-json",
          "mode": "write"
        }
      ],
      "sideEffects": [
        {
          "type": "filesystem",
          "description": "Writes final published catalogue and stats JSON artifacts."
        }
      ]
    },
    {
      "id": "generate-result-markdown",
      "name": "Generate result markdown",
      "description": "Render website/result.md from the in-memory analysis payload and published stats.",
      "command": {
        "executable": "node",
        "args": ["scripts/generate-result-markdown.ts"]
      },
      "dependsOn": ["aggregate-catalogue"],
      "inputs": [
        {
          "artifact": "stats-json",
          "mode": "read"
        }
      ],
      "outputs": [
        {
          "artifact": "result-markdown",
          "mode": "write"
        }
      ],
      "sideEffects": [
        {
          "type": "filesystem",
          "description": "Writes the published result markdown page for module issue review."
        }
      ]
    }
  ]
}