diff --git a/.github/skills/micro-environment-simulator/simulator.js b/.github/skills/micro-environment-simulator/simulator.js index 0b472ded..e06cd1aa 100644 --- a/.github/skills/micro-environment-simulator/simulator.js +++ b/.github/skills/micro-environment-simulator/simulator.js @@ -26,6 +26,13 @@ const COMMAND_LINE_TERMINAL_WEIGHT = 0.03; const TERMINAL_DEMAND_CAP = 0.95; const UI_ALTERNATIVE_TERMINAL_DISCOUNT = 0.1; const UI_ALTERNATIVE_TERMINAL_DISCOUNT_CAP = 0.3; +const AGENT_ADJUSTMENT_LIMIT = 0.15; +const SEMANTIC_SCORE_WEIGHTS = { + stateReadiness: 0.5, + pathClarity: 0.3, + recoverySupport: 0.2 +}; +const SEMANTIC_SCORE_PROBABILITY_SCALE = 0.002; function clamp(value, min, max) { return Math.max(min, Math.min(max, value)); @@ -322,6 +329,54 @@ function normalizeNumericObject(container, fieldName, stepId) { } } +function normalizeBoundedAdjustments(container, fieldName, stepId, allowedKeys) { + normalizeNumericObject(container, fieldName, stepId); + if (!isPlainObject(container[fieldName])) { + return; + } + const allowedKeySet = new Set(allowedKeys); + for (const key of Object.keys(container[fieldName])) { + if (!allowedKeySet.has(key)) { + console.warn(`[simulator] Ignoring unknown adjustment '${stepId}.${fieldName}.${key}'.`); + delete container[fieldName][key]; + continue; + } + container[fieldName][key] = clamp( + container[fieldName][key], + -AGENT_ADJUSTMENT_LIMIT, + AGENT_ADJUSTMENT_LIMIT + ); + } +} + +function normalizeSemanticScores(container, stepId) { + if (container.semanticScores == null) { + return; + } + if (!isPlainObject(container.semanticScores)) { + console.warn(`[simulator] Agent insight '${stepId}.semanticScores' should be an object.`); + delete container.semanticScores; + return; + } + const scores = {}; + let valid = true; + for (const field of Object.keys(SEMANTIC_SCORE_WEIGHTS)) { + const rawValue = container.semanticScores[field]; + const value = Number(rawValue); + if (rawValue == null || !Number.isFinite(value)) { + console.warn(`[simulator] Agent insight '${stepId}.semanticScores.${field}' should be numeric.`); + valid = false; + continue; + } + scores[field] = clamp(value, 0, 100); + } + if (valid) { + container.semanticScores = scores; + } else { + delete container.semanticScores; + } +} + function normalizeAgentInsightsByStep(input) { if (!isPlainObject(input)) { return {}; @@ -341,9 +396,30 @@ function normalizeAgentInsightsByStep(input) { console.warn(`[simulator] Agent insight '${stepId}.summary' should be a string.`); delete nextInsight.summary; } + normalizeNumericField(nextInsight, "rank", stepId); normalizeNumericField(nextInsight, "bias", stepId); - normalizeNumericObject(nextInsight, "signalAdjustments", stepId); - normalizeNumericObject(nextInsight, "pathAdjustments", stepId); + if (nextInsight.bias != null) { + nextInsight.bias = clamp(nextInsight.bias, -AGENT_ADJUSTMENT_LIMIT, AGENT_ADJUSTMENT_LIMIT); + } + normalizeSemanticScores(nextInsight, stepId); + normalizeBoundedAdjustments(nextInsight, "signalAdjustments", stepId, [ + "complexity", + "terminalDemand", + "browserSupport", + "authDemand", + "troubleshootingSupport", + "conceptDemand", + "enterpriseDemand" + ]); + normalizeBoundedAdjustments(nextInsight, "pathAdjustments", stepId, [ + "browser", + "cli", + "codespaces", + "local", + "mobile", + "uiPreferred", + "enterprise" + ]); normalizeNumericField(nextInsight, "evaluatedContentHash", stepId); if (nextInsight.evaluations !== null && nextInsight.evaluations !== undefined) { if (!isPlainObject(nextInsight.evaluations)) { @@ -418,15 +494,19 @@ function buildStepContentById({ markdown, fileContents.map(({ file, title }) => ({ file, title })) ); - const agentInsight = agentInsightsByStep[stepId] ? { ...agentInsightsByStep[stepId] } : null; + const rawAgentInsight = agentInsightsByStep[stepId] || null; + let agentInsight = rawAgentInsight; + // Every model-affecting insight is tied to the exact page revision it evaluated. if ( - agentInsight?.evaluations && - Number(agentInsight.evaluatedContentHash) !== contentAnalysis.contentHash + rawAgentInsight && + Number(rawAgentInsight.evaluatedContentHash) !== contentAnalysis.contentHash ) { console.warn( - `[simulator] Ignoring stale agent evaluations for step '${stepId}': content hash does not match.` + `[simulator] Ignoring stale agent insight for step '${stepId}': content hash does not match.` ); - delete agentInsight.evaluations; + agentInsight = null; + } else if (rawAgentInsight) { + agentInsight = { ...rawAgentInsight }; } stepContentById[stepId] = deepFreeze({ ...contentAnalysis, @@ -993,6 +1073,8 @@ const exportedApi = { VALID_TERMINALS, INFERENCE_PROVIDERS, PROVIDER_SECRET_BY_NAME, + SEMANTIC_SCORE_WEIGHTS, + SEMANTIC_SCORE_PROBABILITY_SCALE, ensure, clamp, stableHash, diff --git a/.github/skills/micro-environment-simulator/simulator.test.js b/.github/skills/micro-environment-simulator/simulator.test.js index 781d166e..f4ce5360 100644 --- a/.github/skills/micro-environment-simulator/simulator.test.js +++ b/.github/skills/micro-environment-simulator/simulator.test.js @@ -143,7 +143,72 @@ test("agent assumption evaluations are normalized and stale evaluations are igno repoRoot, agentInsightsByStep: normalized }); - assert.equal(updated.step.agentInsight.evaluations, undefined); + assert.equal(updated.step.agentInsight, null); +}); + +test("agent adjustments are bounded and unknown adjustment keys are ignored", () => { + const normalized = simulator.normalizeAgentInsightsByStep({ + stepInsightsById: { + step: { + evaluatedContentHash: 1, + bias: -4, + semanticScores: { + stateReadiness: -10, + pathClarity: 60, + recoverySupport: 120, + ignored: "not-a-score" + }, + signalAdjustments: { complexity: 3, invented: 1 }, + pathAdjustments: { browser: -3, invented: -1 } + }, + malformed: { + semanticScores: { + stateReadiness: null, + pathClarity: "not-a-score" + } + } + } + }); + + assert.equal(normalized.step.bias, -0.15); + assert.deepEqual(normalized.step.semanticScores, { + stateReadiness: 0, + pathClarity: 60, + recoverySupport: 100 + }); + assert.deepEqual(normalized.step.signalAdjustments, { complexity: 0.15 }); + assert.deepEqual(normalized.step.pathAdjustments, { browser: -0.15 }); + assert.equal(normalized.malformed.semanticScores, undefined); +}); + +test("semantic page scores change the statistical readiness outcome", () => { + const student = { + id: 9, + level: "beginner", + personality: "curious", + background: "no-coding", + goal: "personal-learning", + tool: "cli", + ui_preferred: false + }; + const state = simulator.defaultEnvironmentForStudent(student, 120, 0); + const transition = journey.transitions["04-actions-intro"]; + const context = { + stepId: "04-actions-intro", + random: () => 0.65, + stepContent: { + agentInsight: { + semanticScores: { + stateReadiness: 0, + pathClarity: 0, + recoverySupport: 0 + } + } + } + }; + + assert.equal(transition(state, { ...context, stepContent: {} }).ok, true); + assert.equal(transition(state, context).ok, false); }); function firstWorkflowState(centralizedCopilotBilling) { diff --git a/.github/skills/micro-environment-simulator/workshop-student-journey.js b/.github/skills/micro-environment-simulator/workshop-student-journey.js index 2ee915d6..5b7fbbac 100644 --- a/.github/skills/micro-environment-simulator/workshop-student-journey.js +++ b/.github/skills/micro-environment-simulator/workshop-student-journey.js @@ -1,6 +1,11 @@ "use strict"; -const { VALID_TERMINALS, ensure } = require("./simulator"); +const { + VALID_TERMINALS, + SEMANTIC_SCORE_WEIGHTS, + SEMANTIC_SCORE_PROBABILITY_SCALE, + ensure +} = require("./simulator"); const MODEL_VERSION = "2026-07-survival-model-v2"; @@ -338,6 +343,7 @@ function evaluateStepProbability(state, context, options = {}) { const insight = agentInsight(context); const signalAdjustments = ensurePlainObject(insight.signalAdjustments); const pathAdjustments = ensurePlainObject(insight.pathAdjustments); + const semanticScores = ensurePlainObject(insight.semanticScores); const usingBrowserPath = prefersBrowserPath(state, context); const emphasis = { ...(options.emphasis || {}), @@ -345,6 +351,17 @@ function evaluateStepProbability(state, context, options = {}) { }; let probability = computeSuccessProbability(state, context, emphasis); + const scoredDimensions = Object.entries(SEMANTIC_SCORE_WEIGHTS).filter(([dimension]) => + Number.isFinite(Number(semanticScores[dimension])) + ); + if (scoredDimensions.length === Object.keys(SEMANTIC_SCORE_WEIGHTS).length) { + const contentSupportScore = scoredDimensions.reduce( + (total, [dimension, weight]) => total + Number(semanticScores[dimension]) * weight, + 0 + ); + probability += (contentSupportScore - 50) * SEMANTIC_SCORE_PROBABILITY_SCALE; + } + for (const signal of [ "complexity", "terminalDemand", diff --git a/.github/workflows/workshop-student-simulator.lock.yml b/.github/workflows/workshop-student-simulator.lock.yml index ae5d34c3..f888954a 100644 --- a/.github/workflows/workshop-student-simulator.lock.yml +++ b/.github/workflows/workshop-student-simulator.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"c65739738933eefde1400116413dd4a77cb5beacad573916c961623680331f74","body_hash":"373721c2d75981422f54f13a63b21899f9e230df611f8fe9c91f51656ee4d07c","compiler_version":"v0.82.13","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.70"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"c65739738933eefde1400116413dd4a77cb5beacad573916c961623680331f74","body_hash":"4b7c4ec94232426f0231a2a095c8d6a852c5ad569e0dadcdeea2e1b68bdc0c49","compiler_version":"v0.82.13","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.70"}} # gh-aw-manifest: {"version":1,"secrets":["COPILOT_GITHUB_TOKEN","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache/restore","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/save","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/checkout","sha":"9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0","version":"v7.0.0"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"820762786026740c76f36085b0efc47a31fe5020","version":"v7.0.0"},{"repo":"actions/setup-python","sha":"ece7cb06caefa5fff74198d8649806c4678c61a1","version":"v6.3.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"github/gh-aw-actions/setup","sha":"a5d8b7df9d4f9137bedb3d074d0a0b005f1e1996","version":"v0.82.13"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.27.35","digest":"sha256:2202f63e8650b2b8b0d38033b44a05387b2b71ad3e690c4d23a34786f5462aed","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.27.35@sha256:2202f63e8650b2b8b0d38033b44a05387b2b71ad3e690c4d23a34786f5462aed"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.35","digest":"sha256:755b79d0dfda82bd6b43a208d68666721e504110c5d342a4eeb199802644ff04","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.35@sha256:755b79d0dfda82bd6b43a208d68666721e504110c5d342a4eeb199802644ff04"},{"image":"ghcr.io/github/gh-aw-firewall/cli-proxy:0.27.35","digest":"sha256:fe83cd274636efa9de3f456e2b078fae137328b9bb6ee4986ae510acaef0cec5","pinned_image":"ghcr.io/github/gh-aw-firewall/cli-proxy:0.27.35@sha256:fe83cd274636efa9de3f456e2b078fae137328b9bb6ee4986ae510acaef0cec5"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.27.35","digest":"sha256:f69282ec7b1326ba53891c399cf5b10475c0d3ccf4e1519b33d234a5427b57d3","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.27.35@sha256:f69282ec7b1326ba53891c399cf5b10475c0d3ccf4e1519b33d234a5427b57d3"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.1","digest":"sha256:ad2a979c2cd8b50098e84938ca9c9c1580eb8e91526f101a90adfba7859b2c32","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.1@sha256:ad2a979c2cd8b50098e84938ca9c9c1580eb8e91526f101a90adfba7859b2c32"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:529d02eb970b1161aa25c593a9c3df57fdfad5a8add328cb3b6eccef66f3183b","pinned_image":"ghcr.io/github/gh-aw-node@sha256:529d02eb970b1161aa25c593a9c3df57fdfad5a8add328cb3b6eccef66f3183b"},{"image":"ghcr.io/github/github-mcp-server:v1.6.0","digest":"sha256:2b0c48b070f61e9d3969269ead600f62d00fb237b60ac849ef3d166ee7de9ad3","pinned_image":"ghcr.io/github/github-mcp-server:v1.6.0@sha256:2b0c48b070f61e9d3969269ead600f62d00fb237b60ac849ef3d166ee7de9ad3"}]} # This file was automatically generated by gh-aw (v0.82.13). DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # diff --git a/.github/workflows/workshop-student-simulator.md b/.github/workflows/workshop-student-simulator.md index 5379cdb0..d0ba0552 100644 --- a/.github/workflows/workshop-student-simulator.md +++ b/.github/workflows/workshop-student-simulator.md @@ -256,7 +256,7 @@ The profiles are generated from `.github/skills/micro-environment-simulator/work For each student in the generated cohort, simulate their experience step-by-step using the following rules: -First read the baseline Monte Carlo output that was already written to `/tmp/gh-aw/agent/sim/data/monte-carlo-replay.json` to identify the highest dropout or highest-risk steps. Then read both `curriculum.json` and `curriculum-quality-metrics.json`. For each high-risk step, inspect that step and the preceding activities that produce its required state before writing `/tmp/gh-aw/agent/sim/data/agent-step-insights.json`. For example, inspect the learner's Step 7 authoring path (Terminal, GitHub UI, or GitHub Copilot) and the shared Step 7d model-access activity before adjusting Step 8. +First read the baseline Monte Carlo output that was already written to `/tmp/gh-aw/agent/sim/data/monte-carlo-replay.json`, then read both `curriculum.json` and `curriculum-quality-metrics.json`. Build a candidate set from the five highest-dropout steps and the five lowest-scoring curriculum steps. Semantically score and rank every candidate using the `stateReadiness`, `pathClarity`, and `recoverySupport` dimensions defined below, then select the three highest-risk steps for detailed analysis. Inspect every workshop page mapped to each selected step, plus the preceding activities that produce its required state, before writing `/tmp/gh-aw/agent/sim/data/agent-step-insights.json`. A simulated step may map to several pages; do not treat a file limit as a step limit. For example, inspect the learner's Step 7 authoring path (Terminal, GitHub UI, or GitHub Copilot) and the shared Step 7d model-access activity before adjusting Step 8. Evaluate content assumptions semantically instead of inferring state from keyword counts. Use one focused yes/no question per assumption, answer only `YES`, `NO`, or `UNKNOWN`, and cite `file:line` evidence. Copy each step's `contentHash` from the baseline output into `evaluatedContentHash`. The simulator ignores evaluations when that hash does not match the current mapped page content, so page edits require fresh evaluations. @@ -268,6 +268,12 @@ Use this JSON shape: "stepInsightsById": { "": { "evaluatedContentHash": 123456789, + "rank": 1, + "semanticScores": { + "stateReadiness": 35, + "pathClarity": 45, + "recoverySupport": 30 + }, "evaluations": { "": { "question": "Does the applicable content ensure this state is true before the next step?", @@ -302,6 +308,9 @@ Use this JSON shape: ``` - `schemaVersion: 1` is the initial assumption-evaluation format; increment it when a future change is not backward compatible. +- `rank` orders the selected steps from highest to lowest content risk. +- Score `stateReadiness`, `pathClarity`, and `recoverySupport` from 0 (unsupported) to 100 (fully supported) using evidence from the mapped current pages. The simulator combines these scores at weights 50%, 30%, and 20%, then converts the weighted score to a bounded probability adjustment of -0.10 to +0.10. This semantic score is the required agentic contribution to the hybrid model. +- Include all three semantic scores for each selected step, even when the resulting probability adjustment is neutral. Explain the score in `summary` and cite page evidence in the evaluations. - Always evaluate these Step 7 assumptions because they determine the state required by Step 8: - `workflow_source_created_terminal` - `workflow_compiled_terminal` @@ -319,6 +328,7 @@ Use this JSON shape: - You may add focused evaluations for other high-risk steps. Unknown evaluation IDs remain available as report evidence but cannot directly patch arbitrary simulator state. - Omit steps where you have no meaningful adjustment. - Use positive numbers to increase success probability and negative numbers to decrease it. +- Keep `bias`, each signal adjustment, and each path adjustment between `-0.15` and `0.15`; the simulator clamps larger values and ignores unknown adjustment keys. - Base these adjustments on the actual wording, path structure, fallbacks, and recovery guidance in the files you inspect — not only on the numeric signals already present in the simulator. - For Copilot / Agents-tab paths, verify that the content treats the surface as prompt-driven chat and explicitly calls out `/agentic-workflows` for workflow-authoring tasks. Missing that cue, or presenting shell commands as if they run inside the Agents tab, is an access barrier for `tool: CCA` learners. @@ -335,6 +345,8 @@ node .github/skills/micro-environment-simulator/simulator.js \ --out /tmp/gh-aw/agent/sim/data/monte-carlo-replay.json ``` +After rerunning, verify in the replay output that all three selected steps have a non-null `stepContentById[step].agentInsight`, a matching `evaluatedContentHash`, and all three `semanticScores`. Stop and repair the insights instead of reporting if any selected insight is missing or stale. + The Monte Carlo simulation written to `/tmp/gh-aw/agent/sim/data/monte-carlo-replay.json` should therefore reflect both the simulator's constraint model and your content-aware adjustments: ```json @@ -442,7 +454,7 @@ Update `/tmp/gh-aw/cache-memory/profiles.json`: ### Read workshop files (if available) -If `${{ env.WORKSHOP_STEP_COUNT }}` > 0, use the available tools to read up to 3 of the workshop step files to ground both your `agent-step-insights.json` adjustments and your improvement suggestions in the actual content. Focus on the highest-dropout or highest-risk steps. +If `${{ env.WORKSHOP_STEP_COUNT }}` > 0, use the available tools to read every mapped page for the three semantically ranked high-risk steps, plus prerequisite pages that establish their required state. Ground both `agent-step-insights.json` and improvement suggestions in that current content. ### Create a concise report issue