Make student simulation content-aware and hybrid

.github/skills/micro-environment-simulator/simulator.js

@@ -26,6 +26,13 @@ const COMMAND_LINE_TERMINAL_WEIGHT = 0.03;
 const TERMINAL_DEMAND_CAP = 0.95;
 const UI_ALTERNATIVE_TERMINAL_DISCOUNT = 0.1;
 const UI_ALTERNATIVE_TERMINAL_DISCOUNT_CAP = 0.3;
+const AGENT_ADJUSTMENT_LIMIT = 0.15;
+const SEMANTIC_SCORE_WEIGHTS = {
+  stateReadiness: 0.5,
+  pathClarity: 0.3,
+  recoverySupport: 0.2
+};
+const SEMANTIC_SCORE_PROBABILITY_SCALE = 0.002;
 
 function clamp(value, min, max) {
   return Math.max(min, Math.min(max, value));
@@ -322,6 +329,54 @@ function normalizeNumericObject(container, fieldName, stepId) {
   }
 }
 
+function normalizeBoundedAdjustments(container, fieldName, stepId, allowedKeys) {
+  normalizeNumericObject(container, fieldName, stepId);
+  if (!isPlainObject(container[fieldName])) {
+    return;
+  }
+  const allowedKeySet = new Set(allowedKeys);
+  for (const key of Object.keys(container[fieldName])) {
+    if (!allowedKeySet.has(key)) {
+      console.warn(`[simulator] Ignoring unknown adjustment '${stepId}.${fieldName}.${key}'.`);
+      delete container[fieldName][key];
+      continue;
+    }
+    container[fieldName][key] = clamp(
+      container[fieldName][key],
+      -AGENT_ADJUSTMENT_LIMIT,
+      AGENT_ADJUSTMENT_LIMIT
+    );
+  }
+}
+
+function normalizeSemanticScores(container, stepId) {
+  if (container.semanticScores == null) {
+    return;
+  }
+  if (!isPlainObject(container.semanticScores)) {
+    console.warn(`[simulator] Agent insight '${stepId}.semanticScores' should be an object.`);
+    delete container.semanticScores;
+    return;
+  }
+  const scores = {};
+  let valid = true;
+  for (const field of Object.keys(SEMANTIC_SCORE_WEIGHTS)) {
+    const rawValue = container.semanticScores[field];
+    const value = Number(rawValue);
+    if (rawValue == null || !Number.isFinite(value)) {
+      console.warn(`[simulator] Agent insight '${stepId}.semanticScores.${field}' should be numeric.`);
+      valid = false;
+      continue;
+    }
+    scores[field] = clamp(value, 0, 100);
+  }
+  if (valid) {
+    container.semanticScores = scores;
+  } else {
+    delete container.semanticScores;
+  }
+}
+
 function normalizeAgentInsightsByStep(input) {
   if (!isPlainObject(input)) {
     return {};
@@ -341,9 +396,30 @@ function normalizeAgentInsightsByStep(input) {
       console.warn(`[simulator] Agent insight '${stepId}.summary' should be a string.`);
       delete nextInsight.summary;
     }
+    normalizeNumericField(nextInsight, "rank", stepId);
     normalizeNumericField(nextInsight, "bias", stepId);
-    normalizeNumericObject(nextInsight, "signalAdjustments", stepId);
-    normalizeNumericObject(nextInsight, "pathAdjustments", stepId);
+    if (nextInsight.bias != null) {
+      nextInsight.bias = clamp(nextInsight.bias, -AGENT_ADJUSTMENT_LIMIT, AGENT_ADJUSTMENT_LIMIT);
+    }
+    normalizeSemanticScores(nextInsight, stepId);
+    normalizeBoundedAdjustments(nextInsight, "signalAdjustments", stepId, [
+      "complexity",
+      "terminalDemand",
+      "browserSupport",
+      "authDemand",
+      "troubleshootingSupport",
+      "conceptDemand",
+      "enterpriseDemand"
+    ]);
+    normalizeBoundedAdjustments(nextInsight, "pathAdjustments", stepId, [
+      "browser",
+      "cli",
+      "codespaces",
+      "local",
+      "mobile",
+      "uiPreferred",
+      "enterprise"
+    ]);
     normalizeNumericField(nextInsight, "evaluatedContentHash", stepId);
     if (nextInsight.evaluations !== null && nextInsight.evaluations !== undefined) {
       if (!isPlainObject(nextInsight.evaluations)) {
@@ -418,15 +494,19 @@ function buildStepContentById({
       markdown,
       fileContents.map(({ file, title }) => ({ file, title }))
     );
-    const agentInsight = agentInsightsByStep[stepId] ? { ...agentInsightsByStep[stepId] } : null;
+    const rawAgentInsight = agentInsightsByStep[stepId] || null;
+    let agentInsight = rawAgentInsight;
+    // Every model-affecting insight is tied to the exact page revision it evaluated.
     if (
-      agentInsight?.evaluations &&
-      Number(agentInsight.evaluatedContentHash) !== contentAnalysis.contentHash
+      rawAgentInsight &&
+      Number(rawAgentInsight.evaluatedContentHash) !== contentAnalysis.contentHash
     ) {
       console.warn(
-        `[simulator] Ignoring stale agent evaluations for step '${stepId}': content hash does not match.`
+        `[simulator] Ignoring stale agent insight for step '${stepId}': content hash does not match.`
       );
-      delete agentInsight.evaluations;
+      agentInsight = null;
+    } else if (rawAgentInsight) {
+      agentInsight = { ...rawAgentInsight };
     }
     stepContentById[stepId] = deepFreeze({
       ...contentAnalysis,
@@ -993,6 +1073,8 @@ const exportedApi = {
   VALID_TERMINALS,
   INFERENCE_PROVIDERS,
   PROVIDER_SECRET_BY_NAME,
+  SEMANTIC_SCORE_WEIGHTS,
+  SEMANTIC_SCORE_PROBABILITY_SCALE,
   ensure,
   clamp,
   stableHash,

.github/skills/micro-environment-simulator/simulator.test.js

@@ -143,7 +143,72 @@ test("agent assumption evaluations are normalized and stale evaluations are igno
     repoRoot,
     agentInsightsByStep: normalized
   });
-  assert.equal(updated.step.agentInsight.evaluations, undefined);
+  assert.equal(updated.step.agentInsight, null);
+});
+
+test("agent adjustments are bounded and unknown adjustment keys are ignored", () => {
+  const normalized = simulator.normalizeAgentInsightsByStep({
+    stepInsightsById: {
+      step: {
+        evaluatedContentHash: 1,
+        bias: -4,
+        semanticScores: {
+          stateReadiness: -10,
+          pathClarity: 60,
+          recoverySupport: 120,
+          ignored: "not-a-score"
+        },
+        signalAdjustments: { complexity: 3, invented: 1 },
+        pathAdjustments: { browser: -3, invented: -1 }
+      },
+      malformed: {
+        semanticScores: {
+          stateReadiness: null,
+          pathClarity: "not-a-score"
+        }
+      }
+    }
+  });
+
+  assert.equal(normalized.step.bias, -0.15);
+  assert.deepEqual(normalized.step.semanticScores, {
+    stateReadiness: 0,
+    pathClarity: 60,
+    recoverySupport: 100
+  });
+  assert.deepEqual(normalized.step.signalAdjustments, { complexity: 0.15 });
+  assert.deepEqual(normalized.step.pathAdjustments, { browser: -0.15 });
+  assert.equal(normalized.malformed.semanticScores, undefined);
+});
+
+test("semantic page scores change the statistical readiness outcome", () => {
+  const student = {
+    id: 9,
+    level: "beginner",
+    personality: "curious",
+    background: "no-coding",
+    goal: "personal-learning",
+    tool: "cli",
+    ui_preferred: false
+  };
+  const state = simulator.defaultEnvironmentForStudent(student, 120, 0);
+  const transition = journey.transitions["04-actions-intro"];
+  const context = {
+    stepId: "04-actions-intro",
+    random: () => 0.65,
+    stepContent: {
+      agentInsight: {
+        semanticScores: {
+          stateReadiness: 0,
+          pathClarity: 0,
+          recoverySupport: 0
+        }
+      }
+    }
+  };
+
+  assert.equal(transition(state, { ...context, stepContent: {} }).ok, true);
+  assert.equal(transition(state, context).ok, false);
 });
 
 function firstWorkflowState(centralizedCopilotBilling) {

.github/skills/micro-environment-simulator/workshop-student-journey.js

@@ -1,6 +1,11 @@
 "use strict";
 
-const { VALID_TERMINALS, ensure } = require("./simulator");
+const {
+  VALID_TERMINALS,
+  SEMANTIC_SCORE_WEIGHTS,
+  SEMANTIC_SCORE_PROBABILITY_SCALE,
+  ensure
+} = require("./simulator");
 
 const MODEL_VERSION = "2026-07-survival-model-v2";
 
@@ -338,13 +343,25 @@ function evaluateStepProbability(state, context, options = {}) {
   const insight = agentInsight(context);
   const signalAdjustments = ensurePlainObject(insight.signalAdjustments);
   const pathAdjustments = ensurePlainObject(insight.pathAdjustments);
+  const semanticScores = ensurePlainObject(insight.semanticScores);
   const usingBrowserPath = prefersBrowserPath(state, context);
   const emphasis = {
     ...(options.emphasis || {}),
     bias: (options.emphasis?.bias || 0) + Number(insight.bias || 0)
   };
   let probability = computeSuccessProbability(state, context, emphasis);
 
+  const scoredDimensions = Object.entries(SEMANTIC_SCORE_WEIGHTS).filter(([dimension]) =>
+    Number.isFinite(Number(semanticScores[dimension]))
+  );
+  if (scoredDimensions.length === Object.keys(SEMANTIC_SCORE_WEIGHTS).length) {
+    const contentSupportScore = scoredDimensions.reduce(
+      (total, [dimension, weight]) => total + Number(semanticScores[dimension]) * weight,
+      0
+    );
+    probability += (contentSupportScore - 50) * SEMANTIC_SCORE_PROBABILITY_SCALE;
+  }
+
   for (const signal of [
     "complexity",
     "terminalDemand",

.github/workflows/workshop-student-simulator.md

@@ -256,7 +256,7 @@ The profiles are generated from `.github/skills/micro-environment-simulator/work
 
 For each student in the generated cohort, simulate their experience step-by-step using the following rules:
 
-First read the baseline Monte Carlo output that was already written to `/tmp/gh-aw/agent/sim/data/monte-carlo-replay.json` to identify the highest dropout or highest-risk steps. Then read both `curriculum.json` and `curriculum-quality-metrics.json`. For each high-risk step, inspect that step and the preceding activities that produce its required state before writing `/tmp/gh-aw/agent/sim/data/agent-step-insights.json`. For example, inspect the learner's Step 7 authoring path (Terminal, GitHub UI, or GitHub Copilot) and the shared Step 7d model-access activity before adjusting Step 8.
+First read the baseline Monte Carlo output that was already written to `/tmp/gh-aw/agent/sim/data/monte-carlo-replay.json`, then read both `curriculum.json` and `curriculum-quality-metrics.json`. Build a candidate set from the five highest-dropout steps and the five lowest-scoring curriculum steps. Semantically score and rank every candidate using the `stateReadiness`, `pathClarity`, and `recoverySupport` dimensions defined below, then select the three highest-risk steps for detailed analysis. Inspect every workshop page mapped to each selected step, plus the preceding activities that produce its required state, before writing `/tmp/gh-aw/agent/sim/data/agent-step-insights.json`. A simulated step may map to several pages; do not treat a file limit as a step limit. For example, inspect the learner's Step 7 authoring path (Terminal, GitHub UI, or GitHub Copilot) and the shared Step 7d model-access activity before adjusting Step 8.
 
 Evaluate content assumptions semantically instead of inferring state from keyword counts. Use one focused yes/no question per assumption, answer only `YES`, `NO`, or `UNKNOWN`, and cite `file:line` evidence. Copy each step's `contentHash` from the baseline output into `evaluatedContentHash`. The simulator ignores evaluations when that hash does not match the current mapped page content, so page edits require fresh evaluations.
 
@@ -268,6 +268,12 @@ Use this JSON shape:
   "stepInsightsById": {
     "<step-id>": {
       "evaluatedContentHash": 123456789,
+      "rank": 1,
+      "semanticScores": {
+        "stateReadiness": 35,
+        "pathClarity": 45,
+        "recoverySupport": 30
+      },
       "evaluations": {
         "<assumption-id>": {
           "question": "Does the applicable content ensure this state is true before the next step?",
@@ -302,6 +308,9 @@ Use this JSON shape:
 ```
 
 - `schemaVersion: 1` is the initial assumption-evaluation format; increment it when a future change is not backward compatible.
+- `rank` orders the selected steps from highest to lowest content risk.
+- Score `stateReadiness`, `pathClarity`, and `recoverySupport` from 0 (unsupported) to 100 (fully supported) using evidence from the mapped current pages. The simulator combines these scores at weights 50%, 30%, and 20%, then converts the weighted score to a bounded probability adjustment of -0.10 to +0.10. This semantic score is the required agentic contribution to the hybrid model.
+- Include all three semantic scores for each selected step, even when the resulting probability adjustment is neutral. Explain the score in `summary` and cite page evidence in the evaluations.
 - Always evaluate these Step 7 assumptions because they determine the state required by Step 8:
   - `workflow_source_created_terminal`
   - `workflow_compiled_terminal`
@@ -319,6 +328,7 @@ Use this JSON shape:
 - You may add focused evaluations for other high-risk steps. Unknown evaluation IDs remain available as report evidence but cannot directly patch arbitrary simulator state.
 - Omit steps where you have no meaningful adjustment.
 - Use positive numbers to increase success probability and negative numbers to decrease it.
+- Keep `bias`, each signal adjustment, and each path adjustment between `-0.15` and `0.15`; the simulator clamps larger values and ignores unknown adjustment keys.
 - Base these adjustments on the actual wording, path structure, fallbacks, and recovery guidance in the files you inspect — not only on the numeric signals already present in the simulator.
 - For Copilot / Agents-tab paths, verify that the content treats the surface as prompt-driven chat and explicitly calls out `/agentic-workflows` for workflow-authoring tasks. Missing that cue, or presenting shell commands as if they run inside the Agents tab, is an access barrier for `tool: CCA` learners.
 
@@ -335,6 +345,8 @@ node .github/skills/micro-environment-simulator/simulator.js \
   --out /tmp/gh-aw/agent/sim/data/monte-carlo-replay.json
 ```
 
+After rerunning, verify in the replay output that all three selected steps have a non-null `stepContentById[step].agentInsight`, a matching `evaluatedContentHash`, and all three `semanticScores`. Stop and repair the insights instead of reporting if any selected insight is missing or stale.
+
 The Monte Carlo simulation written to `/tmp/gh-aw/agent/sim/data/monte-carlo-replay.json` should therefore reflect both the simulator's constraint model and your content-aware adjustments:
 
 ```json
@@ -442,7 +454,7 @@ Update `/tmp/gh-aw/cache-memory/profiles.json`:
 
 ### Read workshop files (if available)
 
-If `${{ env.WORKSHOP_STEP_COUNT }}` > 0, use the available tools to read up to 3 of the workshop step files to ground both your `agent-step-insights.json` adjustments and your improvement suggestions in the actual content. Focus on the highest-dropout or highest-risk steps.
+If `${{ env.WORKSHOP_STEP_COUNT }}` > 0, use the available tools to read every mapped page for the three semantically ranked high-risk steps, plus prerequisite pages that establish their required state. Ground both `agent-step-insights.json` and improvement suggestions in that current content.
 
 ### Create a concise report issue