Skip to content

Commit 28f45e2

Browse files
Copilotpelikhan
Copilot
and
pelikhan
authored
Feed page assumption evaluations into student simulator state (#1735)
* Add semantic simulator assumption evaluations Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> * Teach simulator agent to evaluate page state Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> * Address simulator eval review feedback Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> * Finalize simulator eval validation Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> * Clarify simulator eval fallback semantics Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com>
1 parent f9f6fb5 commit 28f45e2

6 files changed

Lines changed: 268 additions & 14 deletions

File tree

‎.github/skills/micro-environment-simulator/SKILL.md‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,3 +105,12 @@ Return concise JSON-friendly results that workflows can aggregate:
105105
Treat population distributions and transition coefficients as assumptions rather than observed learner data. Report that the confidence intervals exclude model and population-assumption uncertainty.
106106

107107
If assumptions hold for a full replay, mark the run successful and include the final state summary.
108+
109+
## Agent-Evaluated Content Assumptions
110+
111+
When an agent evaluates whether current workshop pages establish simulator state, pass the results with `--agent-insights`. Each step insight may include:
112+
113+
- `evaluatedContentHash`: the step's `contentHash` from a simulator run over the current pages
114+
- `evaluations`: named, single-claim records with an `answer` of `YES`, `NO`, or `UNKNOWN` and `file:line` evidence
115+
116+
The simulator ignores evaluations whose content hash no longer matches the mapped workshop pages. Journey code must map known evaluation IDs to specific state fields; never apply arbitrary agent-provided state patches. Keep probability adjustments separate from state assumptions so a favorable score cannot bypass a failed prerequisite.

‎.github/skills/micro-environment-simulator/simulator.js‎

Lines changed: 49 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -344,6 +344,38 @@ function normalizeAgentInsightsByStep(input) {
344344
normalizeNumericField(nextInsight, "bias", stepId);
345345
normalizeNumericObject(nextInsight, "signalAdjustments", stepId);
346346
normalizeNumericObject(nextInsight, "pathAdjustments", stepId);
347+
normalizeNumericField(nextInsight, "evaluatedContentHash", stepId);
348+
if (nextInsight.evaluations !== null && nextInsight.evaluations !== undefined) {
349+
if (!isPlainObject(nextInsight.evaluations)) {
350+
console.warn(`[simulator] Agent insight '${stepId}.evaluations' should be an object.`);
351+
delete nextInsight.evaluations;
352+
} else {
353+
const evaluations = {};
354+
for (const [evaluationId, evaluation] of Object.entries(nextInsight.evaluations)) {
355+
if (!isPlainObject(evaluation)) {
356+
console.warn(
357+
`[simulator] Ignoring malformed evaluation '${stepId}.${evaluationId}': expected an object.`
358+
);
359+
continue;
360+
}
361+
const answer = String(evaluation.answer ?? "").toUpperCase();
362+
if (!["YES", "NO", "UNKNOWN"].includes(answer)) {
363+
console.warn(
364+
`[simulator] Ignoring malformed evaluation '${stepId}.${evaluationId}': answer must be YES, NO, or UNKNOWN.`
365+
);
366+
continue;
367+
}
368+
evaluations[evaluationId] = {
369+
answer,
370+
...(typeof evaluation.question === "string" ? { question: evaluation.question } : {}),
371+
evidence: Array.isArray(evaluation.evidence)
372+
? evaluation.evidence.filter((item) => typeof item === "string")
373+
: []
374+
};
375+
}
376+
nextInsight.evaluations = evaluations;
377+
}
378+
}
347379
if (nextInsight.riskTags != null && !Array.isArray(nextInsight.riskTags)) {
348380
console.warn(`[simulator] Agent insight '${stepId}.riskTags' should be an array.`);
349381
delete nextInsight.riskTags;
@@ -381,20 +413,31 @@ function buildStepContentById({
381413
markdown: fs.readFileSync(path.resolve(workshopDir, file), "utf8")
382414
}));
383415
const markdown = fileContents.map(({ markdown: fileMarkdown }) => fileMarkdown).join("\n\n");
416+
const contentAnalysis = analyzeStepMarkdown(
417+
stepId,
418+
markdown,
419+
fileContents.map(({ file, title }) => ({ file, title }))
420+
);
421+
const agentInsight = agentInsightsByStep[stepId] ? { ...agentInsightsByStep[stepId] } : null;
422+
if (
423+
agentInsight?.evaluations &&
424+
Number(agentInsight.evaluatedContentHash) !== contentAnalysis.contentHash
425+
) {
426+
console.warn(
427+
`[simulator] Ignoring stale agent evaluations for step '${stepId}': content hash does not match.`
428+
);
429+
delete agentInsight.evaluations;
430+
}
384431
stepContentById[stepId] = deepFreeze({
385-
...analyzeStepMarkdown(
386-
stepId,
387-
markdown,
388-
fileContents.map(({ file, title }) => ({ file, title }))
389-
),
432+
...contentAnalysis,
390433
fileSignals: fileContents.map(({ file, title, markdown: fileMarkdown }) =>
391434
deepFreeze({
392435
file,
393436
title,
394437
...analyzeStepMarkdown(file, fileMarkdown, [{ file, title }])
395438
})
396439
),
397-
agentInsight: agentInsightsByStep[stepId] || null
440+
agentInsight
398441
});
399442
}
400443

‎.github/skills/micro-environment-simulator/simulator.test.js‎

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,12 @@
22

33
const assert = require("node:assert/strict");
44
const fs = require("node:fs");
5+
const os = require("node:os");
56
const path = require("node:path");
67
const test = require("node:test");
78

89
const simulator = require("./simulator");
10+
const journey = require("./workshop-student-journey");
911

1012
const populationModel = JSON.parse(
1113
fs.readFileSync(path.join(__dirname, "workshop-student-population.json"), "utf8")
@@ -101,3 +103,132 @@ test("synthetic simulation history does not increase learner confidence", () =>
101103
assert.equal(freshState.learner.sessionEffect, experiencedState.learner.sessionEffect);
102104
assert.deepEqual(freshState.learner.mastery, experiencedState.learner.mastery);
103105
});
106+
107+
test("agent assumption evaluations are normalized and stale evaluations are ignored", (t) => {
108+
const repoRoot = fs.mkdtempSync(path.join(os.tmpdir(), "simulator-evals-"));
109+
t.after(() => fs.rmSync(repoRoot, { recursive: true, force: true }));
110+
fs.mkdirSync(path.join(repoRoot, "workshop"));
111+
fs.writeFileSync(path.join(repoRoot, "workshop", "step.md"), "# Current lesson\n");
112+
const contentHash = simulator.analyzeStepMarkdown("step", "# Current lesson\n").contentHash;
113+
const normalized = simulator.normalizeAgentInsightsByStep({
114+
stepInsightsById: {
115+
step: {
116+
evaluatedContentHash: String(contentHash),
117+
evaluations: {
118+
valid: { answer: "yes", evidence: ["step.md:1", 2] },
119+
unknown: { answer: "UNKNOWN" },
120+
invalid: { answer: "MAYBE" }
121+
}
122+
}
123+
}
124+
});
125+
126+
assert.equal(normalized.step.evaluations.valid.answer, "YES");
127+
assert.deepEqual(normalized.step.evaluations.valid.evidence, ["step.md:1"]);
128+
assert.equal(normalized.step.evaluations.unknown.answer, "UNKNOWN");
129+
assert.equal(normalized.step.evaluations.invalid, undefined);
130+
131+
const current = simulator.buildStepContentById({
132+
steps: ["step"],
133+
stepFilesById: { step: ["step.md"] },
134+
repoRoot,
135+
agentInsightsByStep: normalized
136+
});
137+
assert.equal(current.step.agentInsight.evaluations.valid.answer, "YES");
138+
139+
fs.writeFileSync(path.join(repoRoot, "workshop", "step.md"), "# Updated lesson\n");
140+
const updated = simulator.buildStepContentById({
141+
steps: ["step"],
142+
stepFilesById: { step: ["step.md"] },
143+
repoRoot,
144+
agentInsightsByStep: normalized
145+
});
146+
assert.equal(updated.step.agentInsight.evaluations, undefined);
147+
});
148+
149+
function firstWorkflowState(centralizedCopilotBilling) {
150+
const student = {
151+
id: 1,
152+
level: "actions-user",
153+
personality: "methodical",
154+
background: "web-dev",
155+
goal: "personal-learning",
156+
tool: "CCA",
157+
ui_preferred: true
158+
};
159+
const state = JSON.parse(
160+
JSON.stringify(simulator.defaultEnvironmentForStudent(student, 120, 0))
161+
);
162+
state.auth.hasGithubSession = true;
163+
state.auth.hasCopilotAccess = true;
164+
state.github.deployment = "github.com";
165+
state.flags.hasRepo = true;
166+
state.flags.repoHasReadme = true;
167+
state.actions.centralizedCopilotBilling = centralizedCopilotBilling;
168+
return state;
169+
}
170+
171+
function firstWorkflowContext(evaluations) {
172+
return {
173+
stepId: "07-first-workflow",
174+
random: () => 0,
175+
stepContent: {
176+
browserSupport: 1,
177+
terminalDemand: 0,
178+
complexity: 0,
179+
authDemand: 0,
180+
troubleshootingSupport: 0,
181+
conceptDemand: 0,
182+
enterpriseDemand: 0,
183+
workflowCompileCueCount: 0,
184+
agentInsight: { evaluations }
185+
}
186+
};
187+
}
188+
189+
test("semantic evaluations update compiled workflow and centralized billing state", () => {
190+
const result = journey.transitions["07-first-workflow"](
191+
firstWorkflowState(true),
192+
firstWorkflowContext({
193+
cca_authoring_guidance: { answer: "YES" },
194+
workflow_source_created_copilot: { answer: "YES" },
195+
workflow_compiled_copilot: { answer: "YES" },
196+
workflow_published_copilot: { answer: "YES" },
197+
copilot_centralized_billing_configured: { answer: "YES" }
198+
})
199+
);
200+
201+
assert.equal(result.ok, true);
202+
assert.equal(result.state.flags.workflowReadyToRun, true);
203+
assert.equal(result.state.actions.permissions.copilotRequestsWrite, true);
204+
assert.equal(result.state.actions.secrets.COPILOT_GITHUB_TOKEN, false);
205+
});
206+
207+
test("semantic evaluations update personal billing state and fail closed on UNKNOWN", () => {
208+
const configured = journey.transitions["07-first-workflow"](
209+
firstWorkflowState(false),
210+
firstWorkflowContext({
211+
cca_authoring_guidance: { answer: "YES" },
212+
workflow_source_created_copilot: { answer: "YES" },
213+
workflow_compiled_copilot: { answer: "YES" },
214+
workflow_published_copilot: { answer: "YES" },
215+
copilot_personal_billing_configured: { answer: "YES" }
216+
})
217+
);
218+
const notPublished = journey.transitions["07-first-workflow"](
219+
firstWorkflowState(false),
220+
firstWorkflowContext({
221+
cca_authoring_guidance: { answer: "YES" },
222+
workflow_source_created_copilot: { answer: "YES" },
223+
workflow_compiled_copilot: { answer: "YES" },
224+
workflow_published_copilot: { answer: "UNKNOWN" },
225+
copilot_personal_billing_configured: { answer: "UNKNOWN" }
226+
})
227+
);
228+
229+
assert.equal(configured.ok, true);
230+
assert.equal(configured.state.actions.permissions.copilotRequestsWrite, false);
231+
assert.equal(configured.state.actions.secrets.COPILOT_GITHUB_TOKEN, true);
232+
assert.equal(notPublished.state.flags.workflowReadyToRun, false);
233+
assert.equal(notPublished.state.actions.secrets.COPILOT_GITHUB_TOKEN, false);
234+
});

‎.github/skills/micro-environment-simulator/workshop-student-journey.js‎

Lines changed: 43 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,16 @@ function agentInsight(context) {
123123
return insight && typeof insight === "object" ? insight : {};
124124
}
125125

126+
// Missing evaluations return undefined so legacy lexical inference remains available.
127+
// UNKNOWN returns null so evaluated paths can fail closed without treating the answer as NO.
128+
function evaluatedAssumption(context, evaluationId) {
129+
const evaluation = agentInsight(context).evaluations?.[evaluationId];
130+
if (!evaluation) return undefined;
131+
if (evaluation.answer === "YES") return true;
132+
if (evaluation.answer === "NO") return false;
133+
return null;
134+
}
135+
126136
function ensurePlainObject(value) {
127137
return value !== null && typeof value === "object" && !Array.isArray(value) ? value : {};
128138
}
@@ -154,6 +164,8 @@ function needsCcaPromptGuidance(state, context) {
154164
}
155165

156166
function hasCcaPromptGuidance(state, context) {
167+
const evaluated = evaluatedAssumption(context, "cca_authoring_guidance");
168+
if (evaluated !== undefined) return evaluated === true;
157169
return (
158170
stepMetric(state, context, "agentsPromptCueCount") > 0 &&
159171
stepMetric(state, context, "agenticWorkflowSkillCueCount") > 0
@@ -182,17 +194,31 @@ function stepMetric(state, context, metric) {
182194

183195
function updateWorkflowCompileState(state, context, options = {}) {
184196
const next = cloneState(state);
197+
const path = prefersBrowserPath(state, context) ? "copilot" : "terminal";
198+
const sourceCreated = evaluatedAssumption(context, `workflow_source_created_${path}`);
185199
next.flags.hasWorkflowFile = true;
200+
if (sourceCreated !== undefined) {
201+
next.flags.hasWorkflowFile = sourceCreated === true;
202+
}
203+
const compiledEvaluation = evaluatedAssumption(context, `workflow_compiled_${path}`);
204+
const publishedEvaluation = evaluatedAssumption(context, `workflow_published_${path}`);
186205
// A lesson without compile instructions does not invalidate a lock file completed earlier.
187-
if (stepMetric(state, context, "workflowCompileCueCount") <= 0) {
206+
if (
207+
compiledEvaluation === undefined &&
208+
stepMetric(state, context, "workflowCompileCueCount") <= 0
209+
) {
188210
return next;
189211
}
190212
const hasCompiledWorkflowLock =
191-
canCompileWorkflow(state, context, options);
213+
compiledEvaluation === undefined
214+
? canCompileWorkflow(state, context, options)
215+
: compiledEvaluation === true;
192216
const hasPushedCompiledWorkflowLock =
193217
hasCompiledWorkflowLock &&
194-
(stepMetric(state, context, "workflowLockPublishCueCount") > 0 ||
195-
state.flags.hasPushedCompiledWorkflowLock);
218+
(publishedEvaluation === undefined
219+
? stepMetric(state, context, "workflowLockPublishCueCount") > 0 ||
220+
state.flags.hasPushedCompiledWorkflowLock
221+
: publishedEvaluation === true);
196222
next.flags.hasCompiledWorkflowLock = hasCompiledWorkflowLock;
197223
next.flags.hasPushedCompiledWorkflowLock = hasPushedCompiledWorkflowLock;
198224
next.flags.workflowReadyToRun = hasPushedCompiledWorkflowLock;
@@ -205,10 +231,21 @@ function configureFirstWorkflowAuth(state, context) {
205231
return next;
206232
}
207233
const usesCentralizedBilling = next.actions?.centralizedCopilotBilling === true;
234+
const centralizedEvaluation = evaluatedAssumption(
235+
context,
236+
"copilot_centralized_billing_configured"
237+
);
238+
const personalEvaluation = evaluatedAssumption(context, "copilot_personal_billing_configured");
208239
next.actions.permissions.copilotRequestsWrite =
209-
usesCentralizedBilling && stepMetric(state, context, "copilotRequestsWriteCueCount") > 0;
240+
usesCentralizedBilling &&
241+
(centralizedEvaluation === undefined
242+
? stepMetric(state, context, "copilotRequestsWriteCueCount") > 0
243+
: centralizedEvaluation === true);
210244
next.actions.secrets.COPILOT_GITHUB_TOKEN =
211-
!usesCentralizedBilling && stepMetric(state, context, "copilotGithubTokenCueCount") > 0;
245+
!usesCentralizedBilling &&
246+
(personalEvaluation === undefined
247+
? stepMetric(state, context, "copilotGithubTokenCueCount") > 0
248+
: personalEvaluation === true);
212249
return next;
213250
}
214251

‎.github/workflows/workshop-student-simulator.lock.yml‎

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)