Skip to content

Commit a71eb30

Browse files
committed
feat(config): hard-break dashboard project schema
1 parent 907ac6d commit a71eb30

10 files changed

Lines changed: 143 additions & 139 deletions

File tree

apps/cli/src/commands/results/remote.ts

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -154,8 +154,8 @@ async function loadNormalizedResultsConfig(
154154
const projectResults = project?.results
155155
? {
156156
mode: 'github' as const,
157-
repo: project.results.repository,
158-
path: project.results.localPath,
157+
repo: project.results.repoUrl,
158+
path: project.results.path,
159159
auto_push: project.results.sync?.autoPush,
160160
branch_prefix: project.results.branchPrefix,
161161
}

apps/cli/test/commands/results/serve.test.ts

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1272,8 +1272,8 @@ describe('serve app', () => {
12721272
name: 'Project No Publish',
12731273
path: projectDir,
12741274
results: {
1275-
repository: `file://${remoteDir}`,
1276-
localPath: missingCloneDir,
1275+
repoUrl: `file://${remoteDir}`,
1276+
path: missingCloneDir,
12771277
sync: { autoPush: true },
12781278
},
12791279
addedAt: '2026-01-01T00:00:00.000Z',
@@ -1412,8 +1412,8 @@ describe('serve app', () => {
14121412
name: 'AgentV',
14131413
path: projectDir,
14141414
results: {
1415-
repository: 'EntityProcess/agentv-examples-eval-results',
1416-
localPath: '/home/entity/projects/EntityProcess/agentv-examples-eval-results',
1415+
repoUrl: 'EntityProcess/agentv-examples-eval-results',
1416+
path: '/home/entity/projects/EntityProcess/agentv-examples-eval-results',
14171417
sync: { autoPush: true },
14181418
},
14191419
addedAt: '2026-01-01T00:00:00.000Z',
@@ -1464,8 +1464,8 @@ describe('serve app', () => {
14641464
name: 'Project Sync Pull',
14651465
path: projectDir,
14661466
results: {
1467-
repository: `file://${remoteDir}`,
1468-
localPath: cloneDir,
1467+
repoUrl: `file://${remoteDir}`,
1468+
path: cloneDir,
14691469
sync: { autoPush: false },
14701470
},
14711471
addedAt: '2026-01-01T00:00:00.000Z',
@@ -1543,8 +1543,8 @@ describe('serve app', () => {
15431543
name: 'Project Sync Push',
15441544
path: projectDir,
15451545
results: {
1546-
repository: `file://${remoteDir}`,
1547-
localPath: cloneDir,
1546+
repoUrl: `file://${remoteDir}`,
1547+
path: cloneDir,
15481548
sync: { autoPush: true },
15491549
},
15501550
addedAt: '2026-01-01T00:00:00.000Z',
@@ -1610,8 +1610,8 @@ describe('serve app', () => {
16101610
name: 'Project Sync Offline',
16111611
path: projectDir,
16121612
results: {
1613-
repository: `file://${remoteDir}`,
1614-
localPath: cloneDir,
1613+
repoUrl: `file://${remoteDir}`,
1614+
path: cloneDir,
16151615
sync: { autoPush: true },
16161616
},
16171617
addedAt: '2026-01-01T00:00:00.000Z',
@@ -1669,8 +1669,8 @@ describe('serve app', () => {
16691669
name: 'Project Sync Conflict',
16701670
path: projectDir,
16711671
results: {
1672-
repository: `file://${remoteDir}`,
1673-
localPath: cloneDir,
1672+
repoUrl: `file://${remoteDir}`,
1673+
path: cloneDir,
16741674
sync: { autoPush: true },
16751675
},
16761676
addedAt: '2026-01-01T00:00:00.000Z',

apps/web/src/content/docs/docs/tools/dashboard.mdx

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -189,13 +189,13 @@ agentv dashboard --add /path/to/other-evals
189189

190190
Each path must contain a `.agentv/` directory. Registered projects are stored under `projects:` in `$AGENTV_HOME/config.yaml`, or `~/.agentv/config.yaml` when `AGENTV_HOME` is unset.
191191

192-
To register a remote repo and keep it synced automatically, add `repository` and `ref` to the entry in `$AGENTV_HOME/config.yaml`. `repository` uses GitHub's standard owner/name form and AgentV resolves it to `https://github.com/<owner>/<name>.git` for clone and pull operations:
192+
To register a remote repo and keep it synced automatically, add `repo_url` and `ref` to the entry in `$AGENTV_HOME/config.yaml`. `repo_url` is the Git remote URL AgentV passes to `git clone`, so it can be HTTPS or SSH:
193193

194194
```yaml
195195
projects:
196196
- id: my-evals
197197
name: My Evals
198-
repository: example/my-evals
198+
repo_url: https://github.com/example/my-evals.git
199199
path: /srv/agentv/my-evals
200200
ref: main
201201
```
@@ -255,17 +255,17 @@ For a registered project, put results repo settings on that project's entry in `
255255
projects:
256256
- id: agentv
257257
name: AgentV
258-
repository: EntityProcess/agentv
258+
repo_url: https://github.com/EntityProcess/agentv.git
259259
path: /home/entity/projects/EntityProcess/agentv
260260
ref: main
261261
results:
262-
repository: EntityProcess/agentv-examples-eval-results
263-
local_path: /home/entity/projects/EntityProcess/agentv-examples-eval-results
262+
repo_url: git@github.com:EntityProcess/agentv-examples-eval-results.git
263+
path: /home/entity/projects/EntityProcess/agentv-examples-eval-results
264264
sync:
265265
auto_push: true
266266
```
267267

268-
`results.local_path` is the filesystem location of the local clone AgentV manages for the results repo. It is **not** a subdirectory inside the remote repo. `results.repository` uses GitHub owner/name form and resolves to `https://github.com/<owner>/<name>.git` for clone and push operations.
268+
`results.path` is the filesystem location of the local clone AgentV manages for the results repo. It uses the same local-path field name as the source project. `results.repo_url` is the Git remote URL used for clone and push operations, so use HTTPS when credentials are HTTP-token based and SSH when the runtime has SSH keys configured.
269269

270270
You can also set a top-level global fallback in the same file. This is used when the current project is not registered or its registry entry has no `results` block:
271271

@@ -279,10 +279,10 @@ results:
279279

280280
Project-local `.agentv/config.yaml` is for portable eval defaults such as `execution`, `eval_patterns`, and `dashboard`. Do not put `projects` in project-local config; AgentV warns and ignores it there. `results_by_project` is deprecated; use `projects[].results` in `$AGENTV_HOME/config.yaml`.
281281

282-
The project `repository` and the `results` block sync different repositories:
282+
The project `repo_url` and the `results` block sync different repositories:
283283

284-
- `projects[].repository` is the eval source project. Dashboard startup clones or fast-forwards the project checkout so eval YAML, scripts, and project-local `.agentv/config.yaml` stay current.
285-
- `projects[].results.repository` is the git-backed results store. **Sync Project** fetches, fast-forwards, and, when configured, pushes run artifacts and mutable metadata in that results repo clone.
284+
- `projects[].repo_url` is the eval source project remote. Dashboard startup clones or fast-forwards the project checkout so eval YAML, scripts, and project-local `.agentv/config.yaml` stay current.
285+
- `projects[].results.repo_url` is the git-backed results store remote. **Sync Project** fetches, fast-forwards, and, when configured, pushes run artifacts and mutable metadata in that results repo clone.
286286

287287
#### Migration from the legacy project schema
288288

@@ -309,17 +309,17 @@ After:
309309
projects:
310310
- id: agentv
311311
name: AgentV
312-
repository: EntityProcess/agentv
312+
repo_url: https://github.com/EntityProcess/agentv.git
313313
path: /home/entity/projects/EntityProcess/agentv
314314
ref: main
315315
results:
316-
repository: EntityProcess/agentv-eval-results
317-
local_path: /home/entity/projects/EntityProcess/agentv-eval-results
316+
repo_url: https://github.com/EntityProcess/agentv-eval-results.git
317+
path: /home/entity/projects/EntityProcess/agentv-eval-results
318318
sync:
319319
auto_push: true
320320
```
321321

322-
Legacy project fields (`source`, `results.mode`, `results.repo`, `results.path`, and `results.auto_push`) fail validation with migration guidance.
322+
Legacy project fields (`source`, `repository`, `results.mode`, `results.repo`, `results.repository`, `results.local_path`, and `results.auto_push`) fail validation with migration guidance.
323323

324324
Use project-level **Sync Project** as the results exchange workflow. It handles pulled remote runs, locally edited metadata, dirty state, and blocked conflict feedback in one project-scoped action.
325325

packages/core/src/evaluation/validation/config-validator.ts

Lines changed: 27 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -175,17 +175,21 @@ function validateProjects(errors: ValidationError[], filePath: string, projects:
175175
severity: 'error',
176176
filePath,
177177
location: `${location}.source`,
178-
message: `Field '${location}.source' was removed. Move 'source.url' to '${location}.repository' as a GitHub owner/name value (for example, 'example/repo') and move 'source.ref' to '${location}.ref'.`,
178+
message: `Field '${location}.source' was removed. Move 'source.url' to '${location}.repo_url' and move 'source.ref' to '${location}.ref'. Use a Git remote URL such as https://github.com/example/repo.git or git@github.com:example/repo.git.`,
179179
});
180180
}
181181

182182
if (projectRecord.repository !== undefined) {
183-
validateGitHubRepository(
184-
errors,
183+
errors.push({
184+
severity: 'error',
185185
filePath,
186-
projectRecord.repository,
187-
`${location}.repository`,
188-
);
186+
location: `${location}.repository`,
187+
message: `Field '${location}.repository' was removed. Use '${location}.repo_url' with a Git remote URL instead.`,
188+
});
189+
}
190+
191+
if (projectRecord.repo_url !== undefined) {
192+
validateGitRemoteUrl(errors, filePath, projectRecord.repo_url, `${location}.repo_url`);
189193
}
190194

191195
if (projectRecord.ref !== undefined) {
@@ -212,7 +216,7 @@ function validateRequiredString(
212216
}
213217
}
214218

215-
function validateGitHubRepository(
219+
function validateGitRemoteUrl(
216220
errors: ValidationError[],
217221
filePath: string,
218222
value: unknown,
@@ -223,18 +227,18 @@ function validateGitHubRepository(
223227
severity: 'error',
224228
filePath,
225229
location,
226-
message: `Field '${location}' must be a non-empty GitHub owner/name repository (e.g., EntityProcess/agentv)`,
230+
message: `Field '${location}' must be a non-empty Git remote URL (e.g., https://github.com/EntityProcess/agentv.git or git@github.com:EntityProcess/agentv.git)`,
227231
});
228232
return;
229233
}
230234

231-
const repository = value.trim();
232-
if (!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(repository)) {
235+
const repoUrl = value.trim();
236+
if (!/^(https?:\/\/|ssh:\/\/|git@|file:\/\/).+/.test(repoUrl)) {
233237
errors.push({
234238
severity: 'error',
235239
filePath,
236240
location,
237-
message: `Field '${location}' must use GitHub owner/name format (e.g., EntityProcess/agentv), not a URL. It resolves to https://github.com/<owner>/<name>.git for git operations.`,
241+
message: `Field '${location}' must be a Git remote URL, not an owner/name shorthand. Use https://github.com/owner/repo.git or git@github.com:owner/repo.git.`,
238242
});
239243
}
240244
}
@@ -262,9 +266,10 @@ function validateProjectResultsConfig(
262266
const resultsRecord = rawResults as Record<string, unknown>;
263267

264268
const removedFields: Record<string, string> = {
265-
mode: `Remove '${location}.mode'; project results are GitHub-backed by '${location}.repository'.`,
266-
repo: `Field '${location}.repo' was removed. Use '${location}.repository' with GitHub owner/name format instead.`,
267-
path: `Field '${location}.path' was removed. Use '${location}.local_path' for the local clone path instead.`,
269+
mode: `Remove '${location}.mode'; project results use '${location}.repo_url' as the Git remote URL.`,
270+
repo: `Field '${location}.repo' was removed. Use '${location}.repo_url' with a Git remote URL instead.`,
271+
repository: `Field '${location}.repository' was removed. Use '${location}.repo_url' with a Git remote URL instead.`,
272+
local_path: `Field '${location}.local_path' was removed. Use '${location}.path' for the local clone path instead.`,
268273
auto_push: `Field '${location}.auto_push' was removed. Use '${location}.sync.auto_push' instead.`,
269274
};
270275

@@ -279,25 +284,22 @@ function validateProjectResultsConfig(
279284
}
280285
}
281286

282-
validateGitHubRepository(errors, filePath, resultsRecord.repository, `${location}.repository`);
287+
validateGitRemoteUrl(errors, filePath, resultsRecord.repo_url, `${location}.repo_url`);
283288

284-
if (resultsRecord.local_path !== undefined) {
285-
if (
286-
typeof resultsRecord.local_path !== 'string' ||
287-
resultsRecord.local_path.trim().length === 0
288-
) {
289+
if (resultsRecord.path !== undefined) {
290+
if (typeof resultsRecord.path !== 'string' || resultsRecord.path.trim().length === 0) {
289291
errors.push({
290292
severity: 'error',
291293
filePath,
292-
location: `${location}.local_path`,
293-
message: `Field '${location}.local_path' must be a non-empty string`,
294+
location: `${location}.path`,
295+
message: `Field '${location}.path' must be a non-empty string`,
294296
});
295-
} else if (!isFilesystemPath(resultsRecord.local_path.trim())) {
297+
} else if (!isFilesystemPath(resultsRecord.path.trim())) {
296298
errors.push({
297299
severity: 'error',
298300
filePath,
299-
location: `${location}.local_path`,
300-
message: `'${location}.local_path' must be an absolute or home-relative filesystem path (e.g., ~/data/agentv-results).`,
301+
location: `${location}.path`,
302+
message: `'${location}.path' must be an absolute or home-relative filesystem path (e.g., ~/data/agentv-results).`,
301303
});
302304
}
303305
}

packages/core/src/index.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -109,7 +109,7 @@ export {
109109
deriveProjectId,
110110
getProjectsRegistryPath,
111111
} from './projects.js';
112-
export { syncProject, syncProjects, resolveGitHubRepositoryUrl } from './project-sync.js';
112+
export { syncProject, syncProjects } from './project-sync.js';
113113
export { trimBaselineResult } from './evaluation/baseline.js';
114114
export { DEFAULT_CATEGORY, deriveCategory } from './evaluation/category.js';
115115
export * from './observability/index.js';

packages/core/src/project-sync.ts

Lines changed: 11 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,11 @@
11
/**
2-
* Project sync — pulls remote GitHub repos to the local path declared in the
2+
* Project sync — pulls remote Git repos to the local path declared in the
33
* project registry before Dashboard/eval startup.
44
*
55
* Sync is oneshot only, triggered by the Dashboard UI "Sync" button or the
66
* `agentv project sync` CLI command. There is no daemon or continuous mode.
77
*
8-
* First run — git clone --depth 1 --filter=blob:none --branch <ref> <url> <path>
8+
* First run — git clone --depth 1 --filter=blob:none --branch <ref> <repoUrl> <path>
99
* Subsequent — git pull --ff-only (when <path>/.git already exists)
1010
*
1111
* Usage:
@@ -18,45 +18,40 @@ import { existsSync } from 'node:fs';
1818

1919
import type { ProjectEntry } from './projects.js';
2020

21-
export function resolveGitHubRepositoryUrl(repository: string): string {
22-
return `https://github.com/${repository.trim().replace(/\.git$/, '')}.git`;
23-
}
24-
2521
/**
26-
* Clone or pull a single project entry from its declared repository.
22+
* Clone or pull a single project entry from its declared repo URL.
2723
* - No .git present: shallow clone into entry.path.
2824
* - .git present: git pull --ff-only to update in place.
29-
* Throws on git error or missing repository/ref.
25+
* Throws on git error or missing repoUrl/ref.
3026
*/
3127
export async function syncProject(entry: ProjectEntry): Promise<void> {
32-
if (!entry.repository) {
33-
throw new Error(`Project '${entry.id}' has no repository defined`);
28+
if (!entry.repoUrl) {
29+
throw new Error(`Project '${entry.id}' has no repo_url defined`);
3430
}
3531
if (!entry.ref) {
3632
throw new Error(`Project '${entry.id}' has no ref defined`);
3733
}
38-
const url = resolveGitHubRepositoryUrl(entry.repository);
3934
const dest = entry.path;
4035

4136
if (existsSync(`${dest}/.git`)) {
4237
childProcess.execFileSync('git', ['-C', dest, 'pull', '--ff-only'], { stdio: 'inherit' });
4338
} else {
4439
childProcess.execFileSync(
4540
'git',
46-
['clone', '--depth', '1', '--filter=blob:none', '--branch', entry.ref, url, dest],
41+
['clone', '--depth', '1', '--filter=blob:none', '--branch', entry.ref, entry.repoUrl, dest],
4742
{ stdio: 'inherit' },
4843
);
4944
}
5045
}
5146

5247
/**
53-
* Iterate project entries and sync any that have a repository declared.
54-
* Entries without repository are skipped silently.
48+
* Iterate project entries and sync any that have a repo URL declared.
49+
* Entries without repoUrl are skipped silently.
5550
*/
5651
export async function syncProjects(entries: ProjectEntry[]): Promise<void> {
5752
for (const entry of entries) {
58-
if (!entry.repository) continue;
59-
console.log(`Syncing project '${entry.id}' from ${entry.repository}...`);
53+
if (!entry.repoUrl) continue;
54+
console.log(`Syncing project '${entry.id}' from ${entry.repoUrl}...`);
6055
await syncProject(entry);
6156
console.log(`Project '${entry.id}' synced.`);
6257
}

0 commit comments

Comments
 (0)