Skip to content

Commit e01fa30

Browse files
Patrick-ErichsenPatrick-Erichsen
committed
feat: add canonical ClawScan verdict fields
1 parent 01e4418 commit e01fa30

65 files changed

Lines changed: 2517 additions & 676 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎convex/_generated/api.d.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@ import type * as lib_badges from "../lib/badges.js";
4949
import type * as lib_batching from "../lib/batching.js";
5050
import type * as lib_changelog from "../lib/changelog.js";
5151
import type * as lib_clawScanNote from "../lib/clawScanNote.js";
52+
import type * as lib_clawScanVerdict from "../lib/clawScanVerdict.js";
5253
import type * as lib_clawpack from "../lib/clawpack.js";
5354
import type * as lib_commentScamPrompt from "../lib/commentScamPrompt.js";
5455
import type * as lib_contentTypes from "../lib/contentTypes.js";
@@ -182,6 +183,7 @@ declare const fullApi: ApiFromModules<{
182183
"lib/batching": typeof lib_batching;
183184
"lib/changelog": typeof lib_changelog;
184185
"lib/clawScanNote": typeof lib_clawScanNote;
186+
"lib/clawScanVerdict": typeof lib_clawScanVerdict;
185187
"lib/clawpack": typeof lib_clawpack;
186188
"lib/commentScamPrompt": typeof lib_commentScamPrompt;
187189
"lib/contentTypes": typeof lib_contentTypes;

‎convex/autobanRemediation.test.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -969,6 +969,7 @@ describe("autoban remediation package restore", () => {
969969
verification: {},
970970
softDeletedAt: bannedAt,
971971
staticScan: { status: "malicious" },
972+
clawScanVerdict: "malicious",
972973
distTags: ["latest"],
973974
createdAt: 2,
974975
},
@@ -1099,6 +1100,7 @@ describe("autoban remediation package restore", () => {
10991100
version: "1.0.0",
11001101
softDeletedAt: bannedAt,
11011102
staticScan: { status: "malicious" },
1103+
clawScanVerdict: "malicious",
11021104
distTags: ["latest"],
11031105
},
11041106
]),

‎convex/httpApiV1.handlers.test.ts‎

Lines changed: 94 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1828,7 +1828,7 @@ describe("httpApiV1 handlers", () => {
18281828
expect(response.status).toBe(200);
18291829
const json = await response.json();
18301830
expect(json.version.security.status).toBe("pending");
1831-
expect(json.version.security.scanners.vt.normalizedStatus).toBe("suspicious");
1831+
expect(json.version.security.scanners.vt.normalizedStatus).toBe("review");
18321832
expect(json.version.security.virustotalUrl).toContain("virustotal.com/gui/file/");
18331833
});
18341834

@@ -1880,12 +1880,12 @@ describe("httpApiV1 handlers", () => {
18801880
expect(json.version.security.status).toBe("clean");
18811881
expect(json.version.security.hasWarnings).toBe(false);
18821882
expect(json.version.security.hasScanResult).toBe(true);
1883-
expect(json.version.security.scanners.static.normalizedStatus).toBe("pending");
1883+
expect(json.version.security.scanners.static.normalizedStatus).toBe("review");
18841884
expect(json.version.security.scanners.vt.normalizedStatus).toBe("clean");
18851885
expect(json.version.security.scanners.llm.normalizedStatus).toBe("clean");
18861886
});
18871887

1888-
it("lets static-scan malicious status dominate benign vt and llm results", async () => {
1888+
it("keeps static-scan malicious status advisory when ClawScan is clean", async () => {
18891889
const runQuery = vi.fn(async (_query: unknown, args: Record<string, unknown>) => {
18901890
if ("slug" in args) {
18911891
return {
@@ -1918,6 +1918,8 @@ describe("httpApiV1 handlers", () => {
19181918
verdict: "benign",
19191919
checkedAt: 222,
19201920
},
1921+
clawScanVerdict: "clean",
1922+
clawScanState: "complete",
19211923
files: [],
19221924
};
19231925
}
@@ -1930,13 +1932,93 @@ describe("httpApiV1 handlers", () => {
19301932
);
19311933
expect(response.status).toBe(200);
19321934
const json = await response.json();
1933-
expect(json.version.security.status).toBe("malicious");
1934-
expect(json.version.security.hasWarnings).toBe(true);
1935+
expect(json.version.security.status).toBe("clean");
1936+
expect(json.version.security.hasWarnings).toBe(false);
19351937
expect(json.version.security.hasScanResult).toBe(true);
19361938
expect(json.version.security.checkedAt).toBe(555);
19371939
expect(json.version.security.scanners.static.normalizedStatus).toBe("malicious");
19381940
});
19391941

1942+
it("reports pending ClawScan state over stale completed scanner details", async () => {
1943+
const runQuery = vi.fn(async (_query: unknown, args: Record<string, unknown>) => {
1944+
if ("slug" in args) {
1945+
return {
1946+
skill: { _id: "skills:1", slug: "demo", displayName: "Demo" },
1947+
latestVersion: null,
1948+
owner: { handle: "owner", displayName: "Owner", image: null },
1949+
};
1950+
}
1951+
if ("skillId" in args && "version" in args) {
1952+
return {
1953+
version: "1.0.0",
1954+
createdAt: 1,
1955+
changelog: "c",
1956+
changelogSource: "auto",
1957+
sha256hash: "a".repeat(64),
1958+
clawScanVerdict: "clean",
1959+
clawScanState: "running",
1960+
llmAnalysis: {
1961+
status: "completed",
1962+
verdict: "benign",
1963+
checkedAt: 222,
1964+
},
1965+
files: [],
1966+
};
1967+
}
1968+
return null;
1969+
});
1970+
const runMutation = vi.fn().mockResolvedValue(okRate());
1971+
const response = await __handlers.skillsGetRouterV1Handler(
1972+
makeCtx({ runQuery, runMutation }),
1973+
new Request("https://example.com/api/v1/skills/demo/versions/1.0.0"),
1974+
);
1975+
expect(response.status).toBe(200);
1976+
const json = await response.json();
1977+
expect(json.version.security.status).toBe("pending");
1978+
expect(json.version.security.hasScanResult).toBe(false);
1979+
expect(json.version.security.scanners.llm.normalizedStatus).toBe("clean");
1980+
});
1981+
1982+
it("keeps malicious ClawScan verdict authoritative during rescans", async () => {
1983+
const runQuery = vi.fn(async (_query: unknown, args: Record<string, unknown>) => {
1984+
if ("slug" in args) {
1985+
return {
1986+
skill: { _id: "skills:1", slug: "demo", displayName: "Demo" },
1987+
latestVersion: null,
1988+
owner: { handle: "owner", displayName: "Owner", image: null },
1989+
};
1990+
}
1991+
if ("skillId" in args && "version" in args) {
1992+
return {
1993+
version: "1.0.0",
1994+
createdAt: 1,
1995+
changelog: "c",
1996+
changelogSource: "auto",
1997+
sha256hash: "a".repeat(64),
1998+
clawScanVerdict: "malicious",
1999+
clawScanState: "running",
2000+
llmAnalysis: {
2001+
status: "malicious",
2002+
verdict: "malicious",
2003+
checkedAt: 222,
2004+
},
2005+
files: [],
2006+
};
2007+
}
2008+
return null;
2009+
});
2010+
const runMutation = vi.fn().mockResolvedValue(okRate());
2011+
const response = await __handlers.skillsGetRouterV1Handler(
2012+
makeCtx({ runQuery, runMutation }),
2013+
new Request("https://example.com/api/v1/skills/demo/versions/1.0.0"),
2014+
);
2015+
expect(response.status).toBe(200);
2016+
const json = await response.json();
2017+
expect(json.version.security.status).toBe("malicious");
2018+
expect(json.version.security.hasWarnings).toBe(true);
2019+
expect(json.version.security.hasScanResult).toBe(true);
2020+
});
2021+
19402022
it("does not treat a static scan by itself as a definitive scan result", async () => {
19412023
const runQuery = vi.fn(async (_query: unknown, args: Record<string, unknown>) => {
19422024
if ("slug" in args) {
@@ -1975,7 +2057,7 @@ describe("httpApiV1 handlers", () => {
19752057
expect(json.version.security.hasWarnings).toBe(false);
19762058
expect(json.version.security.hasScanResult).toBe(false);
19772059
expect(json.version.security.virustotalUrl).toBeNull();
1978-
expect(json.version.security.scanners.static.normalizedStatus).toBe("pending");
2060+
expect(json.version.security.scanners.static.normalizedStatus).toBe("clean");
19792061
expect(json.version.security.scanners.vt).toBeNull();
19802062
expect(json.version.security.scanners.llm).toBeNull();
19812063
});
@@ -2078,7 +2160,7 @@ describe("httpApiV1 handlers", () => {
20782160
);
20792161
expect(response.status).toBe(200);
20802162
const json = await response.json();
2081-
expect(json.security.status).toBe("suspicious");
2163+
expect(json.security.status).toBe("review");
20822164
expect(json.security.hasScanResult).toBe(true);
20832165
expect(json.security.capabilityTags).toEqual([
20842166
"crypto",
@@ -2311,7 +2393,7 @@ describe("httpApiV1 handlers", () => {
23112393
expect(response.status).toBe(200);
23122394
const json = await response.json();
23132395
expect(json.version.version).toBe("1.0.0");
2314-
expect(json.security.status).toBe("suspicious");
2396+
expect(json.security.status).toBe("review");
23152397
expect(json.moderation.scope).toBe("skill");
23162398
expect(json.moderation.sourceVersion).toEqual({
23172399
version: "2.0.0",
@@ -5355,6 +5437,8 @@ describe("httpApiV1 handlers", () => {
53555437
{
53565438
name: "malicious",
53575439
release: {
5440+
clawScanVerdict: "malicious",
5441+
clawScanState: "complete",
53585442
staticScan: {
53595443
status: "malicious",
53605444
reasonCodes: ["malicious.test"],
@@ -7156,6 +7240,8 @@ describe("httpApiV1 handlers", () => {
71567240
version: "1.0.0",
71577241
createdAt: 1,
71587242
changelog: "init",
7243+
clawScanVerdict: "malicious",
7244+
clawScanState: "complete",
71597245
verification: { scanStatus: "malicious" },
71607246
files: [
71617247
{

‎convex/httpApiV1/packagesV1.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -412,6 +412,8 @@ type ReleaseLike = {
412412
verification?: Doc<"packageReleases">["verification"];
413413
extractedPackageJson?: Doc<"packageReleases">["extractedPackageJson"];
414414
sha256hash?: string;
415+
clawScanVerdict?: Doc<"packageReleases">["clawScanVerdict"];
416+
clawScanState?: Doc<"packageReleases">["clawScanState"];
415417
vtAnalysis?: Doc<"packageReleases">["vtAnalysis"];
416418
llmAnalysis?: Doc<"packageReleases">["llmAnalysis"];
417419
clawScanNote?: string;
@@ -2900,6 +2902,8 @@ export async function packagesGetRouterV1Handler(ctx: ActionCtx, request: Reques
29002902
verification: result.version.verification ?? null,
29012903
artifact: toReleaseArtifact(result.version, result.package.name),
29022904
sha256hash: result.version.sha256hash ?? null,
2905+
clawScanVerdict: result.version.clawScanVerdict ?? null,
2906+
clawScanState: result.version.clawScanState ?? null,
29032907
vtAnalysis: result.version.vtAnalysis ?? null,
29042908
llmAnalysis: result.version.llmAnalysis ?? null,
29052909
clawScanNote: result.version.clawScanNote ?? null,

‎convex/httpApiV1/skillsV1.ts‎

Lines changed: 74 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import { api, internal } from "../_generated/api";
1111
import type { Doc, Id } from "../_generated/dataModel";
1212
import type { ActionCtx } from "../_generated/server";
1313
import { getOptionalApiTokenUserId, requireApiTokenUser } from "../lib/apiTokenAuth";
14+
import { normalizeClawScanVerdict, type ClawScanState } from "../lib/clawScanVerdict";
1415
import { applyRateLimit } from "../lib/httpRateLimit";
1516
import { parseBooleanQueryParam, resolveBooleanQueryParam } from "../lib/httpUtils";
1617
import type {
@@ -106,6 +107,8 @@ type PublicSkillVersionResponse = {
106107
parsed?: PublicSkillVersionParsed;
107108
softDeletedAt?: number;
108109
sha256hash?: string;
110+
clawScanVerdict?: Doc<"skillVersions">["clawScanVerdict"];
111+
clawScanState?: Doc<"skillVersions">["clawScanState"];
109112
vtAnalysis?: Doc<"skillVersions">["vtAnalysis"];
110113
llmAnalysis?: Doc<"skillVersions">["llmAnalysis"];
111114
staticScan?: PublicSkillVersionStaticScan;
@@ -208,7 +211,14 @@ function normalizeModerationFromSkill(skill: SkillModerationShape) {
208211
};
209212
}
210213

211-
type NormalizedSecurityStatus = "clean" | "suspicious" | "malicious" | "pending" | "error";
214+
type NormalizedSecurityStatus =
215+
| "clean"
216+
| "review"
217+
| "warn"
218+
| "suspicious"
219+
| "malicious"
220+
| "pending"
221+
| "error";
212222

213223
type SkillSecuritySnapshot = {
214224
status: NormalizedSecurityStatus;
@@ -277,26 +287,29 @@ async function runMutationRef<T>(ctx: ActionCtx, ref: unknown, args: unknown): P
277287

278288
function isDefinitiveSecurityStatus(
279289
status: NormalizedSecurityStatus | null | undefined,
280-
): status is "clean" | "suspicious" | "malicious" {
281-
return status === "clean" || status === "suspicious" || status === "malicious";
290+
): status is "clean" | "review" | "warn" | "suspicious" | "malicious" {
291+
return (
292+
status === "clean" ||
293+
status === "review" ||
294+
status === "warn" ||
295+
status === "suspicious" ||
296+
status === "malicious"
297+
);
282298
}
283299

284-
const SECURITY_STATUS_PRIORITY: Record<NormalizedSecurityStatus, number> = {
285-
clean: 0,
286-
error: 1,
287-
pending: 2,
288-
suspicious: 3,
289-
malicious: 4,
290-
};
291-
292300
function normalizeSecurityStatus(value: string | null | undefined): NormalizedSecurityStatus {
293301
const normalized = value?.trim().toLowerCase();
294302
switch (normalized) {
295303
case "benign":
296304
case "clean":
297305
return "clean";
306+
case "review":
307+
return "review";
308+
case "warn":
309+
case "warning":
310+
return "warn";
298311
case "suspicious":
299-
return "suspicious";
312+
return "review";
300313
case "malicious":
301314
return "malicious";
302315
case "error":
@@ -314,11 +327,17 @@ function normalizeSecurityStatus(value: string | null | undefined): NormalizedSe
314327
}
315328
}
316329

317-
function mergeSecurityStatuses(statuses: NormalizedSecurityStatus[]) {
318-
if (statuses.length === 0) return "pending" satisfies NormalizedSecurityStatus;
319-
return statuses.reduce((current, candidate) =>
320-
SECURITY_STATUS_PRIORITY[candidate] > SECURITY_STATUS_PRIORITY[current] ? candidate : current,
321-
);
330+
function normalizeClawScanState(value: string | null | undefined): ClawScanState | null {
331+
const normalized = value?.trim().toLowerCase();
332+
if (
333+
normalized === "pending" ||
334+
normalized === "running" ||
335+
normalized === "complete" ||
336+
normalized === "error"
337+
) {
338+
return normalized;
339+
}
340+
return null;
322341
}
323342

324343
function hasLlmDimensionWarnings(dimensions: LlmEvalDimension[] | undefined) {
@@ -333,7 +352,13 @@ function hasLlmDimensionWarnings(dimensions: LlmEvalDimension[] | undefined) {
333352
function buildSkillSecuritySnapshot(
334353
version: Pick<
335354
PublicSkillVersionResponse,
336-
"sha256hash" | "vtAnalysis" | "llmAnalysis" | "staticScan" | "capabilityTags"
355+
| "sha256hash"
356+
| "clawScanVerdict"
357+
| "clawScanState"
358+
| "vtAnalysis"
359+
| "llmAnalysis"
360+
| "staticScan"
361+
| "capabilityTags"
337362
>,
338363
): SkillSecuritySnapshot | null {
339364
const capabilityTags = version.capabilityTags ?? [];
@@ -344,22 +369,40 @@ function buildSkillSecuritySnapshot(
344369

345370
if (!sha256hash && !vt && !llm && !staticScan && capabilityTags.length === 0) return null;
346371

347-
const staticStatus =
348-
staticScan?.status?.trim().toLowerCase() === "malicious"
349-
? ("malicious" satisfies NormalizedSecurityStatus)
350-
: null;
372+
const staticStatus = staticScan ? normalizeSecurityStatus(staticScan.status) : null;
351373
const vtStatus = vt ? normalizeSecurityStatus(vt.verdict ?? vt.status) : null;
352374
const llmStatus = llm ? normalizeSecurityStatus(llm.verdict ?? llm.status) : null;
353-
354-
const statuses: NormalizedSecurityStatus[] = [];
355-
if (staticStatus) statuses.push(staticStatus);
356-
if (llmStatus) statuses.push(llmStatus);
357-
if (statuses.length === 0 && sha256hash) statuses.push("pending");
358-
const status = mergeSecurityStatuses(statuses);
359-
const hasScanResult =
360-
isDefinitiveSecurityStatus(staticStatus) || isDefinitiveSecurityStatus(llmStatus);
375+
const clawScanVerdict = normalizeClawScanVerdict(version.clawScanVerdict);
376+
const clawScanStatus = clawScanVerdict
377+
? (clawScanVerdict satisfies NormalizedSecurityStatus)
378+
: null;
379+
const clawScanState = normalizeClawScanState(version.clawScanState);
380+
381+
let status: NormalizedSecurityStatus = "pending";
382+
let hasScanResult = false;
383+
if (clawScanStatus === "malicious") {
384+
status = "malicious";
385+
hasScanResult = true;
386+
} else if (clawScanState === "pending" || clawScanState === "running") {
387+
status = "pending";
388+
} else if (clawScanState === "error") {
389+
status = "error";
390+
} else if (clawScanStatus) {
391+
status = clawScanStatus;
392+
hasScanResult = true;
393+
} else if (clawScanState === "complete") {
394+
status = isDefinitiveSecurityStatus(llmStatus) ? llmStatus : "error";
395+
hasScanResult = isDefinitiveSecurityStatus(llmStatus);
396+
} else if (llmStatus) {
397+
status = llmStatus;
398+
hasScanResult = isDefinitiveSecurityStatus(llmStatus);
399+
}
361400
const hasWarnings =
362-
status === "suspicious" || status === "malicious" || hasLlmDimensionWarnings(llm?.dimensions);
401+
status === "review" ||
402+
status === "warn" ||
403+
status === "suspicious" ||
404+
status === "malicious" ||
405+
hasLlmDimensionWarnings(llm?.dimensions);
363406

364407
const checkedAtCandidates = [staticScan?.checkedAt, vt?.checkedAt, llm?.checkedAt].filter(
365408
(value): value is number => typeof value === "number",

0 commit comments

Comments
 (0)