Skip to content

Commit 330bd44

Browse files
RafaelGSSRafaelGSS
committed
feat: add git node security --validate-reports
1 parent f0587fa commit 330bd44

3 files changed

Lines changed: 1353 additions & 0 deletions

File tree

‎components/git/security.js‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ import PrepareSecurityRelease from '../../lib/prepare_security.js';
33
import UpdateSecurityRelease from '../../lib/update_security_release.js';
44
import SecurityBlog from '../../lib/security_blog.js';
55
import SecurityAnnouncement from '../../lib/security-announcement.js';
6+
import ValidateReports from '../../lib/validate_reports.js';
67

78
export const command = 'security [options]';
89
export const describe = 'Manage an in-progress security release or start a new one.';
@@ -40,6 +41,52 @@ const securityOptions = {
4041
describe: 'Request CVEs for a security release',
4142
type: 'boolean'
4243
},
44+
'validate-reports': {
45+
describe: 'Validate triaged HackerOne reports against the Node.js threat model',
46+
type: 'boolean'
47+
},
48+
'validate-reports-format': {
49+
choices: ['markdown', 'json'],
50+
default: 'markdown',
51+
describe: 'Output format for --validate-reports',
52+
type: 'string'
53+
},
54+
'validate-reports-output': {
55+
describe: 'Write --validate-reports output to a file instead of stdout',
56+
type: 'string'
57+
},
58+
'validate-reports-limit': {
59+
describe: 'Maximum number of triaged reports to validate',
60+
type: 'number'
61+
},
62+
'validate-reports-confirm': {
63+
default: true,
64+
describe: 'Ask before continuing to the next report after each LLM assessment',
65+
type: 'boolean'
66+
},
67+
'validate-reports-cache': {
68+
default: true,
69+
describe: 'Reuse cached LLM assessments for the same report, model, and prompt',
70+
type: 'boolean'
71+
},
72+
llm: {
73+
choices: ['codex', 'claude', 'copilot'],
74+
describe: 'Ask an LLM CLI to assess each triaged report',
75+
type: 'string'
76+
},
77+
'llm-model': {
78+
describe: 'Override the LLM model used for command construction and cache identity',
79+
type: 'string'
80+
},
81+
'llm-command': {
82+
describe: 'Override the command used for --llm. The report prompt is sent on stdin.',
83+
type: 'string'
84+
},
85+
'node-repo': {
86+
default: process.cwd(),
87+
describe: 'Node.js checkout path the LLM should use to read SECURITY.md and doc/',
88+
type: 'string'
89+
},
4390
'post-release': {
4491
describe: 'Create the post-release announcement to the given nodejs.org folder',
4592
type: 'string'
@@ -82,6 +129,9 @@ export function builder(yargs) {
82129
'git node security --request-cve',
83130
'Request CVEs for a security release of Node.js based on' +
84131
' the next-security-release/vulnerabilities.json'
132+
).example(
133+
'git node security --validate-reports',
134+
'Validate triaged HackerOne reports against the Node.js threat model'
85135
).example(
86136
'git node security --post-release="../nodejs.org/"',
87137
'Create the post-release announcement on the Nodejs.org repo'
@@ -119,6 +169,9 @@ export function handler(argv) {
119169
if (argv['request-cve']) {
120170
return requestCVEs(cli, argv);
121171
}
172+
if (argv['validate-reports']) {
173+
return validateReports(cli, argv);
174+
}
122175
if (argv['post-release']) {
123176
return createPostRelease(cli, argv);
124177
}
@@ -157,6 +210,11 @@ async function requestCVEs(cli) {
157210
return hackerOneCve.requestCVEs();
158211
}
159212

213+
async function validateReports(cli, argv) {
214+
const validator = new ValidateReports(cli, argv);
215+
return validator.validate();
216+
}
217+
160218
async function createPostRelease(cli, argv) {
161219
const nodejsOrgFolder = argv['post-release'];
162220
const blog = new SecurityBlog(cli);

‎docs/git-node.md‎

Lines changed: 130 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -506,6 +506,136 @@ Example:
506506
git node security --remove-report=12345
507507
```
508508

509+
### `git node security --validate-reports`
510+
511+
This command retrieves all triaged HackerOne reports for the Node.js program and
512+
produces a local validation report for each one. It is intended to help the
513+
security team review whether a report still looks valid under the Node.js threat
514+
model and whether the current HackerOne severity/CVSS is consistent with the
515+
available evidence.
516+
517+
The command uses the existing HackerOne credentials configured in `.ncurc`. It
518+
does not modify HackerOne reports, labels, comments, or severity. The output is
519+
only a local triage aid and still requires human review.
520+
521+
```sh
522+
git node security --validate-reports
523+
git node security --validate-reports --validate-reports-format=json
524+
git node security --validate-reports --validate-reports-output=reports.md
525+
git node security --validate-reports --llm=codex --node-repo=/path/to/node
526+
git node security --validate-reports --llm=codex --llm-model=gpt-5.5
527+
git node security --validate-reports --llm=codex --no-validate-reports-confirm
528+
git node security --validate-reports --llm=codex --no-validate-reports-cache
529+
git node security --validate-reports --llm=claude --node-repo=/path/to/node
530+
git node security --validate-reports --llm=copilot --node-repo=/path/to/node
531+
git node security --validate-reports --llm=copilot --llm-command="copilot -p"
532+
```
533+
534+
By default, the command runs a heuristic pass only. The heuristic checks the
535+
report title, vulnerability information, impact, description, comments, current
536+
severity, CVSS vector, and weakness metadata for common Node.js security topics.
537+
It can identify obvious mismatches, such as a CVSS vector whose calculated
538+
rating does not match the HackerOne rating. Keyword matches are treated only as
539+
topic hints, not as proof that a report is valid or invalid. This is deliberate:
540+
HackerOne report text is reporter-controlled, so words like `request smuggling`
541+
or `permission model` are not enough to make a threat-model decision. The
542+
heuristic output is deliberately conservative and always leaves threat-model
543+
validity as `needs-manual-review`.
544+
545+
Use `--llm=<provider>` to ask an LLM CLI to produce a structured assessment for
546+
each report. Supported providers are `codex`, `claude`, and `copilot`.
547+
548+
When LLM mode is enabled, the command asks before assessing each report and shows
549+
the report title, current severity, CVSS vector, and weakness. After each LLM
550+
assessment, it prints a readable summary with:
551+
552+
- the report URL and title
553+
- the provider and model/cache identity
554+
- validity under the Node.js threat model
555+
- whether the current severity is correct
556+
- current severity/CVSS and suggested severity/CVSS
557+
- a colored CVSS metric diff when the suggested vector differs
558+
- CWE
559+
- confidence from 0 to 100
560+
- threat model/documentation references used by the model
561+
- reasoning
562+
563+
Use `--no-validate-reports-confirm` for batch mode without the per-report
564+
prompts. Use `--validate-reports-limit=<n>` to test the flow against a smaller
565+
number of reports.
566+
567+
#### LLM prompt and reasoning
568+
569+
The LLM prompt is designed to keep the model anchored to the Node.js threat
570+
model and local documentation instead of only reasoning from the report text.
571+
For each report, the prompt instructs the model to:
572+
573+
- read `SECURITY.md` from the Node.js checkout supplied through `--node-repo`
574+
- inspect relevant files under `doc/` for the affected API or subsystem
575+
- apply the documented Node.js threat model, including treatment of application
576+
code, caller-supplied API inputs, third-party modules, unsupported platforms,
577+
and inspector/debugger access
578+
- decide whether the report is valid under that threat model
579+
- decide whether the current HackerOne severity/CVSS is correct
580+
- use reports with the same CWE/weakness as precedent context when available
581+
- avoid copying precedent blindly when `SECURITY.md` or `doc/` point to a
582+
different result
583+
- return only JSON matching the schema expected by the command
584+
585+
The report payload sent to the model contains the HackerOne report id, title,
586+
URL, state, current severity, current CVSS vector, weakness metadata, reporter,
587+
report body fields, comments, heuristic findings, and comparable reports with
588+
the same weakness. Comparable reports include their title, URL, current
589+
severity/CVSS, state, and any team summary available through HackerOne. This
590+
helps the model account for previous team decisions while still checking the
591+
current report against the threat model.
592+
593+
The model must return these fields:
594+
595+
- `validity`: `valid`, `invalid`, or `needs-more-info`
596+
- `severity_correct`: boolean
597+
- `suggested_severity`: `none`, `low`, `medium`, `high`, `critical`, or
598+
`informational`
599+
- `suggested_cvss`: a CVSS vector or `N/A`
600+
- `cwe`: the best matching CWE
601+
- `confidence`: a number from 0 to 100
602+
- `reasoning`: a concise explanation
603+
- `threat_model_references`: references to `SECURITY.md` and relevant `doc/`
604+
material used for the decision
605+
606+
#### LLM commands and cache
607+
608+
The `--llm-command` option can override the default provider command. The prompt
609+
is sent on stdin and the command must print a JSON object matching the expected
610+
schema.
611+
612+
The model label is inferred from the local LLM CLI configuration when possible.
613+
For example, Codex reads `~/.codex/config.toml` and includes
614+
`model_reasoning_effort` in the cache label. Use `--llm-model` to override the
615+
provider command model and the cache identity. If the model cannot be inferred,
616+
the cache entry includes a comment explaining that `default` was used.
617+
618+
Successful LLM assessments are cached locally in
619+
`.ncu-cache/security-report-validation` using the report, provider, model, and
620+
prompt as the cache key. Use `--no-validate-reports-cache` to force a fresh LLM
621+
assessment.
622+
623+
#### Options
624+
625+
| Option | Description |
626+
| --- | --- |
627+
| `--validate-reports-format=markdown\|json` | Select the final output format. Defaults to `markdown`. |
628+
| `--validate-reports-output=<file>` | Write the final output to a file instead of stdout. |
629+
| `--validate-reports-limit=<n>` | Validate at most `n` triaged reports. Useful for testing the flow. |
630+
| `--validate-reports-confirm` | Ask before each LLM assessment and before continuing to the next report. Enabled by default. |
631+
| `--no-validate-reports-confirm` | Disable interactive prompts for batch runs. |
632+
| `--validate-reports-cache` | Reuse cached successful LLM assessments. Enabled by default. |
633+
| `--no-validate-reports-cache` | Ignore existing LLM cache entries and do not reuse them. |
634+
| `--llm=codex\|claude\|copilot` | Ask an LLM CLI to assess each report. |
635+
| `--llm-model=<model>` | Override the provider model and cache identity. |
636+
| `--llm-command=<command>` | Override the command used for LLM assessment. The prompt is sent on stdin. |
637+
| `--node-repo=<path>` | Path to a Node.js checkout containing `SECURITY.md` and `doc/`. Defaults to the current directory. |
638+
509639
## `git node status`
510640

511641
Return status and information about the current git-node land session. Shows the following information:

0 commit comments

Comments
 (0)