diff --git a/src/reviewer/cli.py b/src/reviewer/cli.py index f4b49ad..d90f4ac 100644 --- a/src/reviewer/cli.py +++ b/src/reviewer/cli.py @@ -188,6 +188,7 @@ def _build_paper_json( "explanation": c.explanation, "comment_type": c.comment_type, "paragraph_index": c.paragraph_index, + "severity": c.severity, }) model_short = _model_short_name(result.model) if result.model else "" diff --git a/src/reviewer/models.py b/src/reviewer/models.py index 671bbc0..b00e9bb 100644 --- a/src/reviewer/models.py +++ b/src/reviewer/models.py @@ -2,6 +2,8 @@ from dataclasses import dataclass, field +SEVERITY_VALUES = ("minor", "moderate", "major") + @dataclass class Comment: @@ -11,6 +13,7 @@ class Comment: explanation: str # reviewer's explanation comment_type: str # "technical" or "logical" paragraph_index: int | None = None # 0-based index in split paragraphs + severity: str | None = None # one of SEVERITY_VALUES, or None when unknown def to_dict(self) -> dict: d = { @@ -21,6 +24,8 @@ def to_dict(self) -> dict: } if self.paragraph_index is not None: d["paragraph_index"] = self.paragraph_index + if self.severity: + d["severity"] = self.severity return d diff --git a/src/reviewer/prompts.py b/src/reviewer/prompts.py index ea7d7d7..d46425d 100644 --- a/src/reviewer/prompts.py +++ b/src/reviewer/prompts.py @@ -48,6 +48,12 @@ DO_NOT_FLAG_CHUNKED = DO_NOT_FLAG_BASE.rstrip() + """ - Incomplete text at passage boundaries""" +SEVERITY_RUBRIC = """\ +For each issue, assign a "severity" tier: +- "major": undermines a key claim, methodology, or comparison; affects conclusions +- "moderate": real error or gap, but localized and fixable +- "minor": framing concern, mild overclaim, or ambiguity resolvable from context""" + DO_NOT_FLAG_PROGRESSIVE = DO_NOT_FLAG_CHUNKED.rstrip() + """ - Notation not yet in the summary — it may be introduced later""" @@ -57,6 +63,7 @@ - "quote": the exact verbatim text (preserving LaTeX) - "explanation": deep reasoning — what you initially thought, whether context resolves it, and what specifically remains problematic - "type": "technical" or "logical" +- "severity": "minor", "moderate", or "major" """ JSON_OBJECT_OUTPUT = """\ @@ -99,6 +106,8 @@ {DO_NOT_FLAG_CHUNKED} +{SEVERITY_RUBRIC} + {JSON_ARRAY_OUTPUT}""" DEEP_CHECK_PROGRESSIVE_PROMPT = f"""{REVIEWER_PREAMBLE} @@ -149,6 +158,8 @@ {DO_NOT_FLAG_BASE} +{SEVERITY_RUBRIC} + Return a JSON object with this structure: {{{{ "overall_feedback": "One paragraph high-level assessment of the paper's quality and main issues", @@ -157,7 +168,8 @@ "title": "short descriptive title of the issue", "quote": "the exact verbatim text from the paper containing the issue (copy it exactly, preserving LaTeX)", "explanation": "deep reasoning — what you initially thought, whether context resolves it, and what specifically remains problematic", - "type": "technical" or "logical" + "type": "technical" or "logical", + "severity": "minor" or "moderate" or "major" }}}} ] }}}} @@ -194,6 +206,8 @@ {DO_NOT_FLAG_CHUNKED} +{SEVERITY_RUBRIC} + Return a JSON object: {{{{ "overall_feedback": "brief assessment of this section", @@ -202,7 +216,8 @@ "title": "short descriptive title of the issue", "quote": "the exact verbatim text from the paper containing the issue (copy it exactly, preserving LaTeX)", "explanation": "deep reasoning — what you initially thought, whether context resolves it, and what specifically remains problematic", - "type": "technical" or "logical" + "type": "technical" or "logical", + "severity": "minor" or "moderate" or "major" }}}} ] }}}} diff --git a/src/reviewer/utils.py b/src/reviewer/utils.py index ec62cce..347e926 100644 --- a/src/reviewer/utils.py +++ b/src/reviewer/utils.py @@ -6,7 +6,7 @@ import tiktoken -from .models import Comment +from .models import Comment, SEVERITY_VALUES def _get_encoding(): @@ -205,12 +205,18 @@ def parse_comments_from_list(items: list[dict]) -> list[Comment]: paragraph_index = item.get("paragraph_index", None) if paragraph_index is not None: paragraph_index = int(paragraph_index) + severity = item.get("severity") + if isinstance(severity, str): + severity = severity.strip().lower() + if severity not in SEVERITY_VALUES: + severity = "moderate" comments.append(Comment( title=title, quote=quote, explanation=explanation, comment_type=comment_type, paragraph_index=paragraph_index, + severity=severity, )) return comments