Skip to content

Commit a70572d

Browse files
GangGreenTemperTatummonoxgasrdheekonda
authored
chore: add custom system prompt capability to judge (#294)
* chore: add custom system prompt capability to judge * Fixing system prompt for judge * leverage inject system content util function in judge module --------- Co-authored-by: monoxgas <monoxgas@gmail.com> Co-authored-by: Raja Sekhar Rao Dheekonda <raja@dreadnode.io>
1 parent 7a2e80f commit a70572d

1 file changed

Lines changed: 7 additions & 1 deletion

File tree

dreadnode/scorers/judge.py

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ def llm_judge(
4646
min_score: float | None = None,
4747
max_score: float | None = None,
4848
name: str = "llm_judge",
49+
system_prompt: str | None = None,
4950
) -> "Scorer[t.Any]":
5051
"""
5152
Score the output of a task using an LLM to judge it against a rubric.
@@ -60,6 +61,7 @@ def llm_judge(
6061
min_score: Optional minimum score for the judgement - if provided, the score will be clamped to this value.
6162
max_score: Optional maximum score for the judgement - if provided, the score will be clamped to this value.
6263
name: The name of the scorer.
64+
system_prompt: Optional custom system prompt for the judge. If None, uses default.
6365
"""
6466

6567
async def evaluate(
@@ -74,6 +76,7 @@ async def evaluate(
7476
model_params: rg.GenerateParams | AnyDict | None = model_params,
7577
min_score: float | None = min_score,
7678
max_score: float | None = max_score,
79+
system_prompt: str | None = system_prompt,
7780
) -> list[Metric]:
7881
generator: rg.Generator
7982
if isinstance(model, str):
@@ -97,7 +100,10 @@ async def evaluate(
97100
rubric=rubric,
98101
)
99102

100-
judgement = await judge.bind(generator)(input_data)
103+
pipeline = generator.chat([])
104+
if system_prompt:
105+
pipeline.chat.inject_system_content(system_prompt)
106+
judgement = await judge.bind(pipeline)(input_data)
101107

102108
if min_score is not None:
103109
judgement.score = max(min_score, judgement.score)

0 commit comments

Comments
 (0)