Merge pull request #30 from naaa760/feat/feasibility-validator-selection

feat: make feasibility agent choose validators from catalog

Author: dkargatzis

src/agents/feasibility_agent/agent.py

@@ -87,6 +87,7 @@ async def execute(self, rule_description: str) -> AgentResult:
                     "is_feasible": result.is_feasible,
                     "yaml_content": result.yaml_content,
                     "confidence_score": result.confidence_score,
+                    "chosen_validators": result.chosen_validators,
                     "rule_type": result.rule_type,
                     "analysis_steps": result.analysis_steps,
                 },

src/agents/feasibility_agent/models.py

@@ -10,6 +10,10 @@ class FeasibilityAnalysis(BaseModel):
 
     is_feasible: bool = Field(description="Whether the rule is feasible to implement with Watchflow")
     rule_type: str = Field(description="Type of rule (time_restriction, branch_pattern, title_pattern, etc.)")
+    chosen_validators: list[str] = Field(
+        description="Names of validators from the catalog that can implement this rule",
+        default_factory=list,
+    )
     confidence_score: float = Field(description="Confidence score from 0.0 to 1.0", ge=0.0, le=1.0)
     feedback: str = Field(description="Detailed feedback on implementation considerations")
     analysis_steps: list[str] = Field(description="Step-by-step analysis breakdown", default_factory=list)
@@ -30,4 +34,5 @@ class FeasibilityState(BaseModel):
     feedback: str = ""
     confidence_score: float = 0.0
     rule_type: str = ""
+    chosen_validators: list[str] = Field(default_factory=list)
     analysis_steps: list[str] = Field(default_factory=list)

src/agents/feasibility_agent/nodes.py

@@ -7,6 +7,7 @@
 from src.agents.feasibility_agent.models import FeasibilityAnalysis, FeasibilityState, YamlGeneration
 from src.agents.feasibility_agent.prompts import RULE_FEASIBILITY_PROMPT, YAML_GENERATION_PROMPT
 from src.integrations.providers import get_chat_model
+from src.rules.validators import get_validator_descriptions
 
 logger = logging.getLogger(__name__)
 
@@ -23,15 +24,30 @@ async def analyze_rule_feasibility(state: FeasibilityState) -> FeasibilityState:
         # Use structured output instead of manual JSON parsing
         structured_llm = llm.with_structured_output(FeasibilityAnalysis)
 
-        # Analyze rule feasibility
-        prompt = RULE_FEASIBILITY_PROMPT.format(rule_description=state.rule_description)
+        # Build validator catalog text for the prompt
+        validator_catalog = []
+        for v in get_validator_descriptions():
+            validator_catalog.append(
+                f"- name: {v.get('name')}\n"
+                f"  event_types: {v.get('event_types')}\n"
+                f"  parameter_patterns: {v.get('parameter_patterns')}\n"
+                f"  description: {v.get('description')}"
+            )
+        validators_text = "\n".join(validator_catalog)
+
+        # Analyze rule feasibility with awareness of available validators
+        prompt = RULE_FEASIBILITY_PROMPT.format(
+            rule_description=state.rule_description,
+            validator_catalog=validators_text,
+        )
 
         # Get structured response with retry logic
         result = await structured_llm.ainvoke(prompt)
 
         # Update state with analysis results - now type-safe!
         state.is_feasible = result.is_feasible
         state.rule_type = result.rule_type
+        state.chosen_validators = result.chosen_validators
         state.confidence_score = result.confidence_score
         state.feedback = result.feedback
         state.analysis_steps = result.analysis_steps
@@ -67,7 +83,11 @@ async def generate_yaml_config(state: FeasibilityState) -> FeasibilityState:
         # Use structured output for YAML generation
         structured_llm = llm.with_structured_output(YamlGeneration)
 
-        prompt = YAML_GENERATION_PROMPT.format(rule_type=state.rule_type, rule_description=state.rule_description)
+        prompt = YAML_GENERATION_PROMPT.format(
+            rule_type=state.rule_type,
+            rule_description=state.rule_description,
+            chosen_validators=", ".join(state.chosen_validators),
+        )
 
         # Get structured response with retry logic
         result = await structured_llm.ainvoke(prompt)

src/agents/feasibility_agent/prompts.py

@@ -3,71 +3,44 @@
 """
 
 RULE_FEASIBILITY_PROMPT = """
-You are an expert in Watchflow rules and GitHub automation. Analyze whether a natural language rule description is feasible to implement using Watchflow.
+You are an expert in Watchflow rules and GitHub automation. Analyze whether a natural language rule description is feasible to implement using Watchflow’s existing validator catalog. Do NOT invent custom logic; choose from the provided validators. If none fit, mark as not feasible.
 
-Rule Description: {rule_description}
+Rule Description:
+{rule_description}
 
-Analyze this rule and determine:
-1. Is it feasible to implement with Watchflow's rule system?
-2. What type of rule is it?
-3. Provide concise feedback on implementation considerations
+Available validators (name, event_types, parameter_patterns, description):
+{validator_catalog}
 
-Available rule types:
-- label_requirement: Rules requiring specific labels
-- time_restriction: Rules about when actions can occur (weekends, hours, days)
-- approval_requirement: Rules about required approvals
-- title_pattern: Rules about PR title formatting
-- branch_pattern: Rules about branch naming conventions
-- file_size: Rules about file size limits
-- commit_message: Rules about commit message format
-- branch_protection: Rules about protected branches
-
-Focus on:
-- Practical implementation with Watchflow
-- Key configuration considerations
-- Severity and enforcement level recommendations
-- Keep feedback under 150 words
+Decide:
+1) is_feasible (true/false)
+2) rule_type (short label you infer)
+3) chosen_validators (list of validator names from the catalog that can implement this rule; empty if not feasible)
+4) feedback (practical, under 120 words)
+5) analysis_steps (succinct bullets)
 """
 
 YAML_GENERATION_PROMPT = """
-Generate a complete Watchflow rule configuration for the following rule:
+Generate a complete Watchflow rules.yaml for the rule below using ONLY the selected validators. Do not introduce parameters that the chosen validators do not support.
 
 Rule Type: {rule_type}
 Description: {rule_description}
+Chosen Validators: {chosen_validators}
 
-Generate a complete rules.yaml file that follows this EXACT structure:
-
+Rules YAML format:
 ```yaml
 rules:
-  - description: "Clear description of what this rule does"
+  - description: "<concise description>"
     enabled: true
     severity: "medium"
     event_types: ["pull_request"]
     parameters:
-      required_labels: ["security", "review"]
+      <validator-appropriate-parameters>
 ```
 
-IMPORTANT REQUIREMENTS:
-- Generate the COMPLETE rules.yaml file including the "rules:" wrapper
-- Use the rule description as the primary identifier
-- Include enabled: true (allows rule activation/deactivation)
-- Set appropriate severity (low, medium, high, critical)
-- Include relevant event_types
-- Add correct parameters based on rule type
-- For regex patterns, use single quotes to avoid YAML parsing issues
-
-Rule type parameters:
-- label_requirement: use "required_labels" with array of labels
-- time_restriction: use "days" for restricted days or "allowed_hours" for restricted hours
-- approval_requirement: use "min_approvals" with number
-- title_pattern: use "title_pattern" with regex pattern (use single quotes for regex)
-- file_size: use "max_file_size_mb" with number
-- commit_message: use "pattern" with regex pattern (use single quotes for regex)
-- branch_protection: use "protected_branches" with array of branch names
-
-Examples of proper regex patterns:
-- title_pattern: '^feat|^fix|^docs'  # Use single quotes
-- pattern: '^[A-Z]+-\\d+'  # Use single quotes for regex
-
-Return the complete YAML file content.
+Guidelines:
+- Keep severity appropriate (low/medium/high/critical).
+- event_types must align with the validators chosen.
+- For regex, use single quotes.
+- If no validators fit, return an empty yaml_content.
+Return only the YAML content.
 """