code_evaluation.py

from deepeval import evaluate
from deepeval.metrics import GEval
from deepeval.test_case import LLMTestCase, LLMTestCaseParams
from deepeval.metrics.g_eval import Rubric
from typing import Dict, Any

def evaluate_code(generated_code: str, reference_code: str = None):
    try:
        # Initialize test case
        test_case = LLMTestCase(
            input="Code Generation Task",
            actual_output=generated_code,
            expected_output=reference_code if reference_code else ""
        )

        # Code Correctness Metric
        correctness_metric = GEval(
            name="Code Correctness",
            criteria="Evaluate if the code is functionally correct, properly handles edge cases, and implements the required functionality completely.",
            evaluation_steps=[
                "Check if the code implements all required functionality",
                "Verify proper handling of edge cases",
                "Check for potential runtime errors",
                "Assess if the code produces expected outputs"
            ],
            evaluation_params=[LLMTestCaseParams.ACTUAL_OUTPUT, LLMTestCaseParams.EXPECTED_OUTPUT],
            rubric=[
                Rubric(score_range=(0,2), expected_outcome="Code is non-functional or has critical errors"),
                Rubric(score_range=(3,5), expected_outcome="Code works but misses key functionality"),
                Rubric(score_range=(6,8), expected_outcome="Code is mostly correct with minor issues"),
                Rubric(score_range=(9,10), expected_outcome="Code is completely correct")
            ],
            threshold=0.7
        )

        # Code Readability Metric
        readability_metric = GEval(
            name="Code Readability",
            criteria="Evaluate code readability including proper naming, formatting, and documentation.",
            evaluation_steps=[
                "Check for clear and consistent naming conventions",
                "Verify proper code formatting and indentation",
                "Assess quality and completeness of comments and docstrings",
                "Check for code organization and logical structure"
            ],
            evaluation_params=[LLMTestCaseParams.ACTUAL_OUTPUT],
            rubric=[
                Rubric(score_range=(0,2), expected_outcome="Code is poorly formatted and hard to read"),
                Rubric(score_range=(3,5), expected_outcome="Code has basic formatting but lacks clarity"),
                Rubric(score_range=(6,8), expected_outcome="Code is well formatted with minor issues"),
                Rubric(score_range=(9,10), expected_outcome="Code is exceptionally readable and well documented")
            ],
            threshold=0.7
        )

        # Code Best Practices Metric
        best_practices_metric = GEval(
            name="Code Best Practices",
            criteria="Evaluate adherence to coding best practices, including error handling, security, and efficiency.",
            evaluation_steps=[
                "Check for proper error handling and exceptions",
                "Verify security best practices",
                "Assess code efficiency and performance considerations",
                "Check for code reusability and modularity"
            ],
            evaluation_params=[LLMTestCaseParams.ACTUAL_OUTPUT],
            rubric=[
                Rubric(score_range=(0,2), expected_outcome="Code ignores best practices"),
                Rubric(score_range=(3,5), expected_outcome="Code follows basic practices with gaps"),
                Rubric(score_range=(6,8), expected_outcome="Code mostly follows best practices"),
                Rubric(score_range=(9,10), expected_outcome="Code perfectly follows all best practices")
            ],
            threshold=0.7
        )

        # Run evaluation
        metrics = [correctness_metric, readability_metric, best_practices_metric]
        for metric in metrics:
            metric.measure(test_case)

        # Calculate overall score
        overall_score = (correctness_metric.score + readability_metric.score + best_practices_metric.score) / 3

        # Prepare detailed metrics
        detailed_metrics = {
            "correctness": {
                "score": correctness_metric.score,
                "reason": correctness_metric.reason
            },
            "readability": {
                "score": readability_metric.score,
                "reason": readability_metric.reason
            },
            "best_practices": {
                "score": best_practices_metric.score,
                "reason": best_practices_metric.reason
            }
        }

        return {
            "overall_score": overall_score,
            "detailed_metrics": detailed_metrics,
            "passed": overall_score >= 0.7 
        }

    except Exception as e:
        return {
            "error": f"Error evaluating code: {str(e)}",
            "overall_score": 0.0,
            "detailed_metrics": {},
            "passed": False
        }