Merge pull request #363 from raifdmueller/feat/new-anchor-evals

feat: evaluation specs + results for PERT, GRASP, VSA

Author: rdmueller

evaluations/pilot.py

@@ -295,9 +295,11 @@ def save_results(all_results, out_file):
 def run_pilot(models, dry_run=False, verbose=False, ollama_model="qwen3:4b", no_think=False,
               ollama_url="http://localhost:11434", openai_model="gpt-4o-mini",
               mistral_model="mistral-large-latest", deepseek_model="deepseek-chat",
-              claude_model="claude-sonnet-4-20250514"):
+              claude_model="claude-sonnet-4-20250514", anchor_filter=None):
     start_time = time.time()
     specs = load_specs()
+    if anchor_filter:
+        specs = [s for s in specs if s["anchor"] in anchor_filter]
     print(f"Loaded {len(specs)} anchor specs")
     print(f"Models: {', '.join(models)}")
     print(f"Temperature: {TEMPERATURE}")
@@ -522,6 +524,8 @@ def append_and_save(r):
                         help="Sampling temperature (default: 0.0). Note: claude-cli/claude-haiku ignore this.")
     parser.add_argument("--no-think", action="store_true",
                         help="Disable reasoning/thinking for Ollama models (faster, fewer tokens)")
+    parser.add_argument("--anchor", nargs="+",
+                        help="Only evaluate specific anchors (e.g., --anchor pert grasp)")
     parser.add_argument("--dry-run", action="store_true",
                         help="Show prompts without sending")
     parser.add_argument("--verbose", action="store_true",
@@ -530,4 +534,4 @@ def append_and_save(r):
     set_temperature(args.temperature)
     run_pilot(args.model, args.dry_run, args.verbose, args.ollama_model, args.no_think,
               args.ollama_url, args.openai_model, args.mistral_model, args.deepseek_model,
-              args.claude_model)
+              args.claude_model, args.anchor)

evaluations/report.html

@@ -0,0 +1,18 @@
+anchor: explicit-contract-surface
+tier: 3
+
+questions:
+  recognition:
+    question: 'Which of the following best describes "Explicit Contract Surface"?
+
+      '
+    options:
+      A: A legal framework for defining service-level agreements between microservices
+        with automated compliance monitoring and penalty enforcement
+      B: The practice of exposing only well-defined DTOs and contracts at layer
+        boundaries, never domain entities, with explicit mapping at every seam
+      C: A UI design pattern that makes all system constraints and validation rules
+        visible to users through progressive disclosure and inline feedback
+      D: A versioning strategy that maintains backward compatibility by exposing
+        multiple API versions simultaneously through content negotiation
+    correct: B

evaluations/specs/explicit-contract-surface.yaml

@@ -0,0 +1,39 @@
+anchor: grasp
+tier: 3
+questions:
+  recognition:
+    question: 'Which of the following best describes "GRASP"?
+
+      '
+    options:
+      A: Ensure that a class has only one reason to change by separating concerns
+        and maintaining high cohesion within modules while minimizing dependencies
+        between them
+      B: Assign responsibility to the class that has the information needed to fulfill
+        it; assign responsibility for creating an object to the class that aggregates,
+        contains, or closely uses it
+      C: Define a family of algorithms, encapsulate each one, and make them interchangeable
+        so that the algorithm can vary independently from clients that use it
+      D: Depend on abstractions rather than concretions by inverting the direction
+        of dependencies so that high-level modules do not depend on low-level modules
+    correct: B
+  application:
+    scenario: You're designing an e-commerce system where customers can place orders
+      containing multiple items. The system needs to calculate the total cost of an
+      order, including item prices, taxes, and shipping fees. Currently, the OrderController
+      class is handling all calculations, validation, and persistence logic.
+    anchor_prompt: using GRASP
+    paraphrase_prompt: How should you restructure the responsibility assignments to
+      create a more maintainable and cohesive design?
+    options:
+      A: Keep all calculation logic in OrderController but extract tax and shipping
+        calculations into separate static utility classes to reduce the controller's
+        size
+      B: Move cost calculation to the Order class (which contains the items), create
+        a TaxCalculator service for tax logic, and let OrderItem handle its own price
+        calculations
+      C: Create a single OrderProcessor class that handles all order-related operations
+        including calculations, validation, and persistence to centralize order logic
+      D: Move all calculation logic to the database using stored procedures and have
+        the OrderController only handle data retrieval and UI coordination
+    correct: B

evaluations/specs/grasp.yaml

@@ -0,0 +1,18 @@
+anchor: kiss-principle
+tier: 1
+
+questions:
+  recognition:
+    question: 'Which of the following best describes the "KISS Principle"?
+
+      '
+    options:
+      A: A design principle favoring simplicity over complexity, avoiding over-engineering
+        and unnecessary features in software design
+      B: A testing methodology that focuses on verifying the simplest possible test
+        cases before progressing to complex integration scenarios
+      C: A refactoring pattern that reduces code complexity by extracting common logic
+        into reusable utility functions and shared libraries
+      D: A project management approach that prioritizes delivering minimal viable
+        features with the shortest possible development cycles
+    correct: A

evaluations/specs/kiss-principle.yaml

@@ -0,0 +1,18 @@
+anchor: para-method
+tier: 3
+
+questions:
+  recognition:
+    question: 'Which of the following best describes the "P.A.R.A. Method"?
+
+      '
+    options:
+      A: A software architecture pattern that separates concerns into Presentation,
+        Application, Repository, and Adapter layers
+      B: A knowledge organization framework that categorizes information into Projects,
+        Areas, Resources, and Archive based on actionability
+      C: A prioritization technique that evaluates items by their Probability,
+        Applicability, Risk, and Achievability scores
+      D: A documentation standard that structures technical content into Purpose,
+        Audience, Requirements, and Appendix sections
+    correct: B

evaluations/specs/para-method.yaml

@@ -0,0 +1,36 @@
+anchor: pert
+tier: 2
+questions:
+  recognition:
+    question: 'Which of the following best describes "PERT"?
+
+      '
+    options:
+      A: An iterative requirements prioritization framework that categorizes features
+        into must-have, should-have, could-have, and won't-have buckets
+      B: A stochastic project scheduling method using three-point estimation (optimistic,
+        most likely, pessimistic) to calculate expected duration and variance for
+        activities in a network diagram
+      C: A process improvement methodology that uses statistical control charts to
+        monitor and reduce variation in manufacturing and software delivery processes
+      D: A stakeholder mapping technique that visualizes the impact and influence
+        of different actors to prioritize engagement strategies
+    correct: B
+  application:
+    scenario: Your team is planning a critical software release with several interdependent
+      features. The database migration task has estimates of 2 days (optimistic),
+      5 days (most likely), and 14 days (pessimistic). The API development task estimates
+      are 3 days (optimistic), 7 days (most likely), and 11 days (pessimistic).
+    anchor_prompt: using PERT
+    paraphrase_prompt: What should be the expected duration estimates for these two
+      tasks when accounting for uncertainty in a probabilistic scheduling approach?
+    options:
+      A: 'Database migration: 7 days, API development: 7 days (using simple averages
+        of the three estimates)'
+      B: 'Database migration: 6 days, API development: 7 days (using weighted average
+        formula with most likely estimate weighted 4x)'
+      C: 'Database migration: 5 days, API development: 7 days (using the most likely
+        estimates as the expected values)'
+      D: 'Database migration: 8 days, API development: 8.5 days (using average of
+        optimistic and pessimistic estimates plus buffer)'
+    correct: B

evaluations/specs/pert.yaml

@@ -0,0 +1,18 @@
+anchor: spec-driven-development
+tier: 3
+
+questions:
+  recognition:
+    question: 'Which of the following best describes "Spec-Driven Development"?
+
+      '
+    options:
+      A: An approach where software is defined, constrained, and validated through
+        explicit specifications, with specs as the source of truth rather than code
+      B: A testing methodology that generates test cases automatically from formal
+        mathematical specifications of expected behavior
+      C: A documentation practice that requires detailed technical specifications
+        to be written and approved before any implementation begins
+      D: A deployment strategy that uses infrastructure specifications to provision
+        and configure environments automatically
+    correct: A

evaluations/specs/spec-driven-development.yaml

@@ -0,0 +1,46 @@
+anchor: vertical-slice-architecture
+tier: 3
+questions:
+  recognition:
+    question: 'Which of the following best describes "Vertical Slice Architecture
+      (VSA)"?
+
+      '
+    options:
+      A: Code is organized into distinct horizontal layers (presentation, business,
+        data) with clear separation of concerns and dependencies flowing downward
+        through each technical layer
+      B: Each feature is organized as an end-to-end slice spanning all layers (request,
+        validation, domain logic, persistence, API); code for a feature lives together
+        regardless of technical layer
+      C: The application core is isolated from external dependencies through ports
+        and adapters, with business logic at the center surrounded by interface adapters
+        that connect to external systems
+      D: Commands and queries are separated into distinct models and handlers, with
+        write operations handled differently from read operations to optimize for
+        different access patterns and scalability requirements
+    correct: B
+  application:
+    scenario: Your team is building an e-commerce platform with features like user
+      registration, product catalog, shopping cart, order processing, and payment
+      handling. Currently, the codebase is organized into traditional layers (Controllers,
+      Services, Repositories) but developers are experiencing frequent merge conflicts
+      and find it difficult to work on features independently. Changes to one feature
+      often require modifications across multiple layers, slowing down development.
+    anchor_prompt: using Vertical Slice Architecture (VSA)
+    paraphrase_prompt: How should you reorganize the codebase to enable independent
+      feature development and reduce cross-feature coupling?
+    options:
+      A: Create separate microservices for each feature (UserService, ProductService,
+        CartService, etc.) with their own databases and API gateways to ensure complete
+        isolation between features.
+      B: Organize code by feature slices where each feature (Registration, Catalog,
+        Cart, Orders, Payments) contains its own request handlers, validation, business
+        logic, and data access in a single vertical slice.
+      C: Implement a strict layered architecture with clear interfaces between Controller,
+        Service, and Repository layers, using dependency injection to manage cross-layer
+        dependencies more effectively.
+      D: Apply Domain-Driven Design by creating bounded contexts for each business
+        domain and organizing code into aggregate roots with separate application
+        services for each domain.
+    correct: B