Merge branch 'main' into kaylieee/release/2.1.0

Author: kaylieee

.github/skills/azsdk-common-apiview-feedback-resolution/SKILL.md

@@ -5,8 +5,7 @@ metadata:
   version: "1.0.0"
   distribution: shared
 description: "Analyze and resolve APIView review feedback on Azure SDK PRs. **UTILITY SKILL**. USE FOR: APIView comments, API review feedback, SDK API surface changes. DO NOT USE FOR: general code review, non-APIView feedback. INVOKES: azure-sdk-mcp:azsdk_apiview_get_comments, azure-sdk-mcp:azsdk_customized_code_update."
-compatibility:
-  requires: "azure-sdk-mcp server, SDK pull request with APIView review link"
+compatibility: "azure-sdk-mcp server, SDK pull request with APIView review link"
 ---
 
 # APIView Feedback Resolution

.github/skills/azsdk-common-apiview-feedback-resolution/eval.yaml

@@ -1,29 +1,82 @@
-name: azsdk-common-apiview-feedback-resolution-evals
-description: Evaluations for the APIView feedback resolution skill
-skill: azsdk-common-apiview-feedback-resolution
-version: "1.0"
+name: azsdk-common-apiview-feedback-resolution-eval
+description: Evaluation suite for the azsdk-common-apiview-feedback-resolution skill
+type: capability
+
+environment: azsdk-mcp-mock
+
+tags:
+  area: azsdk-common-apiview-feedback-resolution
+  type: ci-gate
+
 config:
-  model: gpt-4o
-  executor: mock
-  trials_per_task: 1
-  timeout_seconds: 300
-  parallel: false
-metrics:
-  - name: task_completion
-    weight: 0.7
-    threshold: 0.8
-    description: Did the skill complete trigger and anti-trigger checks?
-  - name: efficiency
-    weight: 0.3
-    threshold: 0.7
-    description: Did the skill stay within behavior limits?
-graders:
-  - type: text
-    name: no_fatal_errors
-    config:
-      regex_not_match:
-        - (?i)fatal error
-        - (?i)unhandled exception
-        - '(?i)panic:'
-tasks:
-  - "tasks/*.yaml"
+  runs: 1
+  timeout: "90s"
+  model: claude-opus-4.6
+  executor: copilot-sdk
+
+scoring:
+  threshold: 0.8
+
+stimuli:
+  - name: apiview-basic-001
+    prompt: "My Python SDK PR has APIView comments that I need to address. Can you help me retrieve and resolve them?"
+    graders:
+      - type: tool-calls
+        config:
+          required:
+            - name: azure-sdk-mcp-azsdk_apiview_get_comments
+
+  - name: apiview-edge-001
+    prompt: "An APIView reviewer says I need to rename 'ComputeProvisioningState' to 'ProvisioningState' in my SDK. Use the SDK customization tools to apply this rename."
+    graders:
+      - type: output-not-contains
+        config:
+          substring: "cannot help"
+
+  - name: apiview-no-feedback-001
+    prompt: "Check if there are any APIView comments on my .NET SDK PR that I need to address before release."
+    graders:
+      - type: output-contains
+        config:
+          substring: "APIView"
+
+  - name: apiview-negative-001
+    prompt: "I need to create a new TypeSpec project for my Azure Storage service."
+    graders:
+      - type: tool-calls
+        config:
+          disallowed:
+            - name: azure-sdk-mcp-azsdk_apiview_get_comments
+      - type: output-not-contains
+        config:
+          substring: "APIView"
+
+  - name: basic-trigger
+    prompt: "I have APIView comments on my SDK PR that need to be resolved. Can you help?"
+    graders:
+      - type: output-contains
+        config:
+          substring: "APIView"
+
+  - name: anti-trigger
+    prompt: "How do I set up a new Azure Storage account?"
+    graders:
+      - type: output-not-contains
+        config:
+          substring: "APIView"
+      - type: output-not-contains
+        config:
+          substring: "API review"
+      - type: output-not-contains
+        config:
+          substring: "apiview.dev"
+      - type: output-not-contains
+        config:
+          substring: "apiviewstagingtest.com"
+
+  - name: url-trigger
+    prompt: "I have review comments on my SDK PR from this page: https://spa.apiview.dev/review/abc123?activeApiRevisionId=def456 — can you help me resolve them?"
+    graders:
+      - type: output-contains
+        config:
+          substring: "apiview"

.github/skills/azsdk-common-apiview-feedback-resolution/evals/eval.yaml

@@ -0,0 +1,99 @@
+name: azsdk-common-apiview-feedback-resolution-trigger-eval
+description: Trigger and anti-trigger tests for the azsdk-common-apiview-feedback-resolution skill
+type: capability
+
+environment: azsdk-mcp-mock
+
+tags:
+  area: azsdk-common-apiview-feedback-resolution
+  type: ci-gate
+
+config:
+  runs: 1
+  timeout: "90s"
+  model: claude-opus-4.6
+  executor: copilot-sdk
+
+scoring:
+  threshold: 0.8
+
+stimuli:
+  - name: trigger-resolve-apiview-feedback-on-my-pr
+    prompt: "resolve APIView feedback on my PR"
+    graders:
+      - type: skill-invocation
+        config:
+          required: ["azsdk-common-apiview-feedback-resolution"]
+  - name: trigger-fix-apiview-comments
+    prompt: "fix APIView comments"
+    graders:
+      - type: skill-invocation
+        config:
+          required: ["azsdk-common-apiview-feedback-resolution"]
+  - name: trigger-address-api-review-feedback
+    prompt: "address API review feedback"
+    graders:
+      - type: skill-invocation
+        config:
+          required: ["azsdk-common-apiview-feedback-resolution"]
+  - name: trigger-apiview-shows-issues-with-my-sdk
+    prompt: "APIView shows issues with my SDK"
+    graders:
+      - type: skill-invocation
+        config:
+          required: ["azsdk-common-apiview-feedback-resolution"]
+  - name: trigger-help-me-with-apiview-suggestions
+    prompt: "help me with APIView suggestions"
+    graders:
+      - type: skill-invocation
+        config:
+          required: ["azsdk-common-apiview-feedback-resolution"]
+  - name: trigger-fix-comments-apiview-url-1
+    prompt: "Help me fix these comments: https://spa.apiview.dev/review/0qw98r9qwur9q823r?activeApiRevisionId=afiq948tquajf8jq9f"
+    graders:
+      - type: skill-invocation
+        config:
+          required: ["azsdk-common-apiview-feedback-resolution"]
+  - name: trigger-address-feedback-apiview-url-2
+    prompt: "Address this feedback: https://spa.apiview.dev/review/akldfja9qjta498tjaejg0?activeApiRevisionId=odiufa9sefighj98439"
+    graders:
+      - type: skill-invocation
+        config:
+          required: ["azsdk-common-apiview-feedback-resolution"]
+  - name: trigger-resolve-comments-staging-url
+    prompt: "Resolve these comments: https://apiviewstagingtest.com/review/akldfja9qjta498tjaejg0?activeApiRevisionId=odiufa9sefighj98439"
+    graders:
+      - type: skill-invocation
+        config:
+          required: ["azsdk-common-apiview-feedback-resolution"]
+
+  - name: anti-trigger-deploy-my-azure-function
+    prompt: "deploy my Azure Function"
+    graders:
+      - type: skill-invocation
+        config:
+          disallowed: ["azsdk-common-apiview-feedback-resolution"]
+  - name: anti-trigger-fix-my-pipeline-build-error
+    prompt: "fix my pipeline build error"
+    graders:
+      - type: skill-invocation
+        config:
+          disallowed: ["azsdk-common-apiview-feedback-resolution"]
+  - name: anti-trigger-create-a-new-typespec-definition
+    prompt: "create a new TypeSpec definition"
+    graders:
+      - type: skill-invocation
+        config:
+          disallowed: ["azsdk-common-apiview-feedback-resolution"]
+  - name: anti-trigger-how-do-i-release-an-sdk-package
+    prompt: "how do I release an SDK package"
+    graders:
+      - type: skill-invocation
+        config:
+          disallowed: ["azsdk-common-apiview-feedback-resolution"]
+  - name: anti-trigger-set-up-azure-storage-account
+    prompt: "set up Azure Storage account"
+    graders:
+      - type: skill-invocation
+        config:
+          disallowed: ["azsdk-common-apiview-feedback-resolution"]