diff --git a/.gitignore b/.gitignore index 364b391a012..78aff7a1e3d 100644 --- a/.gitignore +++ b/.gitignore @@ -35,6 +35,9 @@ docs/_site/ logs *.log +# Oncall incident logs +incidents/ + # Vite development .vite-port diff --git a/.roo/commands/k8s-troubleshoot.md b/.roo/commands/k8s-troubleshoot.md new file mode 100644 index 00000000000..0e7548d7d9a --- /dev/null +++ b/.roo/commands/k8s-troubleshoot.md @@ -0,0 +1,8 @@ +--- +description: "Run Kubernetes troubleshooting workflow" +mode: oncall +--- + +Execute the Kubernetes troubleshooting workflow from `.roo/rules-oncall/k8s-troubleshooting.md`. + +Read the workflow file and follow it step-by-step to troubleshoot the Kubernetes issue. diff --git a/.roo/commands/oncall-workflow.md b/.roo/commands/oncall-workflow.md new file mode 100644 index 00000000000..31c50b0dc32 --- /dev/null +++ b/.roo/commands/oncall-workflow.md @@ -0,0 +1,16 @@ +--- +description: "Run a specific oncall workflow" +argument-hint: "[workflow-name]" +mode: oncall +--- + +Execute an oncall workflow. The workflow name is provided as an argument. + +Read and execute the workflow file from `.roo/rules-oncall/{workflow-name}.md` step-by-step. + +If no workflow name is provided, list available workflows in `.roo/rules-oncall/` and ask the user which one to execute. + +Examples: + +- `/oncall-workflow k8s-troubleshooting` - Execute `k8s-troubleshooting.md` +- `/oncall-workflow service-specific/example-service-runbook` - Execute `service-specific/example-service-runbook.md` diff --git a/.roo/rules-oncall/1_workflow_overview.md b/.roo/rules-oncall/1_workflow_overview.md new file mode 100644 index 00000000000..b2bf48d7a54 --- /dev/null +++ b/.roo/rules-oncall/1_workflow_overview.md @@ -0,0 +1,36 @@ +# Oncall Workflow Overview + +This directory contains workflow files that are automatically loaded when using the Oncall mode. These workflows provide structured guidance for handling various oncall scenarios. + +## Adding Your Workflows + +Workflows should be placed in `.roo/rules-oncall/`. Create markdown files for each workflow you need: + +- Incident response procedures +- Page handling workflows +- Escalation procedures +- Service-specific runbooks (in `service-specific/` subdirectory) + +## Workflow Structure + +Each workflow should follow a clear structure: + +1. **Initial Assessment** - Understand the situation +2. **Action Plan** - Determine next steps +3. **Execution** - Follow the plan using MCP tools +4. **Documentation** - Record actions taken +5. **Resolution** - Confirm resolution and update status + +## Using Workflows + +When handling an oncall task: + +1. Identify which workflow applies to your situation +2. Read the workflow file from `.roo/rules-oncall/` +3. Follow the steps sequentially +4. Use MCP tools as specified in the workflow +5. Document all actions in incident logs + +## MCP Tools + +Workflows can reference MCP tools using the `use_mcp_tool` pattern. Configure your MCP servers and reference them in your workflow files. diff --git a/.roo/rules-oncall/k8s-troubleshooting.md b/.roo/rules-oncall/k8s-troubleshooting.md new file mode 100644 index 00000000000..6d13f67dc62 --- /dev/null +++ b/.roo/rules-oncall/k8s-troubleshooting.md @@ -0,0 +1,303 @@ +# Kubernetes Troubleshooting Workflow + +This workflow guides you through troubleshooting Kubernetes-related incidents. + +## Step 1: Assess the Situation + +### 1.1 Gather Initial Information + +- Identify which namespace/service is affected +- Check if it's a pod, deployment, service, or ingress issue +- Determine the severity and user impact + +**Actions:** + +- Use MCP tools to query cluster status +- Check recent deployments or changes +- Review alert details + +### 1.2 Check Pod Status + +Use MCP tools or commands to check pod status: + +**MCP Actions:** + +- Use `k8s` MCP server `list_pods` to see pod status +- Use `k8s` MCP server `get_pod_logs` to check recent logs +- Use `k8s` MCP server `describe_pod` for detailed pod information + +**Manual Commands (if MCP not available):** + +```bash +kubectl get pods -n +kubectl describe pod -n +kubectl logs -n --tail=100 +``` + +## Step 2: Diagnose the Issue + +### 2.1 Check Pod States + +Identify the pod state: + +- **Pending**: Pod can't be scheduled +- **CrashLoopBackOff**: Pod keeps crashing +- **ImagePullBackOff**: Can't pull container image +- **Running but unhealthy**: Pod running but failing health checks +- **Terminating**: Pod stuck in termination + +### 2.2 Common Issues and Solutions + +#### Issue: Pod in CrashLoopBackOff + +**Diagnosis:** + +1. Check pod logs for errors +2. Review container exit codes +3. Check resource limits +4. Verify environment variables and configs + +**Resolution:** + +- Fix application errors in logs +- Adjust resource requests/limits if OOMKilled +- Fix configuration issues +- Check for missing dependencies or secrets + +**MCP Actions:** + +- Use `k8s` MCP `get_pod_logs` to view crash logs +- Use `k8s` MCP `get_pod_events` to see recent events + +#### Issue: ImagePullBackOff + +**Diagnosis:** + +1. Check if image exists and is accessible +2. Verify image pull secrets +3. Check network connectivity to registry + +**Resolution:** + +- Verify image tag exists +- Add/update imagePullSecrets if needed +- Check registry authentication +- Verify network policies allow registry access + +**MCP Actions:** + +- Use `k8s` MCP `describe_pod` to see image pull errors +- Use `k8s` MCP `list_secrets` to check image pull secrets + +#### Issue: Pod Pending + +**Diagnosis:** + +1. Check node resources (CPU, memory) +2. Review node selectors and affinity rules +3. Check for taints and tolerations +4. Verify persistent volume claims + +**Resolution:** + +- Scale cluster if resources exhausted +- Adjust node selectors/affinity +- Add tolerations if needed +- Fix PVC issues + +**MCP Actions:** + +- Use `k8s` MCP `describe_pod` to see scheduling events +- Use `k8s` MCP `get_nodes` to check node resources +- Use `k8s` MCP `get_pvc` to check volume claims + +#### Issue: Service Not Accessible + +**Diagnosis:** + +1. Check service endpoints +2. Verify service selector matches pod labels +3. Check ingress configuration +4. Review network policies + +**Resolution:** + +- Fix label mismatches +- Update service selectors +- Fix ingress rules +- Adjust network policies + +**MCP Actions:** + +- Use `k8s` MCP `get_service_endpoints` to check endpoints +- Use `k8s` MCP `describe_service` for service details +- Use `k8s` MCP `get_ingress` to check ingress rules + +## Step 3: Check Resource Constraints + +### 3.1 Resource Limits + +Check if pods are hitting resource limits: + +**MCP Actions:** + +- Use `k8s` MCP `get_pod_metrics` to see current usage +- Use `k8s` MCP `describe_pod` to check limits/requests + +**Manual Commands:** + +```bash +kubectl top pod -n +kubectl describe pod -n | grep -A 5 "Limits\|Requests" +``` + +### 3.2 Node Resources + +Check cluster-wide resource availability: + +**MCP Actions:** + +- Use `k8s` MCP `get_node_metrics` to see node usage +- Use `k8s` MCP `get_nodes` to check allocatable resources + +## Step 4: Check Dependencies + +### 4.1 ConfigMaps and Secrets + +Verify required configs and secrets exist: + +**MCP Actions:** + +- Use `k8s` MCP `get_configmap` to check configs +- Use `k8s` MCP `get_secret` to verify secrets + +**Manual Commands:** + +```bash +kubectl get configmap -n +kubectl get secret -n +``` + +### 4.2 Service Dependencies + +Check if dependent services are running: + +**MCP Actions:** + +- Use `k8s` MCP `list_services` to see all services +- Use `k8s` MCP `get_service_endpoints` to verify endpoints + +## Step 5: Review Recent Changes + +### 5.1 Check Deployment History + +Look for recent changes that might have caused the issue: + +**MCP Actions:** + +- Use `k8s` MCP `get_deployment_history` to see rollout history +- Use `k8s` MCP `get_events` to see recent cluster events + +**Manual Commands:** + +```bash +kubectl rollout history deployment/ -n +kubectl get events -n --sort-by='.lastTimestamp' +``` + +### 5.2 Rollback if Needed + +If recent deployment caused the issue: + +**MCP Actions:** + +- Use `k8s` MCP `rollback_deployment` to rollback + +**Manual Commands:** + +```bash +kubectl rollout undo deployment/ -n +``` + +## Step 6: Verify Resolution + +### 6.1 Check Pod Status + +Verify pods are running and healthy: + +**MCP Actions:** + +- Use `k8s` MCP `list_pods` to verify status +- Use `k8s` MCP `get_pod_metrics` to check resource usage + +### 6.2 Test Functionality + +- Verify service endpoints respond +- Check application logs for errors +- Test critical user flows + +**MCP Actions:** + +- Use `k8s` MCP `port_forward` to test locally if needed +- Use monitoring MCP to check service metrics + +## Step 7: Document Resolution + +### 7.1 Update Incident Log + +Create or update incident log: `incidents/YYYY-MM-DD-HHMM-k8s-.md` + +Include: + +- Root cause +- Steps taken to resolve +- Any rollbacks performed +- Follow-up actions needed + +### 7.2 Update Runbooks + +If this was a new issue pattern: + +- Add to service-specific runbook +- Document the solution for future reference + +## Quick Reference Commands + +```bash +# Get pod status +kubectl get pods -n + +# Describe pod details +kubectl describe pod -n + +# View logs +kubectl logs -n --tail=100 -f + +# Check events +kubectl get events -n --sort-by='.lastTimestamp' + +# Check resource usage +kubectl top pod -n + +# Check service endpoints +kubectl get endpoints -n + +# Rollback deployment +kubectl rollout undo deployment/ -n + +# Scale deployment +kubectl scale deployment/ --replicas= -n +``` + +## Common MCP Tools Reference + +If you have a Kubernetes MCP server configured, use these tools: + +- `k8s_list_pods` - List pods in namespace +- `k8s_get_pod_logs` - Get pod logs +- `k8s_describe_pod` - Get detailed pod information +- `k8s_get_pod_events` - Get events for a pod +- `k8s_get_service_endpoints` - Check service endpoints +- `k8s_rollback_deployment` - Rollback a deployment +- `k8s_get_nodes` - Check node status and resources +- `k8s_get_pvc` - Check persistent volume claims diff --git a/.roo/rules-oncall/service-specific/example-service-runbook.md b/.roo/rules-oncall/service-specific/example-service-runbook.md new file mode 100644 index 00000000000..18bd5c66161 --- /dev/null +++ b/.roo/rules-oncall/service-specific/example-service-runbook.md @@ -0,0 +1,170 @@ +# Example Service Runbook + +This is an example service-specific runbook. Replace with your actual service details. + +## Service Overview + +- **Service Name:** Example API Service +- **Owner:** Platform Team +- **Dependencies:** + - Database Service + - Cache Service + - Auth Service +- **Critical Paths:** + - User authentication + - Payment processing + - Data retrieval + +## Common Issues + +### Issue 1: High Error Rate + +**Symptoms:** + +- Error rate > 1% +- Increased latency +- User reports of failures + +**Diagnosis:** + +1. Check error rate metrics via monitoring MCP +2. Review error logs for patterns +3. Check recent deployments +4. Verify database connectivity + +**Resolution:** + +1. Identify root cause from logs +2. If deployment-related: Consider rollback +3. If resource-related: Scale up resources +4. If code-related: Apply hotfix +5. Monitor metrics to confirm resolution + +**MCP Tools:** + +- Use `monitoring` MCP `query_metrics` to check error rates +- Use `deployment` MCP `list_recent_deployments` to check changes +- Use `database` MCP `check_connection` to verify DB status + +### Issue 2: Database Connection Pool Exhausted + +**Symptoms:** + +- Timeout errors +- Slow response times +- Database connection errors in logs + +**Diagnosis:** + +1. Check database connection pool metrics +2. Review connection pool configuration +3. Check for connection leaks +4. Verify database server status + +**Resolution:** + +1. Increase connection pool size if needed +2. Restart service to clear stale connections +3. Fix connection leaks in code +4. Scale database if resource-constrained + +**MCP Tools:** + +- Use `database` MCP `check_pool_status` to view connections +- Use `monitoring` MCP `query_metrics` for pool metrics + +## Health Checks + +### Quick Health Check + +- Endpoint: `GET /health` +- Expected: `{"status": "healthy"}` +- MCP Tool: Use `monitoring` MCP `check_health_endpoint` + +### Deep Health Check + +1. Health endpoint responds +2. Database connectivity +3. Cache connectivity +4. All critical dependencies reachable +5. Error rate within normal range + +## Restart Procedures + +### Graceful Restart + +1. Drain traffic from instance (if load balanced) +2. Wait for in-flight requests to complete +3. Stop service gracefully +4. Start new instance +5. Verify health +6. Resume traffic + +**MCP Tools:** + +- Use `deployment` MCP `restart_service` with graceful flag + +### Emergency Restart + +1. Stop service immediately +2. Start new instance +3. Verify health +4. Monitor for issues + +**MCP Tools:** + +- Use `deployment` MCP `restart_service` with force flag + +## Rollback Procedures + +### How to Rollback + +1. Identify last known good deployment +2. Use deployment MCP to rollback +3. Verify service health after rollback +4. Monitor metrics closely + +**MCP Tools:** + +- Use `deployment` MCP `list_deployments` to find version +- Use `deployment` MCP `rollback_deployment` to rollback + +### Rollback Verification + +- Health endpoint returns healthy +- Error rate returns to normal +- Key metrics within expected range +- No user reports of issues + +## Monitoring + +### Key Metrics + +- **Request Rate**: Normal: 1000-5000 req/min +- **Error Rate**: Normal: < 0.1% +- **Latency (p95)**: Normal: < 200ms +- **Database Connections**: Normal: 10-50 active + +### Alert Thresholds + +- **High Error Rate**: > 1% for 5 minutes +- **High Latency**: p95 > 500ms for 5 minutes +- **Service Down**: Health check fails for 2 minutes + +## Contacts + +### Service Owner + +- Name: Platform Team +- Contact: #platform-team Slack channel + +### Oncall Rotation + +- Check PagerDuty for current oncall engineer +- Use `paging` MCP `get_current_oncall` to find oncall + +## Related Documentation + +- Architecture docs: `/docs/architecture/api-service.md` +- Deployment guide: `/docs/deployment/api-service.md` +- Monitoring dashboard: [Link to dashboard] diff --git a/.roomodes b/.roomodes index 01f6ed45050..4756c5c5aad 100644 --- a/.roomodes +++ b/.roomodes @@ -236,3 +236,86 @@ customModes: - command - mcp source: project + - slug: oncall + name: 🚨 Oncall + roleDefinition: |- + You are Roo Code, an oncall incident response specialist with expertise in: + - Following structured incident response workflows + - Using MCP tools to interact with monitoring, ticketing, and alerting systems + - Executing runbooks and standard operating procedures + - Documenting incidents and response actions + - Coordinating with team members during incidents + - Escalating issues when appropriate + - Post-incident analysis and documentation + + You follow predefined workflows stored in markdown files to ensure consistent, + thorough incident response. You leverage MCP tools to gather information from + monitoring systems, create tickets, send notifications, and interact with various + oncall tools. + whenToUse: >- + Use this mode when handling oncall incidents, following runbooks, executing + incident response workflows, or managing operational issues. This mode is ideal + for structured incident response following predefined workflows and using MCP + tools to interact with your monitoring and ticketing infrastructure. + description: Oncall incident response with workflow orchestration + groups: + - read + - - edit + - fileRegex: (incidents/.*\.md$|logs/.*\.md$|\.roo/rules-oncall/.*\.md$) + description: Incident logs, log files, and oncall workflow files + - command + - mcp + customInstructions: |- + ## Workflow Execution Process + + When handling oncall tasks, follow this structured approach: + + 1. **Identify the Workflow** + - Read the appropriate workflow file from `.roo/rules-oncall/` + - For Kubernetes issues: Use `k8s-troubleshooting.md` + - For specific services: Check `service-specific/` directory for service runbooks + - Add your own workflows as needed + + 2. **Execute Workflow Steps** + - Follow the workflow markdown file step-by-step + - Use MCP tools to interact with systems (monitoring, ticketing, etc.) + - Document each action taken in incident logs + - Update status as you progress through the workflow + + 3. **Use MCP Tools** + - **Monitoring MCP**: Check service health, query metrics, view dashboards + - **Ticketing MCP**: Create/update tickets, add comments, change status + - **Alerting MCP**: Acknowledge alerts, update alert status, send notifications + - **Communication MCP**: Send updates to teams, post to channels + + 4. **Document Actions** + - Update incident logs in `incidents/` directory (created automatically when switching to oncall mode) + - Record timestamps for all actions + - Note any decisions made and reasoning + - Document resolution steps + + 5. **Follow Best Practices** + - Always acknowledge alerts/pages promptly + - Assess impact before taking action + - Document everything for post-incident review + - Escalate when appropriate + - Keep stakeholders informed + + ## Available Workflows + + Workflows are stored in `.roo/rules-oncall/` and are automatically loaded as context: + - Example: `k8s-troubleshooting.md` - Kubernetes troubleshooting workflow + - `service-specific/` - Service-specific runbooks (e.g., `example-service-runbook.md`) + - Add your own workflow markdown files as needed + + ## MCP Integration + + This mode has access to MCP tools. Use them to: + - Query monitoring systems for current status + - Create and update tickets in ticketing systems + - Send notifications and alerts + - Check service dependencies and relationships + - Access runbooks and documentation + + Always use MCP tools when available instead of manual commands when possible. + source: project diff --git a/src/core/webview/ClineProvider.ts b/src/core/webview/ClineProvider.ts index bc3f6bd6ef1..ca740108b72 100644 --- a/src/core/webview/ClineProvider.ts +++ b/src/core/webview/ClineProvider.ts @@ -1358,6 +1358,27 @@ export class ClineProvider } } + // Create incidents directory when switching to oncall mode + if (newMode === "oncall") { + try { + const workspacePath = getWorkspacePath() + if (workspacePath) { + const incidentsDir = path.join(workspacePath, "incidents") + const pagesDir = path.join(incidentsDir, "pages") + const escalationsDir = path.join(incidentsDir, "escalations") + + // Create directories if they don't exist + await fs.mkdir(pagesDir, { recursive: true }) + await fs.mkdir(escalationsDir, { recursive: true }) + } + } catch (error) { + // Log error but don't fail mode switch if directory creation fails + this.log( + `Failed to create incidents directory when switching to oncall mode: ${error instanceof Error ? error.message : String(error)}`, + ) + } + } + await this.postStateToWebview() } diff --git a/src/package.json b/src/package.json index 624b5b5b16e..e0d00a1bfd5 100644 --- a/src/package.json +++ b/src/package.json @@ -3,7 +3,7 @@ "displayName": "%extension.displayName%", "description": "%extension.description%", "publisher": "RooVeterinaryInc", - "version": "3.46.1", + "version": "3.46.2", "icon": "assets/icons/icon.png", "galleryBanner": { "color": "#617A91",