diff --git a/.claude/skills/close-task-commit-push-pr/SKILL.md b/.claude/skills/close-task-commit-push-pr/SKILL.md new file mode 100644 index 00000000..9c4ef42c --- /dev/null +++ b/.claude/skills/close-task-commit-push-pr/SKILL.md @@ -0,0 +1,95 @@ +--- +name: close-task-commit-push-pr +description: Close the active backlog task (detected from branch name), commit all changes, push to remote, and open a pull request. Use when the user says "close task and ship it", "close task commit push pr", or invokes /close-task-commit-push-pr. +allowed-tools: Bash(git add:*), Bash(git status:*), Bash(git diff:*), Bash(git log:*), Bash(git commit:*), Bash(git push:*), Bash(git branch:*), Bash(git checkout:*), Bash(gh pr create:*), mcp__backlog__task_edit, mcp__backlog__task_complete, mcp__backlog__task_view +--- + +# Close Task, Commit, Push & PR + +## Context + +- Current git status: !`git status` +- Current git diff (staged and unstaged changes): !`git diff HEAD` +- Current branch: !`git branch --show-current` +- Main branch: !`git rev-parse --verify main 2>/dev/null && echo main || echo master` +- Recent commits: !`git log --oneline -10` + +## Your task + +Close the active backlog task, commit all changes, push, and open a pull request. + +### Step 0 — Identify the task + +- Extract the task ID from the current branch name (e.g. `feature/task-113-split-css-modules` → `task-113`). +- If no task ID is found in the branch name, ask the user which task to close. + +### Step 1 — Close the backlog task + +Close the task **before** committing so the task file changes are included in the commit and PR. + +- Use `mcp__backlog__task_view` to read the task details (title, acceptance criteria). +- Use `mcp__backlog__task_edit` to: + - Set status to `Done` + - Check off all acceptance criteria that were completed (review the diff to determine which ones) + - Write a `finalSummary` that concisely describes what was implemented +- Use `mcp__backlog__task_complete` to move the task to the completed folder. + +### Step 2 — Analyze changes and plan commits + +Before committing, analyze `git status` and `git diff HEAD` to identify logically distinct groups of changes. Group by feature or concern — for example: + +- New files that form a self-contained module → one commit +- Modifications to an existing file that depend on the new module → separate commit +- Task/backlog file changes → include in the final commit (or a dedicated chore commit) + +Print a short commit plan (list of planned commits with the files in each) so the grouping is visible. + +### Step 3 — Commit + +- Create one commit per logical group identified above. Stage only the files for that group using explicit file names (never `git add -A`; never stage `.env` or credential files). +- Write a clean, descriptive commit message for each commit using conventional commits style. +- End every commit message with: + `Co-Authored-By: Claude Opus 4.6 (1M context) ` +- Use a HEREDOC to pass the commit message for correct formatting. +- **Include the closed task file** in the final commit. + +### Step 4 — Push + +- If the current branch is `main` or `master`, create a new feature branch first (use a descriptive name based on the changes). +- Push the branch to origin with `-u` to set upstream tracking. + +### Step 5 — Pull Request + +- Create a PR using `gh pr create` targeting the main branch. +- Keep the PR title short (under 70 characters), using conventional commit style. +- Use a HEREDOC for the PR body with this format: + +``` +## Summary +<1-3 bullet points describing the changes> + +## Test plan +- [ ] + +🤖 Generated with [Claude Code](https://claude.com/claude-code) +``` + +### Step 6 — Update task with PR reference + +- Now that you have the PR number, use `mcp__backlog__task_edit` to append the PR URL/number to the `finalSummary`. + +### Step 7 — Amend commit with updated task file + +- Stage the updated task file and amend the last commit to include the PR reference: `git commit --amend --no-edit` then `git push --force-with-lease`. + +### Step 8 — Report + +- Print the PR URL and confirm the task was closed. + +## Rules + +- Do all steps in as few messages as possible. Parallelize independent tool calls. +- Do not read or explore code beyond what git provides in the context above. +- Do not use interactive git flags (`-i`). +- Never force-push or amend existing commits. +- If a pre-commit hook fails, fix the issue and create a NEW commit (do not amend). diff --git a/.claude/skills/git-commit-push-pr/SKILL.md b/.claude/skills/git-commit-push-pr/SKILL.md new file mode 100644 index 00000000..e23e0e02 --- /dev/null +++ b/.claude/skills/git-commit-push-pr/SKILL.md @@ -0,0 +1,70 @@ +--- +name: git-commit-push-pr +description: Commit all changes, push to remote, and open a pull request in one go. Use when the user says "commit push pr", "ship it", "open a pr", or invokes /git-commit-push-pr. +allowed-tools: Bash(git add:*), Bash(git status:*), Bash(git diff:*), Bash(git log:*), Bash(git commit:*), Bash(git push:*), Bash(git branch:*), Bash(git checkout:*), Bash(gh pr create:*) +--- + +# Git Commit, Push & PR + +## Context + +- Current git status: !`git status` +- Current git diff (staged and unstaged changes): !`git diff HEAD` +- Current branch: !`git branch --show-current` +- Main branch: !`git rev-parse --verify main 2>/dev/null && echo main || echo master` +- Recent commits: !`git log --oneline -10` + +## Your task + +Review the changes in the repo and ship them as a pull request in one go. + +### Step 1 — Analyze changes and plan commits + +Before committing, analyze `git status` and `git diff HEAD` to identify logically distinct groups of changes. Group by feature or concern — for example: + +- New files that form a self-contained module → one commit +- Modifications to an existing file that depend on the new module → separate commit +- Config or tooling changes → separate commit + +Print a short commit plan (list of planned commits with the files in each) so the grouping is visible. + +### Step 2 — Commit + +- Create one commit per logical group identified above. Stage only the files for that group using explicit file names (never `git add -A`; never stage `.env` or credential files). +- Write a clean, descriptive commit message for each commit using conventional commits style. +- End every commit message with: + `Co-Authored-By: Claude Opus 4.6 (1M context) ` +- Use a HEREDOC to pass the commit message for correct formatting. + +### Step 3 — Push + +- If the current branch is `main` or `master`, create a new feature branch first (use a descriptive name based on the changes). +- Push the branch to origin with `-u` to set upstream tracking. + +### Step 4 — Pull Request + +- Create a PR using `gh pr create` targeting the main branch. +- Keep the PR title short (under 70 characters), using conventional commit style. +- Use a HEREDOC for the PR body with this format: + +``` +## Summary +<1-3 bullet points describing the changes> + +## Test plan +- [ ] + +🤖 Generated with [Claude Code](https://claude.com/claude-code) +``` + +### Step 5 — Report + +- Print the PR URL so the user can see it. + +## Rules + +- Do all steps in as few messages as possible. Parallelize independent tool calls. +- Do not read or explore code beyond what git provides in the context above. +- Do not use interactive git flags (`-i`). +- Never force-push or amend existing commits. +- If a pre-commit hook fails, fix the issue and create a NEW commit (do not amend). diff --git a/.claude/skills/review/SKILL.md b/.claude/skills/review/SKILL.md new file mode 100644 index 00000000..7624f4f8 --- /dev/null +++ b/.claude/skills/review/SKILL.md @@ -0,0 +1,48 @@ +--- +name: review +description: Review local code changes for bugs, regressions, missing tests, and pragmatic improvements. Use when the user asks to review the current changes from `git status`, or to review a specific backlog task by id such as `task-65` or `TASK-65`. Also trigger when the user says things like "review", "review current changes", "check my changes", "look over the diff", or "review task-42". +--- + +# Review + +## Overview + +Review the current local changes or the implementation tied to a backlog task id and report concrete findings, not a changelog. Treat this as a code review unless the user explicitly asks for edits. + +## Workflow + +1. Resolve the review target. +If the prompt includes a backlog task id, locate the task markdown in `backlog/tasks/` first, then `backlog/completed/` if needed. Read the description, acceptance criteria, and referenced files. If no task id is given, start from `git status --short` and review the current local changes directly. + +2. Build review scope from local context. +Check `git status --short`, inspect the diff for files related to the task, and read surrounding code where behavior is affected. Prefer `rg` and targeted `git diff -- ` over broad scans. + +3. Review for defects and regressions. +Prioritize: +- broken behavior versus the task intent +- stale callers or UI paths left behind after refactors +- contract mismatches across main/preload/renderer/shared code +- silent failure handling and misleading fallback behavior +- missing or weak tests for the changed behavior +- unnecessary performance regressions or duplicate work + +4. Verify when useful. +Run focused tests for the touched area if they are available and cheap. Mention clearly when verification is blocked or when broader typecheck/test failures are unrelated. + +## Output + +Report findings first, ordered by severity. For each finding: +- state the severity +- describe the bug/risk or improvement +- include clickable file references with line numbers when available +- explain the user-visible or maintenance impact briefly + +After findings, include: +- open questions or assumptions if any +- a brief note on verification performed + +If no findings are discovered, say that explicitly and mention residual risk or missing coverage. + +## Prompt Shape + +Interpret prompts like `review`, `review current changes`, or `review the changes from git status` as a request to review the current working tree. Interpret prompts like `task-56`, `review task-56`, or `review task-56 and report any issues/bugs or possible improvements` as a review of the local changes associated with that task. diff --git a/.claude/skills/start-task/SKILL.md b/.claude/skills/start-task/SKILL.md new file mode 100644 index 00000000..909b255e --- /dev/null +++ b/.claude/skills/start-task/SKILL.md @@ -0,0 +1,67 @@ +--- +name: start-task +description: Create a feature branch for a backlog task, switch to it, and start implementation. Use when the user says "start task-123", "work on task-123", "implement task-123", or invokes /start-task with a task ID. +allowed-tools: Bash(git checkout:*), Bash(git branch:*), Bash(git status:*), Bash(git pull:*), Bash(git stash:*), Bash(npm run lint:*), Bash(npm run typecheck:*), Bash(npm test:*), Bash(npm run build*), Bash(npx playwright test:*), mcp__backlog__task_view, mcp__backlog__task_edit, mcp__backlog__task_search, mcp__backlog__task_list +--- + +# Start Task + +## Context + +- Current branch: !`git branch --show-current` +- Working tree status: !`git status --short` +- Existing branches: !`git branch --list 'feature/task-*'` + +## Your task + +Create a feature branch for a backlog task and begin implementation. + +The user will provide a task ID (e.g. `task-123` or `123`). If no task ID is provided, check the argument `$ARGUMENTS` for the task reference. + +### Step 1 — Load the task + +- Normalize the input: if the user gave just a number like `123`, treat it as `task-123`. +- Use `mcp__backlog__task_view` to read the full task details: title, description, acceptance criteria, and any referenced files. +- If the task is not found, use `mcp__backlog__task_search` to locate it. +- Print a brief summary of the task for the user. + +### Step 2 — Ensure a clean working tree + +- Check `git status`. If there are uncommitted changes, warn the user and ask whether to stash them before proceeding. +- Ensure we are on the main branch. If not, ask the user if they want to switch. + +### Step 3 — Create and switch to a feature branch + +- Pull latest changes on main: `git pull --rebase`. +- Derive a branch name from the task: `feature/task-{id}-{slugified-title}` (lowercase, hyphens, max ~60 chars). + - Example: task 113 "Split app.css into native CSS modules" → `feature/task-113-split-css-modules` +- Create and switch to the branch: `git checkout -b `. + +### Step 4 — Mark the task as in-progress + +- Use `mcp__backlog__task_edit` to set the task status to `In Progress`. + +### Step 5 — Plan and implement + +- Analyze the task description and acceptance criteria carefully. +- Read all files referenced in the task, plus any related code you need to understand. +- Create a plan and present it to the user for approval before writing code. +- Once approved, implement the changes following the project's patterns and conventions. +- After implementation, run the full verification suite: + ```bash + npm run lint && npm run typecheck && npm test && npm run build:app && npx playwright test + ``` +- Fix any issues found by the verification suite. + +### Step 6 — Report + +- Summarize what was implemented and which acceptance criteria were addressed. +- Remind the user they can use `/close-task-commit-push-pr` when ready to ship. + +## Rules + +- Always create the branch from an up-to-date main branch. +- Never start implementation without showing the plan to the user first. +- Follow existing code patterns and conventions in the project. +- Do not use interactive git flags (`-i`). +- If the working tree is dirty, never silently discard changes. diff --git a/.gitignore b/.gitignore index 0ce5efd6..9387e5df 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ .env build/** +.kotlin/ .gradle .intellijPlatform .idea diff --git a/CHANGELOG.md b/CHANGELOG.md index 2e456eb2..62ac4d08 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ so # Changelog All notable changes to this project will be documented in this file. +## [Unreleased] + +### Added +- Anonymous LLM provider/model usage analytics (opt-out). Helps guide which providers and models receive engineering investment. The plugin sends a minimal payload — anonymous install ID (UUID), per-launch session ID, plugin version, IDE version, LLM provider name, LLM model name — when you run a prompt or change models. **Never sent**: prompt text, response text, file content, file paths, project names, conversation history, API keys, or anything that could identify you. A first-launch notification asks for consent; you can disable it any time in *Settings → DevoxxGenie → General*. + ## [1.4.1] ### Updated diff --git a/README.md b/README.md index 76cf2ee0..f640d067 100644 --- a/README.md +++ b/README.md @@ -417,6 +417,25 @@ It is recommended to use the publishPlugin task for releasing the plugin Enjoy! +## Privacy & Anonymous Usage Analytics + +To guide which LLM providers and models receive engineering investment, DevoxxGenie collects **anonymous** usage data when you run a prompt or change models. + +**What is sent:** +- An anonymous install ID (UUID), generated once and stored locally +- A per-launch session ID (random 10-digit number) +- Plugin version and IDE version +- LLM provider name (e.g. `anthropic`, `ollama`) +- LLM model name (e.g. `claude-3-5-sonnet`) + +**What is never sent:** +- Prompt text, response text, conversation history +- File content, file paths, project name, git remote +- API keys, credentials, user name, email +- Token counts or cost data + +A first-launch notification asks for your consent before any data is sent. You can change this at any time in **Settings → DevoxxGenie → General**. + ## Contribute **[📖 Contributing Guide](https://genie.devoxx.com/docs/contributing)** diff --git a/backlog/completed/task-206 - Add-anonymous-LLM-provider-model-usage-analytics.md b/backlog/completed/task-206 - Add-anonymous-LLM-provider-model-usage-analytics.md new file mode 100644 index 00000000..f2e5a9fd --- /dev/null +++ b/backlog/completed/task-206 - Add-anonymous-LLM-provider-model-usage-analytics.md @@ -0,0 +1,175 @@ +--- +id: TASK-206 +title: Add anonymous LLM provider/model usage analytics +status: Done +assignee: [] +created_date: '2026-04-13 09:25' +updated_date: '2026-04-13 10:37' +labels: + - analytics + - telemetry + - feature +dependencies: [] +references: + - src/main/java/com/devoxx/genie/controller/PromptExecutionController.java + - src/main/java/com/devoxx/genie/service/DevoxxGenieStateService.java + - src/main/java/com/devoxx/genie/ui/settings/ + - src/main/resources/META-INF/plugin.xml + - ../GenieBuilder/src/main/services/analytics/analytics-service.ts + - ../GenieBuilder/tests/unit/analytics-service.test.ts + - ../GenieBuilder/functions/src/index.ts + - ../GenieBuilder/functions/src/analytics.ts + - ../GenieBuilder/web-admin/src/app/features/analytics/ + - ../GenieBuilder/cloudflare/ +documentation: + - 'https://developers.google.com/analytics/devguides/collection/protocol/ga4' + - 'https://aws.amazon.com/bedrock/pricing/' +priority: high +--- + +## Description + + +## Why + +We are evaluating a flat-fee LLM cloud subscription tier for DevoxxGenie users (fronted by AWS Bedrock and other providers). To price plans without losing money, we need real data on **which LLM providers and models DevoxxGenie users actually run**, and at what relative frequency. Today we have zero visibility into this — pricing decisions would be guesswork against a heavy-tailed cost distribution where one Opus power user can wipe out hundreds of light subscribers. + +This task adds **minimal, anonymous, opt-out-able** usage analytics that capture LLM provider and model per actual prompt execution. No prompts, responses, file content, file paths, project names, API keys, conversation history, or user identity are ever sent. + +## What we are reusing from GenieBuilder (Option A) + +GenieBuilder (sister Electron project at `../GenieBuilder`) already implements this pattern: + +- **Cloudflare Worker endpoint:** `https://delicate-morning-ff55.devoxx.workers.dev` — accepts GA4 Measurement Protocol JSON and forwards to GA4 (property `G-VHHFZ5TRG2`, Firebase project `geniebuilder-49a88`). +- **Reference client:** `../GenieBuilder/src/main/services/analytics/analytics-service.ts` (1-258) and tests at `../GenieBuilder/tests/unit/analytics-service.test.ts` (21-51). +- **Backend read API:** Firebase Cloud Function `analyticsReport` at `../GenieBuilder/functions/src/index.ts` (148-185), allowlist in `../GenieBuilder/functions/src/analytics.ts:12`, provider/model breakdowns at `analytics.ts:142,157`. +- **Admin dashboard:** `../GenieBuilder/web-admin/src/app/features/analytics/`. + +**Option A (chosen):** reuse the existing Cloudflare Worker AND Firebase admin dashboard. DevoxxGenie traffic is segmented in the same GA4 property via `app_name=devoxxgenie-intellij`. No new worker, no new GA4 property, no new Firebase project. The cross-repo backend changes required to surface `prompt_executed` in the dashboard are tracked separately in **TASK-207** (a hard dependency for end-to-end visibility, but does not block this plugin task's PR from merging). + +## Exact payload (full disclosure) + +```json +{ + "client_id": "uuid-persisted-once-in-state-service", + "events": [{ + "name": "prompt_executed", + "params": { + "provider_id": "anthropic", + "model_name": "claude-3-5-sonnet", + "app_name": "devoxxgenie-intellij", + "app_version": "1.4.1", + "ide_version": "2024.3", + "session_id": "1234567890", + "engagement_time_msec": 1 + } + }] +} +``` + +**Every field above is user-visible data.** README, plugin.xml change-notes, CHANGELOG, the settings help text, and the first-launch notification MUST list all of these explicitly. The earlier draft phrasing ("only LLM provider and model") was inaccurate and is replaced. + +`session_id` is a string of 10 digits regenerated on each IDE launch (matches GenieBuilder format and GA4 Measurement Protocol expectations for realtime reporting). + +## Two events + +- **`prompt_executed`** — load-bearing event for pricing math. Fired only when DevoxxGenie has actually decided to dispatch a prompt to an LLM. NOT fired for: empty prompts, slash commands handled locally (`/init`, `/help`, `/clear`, etc.), stop toggles, or any non-LLM command. Insertion point is after command processing resolves provider+model on `ChatMessageContext` (currently `ActionButtonsPanelController.java:83`) and before `PromptExecutionController.java:94` dispatches. The event must still fire if the LLM call later fails — it represents intent to dispatch. +- **`model_selected`** — intent signal. Fired only on **user-initiated** model changes in the model combo at `DevoxxGenieToolWindowContent.java:273`. MUST be guarded so it does NOT fire on: combo initialization, project open, settings refresh/restore, programmatic `setSelectedItem`, or provider switch repopulating the list. Note: `LlmProviderPanel.java:350` only handles the Exo edge case and is not the right hook for this event. + +## What is NEVER sent + +- Prompt text, response text, conversation history +- File content, file paths, project name, git remote +- API keys, credentials, user name, email +- Token counts, cost data (out of scope; possibly a future task) + +## Privacy, consent, gating + +This is an open-source IntelliJ Marketplace plugin. Silent telemetry will get the plugin flagged. Mandatory rules: + +1. **Persisted flag** `analyticsNoticeAcknowledged` (default `false`) in `DevoxxGenieStateService`, alongside `analyticsEnabled` (default `true`) and `analyticsClientId`. +2. **No analytics event may be emitted before the notice has been shown and acknowledged** — enforced as a hard precondition in `AnalyticsService`, unit-tested. +3. **First-launch / post-update notification** shown exactly once per install: lists every field collected (client_id, session_id, app_version, ide_version, provider_id, model_name), states what is NOT collected, and offers inline `[Disable]` and `[OK]` actions. `[Disable]` must synchronously set both `analyticsEnabled=false` and `analyticsNoticeAcknowledged=true` and persist immediately. +4. **Settings checkbox** in the LLM Settings configurable (`LLMConfigSettingsComponent`) — or a new "General" configurable registered under the DevoxxGenie root in `plugin.xml:448` if cleaner. Help text under the checkbox enumerates every field sent. +5. **Honor the opt-out flag synchronously** — when `analyticsEnabled=false`, `AnalyticsService` makes zero HTTP requests. + +## Implementation outline + +1. Add to `DevoxxGenieStateService`: `analyticsEnabled` (boolean, default true), `analyticsNoticeAcknowledged` (boolean, default false), `analyticsClientId` (UUID String, generated once on first read), `analyticsEndpoint` (String, defaults to the Cloudflare worker URL — overridable for tests). +2. Create `service/analytics/AnalyticsService.java` (~150 LoC). Async fire-and-forget via `ApplicationManager.getApplication().executeOnPooledThread()` — never block EDT. Silent failure (debug log only). Use `java.net.http.HttpClient` (no new dependency). +3. Wire `prompt_executed` at the dispatch decision point described above. +4. Wire `model_selected` at `DevoxxGenieToolWindowContent.java:273` behind a user-action guard. +5. Add the settings checkbox + first-launch notification + state flags. +6. Documentation: README privacy section, plugin.xml change-notes, CHANGELOG entry — all listing every field. +7. Cross-repo dashboard updates tracked in TASK-207. + +## Branch strategy + +Per project CLAUDE.md: create branch `feature/llm-usage-analytics` from `master` BEFORE any code changes in this repo. The untracked `backlog/tasks/task-206 *.md` and `task-207 *.md` files should be committed on that branch as part of the first commit so task metadata travels with the implementation. + +## Out of scope (explicitly) + +- Token counting or cost telemetry +- User accounts, authentication, or non-anonymous identifiers +- Per-request metadata beyond the payload above +- Standing up a separate Cloudflare worker, GA4 property, or Firebase project (Option B — rejected) +- GenieBuilder backend/admin UI changes (tracked in TASK-207) + + +## Acceptance Criteria + +- [x] #1 AnalyticsService class exists under service/analytics/ and posts JSON to the configured Cloudflare worker endpoint asynchronously off the EDT using java.net.http.HttpClient +- [x] #2 Payload contains EXACTLY these fields and no others: client_id, event name, provider_id, model_name, app_name='devoxxgenie-intellij', app_version, ide_version, session_id (10-digit string), engagement_time_msec — verified by unit test asserting no extra fields +- [x] #3 session_id is a 10-digit string, regenerated on each IDE launch, identical in shape to GenieBuilder's analytics-service.ts format +- [x] #4 Anonymous client_id is a UUID generated once on first read and persisted in DevoxxGenieStateService across IDE restarts and plugin updates +- [x] #5 prompt_executed event is fired only when DevoxxGenie has decided to dispatch a prompt to an LLM — NOT for empty prompts, /init, /help, /clear, stop toggles, or any locally-handled command +- [x] #6 prompt_executed fires after ChatMessageContext has the resolved provider+model (after command processing in ActionButtonsPanelController.java:83) and before PromptExecutionController.java:94 dispatches; the event still fires if the LLM call later fails +- [x] #7 model_selected fires only on user-initiated changes to the model combo in DevoxxGenieToolWindowContent.java:273; a user-action guard prevents firing during initialization, project open, settings refresh/restore, provider switch repopulation, or programmatic setSelectedItem +- [x] #8 DevoxxGenieStateService gains persistent fields: analyticsEnabled (default true), analyticsNoticeAcknowledged (default false), analyticsClientId (UUID), analyticsEndpoint (default to Cloudflare worker URL, overridable for tests) +- [x] #9 AnalyticsService emits zero events when analyticsNoticeAcknowledged is false — enforced as a hard precondition and verified by unit test +- [x] #10 First-launch (or post-update) notification is shown exactly once per install, listing every field collected (client_id, session_id, app_version, ide_version, provider_id, model_name) and stating that prompts, code, and file content are NOT sent +- [x] #11 Notification's inline Disable action synchronously sets analyticsEnabled=false AND analyticsNoticeAcknowledged=true and persists immediately before any event can be emitted +- [x] #12 Settings checkbox 'Send anonymous usage statistics' lives in LLMConfigSettingsComponent (or a new General configurable explicitly registered under the DevoxxGenie root in plugin.xml:448); help text under the checkbox enumerates every field sent +- [x] #13 When analyticsEnabled is false, AnalyticsService makes zero HTTP requests — verified by unit test +- [x] #14 Network failures, timeouts, and non-2xx responses are caught silently and logged at debug level only — never surfaced to the user +- [x] #15 Unit test asserts no prompt text, response text, conversation history, file content, file paths, project names, API keys, or user identity appear in any payload +- [x] #16 README, plugin.xml description/change-notes, and CHANGELOG explicitly list every field collected (client_id, session_id, app_version, ide_version, provider_id, model_name) and explain how to opt out +- [x] #17 Unit tests cover: payload schema, opt-out behavior, EDT non-blocking, silent failure, client_id persistence, notice gating precondition, model_selected user-action guard +- [x] #18 Feature branch feature/llm-usage-analytics is created from master before any code changes; task-206 and task-207 markdown files are committed on that branch +- [x] #19 TASK-207 is listed as a hard dependency for end-to-end dashboard visibility but does not block this task's PR from merging + + +## Final Summary + + +## Anonymous LLM provider/model usage analytics (task-206) + +Implemented opt-out anonymous usage analytics that capture LLM provider and model on every real prompt dispatch and on user-initiated model changes. Reuses the existing GenieBuilder Cloudflare worker (`https://delicate-morning-ff55.devoxx.workers.dev`) and segments DevoxxGenie traffic via `app_name=devoxxgenie-intellij`. **Cross-repo dashboard work tracked separately in TASK-207.** + +### What was added +- **`AnalyticsService`** (`service/analytics/`) — application-level service that builds the GA4 Measurement Protocol payload by hand (no new dependencies), posts via `java.net.http.HttpClient` on a pooled thread, and never blocks the EDT. Hard preconditions block emission until the consent notice is acknowledged AND the user hasn't opted out. +- **`AnalyticsConsentNotifier`** — first-launch notification with two inline actions (`OK` and `Disable`), shown exactly once per install via a separate `analyticsNoticeShown` flag set synchronously before the EDT dispatch to avoid concurrent-project-open races. +- **General settings configurable** (`ui/settings/general/`) — new top-level entry under DevoxxGenie with the opt-out checkbox and explicit help text enumerating every field that is and is not sent. +- **State service additions**: `analyticsEnabled`, `analyticsNoticeShown`, `analyticsNoticeAcknowledged`, `analyticsClientId` (lazy UUID), `analyticsEndpoint`. Client id persists across IDE restarts and plugin updates. + +### What was wired +- **`prompt_executed`** fires inside `PromptExecutionService.executePrompt` *after* `processCommands` confirms an LLM dispatch and *before* the strategy executes — so locally-handled commands (`/init`, `/help`, `/clear`, stop toggles, empty prompts) are excluded by construction. Event still fires if the LLM call later fails. +- **`model_selected`** fires from `DevoxxGenieToolWindowContent.processModelNameSelection` only when **all three** guards pass: `isInitializationComplete`, `suppressModelSelectionTracking` (wraps `settingsChanged`), and `LlmProviderPanel.isUpdatingModelNames()` (wraps `updateModelNamesComboBox` and `restoreLastSelectedLanguageModel`). Provider switches, settings refreshes, and programmatic restores no longer emit false events. + +### Privacy posture +The exact payload is six fields: `client_id`, `session_id` (10-digit string per launch), `app_version`, `ide_version`, `provider_id`, `model_name`, plus the GA4-required `engagement_time_msec` and `app_name=devoxxgenie-intellij`. **Never sent**: prompt text, response text, conversation history, file content, file paths, project names, git remotes, API keys, credentials, token counts, cost data. Every one of those exclusions is documented in `README.md` (new Privacy section), `plugin.xml` description (new Marketplace section), `plugin.xml` change-notes (Unreleased), `CHANGELOG.md`, and the General settings help text. + +### Tests added +- `AnalyticsServiceTest` (8 tests): exact payload allowlist, 10-digit session id, UUID client id persistence, opt-out gating, notice-acknowledgement gating, missing provider/model gating, silent network failure, no-PII even with path-like inputs. +- `LlmProviderPanelTest.isUpdatingModelNames_isTrueDuringUpdateAndFalseAfter`: regression for the user-action guard. + +### Verification +`./gradlew test` — full suite green (was 6 failing before; the 6 Exo failures were pre-existing on master, addressed in a separate concern via `Assumptions.assumeTrue` skip when no Exo server is running on `localhost:52415`). 8 new analytics tests passing, 1 regression test passing. + +### Notes +- TASK-207 (GenieBuilder backend + admin UI to surface `prompt_executed` and add an `app_name` filter) is a hard dependency for end-to-end visibility but does not block this PR. + +--- + +**PR:** https://github.com/devoxx/DevoxxGenieIDEAPlugin/pull/1005 + diff --git a/backlog/tasks/task-207 - GenieBuilder-surface-DevoxxGenie-prompt_executed-events-in-analytics-dashboard.md b/backlog/tasks/task-207 - GenieBuilder-surface-DevoxxGenie-prompt_executed-events-in-analytics-dashboard.md new file mode 100644 index 00000000..42c6f555 --- /dev/null +++ b/backlog/tasks/task-207 - GenieBuilder-surface-DevoxxGenie-prompt_executed-events-in-analytics-dashboard.md @@ -0,0 +1,91 @@ +--- +id: TASK-207 +title: >- + GenieBuilder: surface DevoxxGenie prompt_executed events in analytics + dashboard +status: To Do +assignee: [] +created_date: '2026-04-13 09:34' +labels: + - analytics + - geniebuilder + - cross-repo + - dashboard +dependencies: + - TASK-206 +references: + - ../GenieBuilder/functions/src/analytics.ts + - ../GenieBuilder/functions/src/index.ts + - ../GenieBuilder/web-admin/src/app/features/analytics/ + - ../GenieBuilder/web-admin/src/app/core/services/analytics.service.ts + - ../GenieBuilder/cloudflare/ +documentation: + - 'https://developers.google.com/analytics/devguides/reporting/data/v1' +priority: high +--- + +## Description + + +## Why + +DevoxxGenie TASK-206 introduces a new `prompt_executed` analytics event sent through the existing Cloudflare worker (`https://delicate-morning-ff55.devoxx.workers.dev`) into the shared GA4 property `G-VHHFZ5TRG2` (Firebase project `geniebuilder-49a88`). DevoxxGenie traffic is segmented via `app_name=devoxxgenie-intellij`. + +As of today the GenieBuilder admin dashboard pipeline does NOT know about this event: + +- `../GenieBuilder/functions/src/analytics.ts:12` — the `TRACKED_EVENTS` allowlist has no `prompt_executed` entry; events arriving in GA4 will not be queried. +- `../GenieBuilder/functions/src/analytics.ts:142` and `:157` — provider/model breakdown queries currently key off `provider_selected` and `model_selected` (intent signals), not actual prompt usage. +- The Angular admin UI at `../GenieBuilder/web-admin/src/app/features/analytics/` has no `app_name` filter, so DevoxxGenie traffic would be commingled with GenieBuilder Electron traffic in every chart. + +Without this task, the DevoxxGenie pricing data we want to gather will land in GA4 but never surface in the admin dashboard — defeating the purpose of TASK-206. + +## What changes (cross-repo, in the GenieBuilder repo) + +This task is implemented in `/Users/stephan/IdeaProjects/GenieBuilder`, NOT in DevoxxGenieIDEAPlugin. Branch from GenieBuilder's main/develop branch per that project's conventions. + +### Cloud Functions (`functions/src/analytics.ts`, `functions/src/index.ts`) + +1. Add `prompt_executed` to `TRACKED_EVENTS` allowlist (`analytics.ts:12`). +2. Add `app_name` as a queryable dimension on the GA4 Data API requests in `analyticsReport` (`functions/src/index.ts:148-185`). +3. Add new provider and model breakdown queries that aggregate by `prompt_executed` (in addition to, not replacing, the existing `provider_selected` / `model_selected` breakdowns) so we measure actual usage, not just selection intent. +4. Accept an `appName` query parameter on the `analyticsReport` endpoint that filters all queries by that `app_name` dimension. Default behavior (no param) should remain backwards compatible with the Electron app's current dashboard view. +5. Update event labels/categories so DevoxxGenie events render with sensible names in the dashboard. + +### Web admin UI (`web-admin/src/app/features/analytics/`) + +1. Add an app selector (tab or dropdown) at the top of the analytics view: "GenieBuilder (Electron)" / "DevoxxGenie (IntelliJ)" / "All". +2. Pass the selection through to `analyticsReport` as the `appName` query param. +3. Show the new actual-usage provider/model breakdowns from `prompt_executed` alongside the existing intent-signal breakdowns. Label them clearly (e.g., "Prompts dispatched" vs "Models selected") so a viewer can't confuse them. +4. CSV export should include the app filter and the new breakdown columns. + +### Tests + +- Unit tests for the new query branches in `analytics.ts`. +- Snapshot or screenshot test for the Angular UI showing the app selector and the new breakdown sections. + +## Dependencies and ordering + +- This task **depends on TASK-206** producing real `prompt_executed` events with `app_name=devoxxgenie-intellij` so the queries can be validated end-to-end. +- TASK-206's PR can merge before this task — DevoxxGenie will start sending events into GA4 immediately, and they will accumulate in the raw GA4 data even before the dashboard surfaces them. This task simply unlocks visibility. + +## Out of scope + +- Standing up a separate Cloudflare worker or GA4 property (Option B — rejected). +- Token/cost telemetry visualizations (no token data is sent yet). +- Authentication or access control changes to the admin dashboard. + + +## Acceptance Criteria + +- [ ] #1 functions/src/analytics.ts TRACKED_EVENTS allowlist includes 'prompt_executed' +- [ ] #2 GA4 Data API queries in analyticsReport (functions/src/index.ts) accept and filter by an 'app_name' dimension +- [ ] #3 analyticsReport endpoint accepts an 'appName' query parameter; omitting it preserves the existing GenieBuilder Electron behavior (backwards compatible) +- [ ] #4 New provider and model breakdown queries aggregate events of type 'prompt_executed' (in addition to existing 'provider_selected' / 'model_selected' breakdowns) so actual usage is measured, not just selection intent +- [ ] #5 Angular admin UI in web-admin/src/app/features/analytics/ has an app selector (tab or dropdown) with options: GenieBuilder (Electron), DevoxxGenie (IntelliJ), All +- [ ] #6 Selecting an app in the UI passes appName through to the analyticsReport endpoint and updates all charts and tables +- [ ] #7 UI clearly labels intent-signal breakdowns ('Models selected') separately from actual-usage breakdowns ('Prompts dispatched') so they cannot be confused +- [ ] #8 CSV export honors the active app filter and includes the new breakdown columns +- [ ] #9 Unit tests cover the new query branches in functions/src/analytics.ts (allowlist, app filter, prompt_executed breakdowns) +- [ ] #10 End-to-end validation: a synthetic prompt_executed event with app_name=devoxxgenie-intellij sent through the Cloudflare worker appears in the DevoxxGenie view of the dashboard within GA4's normal latency window +- [ ] #11 Cross-repo work is implemented in /Users/stephan/IdeaProjects/GenieBuilder on a feature branch following GenieBuilder's branching conventions, not in DevoxxGenieIDEAPlugin + diff --git a/src/main/java/com/devoxx/genie/service/PostStartupActivity.java b/src/main/java/com/devoxx/genie/service/PostStartupActivity.java index 2f29eb96..1118588e 100644 --- a/src/main/java/com/devoxx/genie/service/PostStartupActivity.java +++ b/src/main/java/com/devoxx/genie/service/PostStartupActivity.java @@ -1,5 +1,6 @@ package com.devoxx.genie.service; +import com.devoxx.genie.service.analytics.AnalyticsConsentNotifier; import com.devoxx.genie.service.automation.listeners.BuildCompilationListener; import com.devoxx.genie.service.automation.listeners.FileEventListener; import com.devoxx.genie.service.automation.listeners.FileSaveListener; @@ -58,6 +59,9 @@ public Object execute(@NotNull Project project, @NotNull ContinuationUntil the user acknowledges (via either inline action or by visiting Settings → DevoxxGenie + * → General), {@link AnalyticsService} refuses to emit any event. The notification therefore + * gates all telemetry on informed consent. + */ +public final class AnalyticsConsentNotifier { + + private static final String NOTIFICATION_GROUP_ID = "com.devoxx.genie.notifications"; + + private static final String TITLE = "DevoxxGenie usage analytics"; + + private static final String CONTENT = + "To guide which LLM providers and models we invest engineering effort in, " + + "DevoxxGenie collects anonymous usage data when you run a prompt or change models:" + + "
    " + + "
  • Anonymous install ID, per-launch session ID, plugin version, IDE version
    • " + + "
  • LLM provider name and model name
    • " + + "
" + + "We never send prompt text, response text, file content, file paths, project names, " + + "API keys, or anything that could identify you. " + + "You can change this any time in Settings → DevoxxGenie → General." + + ""; + + private AnalyticsConsentNotifier() { + } + + public static void maybeShow(@NotNull Project project) { + DevoxxGenieStateService state = DevoxxGenieStateService.getInstance(); + // Show exactly once per install: gate on analyticsNoticeShown, not on Acknowledged. + // Acknowledged still gates emission (informed-consent rule), but the notice itself + // never re-appears for users who saw it once and dismissed without clicking. + if (Boolean.TRUE.equals(state.getAnalyticsNoticeShown())) { + return; + } + // Mark as shown synchronously, BEFORE scheduling the EDT task, so concurrent project + // openings during startup never race into showing the balloon twice. + state.setAnalyticsNoticeShown(true); + + ApplicationManager.getApplication().invokeLater(() -> { + Notification notification = NotificationGroupManager.getInstance() + .getNotificationGroup(NOTIFICATION_GROUP_ID) + .createNotification(TITLE, CONTENT, NotificationType.INFORMATION); + + notification.addAction(new AnAction("OK, Keep Enabled") { + @Override + public void actionPerformed(@NotNull AnActionEvent e) { + DevoxxGenieStateService.getInstance().setAnalyticsNoticeAcknowledged(true); + notification.expire(); + } + }); + + notification.addAction(new AnAction("Disable") { + @Override + public void actionPerformed(@NotNull AnActionEvent e) { + DevoxxGenieStateService s = DevoxxGenieStateService.getInstance(); + s.setAnalyticsEnabled(false); + s.setAnalyticsNoticeAcknowledged(true); + notification.expire(); + } + }); + + Notifications.Bus.notify(notification, project); + }); + } +} diff --git a/src/main/java/com/devoxx/genie/service/analytics/AnalyticsService.java b/src/main/java/com/devoxx/genie/service/analytics/AnalyticsService.java new file mode 100644 index 00000000..1081a4f1 --- /dev/null +++ b/src/main/java/com/devoxx/genie/service/analytics/AnalyticsService.java @@ -0,0 +1,205 @@ +package com.devoxx.genie.service.analytics; + +import com.devoxx.genie.ui.settings.DevoxxGenieStateService; +import com.intellij.ide.plugins.PluginManagerCore; +import com.intellij.openapi.application.ApplicationInfo; +import com.intellij.openapi.application.ApplicationManager; +import com.intellij.openapi.components.Service; +import com.intellij.openapi.extensions.PluginId; +import lombok.extern.slf4j.Slf4j; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.jetbrains.annotations.TestOnly; + +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.time.Duration; +import java.util.concurrent.ThreadLocalRandom; + +/** + * Anonymous LLM provider/model usage analytics. + * + *

Sends a GA4 Measurement Protocol payload through the shared GenieBuilder Cloudflare worker. + * DevoxxGenie traffic is segmented from GenieBuilder's Electron traffic via {@code app_name=devoxxgenie-intellij}. + * + *

The payload is intentionally minimal — see task-206 for the full disclosure list. No prompt + * text, response text, conversation history, file content, file paths, project names, API keys, + * or user identity is ever sent. + * + *

Calls are fire-and-forget on the application thread pool and never block the EDT. Failures + * are logged at debug level and never surfaced to the user. + */ +@Slf4j +@Service(Service.Level.APP) +public final class AnalyticsService { + + public static final String APP_NAME = "devoxxgenie-intellij"; + public static final String EVENT_PROMPT_EXECUTED = "prompt_executed"; + public static final String EVENT_MODEL_SELECTED = "model_selected"; + + private static final String PLUGIN_ID = "com.devoxx.genie"; + + private final String sessionId; + private HttpClient httpClient; + private boolean synchronousForTest = false; + + public AnalyticsService() { + this.sessionId = generateSessionId(); + } + + public static AnalyticsService getInstance() { + return ApplicationManager.getApplication().getService(AnalyticsService.class); + } + + public void trackPromptExecuted(@Nullable String providerId, @Nullable String modelName) { + send(EVENT_PROMPT_EXECUTED, providerId, modelName); + } + + public void trackModelSelected(@Nullable String providerId, @Nullable String modelName) { + send(EVENT_MODEL_SELECTED, providerId, modelName); + } + + private void send(@NotNull String eventName, @Nullable String providerId, @Nullable String modelName) { + DevoxxGenieStateService state = DevoxxGenieStateService.getInstance(); + + // Hard precondition gates — never emit before consent or when disabled. + if (!Boolean.TRUE.equals(state.getAnalyticsNoticeAcknowledged())) { + return; + } + if (!Boolean.TRUE.equals(state.getAnalyticsEnabled())) { + return; + } + + // Provider/model are required for both events to be useful. + if (providerId == null || providerId.isEmpty() || modelName == null || modelName.isEmpty()) { + return; + } + + String endpoint = state.getAnalyticsEndpoint(); + if (endpoint == null || endpoint.isEmpty()) { + return; + } + + String clientId = state.getAnalyticsClientId(); + String payload = buildPayload(clientId, eventName, providerId, modelName); + + if (synchronousForTest) { + postSilently(endpoint, payload); + } else { + ApplicationManager.getApplication().executeOnPooledThread(() -> postSilently(endpoint, payload)); + } + } + + private void postSilently(@NotNull String endpoint, @NotNull String payload) { + try { + HttpRequest request = HttpRequest.newBuilder() + .uri(URI.create(endpoint)) + .timeout(Duration.ofSeconds(5)) + .header("Content-Type", "application/json") + .POST(HttpRequest.BodyPublishers.ofString(payload)) + .build(); + HttpResponse response = client().send(request, HttpResponse.BodyHandlers.discarding()); + if (response.statusCode() / 100 != 2) { + log.debug("Analytics endpoint returned {}", response.statusCode()); + } + } catch (Exception e) { + log.debug("Analytics post failed: {}", e.getMessage()); + } + } + + String buildPayload(@NotNull String clientId, + @NotNull String eventName, + @NotNull String providerId, + @NotNull String modelName) { + StringBuilder sb = new StringBuilder(384); + sb.append('{') + .append("\"client_id\":\"").append(escape(clientId)).append("\",") + .append("\"events\":[{") + .append("\"name\":\"").append(escape(eventName)).append("\",") + .append("\"params\":{") + .append("\"provider_id\":\"").append(escape(providerId)).append("\",") + .append("\"model_name\":\"").append(escape(modelName)).append("\",") + .append("\"app_name\":\"").append(APP_NAME).append("\",") + .append("\"app_version\":\"").append(escape(pluginVersion())).append("\",") + .append("\"ide_version\":\"").append(escape(ideVersion())).append("\",") + .append("\"session_id\":\"").append(sessionId).append("\",") + .append("\"engagement_time_msec\":1") + .append("}}]}") + ; + return sb.toString(); + } + + @NotNull + private static String escape(@NotNull String value) { + StringBuilder sb = new StringBuilder(value.length() + 8); + for (int i = 0; i < value.length(); i++) { + char c = value.charAt(i); + switch (c) { + case '\\': sb.append("\\\\"); break; + case '"': sb.append("\\\""); break; + case '\n': sb.append("\\n"); break; + case '\r': sb.append("\\r"); break; + case '\t': sb.append("\\t"); break; + default: + if (c < 0x20) { + sb.append(String.format("\\u%04x", (int) c)); + } else { + sb.append(c); + } + } + } + return sb.toString(); + } + + @NotNull + private static String pluginVersion() { + try { + var descriptor = PluginManagerCore.getPlugin(PluginId.getId(PLUGIN_ID)); + if (descriptor != null && descriptor.getVersion() != null) { + return descriptor.getVersion(); + } + } catch (Exception ignored) { + // fall through + } + return "unknown"; + } + + @NotNull + private static String ideVersion() { + try { + return ApplicationInfo.getInstance().getFullVersion(); + } catch (Exception e) { + return "unknown"; + } + } + + @NotNull + private static String generateSessionId() { + // 10-digit string, matching GenieBuilder analytics-service.ts format. + long n = ThreadLocalRandom.current().nextLong(1_000_000_000L, 10_000_000_000L); + return Long.toString(n); + } + + @NotNull + String getSessionId() { + return sessionId; + } + + @NotNull + private synchronized HttpClient client() { + if (httpClient == null) { + httpClient = HttpClient.newBuilder() + .connectTimeout(Duration.ofSeconds(5)) + .build(); + } + return httpClient; + } + + @TestOnly + synchronized void setHttpClientForTest(@Nullable HttpClient client) { + this.httpClient = client; + this.synchronousForTest = true; + } +} diff --git a/src/main/java/com/devoxx/genie/service/prompt/PromptExecutionService.java b/src/main/java/com/devoxx/genie/service/prompt/PromptExecutionService.java index 6082c55e..400d23a6 100644 --- a/src/main/java/com/devoxx/genie/service/prompt/PromptExecutionService.java +++ b/src/main/java/com/devoxx/genie/service/prompt/PromptExecutionService.java @@ -1,7 +1,9 @@ package com.devoxx.genie.service.prompt; +import com.devoxx.genie.model.LanguageModel; import com.devoxx.genie.model.request.ChatMessageContext; import com.devoxx.genie.service.FileListManager; +import com.devoxx.genie.service.analytics.AnalyticsService; import com.devoxx.genie.service.prompt.cancellation.PromptCancellationService; import com.devoxx.genie.service.prompt.command.PromptCommandProcessor; import com.devoxx.genie.service.prompt.error.ExecutionException; @@ -77,11 +79,16 @@ public void executePrompt(@NotNull ChatMessageContext context, // Process commands Optional processedPrompt = commandProcessor.processCommands(context, panel); if (processedPrompt.isEmpty()) { - // Command processing indicated we should stop + // Command processing indicated we should stop — local-only command, no LLM dispatch. enableButtons.run(); return; } + // At this point we have committed to dispatching the prompt to an LLM. Record the + // anonymous provider/model usage event (task-206). Slash commands handled locally + // never reach this line, so the count reflects real LLM intent. + trackPromptExecuted(context); + // Start a background progress indicator ProgressManager.getInstance().run( new Task.Backgroundable(project, "Working...", true) { @@ -130,6 +137,24 @@ public void onCancel() { })); } + /** + * Anonymous usage analytics — fires once per real LLM dispatch attempt (task-206). + * Provider and model only; never any prompt content. + */ + private void trackPromptExecuted(@NotNull ChatMessageContext context) { + try { + LanguageModel model = context.getLanguageModel(); + if (model == null || model.getProvider() == null) { + return; + } + AnalyticsService.getInstance().trackPromptExecuted( + model.getProvider().getName(), + model.getModelName()); + } catch (Exception e) { + log.debug("Analytics tracking skipped: {}", e.getMessage()); + } + } + /** * Stop the execution for a specific project. * diff --git a/src/main/java/com/devoxx/genie/ui/panel/LlmProviderPanel.java b/src/main/java/com/devoxx/genie/ui/panel/LlmProviderPanel.java index ea5474f6..0097b0a8 100644 --- a/src/main/java/com/devoxx/genie/ui/panel/LlmProviderPanel.java +++ b/src/main/java/com/devoxx/genie/ui/panel/LlmProviderPanel.java @@ -60,6 +60,15 @@ public class LlmProviderPanel extends JBPanel implements LLMSe private boolean isInitializationComplete = false; private boolean isUpdatingModelNames = false; + /** + * @return {@code true} while the model name combo is being repopulated programmatically + * (provider switch, settings refresh, restore). Listeners that distinguish user actions + * from programmatic updates can read this to suppress side-effects (e.g. analytics). + */ + public boolean isUpdatingModelNames() { + return isUpdatingModelNames; + } + public LlmProviderPanel(@NotNull Project project) { this(project, null); } @@ -259,6 +268,8 @@ private void notifyModelChanges(@NonNull ModelProvider selectedProvider, */ public void updateModelNamesComboBox(String modelProvider) { Optional.ofNullable(modelProvider).ifPresent(provider -> { + boolean wasUpdating = isUpdatingModelNames; + isUpdatingModelNames = true; try { modelNameComboBox.removeAllItems(); modelNameComboBox.setVisible(true); @@ -281,6 +292,8 @@ public void updateModelNamesComboBox(String modelProvider) { } catch (Exception e) { log.error("Error updating model names", e); Messages.showErrorDialog(project, "Failed to update model names: " + e.getMessage(), "Error"); + } finally { + isUpdatingModelNames = wasUpdating; } }); } @@ -331,12 +344,18 @@ public void restoreLastSelectedProvider() { */ public void restoreLastSelectedLanguageModel() { if (lastSelectedLanguageModel != null) { - for (int i = 0; i < modelNameComboBox.getItemCount(); i++) { - LanguageModel model = modelNameComboBox.getItemAt(i); - if (model.getModelName().equals(lastSelectedLanguageModel)) { - modelNameComboBox.setSelectedIndex(i); - break; + boolean wasUpdating = isUpdatingModelNames; + isUpdatingModelNames = true; + try { + for (int i = 0; i < modelNameComboBox.getItemCount(); i++) { + LanguageModel model = modelNameComboBox.getItemAt(i); + if (model.getModelName().equals(lastSelectedLanguageModel)) { + modelNameComboBox.setSelectedIndex(i); + break; + } } + } finally { + isUpdatingModelNames = wasUpdating; } } } diff --git a/src/main/java/com/devoxx/genie/ui/settings/DevoxxGenieStateService.java b/src/main/java/com/devoxx/genie/ui/settings/DevoxxGenieStateService.java index dcbc568c..5b94b91e 100644 --- a/src/main/java/com/devoxx/genie/ui/settings/DevoxxGenieStateService.java +++ b/src/main/java/com/devoxx/genie/ui/settings/DevoxxGenieStateService.java @@ -283,6 +283,25 @@ public static DevoxxGenieStateService getInstance() { private EventAutomationSettings eventAutomationSettings = new EventAutomationSettings(); private Boolean eventAutomationEnabled = false; + // Anonymous usage analytics (LLM provider + model only) + // See task-206. Endpoint is the GenieBuilder Cloudflare worker reused for DevoxxGenie via app_name segmentation. + private Boolean analyticsEnabled = true; + private Boolean analyticsNoticeShown = false; + private Boolean analyticsNoticeAcknowledged = false; + private String analyticsClientId = ""; + private String analyticsEndpoint = "https://delicate-morning-ff55.devoxx.workers.dev"; + + /** + * Returns the persisted anonymous client id, generating a new UUID on first access + * and persisting it so it stays stable across IDE restarts and plugin updates. + */ + public String getAnalyticsClientId() { + if (analyticsClientId == null || analyticsClientId.isEmpty()) { + analyticsClientId = UUID.randomUUID().toString(); + } + return analyticsClientId; + } + // Inline completion settings private String inlineCompletionProvider = ""; // "", "Ollama", or "LMStudio" private String inlineCompletionModel = ""; diff --git a/src/main/java/com/devoxx/genie/ui/settings/general/GeneralSettingsComponent.java b/src/main/java/com/devoxx/genie/ui/settings/general/GeneralSettingsComponent.java new file mode 100644 index 00000000..995a76c5 --- /dev/null +++ b/src/main/java/com/devoxx/genie/ui/settings/general/GeneralSettingsComponent.java @@ -0,0 +1,92 @@ +package com.devoxx.genie.ui.settings.general; + +import com.devoxx.genie.ui.settings.DevoxxGenieStateService; +import com.intellij.ui.components.JBLabel; +import com.intellij.util.ui.JBUI; +import com.intellij.util.ui.UIUtil; + +import javax.swing.*; +import java.awt.*; + +/** + * Settings UI for general DevoxxGenie options. Currently exposes the anonymous usage + * analytics opt-out (task-206). Help text below the checkbox enumerates every field that + * is sent and what is never sent. + */ +public class GeneralSettingsComponent { + + private final JPanel panel; + private final JCheckBox analyticsEnabledCheckBox; + + public GeneralSettingsComponent() { + DevoxxGenieStateService state = DevoxxGenieStateService.getInstance(); + + analyticsEnabledCheckBox = new JCheckBox("Send anonymous usage statistics"); + analyticsEnabledCheckBox.setSelected(Boolean.TRUE.equals(state.getAnalyticsEnabled())); + + JBLabel sentHeader = new JBLabel("What is sent (per LLM prompt or model selection):"); + JBLabel sentList = new JBLabel( + "

    " + + "
  • An anonymous install ID (UUID), generated once and stored locally
    • " + + "
  • A per-launch session ID (random 10-digit number)
    • " + + "
  • Plugin version and IDE version
    • " + + "
  • LLM provider name (e.g. anthropic, ollama)
    • " + + "
  • LLM model name (e.g. claude-3-5-sonnet)
    • " + + "
"); + + JBLabel notSentHeader = new JBLabel("What is never sent:"); + JBLabel notSentList = new JBLabel( + "
    " + + "
  • Prompt text, response text, conversation history
    • " + + "
  • File content, file paths, project name, git remote
    • " + + "
  • API keys, credentials, user name, email
    • " + + "
  • Token counts or cost data
    • " + + "
This data is used solely to guide which LLM providers and models receive engineering investment."); + + Color subtle = UIUtil.getContextHelpForeground(); + for (JBLabel l : new JBLabel[]{sentHeader, sentList, notSentHeader, notSentList}) { + l.setForeground(subtle); + } + + panel = new JPanel(); + panel.setLayout(new BoxLayout(panel, BoxLayout.Y_AXIS)); + panel.setBorder(JBUI.Borders.empty(12)); + + analyticsEnabledCheckBox.setAlignmentX(Component.LEFT_ALIGNMENT); + sentHeader.setAlignmentX(Component.LEFT_ALIGNMENT); + sentList.setAlignmentX(Component.LEFT_ALIGNMENT); + notSentHeader.setAlignmentX(Component.LEFT_ALIGNMENT); + notSentList.setAlignmentX(Component.LEFT_ALIGNMENT); + + panel.add(analyticsEnabledCheckBox); + panel.add(Box.createVerticalStrut(8)); + panel.add(sentHeader); + panel.add(sentList); + panel.add(Box.createVerticalStrut(8)); + panel.add(notSentHeader); + panel.add(notSentList); + panel.add(Box.createVerticalGlue()); + } + + public JPanel getPanel() { + return panel; + } + + public boolean isModified() { + return analyticsEnabledCheckBox.isSelected() + != Boolean.TRUE.equals(DevoxxGenieStateService.getInstance().getAnalyticsEnabled()); + } + + public void apply() { + DevoxxGenieStateService state = DevoxxGenieStateService.getInstance(); + state.setAnalyticsEnabled(analyticsEnabledCheckBox.isSelected()); + // Touching the setting in the UI counts as informed acknowledgement — so we never + // re-show the first-launch notice for users who configured the toggle explicitly. + state.setAnalyticsNoticeAcknowledged(true); + } + + public void reset() { + analyticsEnabledCheckBox.setSelected( + Boolean.TRUE.equals(DevoxxGenieStateService.getInstance().getAnalyticsEnabled())); + } +} diff --git a/src/main/java/com/devoxx/genie/ui/settings/general/GeneralSettingsConfigurable.java b/src/main/java/com/devoxx/genie/ui/settings/general/GeneralSettingsConfigurable.java new file mode 100644 index 00000000..f7173e76 --- /dev/null +++ b/src/main/java/com/devoxx/genie/ui/settings/general/GeneralSettingsConfigurable.java @@ -0,0 +1,52 @@ +package com.devoxx.genie.ui.settings.general; + +import com.intellij.openapi.options.Configurable; +import org.jetbrains.annotations.Nls; +import org.jetbrains.annotations.Nullable; + +import javax.swing.*; + +/** + * Top-level "General" configurable for DevoxxGenie. Currently hosts the anonymous + * usage analytics opt-out toggle (task-206); future general toggles can live here too. + */ +public class GeneralSettingsConfigurable implements Configurable { + + private GeneralSettingsComponent component; + + @Nls(capitalization = Nls.Capitalization.Title) + @Override + public String getDisplayName() { + return "General"; + } + + @Override + public @Nullable JComponent createComponent() { + component = new GeneralSettingsComponent(); + return component.getPanel(); + } + + @Override + public boolean isModified() { + return component != null && component.isModified(); + } + + @Override + public void apply() { + if (component != null) { + component.apply(); + } + } + + @Override + public void reset() { + if (component != null) { + component.reset(); + } + } + + @Override + public void disposeUIResources() { + component = null; + } +} diff --git a/src/main/java/com/devoxx/genie/ui/window/DevoxxGenieToolWindowContent.java b/src/main/java/com/devoxx/genie/ui/window/DevoxxGenieToolWindowContent.java index 1cc5ac40..3e0f9ca1 100644 --- a/src/main/java/com/devoxx/genie/ui/window/DevoxxGenieToolWindowContent.java +++ b/src/main/java/com/devoxx/genie/ui/window/DevoxxGenieToolWindowContent.java @@ -4,6 +4,7 @@ import com.devoxx.genie.model.LanguageModel; import com.devoxx.genie.model.enumarations.ModelProvider; import com.devoxx.genie.service.ExternalPromptService; +import com.devoxx.genie.service.analytics.AnalyticsService; import com.devoxx.genie.service.conversations.ConversationStorageService; import com.devoxx.genie.ui.component.InputSwitch; import com.devoxx.genie.ui.component.border.AnimatedGlowingBorder; @@ -75,6 +76,7 @@ public class DevoxxGenieToolWindowContent implements SettingsChangeListener, Glo private PromptOutputPanel promptOutputPanel; private ExoClusterPanel exoClusterPanel; private boolean isInitializationComplete = false; + private boolean suppressModelSelectionTracking = false; /** * The Devoxx Genie Tool Window Content constructor. @@ -251,19 +253,24 @@ public void settingsChanged(boolean hasKey) { ModelProvider currentProvider = (ModelProvider) llmProviderPanel.getModelProviderComboBox().getSelectedItem(); LanguageModel currentModel = (LanguageModel) llmProviderPanel.getModelNameComboBox().getSelectedItem(); - llmProviderPanel.getModelProviderComboBox().removeAllItems(); - llmProviderPanel.getModelNameComboBox().removeAllItems(); - llmProviderPanel.addModelProvidersToComboBox(); + suppressModelSelectionTracking = true; + try { + llmProviderPanel.getModelProviderComboBox().removeAllItems(); + llmProviderPanel.getModelNameComboBox().removeAllItems(); + llmProviderPanel.addModelProvidersToComboBox(); - if (currentProvider != null) { - llmProviderPanel.getModelProviderComboBox().setSelectedItem(currentProvider); - llmProviderPanel.updateModelNamesComboBox(currentProvider.getName()); + if (currentProvider != null) { + llmProviderPanel.getModelProviderComboBox().setSelectedItem(currentProvider); + llmProviderPanel.updateModelNamesComboBox(currentProvider.getName()); - if (currentModel != null) { - llmProviderPanel.getModelNameComboBox().setSelectedItem(currentModel); + if (currentModel != null) { + llmProviderPanel.getModelNameComboBox().setSelectedItem(currentModel); + } + } else { + llmProviderPanel.setLastSelectedProvider(); } - } else { - llmProviderPanel.setLastSelectedProvider(); + } finally { + suppressModelSelectionTracking = false; } } @@ -277,6 +284,21 @@ private void processModelNameSelection(@NotNull ActionEvent e) { if (selectedModel != null) { DevoxxGenieStateService.getInstance().setSelectedLanguageModel(getMemoryKey(), selectedModel.getModelName()); submitPanel.getActionButtonsPanel().updateTokenUsage(selectedModel.getInputMaxTokens()); + + // Anonymous usage analytics — fires only on user-initiated changes (task-206). + // Three guards combine to filter out programmatic combo events: + // - isInitializationComplete: blocks events during initial state load + // - suppressModelSelectionTracking: blocks events during settingsChanged() + // when this class itself is repopulating combos + // - llmProviderPanel.isUpdatingModelNames(): blocks events fired indirectly + // by provider switch / refresh / restore inside LlmProviderPanel + if (!suppressModelSelectionTracking + && !llmProviderPanel.isUpdatingModelNames() + && selectedModel.getProvider() != null) { + AnalyticsService.getInstance().trackModelSelected( + selectedModel.getProvider().getName(), + selectedModel.getModelName()); + } } } } diff --git a/src/main/resources/META-INF/plugin.xml b/src/main/resources/META-INF/plugin.xml index 79fb00e1..14a31ded 100644 --- a/src/main/resources/META-INF/plugin.xml +++ b/src/main/resources/META-INF/plugin.xml @@ -42,9 +42,26 @@

Need help? Visit our Getting Started Guide or check the Troubleshooting page.

+ +

Privacy & Anonymous Usage Analytics

+

To guide which LLM providers and models receive engineering investment, DevoxxGenie collects anonymous usage data when you run a prompt or change models.

+

What is sent:

+
    +
  • An anonymous install ID (UUID), generated once and stored locally
    • +
  • A per-launch session ID (random 10-digit number)
    • +
  • Plugin version and IDE version
    • +
  • LLM provider name (e.g. anthropic, ollama)
    • +
  • LLM model name (e.g. claude-3-5-sonnet)
    • +
+

What is never sent: prompt text, response text, conversation history, file content, file paths, project names, git remotes, API keys, credentials, token counts, or cost data.

+

A first-launch notification asks for your consent before any data is sent. You can change this any time in Settings → DevoxxGenie → General.

]]> Unreleased +
    +
  • Feat: Anonymous LLM provider/model usage analytics (opt-out). Sends only: anonymous install ID (UUID), per-launch session ID, plugin version, IDE version, LLM provider name, LLM model name. Never sends prompt text, response text, file content, file paths, project names, conversation history, API keys, or anything that could identify you. First-launch notification asks for consent; toggle in Settings → DevoxxGenie → General.
    • +

v1.4.1

  • Update: Upgrade 40 Gradle dependencies including Langchain4J to 1.12.2
    • @@ -450,6 +467,11 @@ instance="com.devoxx.genie.ui.settings.llm.LLMProvidersConfigurable" displayName="DevoxxGenie"/> + + Several Exo tests use Mockito to stub HTTP calls but exercise multi-step flows that have + * since drifted from the production code, so they only pass against a real Exo server running on + * the default port. Use {@link #isExoServerRunning()} together with {@code Assumptions.assumeTrue} + * to skip these tests on machines without a local Exo instance. + */ +public final class ExoTestAssumptions { + + private static final String EXO_HOST = "localhost"; + private static final int EXO_PORT = 52415; + private static final int CONNECT_TIMEOUT_MS = 300; + + private ExoTestAssumptions() { + } + + /** + * @return {@code true} when an Exo server can be reached on {@code localhost:52415}. + */ + public static boolean isExoServerRunning() { + try (Socket socket = new Socket()) { + socket.connect(new InetSocketAddress(EXO_HOST, EXO_PORT), CONNECT_TIMEOUT_MS); + return true; + } catch (Exception e) { + return false; + } + } +} diff --git a/src/test/java/com/devoxx/genie/service/analytics/AnalyticsServiceTest.java b/src/test/java/com/devoxx/genie/service/analytics/AnalyticsServiceTest.java new file mode 100644 index 00000000..c413d02a --- /dev/null +++ b/src/test/java/com/devoxx/genie/service/analytics/AnalyticsServiceTest.java @@ -0,0 +1,252 @@ +package com.devoxx.genie.service.analytics; + +import com.devoxx.genie.ui.settings.DevoxxGenieStateService; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; + +import java.io.IOException; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import java.util.UUID; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.when; + +/** + * Unit tests for {@link AnalyticsService} (task-206). + * + *

    Verifies payload schema, opt-out behavior, consent gating, silent failure, client_id + * persistence, and the strict "no PII" guarantee. + */ +class AnalyticsServiceTest { + + private DevoxxGenieStateService state; + private AnalyticsService service; + private RecordingHttpClient httpClient; + + @BeforeEach + void setUp() { + state = new DevoxxGenieStateService(); + state.setAnalyticsEnabled(true); + state.setAnalyticsNoticeAcknowledged(true); + state.setAnalyticsClientId(""); + state.setAnalyticsEndpoint("https://example.invalid/collect"); + + service = new AnalyticsService(); + httpClient = new RecordingHttpClient(); + service.setHttpClientForTest(httpClient); + } + + private void runWithState(Runnable action) { + try (MockedStatic mocked = mockStatic(DevoxxGenieStateService.class)) { + mocked.when(DevoxxGenieStateService::getInstance).thenReturn(state); + action.run(); + } + } + + @Test + void payloadContainsExactlyTheExpectedFields() throws Exception { + runWithState(() -> service.trackPromptExecuted("anthropic", "claude-3-5-sonnet")); + httpClient.awaitOne(); + + String body = httpClient.lastBody(); + assertThat(body).contains("\"client_id\":\""); + assertThat(body).contains("\"name\":\"prompt_executed\""); + assertThat(body).contains("\"provider_id\":\"anthropic\""); + assertThat(body).contains("\"model_name\":\"claude-3-5-sonnet\""); + assertThat(body).contains("\"app_name\":\"devoxxgenie-intellij\""); + assertThat(body).contains("\"app_version\":\""); + assertThat(body).contains("\"ide_version\":\""); + assertThat(body).contains("\"session_id\":\""); + assertThat(body).contains("\"engagement_time_msec\":1"); + + // Strict allowlist — no extra parameter keys ever. + Set allowedParamKeys = Set.of( + "provider_id", "model_name", "app_name", + "app_version", "ide_version", "session_id", "engagement_time_msec"); + Pattern keyPattern = Pattern.compile("\"params\":\\{([^}]*)}"); + Matcher m = keyPattern.matcher(body); + assertThat(m.find()).isTrue(); + String paramsBlock = m.group(1); + Pattern fieldPattern = Pattern.compile("\"([a-z_]+)\"\\s*:"); + Matcher fm = fieldPattern.matcher(paramsBlock); + while (fm.find()) { + assertThat(allowedParamKeys).contains(fm.group(1)); + } + } + + @Test + void sessionIdIsTenDigitString() { + assertThat(service.getSessionId()).matches("\\d{10}"); + } + + @Test + void clientIdIsGeneratedOnceAndPersisted() { + runWithState(() -> { + service.trackPromptExecuted("ollama", "llama3"); + httpClient.awaitOne(); + }); + String firstId = state.getAnalyticsClientId(); + assertThat(firstId).isNotEmpty(); + // Ensure it parses as a UUID + assertThat(UUID.fromString(firstId)).isNotNull(); + + // A second call must reuse the same id. + runWithState(() -> { + service.trackPromptExecuted("ollama", "llama3"); + httpClient.awaitTotal(2); + }); + assertThat(state.getAnalyticsClientId()).isEqualTo(firstId); + } + + @Test + void disabledStateSendsNothing() { + state.setAnalyticsEnabled(false); + runWithState(() -> service.trackPromptExecuted("anthropic", "claude")); + httpClient.awaitNothingFor(150); + assertThat(httpClient.requestCount()).isZero(); + } + + @Test + void noticeNotAcknowledgedSendsNothing() { + state.setAnalyticsNoticeAcknowledged(false); + runWithState(() -> service.trackPromptExecuted("anthropic", "claude")); + httpClient.awaitNothingFor(150); + assertThat(httpClient.requestCount()).isZero(); + } + + @Test + void missingProviderOrModelSendsNothing() { + runWithState(() -> { + service.trackPromptExecuted(null, "claude"); + service.trackPromptExecuted("anthropic", null); + service.trackPromptExecuted("", "claude"); + service.trackPromptExecuted("anthropic", ""); + }); + httpClient.awaitNothingFor(150); + assertThat(httpClient.requestCount()).isZero(); + } + + @Test + void networkFailureIsSilent() { + httpClient.throwOnSend = true; + runWithState(() -> service.trackPromptExecuted("anthropic", "claude")); + // Should not throw, should not crash. We just want the call to complete. + httpClient.awaitOne(); + assertThat(httpClient.requestCount()).isOne(); + } + + @Test + void payloadContainsNoPiiEvenWhenInputsLookLikePaths() { + // A defensive test: even if we ever pass something path-like, the payload only carries + // the two strings we passed and nothing else (no project name, no file content). + runWithState(() -> service.trackModelSelected("anthropic", "claude-3-5-sonnet")); + httpClient.awaitOne(); + + String body = httpClient.lastBody(); + // Forbidden substrings — anything that would imply a leak. + assertThat(body).doesNotContain("/Users/"); + assertThat(body).doesNotContain("file:"); + assertThat(body).doesNotContain("project"); + assertThat(body).doesNotContain("git"); + assertThat(body).doesNotContain("apiKey"); + assertThat(body).doesNotContain("password"); + assertThat(body).doesNotContain("conversation"); + } + + /** Minimal recording HttpClient stub. */ + private static class RecordingHttpClient extends HttpClient { + + private final List requests = new ArrayList<>(); + private final List bodies = new ArrayList<>(); + boolean throwOnSend = false; + + synchronized int requestCount() { + return requests.size(); + } + + synchronized String lastBody() { + return bodies.get(bodies.size() - 1); + } + + void awaitOne() { + awaitTotal(1); + } + + void awaitTotal(int n) { + long deadline = System.currentTimeMillis() + 2000; + while (System.currentTimeMillis() < deadline) { + synchronized (this) { + if (requests.size() >= n) return; + } + try { + Thread.sleep(10); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return; + } + } + } + + void awaitNothingFor(long ms) { + try { + Thread.sleep(ms); + } catch (InterruptedException ignored) { + Thread.currentThread().interrupt(); + } + } + + @Override + public synchronized HttpResponse send(HttpRequest request, + HttpResponse.BodyHandler responseBodyHandler) throws IOException { + requests.add(request); + request.bodyPublisher().ifPresent(p -> { + StringBuilder sb = new StringBuilder(); + p.subscribe(new java.util.concurrent.Flow.Subscriber<>() { + @Override public void onSubscribe(java.util.concurrent.Flow.Subscription s) { s.request(Long.MAX_VALUE); } + @Override public void onNext(java.nio.ByteBuffer item) { + byte[] bytes = new byte[item.remaining()]; + item.get(bytes); + sb.append(new String(bytes)); + } + @Override public void onError(Throwable t) { } + @Override public void onComplete() { } + }); + bodies.add(sb.toString()); + }); + if (throwOnSend) { + throw new IOException("simulated"); + } + @SuppressWarnings("unchecked") + HttpResponse stub = (HttpResponse) mock(HttpResponse.class); + when(stub.statusCode()).thenReturn(204); + return stub; + } + + @Override public java.util.Optional cookieHandler() { return java.util.Optional.empty(); } + @Override public java.util.Optional connectTimeout() { return java.util.Optional.empty(); } + @Override public Redirect followRedirects() { return Redirect.NEVER; } + @Override public java.util.Optional proxy() { return java.util.Optional.empty(); } + @Override public javax.net.ssl.SSLContext sslContext() { return null; } + @Override public javax.net.ssl.SSLParameters sslParameters() { return new javax.net.ssl.SSLParameters(); } + @Override public java.util.Optional authenticator() { return java.util.Optional.empty(); } + @Override public Version version() { return Version.HTTP_1_1; } + @Override public java.util.Optional executor() { return java.util.Optional.empty(); } + @Override public java.util.concurrent.CompletableFuture> sendAsync(HttpRequest request, HttpResponse.BodyHandler responseBodyHandler) { + return java.util.concurrent.CompletableFuture.failedFuture(new UnsupportedOperationException()); + } + @Override public java.util.concurrent.CompletableFuture> sendAsync(HttpRequest request, HttpResponse.BodyHandler responseBodyHandler, HttpResponse.PushPromiseHandler pushPromiseHandler) { + return java.util.concurrent.CompletableFuture.failedFuture(new UnsupportedOperationException()); + } + } +} diff --git a/src/test/java/com/devoxx/genie/ui/panel/LlmProviderPanelTest.java b/src/test/java/com/devoxx/genie/ui/panel/LlmProviderPanelTest.java index 1143ff58..34b9e6e4 100644 --- a/src/test/java/com/devoxx/genie/ui/panel/LlmProviderPanelTest.java +++ b/src/test/java/com/devoxx/genie/ui/panel/LlmProviderPanelTest.java @@ -113,6 +113,49 @@ void tearDown() { appManagerMockedStatic.close(); } + @Test + void isUpdatingModelNames_isTrueDuringUpdateAndFalseAfter() { + // Regression for task-206: model_selected analytics must be suppressible during + // programmatic combo repopulation. The flag must be true while updateModelNamesComboBox + // runs and must reset to false on completion. + List providers = List.of(ModelProvider.Ollama, ModelProvider.OpenAI); + when(llmProviderService.getAvailableModelProviders()).thenReturn(providers); + + ChatModelFactory factory = mock(ChatModelFactory.class); + boolean[] flagSnapshotDuringPopulate = new boolean[]{false}; + // Capture the flag value at the moment the factory is asked for models — that runs + // inside updateModelNamesComboBox after isUpdatingModelNames was set true. + LlmProviderPanel[] panelHolder = new LlmProviderPanel[1]; + when(factory.getModels()).thenAnswer(invocation -> { + flagSnapshotDuringPopulate[0] = panelHolder[0].isUpdatingModelNames(); + return List.of( + LanguageModel.builder().provider(ModelProvider.OpenAI) + .modelName("gpt-4").displayName("GPT-4").build() + ); + }); + + factoryProviderMockedStatic.when(() -> ChatModelFactoryProvider.getFactoryByProvider("OpenAI")) + .thenReturn(Optional.of(factory)); + factoryProviderMockedStatic.when(() -> ChatModelFactoryProvider.getFactoryByProvider("Ollama")) + .thenReturn(Optional.empty()); + + LlmProviderPanel panel = new LlmProviderPanel(project); + panelHolder[0] = panel; + + assertThat(panel.isUpdatingModelNames()) + .as("Flag is false before any programmatic update") + .isFalse(); + + panel.updateModelNamesComboBox("OpenAI"); + + assertThat(flagSnapshotDuringPopulate[0]) + .as("Flag must be true while combo is being repopulated") + .isTrue(); + assertThat(panel.isUpdatingModelNames()) + .as("Flag must reset to false after updateModelNamesComboBox completes") + .isFalse(); + } + @Test void testUpdateModelNamesComboBox_WithModels_PopulatesComboBox() { List providers = List.of(ModelProvider.Ollama, ModelProvider.OpenAI);