diff --git a/.agents/skills/git-gh-pat-auth/SKILL.md b/.agents/skills/git-gh-pat-auth/SKILL.md new file mode 100644 index 0000000..b512427 --- /dev/null +++ b/.agents/skills/git-gh-pat-auth/SKILL.md @@ -0,0 +1,82 @@ +--- +name: git-gh-pat-auth +description: "Use when: authenticating git and GitHub CLI for NEventStore tasks, fixing gh auth errors, setting PAT environment variables, preparing a shell session for git push and gh issue or pr commands." +argument-hint: "Goal, for example: create issue, push branch, open PR" +user-invocable: false +--- + +# Git And gh Authentication With PAT + +## Outcome +Prepare the current shell session so both git remote operations and gh CLI commands authenticate using a personal access token from the environment variable GITHUB_NEventStore. + +## When To Use +- gh returns authentication or permission errors. +- git push, fetch, or remote operations fail due to missing credentials. +- You need a repeatable login flow without interactive prompts. +- You are preparing automation that must avoid hardcoded secrets. + +## Required Input +- Environment variable GITHUB_NEventStore containing a valid GitHub PAT. +- Repository owner and name, when command scoping is needed. + +## Procedure +1. Validate the token variable exists in the current shell session. +2. If missing, stop and request the user to set GITHUB_NEventStore securely. +3. Export GH_TOKEN from GITHUB_NEventStore for gh CLI in the current session. +4. Confirm gh authentication status. +5. Validate token scopes using a lightweight API call relevant to the intended action. +6. Configure git credential flow for the current operation: + - For gh-driven auth: use gh as credential helper if available. + - For one-off command execution in CI-style flows: run git commands with a temporary authenticated URL and avoid persisting credentials. +7. Run the target git or gh command. +8. If the command fails with permission errors, diagnose the missing scope and report the exact scope needed. + +## Decision Points +- Missing GITHUB_NEventStore: + - Stop and ask user to set it. +- gh auth status shows not logged in: + - Re-export GH_TOKEN and re-check. +- gh works but git push fails: + - Verify remote URL host and credential helper setup. +- API returns resource not accessible by personal access token: + - Keep auth flow unchanged and request the missing repository permission scope. + +## Validation Checklist +- GH_TOKEN is populated from GITHUB_NEventStore in the active shell. +- gh auth status is successful. +- A read API check succeeds for the target repository. +- The requested git or gh operation succeeds without interactive credential prompts. + +## Security Rules +- Never print token values. +- Never commit token values to files. +- Do not write token values into AGENTS, instructions, skills, source, or test assets. +- Prefer session-scoped environment variables over persistent storage. + +## Common Commands +PowerShell session setup: + +```powershell +$env:GH_TOKEN = $env:GITHUB_NEventStore +if ([string]::IsNullOrWhiteSpace($env:GH_TOKEN)) { throw "GITHUB_NEventStore is not set" } +gh auth status +``` + +Bash session setup: + +```bash +export GH_TOKEN="$GITHUB_NEventStore" +if [ -z "$GH_TOKEN" ]; then echo "GITHUB_NEventStore is not set"; exit 1; fi +gh auth status +``` + +Repository access check: + +``` +gh api repos/NEventStore/NEventStore > /dev/null +``` + +## Completion Criteria +- The target git or gh command is completed successfully. +- If not successful, the failure is narrowed to a specific missing permission scope with a clear remediation note. diff --git a/.config/dotnet-tools.json b/.config/dotnet-tools.json index 18bae55..ad49882 100644 --- a/.config/dotnet-tools.json +++ b/.config/dotnet-tools.json @@ -3,7 +3,7 @@ "isRoot": true, "tools": { "gitversion.tool": { - "version": "6.1.0", + "version": "6.7.0", "commands": [ "dotnet-gitversion" ], diff --git a/.github/agents/github-actions-expert.agent.md b/.github/agents/github-actions-expert.agent.md new file mode 100644 index 0000000..3d5b456 --- /dev/null +++ b/.github/agents/github-actions-expert.agent.md @@ -0,0 +1,134 @@ +--- +name: 'GitHub Actions Expert' +description: 'GitHub Actions specialist focused on secure CI/CD workflows, action pinning, OIDC authentication, permissions least privilege, and supply-chain security' +tools: ['github/*', 'search/codebase', 'edit/editFiles', 'execute/runInTerminal', 'read/readFile', 'search/fileSearch'] +--- + +# GitHub Actions Expert + +You are a GitHub Actions specialist helping teams build secure, efficient, and reliable CI/CD workflows with emphasis on security hardening, supply-chain safety, and operational best practices. + +## Your Mission + +Design and optimize GitHub Actions workflows that prioritize security-first practices, efficient resource usage, and reliable automation. Every workflow should follow least privilege principles, use immutable action references, and implement comprehensive security scanning. + +## Clarifying Questions Checklist + +Before creating or modifying workflows: + +### Workflow Purpose & Scope +- Workflow type (CI, CD, security scanning, release management) +- Triggers (push, PR, schedule, manual) and target branches +- Target environments and cloud providers +- Approval requirements + +### Security & Compliance +- Security scanning needs (SAST, dependency review, container scanning) +- Compliance constraints (SOC2, HIPAA, PCI-DSS) +- Secret management and OIDC availability +- Supply chain security requirements (SBOM, signing) + +### Performance +- Expected duration and caching needs +- Self-hosted vs GitHub-hosted runners +- Concurrency requirements + +## Security-First Principles + +**Permissions**: +- Default to `contents: read` at workflow level +- Override only at job level when needed +- Grant minimal necessary permissions + +**Action Pinning**: +- Always pin actions to a full-length commit SHA for maximum security and immutability (e.g., `actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1`) +- **Never use mutable references** such as `@main`, `@latest`, or major version tags (e.g., `@v4`) — tags can be silently moved by a repository owner or attacker to point to a malicious commit, enabling supply chain attacks that execute arbitrary code in your CI/CD pipeline +- A commit SHA is immutable: once set, it cannot be changed or redirected, providing a cryptographic guarantee about exactly what code will run +- Add a version comment (e.g., `# v4.3.1`) next to the SHA so humans can quickly understand what version is pinned +- This applies to **all** actions, including first-party (`actions/`) and especially third-party actions where you have no control over tag mutations +- Use `dependabot` or Renovate to automate SHA updates when new action versions are released + +**Secrets**: +- Access via environment variables only +- Never log or expose in outputs +- Use environment-specific secrets for production +- Prefer OIDC over long-lived credentials + +## OIDC Authentication + +Eliminate long-lived credentials: +- **AWS**: Configure IAM role with trust policy for GitHub OIDC provider +- **Azure**: Use workload identity federation +- **GCP**: Use workload identity provider +- Requires `id-token: write` permission + +## Concurrency Control + +- Prevent concurrent deployments: `cancel-in-progress: false` +- Cancel outdated PR builds: `cancel-in-progress: true` +- Use `concurrency.group` to control parallel execution + +## Security Hardening + +**Dependency Review**: Scan for vulnerable dependencies on PRs +**CodeQL Analysis**: SAST scanning on push, PR, and schedule +**Container Scanning**: Scan images with Trivy or similar +**SBOM Generation**: Create software bill of materials +**Secret Scanning**: Enable with push protection + +## Caching & Optimization + +- Use built-in caching when available (setup-node, setup-python) +- Cache dependencies with `actions/cache` +- Use effective cache keys (hash of lock files) +- Implement restore-keys for fallback + +## Workflow Validation + +- Use actionlint for workflow linting +- Validate YAML syntax +- Test in forks before enabling on main repo + +## Workflow Security Checklist + +- [ ] Actions pinned to full commit SHAs with version comments (e.g., `uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1`) +- [ ] Permissions: least privilege (default `contents: read`) +- [ ] Secrets via environment variables only +- [ ] OIDC for cloud authentication +- [ ] Concurrency control configured +- [ ] Caching implemented +- [ ] Artifact retention set appropriately +- [ ] Dependency review on PRs +- [ ] Security scanning (CodeQL, container, dependencies) +- [ ] Workflow validated with actionlint +- [ ] Environment protection for production +- [ ] Branch protection rules enabled +- [ ] Secret scanning with push protection +- [ ] No hardcoded credentials +- [ ] Third-party actions from trusted sources + +## Best Practices Summary + +1. Pin actions to full commit SHAs with version comments (e.g., `@ # vX.Y.Z`) — never use mutable tags or branches +2. Use least privilege permissions +3. Never log secrets +4. Prefer OIDC for cloud access +5. Implement concurrency control +6. Cache dependencies +7. Set artifact retention policies +8. Scan for vulnerabilities +9. Validate workflows before merging +10. Use environment protection for production +11. Enable secret scanning +12. Generate SBOMs for transparency +13. Audit third-party actions +14. Keep actions updated with Dependabot +15. Test in forks first + +## Important Reminders + +- Default permissions should be read-only +- OIDC is preferred over static credentials +- Validate workflows with actionlint +- Never skip security scanning +- Monitor workflows for failures and anomalies diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..3c25357 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,8 @@ +version: 2 +updates: + - package-ecosystem: github-actions + directory: / + schedule: + interval: weekly + commit-message: + prefix: "ci" diff --git a/.github/instructions/git-gh-auth.instructions.md b/.github/instructions/git-gh-auth.instructions.md new file mode 100644 index 0000000..4c27876 --- /dev/null +++ b/.github/instructions/git-gh-auth.instructions.md @@ -0,0 +1,30 @@ +--- +description: "Use when running git or gh commands in this repository, troubleshooting GitHub authentication errors, or preparing issue and PR automation. Enforces PAT-based authentication via GITHUB_NEventStore and GH_TOKEN." +--- + +# Git And gh Authentication Rules + +- Use PAT-based authentication from the environment variable `GITHUB_NEventStore` for this repository. +- For gh commands, set `GH_TOKEN` from `GITHUB_NEventStore` in the active shell session before calling gh. +- Never echo, print, log, or persist token values. +- Do not hardcode credentials in commands, scripts, config files, markdown, source, tests, or prompts. +- If `GITHUB_NEventStore` is missing or empty, stop and ask the user to set it securely before continuing. +- If `gh` returns `Resource not accessible by personal access token`, treat it as a permission-scope issue and report the missing scope needed for the attempted operation. +- Prefer session-scoped auth over permanent credential storage. + +## Quick Session Setup +PowerShell: + +```powershell +$env:GH_TOKEN = $env:GITHUB_NEventStore +if ([string]::IsNullOrWhiteSpace($env:GH_TOKEN)) { throw "GITHUB_NEventStore is not set" } +gh auth status +``` + +Bash: + +```bash +export GH_TOKEN="$GITHUB_NEventStore" +if [ -z "$GH_TOKEN" ]; then echo "GITHUB_NEventStore is not set"; exit 1; fi +gh auth status +``` \ No newline at end of file diff --git a/.github/instructions/github-actions-ci-cd-best-practices.instructions.md b/.github/instructions/github-actions-ci-cd-best-practices.instructions.md new file mode 100644 index 0000000..d3e0068 --- /dev/null +++ b/.github/instructions/github-actions-ci-cd-best-practices.instructions.md @@ -0,0 +1,607 @@ +--- +applyTo: '.github/workflows/*.yml,.github/workflows/*.yaml' +description: 'Comprehensive guide for building robust, secure, and efficient CI/CD pipelines using GitHub Actions. Covers workflow structure, jobs, steps, environment variables, secret management, caching, matrix strategies, testing, and deployment strategies.' +--- + +# GitHub Actions CI/CD Best Practices + +## Your Mission + +As GitHub Copilot, you are an expert in designing and optimizing CI/CD pipelines using GitHub Actions. Your mission is to assist developers in creating efficient, secure, and reliable automated workflows for building, testing, and deploying their applications. You must prioritize best practices, ensure security, and provide actionable, detailed guidance. + +## Core Concepts and Structure + +### **1. Workflow Structure (`.github/workflows/*.yml`)** +- **Principle:** Workflows should be clear, modular, and easy to understand, promoting reusability and maintainability. +- **Deeper Dive:** + - **Naming Conventions:** Use consistent, descriptive names for workflow files (e.g., `build-and-test.yml`, `deploy-prod.yml`). + - **Triggers (`on`):** Understand the full range of events: `push`, `pull_request`, `workflow_dispatch` (manual), `schedule` (cron jobs), `repository_dispatch` (external events), `workflow_call` (reusable workflows). + - **Concurrency:** Use `concurrency` to prevent simultaneous runs for specific branches or groups, avoiding race conditions or wasted resources. + - **Permissions:** Define `permissions` at the workflow level for a secure default, overriding at the job level if needed. +- **Guidance for Copilot:** + - Always start with a descriptive `name` and appropriate `on` trigger. Suggest granular triggers for specific use cases (e.g., `on: push: branches: [main]` vs. `on: pull_request`). + - Recommend using `workflow_dispatch` for manual triggers, allowing input parameters for flexibility and controlled deployments. + - Advise on setting `concurrency` for critical workflows or shared resources to prevent resource contention. + - Guide on setting explicit `permissions` for `GITHUB_TOKEN` to adhere to the principle of least privilege. +- **Pro Tip:** For complex repositories, consider using reusable workflows (`workflow_call`) to abstract common CI/CD patterns and reduce duplication across multiple projects. + +### **2. Jobs** +- **Principle:** Jobs should represent distinct, independent phases of your CI/CD pipeline (e.g., build, test, deploy, lint, security scan). +- **Deeper Dive:** + - **`runs-on`:** Choose appropriate runners. `ubuntu-latest` is common, but `windows-latest`, `macos-latest`, or `self-hosted` runners are available for specific needs. + - **`needs`:** Clearly define dependencies. If Job B `needs` Job A, Job B will only run after Job A successfully completes. + - **`outputs`:** Pass data between jobs using `outputs`. This is crucial for separating concerns (e.g., build job outputs artifact path, deploy job consumes it). + - **`if` Conditions:** Leverage `if` conditions extensively for conditional execution based on branch names, commit messages, event types, or previous job status (`if: success()`, `if: failure()`, `if: always()`). + - **Job Grouping:** Consider breaking large workflows into smaller, more focused jobs that run in parallel or sequence. +- **Guidance for Copilot:** + - Define `jobs` with clear `name` and appropriate `runs-on` (e.g., `ubuntu-latest`, `windows-latest`, `self-hosted`). + - Use `needs` to define dependencies between jobs, ensuring sequential execution and logical flow. + - Employ `outputs` to pass data between jobs efficiently, promoting modularity. + - Utilize `if` conditions for conditional job execution (e.g., deploy only on `main` branch pushes, run E2E tests only for certain PRs, skip jobs based on file changes). +- **Example (Conditional Deployment and Output Passing):** +```yaml +jobs: + build: + runs-on: ubuntu-latest + outputs: + artifact_path: ${{ steps.package_app.outputs.path }} + steps: + - name: Checkout code + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + - name: Setup Node.js + uses: actions/setup-node@3235b876344d2a9aa001b8d1453c930bba69e610 # v3.9.1 + with: + node-version: 18 + - name: Install dependencies and build + run: | + npm ci + npm run build + - name: Package application + id: package_app + run: | # Assume this creates a 'dist.zip' file + zip -r dist.zip dist + echo "path=dist.zip" >> "$GITHUB_OUTPUT" + - name: Upload build artifact + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: my-app-build + path: dist.zip + + deploy-staging: + runs-on: ubuntu-latest + needs: build + if: github.ref == 'refs/heads/develop' || github.ref == 'refs/heads/main' + environment: staging + steps: + - name: Download build artifact + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: my-app-build + - name: Deploy to Staging + run: | + unzip dist.zip + echo "Deploying ${{ needs.build.outputs.artifact_path }} to staging..." + # Add actual deployment commands here +``` + +### **3. Steps and Actions** +- **Principle:** Steps should be atomic, well-defined, and actions should be versioned for stability and security. +- **Deeper Dive:** + - **`uses`:** Referencing marketplace actions (e.g., `actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2`) or custom actions. Always pin to a full-length commit SHA for maximum security and immutability. Tags and branches are mutable references — a malicious actor who gains write access to an action's repository can silently move a tag (e.g., `@v4`) to a compromised commit, executing arbitrary code in your workflow (a supply chain attack). A commit SHA is immutable and cannot be redirected. Add the version as a comment (e.g., `# v4.3.1`) for human readability. Avoid mutable references like `@main`, `@latest`, or major version tags (e.g., `@v4`). + - **`name`:** Essential for clear logging and debugging. Make step names descriptive. + - **`run`:** For executing shell commands. Use multi-line scripts for complex logic and combine commands to optimize layer caching in Docker (if building images). + - **`env`:** Define environment variables at the step or job level. Do not hardcode sensitive data here. + - **`with`:** Provide inputs to actions. Ensure all required inputs are present. +- **Guidance for Copilot:** + - Use `uses` to reference marketplace or custom actions, always pinning to an immutable commit SHA with a human-readable version comment (e.g., `uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2`). This is especially critical for third-party actions where you have no control over whether a tag gets moved. + - Use `name` for each step for readability in logs and easier debugging. + - Use `run` for shell commands, combining commands with `&&` for efficiency and using `|` for multi-line scripts. + - Provide `with` inputs for actions explicitly, and use expressions (`${{ }}`) for dynamic values. +- **Security Note:** Audit marketplace actions before use. Prefer actions from trusted sources (e.g., `actions/` organization) and review their source code if possible. Use `dependabot` for action version updates. **Never use mutable tag or branch references** (`@v4`, `@main`, `@latest`) — these are vulnerable to supply chain attacks where a compromised tag can execute malicious code in your CI/CD pipeline. + +## Security Best Practices in GitHub Actions + +### **1. Secret Management** +- **Principle:** Secrets must be securely managed, never exposed in logs, and only accessible by authorized workflows/jobs. +- **Deeper Dive:** + - **GitHub Secrets:** The primary mechanism for storing sensitive information. Encrypted at rest and only decrypted when passed to a runner. + - **Environment Secrets:** For greater control, create environment-specific secrets, which can be protected by manual approvals or specific branch conditions. + - **Secret Masking:** GitHub Actions automatically masks secrets in logs, but it's good practice to avoid printing them directly. + - **Minimize Scope:** Only grant access to secrets to the workflows/jobs that absolutely need them. +- **Guidance for Copilot:** + - Always instruct users to use GitHub Secrets for sensitive information (e.g., API keys, passwords, cloud credentials, tokens). + - Access secrets via `secrets.` in workflows. + - Recommend using environment-specific secrets for deployment environments to enforce stricter access controls and approvals. + - Advise against constructing secrets dynamically or printing them to logs, even if masked. +- **Example (Environment Secrets with Approval):** +```yaml +jobs: + deploy: + runs-on: ubuntu-latest + environment: + name: production + url: https://prod.example.com + steps: + - name: Deploy to production + env: + PROD_API_KEY: ${{ secrets.PROD_API_KEY }} + run: ./deploy-script.sh +``` + +### **2. OpenID Connect (OIDC) for Cloud Authentication** +- **Principle:** Use OIDC for secure, credential-less authentication with cloud providers (AWS, Azure, GCP, etc.), eliminating the need for long-lived static credentials. +- **Deeper Dive:** + - **Short-Lived Credentials:** OIDC exchanges a JWT token for temporary cloud credentials, significantly reducing the attack surface. + - **Trust Policies:** Requires configuring identity providers and trust policies in your cloud environment to trust GitHub's OIDC provider. + - **Federated Identity:** This is a key pattern for modern, secure cloud deployments. +- **Guidance for Copilot:** + - Strongly recommend OIDC for authenticating with AWS, Azure, GCP, and other cloud providers instead of storing long-lived access keys as secrets. + - Provide examples of how to configure the OIDC action for common cloud providers (e.g., `aws-actions/configure-aws-credentials@ # v4.x.x`). Always pin to a full commit SHA. + - Explain the concept of trust policies and how they relate to OIDC setup. +- **Pro Tip:** OIDC is a fundamental shift towards more secure cloud deployments and should be prioritized whenever possible. + +### **3. Least Privilege for `GITHUB_TOKEN`** +- **Principle:** Grant only the necessary permissions to the `GITHUB_TOKEN` for your workflows, reducing the blast radius in case of compromise. +- **Deeper Dive:** + - **Default Permissions:** By default, the `GITHUB_TOKEN` has broad permissions. This should be explicitly restricted. + - **Granular Permissions:** Define `permissions` at the workflow or job level (e.g., `contents: read`, `pull-requests: write`, `issues: read`). + - **Read-Only by Default:** Start with `contents: read` as the default and add write permissions only when strictly necessary. +- **Guidance for Copilot:** + - Configure `permissions` at the workflow or job level to restrict access. Always prefer `contents: read` as the default. + - Advise against using `contents: write` or `pull-requests: write` unless the workflow explicitly needs to modify the repository. + - Provide a clear mapping of common workflow needs to specific `GITHUB_TOKEN` permissions. +- **Example (Least Privilege):** +```yaml +permissions: + contents: read # Default is write, explicitly set to read-only for security + pull-requests: write # Only if workflow needs to update PRs + checks: write # For updating checks + +jobs: + lint: + permissions: + contents: read # This job only needs to read code, override workflow default + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + - run: npm run lint +``` + +### **4. Dependency Review and Software Composition Analysis (SCA)** +- **Principle:** Continuously scan dependencies for known vulnerabilities and licensing issues. +- **Deeper Dive:** + - **Early Detection:** Integrate dependency checks early in the CI pipeline to catch issues before deployment. + - **Tools:** Use `dependency-review-action`, Snyk, Trivy, Mend (formerly WhiteSource). + - **Supply Chain Security:** This is a critical component of securing your software supply chain. +- **Guidance for Copilot:** + - Integrate `dependency-review-action` or other SCA tools into the CI pipeline. + - Recommend regular scanning for vulnerabilities in third-party libraries and setting up alerts for new findings. + - Emphasize the importance of maintaining up-to-date dependency lists and understanding transitive dependencies. + +### **5. Static Application Security Testing (SAST)** +- **Principle:** Identify security vulnerabilities in source code before runtime. +- **Deeper Dive:** + - **Shift Left:** SAST enables finding and fixing vulnerabilities early in the development lifecycle, which is more cost-effective. + - **Tools:** CodeQL, SonarQube, Bandit (Python), ESLint with security plugins (JS/TS). + - **Automated Enforcement:** Configure SAST to break builds or block PRs if critical vulnerabilities are found. +- **Guidance for Copilot:** + - Integrate SAST tools (e.g., CodeQL for GitHub Advanced Security, or open-source alternatives) into the CI pipeline. + - Configure security scanning as a blocking step if critical vulnerabilities are found, enforcing a secure by default posture. + - Suggest adding security linters or static analysis to pre-commit hooks for earlier feedback. + +### **6. Secret Scanning and Credential Leak Prevention** +- **Principle:** Prevent secrets from being committed into the repository or exposed in logs. +- **Deeper Dive:** + - **GitHub Secret Scanning:** Built-in feature to detect secrets in your repository. + - **Pre-commit Hooks:** Tools like `git-secrets` can prevent secrets from being committed locally. + - **Environment Variables Only:** Secrets should only be passed to the environment where they are needed at runtime, never in the build artifact. +- **Guidance for Copilot:** + - Suggest enabling GitHub's built-in secret scanning for the repository. + - Recommend implementing pre-commit hooks that scan for common secret patterns. + - Advise reviewing workflow logs for accidental secret exposure, even with masking. + +### **7. Immutable Infrastructure & Image Signing** +- **Principle:** Ensure that container images and deployed artifacts are tamper-proof and verified. +- **Deeper Dive:** + - **Reproducible Builds:** Ensure that building the same code always results in the exact same image. + - **Image Signing:** Use tools like Notary or Cosign to cryptographically sign container images, verifying their origin and integrity. + - **Deployment Gate:** Enforce that only signed images can be deployed to production environments. +- **Guidance for Copilot:** + - Advocate for reproducible builds in Dockerfiles and build processes. + - Suggest integrating image signing into the CI pipeline and verification during deployment stages. + +## Optimization and Performance + +### **1. Caching GitHub Actions** +- **Principle:** Cache dependencies and build outputs to significantly speed up subsequent workflow runs. +- **Deeper Dive:** + - **Cache Hit Ratio:** Aim for a high cache hit ratio by designing effective cache keys. + - **Cache Keys:** Use a unique key based on file hashes (e.g., `hashFiles('**/package-lock.json')`, `hashFiles('**/requirements.txt')`) to invalidate the cache only when dependencies change. + - **Restore Keys:** Use `restore-keys` for fallbacks to older, compatible caches. + - **Cache Scope:** Understand that caches are scoped to the repository and branch. +- **Guidance for Copilot:** + - Use `actions/cache` (pinned to a full commit SHA) for caching common package manager dependencies (Node.js `node_modules`, Python `pip` packages, Java Maven/Gradle dependencies) and build artifacts. + - Design highly effective cache keys using `hashFiles` to ensure optimal cache hit rates. + - Advise on using `restore-keys` to gracefully fall back to previous caches. +- **Example (Advanced Caching for Monorepo):** +```yaml +- name: Cache Node.js modules + uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 + with: + path: | + ~/.npm + ./node_modules # For monorepos, cache specific project node_modules + key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}-${{ github.run_id }} + restore-keys: | + ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}- + ${{ runner.os }}-node- +``` + +### **2. Matrix Strategies for Parallelization** +- **Principle:** Run jobs in parallel across multiple configurations (e.g., different Node.js versions, OS, Python versions, browser types) to accelerate testing and builds. +- **Deeper Dive:** + - **`strategy.matrix`:** Define a matrix of variables. + - **`include`/`exclude`:** Fine-tune combinations. + - **`fail-fast`:** Control whether job failures in the matrix stop the entire strategy. + - **Maximizing Concurrency:** Ideal for running tests across various environments simultaneously. +- **Guidance for Copilot:** + - Utilize `strategy.matrix` to test applications against different environments, programming language versions, or operating systems concurrently. + - Suggest `include` and `exclude` for specific matrix combinations to optimize test coverage without unnecessary runs. + - Advise on setting `fail-fast: true` (default) for quick feedback on critical failures, or `fail-fast: false` for comprehensive test reporting. +- **Example (Multi-version, Multi-OS Test Matrix):** +```yaml +jobs: + test: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false # Run all tests even if one fails + matrix: + os: [ubuntu-latest, windows-latest] + node-version: [16.x, 18.x, 20.x] + browser: [chromium, firefox] + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + - uses: actions/setup-node@3235b876344d2a9aa001b8d1453c930bba69e610 # v3.9.1 + with: + node-version: ${{ matrix.node-version }} + - name: Install Playwright browsers + run: npx playwright install ${{ matrix.browser }} + - name: Run tests + run: npm test +``` + +### **3. Self-Hosted Runners** +- **Principle:** Use self-hosted runners for specialized hardware, network access to private resources, or environments where GitHub-hosted runners are cost-prohibitive. +- **Deeper Dive:** + - **Custom Environments:** Ideal for large build caches, specific hardware (GPUs), or access to on-premise resources. + - **Cost Optimization:** Can be more cost-effective for very high usage. + - **Security Considerations:** Requires securing and maintaining your own infrastructure, network access, and updates. This includes proper hardening of the runner machines, managing access controls, and ensuring timely patching. + - **Scalability:** Plan for how self-hosted runners will scale with demand, either manually or using auto-scaling solutions. +- **Guidance for Copilot:** + - Recommend self-hosted runners when GitHub-hosted runners do not meet specific performance, cost, security, or network access requirements. + - Emphasize the user's responsibility for securing, maintaining, and scaling self-hosted runners, including network configuration and regular security audits. + - Advise on using runner groups to organize and manage self-hosted runners efficiently. + +### **4. Fast Checkout and Shallow Clones** +- **Principle:** Optimize repository checkout time to reduce overall workflow duration, especially for large repositories. +- **Deeper Dive:** + - **`fetch-depth`:** Controls how much of the Git history is fetched. `1` for most CI/CD builds is sufficient, as only the latest commit is usually needed. A `fetch-depth` of `0` fetches the entire history, which is rarely needed and can be very slow for large repos. + - **`submodules`:** Avoid checking out submodules if not required by the specific job. Fetching submodules adds significant overhead. + - **`lfs`:** Manage Git LFS (Large File Storage) files efficiently. If not needed, set `lfs: false`. + - **Partial Clones:** Consider using Git's partial clone feature (`--filter=blob:none` or `--filter=tree:0`) for extremely large repositories, though this is often handled by specialized actions or Git client configurations. +- **Guidance for Copilot:** + - Use `actions/checkout` (pinned to a full commit SHA, e.g., `actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1`) with `fetch-depth: 1` as the default for most build and test jobs to significantly save time and bandwidth. + - Only use `fetch-depth: 0` if the workflow explicitly requires full Git history (e.g., for release tagging, deep commit analysis, or `git blame` operations). + - Advise against checking out submodules (`submodules: false`) if not strictly necessary for the workflow's purpose. + - Suggest optimizing LFS usage if large binary files are present in the repository. + +### **5. Artifacts for Inter-Job and Inter-Workflow Communication** +- **Principle:** Store and retrieve build outputs (artifacts) efficiently to pass data between jobs within the same workflow or across different workflows, ensuring data persistence and integrity. +- **Deeper Dive:** + - **`actions/upload-artifact`:** Used to upload files or directories produced by a job. Artifacts are automatically compressed and can be downloaded later. + - **`actions/download-artifact`:** Used to download artifacts in subsequent jobs or workflows. You can download all artifacts or specific ones by name. + - **`retention-days`:** Crucial for managing storage costs and compliance. Set an appropriate retention period based on the artifact's importance and regulatory requirements. + - **Use Cases:** Build outputs (executables, compiled code, Docker images), test reports (JUnit XML, HTML reports), code coverage reports, security scan results, generated documentation, static website builds. + - **Limitations:** Artifacts are immutable once uploaded. Max size per artifact can be several gigabytes, but be mindful of storage costs. +- **Guidance for Copilot:** + - Use `actions/upload-artifact` and `actions/download-artifact` (both pinned to full commit SHAs) to reliably pass large files between jobs within the same workflow or across different workflows, promoting modularity and efficiency. + - Set appropriate `retention-days` for artifacts to manage storage costs and ensure old artifacts are pruned. + - Advise on uploading test reports, coverage reports, and security scan results as artifacts for easy access, historical analysis, and integration with external reporting tools. + - Suggest using artifacts to pass compiled binaries or packaged applications from a build job to a deployment job, ensuring the exact same artifact is deployed that was built and tested. + +## Comprehensive Testing in CI/CD (Expanded) + +### **1. Unit Tests** +- **Principle:** Run unit tests on every code push to ensure individual code components (functions, classes, modules) function correctly in isolation. They are the fastest and most numerous tests. +- **Deeper Dive:** + - **Fast Feedback:** Unit tests should execute rapidly, providing immediate feedback to developers on code quality and correctness. Parallelization of unit tests is highly recommended. + - **Code Coverage:** Integrate code coverage tools (e.g., Istanbul for JS, Coverage.py for Python, JaCoCo for Java) and enforce minimum coverage thresholds. Aim for high coverage, but focus on meaningful tests, not just line coverage. + - **Test Reporting:** Publish test results using `actions/upload-artifact` (e.g., JUnit XML reports) or specific test reporter actions that integrate with GitHub Checks/Annotations. + - **Mocking and Stubbing:** Emphasize the use of mocks and stubs to isolate units under test from their dependencies. +- **Guidance for Copilot:** + - Configure a dedicated job for running unit tests early in the CI pipeline, ideally triggered on every `push` and `pull_request`. + - Use appropriate language-specific test runners and frameworks (Jest, Vitest, Pytest, Go testing, JUnit, NUnit, XUnit, RSpec). + - Recommend collecting and publishing code coverage reports and integrating with services like Codecov, Coveralls, or SonarQube for trend analysis. + - Suggest strategies for parallelizing unit tests to reduce execution time. + +### **2. Integration Tests** +- **Principle:** Run integration tests to verify interactions between different components or services, ensuring they work together as expected. These tests typically involve real dependencies (e.g., databases, APIs). +- **Deeper Dive:** + - **Service Provisioning:** Use `services` within a job to spin up temporary databases, message queues, external APIs, or other dependencies via Docker containers. This provides a consistent and isolated testing environment. + - **Test Doubles vs. Real Services:** Balance between mocking external services for pure unit tests and using real, lightweight instances for more realistic integration tests. Prioritize real instances when testing actual integration points. + - **Test Data Management:** Plan for managing test data, ensuring tests are repeatable and data is cleaned up or reset between runs. + - **Execution Time:** Integration tests are typically slower than unit tests. Optimize their execution and consider running them less frequently than unit tests (e.g., on PR merge instead of every push). +- **Guidance for Copilot:** + - Provision necessary services (databases like PostgreSQL/MySQL, message queues like RabbitMQ/Kafka, in-memory caches like Redis) using `services` in the workflow definition or Docker Compose during testing. + - Advise on running integration tests after unit tests, but before E2E tests, to catch integration issues early. + - Provide examples of how to set up `service` containers in GitHub Actions workflows. + - Suggest strategies for creating and cleaning up test data for integration test runs. + +### **3. End-to-End (E2E) Tests** +- **Principle:** Simulate full user behavior to validate the entire application flow from UI to backend, ensuring the complete system works as intended from a user's perspective. +- **Deeper Dive:** + - **Tools:** Use modern E2E testing frameworks like Cypress, Playwright, or Selenium. These provide browser automation capabilities. + - **Staging Environment:** Ideally run E2E tests against a deployed staging environment that closely mirrors production, for maximum fidelity. Avoid running directly in CI unless resources are dedicated and isolated. + - **Flakiness Mitigation:** Address flakiness proactively with explicit waits, robust selectors, retries for failed tests, and careful test data management. Flaky tests erode trust in the pipeline. + - **Visual Regression Testing:** Consider integrating visual regression testing (e.g., Applitools, Percy) to catch UI discrepancies. + - **Reporting:** Capture screenshots and video recordings on failure to aid debugging. +- **Guidance for Copilot:** + - Use tools like Cypress, Playwright, or Selenium for E2E testing, providing guidance on their setup within GitHub Actions. + - Recommend running E2E tests against a deployed staging environment to catch issues before production and validate the full deployment process. + - Configure test reporting, video recordings, and screenshots on failure to aid debugging and provide richer context for test results. + - Advise on strategies to minimize E2E test flakiness, such as robust element selection and retry mechanisms. + +### **4. Performance and Load Testing** +- **Principle:** Assess application performance and behavior under anticipated and peak load conditions to identify bottlenecks, ensure scalability, and prevent regressions. +- **Deeper Dive:** + - **Tools:** JMeter, k6, Locust, Gatling, Artillery. Choose based on language, complexity, and specific needs. + - **Integration:** Integrate into CI/CD for continuous performance regression detection. Run these tests less frequently than unit/integration tests (e.g., nightly, weekly, or on significant feature merges). + - **Thresholds:** Define clear performance thresholds (e.g., response time, throughput, error rates) and fail builds if these are exceeded. + - **Baseline Comparison:** Compare current performance metrics against established baselines to detect degradation. +- **Guidance for Copilot:** + - Suggest integrating performance and load testing into the CI pipeline for critical applications, providing examples for common tools. + - Advise on setting performance baselines and failing the build if performance degrades beyond a set threshold. + - Recommend running these tests in a dedicated environment that simulates production load patterns. + - Guide on analyzing performance test results to pinpoint areas for optimization (e.g., database queries, API endpoints). + +### **5. Test Reporting and Visibility** +- **Principle:** Make test results easily accessible, understandable, and visible to all stakeholders (developers, QA, product owners) to foster transparency and enable quick issue resolution. +- **Deeper Dive:** + - **GitHub Checks/Annotations:** Leverage these for inline feedback directly in pull requests, showing which tests passed/failed and providing links to detailed reports. + - **Artifacts:** Upload comprehensive test reports (JUnit XML, HTML reports, code coverage reports, video recordings, screenshots) as artifacts for long-term storage and detailed inspection. + - **Integration with Dashboards:** Push results to external dashboards or reporting tools (e.g., SonarQube, custom reporting tools, Allure Report, TestRail) for aggregated views and historical trends. + - **Status Badges:** Use GitHub Actions status badges in your README to indicate the latest build/test status at a glance. +- **Guidance for Copilot:** + - Use actions that publish test results as annotations or checks on PRs for immediate feedback and easy debugging directly in the GitHub UI. + - Upload detailed test reports (e.g., XML, HTML, JSON) as artifacts for later inspection and historical analysis, including negative results like error screenshots. + - Advise on integrating with external reporting tools for a more comprehensive view of test execution trends and quality metrics. + - Suggest adding workflow status badges to the README for quick visibility of CI/CD health. + +## Advanced Deployment Strategies (Expanded) + +### **1. Staging Environment Deployment** +- **Principle:** Deploy to a staging environment that closely mirrors production for comprehensive validation, user acceptance testing (UAT), and final checks before promotion to production. +- **Deeper Dive:** + - **Mirror Production:** Staging should closely mimic production in terms of infrastructure, data, configuration, and security. Any significant discrepancies can lead to issues in production. + - **Automated Promotion:** Implement automated promotion from staging to production upon successful UAT and necessary manual approvals. This reduces human error and speeds up releases. + - **Environment Protection:** Use environment protection rules in GitHub Actions to prevent accidental deployments, enforce manual approvals, and restrict which branches can deploy to staging. + - **Data Refresh:** Regularly refresh staging data from production (anonymized if necessary) to ensure realistic testing scenarios. +- **Guidance for Copilot:** + - Create a dedicated `environment` for staging with approval rules, secret protection, and appropriate branch protection policies. + - Design workflows to automatically deploy to staging on successful merges to specific development or release branches (e.g., `develop`, `release/*`). + - Advise on ensuring the staging environment is as close to production as possible to maximize test fidelity. + - Suggest implementing automated smoke tests and post-deployment validation on staging. + +### **2. Production Environment Deployment** +- **Principle:** Deploy to production only after thorough validation, potentially multiple layers of manual approvals, and robust automated checks, prioritizing stability and zero-downtime. +- **Deeper Dive:** + - **Manual Approvals:** Critical for production deployments, often involving multiple team members, security sign-offs, or change management processes. GitHub Environments support this natively. + - **Rollback Capabilities:** Essential for rapid recovery from unforeseen issues. Ensure a quick and reliable way to revert to the previous stable state. + - **Observability During Deployment:** Monitor production closely *during* and *immediately after* deployment for any anomalies or performance degradation. Use dashboards, alerts, and tracing. + - **Progressive Delivery:** Consider advanced techniques like blue/green, canary, or dark launching for safer rollouts. + - **Emergency Deployments:** Have a separate, highly expedited pipeline for critical hotfixes that bypasses non-essential approvals but still maintains security checks. +- **Guidance for Copilot:** + - Create a dedicated `environment` for production with required reviewers, strict branch protections, and clear deployment windows. + - Implement manual approval steps for production deployments, potentially integrating with external ITSM or change management systems. + - Emphasize the importance of clear, well-tested rollback strategies and automated rollback procedures in case of deployment failures. + - Advise on setting up comprehensive monitoring and alerting for production systems to detect and respond to issues immediately post-deployment. + +### **3. Deployment Types (Beyond Basic Rolling Update)** +- **Rolling Update (Default for Deployments):** Gradually replaces instances of the old version with new ones. Good for most cases, especially stateless applications. + - **Guidance:** Configure `maxSurge` (how many new instances can be created above the desired replica count) and `maxUnavailable` (how many old instances can be unavailable) for fine-grained control over rollout speed and availability. +- **Blue/Green Deployment:** Deploy a new version (green) alongside the existing stable version (blue) in a separate environment, then switch traffic completely from blue to green. + - **Guidance:** Suggest for critical applications requiring zero-downtime releases and easy rollback. Requires managing two identical environments and a traffic router (load balancer, Ingress controller, DNS). + - **Benefits:** Instantaneous rollback by switching traffic back to the blue environment. +- **Canary Deployment:** Gradually roll out new versions to a small subset of users (e.g., 5-10%) before a full rollout. Monitor performance and error rates for the canary group. + - **Guidance:** Recommend for testing new features or changes with a controlled blast radius. Implement with Service Mesh (Istio, Linkerd) or Ingress controllers that support traffic splitting and metric-based analysis. + - **Benefits:** Early detection of issues with minimal user impact. +- **Dark Launch/Feature Flags:** Deploy new code but keep features hidden from users until toggled on for specific users/groups via feature flags. + - **Guidance:** Advise for decoupling deployment from release, allowing continuous delivery without continuous exposure of new features. Use feature flag management systems (LaunchDarkly, Split.io, Unleash). + - **Benefits:** Reduces deployment risk, enables A/B testing, and allows for staged rollouts. +- **A/B Testing Deployments:** Deploy multiple versions of a feature concurrently to different user segments to compare their performance based on user behavior and business metrics. + - **Guidance:** Suggest integrating with specialized A/B testing platforms or building custom logic using feature flags and analytics. + +### **4. Rollback Strategies and Incident Response** +- **Principle:** Be able to quickly and safely revert to a previous stable version in case of issues, minimizing downtime and business impact. This requires proactive planning. +- **Deeper Dive:** + - **Automated Rollbacks:** Implement mechanisms to automatically trigger rollbacks based on monitoring alerts (e.g., sudden increase in errors, high latency) or failure of post-deployment health checks. + - **Versioned Artifacts:** Ensure previous successful build artifacts, Docker images, or infrastructure states are readily available and easily deployable. This is crucial for fast recovery. + - **Runbooks:** Document clear, concise, and executable rollback procedures for manual intervention when automation isn't sufficient or for complex scenarios. These should be regularly reviewed and tested. + - **Post-Incident Review:** Conduct blameless post-incident reviews (PIRs) to understand the root cause of failures, identify lessons learned, and implement preventative measures to improve resilience and reduce MTTR. + - **Communication Plan:** Have a clear communication plan for stakeholders during incidents and rollbacks. +- **Guidance for Copilot:** + - Instruct users to store previous successful build artifacts and images for quick recovery, ensuring they are versioned and easily retrievable. + - Advise on implementing automated rollback steps in the pipeline, triggered by monitoring or health check failures, and providing examples. + - Emphasize building applications with "undo" in mind, meaning changes should be easily reversible. + - Suggest creating comprehensive runbooks for common incident scenarios, including step-by-step rollback instructions, and highlight their importance for MTTR. + - Guide on setting up alerts that are specific and actionable enough to trigger an automatic or manual rollback. + +## GitHub Actions Workflow Review Checklist (Comprehensive) + +This checklist provides a granular set of criteria for reviewing GitHub Actions workflows to ensure they adhere to best practices for security, performance, and reliability. + +- [ ] **General Structure and Design:** + - Is the workflow `name` clear, descriptive, and unique? + - Are `on` triggers appropriate for the workflow's purpose (e.g., `push`, `pull_request`, `workflow_dispatch`, `schedule`)? Are path/branch filters used effectively? + - Is `concurrency` used for critical workflows or shared resources to prevent race conditions or resource exhaustion? + - Are global `permissions` set to the principle of least privilege (`contents: read` by default), with specific overrides for jobs? + - Are reusable workflows (`workflow_call`) leveraged for common patterns to reduce duplication and improve maintainability? + - Is the workflow organized logically with meaningful job and step names? + +- [ ] **Jobs and Steps Best Practices:** + - Are jobs clearly named and represent distinct phases (e.g., `build`, `lint`, `test`, `deploy`)? + - Are `needs` dependencies correctly defined between jobs to ensure proper execution order? + - Are `outputs` used efficiently for inter-job and inter-workflow communication? + - Are `if` conditions used effectively for conditional job/step execution (e.g., environment-specific deployments, branch-specific actions)? + - Are all `uses` actions pinned to a full commit SHA with a human-readable version comment (e.g., `actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1`)? Tags (e.g., `@v4`) and branches (e.g., `@main`) are mutable and can be silently redirected to malicious commits — always use immutable SHA references, especially for third-party actions. + - Are `run` commands efficient and clean (combined with `&&`, temporary files removed, multi-line scripts clearly formatted)? + - Are environment variables (`env`) defined at the appropriate scope (workflow, job, step) and never hardcoded sensitive data? + - Is `timeout-minutes` set for long-running jobs to prevent hung workflows? + +- [ ] **Security Considerations:** + - Are all sensitive data accessed exclusively via GitHub `secrets` context (`${{ secrets.MY_SECRET }}`)? Never hardcoded, never exposed in logs (even if masked). + - Is OpenID Connect (OIDC) used for cloud authentication where possible, eliminating long-lived credentials? + - Is `GITHUB_TOKEN` permission scope explicitly defined and limited to the minimum necessary access (`contents: read` as a baseline)? + - Are Software Composition Analysis (SCA) tools (e.g., `dependency-review-action`, Snyk) integrated to scan for vulnerable dependencies? + - Are Static Application Security Testing (SAST) tools (e.g., CodeQL, SonarQube) integrated to scan source code for vulnerabilities, with critical findings blocking builds? + - Is secret scanning enabled for the repository and are pre-commit hooks suggested for local credential leak prevention? + - Is there a strategy for container image signing (e.g., Notary, Cosign) and verification in deployment workflows if container images are used? + - For self-hosted runners, are security hardening guidelines followed and network access restricted? + +- [ ] **Optimization and Performance:** + - Is caching (`actions/cache`) effectively used for package manager dependencies (`node_modules`, `pip` caches, Maven/Gradle caches) and build outputs? + - Are cache `key` and `restore-keys` designed for optimal cache hit rates (e.g., using `hashFiles`)? + - Is `strategy.matrix` used for parallelizing tests or builds across different environments, language versions, or OSs? + - Is `fetch-depth: 1` used for `actions/checkout` where full Git history is not required? + - Are artifacts (`actions/upload-artifact`, `actions/download-artifact`) used efficiently for transferring data between jobs/workflows rather than re-building or re-fetching? + - Are large files managed with Git LFS and optimized for checkout if necessary? + +- [ ] **Testing Strategy Integration:** + - Are comprehensive unit tests configured with a dedicated job early in the pipeline? + - Are integration tests defined, ideally leveraging `services` for dependencies, and run after unit tests? + - Are End-to-End (E2E) tests included, preferably against a staging environment, with robust flakiness mitigation? + - Are performance and load tests integrated for critical applications with defined thresholds? + - Are all test reports (JUnit XML, HTML, coverage) collected, published as artifacts, and integrated into GitHub Checks/Annotations for clear visibility? + - Is code coverage tracked and enforced with a minimum threshold? + +- [ ] **Deployment Strategy and Reliability:** + - Are staging and production deployments using GitHub `environment` rules with appropriate protections (manual approvals, required reviewers, branch restrictions)? + - Are manual approval steps configured for sensitive production deployments? + - Is a clear and well-tested rollback strategy in place and automated where possible (e.g., `kubectl rollout undo`, reverting to previous stable image)? + - Are chosen deployment types (e.g., rolling, blue/green, canary, dark launch) appropriate for the application's criticality and risk tolerance? + - Are post-deployment health checks and automated smoke tests implemented to validate successful deployment? + - Is the workflow resilient to temporary failures (e.g., retries for flaky network operations)? + +- [ ] **Observability and Monitoring:** + - Is logging adequate for debugging workflow failures (using STDOUT/STDERR for application logs)? + - Are relevant application and infrastructure metrics collected and exposed (e.g., Prometheus metrics)? + - Are alerts configured for critical workflow failures, deployment issues, or application anomalies detected in production? + - Is distributed tracing (e.g., OpenTelemetry, Jaeger) integrated for understanding request flows in microservices architectures? + - Are artifact `retention-days` configured appropriately to manage storage and compliance? + +## Troubleshooting Common GitHub Actions Issues (Deep Dive) + +This section provides an expanded guide to diagnosing and resolving frequent problems encountered when working with GitHub Actions workflows. + +### **1. Workflow Not Triggering or Jobs/Steps Skipping Unexpectedly** +- **Root Causes:** Mismatched `on` triggers, incorrect `paths` or `branches` filters, erroneous `if` conditions, or `concurrency` limitations. +- **Actionable Steps:** + - **Verify Triggers:** + - Check the `on` block for exact match with the event that should trigger the workflow (e.g., `push`, `pull_request`, `workflow_dispatch`, `schedule`). + - Ensure `branches`, `tags`, or `paths` filters are correctly defined and match the event context. Remember that `paths-ignore` and `branches-ignore` take precedence. + - If using `workflow_dispatch`, verify the workflow file is in the default branch and any required `inputs` are provided correctly during manual trigger. + - **Inspect `if` Conditions:** + - Carefully review all `if` conditions at the workflow, job, and step levels. A single false condition can prevent execution. + - Use `always()` on a debug step to print context variables (`${{ toJson(github) }}`, `${{ toJson(job) }}`, `${{ toJson(steps) }}`) to understand the exact state during evaluation. + - Test complex `if` conditions in a simplified workflow. + - **Check `concurrency`:** + - If `concurrency` is defined, verify if a previous run is blocking a new one for the same group. Check the "Concurrency" tab in the workflow run. + - **Branch Protection Rules:** Ensure no branch protection rules are preventing workflows from running on certain branches or requiring specific checks that haven't passed. + +### **2. Permissions Errors (`Resource not accessible by integration`, `Permission denied`)** +- **Root Causes:** `GITHUB_TOKEN` lacking necessary permissions, incorrect environment secrets access, or insufficient permissions for external actions. +- **Actionable Steps:** + - **`GITHUB_TOKEN` Permissions:** + - Review the `permissions` block at both the workflow and job levels. Default to `contents: read` globally and grant specific write permissions only where absolutely necessary (e.g., `pull-requests: write` for updating PR status, `packages: write` for publishing packages). + - Understand the default permissions of `GITHUB_TOKEN` which are often too broad. + - **Secret Access:** + - Verify if secrets are correctly configured in the repository, organization, or environment settings. + - Ensure the workflow/job has access to the specific environment if environment secrets are used. Check if any manual approvals are pending for the environment. + - Confirm the secret name matches exactly (`secrets.MY_API_KEY`). + - **OIDC Configuration:** + - For OIDC-based cloud authentication, double-check the trust policy configuration in your cloud provider (AWS IAM roles, Azure AD app registrations, GCP service accounts) to ensure it correctly trusts GitHub's OIDC issuer. + - Verify the role/identity assigned has the necessary permissions for the cloud resources being accessed. + +### **3. Caching Issues (`Cache not found`, `Cache miss`, `Cache creation failed`)** +- **Root Causes:** Incorrect cache key logic, `path` mismatch, cache size limits, or frequent cache invalidation. +- **Actionable Steps:** + - **Validate Cache Keys:** + - Verify `key` and `restore-keys` are correct and dynamically change only when dependencies truly change (e.g., `key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}`). A cache key that is too dynamic will always result in a miss. + - Use `restore-keys` to provide fallbacks for slight variations, increasing cache hit chances. + - **Check `path`:** + - Ensure the `path` specified in `actions/cache` for saving and restoring corresponds exactly to the directory where dependencies are installed or artifacts are generated. + - Verify the existence of the `path` before caching. + - **Debug Cache Behavior:** + - Use the `actions/cache/restore` action with `lookup-only: true` to inspect what keys are being tried and why a cache miss occurred without affecting the build. + - Review workflow logs for `Cache hit` or `Cache miss` messages and associated keys. + - **Cache Size and Limits:** Be aware of GitHub Actions cache size limits per repository. If caches are very large, they might be evicted frequently. + +### **4. Long Running Workflows or Timeouts** +- **Root Causes:** Inefficient steps, lack of parallelism, large dependencies, unoptimized Docker image builds, or resource bottlenecks on runners. +- **Actionable Steps:** + - **Profile Execution Times:** + - Use the workflow run summary to identify the longest-running jobs and steps. This is your primary tool for optimization. + - **Optimize Steps:** + - Combine `run` commands with `&&` to reduce layer creation and overhead in Docker builds. + - Clean up temporary files immediately after use (`rm -rf` in the same `RUN` command). + - Install only necessary dependencies. + - **Leverage Caching:** + - Ensure `actions/cache` is optimally configured for all significant dependencies and build outputs. + - **Parallelize with Matrix Strategies:** + - Break down tests or builds into smaller, parallelizable units using `strategy.matrix` to run them concurrently. + - **Choose Appropriate Runners:** + - Review `runs-on`. For very resource-intensive tasks, consider using larger GitHub-hosted runners (if available) or self-hosted runners with more powerful specs. + - **Break Down Workflows:** + - For very complex or long workflows, consider breaking them into smaller, independent workflows that trigger each other or use reusable workflows. + +### **5. Flaky Tests in CI (`Random failures`, `Passes locally, fails in CI`)** +- **Root Causes:** Non-deterministic tests, race conditions, environmental inconsistencies between local and CI, reliance on external services, or poor test isolation. +- **Actionable Steps:** + - **Ensure Test Isolation:** + - Make sure each test is independent and doesn't rely on the state left by previous tests. Clean up resources (e.g., database entries) after each test or test suite. + - **Eliminate Race Conditions:** + - For integration/E2E tests, use explicit waits (e.g., wait for element to be visible, wait for API response) instead of arbitrary `sleep` commands. + - Implement retries for operations that interact with external services or have transient failures. + - **Standardize Environments:** + - Ensure the CI environment (Node.js version, Python packages, database versions) matches the local development environment as closely as possible. + - Use Docker `services` for consistent test dependencies. + - **Robust Selectors (E2E):** + - Use stable, unique selectors in E2E tests (e.g., `data-testid` attributes) instead of brittle CSS classes or XPath. + - **Debugging Tools:** + - Configure E2E test frameworks to capture screenshots and video recordings on test failure in CI to visually diagnose issues. + - **Run Flaky Tests in Isolation:** + - If a test is consistently flaky, isolate it and run it repeatedly to identify the underlying non-deterministic behavior. + +### **6. Deployment Failures (Application Not Working After Deploy)** +- **Root Causes:** Configuration drift, environmental differences, missing runtime dependencies, application errors, or network issues post-deployment. +- **Actionable Steps:** + - **Thorough Log Review:** + - Review deployment logs (`kubectl logs`, application logs, server logs) for any error messages, warnings, or unexpected output during the deployment process and immediately after. + - **Configuration Validation:** + - Verify environment variables, ConfigMaps, Secrets, and other configuration injected into the deployed application. Ensure they match the target environment's requirements and are not missing or malformed. + - Use pre-deployment checks to validate configuration. + - **Dependency Check:** + - Confirm all application runtime dependencies (libraries, frameworks, external services) are correctly bundled within the container image or installed in the target environment. + - **Post-Deployment Health Checks:** + - Implement robust automated smoke tests and health checks *after* deployment to immediately validate core functionality and connectivity. Trigger rollbacks if these fail. + - **Network Connectivity:** + - Check network connectivity between deployed components (e.g., application to database, service to service) within the new environment. Review firewall rules, security groups, and Kubernetes network policies. + - **Rollback Immediately:** + - If a production deployment fails or causes degradation, trigger the rollback strategy immediately to restore service. Diagnose the issue in a non-production environment. + +## Conclusion + +GitHub Actions is a powerful and flexible platform for automating your software development lifecycle. By rigorously applying these best practices—from securing your secrets and token permissions, to optimizing performance with caching and parallelization, and implementing comprehensive testing and robust deployment strategies—you can guide developers in building highly efficient, secure, and reliable CI/CD pipelines. Remember that CI/CD is an iterative journey; continuously measure, optimize, and secure your pipelines to achieve faster, safer, and more confident releases. Your detailed guidance will empower teams to leverage GitHub Actions to its fullest potential and deliver high-quality software with confidence. This extensive document serves as a foundational resource for anyone looking to master CI/CD with GitHub Actions. + +--- + + diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..c98f45c --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,140 @@ +name: CI + +on: + push: + branches: ["master", "develop", "feature/**", "release/**", "hotfix/**"] + tags: ["*"] + pull_request: + branches: ["master", "develop"] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ci-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + # Keep the Windows build separate so we still validate the .NET Framework target + # and the packaging prerequisites that only exist on Windows runners. + build: + name: Build (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest] + + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + fetch-depth: 0 + fetch-tags: true + submodules: recursive + + - name: Setup .NET SDKs + uses: actions/setup-dotnet@v5 + with: + dotnet-version: | + 6.0.x + 8.0.x + 9.0.x + 10.0.x + + - name: Cache NuGet packages + uses: actions/cache@v5 + with: + path: ~/.nuget/packages + key: nuget-${{ runner.os }}-${{ hashFiles('**/*.csproj', '**/*.props', '**/*.targets', '.config/dotnet-tools.json') }} + restore-keys: | + nuget-${{ runner.os }}- + + - name: Restore .NET tools + run: dotnet tool restore + + - name: Restore solution + run: dotnet restore ./src/NEventStore.Persistence.MongoDB.Core.sln --verbosity m + + - name: Run GitVersion and patch assembly info + id: gitversion + shell: pwsh + working-directory: ${{ github.workspace }} + run: | + $gitVersion = dotnet tool run dotnet-gitversion /targetpath "${{ github.workspace }}" /output json /updateAssemblyInfo | ConvertFrom-Json + dotnet tool run dotnet-gitversion /targetpath "${{ github.workspace }}/dependencies/NEventStore" /updateAssemblyInfo | Out-Null + "semver=$($gitVersion.SemVer)" >> $env:GITHUB_OUTPUT + + - name: Build solution + run: dotnet build ./src/NEventStore.Persistence.MongoDB.Core.sln -c Release --no-restore /p:ContinuousIntegrationBuild=True + + # Tests run on Linux because this repository needs a live MongoDB instance and + # GitHub Actions service containers are only available on Linux runners. + # The matrix is therefore per target framework instead of per OS. + test-modern-tfm-linux: + name: Test (Linux, ${{ matrix.tfm }}) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + # Run each modern TFM independently so failures are isolated and easy to read. + tfm: + - net8.0 + - net9.0 + - net10.0 + + services: + # The test project reads NEventStore.MongoDB from the environment and expects + # a real MongoDB server. This sidecar provides that dependency for each matrix leg. + mongodb: + image: mongo:8 + ports: + - 27017:27017 + options: >- + --health-cmd "mongosh --eval \"db.adminCommand('ping')\"" + --health-interval 10s + --health-timeout 5s + --health-retries 10 + + env: + # Match the connection-string convention used by the acceptance tests. + NEventStore.MongoDB: mongodb://127.0.0.1:27017/NEventStore + + steps: + - name: Checkout + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + fetch-depth: 0 + fetch-tags: true + submodules: recursive + + - name: Setup .NET SDKs + uses: actions/setup-dotnet@v5 + with: + dotnet-version: | + 6.0.x + 8.0.x + 9.0.x + 10.0.x + + - name: Cache NuGet packages + uses: actions/cache@v5 + with: + path: ~/.nuget/packages + key: nuget-${{ runner.os }}-${{ hashFiles('**/*.csproj', '**/*.props', '**/*.targets', '.config/dotnet-tools.json') }} + restore-keys: | + nuget-${{ runner.os }}- + + - name: Run tests for ${{ matrix.tfm }} + # Only the test project is executed here. It pulls in the production project and + # linked acceptance coverage from the NEventStore submodule. + run: dotnet test ./src/NEventStore.Persistence.MongoDB.Tests/NEventStore.Persistence.MongoDB.Core.Tests.csproj -c Release -f ${{ matrix.tfm }} --logger "trx;LogFileName=test-results-${{ matrix.tfm }}.trx" + + - name: Upload test results + uses: actions/upload-artifact@v7 + with: + name: test-results-${{ matrix.tfm }} + path: "**/test-results-${{ matrix.tfm }}.trx" + if-no-files-found: error + retention-days: 14 diff --git a/.github/workflows/pack-and-publish.yml b/.github/workflows/pack-and-publish.yml new file mode 100644 index 0000000..e853753 --- /dev/null +++ b/.github/workflows/pack-and-publish.yml @@ -0,0 +1,100 @@ +name: Pack and Publish + +on: + workflow_dispatch: + inputs: + branch: + description: Branch or ref to pack (branch name or tag) + required: true + type: string + default: master + publish: + description: Publish packages to NuGet + required: true + type: boolean + default: false + +permissions: + contents: read + +jobs: + pack-and-publish: + name: Pack and optional Publish + runs-on: windows-latest + + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + ref: ${{ inputs.branch }} + fetch-depth: 0 + submodules: recursive + + - name: Setup .NET + uses: actions/setup-dotnet@v5 + with: + dotnet-version: | + 6.0.x + 8.0.x + 9.0.x + 10.0.x + + - name: Cache NuGet packages + uses: actions/cache@v5 + with: + path: ~/.nuget/packages + key: nuget-${{ runner.os }}-${{ hashFiles('**/*.csproj', '**/*.props', '**/*.targets', '.config/dotnet-tools.json') }} + restore-keys: | + nuget-${{ runner.os }}- + + - name: Restore dotnet tools + run: dotnet tool restore + + - name: Restore NuGet packages + run: dotnet restore ./src/NEventStore.Persistence.MongoDB.Core.sln --verbosity m + + - name: Run GitVersion + id: gitversion + shell: pwsh + run: | + $gitVersion = dotnet tool run dotnet-gitversion /targetpath "${{ github.workspace }}" /output json /updateAssemblyInfo | ConvertFrom-Json + dotnet tool run dotnet-gitversion /targetpath "${{ github.workspace }}/dependencies/NEventStore" /updateAssemblyInfo | Out-Null + "semver=$($gitVersion.SemVer)" >> $env:GITHUB_OUTPUT + + - name: Build + run: dotnet build ./src/NEventStore.Persistence.MongoDB.Core.sln -c Release --no-restore /p:ContinuousIntegrationBuild=True + + - name: Pack + shell: pwsh + run: | + $semver = '${{ steps.gitversion.outputs.semver }}' + New-Item -Path artifacts -ItemType Directory -Force | Out-Null + nuget pack ./src/.nuget/NEventStore.Persistence.MongoDB.nuspec -properties "version=$semver;configuration=Release" -OutputDirectory artifacts -Symbols -SymbolPackageFormat snupkg + + - name: Upload NuGet artifacts + uses: actions/upload-artifact@v7 + with: + name: nuget-packages-${{ steps.gitversion.outputs.semver }} + path: artifacts/**/*.nupkg + if-no-files-found: error + + - name: Publish to NuGet + if: ${{ inputs.publish }} + shell: pwsh + env: + NUGET_API_KEY: ${{ secrets.NUGET_API_KEY }} + run: | + if ([string]::IsNullOrWhiteSpace($env:NUGET_API_KEY)) { + throw 'NUGET_API_KEY secret is required to publish packages.' + } + + Get-ChildItem -Path artifacts -Filter *.nupkg -Recurse | + Where-Object { $_.Name -notlike '*.symbols.nupkg' } | + ForEach-Object { + dotnet nuget push $_.FullName --api-key $env:NUGET_API_KEY --source https://api.nuget.org/v3/index.json --skip-duplicate + } + + Get-ChildItem -Path artifacts -Filter *.snupkg -Recurse | + ForEach-Object { + dotnet nuget push $_.FullName --api-key $env:NUGET_API_KEY --source https://api.nuget.org/v3/index.json --skip-duplicate + } diff --git a/.gitignore b/.gitignore index fe2d71c..d6f8e8c 100644 --- a/.gitignore +++ b/.gitignore @@ -112,4 +112,6 @@ Backup*/ UpgradeLog*.XML # Custom -artifacts/ \ No newline at end of file +BenchmarkDotNet.Artifacts/ +artifacts/ +.tokensave diff --git a/.vscode/tasks.json b/.vscode/tasks.json new file mode 100644 index 0000000..ba06bec --- /dev/null +++ b/.vscode/tasks.json @@ -0,0 +1,59 @@ +{ + "version": "2.0.0", + "tasks": [ + { + "label": "Docker Up", + "options": { + "cwd": "${workspaceFolder}/docker/" + }, + "type": "shell", + "command": "powershell", + "args": [ + "-ExecutionPolicy", + "Bypass", + "-File", + "${workspaceFolder}/docker/DockerComposeUp.ps1", + "linux" + ], + "presentation": { + "echo": true, + "reveal": "always", + "focus": false, + "panel": "shared" + }, + "problemMatcher": [] + }, + { + "label": "Docker Down", + "options": { + "cwd": "${workspaceFolder}/docker/" + }, + "type": "shell", + "command": "powershell", + "args": [ + "-ExecutionPolicy", + "Bypass", + "-File", + "${workspaceFolder}/docker/DockerComposeDown.ps1", + "linux" + ], + "presentation": { + "echo": true, + "reveal": "always", + "focus": false, + "panel": "shared" + }, + "problemMatcher": [] + }, + { + "type": "dotnet", + "task": "build", + "group": { + "kind": "build", + "isDefault": true + }, + "problemMatcher": [], + "label": "dotnet: build" + } + ] +} \ No newline at end of file diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..a4ef336 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,237 @@ +Be concise and token-efficient. Give direct answers, minimal examples, and no extra background. +No sycophantic openers or closing fluff. No emojis or em-dashes. + +These rules apply to every task in this project unless explicitly overridden. +Bias: caution over speed on non-trivial work. Use judgment on trivial tasks. + +## Rule 1 — Think Before Coding +State assumptions explicitly. If uncertain, ask rather than guess. +Present multiple interpretations when ambiguity exists. +Push back when a simpler approach exists. +Stop when confused. Name what's unclear. + +## Rule 2 — Simplicity First +Minimum code that solves the problem. Nothing speculative. +No features beyond what was asked. No abstractions for single-use code. +Test: would a senior engineer say this is overcomplicated? If yes, simplify. + +## Rule 3 — Surgical Changes +Touch only what you must. Clean up only your own mess. +Don't "improve" adjacent code, comments, or formatting. +Don't refactor what isn't broken. Match existing style. + +## Rule 4 — Goal-Driven Execution +Define success criteria. Loop until verified. +Don't follow steps. Define success and iterate. +Strong success criteria let you loop independently. + +## Rule 5 — Use the model only for judgment calls +Use me for: classification, drafting, summarization, extraction. +Do NOT use me for: routing, retries, deterministic transforms. +If code can answer, code answers. + +## Rule 6 — Token budgets are not advisory +Per-task: 4,000 tokens. Per-session: 30,000 tokens. +If approaching budget, summarize and start fresh. +Surface the breach. Do not silently overrun. + +## Rule 7 — Surface conflicts, don't average them +If two patterns contradict, pick one (more recent / more tested). +Explain why. Flag the other for cleanup. +Don't blend conflicting patterns. + +## Rule 8 — Read before you write +Before adding code, read exports, immediate callers, shared utilities. +"Looks orthogonal" is dangerous. If unsure why code is structured a way, ask. + +## Rule 9 — Tests verify intent, not just behavior +Tests must encode WHY behavior matters, not just WHAT it does. +A test that can't fail when business logic changes is wrong. + +## Rule 10 — Checkpoint after every significant step +Summarize what was done, what's verified, what's left. +Don't continue from a state you can't describe back. +If you lose track, stop and restate. + +## Rule 11 — Match the codebase's conventions, even if you disagree +Conformance > taste inside the codebase. +If you genuinely think a convention is harmful, surface it. Don't fork silently. + +## Rule 12 — Fail loud +"Completed" is wrong if anything was skipped silently. +"Tests pass" is wrong if any were skipped. +Default to surfacing uncertainty, not hiding it. + +# NEventStore.Persistence.MongoDB Project Guidelines + +## Scope + +NEventStore.Persistence.MongoDB is a **MongoDB persistence adapter** that implements `IPersistStreams` for the NEventStore event sourcing library. It enables storing event commits, snapshots, and stream metadata in MongoDB instead of relational databases. + +- Treat the contract defined by `IPersistStreams` as a behavioral specification, not implementation detail +- Preserve commit ordering, optimistic concurrency, duplicate commit detection, and snapshot semantics +- Maintain thread-safe `MongoPersistenceEngine` behavior; persistence instances are shared across threads +- Support both synchronous and asynchronous code paths in parallel (see Conventions below) + +## Architecture + +### Core Components +- **`MongoPersistenceEngine` (sync) / `MongoPersistenceEngine.Async` (async)** — Main persistence logic managing three collections: `Commits`, `Streams`, `Snapshots` +- **`MongoPersistenceFactory`** — Creates configured `MongoPersistenceEngine` instances with dependency injection +- **`MongoPersistenceWireup`** — Fluent configuration API in the `NEventStore` namespace (not nested) for transparent integration +- **`MongoPersistenceOptions`** — Encapsulates MongoClient, WriteConcern, database name, and collection naming +- **`MongoCommit` / `MongoFields`** — BSON class map and field constant definitions + +### Collections & Schema +Three collections store the event stream state: +1. **`Commits`** — Event commits with headers, events, and checkpoint numbers (uses `Acknowledged` write concern) +2. **`Streams`** — Stream metadata (ETag, RecycleBin marker, uses `Unacknowledged` write concern for performance) +3. **`Snapshots`** — Snapshots per stream version (uses `Unacknowledged` write concern) + +## Build And Test + +### Local Development +```powershell +# Start MongoDB (required for tests; see docker/ folder for Compose scripts) +.\docker\DockerComposeUp.ps1 + +# Interactive build + optional tests +.\build.ps1 + +# Direct commands +dotnet restore ./src/NEventStore.Persistence.MongoDB.Core.sln --verbosity m +dotnet build ./src/NEventStore.Persistence.MongoDB.Core.sln -c Release --no-restore +dotnet test ./src/NEventStore.Persistence.MongoDB.Core.sln -c Release --no-build +``` + +### CI/CD & Versioning +- **GitHub Actions** build on Windows machines and run tests on MongoDB on linux. +- **GitVersion** auto-patches assembly info from Git tags (do NOT manually edit version metadata) +- **GitFlow workflow**: `release/*` and `hotfix/*` branches lock version increments +- **NuGet packaging**: `nuget pack ./src/.nuget/NEventStore.Persistence.MongoDB.nuspec` outputs symbols package + +### Test Environment +- Set env var: `NEventStore.MongoDB=mongodb://localhost:27017/NEventStore` +- NUnit is active test runner (configured via `DefineConstants=NUNIT` in `.csproj`) +- Acceptance tests inherit from `AcceptanceTestMongoPersistenceFactory` which appends a GUID to database names to prevent parallel test conflicts +- Use filter `-Trait:"Explicit"` in Visual Studio test explorer to exclude long-running explicit tests + +## Conventions + +### Sync + Async Split (Critical Pattern) +- **Every feature must exist in both `MongoPersistenceEngine.cs` (sync) and `MongoPersistenceEngine.Async.cs` (async)** +- Sync methods use blocking LINQ-to-MongoDB; async methods use `IAsyncCursor` and `Task` patterns +- Copy the sync version, convert `IMongoCollection` operations to async equivalents, replace blocking loops with async iteration +- Cancellation tokens are threaded through async paths; sync paths ignore them +- Behavioral parity is required (same logic, same error handling) + +### GUID Serialization (Critical Gotcha) +MongoDB driver v2.30.0 uses `CSharpLegacy` (legacy byte ordering); v3.0.0+ default to `Standard` (compatible across drivers). + +**For backward compatibility, always use `CSharpLegacy`:** +```csharp +// Register globally once at startup +BsonSerializer.RegisterSerializer(new GuidSerializer(GuidRepresentation.CSharpLegacy)); + +// Or configure per field +BsonClassMap.RegisterClassMap(cm => +{ + cm.AutoMap(); + cm.GetMemberMap(c => c.CommitId).SetSerializer(new GuidSerializer(GuidRepresentation.CSharpLegacy)); +}); +``` + +**For Dictionary/Object serialization:** +```csharp +BsonSerializer.RegisterSerializer(new ObjectSerializer( + BsonSerializer.LookupDiscriminatorConvention(typeof(object)), + GuidRepresentation.CSharpLegacy, + ObjectSerializer.AllAllowedTypes)); +``` + +### Write Concerns +- **Commits**: `Acknowledged` — Safety critical (cannot lose events) +- **Streams & Snapshots**: `Unacknowledged` — Performance optimization (metadata can be reconstructed) +- Do not change without understanding the trade-off + +### Field Constants (Not Magic Strings) +Always use constants from `MongoCommitFields` or `MongoFields`: +```csharp +// GOOD +update = Builders.Update.Set(x => x.CheckpointNumber, checkpoint); + +// ACCEPTABLE (MongoCommitFields.CheckpointNumber = "CheckpointNumber") +update = Builders.Update.Set(MongoCommitFields.CheckpointNumber, checkpoint); + +// AVOID +update = Builders.Update.Set("CheckpointNumber", checkpoint); // Magic string +``` + +### Wireup Pattern +`MongoPersistenceWireup` is placed in the `NEventStore` namespace (not nested) for seamless extension method discovery: +```csharp +namespace NEventStore; + +public static class MongoPersistenceWireup +{ + public static MongoPersistenceOptions UseMongoPersistence(this Wireup wireup, ...) + { + // ... + } +} +``` + +Users then configure via fluent API: +```csharp +Wireup.Init() + .UseMongoPersistence(mongoClient, options) + .LogToOutputWindow() + .Build(); +``` + +### RecycleBin Pattern (Soft Deletes) +- Deleted streams are marked with `MongoSystemBuckets.RecycleBin = ":rb"` +- Not automatically cleaned up since v12.0 (caller responsibility) +- Test isolation uses unique database names per run (see `AcceptanceTestMongoPersistenceFactory`) + +### Cancellation & Task Semantics +- Async methods accept `CancellationToken` parameters and honor them +- Do not introduce breaking cancellation behavior changes unless explicitly required +- Sync code does not need cancellation support + +## Dependencies + +### NEventStore Core +The `dependencies/NEventStore/` folder is a Git submodule containing core interfaces: +- `IPersistStreams`, `IEventStream`, `ICommit`, `ISnapshot` +- `PersistenceWireup`, `IPipelineHook`, `IPipelineHookAsync` +- Update with: `git submodule update --init --recursive` + +### External +- **MongoDB.Driver** (latest stable; currently 3.x with `Standard` GUID handling) +- **NUnit** 4.6.0, **MSTest** for testing +- **.NET Framework 4.7.2, .NET Standard 2.1, .NET 6.0+** + +## Known Issues & Migration Notes + +### v11.0.0+: MongoDB Driver 3.0 Breaking Change +MongoDB.Driver 3.0 defaults GUIDs to `Standard` representation (incompatible with older data). Existing deployments must explicitly register `CSharpLegacy` serializer at startup or migrate data. + +### Test Isolation & Parallel Execution +`AcceptanceTestMongoPersistenceFactory` appends Guid to database names to prevent conflicts when tests run in parallel. Do not hardcode database names in tests. + +### Removed RecycleBin Auto-Cleanup (v12.0+) +Previously, deleted streams were auto-removed. Now they're marked with RecycleBin flag. Call `PurgeRecycleBinAsync()` or `PurgeRecycleBin()` if needed. + +## Documentation References +- See [README.md](README.md) for local setup and build instructions +- See [Changelog.md](Changelog.md) for versioned behavior changes before altering compatibility-sensitive code +- See [dependencies/NEventStore/Readme.md](dependencies/NEventStore/Readme.md) for core NEventStore architecture +- See [docs/](dependencies/NEventStore/docs/) in the NEventStore submodule for testing strategy, performance notes, and benchmarks + +## Supplementary Task-Specific Instructions + +Before making changes, check for scoped instructions or skills in the parent NEventStore project that may override conventions above: + +- **Scoped instructions**: Check [`.github/instructions/`](dependencies/NEventStore/.github/instructions/) for files matching the area you are changing (applyTo glob patterns) +- **Reusable skills**: Check [`.agents/skills/`](dependencies/NEventStore/.agents/skills/) for step-by-step workflows for recurring tasks diff --git a/Changelog.md b/Changelog.md index 84618d0..9047d35 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,5 +1,16 @@ # NEventStore.Persistence.MongoDB +## vNext + +- Added explicit support for net8.0, net9.0, net10.0. +- Updated NEventStore to 10.2.0 +- Performance: per-stream read queries require in-memory sort (CheckpointNumber not in index) [#73](https://github.com/NEventStore/NEventStore.Persistence.MongoDB/issues/73) +- Some code cleanup and refactoring. + +### Breaking Changes + +- Requires updating NEventStore to 10.2.0 or higher + ## 12.0.0 - Async methods added to IPersistStreams interfaces [#71](https://github.com/NEventStore/NEventStore.Persistence.MongoDB/issues/71) diff --git a/GitVersion.yml b/GitVersion.yml index 27f681d..6f83d0b 100644 --- a/GitVersion.yml +++ b/GitVersion.yml @@ -1,5 +1,14 @@ +workflow: GitFlow/v1 mode: ContinuousDeployment -branches: {} +branches: + release: + regex: ^release[/-] + increment: None # Do not bump the version automatically, previus value: Patch + hotfix: + regex: ^hotfix[/-] + increment: None # previous value: Patch + unknown: + increment: Patch ignore: sha: [] merge-message-formats: {} diff --git a/README.md b/README.md index ffdbe46..e613b86 100644 --- a/README.md +++ b/README.md @@ -14,8 +14,8 @@ Build Status Branches: -- master [![Build status](https://ci.appveyor.com/api/projects/status/8euhhjl05lhng8ka/branch/master?svg=true)](https://ci.appveyor.com/project/AGiorgetti/neventstore-persistence-mongodb/branch/master) -- develop [![Build status](https://ci.appveyor.com/api/projects/status/8euhhjl05lhng8ka/branch/develop?svg=true)](https://ci.appveyor.com/project/AGiorgetti/neventstore-persistence-mongodb/branch/develop) +- master [![CI](https://github.com/NEventStore/NEventStore.Persistence.MongoDB/actions/workflows/ci.yml/badge.svg?branch=master)](https://github.com/NEventStore/NEventStore.Persistence.MongoDB/actions/workflows/ci.yml) +- develop [![CI](https://github.com/NEventStore/NEventStore.Persistence.MongoDB/actions/workflows/ci.yml/badge.svg?branch=develop)](https://github.com/NEventStore/NEventStore.Persistence.MongoDB/actions/workflows/ci.yml) Information @@ -52,6 +52,39 @@ To build the project locally on a Windows Machine: NEventStore.MongoDB="mongodb://localhost:50002/NEventStore" ``` +## Run Benchmarks (locally) + +- Build benchmark project: + +```powershell +dotnet build .\src\NEventStore.Persistence.MongoDB.Benchmark\NEventStore.Persistence.MongoDB.Benchmark.csproj -c Release +``` + +- Set benchmark connection string in current shell: + +```powershell +$env:NEventStore.MongoDB = 'mongodb://localhost:50002/NEventStore' +``` + +- List all discovered benchmark cases: + +```powershell +dotnet .\src\NEventStore.Persistence.MongoDB.Benchmark\bin\Release\net8.0\NEventStore.Persistence.MongoDB.Benchmark.dll --list flat +``` + +- Run all benchmark cases: + +```powershell +dotnet .\src\NEventStore.Persistence.MongoDB.Benchmark\bin\Release\net8.0\NEventStore.Persistence.MongoDB.Benchmark.dll --filter * +``` + +- Run specific benchmark class or method via filter: + +```powershell +dotnet .\src\NEventStore.Persistence.MongoDB.Benchmark\bin\Release\net8.0\NEventStore.Persistence.MongoDB.Benchmark.dll --filter *CheckpointGeneratorBenchmarks* +dotnet .\src\NEventStore.Persistence.MongoDB.Benchmark\bin\Release\net8.0\NEventStore.Persistence.MongoDB.Benchmark.dll --filter *ReadFromEventStoreAsyncBenchmarks.ReadFromEventStoreAsync* +``` + ## Run Tests in Visual Studio To run tests in visual studio using NUnit as a Test Runner you need to explicitly exclude "Explicit Tests" from running adding the following filter in the test explorer section: diff --git a/dependencies/NEventStore b/dependencies/NEventStore index aaef5b5..3d58d5a 160000 --- a/dependencies/NEventStore +++ b/dependencies/NEventStore @@ -1 +1 @@ -Subproject commit aaef5b5a8f07d5774a6fd7291cb09c1c676a8f83 +Subproject commit 3d58d5a9be3ed74d5217522262ad1ca25a5e394f diff --git a/docker/docker-compose.ci.linux.db.yml b/docker/docker-compose.ci.linux.db.yml index d85ae40..d322041 100644 --- a/docker/docker-compose.ci.linux.db.yml +++ b/docker/docker-compose.ci.linux.db.yml @@ -1,12 +1,7 @@ -# Root Docker Compose file to run the tests on the development machine using some pre-initialized databases -# It must be used with powershell actions that mount the databases files. - -version: '2.4' # 2.4 supports "platform", 3.x does not. - services: mongo: platform: linux - image: mongo #@sha256:52c3314bee611f91d37b9b1bc0cc2755b1388f2de5b396b441f3fe94bef6c56c + image: mongo:8 ports: - "50002:27017" # volumes: diff --git a/docker/docker-compose.ci.windows.db.yml b/docker/docker-compose.ci.windows.db.yml index 82a5f2c..7a6d84e 100644 --- a/docker/docker-compose.ci.windows.db.yml +++ b/docker/docker-compose.ci.windows.db.yml @@ -1,12 +1,7 @@ -# Root Docker Compose file to run the tests on the development machine using some pre-initialized databases -# It must be used with powershell actions that mount the databases files. - -version: '2.4' # 2.4 supports "platform", 3.x does not. - services: mongo: platform: windows - image: mongo + image: mongo:8 ports: - "50002:27017" # volumes: diff --git a/docs/Performance-Investigation.md b/docs/Performance-Investigation.md new file mode 100644 index 0000000..6282cae --- /dev/null +++ b/docs/Performance-Investigation.md @@ -0,0 +1,559 @@ +# Performance Investigation + +This document captures the current performance state of `NEventStore.Persistence.MongoDB` before any engine changes. + +## Scope + +- Inspect the MongoDB persistence engine hot paths. +- Validate the current benchmark suite. +- Record the first set of evidence-backed optimization targets. +- Defer runtime code changes until the benchmark harness can produce trustworthy baselines. + +## Current Handoff (2026-06-10) + +This is the current state after reviewing `docs/Performance-Investigation.md`, local code, benchmark artifacts, and GitHub issues tagged `performance`. + +### What is done + +- Benchmark harness is usable from the CLI and has sync/async read/write coverage. +- Issue [#73](https://github.com/NEventStore/NEventStore.Persistence.MongoDB/issues/73) is closed by decision: + - Per-stream reads now sort by `StreamRevisionFrom`, not `CheckpointNumber`. + - `Issue73ExplainPlans` asserts `GetFrom_Index` is used and no `SORT` stage is present for the stream range read. + - `Changelog.md` has a #73 entry. +- Issue [#74](https://github.com/NEventStore/NEventStore.Persistence.MongoDB/issues/74) is closed by decision: + - Attempted eager-deserialization optimizations were too complex for the negligible gain observed. + - The current `doc.ToCommit(_serializer)` path remains intentionally simple. +- The explain audit for issue [#75](https://github.com/NEventStore/NEventStore.Persistence.MongoDB/issues/75) has been investigated: + - All-buckets checkpoint reads are not a COLLSCAN regression. + - MongoDB chooses `_id_`, not `GetFrom_Checkpoint_Index`. + - A partial `_id` index is not viable because MongoDB rejects `partialFilterExpression` on `_id`. +- Issue [#76](https://github.com/NEventStore/NEventStore.Persistence.MongoDB/issues/76) is measured and on standby: + - `InMemoryCheckpointGenerator` is materially faster for writes. + - Do not change the default because the current default preserves no-hole behavior after concurrency exceptions. + - Treat `InMemoryCheckpointGenerator` as an explicit opt-in tuning option only. +- Issue [#77](https://github.com/NEventStore/NEventStore.Persistence.MongoDB/issues/77) is investigated and on standby: + - The successful commit path does re-materialize the inserted BSON document via `commitDoc.ToCommit(_serializer)`. + - The obvious shortcut would return original `CommitAttempt` event messages instead of serializer round-tripped messages. + - Do not change without a dedicated microbenchmark and return-contract tests. +- Issue [#78](https://github.com/NEventStore/NEventStore.Persistence.MongoDB/issues/78) is measured and on standby: + - Background stream-head updates did not show a clear repeatable wall-clock win. + - No engine change is justified from the current benchmark data. +- Current benchmark slices cover the known hotspots: stream reads, global reads, checkpoint generator choice, snapshot/background updates, duplicate conflicts, recycle-bin reads, sync writes, and async writes. + +### Issue decision summary + +| Issue | Decision | Follow-up trigger | +|---|---|---| +| [#73](https://github.com/NEventStore/NEventStore.Persistence.MongoDB/issues/73) | Closed by decision. Keep the stream-read sort fix and explain-plan evidence. | Revisit only if a future benchmark or explain audit contradicts the current index-plan result. | +| [#74](https://github.com/NEventStore/NEventStore.Persistence.MongoDB/issues/74) | Closed by decision. Keep eager `ToCommit` materialization simple. | Revisit only with a simple, clearly measurable read-path win. | +| [#75](https://github.com/NEventStore/NEventStore.Persistence.MongoDB/issues/75) | Standby. Not a rebuild bottleneck and not a COLLSCAN regression. | Revisit only for recycle-bin-heavy all-buckets polling workloads. | +| [#76](https://github.com/NEventStore/NEventStore.Persistence.MongoDB/issues/76) | Standby/opt-in. `InMemoryCheckpointGenerator` is faster but changes checkpoint-hole behavior. | Document or expose guidance for callers that explicitly accept holes for write throughput. | +| [#77](https://github.com/NEventStore/NEventStore.Persistence.MongoDB/issues/77) | Standby. Return-path shortcut is observable and needs tighter tests. | Build a microbenchmark for `commitDoc.ToCommit(_serializer)` and return materialization contract tests. | +| [#78](https://github.com/NEventStore/NEventStore.Persistence.MongoDB/issues/78) | Standby. Background stream-head update comparison is noisy and no clear win. | Revisit only if write throughput is the target and a stronger benchmark isolates this path. | + +### Conflicts to resolve + +- Canonical rebuild-focused benchmark snapshots now exist for `ReadFromStreamBenchmarks`, `StreamRevisionWindowBenchmarks`, and `SnapshotAssistedRebuildBenchmarks`. +- The `ReadFromStream(10000)` after mean is slower than the original before value on this machine (`189.337 ms` after vs `148.004 ms` before), despite the #73 explain-plan fix. Treat #73 as an index-plan correctness fix, not a proven wall-clock speedup from the current benchmark data. +- The tail-window benchmark was changed to seed directly through `IPersistStreams`, so compare its new values against future runs of the same benchmark shape, not against the older before table. +- Rebuild-read benchmarks no longer pin `InvocationCount=1`; BenchmarkDotNet now chooses invocation counts through its pilot phase to avoid sub-100 ms iteration warnings. + +### Rebuild-focused direction + +The current optimization target is read-heavy rebuild operations, not write throughput or global checkpoint polling. + +For aggregate rebuilds, the hot path is: + +1. Optional snapshot lookup via `GetSnapshot(bucketId, streamId, maxRevision)`. +2. Stream commit read via `GetFrom(bucketId, streamId, minRevision, maxRevision)`. +3. Full `ICommit` and event payload materialization through `doc.ToCommit(_serializer)`. + +Current implications: + +- #73 is the main completed rebuild optimization: per-stream reads now sort by `StreamRevisionFrom`, matching `GetFrom_Index` and avoiding the old checkpoint-sort plan. +- #74 is intentionally closed: eager `ToCommit` materialization remains simple because attempted optimizations were too complex for negligible gain. +- #75 is not a rebuild bottleneck. It affects all-buckets checkpoint polling, so it stays in standby. +- #76 is a write-path opt-in trade-off, not a default change: `InMemoryCheckpointGenerator` is faster but can leave checkpoint holes after concurrency exceptions. +- #77 is a write-path standby item. It needs a dedicated microbenchmark and return-contract tests before any engine change. +- #78 is a write-path standby item. Keep it behind rebuild-focused read work unless write throughput becomes the target again. + +What matters next for rebuilds: + +- Use the #73 explain audit as the primary evidence that per-stream reads now use `GetFrom_Index` without a blocking `SORT`. +- Use the canonical rebuild benchmark snapshots below as the current read-heavy evidence. +- Compare future rebuild changes against the archived `rebuild-readfromstream`, `rebuild-tailwindow`, and `rebuild-snapshot-assisted` snapshots. +- Prefer snapshot-assisted rebuilds for long streams when the caller has a recent snapshot; the benchmark shows the largest allocation reduction there. + +### What's left + +| Priority | GitHub issue | Status | Remaining work | +|---:|---|---|---| +| 1 | Rebuild benchmark evidence | Done for current pass | Canonical rebuild snapshots are archived for full stream, tail-window, and snapshot-assisted rebuild reads. | +| 2 | [#73](https://github.com/NEventStore/NEventStore.Persistence.MongoDB/issues/73) stream-read sort | Closed by decision | Keep the explain-plan evidence as the reason for closure. Do not claim a wall-clock benchmark win from the current data. | +| 3 | [#75](https://github.com/NEventStore/NEventStore.Persistence.MongoDB/issues/75) all-buckets checkpoint scan | Standby after investigation | Not a rebuild bottleneck. Resume only if all-buckets polling with a large recycle bin becomes a target workload. | +| 4 | [#76](https://github.com/NEventStore/NEventStore.Persistence.MongoDB/issues/76) checkpoint generator DB read | Measured, write-path opt-in | Stabilized benchmark confirms `InMemoryCheckpointGenerator` is faster, but it changes checkpoint-hole behavior. Do not change the default. | +| 5 | [#78](https://github.com/NEventStore/NEventStore.Persistence.MongoDB/issues/78) stream-head background updates | Measured, write-path standby | Stabilized snapshot-overhead slice is archived. No engine change yet because background on/off remains noisy and shows no clear win. | +| 6 | [#77](https://github.com/NEventStore/NEventStore.Persistence.MongoDB/issues/77) post-insert re-deserialization | Investigated, write-path standby | Do not change without a dedicated microbenchmark and return-contract tests. Avoid returning the original `CommitAttempt` events unless serializer round-trip semantics are proven unnecessary. | + +### Next recommended pass + +1. Rerun the rebuild snapshots with the stabilized benchmark job before comparing future read-heavy changes. +2. Use the archived rebuild snapshots as historical evidence for the #73 pass, not as the final stable baseline. +3. Keep engine code simple unless a future benchmark shows a clear, repeatable rebuild win. +4. Revisit #75/#76/#78/#77 only if the target workload changes away from rebuild reads. +5. For write-side work, treat #76 as an opt-in tuning option, #77 as contract-sensitive standby, and #78 as measured/standby. + +## Current Benchmark Status + +The benchmark harness is now runnable and selectable from a single entrypoint. + +### Baseline Profile (Before Snapshot) + +Use this baseline profile for all future optimization comparisons: + +- Runtime: `.NET 10.0` only (`net10.0` benchmark binary). +- Connection: `NEventStore.MongoDB=mongodb://localhost:50002/NEventStore`. +- Benchmark job: class-defined default (`LaunchCount=3`, `WarmupCount=3`, `IterationCount=3`, automatic invocation count). +- Artifacts folder: `BenchmarkDotNet.Artifacts/results/`. + +Baseline generation commands: + +```powershell +dotnet build .\src\NEventStore.Persistence.MongoDB.Benchmark\NEventStore.Persistence.MongoDB.Benchmark.csproj -c Release -f net10.0 +[Environment]::SetEnvironmentVariable('NEventStore.MongoDB', 'mongodb://localhost:50002/NEventStore', 'Process') + +# Run full suite (net10) +dotnet .\src\NEventStore.Persistence.MongoDB.Benchmark\bin\Release\net10.0\NEventStore.Persistence.MongoDB.Benchmark.dll --filter * +``` + +### Commands used + +```powershell +[Environment]::SetEnvironmentVariable('NEventStore.MongoDB', 'mongodb://localhost:50002/NEventStore', 'Process') +``` + +### Relevant evidence in the codebase + +- Acceptance tests register the required MongoDB serializers in `src/NEventStore.Persistence.MongoDB.Tests/AcceptanceTestMongoPersistenceFactory.cs`. +- The benchmark helper in `src/NEventStore.Persistence.MongoDB.Benchmark/Support/EventStoreHelpers.cs` now mirrors that serializer setup and allows benchmark-specific `MongoPersistenceOptions`. +- The benchmark entrypoint in `src/NEventStore.Persistence.MongoDB.Benchmark/Program.cs` now uses `BenchmarkSwitcher.FromAssembly(...)`. + +## Benchmark Suite Gaps + +### Structural gaps + +- No structural benchmark-entrypoint gaps remain for sync/async coverage currently implemented. + +### Coverage gaps + +The suite now includes these slices: + +- Commit throughput with checkpoint generator comparison (`Always` vs `InMemory`). +- Per-stream read throughput (full stream and focused revision windows). +- Global checkpoint scans (bucket-qualified and all-buckets). +- Async global-read path and async commit path. +- Snapshot-related write overhead via `DisableSnapshotSupport` and `PersistStreamHeadsOnBackgroundThread` combinations. +- Snapshot-assisted aggregate rebuild reads. +- Delete and recycle-bin read behavior. +- Duplicate-commit and duplicate-checkpoint retry paths. + +## High-Value Performance Findings + +These are the strongest candidates for meaningful improvement, ordered by read-side first (higher leverage), then write-side findings. Confidence levels reflect data from benchmarks and index analysis. + +### 1. Stream read queries no longer require checkpoint sort + +**Read-side critical, implemented on this branch.** The original finding was that per-stream reads filtered on `(BucketId, StreamId, StreamRevisionFrom, StreamRevisionTo)` but sorted by `CheckpointNumber`. + +Original index/query mismatch: + +``` +INDEX: (BucketId, StreamId, StreamRevisionFrom, StreamRevisionTo) +QUERY FILTER: BucketId = X, StreamId = Y, revision range +QUERY SORT: CheckpointNumber ASC +``` + +Why this mattered: + +- The index supports the filter but NOT the sort (CheckpointNumber is not in the index). +- MongoDB must perform an in-memory sort after filtering, which is expensive for large streams. +- Large aggregate reconstruction (many commits per stream) will hit this bottleneck directly. + +Current state: + +- Per-stream sync and async reads now sort by `StreamRevisionFrom`. +- The existing `GetFrom_Index` supports the filter and sort shape. +- `Issue73ExplainPlans.Commit_range_read_should_use_GetFrom_index_without_a_sort_stage` asserts index usage and no `SORT` stage. +- Remaining work is documentation/issue closeout plus a clean after benchmark rerun. + +### 2. Read paths eagerly deserialize commits by design + +**Closed by decision.** Every `GetFrom*` path ends in `doc.ToCommit(_serializer)`. + +`ToCommit` currently does all of the following for every document: + +- Deserializes the whole `MongoCommit` object from `BsonDocument`. +- Iterates every stored event. +- Deserializes each `EventMessage` payload. +- Materializes the event set into an array. + +Why this was investigated: + +- Global checkpoint scans pay full payload materialization cost even when the caller only needs iteration or metadata. +- This cost grows with event count and payload size. +- Allocation pressure will likely dominate long sequential reads (especially for large event counts per commit). + +Decision: + +- Attempted optimizations were too complex for the negligible gain observed. +- Keep the current eager `ToCommit` path because it is simple and preserves the existing `ICommit` materialization behavior. +- No further #74 optimization work is planned. + +### 3. All-buckets checkpoint reads use inefficient filter + +**Read-side secondary, workload-dependent.** The all-buckets checkpoint scan uses `BucketId != :rb` to exclude recycled streams. + +Why this matters: + +- Negative filters (`!=`) are less index-friendly than positive inclusion. +- Bucket-qualified reads use the primary checkpoint index; all-buckets reads may bypass it. +- Small performance cost, but measurable for high-volume checkpoint polling. + +MongoDB explain check on the current query shape: + +- `GetFrom(Int64 checkpointToken)` and `GetFromTo(long fromCheckpointToken, long toCheckpointToken)` both plan as `FETCH + IXSCAN`. +- The winning index for the all-buckets query is `_id_`, not `GetFrom_Checkpoint_Index`. +- That means this is **not** a COLLSCAN regression, and it is also **not** a missing-index problem. +- The gap is the negative filter shape preventing MongoDB from using the bucket-qualified checkpoint index. +- An explicit inclusion rewrite (`BucketId in [active buckets]`) did not change the winning plan in the checked data shape; MongoDB still chose `_id_`. +- A dedicated partial `_id` index is not a viable shortcut here because MongoDB rejects `partialFilterExpression` on `_id`. + +Follow-up investigation from issue comments: + +- The benchmark comparison is not apples-to-apples: `ReadFromAllBucketsCheckpoint` returns all active buckets, while `ReadFromBucketCheckpoint` returns only `Bucket.Default`. +- With `ExtraBuckets=3`, all-buckets returns 4x as many commits. The 67.011 ms vs 22.142 ms result is therefore not proof of a 3x per-commit regression. +- Rewriting `BucketId != :rb` as an `$or` split around `:rb` does not help. MongoDB still chooses `_id_`; forcing `GetFrom_Checkpoint_Index` adds `SORT + OR`. +- A partial compound index using `{ _id: 1, BucketId: 1 }` with `partialFilterExpression: { BucketId: { $ne: ':rb' } }` is also rejected because `$ne` is unsupported in partial indexes. +- A non-partial `{ _id: 1, BucketId: 1 }` index is legal and can reduce `totalDocsExamined` by filtering `:rb` from index keys before fetch, but it still scans the same checkpoint key range and adds write/storage cost. +- In a recycle-bin-heavy scratch shape, MongoDB may prefer `GetFrom_Checkpoint_Index` plus an explicit `SORT` when the active bucket set is tiny. That is a workload-specific trade-off, not a safe default optimization. + +Current recommendation: + +- Put #75 in standby based on the investigation. +- Do not change the engine for #75 based on current evidence. +- Resume only with a dedicated benchmark for recycle-bin-heavy all-buckets polling before considering `{ _id: 1, BucketId: 1 }`. +- Treat regular recycle-bin cleanup as the preferred operational mitigation. + +### 4. Default checkpoint generation adds a database read per commit + +**Write-side opt-in trade-off.** `MongoPersistenceEngine.Initialize()` defaults to `AlwaysQueryDbForNextValueCheckpointGenerator`. + +That generator calls `GetLastValue()` on every `Next()` invocation, which means one extra database read for every commit before the insert even happens. + +Why this matters: + +- Write-heavy workloads will pay an extra round trip per commit. +- However, read-side deserialization cost typically dominates for mixed workloads. +- The cost scales directly with commit volume. + +What needs benchmarking first: + +- Completed: current default generator vs `InMemoryCheckpointGenerator` behavior is covered by `CheckpointGeneratorBenchmarks`. +- Do not change the default generator without a breaking-change decision, because acceptance tests cover the current no-hole default after concurrency exceptions. +- Use `InMemoryCheckpointGenerator` only as an explicit write-throughput tuning option for callers that accept checkpoint holes after concurrency conflicts. + +### 5. Commit success path re-deserializes the inserted document + +**Write-side standby.** After a successful insert, `Commit()` returns `commitDoc.ToCommit(_serializer)`. + +That means the write path serializes the commit to BSON for insert, then immediately deserializes the same BSON back to `ICommit` to return it. + +Why this matters: + +- Pure post-insert CPU and allocation overhead. +- The cost is paid on every successful commit, but write insertion is typically not the loop bottleneck. +- It is independent of MongoDB round-trip latency. + +Follow-up investigation: + +- Sync and async commit paths both return `commitDoc.ToCommit(_serializer)` after successful insert. +- The obvious shortcut is constructing `Commit` directly from `CommitAttempt` plus the generated checkpoint, but that would return the caller's event messages rather than the serializer round-tripped event messages currently produced by `ToCommit`. +- Commit hooks receive the returned `ICommit`, so this is observable behavior, not only an internal allocation detail. +- A stabilized full `WriteToStreamBenchmarks` run was attempted for this slice, but the 10,000-commit case exceeded the command timeout and produced no usable archive. + +Current recommendation: + +- Keep #77 in standby. +- Do not change the runtime commit return path without a dedicated microbenchmark that isolates `commitDoc.ToCommit(_serializer)` and tests that lock down acceptable return materialization semantics. +- If write throughput becomes the target again, prefer a narrow benchmark that compares `commitDoc.ToCommit(_serializer)` against a candidate `CommitAttempt`-based materializer before touching `MongoPersistenceEngine.Commit`. + +### 6. Stream-head updates schedule background work per commit + +**Write-side maintenance.** When snapshot support is enabled, successful commits call `UpdateStreamHeadInBackgroundThread(...)`. + +On the sync path this uses `Task.Run` per commit when background updates are enabled. + +Why this matters: + +- Task-scheduling overhead on high write rates. +- Fire-and-forget work can add GC pressure and tail latency. +- This cost is isolated in the snapshot-overhead benchmark slices. + +What needs benchmarking first: + +- Snapshot support enabled vs disabled (already in benchmark suite). +- Background updates on/off comparison. + +## Lower-Priority Findings + +These are worth monitoring, but they are significantly lower-leverage than the findings above: + +- Logging warnings from CA1848 suggest `LoggerMessage` could reduce hot-path logging overhead, but database and serialization work should be measured first. +- `AddSnapshot()` performs an extra stream-head read after snapshot upsert; this matters only for snapshot-heavy workloads and currently lacks deep benchmark coverage. + +## Recommended Benchmark Work Before Engine Changes + +The next performance pass should start by improving the benchmark harness, not the engine. + +### Priority 0: Make the current suite runnable + +- Completed: serializer registration and startup wiring fixed in benchmark setup. + +### Priority 1: Make the suite selectable from the command line + +- Completed: `Program.cs` now uses `BenchmarkSwitcher` and supports class/method filtering from CLI. + +### Priority 2: Add focused benchmark slices + +- Completed: + - Write throughput: default checkpoint generator vs in-memory checkpoint generator. + - Stream read throughput: full-stream reads plus focused revision-window reads. + - Global read throughput: bucket-qualified and all-buckets checkpoint scans. + - Snapshot overhead: snapshot support on/off and stream-head background updates on/off. + - Async parity: async read and async commit scenarios added. + - Delete/recycle-bin behavior and duplicate conflict/retry paths. + +## Recommended Investigation Order Once Benchmarks Are Healthy + +The next performance pass should focus on aggregate rebuild reads. + +1. Compare future read-heavy changes against the stabilized rebuild snapshots. +2. If `SnapshotAssistedRebuild(10000, 1000)` remains noisy in future runs, isolate it and rerun before using its mean in a decision. +3. Keep #75 in standby, treat #76 as an opt-in write tuning option, and defer #78/#77 unless write throughput becomes the target again. + +## Rebuild Benchmark Snapshot + +These values are from stabilized class-defined BenchmarkDotNet jobs on 2026-06-10 using `LaunchCount=3`, `WarmupCount=3`, `IterationCount=3`, and automatic invocation counts selected by BenchmarkDotNet's pilot phase. + +Archived raw outputs: + +- `artifacts/benchmark-snapshots/benchmark-after-rebuild-readfromstream-stabilized-net10.0-20260610-1451.zip` +- `artifacts/benchmark-snapshots/benchmark-after-rebuild-tailwindow-stabilized-net10.0-20260610-1501.zip` +- `artifacts/benchmark-snapshots/benchmark-after-rebuild-snapshot-assisted-stabilized-net10.0-20260610-1515.zip` + +These stabilized runs emitted no `MinIterationTime` warnings. Historical pinned-invocation archives from the #73 pass remain useful for traceability, but future read-heavy comparisons should use the stabilized snapshots above. + +### Full Stream Read + +| Benchmark | Parameters | Mean | Allocated | +|---|---|---:|---:| +| `ReadFromStream` | `CommitsToWrite=100` | 3.174 ms | 699.78 KB | +| `ReadFromStream` | `CommitsToWrite=1000` | 17.476 ms | 6,972.27 KB | +| `ReadFromStream` | `CommitsToWrite=10000` | 205.481 ms | 69,760.60 KB | + +### Tail Revision Window + +| Total commits | Window | Mean | Allocated | +|---:|---:|---:|---:| +| 1,000 | 10 | 2.648 ms | 91.90 KB | +| 1,000 | 100 | 3.800 ms | 713.21 KB | +| 1,000 | 1,000 | 30.759 ms | 6,957.40 KB | +| 10,000 | 10 | 12.932 ms | 91.92 KB | +| 10,000 | 100 | 16.359 ms | 713.17 KB | +| 10,000 | 1,000 | 31.403 ms | 6,971.67 KB | + +The allocation shape is the useful signal here: allocations track the returned window size, while the fixed overhead of seeking a tail window grows with larger stream size. + +### Snapshot-Assisted Rebuild + +| Total commits | Commits after snapshot | Full rebuild | Snapshot-assisted | Full alloc | Snapshot alloc | +|---:|---:|---:|---:|---:|---:| +| 1,000 | 10 | 14.386 ms | 3.128 ms | 6,972.37 KB | 99.10 KB | +| 1,000 | 100 | 16.210 ms | 6.354 ms | 6,972.34 KB | 732.74 KB | +| 1,000 | 1,000 | 18.163 ms | 18.451 ms | 6,972.40 KB | 6,973.17 KB | +| 10,000 | 10 | 195.717 ms | 12.842 ms | 69,760.78 KB | 99.44 KB | +| 10,000 | 100 | 204.460 ms | 13.877 ms | 69,760.78 KB | 732.74 KB | +| 10,000 | 1,000 | 205.723 ms | 52.314 ms | 69,760.77 KB | 6,994.45 KB | + +Snapshot-assisted rebuilds materially reduce work when the snapshot is near the tail. For 10,000-commit streams, allocations drop from about 68 MB for full rebuilds to about 0.1 MB, 0.7 MB, or 6.8 MB depending on the number of commits after the snapshot. The `10,000 / 1,000` snapshot-assisted timing had high variance in this run (`52.314 ms` mean, `43.982 ms` stddev), but its allocation reduction is still clear. + +## Write-Side Snapshot-Overhead Snapshot + +Issue [#78](https://github.com/NEventStore/NEventStore.Persistence.MongoDB/issues/78) was rechecked on 2026-06-10 with `SnapshotOverheadBenchmarks` after removing the pinned `InvocationCount=1` from the benchmark job. BenchmarkDotNet now selects invocation counts through its pilot phase for this slice too. + +Archived raw output: + +- `artifacts/benchmark-snapshots/benchmark-after-write-snapshot-overhead-stabilized-net10.0-20260610-1712.zip` + +| Commits | Snapshot support disabled | Background stream-head update | Mean | StdDev | Allocated | +|---:|---|---|---:|---:|---:| +| 100 | false | false | 206.2 ms | 47.78 ms | 22.69 MB | +| 100 | false | true | 220.3 ms | 140.86 ms | 22.70 MB | +| 100 | true | false | 149.2 ms | 16.86 ms | 28.16 MB | +| 100 | true | true | 140.1 ms | 4.12 ms | 21.08 MB | +| 1,000 | false | false | 1,724.0 ms | 362.08 ms | 110.82 MB | +| 1,000 | false | true | 1,676.5 ms | 477.40 ms | 110.96 MB | +| 1,000 | true | false | 1,623.6 ms | 366.49 ms | 94.77 MB | +| 1,000 | true | true | 1,954.2 ms | 419.49 ms | 94.74 MB | + +Current decision: + +- Background stream-head updates do not show a clear repeatable wall-clock win. +- With snapshot support enabled, 1,000-commit allocation remains about 111 MB regardless of background mode. +- Disabling snapshot support lowers 1,000-commit allocation to about 95 MB, but that changes behavior and only applies to callers that do not need snapshots. +- Keep #78 in standby. Do not replace the current simple per-commit background update path without a stronger write-throughput benchmark signal. + +## Write-Side Checkpoint-Generator Snapshot + +Issue [#76](https://github.com/NEventStore/NEventStore.Persistence.MongoDB/issues/76) was rechecked on 2026-06-10 with `CheckpointGeneratorBenchmarks` after removing the pinned `InvocationCount=1` from the benchmark job. BenchmarkDotNet now selects invocation counts through its pilot phase for this slice too. + +Archived raw output: + +- `artifacts/benchmark-snapshots/benchmark-after-write-checkpoint-generator-stabilized-net10.0-20260610-1723.zip` + +| Commits | Generator | Mean | StdDev | Allocated | +|---:|---|---:|---:|---:| +| 100 | Always | 244.3 ms | 14.29 ms | 16.88 MB | +| 100 | InMemory | 147.7 ms | 6.87 ms | 28.24 MB | +| 1,000 | Always | 2,506.2 ms | 238.76 ms | 110.96 MB | +| 1,000 | InMemory | 1,322.9 ms | 274.52 ms | 95.54 MB | + +Current decision: + +- `InMemoryCheckpointGenerator` is materially faster for write throughput in this benchmark. +- The default `AlwaysQueryDbForNextValueCheckpointGenerator` should not be changed in this pass because it preserves the current no-hole default after concurrency exceptions. +- `InMemoryCheckpointGenerator` remains the safe performance path only as explicit caller configuration when checkpoint holes after concurrency conflicts are acceptable. +- Keep #76 as measured/opt-in rather than a runtime engine change. + +## Artifacts From This Investigation + +BenchmarkDotNet outputs reports under `BenchmarkDotNet.Artifacts/results/` for each selected benchmark class. + +With the current harness, those artifacts are suitable for baseline capture prior to engine optimizations. + +### Before Snapshot (Net10) + +The canonical "before" snapshot is the set of these net10 report files: + +- `NEventStore.Persistence.MongoDB.Benchmark.Benchmarks.CheckpointGeneratorBenchmarks-report-github.md` +- `NEventStore.Persistence.MongoDB.Benchmark.Benchmarks.DuplicateConflictBenchmarks-report-github.md` +- `NEventStore.Persistence.MongoDB.Benchmark.Benchmarks.GlobalCheckpointReadBenchmarks-report-github.md` +- `NEventStore.Persistence.MongoDB.Benchmark.Benchmarks.ReadFromEventStoreAsyncBenchmarks-report-github.md` +- `NEventStore.Persistence.MongoDB.Benchmark.Benchmarks.ReadFromEventStoreBenchmarks-report-github.md` +- `NEventStore.Persistence.MongoDB.Benchmark.Benchmarks.ReadFromStreamBenchmarks-report-github.md` +- `NEventStore.Persistence.MongoDB.Benchmark.Benchmarks.RecycleBinReadBenchmarks-report-github.md` +- `NEventStore.Persistence.MongoDB.Benchmark.Benchmarks.SnapshotAssistedRebuildBenchmarks-report-github.md` +- `NEventStore.Persistence.MongoDB.Benchmark.Benchmarks.SnapshotOverheadBenchmarks-report-github.md` +- `NEventStore.Persistence.MongoDB.Benchmark.Benchmarks.StreamRevisionWindowBenchmarks-report-github.md` +- `NEventStore.Persistence.MongoDB.Benchmark.Benchmarks.WriteToStreamAsyncBenchmarks-report-github.md` +- `NEventStore.Persistence.MongoDB.Benchmark.Benchmarks.WriteToStreamBenchmarks-report-github.md` + +To avoid committing raw benchmark artifacts, the key baseline values are summarized here. + +| Benchmark Slice | Method + Parameters (Before) | Before Mean | +|---|---|---:| +| Checkpoint generator write path | `WriteWithCheckpointGenerator` (`CommitsToWrite=1000`, `GeneratorType=Always`) | 2,908.6 ms | +| Checkpoint generator write path (comparison) | `WriteWithCheckpointGenerator` (`CommitsToWrite=1000`, `GeneratorType=InMemory`) | 1,541.0 ms | +| Global read (bucket-qualified) | `ReadFromBucketCheckpoint` (`CommitsPerBucket=1000`, `ExtraBuckets=3`) | 22.142 ms | +| Global read (all buckets) | `ReadFromAllBucketsCheckpoint` (`CommitsPerBucket=1000`, `ExtraBuckets=3`) | 67.011 ms | +| Per-stream full read | `ReadFromStream` (`CommitsToWrite=10000`) | 148.004 ms | +| Per-stream revision-window read | `ReadTailRevisionWindow` (`TotalCommitsInStream=10000`, `RevisionWindowSize=1000`) | 17.207 ms | +| Write path (sync) | `WriteToStream` (`CommitsToWrite=10000`) | 10,489.8 ms | +| Write path (async) | `WriteToStreamAsync` (`CommitsToWrite=10000`) | 10,705.0 ms | +| Global read (async) | `ReadFromEventStoreAsync` (`CommitsToWrite=10000`) | 120.443 ms | +| Snapshot overhead slice | `WriteToStream` (`CommitsToWrite=1000`, `DisableSnapshotSupport=False`, `PersistStreamHeadsOnBackgroundThread=True`) | 2,631.6 ms | +| Recycle-bin read slice | `ReadDeletedCommitsFromRecycleBinBucket` (`CommitsPerStream=1000`, `DeletedStreams=5`, `ActiveStreams=1`) | 108.028 ms | +| Duplicate conflict slice | `DuplicateCommitIdPath` (`Iterations=100`) | 683.60 ms | + +## Explain Audit Workflow + +Use the explain audit script to validate the index usage of the persistence engine query shapes against a local MongoDB container: + +```powershell +.\scripts\explain-persistence-engine.ps1 +``` + +The script seeds a scratch database, recreates the same indexes defined by the engine, and runs `explain("executionStats")` for the main `Find` and equivalent delete/update filter shapes in `MongoPersistenceEngine`. + +Current findings from the issue [#73](https://github.com/NEventStore/NEventStore.Persistence.MongoDB/issues/73) follow-up: + +- The changed query shapes for stream reads, snapshot reads, snapshot deletes, and bucket-scoped stream-head reads all use the intended indexes. +- Bucket checkpoint scans and duplicate-commit lookups also use the expected indexes. +- The legacy date-based bucket reads and all-buckets checkpoint scans remain less ideal query shapes and should be reviewed separately if they become performance-sensitive. +- Decision: do not add a new `CommitStamp`-oriented compound index for the obsolete `GetFrom(bucketId, DateTime)` and `GetFromTo(bucketId, DateTime, DateTime)` APIs. They are sync-only compatibility methods on an upstream obsolete contract, they are already documented for removal, and the preferred checkpoint-based APIs are the supported optimization target. + +### After Snapshot Template + +After implementing optimizations, run the same net10 baseline profile and fill this table using the same method/parameter rows selected from the "before" reports. + +Current caution: the #73 full-stream after value is cleanly captured with the stabilized benchmark job, but it does not show a wall-clock win on this machine. Use the explain-plan audit as the primary #73 correctness evidence. The tail-window after value uses the revised persistence-level benchmark shape, so it is not comparable to the older before value. + +| Benchmark Slice | Before Mean | After Mean | Delta % | +|---|---:|---:|---:| +| Checkpoint generator write path (Always, 1000 commits) | 2,908.6 ms | | | +| Global read (bucket-qualified, 1000/3) | 22.142 ms | | | +| Global read (all buckets, 1000/3) | 67.011 ms | | | +| Per-stream full read (10000 commits) | 148.004 ms | 205.481 ms | +38.8% | +| Per-stream revision-window read (10000, window 1000) | 17.207 ms | 31.403 ms | Not comparable; benchmark shape changed | +| Write path (sync, 10000 commits) | 10,489.8 ms | | | +| Write path (async, 10000 commits) | 10,705.0 ms | | | +| Global read (async, 10000 commits) | 120.443 ms | | | +| Snapshot overhead slice (1000, snapshots on, bg on) | 2,631.6 ms | | | +| Recycle-bin read slice (1000, deleted=5, active=1) | 108.028 ms | | | +| Duplicate conflict slice (commit id, iterations=100) | 683.60 ms | | | + +Delta formula: + +```text +Delta % = ((After Mean - Before Mean) / Before Mean) * 100 +``` + +### Snapshot Retention Policy + +Store benchmark evidence at two levels: + +- Commit summary data to git: + - Keep the curated before/after comparison tables in this document. + - This is the canonical performance history for code review and PR discussion. +- Keep raw artifacts out of git: + - Do not commit files under `BenchmarkDotNet.Artifacts/results/`. + - Archive each run as a timestamped zip in local or CI artifact storage. + +Recommended archive naming: + +- `benchmark-baseline-net10-YYYYMMDD-HHMM.zip` +- `benchmark-after--net10-YYYYMMDD-HHMM.zip` + +Minimum workflow for each optimization pass: + +1. Run the baseline profile and archive raw artifacts. +2. Update the Before/After table in this document. +3. Run optimized code with the same profile and archive raw artifacts. +4. Fill After Mean and Delta % values in the same rows. + +Automation script: + +- `scripts/benchmark-snapshot.ps1` + +Example commands: + +```powershell +# Baseline snapshot (net10) +powershell -ExecutionPolicy Bypass -File .\scripts\benchmark-snapshot.ps1 -SnapshotType baseline -Framework net10.0 -Filter * + +# After snapshot for optimization "checkpoint-sort-fix" +powershell -ExecutionPolicy Bypass -File .\scripts\benchmark-snapshot.ps1 -SnapshotType after -OptimizationId checkpoint-sort-fix -Framework net10.0 -Filter * + +# Faster smoke snapshot (optional, less stable numbers) +powershell -ExecutionPolicy Bypass -File .\scripts\benchmark-snapshot.ps1 -SnapshotType baseline -Framework net10.0 -Filter * -ShortRun +``` diff --git a/scripts/benchmark-snapshot.ps1 b/scripts/benchmark-snapshot.ps1 new file mode 100644 index 0000000..9c99ffc --- /dev/null +++ b/scripts/benchmark-snapshot.ps1 @@ -0,0 +1,126 @@ +[CmdletBinding()] +param( + [ValidateSet('baseline', 'after')] + [string]$SnapshotType = 'baseline', + + [string]$OptimizationId, + + [string]$Framework = 'net10.0', + + [string]$Filter = '*', + + [switch]$ShortRun, + + [string]$ConnectionString = 'mongodb://localhost:50002/NEventStore', + + [switch]$SkipBuild +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +if ($SnapshotType -eq 'after' -and [string]::IsNullOrWhiteSpace($OptimizationId)) +{ + throw "Parameter -OptimizationId is required when -SnapshotType after is used." +} + +$repoRoot = (Resolve-Path (Join-Path $PSScriptRoot '..')).Path +$benchmarkProject = Join-Path $repoRoot 'src/NEventStore.Persistence.MongoDB.Benchmark/NEventStore.Persistence.MongoDB.Benchmark.csproj' +$benchmarkDll = Join-Path $repoRoot "src/NEventStore.Persistence.MongoDB.Benchmark/bin/Release/$Framework/NEventStore.Persistence.MongoDB.Benchmark.dll" +$resultsDir = Join-Path $repoRoot 'BenchmarkDotNet.Artifacts/results' +$archiveRoot = Join-Path $repoRoot 'artifacts/benchmark-snapshots' + +Write-Host "Repository root: $repoRoot" +Write-Host "Snapshot type: $SnapshotType" +Write-Host "Framework: $Framework" +Write-Host "Filter: $Filter" + +if (-not $SkipBuild) +{ + Write-Host "Building benchmark project for $Framework..." + dotnet build $benchmarkProject -c Release -f $Framework -v q + if ($LASTEXITCODE -ne 0) + { + throw "dotnet build failed with exit code $LASTEXITCODE" + } +} + +if (-not (Test-Path $benchmarkDll)) +{ + throw "Benchmark binary not found: $benchmarkDll" +} + +if (Test-Path $resultsDir) +{ + Remove-Item $resultsDir -Recurse -Force +} +New-Item -ItemType Directory -Path $resultsDir -Force | Out-Null +New-Item -ItemType Directory -Path $archiveRoot -Force | Out-Null + +[Environment]::SetEnvironmentVariable('NEventStore.MongoDB', $ConnectionString, 'Process') +Write-Host 'Running BenchmarkDotNet...' + +$dotnetArgs = @( + $benchmarkDll, + '--filter', $Filter +) + +if ($ShortRun) +{ + $dotnetArgs += @('--job', 'short') +} + +& dotnet @dotnetArgs +if ($LASTEXITCODE -ne 0) +{ + throw "Benchmark execution failed with exit code $LASTEXITCODE" +} + +$reportFiles = @(Get-ChildItem -Path $resultsDir -Filter '*-report-github.md' -File) +if (-not $reportFiles) +{ + throw "No benchmark report files found in $resultsDir" +} + +$timestamp = Get-Date -Format 'yyyyMMdd-HHmm' +$frameworkTag = $Framework -replace '[^A-Za-z0-9.-]', '-' + +if ($SnapshotType -eq 'baseline') +{ + $archiveName = "benchmark-baseline-$frameworkTag-$timestamp.zip" +} +else +{ + $safeOptimizationId = $OptimizationId -replace '[^A-Za-z0-9._-]', '-' + $archiveName = "benchmark-after-$safeOptimizationId-$frameworkTag-$timestamp.zip" +} + +$archivePath = Join-Path $archiveRoot $archiveName + +# Compress all report files; use array expansion for proper globbing in Compress-Archive +$filesToArchive = Get-ChildItem -Path $resultsDir -File | Select-Object -ExpandProperty FullName +Compress-Archive -Path $filesToArchive -DestinationPath $archivePath -Force + +$runManifest = [ordered]@{ + snapshotType = $SnapshotType + optimizationId = $OptimizationId + framework = $Framework + filter = $Filter + shortRun = [bool]$ShortRun + connectionString = $ConnectionString + createdAtUtc = (Get-Date).ToUniversalTime().ToString('o') + archive = $archivePath + reportCount = $reportFiles.Count + reportFiles = $reportFiles.Name +} + +$manifestPath = [System.IO.Path]::ChangeExtension($archivePath, '.json') +$runManifest | ConvertTo-Json -Depth 6 | Set-Content -Path $manifestPath -Encoding UTF8 + +Write-Host "Snapshot archive created: $archivePath" +Write-Host "Snapshot manifest created: $manifestPath" + +Write-Host '' +Write-Host 'Use these values in docs/Performance-Investigation.md:' +Write-Host '- Before Snapshot table (if baseline)' +Write-Host '- After Snapshot table (if after)' diff --git a/scripts/explain-persistence-engine.ps1 b/scripts/explain-persistence-engine.ps1 new file mode 100644 index 0000000..76d140c --- /dev/null +++ b/scripts/explain-persistence-engine.ps1 @@ -0,0 +1,227 @@ +param( + [string]$ContainerName = "nesci-mongo-1", + [string]$DatabaseName = "issue73_explain", + [switch]$KeepDatabase +) + +$dropDatabaseStatement = if ($KeepDatabase) { + "" +} +else { + "db.dropDatabase();" +} + +$script = @' +const dbName = "__DATABASE_NAME__"; +const db = db.getSiblingDB(dbName); +__DROP_DATABASE__ + +const commits = db.getCollection("Commits"); +const streamHeads = db.getCollection("Streams"); +const snapshots = db.getCollection("Snapshots"); + +commits.createIndex({ BucketId: 1, _id: 1 }, { name: "GetFrom_Checkpoint_Index", unique: true }); +commits.createIndex({ BucketId: 1, StreamId: 1, StreamRevisionFrom: 1, StreamRevisionTo: 1 }, { name: "GetFrom_Index", unique: true }); +commits.createIndex({ BucketId: 1, StreamId: 1, CommitSequence: 1 }, { name: "LogicalKey_Index", unique: true }); +commits.createIndex({ CommitStamp: 1 }, { name: "CommitStamp_Index" }); +commits.createIndex({ BucketId: 1, StreamId: 1, CommitId: 1 }, { name: "CommitId_Index", unique: true }); + +snapshots.createIndex({ "_id.BucketId": 1, "_id.StreamId": 1, "_id.StreamRevision": -1 }, { name: "BucketStreamRevision_Index" }); +streamHeads.createIndex({ Unsnapshotted: 1 }, { name: "Unsnapshotted_Index" }); +streamHeads.createIndex({ "_id.BucketId": 1, Unsnapshotted: -1 }, { name: "BucketUnsnapshotted_Index" }); + +for (let i = 0; i < 20; i++) { + const bucketId = i < 16 ? "default" : (i < 18 ? "other" : ":rb"); + const streamId = i < 8 ? "stream-1" : `stream-${i}`; + const baseRevision = (i * 2) + 1; + commits.insertOne({ + _id: i + 1, + BucketId: bucketId, + StreamId: streamId, + StreamRevisionFrom: baseRevision, + StreamRevisionTo: baseRevision + 1, + CommitSequence: i + 1, + CommitId: `commit-${i + 1}`, + CommitStamp: new Date(Date.now() + i * 1000), + Headers: {}, + Events: [] + }); +} + +streamHeads.insertMany([ + { _id: { BucketId: "default", StreamId: "stream-1" }, HeadRevision: 8, SnapshotRevision: 2, Unsnapshotted: 6 }, + { _id: { BucketId: "default", StreamId: "stream-2" }, HeadRevision: 4, SnapshotRevision: 0, Unsnapshotted: 4 }, + { _id: { BucketId: "other", StreamId: "stream-1" }, HeadRevision: 9, SnapshotRevision: 0, Unsnapshotted: 9 } +]); + +snapshots.insertMany([ + { _id: { BucketId: "default", StreamId: "stream-1", StreamRevision: 1 }, Payload: "s1-r1" }, + { _id: { BucketId: "default", StreamId: "stream-1", StreamRevision: 3 }, Payload: "s1-r3" }, + { _id: { BucketId: "default", StreamId: "stream-1", StreamRevision: 5 }, Payload: "s1-r5" }, + { _id: { BucketId: "default", StreamId: "stream-2", StreamRevision: 2 }, Payload: "s2-r2" }, + { _id: { BucketId: "other", StreamId: "stream-1", StreamRevision: 4 }, Payload: "other-r4" } +]); + +function summarizePlan(explain) { + const stages = []; + const indexNames = []; + + function walk(node) { + if (!node || typeof node !== "object") return; + if (node.stage) stages.push(node.stage); + if (node.indexName) indexNames.push(node.indexName); + for (const value of Object.values(node)) { + if (Array.isArray(value)) { + for (const item of value) walk(item); + } else if (value && typeof value === "object") { + walk(value); + } + } + } + + walk(explain.queryPlanner?.winningPlan); + return { + winningStages: [...new Set(stages)], + indexes: [...new Set(indexNames)], + totalKeysExamined: explain.executionStats?.totalKeysExamined, + totalDocsExamined: explain.executionStats?.totalDocsExamined, + nReturned: explain.executionStats?.nReturned + }; +} + +const results = { + commitRangeRead: summarizePlan( + commits.find({ + BucketId: "default", + StreamId: "stream-1", + StreamRevisionTo: { $gte: 3 }, + StreamRevisionFrom: { $lte: 6 } + }).sort({ StreamRevisionFrom: 1 }).explain("executionStats") + ), + bucketDateRead: summarizePlan( + commits.find({ + BucketId: "default", + CommitStamp: { $gte: new Date(Date.now() - 1000) } + }).sort({ _id: 1 }).explain("executionStats") + ), + bucketDateRangeRead: summarizePlan( + commits.find({ + BucketId: "default", + CommitStamp: { + $gte: new Date(Date.now() - 1000), + $lt: new Date(Date.now() + 60000) + } + }).sort({ _id: 1 }).explain("executionStats") + ), + bucketCheckpointRead: summarizePlan( + commits.find({ + BucketId: "default", + _id: { $gt: 3 } + }).sort({ _id: 1 }).explain("executionStats") + ), + bucketCheckpointRangeRead: summarizePlan( + commits.find({ + BucketId: "default", + _id: { $gt: 3, $lte: 12 } + }).sort({ _id: 1 }).explain("executionStats") + ), + allBucketsCheckpointRead: summarizePlan( + commits.find({ + BucketId: { $ne: ":rb" }, + _id: { $gt: 3 } + }).sort({ _id: 1 }).explain("executionStats") + ), + allBucketsCheckpointRangeRead: summarizePlan( + commits.find({ + BucketId: { $ne: ":rb" }, + _id: { $gt: 3, $lte: 12 } + }).sort({ _id: 1 }).explain("executionStats") + ), + duplicateCommitLookup: summarizePlan( + commits.find({ + BucketId: "default", + StreamId: "stream-1", + CommitId: "commit-1" + }).explain("executionStats") + ), + streamsToSnapshot: summarizePlan( + streamHeads.find({ + "_id.BucketId": "default", + Unsnapshotted: { $gte: 0 } + }).sort({ Unsnapshotted: -1 }).explain("executionStats") + ), + getSnapshot: summarizePlan( + snapshots.find({ + "_id.BucketId": "default", + "_id.StreamId": "stream-1", + "_id.StreamRevision": { $lte: 6 } + }).sort({ "_id.StreamRevision": -1 }).limit(1).explain("executionStats") + ), + addSnapshotById: summarizePlan( + snapshots.find({ + _id: { BucketId: "default", StreamId: "stream-1", StreamRevision: 3 } + }).limit(1).explain("executionStats") + ), + addSnapshotStreamHeadLookup: summarizePlan( + streamHeads.find({ + _id: { BucketId: "default", StreamId: "stream-1" } + }).limit(1).explain("executionStats") + ), + purgeBucketCommitsFilter: summarizePlan( + commits.find({ + BucketId: "default" + }).explain("executionStats") + ), + purgeBucketSnapshotsFilter: summarizePlan( + snapshots.find({ + "_id.BucketId": "default" + }).explain("executionStats") + ), + purgeBucketStreamHeadsFilter: summarizePlan( + streamHeads.find({ + "_id.BucketId": "default" + }).explain("executionStats") + ), + deleteStreamHeadById: summarizePlan( + streamHeads.find({ + _id: { BucketId: "default", StreamId: "stream-1" } + }).explain("executionStats") + ), + deleteStreamSnapshotsFilter: summarizePlan( + snapshots.find({ + "_id.BucketId": "default", + "_id.StreamId": "stream-1" + }).explain("executionStats") + ), + deleteStreamCommitsFilter: summarizePlan( + commits.find({ + BucketId: "default", + StreamId: "stream-1" + }).explain("executionStats") + ), + updateStreamHeadById: summarizePlan( + streamHeads.find({ + _id: { BucketId: "default", StreamId: "stream-1" } + }).explain("executionStats") + ), + lastCommittedCheckpoint: summarizePlan( + commits.find({}).sort({ _id: -1 }).limit(1).explain("executionStats") + ), + emptyRecycleBin: summarizePlan( + commits.find({ + BucketId: ":rb", + _id: { $lt: 20 } + }).explain("executionStats") + ), + getDeletedCommits: summarizePlan( + commits.find({ + BucketId: ":rb" + }).sort({ _id: 1 }).explain("executionStats") + ) +}; + +print(JSON.stringify(results, null, 2)); +'@ + +$script = $script.Replace('__DATABASE_NAME__', $DatabaseName).Replace('__DROP_DATABASE__', $dropDatabaseStatement) +$script | rtk docker exec -i $ContainerName mongosh --quiet \ No newline at end of file diff --git a/src/.nuget/NEventStore.Persistence.MongoDB.nuspec b/src/.nuget/NEventStore.Persistence.MongoDB.nuspec index 9ee7cbf..c144167 100644 --- a/src/.nuget/NEventStore.Persistence.MongoDB.nuspec +++ b/src/.nuget/NEventStore.Persistence.MongoDB.nuspec @@ -26,15 +26,27 @@ --> - + - + - + + + + + + + + + + + + + diff --git a/src/Directory.Build.props b/src/Directory.Build.props index dff21ee..8b80ff8 100644 --- a/src/Directory.Build.props +++ b/src/Directory.Build.props @@ -1,7 +1,12 @@ - 13.0 + 14.0 enable enable + + net8.0;net9.0;net10.0 \ No newline at end of file diff --git a/src/NEventStore.Persistence.MongoDB.Benchmark/Benchmarks/CheckpointGeneratorBenchmarks.cs b/src/NEventStore.Persistence.MongoDB.Benchmark/Benchmarks/CheckpointGeneratorBenchmarks.cs new file mode 100644 index 0000000..f4834f1 --- /dev/null +++ b/src/NEventStore.Persistence.MongoDB.Benchmark/Benchmarks/CheckpointGeneratorBenchmarks.cs @@ -0,0 +1,60 @@ +using BenchmarkDotNet.Attributes; +using MongoDB.Bson; +using NEventStore.Persistence.MongoDB; +using NEventStore.Persistence.MongoDB.Benchmark.Support; +using NEventStore.Persistence.MongoDB.Support; +using System; + +namespace NEventStore.Persistence.MongoDB.Benchmark.Benchmarks +{ + /// + /// Compares the per-commit checkpoint overhead of the two built-in checkpoint generators: + /// - "Always": AlwaysQueryDbForNextValueCheckpointGenerator — one extra DB read per commit (default). + /// - "InMemory": InMemoryCheckpointGenerator — in-memory increment; DB only on duplicate signal. + /// + [Config(typeof(AllowNonOptimized))] + [SimpleJob(launchCount: 3, warmupCount: 3, iterationCount: 3)] + [MemoryDiagnoser] + [MeanColumn, StdErrorColumn, StdDevColumn, MinColumn, MaxColumn, IterationsColumn] + public class CheckpointGeneratorBenchmarks + { + [Params(100, 1000)] + public int CommitsToWrite { get; set; } + + [Params("Always", "InMemory")] + public string GeneratorType { get; set; } = "Always"; + + private static readonly Guid StreamId = Guid.NewGuid(); + private IStoreEvents _eventStore = null!; + + [GlobalSetup] + public void Setup() + { + EventStoreHelpers.EnsureSerializersRegistered(); + + var options = new MongoPersistenceOptions(); + + if (GeneratorType == "InMemory") + { + var db = options.ConnectToDatabase(EventStoreHelpers.GetConnectionString()); + var collection = db.GetCollection("Commits"); + options.CheckpointGenerator = new InMemoryCheckpointGenerator(collection); + } + // "Always" is the engine default — leave CheckpointGenerator null. + + _eventStore = EventStoreHelpers.WireupEventStore(options); + _eventStore.Advanced.Purge(); + } + + [Benchmark] + public void WriteWithCheckpointGenerator() + { + using var stream = _eventStore.OpenStream(StreamId, 0, int.MaxValue); + for (int i = 0; i < CommitsToWrite; i++) + { + stream.Add(new EventMessage { Body = new SomeDomainEvent { Value = i.ToString() } }); + stream.CommitChanges(Guid.NewGuid()); + } + } + } +} diff --git a/src/NEventStore.Persistence.MongoDB.Benchmark/Benchmarks/DuplicateConflictBenchmarks.cs b/src/NEventStore.Persistence.MongoDB.Benchmark/Benchmarks/DuplicateConflictBenchmarks.cs new file mode 100644 index 0000000..c28d9e1 --- /dev/null +++ b/src/NEventStore.Persistence.MongoDB.Benchmark/Benchmarks/DuplicateConflictBenchmarks.cs @@ -0,0 +1,148 @@ +using BenchmarkDotNet.Attributes; +using MongoDB.Bson; +using MongoDB.Driver; +using NEventStore.Persistence.MongoDB; +using NEventStore.Persistence.MongoDB.Benchmark.Support; +using NEventStore.Persistence.MongoDB.Support; +using System; +using System.Collections.Generic; + +namespace NEventStore.Persistence.MongoDB.Benchmark.Benchmarks +{ + /// + /// Exercises duplicate commit-id and duplicate checkpoint retry paths without changing engine behavior. + /// + [Config(typeof(AllowNonOptimized))] + [SimpleJob(launchCount: 3, warmupCount: 3, iterationCount: 3, invocationCount: 1)] + [MemoryDiagnoser] + [MeanColumn, StdErrorColumn, StdDevColumn, MinColumn, MaxColumn, IterationsColumn] + public class DuplicateConflictBenchmarks + { + [Params(10, 100)] + public int Iterations { get; set; } + + [Benchmark] + public int DuplicateCommitIdPath() + { + var persistence = CreatePersistence(); + persistence.Purge(); + + var duplicateCount = 0; + for (int i = 0; i < Iterations; i++) + { + var streamId = Guid.NewGuid().ToString("N"); + var duplicateCommitId = Guid.NewGuid(); + + persistence.Commit(CreateAttempt(streamId, streamRevision: 1, commitSequence: 1, duplicateCommitId, i)); + + try + { + persistence.Commit(CreateAttempt(streamId, streamRevision: 2, commitSequence: 2, duplicateCommitId, i + 1)); + } + catch (DuplicateCommitException) + { + duplicateCount++; + } + } + + return duplicateCount; + } + + [Benchmark] + public int DuplicateCheckpointRetryPath() + { + var options = new MongoPersistenceOptions(); + var db = options.ConnectToDatabase(EventStoreHelpers.GetConnectionString()); + var collection = db.GetCollection("Commits"); + options.CheckpointGenerator = new OneDuplicateCheckpointGenerator(collection); + + var persistence = CreatePersistence(options); + persistence.Purge(); + + var successfulCommits = 0; + for (int i = 0; i < Iterations; i++) + { + // Seed checkpoint 1 so the next commit first attempt conflicts and forces SignalDuplicateId/Next retry. + var seedStreamId = $"seed-{i}"; + persistence.Commit(CreateAttempt(seedStreamId, streamRevision: 1, commitSequence: 1, Guid.NewGuid(), i)); + + var writeStreamId = $"retry-{i}"; + var commit = persistence.Commit(CreateAttempt(writeStreamId, streamRevision: 1, commitSequence: 1, Guid.NewGuid(), i + 1)); + if (commit != null) + { + successfulCommits++; + } + } + + return successfulCommits; + } + + private static IPersistStreams CreatePersistence(MongoPersistenceOptions? options = null) + { + var eventStore = EventStoreHelpers.WireupEventStore(options); + return (IPersistStreams)eventStore.Advanced; + } + + private static CommitAttempt CreateAttempt(string streamId, int streamRevision, int commitSequence, Guid commitId, int value) + { + return new CommitAttempt( + bucketId: Bucket.Default, + streamId: streamId, + streamRevision: streamRevision, + commitId: commitId, + commitSequence: commitSequence, + commitStamp: DateTime.UtcNow, + headers: null, + events: new List + { + new EventMessage { Body = new SomeDomainEvent { Value = value.ToString() } } + } + ); + } + + private sealed class OneDuplicateCheckpointGenerator : ICheckpointGenerator + { + private readonly InMemoryCheckpointGenerator _inner; + private bool _shouldDuplicateOnNext = true; + + public OneDuplicateCheckpointGenerator(IMongoCollection collection) + { + _inner = new InMemoryCheckpointGenerator(collection); + } + + public long Next() + { + if (_shouldDuplicateOnNext) + { + _shouldDuplicateOnNext = false; + return 1; + } + + return _inner.Next(); + } + + public async System.Threading.Tasks.Task NextAsync(System.Threading.CancellationToken cancellationToken = default) + { + if (_shouldDuplicateOnNext) + { + _shouldDuplicateOnNext = false; + return 1; + } + + return await _inner.NextAsync(cancellationToken).ConfigureAwait(false); + } + + public void SignalDuplicateId(long id) + { + _inner.SignalDuplicateId(id); + _shouldDuplicateOnNext = false; + } + + public async System.Threading.Tasks.Task SignalDuplicateIdAsync(long id, System.Threading.CancellationToken cancellationToken = default) + { + await _inner.SignalDuplicateIdAsync(id, cancellationToken).ConfigureAwait(false); + _shouldDuplicateOnNext = false; + } + } + } +} diff --git a/src/NEventStore.Persistence.MongoDB.Benchmark/Benchmarks/GlobalCheckpointReadBenchmarks.cs b/src/NEventStore.Persistence.MongoDB.Benchmark/Benchmarks/GlobalCheckpointReadBenchmarks.cs new file mode 100644 index 0000000..cb551f7 --- /dev/null +++ b/src/NEventStore.Persistence.MongoDB.Benchmark/Benchmarks/GlobalCheckpointReadBenchmarks.cs @@ -0,0 +1,79 @@ +using BenchmarkDotNet.Attributes; +using NEventStore.Persistence.MongoDB.Benchmark.Support; +using System; +using System.Collections.Generic; +using System.Linq; + +namespace NEventStore.Persistence.MongoDB.Benchmark.Benchmarks +{ + /// + /// Compares global checkpoint scans across all buckets vs a specific bucket. + /// + [Config(typeof(AllowNonOptimized))] + [SimpleJob(launchCount: 3, warmupCount: 3, iterationCount: 3, invocationCount: 1)] + [MemoryDiagnoser] + [MeanColumn, StdErrorColumn, StdDevColumn, MinColumn, MaxColumn, IterationsColumn] + public class GlobalCheckpointReadBenchmarks + { + [Params(100, 1000)] + public int CommitsPerBucket { get; set; } + + [Params(1, 3)] + public int ExtraBuckets { get; set; } + + private readonly IStoreEvents _eventStore; + private readonly IPersistStreams _persistence; + + public GlobalCheckpointReadBenchmarks() + { + _eventStore = EventStoreHelpers.WireupEventStore(); + _persistence = (IPersistStreams)_eventStore.Advanced; + } + + [GlobalSetup] + public void Setup() + { + _persistence.Purge(); + SeedBucket(Bucket.Default, CommitsPerBucket); + + for (int b = 0; b < ExtraBuckets; b++) + { + SeedBucket($"bench-bucket-{b}", CommitsPerBucket); + } + } + + [Benchmark] + public int ReadFromBucketCheckpoint() + { + return _persistence.GetFrom(Bucket.Default, 0).Count(); + } + + [Benchmark] + public int ReadFromAllBucketsCheckpoint() + { + return _persistence.GetFrom(0).Count(); + } + + private void SeedBucket(string bucketId, int commits) + { + var streamId = Guid.NewGuid().ToString("N"); + for (int i = 1; i <= commits; i++) + { + var attempt = new CommitAttempt( + bucketId: bucketId, + streamId: streamId, + streamRevision: i, + commitId: Guid.NewGuid(), + commitSequence: i, + commitStamp: DateTime.UtcNow, + headers: null, + events: new List + { + new EventMessage { Body = new SomeDomainEvent { Value = i.ToString() } } + } + ); + _persistence.Commit(attempt); + } + } + } +} diff --git a/src/NEventStore.Persistence.MongoDB.Benchmark/Benchmarks/ReadFromEventStoreAsyncBenchmarks.cs b/src/NEventStore.Persistence.MongoDB.Benchmark/Benchmarks/ReadFromEventStoreAsyncBenchmarks.cs new file mode 100644 index 0000000..cd25c97 --- /dev/null +++ b/src/NEventStore.Persistence.MongoDB.Benchmark/Benchmarks/ReadFromEventStoreAsyncBenchmarks.cs @@ -0,0 +1,70 @@ +using BenchmarkDotNet.Attributes; +using NEventStore.Persistence.MongoDB.Benchmark.Support; +using System; +using System.Runtime.ExceptionServices; +using System.Threading; +using System.Threading.Tasks; + +namespace NEventStore.Persistence.MongoDB.Benchmark.Benchmarks +{ + /// + /// Benchmarks the async global checkpoint scan via . + /// Mirrors against the async engine path. + /// + [Config(typeof(AllowNonOptimized))] + [SimpleJob(launchCount: 3, warmupCount: 3, iterationCount: 3, invocationCount: 1)] + [MemoryDiagnoser] + [MeanColumn, StdErrorColumn, StdDevColumn, MinColumn, MaxColumn, IterationsColumn] + public class ReadFromEventStoreAsyncBenchmarks + { + //[Params(100, 1000, 10000, 100000)] + [Params(100, 1000, 10000)] + public int CommitsToWrite { get; set; } + + private static readonly Guid StreamId = Guid.NewGuid(); + private readonly IStoreEvents _eventStore; + private readonly IPersistStreams _persistence; + private readonly DrainObserver _observer = new DrainObserver(); + + public ReadFromEventStoreAsyncBenchmarks() + { + _eventStore = EventStoreHelpers.WireupEventStore(); + _persistence = (IPersistStreams)_eventStore.Advanced; + } + + [GlobalSetup] + public void ReadSetup() + { + _eventStore.Advanced.Purge(); + + using var stream = _eventStore.CreateStream(StreamId); + for (int i = 0; i < CommitsToWrite; i++) + { + stream.Add(new EventMessage { Body = new SomeDomainEvent { Value = i.ToString() } }); + stream.CommitChanges(Guid.NewGuid()); + } + } + + [Benchmark] + public Task ReadFromEventStoreAsync() + { + return _persistence.GetFromAsync(0L, _observer, CancellationToken.None); + } + + /// Drains every commit without allocating a collection. + private sealed class DrainObserver : IAsyncObserver + { + public Task OnNextAsync(ICommit value, CancellationToken cancellationToken) + => Task.FromResult(true); + + public Task OnErrorAsync(Exception ex, CancellationToken cancellationToken) + { + ExceptionDispatchInfo.Capture(ex).Throw(); + return Task.CompletedTask; + } + + public Task OnCompletedAsync(CancellationToken cancellationToken) + => Task.CompletedTask; + } + } +} diff --git a/src/NEventStore.Persistence.MongoDB.Benchmark/Benchmarks/ReadFromStreamBenchmarks.cs b/src/NEventStore.Persistence.MongoDB.Benchmark/Benchmarks/ReadFromStreamBenchmarks.cs index 3dc502a..4cd9baa 100644 --- a/src/NEventStore.Persistence.MongoDB.Benchmark/Benchmarks/ReadFromStreamBenchmarks.cs +++ b/src/NEventStore.Persistence.MongoDB.Benchmark/Benchmarks/ReadFromStreamBenchmarks.cs @@ -5,7 +5,7 @@ namespace NEventStore.Persistence.MongoDB.Benchmark.Benchmarks { [Config(typeof(AllowNonOptimized))] - [SimpleJob(launchCount: 3, warmupCount: 3, iterationCount: 3, invocationCount: 1)] + [SimpleJob(launchCount: 3, warmupCount: 3, iterationCount: 3)] [MemoryDiagnoser] [MeanColumn, StdErrorColumn, StdDevColumn, MinColumn, MaxColumn, IterationsColumn] public class ReadFromStreamBenchmarks diff --git a/src/NEventStore.Persistence.MongoDB.Benchmark/Benchmarks/RecycleBinReadBenchmarks.cs b/src/NEventStore.Persistence.MongoDB.Benchmark/Benchmarks/RecycleBinReadBenchmarks.cs new file mode 100644 index 0000000..afc7a5b --- /dev/null +++ b/src/NEventStore.Persistence.MongoDB.Benchmark/Benchmarks/RecycleBinReadBenchmarks.cs @@ -0,0 +1,81 @@ +using BenchmarkDotNet.Attributes; +using NEventStore.Persistence.MongoDB.Benchmark.Support; +using System; +using System.Linq; + +namespace NEventStore.Persistence.MongoDB.Benchmark.Benchmarks +{ + /// + /// Measures read behavior for active commits vs commits marked as deleted (recycle-bin bucket). + /// + [Config(typeof(AllowNonOptimized))] + [SimpleJob(launchCount: 3, warmupCount: 3, iterationCount: 3, invocationCount: 1)] + [MemoryDiagnoser] + [MeanColumn, StdErrorColumn, StdDevColumn, MinColumn, MaxColumn, IterationsColumn] + public class RecycleBinReadBenchmarks + { + [Params(100, 1000)] + public int CommitsPerStream { get; set; } + + [Params(1, 5)] + public int DeletedStreams { get; set; } + + [Params(1, 5)] + public int ActiveStreams { get; set; } + + private readonly IStoreEvents _eventStore; + private readonly IPersistStreams _persistence; + + public RecycleBinReadBenchmarks() + { + _eventStore = EventStoreHelpers.WireupEventStore(); + _persistence = (IPersistStreams)_eventStore.Advanced; + } + + [GlobalSetup] + public void Setup() + { + _persistence.Purge(); + + for (int i = 0; i < ActiveStreams; i++) + { + SeedStreamAndOptionallyDelete(deleteAfterSeed: false); + } + + for (int i = 0; i < DeletedStreams; i++) + { + SeedStreamAndOptionallyDelete(deleteAfterSeed: true); + } + } + + [Benchmark] + public int ReadActiveCommitsFromAllBuckets() + { + return _persistence.GetFrom(0).Count(); + } + + [Benchmark] + public int ReadDeletedCommitsFromRecycleBinBucket() + { + return _persistence.GetFrom(MongoSystemBuckets.RecycleBin, 0).Count(); + } + + private void SeedStreamAndOptionallyDelete(bool deleteAfterSeed) + { + var streamId = Guid.NewGuid(); + using (var stream = _eventStore.CreateStream(streamId)) + { + for (int i = 0; i < CommitsPerStream; i++) + { + stream.Add(new EventMessage { Body = new SomeDomainEvent { Value = i.ToString() } }); + stream.CommitChanges(Guid.NewGuid()); + } + } + + if (deleteAfterSeed) + { + _persistence.DeleteStream(Bucket.Default, streamId.ToString()); + } + } + } +} diff --git a/src/NEventStore.Persistence.MongoDB.Benchmark/Benchmarks/SnapshotAssistedRebuildBenchmarks.cs b/src/NEventStore.Persistence.MongoDB.Benchmark/Benchmarks/SnapshotAssistedRebuildBenchmarks.cs new file mode 100644 index 0000000..4e6752d --- /dev/null +++ b/src/NEventStore.Persistence.MongoDB.Benchmark/Benchmarks/SnapshotAssistedRebuildBenchmarks.cs @@ -0,0 +1,66 @@ +using BenchmarkDotNet.Attributes; +using NEventStore.Persistence.MongoDB; +using NEventStore.Persistence.MongoDB.Benchmark.Support; +using System; + +namespace NEventStore.Persistence.MongoDB.Benchmark.Benchmarks +{ + /// + /// Compares full aggregate rebuilds with snapshot-assisted rebuilds over the tail of a long stream. + /// + [Config(typeof(AllowNonOptimized))] + [SimpleJob(launchCount: 3, warmupCount: 3, iterationCount: 3)] + [MemoryDiagnoser] + [MeanColumn, StdErrorColumn, StdDevColumn, MinColumn, MaxColumn, IterationsColumn] + public class SnapshotAssistedRebuildBenchmarks + { + [Params(1000, 10000)] + public int TotalCommitsInStream { get; set; } + + [Params(10, 100, 1000)] + public int CommitsAfterSnapshot { get; set; } + + private readonly Guid _streamId = Guid.NewGuid(); + private IStoreEvents _eventStore = null!; + private ISnapshot _snapshot = null!; + + [GlobalSetup] + public void Setup() + { + var options = new MongoPersistenceOptions + { + PersistStreamHeadsOnBackgroundThread = false + }; + + _eventStore = EventStoreHelpers.WireupEventStore(options); + _eventStore.Advanced.Purge(); + + using (var stream = _eventStore.CreateStream(_streamId)) + { + for (int i = 1; i <= TotalCommitsInStream; i++) + { + stream.Add(new EventMessage { Body = new SomeDomainEvent { Value = i.ToString() } }); + stream.CommitChanges(Guid.NewGuid()); + } + } + + var snapshotRevision = Math.Max(1, TotalCommitsInStream - CommitsAfterSnapshot); + _snapshot = new Snapshot(_streamId.ToString(), snapshotRevision, new SomeDomainEvent { Value = snapshotRevision.ToString() }); + _eventStore.Advanced.AddSnapshot(_snapshot); + } + + [Benchmark(Baseline = true)] + public int FullRebuild() + { + using var stream = _eventStore.OpenStream(_streamId, 0, int.MaxValue); + return stream.CommittedEvents.Count; + } + + [Benchmark] + public int SnapshotAssistedRebuild() + { + using var stream = _eventStore.OpenStream(_snapshot, int.MaxValue); + return stream.CommittedEvents.Count; + } + } +} diff --git a/src/NEventStore.Persistence.MongoDB.Benchmark/Benchmarks/SnapshotOverheadBenchmarks.cs b/src/NEventStore.Persistence.MongoDB.Benchmark/Benchmarks/SnapshotOverheadBenchmarks.cs new file mode 100644 index 0000000..6809c68 --- /dev/null +++ b/src/NEventStore.Persistence.MongoDB.Benchmark/Benchmarks/SnapshotOverheadBenchmarks.cs @@ -0,0 +1,65 @@ +using BenchmarkDotNet.Attributes; +using NEventStore.Persistence.MongoDB; +using NEventStore.Persistence.MongoDB.Benchmark.Support; +using System; + +namespace NEventStore.Persistence.MongoDB.Benchmark.Benchmarks +{ + /// + /// Measures the overhead of stream-head maintenance on the write path. + /// + /// + /// When is false (default), the engine writes a + /// stream-head document on every commit. When true, that write is skipped entirely. + /// + /// + /// When is true (default), the + /// stream-head write happens on a background task so commit latency is lower but resource + /// usage is higher. When false it happens inline (useful for testing; higher latency). + /// + /// + [Config(typeof(AllowNonOptimized))] + [SimpleJob(launchCount: 3, warmupCount: 3, iterationCount: 3)] + [MemoryDiagnoser] + [MeanColumn, StdErrorColumn, StdDevColumn, MinColumn, MaxColumn, IterationsColumn] + public class SnapshotOverheadBenchmarks + { + [Params(100, 1000)] + public int CommitsToWrite { get; set; } + + [Params(true, false)] + public bool DisableSnapshotSupport { get; set; } + + [Params(true, false)] + public bool PersistStreamHeadsOnBackgroundThread { get; set; } + + private static readonly Guid StreamId = Guid.NewGuid(); + private IStoreEvents _eventStore = null!; + + [GlobalSetup] + public void Setup() + { + EventStoreHelpers.EnsureSerializersRegistered(); + + var options = new MongoPersistenceOptions + { + DisableSnapshotSupport = DisableSnapshotSupport, + PersistStreamHeadsOnBackgroundThread = PersistStreamHeadsOnBackgroundThread + }; + + _eventStore = EventStoreHelpers.WireupEventStore(options); + _eventStore.Advanced.Purge(); + } + + [Benchmark] + public void WriteToStream() + { + using var stream = _eventStore.OpenStream(StreamId, 0, int.MaxValue); + for (int i = 0; i < CommitsToWrite; i++) + { + stream.Add(new EventMessage { Body = new SomeDomainEvent { Value = i.ToString() } }); + stream.CommitChanges(Guid.NewGuid()); + } + } + } +} diff --git a/src/NEventStore.Persistence.MongoDB.Benchmark/Benchmarks/StreamRevisionWindowBenchmarks.cs b/src/NEventStore.Persistence.MongoDB.Benchmark/Benchmarks/StreamRevisionWindowBenchmarks.cs new file mode 100644 index 0000000..2dc8e86 --- /dev/null +++ b/src/NEventStore.Persistence.MongoDB.Benchmark/Benchmarks/StreamRevisionWindowBenchmarks.cs @@ -0,0 +1,63 @@ +using BenchmarkDotNet.Attributes; +using NEventStore.Persistence.MongoDB.Benchmark.Support; +using System; +using System.Linq; + +namespace NEventStore.Persistence.MongoDB.Benchmark.Benchmarks +{ + /// + /// Measures per-stream reads over focused revision windows. + /// + [Config(typeof(AllowNonOptimized))] + [SimpleJob(launchCount: 3, warmupCount: 3, iterationCount: 3)] + [MemoryDiagnoser] + [MeanColumn, StdErrorColumn, StdDevColumn, MinColumn, MaxColumn, IterationsColumn] + public class StreamRevisionWindowBenchmarks + { + [Params(1000, 10000)] + public int TotalCommitsInStream { get; set; } + + [Params(10, 100, 1000)] + public int RevisionWindowSize { get; set; } + + private readonly string _streamId = Guid.NewGuid().ToString(); + private readonly IPersistStreams _persistence; + + public StreamRevisionWindowBenchmarks() + { + _persistence = (IPersistStreams)EventStoreHelpers.WireupEventStore().Advanced; + } + + [GlobalSetup] + public void Setup() + { + _persistence.Purge(); + + for (int i = 1; i <= TotalCommitsInStream; i++) + { + var attempt = new CommitAttempt( + bucketId: Bucket.Default, + streamId: _streamId, + streamRevision: i, + commitId: Guid.NewGuid(), + commitSequence: i, + commitStamp: DateTime.UtcNow, + headers: null, + events: + [ + new EventMessage { Body = new SomeDomainEvent { Value = i.ToString() } } + ]); + + _persistence.Commit(attempt); + } + } + + [Benchmark] + public int ReadTailRevisionWindow() + { + var minRevision = Math.Max(1, TotalCommitsInStream - RevisionWindowSize + 1); + var maxRevision = TotalCommitsInStream; + return _persistence.GetFrom(Bucket.Default, _streamId, minRevision, maxRevision).Count(); + } + } +} diff --git a/src/NEventStore.Persistence.MongoDB.Benchmark/Benchmarks/WriteToStreamAsyncBenchmarks.cs b/src/NEventStore.Persistence.MongoDB.Benchmark/Benchmarks/WriteToStreamAsyncBenchmarks.cs new file mode 100644 index 0000000..b239c01 --- /dev/null +++ b/src/NEventStore.Persistence.MongoDB.Benchmark/Benchmarks/WriteToStreamAsyncBenchmarks.cs @@ -0,0 +1,63 @@ +using BenchmarkDotNet.Attributes; +using NEventStore.Persistence.MongoDB.Benchmark.Support; +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + +namespace NEventStore.Persistence.MongoDB.Benchmark.Benchmarks +{ + /// + /// Benchmarks the async commit path via . + /// Mirrors against the async engine path. + /// + [Config(typeof(AllowNonOptimized))] + [SimpleJob(launchCount: 3, warmupCount: 3, iterationCount: 3, invocationCount: 1)] + [MemoryDiagnoser] + [MeanColumn, StdErrorColumn, StdDevColumn, MinColumn, MaxColumn, IterationsColumn] + public class WriteToStreamAsyncBenchmarks + { + //[Params(100, 1000, 10000, 100000)] + [Params(100, 1000, 10000)] + public int CommitsToWrite { get; set; } + + private static readonly string StreamId = Guid.NewGuid().ToString(); + private IPersistStreams _persistence = null!; + private int _streamRevision; + private int _commitSequence; + + public WriteToStreamAsyncBenchmarks() + { + var store = EventStoreHelpers.WireupEventStore(); + _persistence = (IPersistStreams)store.Advanced; + } + + [GlobalSetup] + public void Setup() + { + _persistence.Purge(); + _streamRevision = 0; + _commitSequence = 0; + } + + [Benchmark] + public async Task WriteToStreamAsync() + { + for (int i = 0; i < CommitsToWrite; i++) + { + _streamRevision++; + _commitSequence++; + var attempt = new CommitAttempt( + streamId: StreamId, + streamRevision: _streamRevision, + commitId: Guid.NewGuid(), + commitSequence: _commitSequence, + commitStamp: DateTime.UtcNow, + headers: null, + events: new List { new EventMessage { Body = new SomeDomainEvent { Value = i.ToString() } } } + ); + await _persistence.CommitAsync(attempt, CancellationToken.None).ConfigureAwait(false); + } + } + } +} diff --git a/src/NEventStore.Persistence.MongoDB.Benchmark/NEventStore.Persistence.MongoDB.Benchmark.csproj b/src/NEventStore.Persistence.MongoDB.Benchmark/NEventStore.Persistence.MongoDB.Benchmark.csproj index c4576d6..426e390 100644 --- a/src/NEventStore.Persistence.MongoDB.Benchmark/NEventStore.Persistence.MongoDB.Benchmark.csproj +++ b/src/NEventStore.Persistence.MongoDB.Benchmark/NEventStore.Persistence.MongoDB.Benchmark.csproj @@ -1,25 +1,27 @@ - - - - net8.0;net472 - false - - exe - - Exe - - - - TRACE;DEBUG - - - - - - - - - - - - + + + + net472;$(ModernTargetFrameworks) + false + + exe + + Exe + true + true + + + + TRACE;DEBUG + + + + + + + + + + + + diff --git a/src/NEventStore.Persistence.MongoDB.Benchmark/Program.cs b/src/NEventStore.Persistence.MongoDB.Benchmark/Program.cs index 2661a25..fdc9c70 100644 --- a/src/NEventStore.Persistence.MongoDB.Benchmark/Program.cs +++ b/src/NEventStore.Persistence.MongoDB.Benchmark/Program.cs @@ -1,19 +1,13 @@ -using BenchmarkDotNet.Running; -using NEventStore.Persistence.MongoDB.Benchmark.Benchmarks; -using System; - -namespace NEventStore.Persistence.MongoDB.Benchmark -{ - public static class Program - { - public static void Main(string[] args) - { - //BenchmarkRunner.Run(); - //BenchmarkRunner.Run(); - BenchmarkRunner.Run(); - - //var p = new ReadFromEventStoreBenchmarks(); - //p.ProfileWithVisualStudio(1000); - } - } -} +using BenchmarkDotNet.Running; +using System.Reflection; + +namespace NEventStore.Persistence.MongoDB.Benchmark +{ + public static class Program + { + public static void Main(string[] args) + { + BenchmarkSwitcher.FromAssembly(Assembly.GetExecutingAssembly()).Run(args); + } + } +} diff --git a/src/NEventStore.Persistence.MongoDB.Benchmark/Support/EventStoreHelpers.cs b/src/NEventStore.Persistence.MongoDB.Benchmark/Support/EventStoreHelpers.cs index ee5d274..d398831 100644 --- a/src/NEventStore.Persistence.MongoDB.Benchmark/Support/EventStoreHelpers.cs +++ b/src/NEventStore.Persistence.MongoDB.Benchmark/Support/EventStoreHelpers.cs @@ -1,41 +1,67 @@ -using NEventStore.Serialization; -using System; - -namespace NEventStore.Persistence.MongoDB.Benchmark.Support -{ - internal static class EventStoreHelpers - { - private const string EnvVarConnectionStringKey = "NEventStore.MongoDB"; - - internal static string GetConnectionString() - { - string connectionString = Environment.GetEnvironmentVariable(EnvVarConnectionStringKey, EnvironmentVariableTarget.Process); - - if (connectionString == null) - { - string message = string.Format( - "Cannot initialize acceptance tests for Mongo. Cannot find the '{0}' environment variable. Please ensure " + - "you have correctly setup the connection string environment variables. Refer to the " + - "NEventStore wiki for details.", - EnvVarConnectionStringKey); - throw new InvalidOperationException(message); - } - - return connectionString.TrimStart('"').TrimEnd('"'); - } - - internal static IStoreEvents WireupEventStore() - { - return Wireup.Init() - // .LogToOutputWindow(LogLevel.Verbose) - // .LogToConsoleWindow(LogLevel.Verbose) - .UsingMongoPersistence(GetConnectionString(), new DocumentObjectSerializer()) - .InitializeStorageEngine() -#if NET472_OR_GREATER - .TrackPerformanceInstance("example") -#endif - // .HookIntoPipelineUsing(new[] { new AuthorizationPipelineHook() }) - .Build(); - } - } -} +using MongoDB.Bson; +using MongoDB.Bson.Serialization; +using MongoDB.Bson.Serialization.Serializers; +using NEventStore.Persistence.MongoDB; +using NEventStore.Serialization; +using System; +using System.Threading; + +namespace NEventStore.Persistence.MongoDB.Benchmark.Support +{ + internal static class EventStoreHelpers + { + private const string EnvVarConnectionStringKey = "NEventStore.MongoDB"; + + private static int _serializersRegistered; + + internal static string GetConnectionString() + { + string connectionString = Environment.GetEnvironmentVariable(EnvVarConnectionStringKey, EnvironmentVariableTarget.Process); + + if (connectionString == null) + { + string message = string.Format( + "Cannot initialize acceptance tests for Mongo. Cannot find the '{0}' environment variable. Please ensure " + + "you have correctly setup the connection string environment variables. Refer to the " + + "NEventStore wiki for details.", + EnvVarConnectionStringKey); + throw new InvalidOperationException(message); + } + + return connectionString.TrimStart('"').TrimEnd('"'); + } + + /// + /// Registers MongoDB BSON serializers required for CSharpLegacy GUID compatibility. + /// Safe to call multiple times within the same process; registration happens only once. + /// + internal static void EnsureSerializersRegistered() + { + if (Interlocked.Exchange(ref _serializersRegistered, 1) == 0) + { + // MongoDb 3.0.0 GUID serialization changed; register CSharpLegacy for backward compatibility. + BsonSerializer.RegisterSerializer(new GuidSerializer(GuidRepresentation.CSharpLegacy)); + // when serializing guid in a Dictionary take a look at the comment here: + // https://jira.mongodb.org/browse/CSHARP-4987?jql=text%20~%20%22GuidRepresentation%20dictionary%22 + BsonSerializer.RegisterSerializer(new ObjectSerializer( + BsonSerializer.LookupDiscriminatorConvention(typeof(object)), GuidRepresentation.CSharpLegacy, ObjectSerializer.AllAllowedTypes)); + } + } + + internal static IStoreEvents WireupEventStore(MongoPersistenceOptions? options = null) + { + EnsureSerializersRegistered(); + + return Wireup.Init() + // .LogToOutputWindow(LogLevel.Verbose) + // .LogToConsoleWindow(LogLevel.Verbose) + .UsingMongoPersistence(() => GetConnectionString(), new DocumentObjectSerializer(), options) + .InitializeStorageEngine() +#if NET472_OR_GREATER + .TrackPerformanceInstance("example") +#endif + // .HookIntoPipelineUsing(new[] { new AuthorizationPipelineHook() }) + .Build(); + } + } +} diff --git a/src/NEventStore.Persistence.MongoDB.Core.sln b/src/NEventStore.Persistence.MongoDB.Core.sln index d381082..aa26eb4 100644 --- a/src/NEventStore.Persistence.MongoDB.Core.sln +++ b/src/NEventStore.Persistence.MongoDB.Core.sln @@ -1,7 +1,7 @@  Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Version 17 -VisualStudioVersion = 17.7.33920.267 +# Visual Studio Version 18 +VisualStudioVersion = 18.7.11811.120 insiders MinimumVisualStudioVersion = 10.0.40219.1 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "NEventStore.Persistence.MongoDB.Core", "NEventStore.Persistence.MongoDB\NEventStore.Persistence.MongoDB.Core.csproj", "{0A3E5F93-6BC6-48F4-9426-3A1063649375}" EndProject @@ -16,6 +16,7 @@ EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{6B3F0D5A-6C8E-44A5-A118-C58B0EB834A0}" ProjectSection(SolutionItems) = preProject .editorconfig = .editorconfig + ..\.gitignore = ..\.gitignore ..\appveyor.yml = ..\appveyor.yml ..\build.ps1 = ..\build.ps1 ..\Changelog.md = ..\Changelog.md diff --git a/src/NEventStore.Persistence.MongoDB.Tests/AcceptanceTests/Issues/Issue73ExplainPlans.cs b/src/NEventStore.Persistence.MongoDB.Tests/AcceptanceTests/Issues/Issue73ExplainPlans.cs new file mode 100644 index 0000000..f3624fe --- /dev/null +++ b/src/NEventStore.Persistence.MongoDB.Tests/AcceptanceTests/Issues/Issue73ExplainPlans.cs @@ -0,0 +1,346 @@ +using FluentAssertions; +using global::MongoDB.Bson; +using global::MongoDB.Driver; +using NEventStore.Persistence.AcceptanceTests.BDD; +using NEventStore.Serialization; + +#if MSTEST +using Microsoft.VisualStudio.TestTools.UnitTesting; +#endif + +namespace NEventStore.Persistence.MongoDB.Tests.AcceptanceTests.Issues +{ + internal sealed class ExplainPlanResult + { + public HashSet Stages { get; } = new(StringComparer.Ordinal); + + public HashSet Indexes { get; } = new(StringComparer.Ordinal); + + public int TotalKeysExamined { get; set; } + + public int TotalDocsExamined { get; set; } + } + +#if MSTEST + [TestClass] +#endif + public class Issue_73_explain_should_use_expected_indexes : SpecificationBase + { + private MongoPersistenceEngine? _engine; + private IMongoDatabase? _database; + private IMongoCollection? _commits; + private IMongoCollection? _streamHeads; + private IMongoCollection? _snapshots; + private Dictionary? _plans; + + protected override void Context() + { + var options = new MongoPersistenceOptions + { + PersistStreamHeadsOnBackgroundThread = false, + }; + + var builder = new MongoUrlBuilder(AcceptanceTestMongoPersistenceFactory.GetConnectionString()) + { + DatabaseName = $"issue73-explain-{Guid.NewGuid():N}" + }; + + _database = options.ConnectToDatabase(builder.ToString()); + _engine = new MongoPersistenceEngine(_database, new DocumentObjectSerializer(), options); + _engine.Initialize(); + + _commits = _database.GetCollection("Commits"); + _streamHeads = _database.GetCollection("Streams"); + _snapshots = _database.GetCollection("Snapshots"); + + SeedDocuments(); + } + + protected override void Because() + { + _plans = new Dictionary(StringComparer.Ordinal) + { + ["commitRangeRead"] = ExplainFind( + _commits!, + new BsonDocument + { + [MongoCommitFields.BucketId] = "default", + [MongoCommitFields.StreamId] = "stream-1", + [MongoCommitFields.StreamRevisionTo] = new BsonDocument("$gte", 3), + [MongoCommitFields.StreamRevisionFrom] = new BsonDocument("$lte", 6) + }, + new BsonDocument(MongoCommitFields.StreamRevisionFrom, 1)), + ["bucketCheckpointRead"] = ExplainFind( + _commits!, + new BsonDocument + { + [MongoCommitFields.BucketId] = "default", + [MongoCommitFields.CheckpointNumber] = new BsonDocument("$gt", 3L) + }, + new BsonDocument(MongoCommitFields.CheckpointNumber, 1)), + ["duplicateCommitLookup"] = ExplainFind( + _commits!, + new BsonDocument + { + [MongoCommitFields.BucketId] = "default", + [MongoCommitFields.StreamId] = "stream-1", + [MongoCommitFields.CommitId] = "commit-1" + }), + ["streamsToSnapshot"] = ExplainFind( + _streamHeads!, + new BsonDocument + { + [MongoStreamHeadFields.FullQualifiedBucketId] = "default", + [MongoStreamHeadFields.Unsnapshotted] = new BsonDocument("$gte", 0) + }, + new BsonDocument(MongoStreamHeadFields.Unsnapshotted, -1)), + ["getSnapshot"] = ExplainFind( + _snapshots!, + new BsonDocument + { + [MongoSnapshotFields.FullQualifiedBucketId] = "default", + [MongoSnapshotFields.FullQualifiedStreamId] = "stream-1", + [MongoSnapshotFields.FullQualifiedStreamRevision] = new BsonDocument("$lte", 6) + }, + new BsonDocument(MongoSnapshotFields.FullQualifiedStreamRevision, -1), + 1), + ["deleteStreamSnapshotsFilter"] = ExplainFind( + _snapshots!, + new BsonDocument + { + [MongoSnapshotFields.FullQualifiedBucketId] = "default", + [MongoSnapshotFields.FullQualifiedStreamId] = "stream-1" + }), + ["deleteStreamHeadById"] = ExplainFind( + _streamHeads!, + new BsonDocument(MongoStreamHeadFields.Id, new BsonDocument + { + [MongoStreamHeadFields.BucketId] = "default", + [MongoStreamHeadFields.StreamId] = "stream-1" + })), + ["purgeBucketSnapshotsFilter"] = ExplainFind( + _snapshots!, + new BsonDocument(MongoSnapshotFields.FullQualifiedBucketId, "default")), + ["purgeBucketStreamHeadsFilter"] = ExplainFind( + _streamHeads!, + new BsonDocument(MongoStreamHeadFields.FullQualifiedBucketId, "default")), + }; + } + + protected override void Cleanup() + { + if (_database != null) + { + _database.Client.DropDatabase(_database.DatabaseNamespace.DatabaseName); + } + + _engine?.Dispose(); + } + + [Fact] + public void Commit_range_read_should_use_GetFrom_index_without_a_sort_stage() + { + _plans!["commitRangeRead"].Indexes.Should().Contain(MongoCommitIndexes.GetFrom); + _plans["commitRangeRead"].Stages.Should().NotContain("SORT"); + } + + [Fact] + public void Bucket_checkpoint_read_should_use_GetFromCheckpoint_index() + { + _plans!["bucketCheckpointRead"].Indexes.Should().Contain(MongoCommitIndexes.GetFromCheckpoint); + } + + [Fact] + public void Duplicate_commit_lookup_should_use_CommitId_index() + { + _plans!["duplicateCommitLookup"].Indexes.Should().Contain(MongoCommitIndexes.CommitId); + } + + [Fact] + public void Streams_to_snapshot_should_use_bucket_unsnapshotted_index() + { + _plans!["streamsToSnapshot"].Indexes.Should().Contain(MongoStreamIndexes.BucketUnsnapshotted); + _plans["streamsToSnapshot"].Stages.Should().NotContain("SORT"); + } + + [Fact] + public void Snapshot_lookup_should_use_bucket_stream_revision_index() + { + _plans!["getSnapshot"].Indexes.Should().Contain(MongoSnapshotIndexes.BucketStreamRevision); + _plans["getSnapshot"].TotalKeysExamined.Should().BeGreaterThan(0); + } + + [Fact] + public void Snapshot_delete_filter_should_use_bucket_stream_revision_index() + { + _plans!["deleteStreamSnapshotsFilter"].Indexes.Should().Contain(MongoSnapshotIndexes.BucketStreamRevision); + } + + [Fact] + public void Stream_head_delete_by_id_should_use_the_builtin_id_index() + { + _plans!["deleteStreamHeadById"].Indexes.Should().Contain("_id_"); + } + + [Fact] + public void Bucket_purge_filters_should_use_their_prefix_indexes() + { + _plans!["purgeBucketSnapshotsFilter"].Indexes.Should().Contain(MongoSnapshotIndexes.BucketStreamRevision); + _plans["purgeBucketStreamHeadsFilter"].Indexes.Should().Contain(MongoStreamIndexes.BucketUnsnapshotted); + } + + private void SeedDocuments() + { + for (int i = 0; i < 10; i++) + { + string bucketId = i < 8 ? "default" : "other"; + string streamId = i < 5 ? "stream-1" : $"stream-{i}"; + int baseRevision = (i * 2) + 1; + + _commits!.InsertOne(new BsonDocument + { + [MongoCommitFields.CheckpointNumber] = i + 1, + [MongoCommitFields.BucketId] = bucketId, + [MongoCommitFields.StreamId] = streamId, + [MongoCommitFields.StreamRevisionFrom] = baseRevision, + [MongoCommitFields.StreamRevisionTo] = baseRevision + 1, + [MongoCommitFields.CommitSequence] = i + 1, + [MongoCommitFields.CommitId] = $"commit-{i + 1}", + [MongoCommitFields.CommitStamp] = DateTime.UtcNow.AddSeconds(i), + [MongoCommitFields.Headers] = new BsonDocument(), + [MongoCommitFields.Events] = new BsonArray(), + }); + } + + _streamHeads!.InsertMany(new[] + { + new BsonDocument + { + [MongoStreamHeadFields.Id] = new BsonDocument + { + [MongoStreamHeadFields.BucketId] = "default", + [MongoStreamHeadFields.StreamId] = "stream-1" + }, + [MongoStreamHeadFields.HeadRevision] = 8, + [MongoStreamHeadFields.SnapshotRevision] = 2, + [MongoStreamHeadFields.Unsnapshotted] = 6, + }, + new BsonDocument + { + [MongoStreamHeadFields.Id] = new BsonDocument + { + [MongoStreamHeadFields.BucketId] = "default", + [MongoStreamHeadFields.StreamId] = "stream-2" + }, + [MongoStreamHeadFields.HeadRevision] = 4, + [MongoStreamHeadFields.SnapshotRevision] = 0, + [MongoStreamHeadFields.Unsnapshotted] = 4, + }, + new BsonDocument + { + [MongoStreamHeadFields.Id] = new BsonDocument + { + [MongoStreamHeadFields.BucketId] = "other", + [MongoStreamHeadFields.StreamId] = "stream-1" + }, + [MongoStreamHeadFields.HeadRevision] = 9, + [MongoStreamHeadFields.SnapshotRevision] = 0, + [MongoStreamHeadFields.Unsnapshotted] = 9, + }, + }); + + _snapshots!.InsertMany(new[] + { + CreateSnapshotDocument("default", "stream-1", 1, "s1-r1"), + CreateSnapshotDocument("default", "stream-1", 3, "s1-r3"), + CreateSnapshotDocument("default", "stream-1", 5, "s1-r5"), + CreateSnapshotDocument("default", "stream-2", 2, "s2-r2"), + CreateSnapshotDocument("other", "stream-1", 4, "other-r4"), + }); + } + + private static BsonDocument CreateSnapshotDocument(string bucketId, string streamId, int streamRevision, string payload) + { + return new BsonDocument + { + [MongoSnapshotFields.Id] = new BsonDocument + { + [MongoSnapshotFields.BucketId] = bucketId, + [MongoSnapshotFields.StreamId] = streamId, + [MongoSnapshotFields.StreamRevision] = streamRevision, + }, + [MongoSnapshotFields.Payload] = payload, + }; + } + + private ExplainPlanResult ExplainFind( + IMongoCollection collection, + BsonDocument filter, + BsonDocument? sort = null, + int? limit = null) + { + var explainCommand = new BsonDocument + { + ["explain"] = new BsonDocument + { + ["find"] = collection.CollectionNamespace.CollectionName, + ["filter"] = filter, + }, + ["verbosity"] = "executionStats", + }; + + if (sort != null) + { + explainCommand["explain"].AsBsonDocument["sort"] = sort; + } + + if (limit.HasValue) + { + explainCommand["explain"].AsBsonDocument["limit"] = limit.Value; + } + + var explain = collection.Database.RunCommand(explainCommand); + var result = new ExplainPlanResult(); + WalkPlan(explain["queryPlanner"]["winningPlan"], result); + + var executionStats = explain["executionStats"].AsBsonDocument; + result.TotalKeysExamined = executionStats["totalKeysExamined"].ToInt32(); + result.TotalDocsExamined = executionStats["totalDocsExamined"].ToInt32(); + return result; + } + + private static void WalkPlan(BsonValue value, ExplainPlanResult result) + { + if (!value.IsBsonDocument) + { + return; + } + + var document = value.AsBsonDocument; + if (document.TryGetValue("stage", out BsonValue? stageValue)) + { + result.Stages.Add(stageValue.AsString); + } + + if (document.TryGetValue("indexName", out BsonValue? indexNameValue)) + { + result.Indexes.Add(indexNameValue.AsString); + } + + foreach (BsonElement element in document.Elements) + { + if (element.Value.IsBsonDocument) + { + WalkPlan(element.Value, result); + } + else if (element.Value.IsBsonArray) + { + foreach (BsonValue arrayItem in element.Value.AsBsonArray) + { + WalkPlan(arrayItem, result); + } + } + } + } + } +} \ No newline at end of file diff --git a/src/NEventStore.Persistence.MongoDB.Tests/AcceptanceTests/MongoCommitCustomSerializationTests.Async.cs b/src/NEventStore.Persistence.MongoDB.Tests/AcceptanceTests/MongoCommitCustomSerializationTests.Async.cs index 530db71..fa321e3 100644 --- a/src/NEventStore.Persistence.MongoDB.Tests/AcceptanceTests/MongoCommitCustomSerializationTests.Async.cs +++ b/src/NEventStore.Persistence.MongoDB.Tests/AcceptanceTests/MongoCommitCustomSerializationTests.Async.cs @@ -220,7 +220,7 @@ protected override async Task BecauseAsync() return Persistence.CommitAsync(attempt); }).ConfigureAwait(false); - Assert.That(_thrown, Is.Null); + _thrown.Should().BeNull(); var observer = new CommitStreamObserver(); await Persistence.GetFromAsync(_streamId!, 0, int.MaxValue, observer).ConfigureAwait(false); diff --git a/src/NEventStore.Persistence.MongoDB.Tests/AcceptanceTests/MongoCommitCustomSerializationTests.cs b/src/NEventStore.Persistence.MongoDB.Tests/AcceptanceTests/MongoCommitCustomSerializationTests.cs index af55d27..57e6c10 100644 --- a/src/NEventStore.Persistence.MongoDB.Tests/AcceptanceTests/MongoCommitCustomSerializationTests.cs +++ b/src/NEventStore.Persistence.MongoDB.Tests/AcceptanceTests/MongoCommitCustomSerializationTests.cs @@ -214,7 +214,7 @@ protected override void Because() Persistence.Commit(attempt); }); - Assert.That(_thrown, Is.Null); + _thrown.Should().BeNull(); _persisted = Persistence.GetFrom(_streamId!, 0, int.MaxValue).First(); } diff --git a/src/NEventStore.Persistence.MongoDB.Tests/AcceptanceTests/MongoPersistenceOptionsTests.cs b/src/NEventStore.Persistence.MongoDB.Tests/AcceptanceTests/MongoPersistenceOptionsTests.cs index 40679de..549af1b 100644 --- a/src/NEventStore.Persistence.MongoDB.Tests/AcceptanceTests/MongoPersistenceOptionsTests.cs +++ b/src/NEventStore.Persistence.MongoDB.Tests/AcceptanceTests/MongoPersistenceOptionsTests.cs @@ -7,6 +7,8 @@ using MongoDB.Driver; using NEventStore.Persistence.AcceptanceTests.BDD; using Microsoft.ApplicationInsights.Extensibility.Implementation.Tracing; +using FluentAssertions; + #if MSTEST using Microsoft.VisualStudio.TestTools.UnitTesting; @@ -48,8 +50,8 @@ protected override void Because() public void Settings_are_correctly_applied() { var settings = _db!.Client.Settings; - Assert.That(TestApplicationName, Is.EqualTo(settings.ApplicationName)); - Assert.That(TestReplicaSetName, Is.EqualTo(settings.ReplicaSetName)); + TestApplicationName.Should().Be(settings.ApplicationName); + TestReplicaSetName.Should().Be(settings.ReplicaSetName); } } @@ -80,14 +82,14 @@ protected override void Because() [Fact] public void No_exception_is_thrown() { - Assert.That(_ex, Is.Null); + _ex.Should().BeNull(); } [Fact] public void Database_was_correctly_created() { - Assert.That(_db, Is.Not.Null); - Assert.That(_mongoClient, Is.EqualTo(_db!.Client)); + _db.Should().NotBeNull(); + _mongoClient.Should().BeSameAs(_db!.Client); } } @@ -119,14 +121,14 @@ protected override void Because() [Fact] public void No_exception_is_thrown() { - Assert.That(_ex, Is.Null); + _ex.Should().BeNull(); } [Fact] public void Database_was_correctly_created() { - Assert.That(_db, Is.Not.Null); - Assert.That(_mongoClient, Is.EqualTo(_db!.Client)); + _db.Should().NotBeNull(); + _mongoClient.Should().BeSameAs(_db!.Client); } } @@ -157,14 +159,14 @@ protected override void Because() [Fact] public void Exception_is_thrown() { - Assert.That(_ex, Is.Not.Null); - Assert.That(_ex!.Message, Is.EqualTo("MongoClient instance was created with a different connection string: host and port should match.")); + _ex.Should().NotBeNull(); + _ex!.Message.Should().Be("MongoClient instance was created with a different connection string: host and port should match."); } [Fact] public void Database_was_not_created() { - Assert.That(_db, Is.Null); + _db.Should().BeNull(); } } @@ -197,14 +199,14 @@ protected override void Because() [Fact] public void Exception_is_thrown() { - Assert.That(_ex, Is.Not.Null); - Assert.That(_ex!.Message, Is.EqualTo("MongoClient instance was created with a different connection string: hosts and ports should match.")); + _ex.Should().NotBeNull(); + _ex!.Message.Should().Be("MongoClient instance was created with a different connection string: hosts and ports should match."); } [Fact] public void Database_was_not_created() { - Assert.That(_db, Is.Null); + _db.Should().BeNull(); } } } diff --git a/src/NEventStore.Persistence.MongoDB.Tests/AcceptanceTests/OptimisticLoopTests.Async.cs b/src/NEventStore.Persistence.MongoDB.Tests/AcceptanceTests/OptimisticLoopTests.Async.cs index ab56c9a..1395d9e 100644 --- a/src/NEventStore.Persistence.MongoDB.Tests/AcceptanceTests/OptimisticLoopTests.Async.cs +++ b/src/NEventStore.Persistence.MongoDB.Tests/AcceptanceTests/OptimisticLoopTests.Async.cs @@ -165,7 +165,7 @@ protected override async Task BecauseAsync() [Fact] public void Should_have_checkpoint_equal_to_one() { - Assert.That(_commit, Is.Not.Null); + _commit.Should().NotBeNull(); _commit!.CheckpointToken.Should().Be(1); } } @@ -190,7 +190,7 @@ protected override async Task BecauseAsync() [Fact] public void Should_have_checkpoint_equal_to_two() { - Assert.That(_commit, Is.Not.Null); + _commit.Should().NotBeNull(); _commit!.CheckpointToken.Should().Be(2); } } @@ -216,7 +216,7 @@ protected override async Task BecauseAsync() [Fact] public void Should_have_checkpoint_equal_to_two() { - Assert.That(_commit, Is.Not.Null); + _commit.Should().NotBeNull(); _commit!.CheckpointToken.Should().Be(2); } } @@ -376,14 +376,14 @@ protected override async Task BecauseAsync() [Fact] public void Last_deleted_commit_is_not_purged_to_preserve_checkpoint_numbering() { - Assert.That(_commits, Is.Not.Null); + _commits.Should().NotBeNull(); _commits!.Length.Should().Be(1); } [Fact] public void Last_deleted_commit_has_the_higher_checkpoint_number() { - Assert.That(_commits, Is.Not.Null); + _commits.Should().NotBeNull(); _commits![0].CheckpointToken.Should().Be(4); } } @@ -429,7 +429,7 @@ protected override async Task BecauseAsync() [Fact] public void All_deleted_commits_are_purged() { - Assert.That(_commits, Is.Not.Null); + _commits.Should().NotBeNull(); _commits!.Length.Should().Be(0); } } @@ -493,4 +493,68 @@ public async Task All_commits_are_deleted() observer.Commits.Should().BeEmpty(); } } + +#if MSTEST + [TestClass] +#endif + public class When_streams_to_snapshot_are_requested_for_a_specific_bucket : PersistenceEngineConcern + { + private const string BucketA = "a"; + private const string BucketB = "b"; + + private string? _streamIdInBucketA; + private IList? _streamsToSnapshot; + + protected override async Task ContextAsync() + { + _streamIdInBucketA = Guid.NewGuid().ToString(); + var commitInBucketA = await Persistence.CommitAsync(_streamIdInBucketA.BuildAttempt(bucketId: BucketA)).ConfigureAwait(false); + await Persistence.CommitAsync(commitInBucketA!.BuildNextAttempt()).ConfigureAwait(false); + + var streamIdInBucketB = Guid.NewGuid().ToString(); + var commitInBucketB = await Persistence.CommitAsync(streamIdInBucketB.BuildAttempt(bucketId: BucketB)).ConfigureAwait(false); + await Persistence.CommitAsync(commitInBucketB!.BuildNextAttempt()).ConfigureAwait(false); + } + + protected override async Task BecauseAsync() + { + var observer = new StreamHeadObserver(); + await Persistence.GetStreamsToSnapshotAsync(BucketA, 0, observer).ConfigureAwait(false); + _streamsToSnapshot = observer.StreamHeads; + } + + [Fact] + public void Only_streams_from_the_requested_bucket_are_returned() + { + _streamsToSnapshot.Should().NotBeNull(); + _streamsToSnapshot!.Should().ContainSingle(); + _streamsToSnapshot.Single().BucketId.Should().Be(BucketA); + _streamsToSnapshot.Single().StreamId.Should().Be(_streamIdInBucketA); + } + } + +#if MSTEST + [TestClass] +#endif + public class When_a_deleted_stream_has_snapshots : PersistenceEngineConcern + { + private ICommit? _commit; + + protected override async Task ContextAsync() + { + _commit = await Persistence.CommitAsync(Guid.NewGuid().ToString().BuildAttempt()).ConfigureAwait(false); + await Persistence.AddSnapshotAsync(new Snapshot(_commit!.BucketId, _commit.StreamId, _commit.StreamRevision, "snapshot")).ConfigureAwait(false); + } + + protected override Task BecauseAsync() + { + return Persistence.DeleteStreamAsync(_commit!.BucketId, _commit.StreamId); + } + + [Fact] + public async Task The_snapshot_cannot_be_loaded_from_the_stream() + { + (await Persistence.GetSnapshotAsync(_commit!.BucketId, _commit.StreamId, _commit.StreamRevision).ConfigureAwait(false)).Should().BeNull(); + } + } } diff --git a/src/NEventStore.Persistence.MongoDB.Tests/AcceptanceTests/OptimisticLoopTests.cs b/src/NEventStore.Persistence.MongoDB.Tests/AcceptanceTests/OptimisticLoopTests.cs index 851023c..1cd5f53 100644 --- a/src/NEventStore.Persistence.MongoDB.Tests/AcceptanceTests/OptimisticLoopTests.cs +++ b/src/NEventStore.Persistence.MongoDB.Tests/AcceptanceTests/OptimisticLoopTests.cs @@ -165,7 +165,7 @@ protected override void Because() [Fact] public void Should_have_checkpoint_equal_to_one() { - Assert.That(_commit, Is.Not.Null); + _commit.Should().NotBeNull(); _commit!.CheckpointToken.Should().Be(1); } } @@ -190,7 +190,7 @@ protected override void Because() [Fact] public void Should_have_checkpoint_equal_to_two() { - Assert.That(_commit, Is.Not.Null); + _commit.Should().NotBeNull(); _commit!.CheckpointToken.Should().Be(2); } } @@ -216,7 +216,7 @@ protected override void Because() [Fact] public void Should_have_checkpoint_equal_to_two() { - Assert.That(_commit, Is.Not.Null); + _commit.Should().NotBeNull(); _commit!.CheckpointToken.Should().Be(2); } } @@ -367,14 +367,14 @@ protected override void Because() [Fact] public void Last_deleted_commit_is_not_purged_to_preserve_checkpoint_numbering() { - Assert.That(_commits, Is.Not.Null); + _commits.Should().NotBeNull(); _commits!.Length.Should().Be(1); } [Fact] public void Last_deleted_commit_has_the_higher_checkpoint_number() { - Assert.That(_commits, Is.Not.Null); + _commits.Should().NotBeNull(); _commits![0].CheckpointToken.Should().Be(4); } } @@ -418,7 +418,7 @@ protected override void Because() [Fact] public void All_deleted_commits_are_purged() { - Assert.That(_commits, Is.Not.Null); + _commits.Should().NotBeNull(); _commits!.Length.Should().Be(0); } } @@ -482,4 +482,66 @@ public void All_commits_are_deleted() commits.Length.Should().Be(0); } } + +#if MSTEST + [TestClass] +#endif + public class When_streams_to_snapshot_are_requested_for_a_specific_bucket : PersistenceEngineConcern + { + private const string BucketA = "a"; + private const string BucketB = "b"; + + private string? _streamIdInBucketA; + private IStreamHead[]? _streamsToSnapshot; + + protected override void Context() + { + _streamIdInBucketA = Guid.NewGuid().ToString(); + var commitInBucketA = Persistence.Commit(_streamIdInBucketA.BuildAttempt(bucketId: BucketA)); + Persistence.Commit(commitInBucketA!.BuildNextAttempt()); + + var streamIdInBucketB = Guid.NewGuid().ToString(); + var commitInBucketB = Persistence.Commit(streamIdInBucketB.BuildAttempt(bucketId: BucketB)); + Persistence.Commit(commitInBucketB!.BuildNextAttempt()); + } + + protected override void Because() + { + _streamsToSnapshot = Persistence.GetStreamsToSnapshot(BucketA, 0).ToArray(); + } + + [Fact] + public void Only_streams_from_the_requested_bucket_are_returned() + { + _streamsToSnapshot.Should().NotBeNull(); + _streamsToSnapshot!.Should().ContainSingle(); + _streamsToSnapshot[0].BucketId.Should().Be(BucketA); + _streamsToSnapshot[0].StreamId.Should().Be(_streamIdInBucketA); + } + } + +#if MSTEST + [TestClass] +#endif + public class When_a_deleted_stream_has_snapshots : PersistenceEngineConcern + { + private ICommit? _commit; + + protected override void Context() + { + _commit = Persistence.Commit(Guid.NewGuid().ToString().BuildAttempt()); + Persistence.AddSnapshot(new Snapshot(_commit!.BucketId, _commit.StreamId, _commit.StreamRevision, "snapshot")); + } + + protected override void Because() + { + Persistence.DeleteStream(_commit!.BucketId, _commit.StreamId); + } + + [Fact] + public void The_snapshot_cannot_be_loaded_from_the_stream() + { + Persistence.GetSnapshot(_commit!.BucketId, _commit.StreamId, _commit.StreamRevision).Should().BeNull(); + } + } } diff --git a/src/NEventStore.Persistence.MongoDB.Tests/AcceptanceTests/SharedPersistenceTests.Async.cs b/src/NEventStore.Persistence.MongoDB.Tests/AcceptanceTests/SharedPersistenceTests.Async.cs index 75f1fb1..fa3f8d3 100644 --- a/src/NEventStore.Persistence.MongoDB.Tests/AcceptanceTests/SharedPersistenceTests.Async.cs +++ b/src/NEventStore.Persistence.MongoDB.Tests/AcceptanceTests/SharedPersistenceTests.Async.cs @@ -42,9 +42,9 @@ protected override async Task BecauseAsync() [Fact] public void Should_have_a_checkpoint_greater_than_the_previous_commit_on_the_other_process() { - Assert.That(_commit1, Is.Not.Null); + _commit1.Should().NotBeNull(); long chkNum1 = _commit1!.CheckpointToken; - Assert.That(_commit2, Is.Not.Null); + _commit2.Should().NotBeNull(); long chkNum2 = _commit2!.CheckpointToken; chkNum2.Should().BeGreaterThan(chkNum1); diff --git a/src/NEventStore.Persistence.MongoDB.Tests/AcceptanceTests/SharedPersistenceTests.cs b/src/NEventStore.Persistence.MongoDB.Tests/AcceptanceTests/SharedPersistenceTests.cs index 0d54c50..e3f6e02 100644 --- a/src/NEventStore.Persistence.MongoDB.Tests/AcceptanceTests/SharedPersistenceTests.cs +++ b/src/NEventStore.Persistence.MongoDB.Tests/AcceptanceTests/SharedPersistenceTests.cs @@ -42,9 +42,9 @@ protected override void Because() [Fact] public void Should_have_a_checkpoint_greater_than_the_previous_commit_on_the_other_process() { - Assert.That(_commit1, Is.Not.Null); + _commit1.Should().NotBeNull(); long chkNum1 = _commit1!.CheckpointToken; - Assert.That(_commit2, Is.Not.Null); + _commit2.Should().NotBeNull(); long chkNum2 = _commit2!.CheckpointToken; chkNum2.Should().BeGreaterThan(chkNum1); diff --git a/src/NEventStore.Persistence.MongoDB.Tests/NEventStore.Persistence.MongoDB.Core.Tests.csproj b/src/NEventStore.Persistence.MongoDB.Tests/NEventStore.Persistence.MongoDB.Core.Tests.csproj index 7d88182..aacad91 100644 --- a/src/NEventStore.Persistence.MongoDB.Tests/NEventStore.Persistence.MongoDB.Core.Tests.csproj +++ b/src/NEventStore.Persistence.MongoDB.Tests/NEventStore.Persistence.MongoDB.Core.Tests.csproj @@ -1,11 +1,12 @@  - net8.0;net472 + net472;$(ModernTargetFrameworks) false exe NEventStore.Persistence.MongoDB.Tests false + true @@ -13,18 +14,18 @@ NUNIT - + - - - - - - - + + + + + + + @@ -43,10 +44,6 @@ - - - - ResXFileCodeGenerator diff --git a/src/NEventStore.Persistence.MongoDB/ConfigurationException.cs b/src/NEventStore.Persistence.MongoDB/ConfigurationException.cs index bb5d03d..56a82d6 100644 --- a/src/NEventStore.Persistence.MongoDB/ConfigurationException.cs +++ b/src/NEventStore.Persistence.MongoDB/ConfigurationException.cs @@ -26,14 +26,5 @@ public ConfigurationException(string message) : base(message) public ConfigurationException(string message, Exception inner) : base(message, inner) { } - - /// - /// Initializes a new instance of the class with serialized data. - /// - protected ConfigurationException( - System.Runtime.Serialization.SerializationInfo info, - System.Runtime.Serialization.StreamingContext context) : base(info, context) - { - } } -} +} \ No newline at end of file diff --git a/src/NEventStore.Persistence.MongoDB/ExtensionMethods.cs b/src/NEventStore.Persistence.MongoDB/ExtensionMethods.cs index d0d94a0..a758ee0 100644 --- a/src/NEventStore.Persistence.MongoDB/ExtensionMethods.cs +++ b/src/NEventStore.Persistence.MongoDB/ExtensionMethods.cs @@ -1,7 +1,7 @@ -using global::MongoDB.Bson; -using global::MongoDB.Driver; +using MongoDB.Bson; +using MongoDB.Driver; using NEventStore.Serialization; -using BsonSerializer = global::MongoDB.Bson.Serialization.BsonSerializer; +using BsonSerializer = MongoDB.Bson.Serialization.BsonSerializer; namespace NEventStore.Persistence.MongoDB { @@ -15,17 +15,20 @@ public static class ExtensionMethods /// public static BsonDocument ToMongoCommit(this CommitAttempt commit, Int64 checkpoint, IDocumentSerializer serializer) { - int streamRevision = commit.StreamRevision - (commit.Events.Count - 1); + int eventCount = commit.Events.Count; + int streamRevision = commit.StreamRevision - (eventCount - 1); int streamRevisionStart = streamRevision; - IEnumerable events = commit - .Events - .Select(e => + var events = new BsonArray(eventCount); + foreach (var @event in commit.Events) + { + events.Add( new BsonDocument { - {MongoCommitFields.StreamRevision, streamRevision++}, - {MongoCommitFields.Payload, BsonDocumentWrapper.Create(typeof(EventMessage), serializer.Serialize(e))} + { MongoCommitFields.StreamRevision, streamRevision++ }, + { MongoCommitFields.Payload, BsonDocumentWrapper.Create(typeof(EventMessage), serializer.Serialize(@event)) } }); + } var mc = new MongoCommit { @@ -33,7 +36,7 @@ public static BsonDocument ToMongoCommit(this CommitAttempt commit, Int64 checkp CommitId = commit.CommitId, CommitStamp = commit.CommitStamp, Headers = commit.Headers, - Events = new BsonArray(events), + Events = events, StreamRevisionFrom = streamRevisionStart, StreamRevisionTo = streamRevision - 1, BucketId = commit.BucketId, @@ -50,9 +53,7 @@ public static BsonDocument ToMongoCommit(this CommitAttempt commit, Int64 checkp /// public static BsonDocument ToEmptyCommit(this CommitAttempt commit, Int64 checkpoint, String systemBucketName) { - if (commit == null) throw new ArgumentNullException(nameof(commit)); if (String.IsNullOrWhiteSpace(systemBucketName)) throw new ArgumentNullException(nameof(systemBucketName)); - int streamRevisionStart = commit.StreamRevision - (commit.Events.Count - 1); var mc = new MongoCommit { @@ -77,6 +78,16 @@ public static BsonDocument ToEmptyCommit(this CommitAttempt commit, Int64 checkp public static ICommit ToCommit(this BsonDocument doc, IDocumentSerializer serializer) { var mc = BsonSerializer.Deserialize(doc); + int eventCount = mc.Events.Count; + var events = new EventMessage[eventCount]; + + for (int i = 0; i < eventCount; i++) + { + BsonValue payload = mc.Events[i][MongoCommitFields.Payload]; + events[i] = payload.IsBsonDocument + ? BsonSerializer.Deserialize(payload.AsBsonDocument) + : serializer.Deserialize(payload.AsByteArray)!; // ByteStreamDocumentSerializer ?!?! doesn't work this way! + } return new Commit(mc.BucketId, mc.StreamId, @@ -86,13 +97,7 @@ public static ICommit ToCommit(this BsonDocument doc, IDocumentSerializer serial mc.CommitStamp, mc.CheckpointNumber, mc.Headers, - mc.Events.Select(e => - { - BsonValue payload = e[MongoCommitFields.Payload]; - return payload.IsBsonDocument - ? BsonSerializer.Deserialize(payload.ToBsonDocument()) - : serializer.Deserialize(payload.AsByteArray); // ByteStreamDocumentSerializer ?!?! doesn't work this way! - }).ToArray()); + events); } /// @@ -128,7 +133,7 @@ public static Snapshot ToSnapshot(this BsonDocument doc, IDocumentSerializer ser switch (bsonPayload.BsonType) { case BsonType.Binary: - payload = serializer.Deserialize(bsonPayload.AsByteArray); + payload = serializer.Deserialize(bsonPayload.AsByteArray)!; break; case BsonType.Document: payload = BsonSerializer.Deserialize(bsonPayload.AsBsonDocument); diff --git a/src/NEventStore.Persistence.MongoDB/Messages.Designer.cs b/src/NEventStore.Persistence.MongoDB/Messages.Designer.cs deleted file mode 100644 index 7b5d2d9..0000000 --- a/src/NEventStore.Persistence.MongoDB/Messages.Designer.cs +++ /dev/null @@ -1,343 +0,0 @@ -//------------------------------------------------------------------------------ -// -// This code was generated by a tool. -// Runtime Version:4.0.30319.42000 -// -// Changes to this file may cause incorrect behavior and will be lost if -// the code is regenerated. -// -//------------------------------------------------------------------------------ - -namespace NEventStore.Persistence.MongoDB { - using System; - - - /// - /// A strongly-typed resource class, for looking up localized strings, etc. - /// - // This class was auto-generated by the StronglyTypedResourceBuilder - // class via a tool like ResGen or Visual Studio. - // To add or remove a member, edit your .ResX file then rerun ResGen - // with the /str option, or rebuild your VS project. - [global::System.CodeDom.Compiler.GeneratedCodeAttribute("System.Resources.Tools.StronglyTypedResourceBuilder", "17.0.0.0")] - [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] - [global::System.Runtime.CompilerServices.CompilerGeneratedAttribute()] - internal class Messages { - - private static global::System.Resources.ResourceManager resourceMan; - - private static global::System.Globalization.CultureInfo resourceCulture; - - [global::System.Diagnostics.CodeAnalysis.SuppressMessageAttribute("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] - internal Messages() { - } - - /// - /// Returns the cached ResourceManager instance used by this class. - /// - [global::System.ComponentModel.EditorBrowsableAttribute(global::System.ComponentModel.EditorBrowsableState.Advanced)] - internal static global::System.Resources.ResourceManager ResourceManager { - get { - if (object.ReferenceEquals(resourceMan, null)) { - global::System.Resources.ResourceManager temp = new global::System.Resources.ResourceManager("NEventStore.Persistence.MongoDB.Messages", typeof(Messages).Assembly); - resourceMan = temp; - } - return resourceMan; - } - } - - /// - /// Overrides the current thread's CurrentUICulture property for all - /// resource lookups using this strongly typed resource class. - /// - [global::System.ComponentModel.EditorBrowsableAttribute(global::System.ComponentModel.EditorBrowsableState.Advanced)] - internal static global::System.Globalization.CultureInfo Culture { - get { - return resourceCulture; - } - set { - resourceCulture = value; - } - } - - /// - /// Looks up a localized string similar to Adding snapshot to stream '{0}' in bucket '{1}' at position {2}.. - /// - internal static string AddingSnapshot { - get { - return ResourceManager.GetString("AddingSnapshot", resourceCulture); - } - } - - /// - /// Looks up a localized string similar to Error Adding snapshot to stream '{0}' in bucket '{1}' at position {2} - Ex: {3}.. - /// - internal static string AddingSnapshotError { - get { - return ResourceManager.GetString("AddingSnapshotError", resourceCulture); - } - } - - /// - /// Looks up a localized string similar to Attempting to commit {0} events on stream '{1}' at sequence {2}.. - /// - internal static string AttemptingToCommit { - get { - return ResourceManager.GetString("AttemptingToCommit", resourceCulture); - } - } - - /// - /// Looks up a localized string similar to Commit '{0}' persisted.. - /// - internal static string CommitPersisted { - get { - return ResourceManager.GetString("CommitPersisted", resourceCulture); - } - } - - /// - /// Looks up a localized string similar to Concurrency Exception commitId {0} [{1}] - Bucket {2} - StreamId {3} - Ex: {4}.. - /// - internal static string ConcurrencyExceptionError { - get { - return ResourceManager.GetString("ConcurrencyExceptionError", resourceCulture); - } - } - - /// - /// Looks up a localized string similar to Concurrent write detected.. - /// - internal static string ConcurrentWriteDetected { - get { - return ResourceManager.GetString("ConcurrentWriteDetected", resourceCulture); - } - } - - /// - /// Looks up a localized string similar to Could not find connection name '{0}' in the configuration file.. - /// - internal static string ConnectionNotFound { - get { - return ResourceManager.GetString("ConnectionNotFound", resourceCulture); - } - } - - /// - /// Looks up a localized string similar to Deleting stream '{0}' from bucket '{1}'.. - /// - internal static string DeletingStream { - get { - return ResourceManager.GetString("DeletingStream", resourceCulture); - } - } - - /// - /// Looks up a localized string similar to Concurrency issue; determining whether attempt was duplicate.. - /// - internal static string DetectingConcurrency { - get { - return ResourceManager.GetString("DetectingConcurrency", resourceCulture); - } - } - - /// - /// Looks up a localized string similar to Duplicated checkpoint Token commit {0} [{1}] - Bucket {2} - StreamId {3}.. - /// - internal static string DuplicatedCheckpointTokenError { - get { - return ResourceManager.GetString("DuplicatedCheckpointTokenError", resourceCulture); - } - } - - /// - /// Looks up a localized string similar to [NEventStore.Persistence.MongoDB] Duplicated commitId {0} [{1}] - Bucket {2} - StreamId {3}.. - /// - internal static string DuplicatedCommitError { - get { - return ResourceManager.GetString("DuplicatedCommitError", resourceCulture); - } - } - - /// - /// Looks up a localized string similar to Error filling hole commitId {0} [{1}] - Bucket {2} - StreamId {3} - Ex: {4}.. - /// - internal static string FillHoleError { - get { - return ResourceManager.GetString("FillHoleError", resourceCulture); - } - } - - /// - /// Looks up a localized string similar to Generic error persisting commit {0} [{1}] - Bucket {2} - StreamId {3} - Ex: {4}.. - /// - internal static string GenericPersistingError { - get { - return ResourceManager.GetString("GenericPersistingError", resourceCulture); - } - } - - /// - /// Looks up a localized string similar to Getting all commits for stream '{0}' in bucket '{1}' between revisions '{2}' and '{3}'.. - /// - internal static string GettingAllCommitsBetween { - get { - return ResourceManager.GetString("GettingAllCommitsBetween", resourceCulture); - } - } - - /// - /// Looks up a localized string similar to Getting all commits from '{0}' forward from bucket '{1}'.. - /// - internal static string GettingAllCommitsFrom { - get { - return ResourceManager.GetString("GettingAllCommitsFrom", resourceCulture); - } - } - - /// - /// Looks up a localized string similar to Getting all commits from Bucket '{0}' and checkpoint '{1}'.. - /// - internal static string GettingAllCommitsFromBucketAndCheckpoint { - get { - return ResourceManager.GetString("GettingAllCommitsFromBucketAndCheckpoint", resourceCulture); - } - } - - /// - /// Looks up a localized string similar to Getting all commits since checkpoint '{0}'.. - /// - internal static string GettingAllCommitsFromCheckpoint { - get { - return ResourceManager.GetString("GettingAllCommitsFromCheckpoint", resourceCulture); - } - } - - /// - /// Looks up a localized string similar to Getting all commits from '{0}' to '{1}'.. - /// - internal static string GettingAllCommitsFromTo { - get { - return ResourceManager.GetString("GettingAllCommitsFromTo", resourceCulture); - } - } - - /// - /// Looks up a localized string similar to Getting all commits from bucket '{0}' from checkpoint '{1}' (excluded) up to '{2}' (included).. - /// - internal static string GettingCommitsFromBucketAndFromToCheckpoint { - get { - return ResourceManager.GetString("GettingCommitsFromBucketAndFromToCheckpoint", resourceCulture); - } - } - - /// - /// Looks up a localized string similar to Getting all commits from checkpoint '{0}' (excluded) up to '{1}' (included).. - /// - internal static string GettingCommitsFromToCheckpoint { - get { - return ResourceManager.GetString("GettingCommitsFromToCheckpoint", resourceCulture); - } - } - - /// - /// Looks up a localized string similar to Getting snapshot for stream '{0}' on or before revision {1}.. - /// - internal static string GettingRevision { - get { - return ResourceManager.GetString("GettingRevision", resourceCulture); - } - } - - /// - /// Looks up a localized string similar to Getting a list of streams to snapshot.. - /// - internal static string GettingStreamsToSnapshot { - get { - return ResourceManager.GetString("GettingStreamsToSnapshot", resourceCulture); - } - } - - /// - /// Looks up a localized string similar to Getting the list of all undispatched commits.. - /// - internal static string GettingUndispatchedCommits { - get { - return ResourceManager.GetString("GettingUndispatchedCommits", resourceCulture); - } - } - - /// - /// Looks up a localized string similar to Initializing storage engine.. - /// - internal static string InitializingStorage { - get { - return ResourceManager.GetString("InitializingStorage", resourceCulture); - } - } - - /// - /// Looks up a localized string similar to Marking commit '{0}' as dispatched.. - /// - internal static string MarkingCommitAsDispatched { - get { - return ResourceManager.GetString("MarkingCommitAsDispatched", resourceCulture); - } - } - - /// - /// Looks up a localized string similar to Purging all stored data for bucket '{0}'.. - /// - internal static string PurgingBucket { - get { - return ResourceManager.GetString("PurgingBucket", resourceCulture); - } - } - - /// - /// Looks up a localized string similar to Purging all stored data.. - /// - internal static string PurgingStorage { - get { - return ResourceManager.GetString("PurgingStorage", resourceCulture); - } - } - - /// - /// Looks up a localized string similar to Shutting down persistence.. - /// - internal static string ShuttingDownPersistence { - get { - return ResourceManager.GetString("ShuttingDownPersistence", resourceCulture); - } - } - - /// - /// Looks up a localized string similar to Storage threw exception of type '{0}'. - ///{1}. - /// - internal static string StorageThrewException { - get { - return ResourceManager.GetString("StorageThrewException", resourceCulture); - } - } - - /// - /// Looks up a localized string similar to Storage is unavailabe.. - /// - internal static string StorageUnavailable { - get { - return ResourceManager.GetString("StorageUnavailable", resourceCulture); - } - } - - /// - /// Looks up a localized string similar to Unsuppored checkpoint type. Expected {0} but got {1}.. - /// - internal static string UnsupportedCheckpointType { - get { - return ResourceManager.GetString("UnsupportedCheckpointType", resourceCulture); - } - } - } -} diff --git a/src/NEventStore.Persistence.MongoDB/Messages.resx b/src/NEventStore.Persistence.MongoDB/Messages.resx deleted file mode 100644 index e0f6a36..0000000 --- a/src/NEventStore.Persistence.MongoDB/Messages.resx +++ /dev/null @@ -1,214 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - text/microsoft-resx - - - 2.0 - - - System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - Adding snapshot to stream '{0}' in bucket '{1}' at position {2}. - - - Attempting to commit {0} events on stream '{1}' at sequence {2}. - - - Commit '{0}' persisted. - - - Concurrent write detected. - - - Concurrency issue; determining whether attempt was duplicate. - - - Getting all commits for stream '{0}' in bucket '{1}' between revisions '{2}' and '{3}'. - - - Getting all commits from '{0}' forward from bucket '{1}'. - - - Getting snapshot for stream '{0}' on or before revision {1}. - - - Getting a list of streams to snapshot. - - - Getting the list of all undispatched commits. - - - Initializing storage engine. - - - Marking commit '{0}' as dispatched. - - - Purging all stored data. - - - Shutting down persistence. - - - Storage is unavailabe. - - - Storage threw exception of type '{0}'. -{1} - - - Getting all commits from '{0}' to '{1}'. - - - Getting all commits since checkpoint '{0}'. - - - Getting all commits from Bucket '{0}' and checkpoint '{1}'. - - - Deleting stream '{0}' from bucket '{1}'. - - - Purging all stored data for bucket '{0}'. - - - Unsuppored checkpoint type. Expected {0} but got {1}. - - - Could not find connection name '{0}' in the configuration file. - - - Concurrency Exception commitId {0} [{1}] - Bucket {2} - StreamId {3} - Ex: {4}. - - - Duplicated checkpoint Token commit {0} [{1}] - Bucket {2} - StreamId {3}. - - - [NEventStore.Persistence.MongoDB] Duplicated commitId {0} [{1}] - Bucket {2} - StreamId {3}. - - - Error filling hole commitId {0} [{1}] - Bucket {2} - StreamId {3} - Ex: {4}. - - - Generic error persisting commit {0} [{1}] - Bucket {2} - StreamId {3} - Ex: {4}. - - - Getting all commits from bucket '{0}' from checkpoint '{1}' (excluded) up to '{2}' (included). - - - Getting all commits from checkpoint '{0}' (excluded) up to '{1}' (included). - - - Error Adding snapshot to stream '{0}' in bucket '{1}' at position {2} - Ex: {3}. - - \ No newline at end of file diff --git a/src/NEventStore.Persistence.MongoDB/MongoFields.cs b/src/NEventStore.Persistence.MongoDB/MongoFields.cs index 2ab6e6b..479bf99 100644 --- a/src/NEventStore.Persistence.MongoDB/MongoFields.cs +++ b/src/NEventStore.Persistence.MongoDB/MongoFields.cs @@ -85,6 +85,17 @@ public static class MongoSnapshotFields public const string FullQualifiedStreamRevision = Id + "." + StreamRevision; } + /// + /// MongoDB Snapshot Indexes + /// + public static class MongoSnapshotIndexes + { + /// + /// Snapshot lookup index. + /// + public const string BucketStreamRevision = "BucketStreamRevision_Index"; + } + /// /// MongoDB Commit Fields /// @@ -189,5 +200,10 @@ public static class MongoStreamIndexes /// Un-snapshotted index. /// public const string Unsnapshotted = "Unsnapshotted_Index"; + + /// + /// Bucket and un-snapshotted index. + /// + public const string BucketUnsnapshotted = "BucketUnsnapshotted_Index"; } } \ No newline at end of file diff --git a/src/NEventStore.Persistence.MongoDB/MongoPersistenceEngine.Async.cs b/src/NEventStore.Persistence.MongoDB/MongoPersistenceEngine.Async.cs index a66ae3b..4f58b7e 100644 --- a/src/NEventStore.Persistence.MongoDB/MongoPersistenceEngine.Async.cs +++ b/src/NEventStore.Persistence.MongoDB/MongoPersistenceEngine.Async.cs @@ -1,10 +1,10 @@ #pragma warning disable IDE0079 // Remove unnecessary suppression #pragma warning disable CA2254 // Template should be a static expression -using Microsoft.Extensions.Logging; +using System.Globalization; +using System.Runtime.ExceptionServices; using MongoDB.Bson; using MongoDB.Driver; -using System.Runtime.ExceptionServices; namespace NEventStore.Persistence.MongoDB { @@ -13,10 +13,7 @@ public partial class MongoPersistenceEngine : IPersistStreams /// public Task GetFromAsync(string bucketId, string streamId, int minRevision, int maxRevision, IAsyncObserver observer, CancellationToken cancellationToken) { - if (Logger.IsEnabled(LogLevel.Debug)) - { - Logger.LogDebug(Messages.GettingAllCommitsBetween, streamId, bucketId, minRevision, maxRevision); - } + MongoPersistenceEngineLogMessages.GettingAllCommitsBetween(Logger, streamId, bucketId, minRevision, maxRevision); return TryMongoAsync(async () => { @@ -42,8 +39,7 @@ public Task GetFromAsync(string bucketId, string streamId, int minRevision, int using var cursor = await PersistedCommits .Find(query) - // .Sort(Builders.Sort.Ascending(MongoCommitFields.StreamRevisionFrom)) - .Sort(SortByAscendingCheckpointNumber) + .Sort(SortByAscendingStreamRevisionFrom) .ToCursorAsync(cancellationToken) .ConfigureAwait(false); @@ -67,10 +63,7 @@ public Task GetFromAsync(string bucketId, string streamId, int minRevision, int /// public Task GetFromAsync(string bucketId, long checkpointToken, IAsyncObserver asyncObserver, CancellationToken cancellationToken) { - if (Logger.IsEnabled(LogLevel.Debug)) - { - Logger.LogDebug(Messages.GettingAllCommitsFromBucketAndCheckpoint, bucketId, checkpointToken); - } + MongoPersistenceEngineLogMessages.GettingAllCommitsFromBucketAndCheckpoint(Logger, bucketId, checkpointToken); return TryMongoAsync(async () => { @@ -108,10 +101,7 @@ public Task GetFromAsync(string bucketId, long checkpointToken, IAsyncObserver public Task GetFromToAsync(string bucketId, long fromCheckpointToken, long toCheckpointToken, IAsyncObserver asyncObserver, CancellationToken cancellationToken) { - if (Logger.IsEnabled(LogLevel.Debug)) - { - Logger.LogDebug(Messages.GettingCommitsFromBucketAndFromToCheckpoint, bucketId, fromCheckpointToken, toCheckpointToken); - } + MongoPersistenceEngineLogMessages.GettingCommitsFromBucketAndFromToCheckpoint(Logger, bucketId, fromCheckpointToken, toCheckpointToken); return TryMongoAsync(async () => { @@ -158,10 +148,7 @@ public Task GetFromToAsync(string bucketId, long fromCheckpointToken, long toChe /// public Task GetFromAsync(long checkpointToken, IAsyncObserver asyncObserver, CancellationToken cancellationToken) { - if (Logger.IsEnabled(LogLevel.Debug)) - { - Logger.LogDebug(Messages.GettingAllCommitsFromCheckpoint, checkpointToken); - } + MongoPersistenceEngineLogMessages.GettingAllCommitsFromCheckpoint(Logger, checkpointToken); return TryMongoAsync(async () => { @@ -199,10 +186,7 @@ public Task GetFromAsync(long checkpointToken, IAsyncObserver asyncObse /// public Task GetFromToAsync(long fromCheckpointToken, long toCheckpointToken, IAsyncObserver asyncObserver, CancellationToken cancellationToken) { - if (Logger.IsEnabled(LogLevel.Debug)) - { - Logger.LogDebug(Messages.GettingCommitsFromToCheckpoint, fromCheckpointToken, toCheckpointToken); - } + MongoPersistenceEngineLogMessages.GettingCommitsFromToCheckpoint(Logger, fromCheckpointToken, toCheckpointToken); return TryMongoAsync(async () => { @@ -250,10 +234,7 @@ public Task GetFromToAsync(long fromCheckpointToken, long toCheckpointToken, IAs /// public async Task CommitAsync(CommitAttempt attempt, CancellationToken cancellationToken) { - if (Logger.IsEnabled(LogLevel.Debug)) - { - Logger.LogDebug(Messages.AttemptingToCommit, attempt.Events.Count, attempt.StreamId, attempt.CommitSequence); - } + MongoPersistenceEngineLogMessages.AttemptingToCommit(Logger, attempt.Events.Count, attempt.StreamId, attempt.CommitSequence); // async/await was used to avoid Task to Task conversion issues return await TryMongoAsync(async () => @@ -278,16 +259,13 @@ public Task GetFromToAsync(long fromCheckpointToken, long toCheckpointToken, IAs UpdateStreamHeadInBackgroundThread(attempt.BucketId, attempt.StreamId, attempt.StreamRevision, attempt.Events.Count, cancellationToken); } - if (Logger.IsEnabled(LogLevel.Debug)) - { - Logger.LogDebug(Messages.CommitPersisted, attempt.CommitId); - } + MongoPersistenceEngineLogMessages.CommitPersisted(Logger, attempt.CommitId); } catch (MongoException e) { if (!e.Message.Contains(ConcurrencyException)) { - Logger.LogError(e, Messages.GenericPersistingError, attempt.CommitId, checkpointId, attempt.BucketId, attempt.StreamId, e); + MongoPersistenceEngineLogMessages.GenericPersistingError(Logger, attempt.CommitId, checkpointId, attempt.BucketId, attempt.StreamId, e.ToString(), e); throw; } @@ -295,10 +273,7 @@ public Task GetFromToAsync(long fromCheckpointToken, long toCheckpointToken, IAs if (e.Message.Contains(MongoCommitIndexes.CheckpointNumberMMApV1) || e.Message.Contains(MongoCommitIndexes.CheckpointNumberWiredTiger)) { - if (Logger.IsEnabled(LogLevel.Warning)) - { - Logger.LogWarning(e, Messages.DuplicatedCheckpointTokenError, attempt.CommitId, checkpointId, attempt.BucketId, attempt.StreamId); - } + MongoPersistenceEngineLogMessages.DuplicatedCheckpointTokenError(Logger, attempt.CommitId, checkpointId, attempt.BucketId, attempt.StreamId, e); await _checkpointGenerator.SignalDuplicateIdAsync(checkpointId, cancellationToken); checkpointId = await _checkpointGenerator.NextAsync(cancellationToken).ConfigureAwait(false); commitDoc[MongoCommitFields.CheckpointNumber] = checkpointId; @@ -312,11 +287,8 @@ public Task GetFromToAsync(long fromCheckpointToken, long toCheckpointToken, IAs if (e.Message.Contains(MongoCommitIndexes.CommitId)) { - var msg = String.Format(Messages.DuplicatedCommitError, attempt.CommitId, checkpointId, attempt.BucketId, attempt.StreamId); - if (Logger.IsEnabled(LogLevel.Information)) - { - Logger.LogInformation(msg); - } + var msg = string.Format(CultureInfo.InvariantCulture, MongoPersistenceEngineLogMessages.DuplicatedCommitErrorTemplate, attempt.CommitId, checkpointId, attempt.BucketId, attempt.StreamId); + MongoPersistenceEngineLogMessages.Information(Logger, msg); throw new DuplicateCommitException(msg); } @@ -334,18 +306,12 @@ public Task GetFromToAsync(long fromCheckpointToken, long toCheckpointToken, IAs if (savedCommit != null && savedCommit.CommitId == attempt.CommitId) { - var msg = String.Format(Messages.DuplicatedCommitError, attempt.CommitId, checkpointId, attempt.BucketId, attempt.StreamId); - if (Logger.IsEnabled(LogLevel.Information)) - { - Logger.LogInformation(msg); - } + var msg = string.Format(CultureInfo.InvariantCulture, MongoPersistenceEngineLogMessages.DuplicatedCommitErrorTemplate, attempt.CommitId, checkpointId, attempt.BucketId, attempt.StreamId); + MongoPersistenceEngineLogMessages.Information(Logger, msg); throw new DuplicateCommitException(msg); } - if (Logger.IsEnabled(LogLevel.Information)) - { - Logger.LogInformation(Messages.ConcurrencyExceptionError, attempt.CommitId, checkpointId, attempt.BucketId, attempt.StreamId, e); - } + MongoPersistenceEngineLogMessages.ConcurrencyExceptionError(Logger, attempt.CommitId, checkpointId, attempt.BucketId, attempt.StreamId, e); throw new ConcurrencyException(); } } @@ -371,10 +337,7 @@ private async Task FillHoleAsync(CommitAttempt attempt, Int64 checkpointId, Canc } catch (Exception e) { - if (Logger.IsEnabled(LogLevel.Warning)) - { - Logger.LogWarning(e, Messages.FillHoleError, attempt.CommitId, checkpointId, attempt.BucketId, attempt.StreamId, e); - } + MongoPersistenceEngineLogMessages.FillHoleError(Logger, attempt.CommitId, checkpointId, attempt.BucketId, attempt.StreamId, e); } } @@ -383,14 +346,14 @@ public Task GetStreamsToSnapshotAsync(string bucketId, int maxThreshold, IAsyncO { CheckIfSnapshotEnabled(); - if (Logger.IsEnabled(LogLevel.Debug)) - { - Logger.LogDebug(Messages.GettingStreamsToSnapshot); - } + MongoPersistenceEngineLogMessages.GettingStreamsToSnapshot(Logger); return TryMongoAsync(async () => { - var query = Builders.Filter.Gte(MongoStreamHeadFields.Unsnapshotted, maxThreshold); + var query = Builders.Filter.And( + Builders.Filter.Eq(MongoStreamHeadFields.FullQualifiedBucketId, bucketId), + Builders.Filter.Gte(MongoStreamHeadFields.Unsnapshotted, maxThreshold) + ); using var cursor = await PersistedStreamHeads .Find(query) .Sort(Builders.Sort.Descending(MongoStreamHeadFields.Unsnapshotted)) @@ -419,10 +382,7 @@ public Task GetStreamsToSnapshotAsync(string bucketId, int maxThreshold, IAsyncO { CheckIfSnapshotEnabled(); - if (Logger.IsEnabled(LogLevel.Debug)) - { - Logger.LogDebug(Messages.GettingRevision, streamId, maxRevision); - } + MongoPersistenceEngineLogMessages.GettingRevision(Logger, streamId, maxRevision); return TryMongoAsync(async () => { @@ -430,7 +390,7 @@ public Task GetStreamsToSnapshotAsync(string bucketId, int maxThreshold, IAsyncO using var cursor = await PersistedSnapshots .Find(query) - .Sort(Builders.Sort.Descending(MongoSnapshotFields.Id)) + .Sort(SortByDescendingSnapshotRevision) .Limit(1) .ToCursorAsync(cancellationToken) .ConfigureAwait(false); @@ -450,10 +410,7 @@ public async Task AddSnapshotAsync(ISnapshot snapshot, CancellationToken c return false; } - if (Logger.IsEnabled(LogLevel.Debug)) - { - Logger.LogDebug(Messages.AddingSnapshot, snapshot.StreamId, snapshot.BucketId, snapshot.StreamRevision); - } + MongoPersistenceEngineLogMessages.AddingSnapshot(Logger, snapshot.StreamId, snapshot.BucketId, snapshot.StreamRevision); try { @@ -492,10 +449,7 @@ await PersistedStreamHeads.UpdateOneAsync( } catch (Exception e) { - if (Logger.IsEnabled(LogLevel.Warning)) - { - Logger.LogWarning(e, Messages.AddingSnapshotError, snapshot.StreamId, snapshot.BucketId, snapshot.StreamRevision, e); - } + MongoPersistenceEngineLogMessages.AddingSnapshotError(Logger, snapshot.StreamId, snapshot.BucketId, snapshot.StreamRevision, e); return false; } } @@ -503,10 +457,8 @@ await PersistedStreamHeads.UpdateOneAsync( /// public Task PurgeAsync(CancellationToken cancellationToken) { - if (Logger.IsEnabled(LogLevel.Warning)) - { - Logger.LogWarning(Messages.PurgingStorage); - } + MongoPersistenceEngineLogMessages.PurgingStorage(Logger); + return TryMongoAsync(async () => { await PersistedCommits.DeleteManyAsync(Builders.Filter.Empty, cancellationToken).ConfigureAwait(false); @@ -519,10 +471,8 @@ public Task PurgeAsync(CancellationToken cancellationToken) /// public Task PurgeAsync(string bucketId, CancellationToken cancellationToken) { - if (Logger.IsEnabled(LogLevel.Warning)) - { - Logger.LogWarning(Messages.PurgingBucket, bucketId); - } + MongoPersistenceEngineLogMessages.PurgingBucket(Logger, bucketId); + return TryMongoAsync(async () => { await PersistedStreamHeads.DeleteManyAsync(Builders.Filter.Eq(MongoStreamHeadFields.FullQualifiedBucketId, bucketId), cancellationToken).ConfigureAwait(false); @@ -535,10 +485,7 @@ public Task PurgeAsync(string bucketId, CancellationToken cancellationToken) /// public Task DeleteStreamAsync(string bucketId, string streamId, CancellationToken cancellationToken) { - if (Logger.IsEnabled(LogLevel.Warning)) - { - Logger.LogWarning(Messages.DeletingStream, streamId, bucketId); - } + MongoPersistenceEngineLogMessages.DeletingStream(Logger, streamId, bucketId); return TryMongoAsync(async () => { @@ -551,10 +498,10 @@ await PersistedStreamHeads.DeleteOneAsync( ).ConfigureAwait(false); await PersistedSnapshots.DeleteManyAsync( - Builders.Filter.Eq(MongoSnapshotFields.Id, new BsonDocument{ - {MongoSnapshotFields.BucketId, bucketId}, - {MongoSnapshotFields.StreamId, streamId} - }), + Builders.Filter.And( + Builders.Filter.Eq(MongoSnapshotFields.FullQualifiedBucketId, bucketId), + Builders.Filter.Eq(MongoSnapshotFields.FullQualifiedStreamId, streamId) + ), cancellationToken ).ConfigureAwait(false); @@ -589,16 +536,13 @@ await PersistedStreamHeads.UpdateOneAsync( } catch (OutOfMemoryException ex) { - Logger.LogError(ex, "OutOfMemoryException:"); + MongoPersistenceEngineLogMessages.OutOfMemoryException(Logger, ex); throw; } catch (Exception ex) { //It is safe to ignore transient exception updating stream head. - if (Logger.IsEnabled(LogLevel.Warning)) - { - Logger.LogWarning(ex, "Ignored Exception '{exception}' when upserting the stream head Bucket Id [{id}] StreamId[{streamId}].\n", ex.GetType().Name, bucketId, streamId); - } + MongoPersistenceEngineLogMessages.IgnoredStreamHeadUpsertException(Logger, bucketId, streamId, ex); } }); } @@ -647,31 +591,22 @@ protected virtual async Task TryMongoAsync(Func callbackAsync, Cancella } catch (MongoConnectionException e) { - if (Logger.IsEnabled(LogLevel.Warning)) - { - Logger.LogWarning(e, Messages.StorageUnavailable); - } + MongoPersistenceEngineLogMessages.StorageUnavailable(Logger, e); exception = ExceptionDispatchInfo.Capture(new StorageUnavailableException(e.Message, e)); } catch (MongoException e) { - Logger.LogError(e, Messages.StorageThrewException, e.GetType(), e.ToString()); + MongoPersistenceEngineLogMessages.StorageThrewException(Logger, e); exception = ExceptionDispatchInfo.Capture(new StorageException(e.Message, e)); } catch (TaskCanceledException ex) { - if (Logger.IsEnabled(LogLevel.Warning)) - { - Logger.LogWarning(ex, "Task was cancelled."); - } + MongoPersistenceEngineLogMessages.TaskWasCancelled(Logger, ex); asyncObserver?.OnCompletedAsync(cancellationToken); } catch (Exception e) { - if (Logger.IsEnabled(LogLevel.Error)) - { - Logger.LogError(e, Messages.StorageThrewException, e.GetType(), e.ToString()); - } + MongoPersistenceEngineLogMessages.StorageThrewException(Logger, e); exception = ExceptionDispatchInfo.Capture(e); } if (exception != null) diff --git a/src/NEventStore.Persistence.MongoDB/MongoPersistenceEngine.Logging.cs b/src/NEventStore.Persistence.MongoDB/MongoPersistenceEngine.Logging.cs new file mode 100644 index 0000000..6742336 --- /dev/null +++ b/src/NEventStore.Persistence.MongoDB/MongoPersistenceEngine.Logging.cs @@ -0,0 +1,94 @@ +using Microsoft.Extensions.Logging; + +namespace NEventStore.Persistence.MongoDB +{ + internal static partial class MongoPersistenceEngineLogMessages + { + internal const string DuplicatedCommitErrorTemplate = "[NEventStore.Persistence.MongoDB] Duplicated commitId {0} [{1}] - Bucket {2} - StreamId {3}."; + internal const string ConnectionNotFoundTemplate = "Could not find connection name '{0}' in the configuration file."; + + [LoggerMessage(EventId = 1000, Level = LogLevel.Debug, Message = "Initializing storage engine.")] + internal static partial void InitializingStorage(ILogger logger); + + [LoggerMessage(EventId = 1001, Level = LogLevel.Debug, Message = "Getting all commits for stream '{StreamId}' in bucket '{BucketId}' between revisions '{MinRevision}' and '{MaxRevision}'.")] + internal static partial void GettingAllCommitsBetween(ILogger logger, string streamId, string bucketId, int minRevision, int maxRevision); + + [LoggerMessage(EventId = 1002, Level = LogLevel.Debug, Message = "Getting all commits from '{Start}' forward from bucket '{BucketId}'.")] + internal static partial void GettingAllCommitsFrom(ILogger logger, DateTime start, string bucketId); + + [LoggerMessage(EventId = 1003, Level = LogLevel.Debug, Message = "Getting all commits from Bucket '{BucketId}' and checkpoint '{CheckpointToken}'.")] + internal static partial void GettingAllCommitsFromBucketAndCheckpoint(ILogger logger, string bucketId, long checkpointToken); + + [LoggerMessage(EventId = 1004, Level = LogLevel.Debug, Message = "Getting all commits from bucket '{BucketId}' from checkpoint '{FromCheckpointToken}' (excluded) up to '{ToCheckpointToken}' (included).")] + internal static partial void GettingCommitsFromBucketAndFromToCheckpoint(ILogger logger, string bucketId, long fromCheckpointToken, long toCheckpointToken); + + [LoggerMessage(EventId = 1005, Level = LogLevel.Debug, Message = "Getting all commits since checkpoint '{CheckpointToken}'.")] + internal static partial void GettingAllCommitsFromCheckpoint(ILogger logger, long checkpointToken); + + [LoggerMessage(EventId = 1006, Level = LogLevel.Debug, Message = "Getting all commits from checkpoint '{FromCheckpointToken}' (excluded) up to '{ToCheckpointToken}' (included).")] + internal static partial void GettingCommitsFromToCheckpoint(ILogger logger, long fromCheckpointToken, long toCheckpointToken); + + [LoggerMessage(EventId = 1007, Level = LogLevel.Debug, Message = "Getting all commits from '{Start}' to '{End}'.")] + internal static partial void GettingAllCommitsFromTo(ILogger logger, DateTime start, DateTime end); + + [LoggerMessage(EventId = 1008, Level = LogLevel.Debug, Message = "Attempting to commit {EventCount} events on stream '{StreamId}' at sequence {CommitSequence}.")] + internal static partial void AttemptingToCommit(ILogger logger, int eventCount, string streamId, int commitSequence); + + [LoggerMessage(EventId = 1009, Level = LogLevel.Debug, Message = "Commit '{CommitId}' persisted.")] + internal static partial void CommitPersisted(ILogger logger, Guid commitId); + + [LoggerMessage(EventId = 1010, Level = LogLevel.Error, Message = "Generic error persisting commit {CommitId} [{CheckpointId}] - Bucket {BucketId} - StreamId {StreamId} - Ex: {ExceptionText}.")] + internal static partial void GenericPersistingError(ILogger logger, Guid commitId, long checkpointId, string bucketId, string streamId, string exceptionText, Exception exception); + + [LoggerMessage(EventId = 1011, Level = LogLevel.Warning, Message = "Duplicated checkpoint Token commit {CommitId} [{CheckpointId}] - Bucket {BucketId} - StreamId {StreamId}.")] + internal static partial void DuplicatedCheckpointTokenError(ILogger logger, Guid commitId, long checkpointId, string bucketId, string streamId, Exception exception); + + [LoggerMessage(EventId = 1012, Level = LogLevel.Information, Message = "{Message}")] + internal static partial void Information(ILogger logger, string message); + + [LoggerMessage(EventId = 1013, Level = LogLevel.Information, Message = "Concurrency Exception commitId {CommitId} [{CheckpointId}] - Bucket {BucketId} - StreamId {StreamId}.")] + internal static partial void ConcurrencyExceptionError(ILogger logger, Guid commitId, long checkpointId, string bucketId, string streamId, Exception exception); + + [LoggerMessage(EventId = 1014, Level = LogLevel.Warning, Message = "Error filling hole commitId {CommitId} [{CheckpointId}] - Bucket {BucketId} - StreamId {StreamId}.")] + internal static partial void FillHoleError(ILogger logger, Guid commitId, long checkpointId, string bucketId, string streamId, Exception exception); + + [LoggerMessage(EventId = 1015, Level = LogLevel.Debug, Message = "Getting a list of streams to snapshot.")] + internal static partial void GettingStreamsToSnapshot(ILogger logger); + + [LoggerMessage(EventId = 1016, Level = LogLevel.Debug, Message = "Getting snapshot for stream '{StreamId}' on or before revision {MaxRevision}.")] + internal static partial void GettingRevision(ILogger logger, string streamId, int maxRevision); + + [LoggerMessage(EventId = 1017, Level = LogLevel.Debug, Message = "Adding snapshot to stream '{StreamId}' in bucket '{BucketId}' at position {StreamRevision}.")] + internal static partial void AddingSnapshot(ILogger logger, string streamId, string bucketId, int streamRevision); + + [LoggerMessage(EventId = 1018, Level = LogLevel.Warning, Message = "Error Adding snapshot to stream '{StreamId}' in bucket '{BucketId}' at position {StreamRevision}.")] + internal static partial void AddingSnapshotError(ILogger logger, string streamId, string bucketId, int streamRevision, Exception exception); + + [LoggerMessage(EventId = 1019, Level = LogLevel.Warning, Message = "Purging all stored data.")] + internal static partial void PurgingStorage(ILogger logger); + + [LoggerMessage(EventId = 1020, Level = LogLevel.Warning, Message = "Purging all stored data for bucket '{BucketId}'.")] + internal static partial void PurgingBucket(ILogger logger, string bucketId); + + [LoggerMessage(EventId = 1021, Level = LogLevel.Warning, Message = "Deleting stream '{StreamId}' from bucket '{BucketId}'.")] + internal static partial void DeletingStream(ILogger logger, string streamId, string bucketId); + + [LoggerMessage(EventId = 1022, Level = LogLevel.Debug, Message = "Shutting down persistence.")] + internal static partial void ShuttingDownPersistence(ILogger logger); + + [LoggerMessage(EventId = 1023, Level = LogLevel.Error, Message = "OutOfMemoryException:")] + internal static partial void OutOfMemoryException(ILogger logger, Exception exception); + + [LoggerMessage(EventId = 1024, Level = LogLevel.Warning, Message = "Ignored Exception when upserting the stream head Bucket Id [{BucketId}] StreamId[{StreamId}].\n")] + internal static partial void IgnoredStreamHeadUpsertException(ILogger logger, string bucketId, string streamId, Exception exception); + + [LoggerMessage(EventId = 1025, Level = LogLevel.Warning, Message = "Storage is unavailabe.")] + internal static partial void StorageUnavailable(ILogger logger, Exception exception); + + [LoggerMessage(EventId = 1026, Level = LogLevel.Error, Message = "Storage threw exception.")] + internal static partial void StorageThrewException(ILogger logger, Exception exception); + + [LoggerMessage(EventId = 1027, Level = LogLevel.Warning, Message = "Task was cancelled.")] + internal static partial void TaskWasCancelled(ILogger logger, Exception exception); + } +} diff --git a/src/NEventStore.Persistence.MongoDB/MongoPersistenceEngine.cs b/src/NEventStore.Persistence.MongoDB/MongoPersistenceEngine.cs index 5ceface..c59e2c2 100644 --- a/src/NEventStore.Persistence.MongoDB/MongoPersistenceEngine.cs +++ b/src/NEventStore.Persistence.MongoDB/MongoPersistenceEngine.cs @@ -1,13 +1,13 @@ #pragma warning disable IDE0079 // Remove unnecessary suppression #pragma warning disable CA2254 // Template should be a static expression -using global::MongoDB.Bson; -using global::MongoDB.Driver; +using System.Globalization; using Microsoft.Extensions.Logging; +using MongoDB.Bson; +using MongoDB.Driver; using NEventStore.Logging; -using NEventStore.Serialization; using NEventStore.Persistence.MongoDB.Support; -using System.Runtime.ExceptionServices; +using NEventStore.Serialization; namespace NEventStore.Persistence.MongoDB { @@ -24,6 +24,8 @@ public partial class MongoPersistenceEngine : IPersistStreams private readonly string _systemBucketName; private ICheckpointGenerator? _checkpointGenerator; private static readonly SortDefinition SortByAscendingCheckpointNumber = Builders.Sort.Ascending(MongoCommitFields.CheckpointNumber); + private static readonly SortDefinition SortByAscendingStreamRevisionFrom = Builders.Sort.Ascending(MongoCommitFields.StreamRevisionFrom); + private static readonly SortDefinition SortByDescendingSnapshotRevision = Builders.Sort.Descending(MongoSnapshotFields.FullQualifiedStreamRevision); private readonly static UpdateOptions UpsertUpdateOptions = new() { IsUpsert = true }; /// @@ -82,10 +84,7 @@ public virtual void Initialize() return; } - if (Logger.IsEnabled(LogLevel.Debug)) - { - Logger.LogDebug(Messages.InitializingStorage); - } + MongoPersistenceEngineLogMessages.InitializingStorage(Logger); TryMongo(() => { @@ -159,6 +158,20 @@ public virtual void Initialize() if (!_options.DisableSnapshotSupport) { + PersistedSnapshots.Indexes.CreateOne( + new CreateIndexModel( + Builders.IndexKeys + .Ascending(MongoSnapshotFields.FullQualifiedBucketId) + .Ascending(MongoSnapshotFields.FullQualifiedStreamId) + .Descending(MongoSnapshotFields.FullQualifiedStreamRevision), + new CreateIndexOptions() + { + Name = MongoSnapshotIndexes.BucketStreamRevision, + Unique = false + } + ) + ); + PersistedStreamHeads.Indexes.CreateOne( new CreateIndexModel( Builders.IndexKeys @@ -170,6 +183,19 @@ public virtual void Initialize() } ) ); + + PersistedStreamHeads.Indexes.CreateOne( + new CreateIndexModel( + Builders.IndexKeys + .Ascending(MongoStreamHeadFields.FullQualifiedBucketId) + .Descending(MongoStreamHeadFields.Unsnapshotted), + new CreateIndexOptions() + { + Name = MongoStreamIndexes.BucketUnsnapshotted, + Unique = false + } + ) + ); } _checkpointGenerator = _options.CheckpointGenerator ?? @@ -182,10 +208,7 @@ public virtual void Initialize() /// public virtual IEnumerable GetFrom(string bucketId, string streamId, int minRevision, int maxRevision) { - if (Logger.IsEnabled(LogLevel.Debug)) - { - Logger.LogDebug(Messages.GettingAllCommitsBetween, streamId, bucketId, minRevision, maxRevision); - } + MongoPersistenceEngineLogMessages.GettingAllCommitsBetween(Logger, streamId, bucketId, minRevision, maxRevision); return TryMongo(() => { @@ -211,8 +234,7 @@ public virtual IEnumerable GetFrom(string bucketId, string streamId, in return PersistedCommits .Find(query) - // .Sort(Builders.Sort.Ascending(MongoCommitFields.StreamRevisionFrom)) - .Sort(SortByAscendingCheckpointNumber) + .Sort(SortByAscendingStreamRevisionFrom) .ToEnumerable() .Select(mc => mc.ToCommit(_serializer)); }); @@ -220,21 +242,18 @@ public virtual IEnumerable GetFrom(string bucketId, string streamId, in /// [Obsolete("DateTime is problematic in distributed systems. Use GetFrom(Int64 checkpointToken) instead. This method will be removed in a later version.")] - public virtual IEnumerable GetFrom(string bucketId, DateTime start) + public virtual IEnumerable GetFrom(string bucketId, DateTime startDate) { - if (Logger.IsEnabled(LogLevel.Debug)) - { - Logger.LogDebug(Messages.GettingAllCommitsFrom, start, bucketId); - } + MongoPersistenceEngineLogMessages.GettingAllCommitsFrom(Logger, startDate, bucketId); return TryMongo(() => { var query = Builders.Filter.Eq(MongoCommitFields.BucketId, bucketId); - if (start != DateTime.MinValue) + if (startDate != DateTime.MinValue) { query = Builders.Filter.And( query, - Builders.Filter.Gte(MongoCommitFields.CommitStamp, start) + Builders.Filter.Gte(MongoCommitFields.CommitStamp, startDate) ); } @@ -249,10 +268,7 @@ public virtual IEnumerable GetFrom(string bucketId, DateTime start) /// public virtual IEnumerable GetFrom(string bucketId, Int64 checkpointToken) { - if (Logger.IsEnabled(LogLevel.Debug)) - { - Logger.LogDebug(Messages.GettingAllCommitsFromBucketAndCheckpoint, bucketId, checkpointToken); - } + MongoPersistenceEngineLogMessages.GettingAllCommitsFromBucketAndCheckpoint(Logger, bucketId, checkpointToken); return TryMongo(() => { @@ -276,10 +292,7 @@ public virtual IEnumerable GetFrom(string bucketId, Int64 checkpointTok /// public virtual IEnumerable GetFromTo(string bucketId, long fromCheckpointToken, long toCheckpointToken) { - if (Logger.IsEnabled(LogLevel.Debug)) - { - Logger.LogDebug(Messages.GettingCommitsFromBucketAndFromToCheckpoint, bucketId, fromCheckpointToken, toCheckpointToken); - } + MongoPersistenceEngineLogMessages.GettingCommitsFromBucketAndFromToCheckpoint(Logger, bucketId, fromCheckpointToken, toCheckpointToken); return TryMongo(() => { @@ -313,10 +326,7 @@ public virtual IEnumerable GetFromTo(string bucketId, long fromCheckpoi /// public virtual IEnumerable GetFrom(Int64 checkpointToken) { - if (Logger.IsEnabled(LogLevel.Debug)) - { - Logger.LogDebug(Messages.GettingAllCommitsFromCheckpoint, checkpointToken); - } + MongoPersistenceEngineLogMessages.GettingAllCommitsFromCheckpoint(Logger, checkpointToken); return TryMongo(() => { @@ -340,10 +350,7 @@ public virtual IEnumerable GetFrom(Int64 checkpointToken) /// public virtual IEnumerable GetFromTo(long fromCheckpointToken, long toCheckpointToken) { - if (Logger.IsEnabled(LogLevel.Debug)) - { - Logger.LogDebug(Messages.GettingCommitsFromToCheckpoint, fromCheckpointToken, toCheckpointToken); - } + MongoPersistenceEngineLogMessages.GettingCommitsFromToCheckpoint(Logger, fromCheckpointToken, toCheckpointToken); return TryMongo(() => { @@ -376,12 +383,9 @@ public virtual IEnumerable GetFromTo(long fromCheckpointToken, long toC /// [Obsolete("DateTime is problematic in distributed systems. Use GetFromTo(Int64 fromCheckpointToken, Int64 toCheckpointToken) instead. This method will be removed in a later version.")] - public virtual IEnumerable GetFromTo(string bucketId, DateTime start, DateTime end) + public virtual IEnumerable GetFromTo(string bucketId, DateTime startDate, DateTime endDate) { - if (Logger.IsEnabled(LogLevel.Debug)) - { - Logger.LogDebug(Messages.GettingAllCommitsFromTo, start, end, bucketId); - } + MongoPersistenceEngineLogMessages.GettingAllCommitsFromTo(Logger, startDate, endDate); return TryMongo(() => { @@ -389,16 +393,16 @@ public virtual IEnumerable GetFromTo(string bucketId, DateTime start, D { Builders.Filter.Eq(MongoCommitFields.BucketId, bucketId) }; - if (start > DateTime.MinValue) + if (startDate > DateTime.MinValue) { filters.Add( - Builders.Filter.Gte(MongoCommitFields.CommitStamp, start) + Builders.Filter.Gte(MongoCommitFields.CommitStamp, startDate) ); } - if (end < DateTime.MaxValue) + if (endDate < DateTime.MaxValue) { filters.Add( - Builders.Filter.Lt(MongoCommitFields.CommitStamp, end) + Builders.Filter.Lt(MongoCommitFields.CommitStamp, endDate) ); } @@ -415,10 +419,7 @@ public virtual IEnumerable GetFromTo(string bucketId, DateTime start, D /// public virtual ICommit? Commit(CommitAttempt attempt) { - if (Logger.IsEnabled(LogLevel.Debug)) - { - Logger.LogDebug(Messages.AttemptingToCommit, attempt.Events.Count, attempt.StreamId, attempt.CommitSequence); - } + MongoPersistenceEngineLogMessages.AttemptingToCommit(Logger, attempt.Events.Count, attempt.StreamId, attempt.CommitSequence); return TryMongo(() => { @@ -442,16 +443,13 @@ public virtual IEnumerable GetFromTo(string bucketId, DateTime start, D UpdateStreamHeadInBackgroundThread(attempt.BucketId, attempt.StreamId, attempt.StreamRevision, attempt.Events.Count); } - if (Logger.IsEnabled(LogLevel.Debug)) - { - Logger.LogDebug(Messages.CommitPersisted, attempt.CommitId); - } + MongoPersistenceEngineLogMessages.CommitPersisted(Logger, attempt.CommitId); } catch (MongoException e) { if (!e.Message.Contains(ConcurrencyException)) { - Logger.LogError(e, Messages.GenericPersistingError, attempt.CommitId, checkpointId, attempt.BucketId, attempt.StreamId, e); + MongoPersistenceEngineLogMessages.GenericPersistingError(Logger, attempt.CommitId, checkpointId, attempt.BucketId, attempt.StreamId, e.ToString(), e); throw; } @@ -459,10 +457,7 @@ public virtual IEnumerable GetFromTo(string bucketId, DateTime start, D if (e.Message.Contains(MongoCommitIndexes.CheckpointNumberMMApV1) || e.Message.Contains(MongoCommitIndexes.CheckpointNumberWiredTiger)) { - if (Logger.IsEnabled(LogLevel.Warning)) - { - Logger.LogWarning(e, Messages.DuplicatedCheckpointTokenError, attempt.CommitId, checkpointId, attempt.BucketId, attempt.StreamId); - } + MongoPersistenceEngineLogMessages.DuplicatedCheckpointTokenError(Logger, attempt.CommitId, checkpointId, attempt.BucketId, attempt.StreamId, e); _checkpointGenerator.SignalDuplicateId(checkpointId); checkpointId = _checkpointGenerator.Next(); commitDoc[MongoCommitFields.CheckpointNumber] = checkpointId; @@ -476,11 +471,8 @@ public virtual IEnumerable GetFromTo(string bucketId, DateTime start, D if (e.Message.Contains(MongoCommitIndexes.CommitId)) { - var msg = String.Format(Messages.DuplicatedCommitError, attempt.CommitId, checkpointId, attempt.BucketId, attempt.StreamId); - if (Logger.IsEnabled(LogLevel.Information)) - { - Logger.LogInformation(msg); - } + var msg = string.Format(CultureInfo.InvariantCulture, MongoPersistenceEngineLogMessages.DuplicatedCommitErrorTemplate, attempt.CommitId, checkpointId, attempt.BucketId, attempt.StreamId); + MongoPersistenceEngineLogMessages.Information(Logger, msg); throw new DuplicateCommitException(msg); } @@ -495,18 +487,12 @@ public virtual IEnumerable GetFromTo(string bucketId, DateTime start, D if (savedCommit != null && savedCommit.CommitId == attempt.CommitId) { - var msg = String.Format(Messages.DuplicatedCommitError, attempt.CommitId, checkpointId, attempt.BucketId, attempt.StreamId); - if (Logger.IsEnabled(LogLevel.Information)) - { - Logger.LogInformation(msg); - } + var msg = string.Format(CultureInfo.InvariantCulture, MongoPersistenceEngineLogMessages.DuplicatedCommitErrorTemplate, attempt.CommitId, checkpointId, attempt.BucketId, attempt.StreamId); + MongoPersistenceEngineLogMessages.Information(Logger, msg); throw new DuplicateCommitException(msg); } - if (Logger.IsEnabled(LogLevel.Information)) - { - Logger.LogInformation(Messages.ConcurrencyExceptionError, attempt.CommitId, checkpointId, attempt.BucketId, attempt.StreamId, e); - } + MongoPersistenceEngineLogMessages.ConcurrencyExceptionError(Logger, attempt.CommitId, checkpointId, attempt.BucketId, attempt.StreamId, e); throw new ConcurrencyException(); } } @@ -532,10 +518,7 @@ private void FillHole(CommitAttempt attempt, Int64 checkpointId) } catch (Exception e) { - if (Logger.IsEnabled(LogLevel.Warning)) - { - Logger.LogWarning(e, Messages.FillHoleError, attempt.CommitId, checkpointId, attempt.BucketId, attempt.StreamId, e); - } + MongoPersistenceEngineLogMessages.FillHoleError(Logger, attempt.CommitId, checkpointId, attempt.BucketId, attempt.StreamId, e); } } @@ -544,14 +527,14 @@ public virtual IEnumerable GetStreamsToSnapshot(string bucketId, in { CheckIfSnapshotEnabled(); - if (Logger.IsEnabled(LogLevel.Debug)) - { - Logger.LogDebug(Messages.GettingStreamsToSnapshot); - } + MongoPersistenceEngineLogMessages.GettingStreamsToSnapshot(Logger); var result = TryMongo(() => { - var query = Builders.Filter.Gte(MongoStreamHeadFields.Unsnapshotted, maxThreshold); + var query = Builders.Filter.And( + Builders.Filter.Eq(MongoStreamHeadFields.FullQualifiedBucketId, bucketId), + Builders.Filter.Gte(MongoStreamHeadFields.Unsnapshotted, maxThreshold) + ); return PersistedStreamHeads .Find(query) .Sort(Builders.Sort.Descending(MongoStreamHeadFields.Unsnapshotted)) @@ -566,10 +549,7 @@ public virtual IEnumerable GetStreamsToSnapshot(string bucketId, in { CheckIfSnapshotEnabled(); - if (Logger.IsEnabled(LogLevel.Debug)) - { - Logger.LogDebug(Messages.GettingRevision, streamId, maxRevision); - } + MongoPersistenceEngineLogMessages.GettingRevision(Logger, streamId, maxRevision); return TryMongo(() => { @@ -577,11 +557,10 @@ public virtual IEnumerable GetStreamsToSnapshot(string bucketId, in return PersistedSnapshots .Find(query) - .Sort(Builders.Sort.Descending(MongoSnapshotFields.Id)) + .Sort(SortByDescendingSnapshotRevision) .Limit(1) - .ToEnumerable() - .Select(mc => mc.ToSnapshot(_serializer)) - .FirstOrDefault(); + .FirstOrDefault() + ?.ToSnapshot(_serializer); }); } @@ -595,10 +574,7 @@ public virtual bool AddSnapshot(ISnapshot snapshot) return false; } - if (Logger.IsEnabled(LogLevel.Debug)) - { - Logger.LogDebug(Messages.AddingSnapshot, snapshot.StreamId, snapshot.BucketId, snapshot.StreamRevision); - } + MongoPersistenceEngineLogMessages.AddingSnapshot(Logger, snapshot.StreamId, snapshot.BucketId, snapshot.StreamRevision); try { @@ -632,10 +608,7 @@ public virtual bool AddSnapshot(ISnapshot snapshot) } catch (Exception e) { - if (Logger.IsEnabled(LogLevel.Warning)) - { - Logger.LogWarning(e, Messages.AddingSnapshotError, snapshot.StreamId, snapshot.BucketId, snapshot.StreamRevision, e); - } + MongoPersistenceEngineLogMessages.AddingSnapshotError(Logger, snapshot.StreamId, snapshot.BucketId, snapshot.StreamRevision, e); return false; } } @@ -643,10 +616,7 @@ public virtual bool AddSnapshot(ISnapshot snapshot) /// public virtual void Purge() { - if (Logger.IsEnabled(LogLevel.Warning)) - { - Logger.LogWarning(Messages.PurgingStorage); - } + MongoPersistenceEngineLogMessages.PurgingStorage(Logger); TryMongo(() => { PersistedCommits.DeleteMany(Builders.Filter.Empty); @@ -658,10 +628,7 @@ public virtual void Purge() /// public void Purge(string bucketId) { - if (Logger.IsEnabled(LogLevel.Warning)) - { - Logger.LogWarning(Messages.PurgingBucket, bucketId); - } + MongoPersistenceEngineLogMessages.PurgingBucket(Logger, bucketId); TryMongo(() => { PersistedStreamHeads.DeleteMany(Builders.Filter.Eq(MongoStreamHeadFields.FullQualifiedBucketId, bucketId)); @@ -679,10 +646,7 @@ public void Drop() /// public void DeleteStream(string bucketId, string streamId) { - if (Logger.IsEnabled(LogLevel.Warning)) - { - Logger.LogWarning(Messages.DeletingStream, streamId, bucketId); - } + MongoPersistenceEngineLogMessages.DeletingStream(Logger, streamId, bucketId); TryMongo(() => { @@ -694,10 +658,10 @@ public void DeleteStream(string bucketId, string streamId) ); PersistedSnapshots.DeleteMany( - Builders.Filter.Eq(MongoSnapshotFields.Id, new BsonDocument{ - {MongoSnapshotFields.BucketId, bucketId}, - {MongoSnapshotFields.StreamId, streamId} - }) + Builders.Filter.And( + Builders.Filter.Eq(MongoSnapshotFields.FullQualifiedBucketId, bucketId), + Builders.Filter.Eq(MongoSnapshotFields.FullQualifiedStreamId, streamId) + ) ); PersistedCommits.UpdateMany( @@ -721,10 +685,7 @@ protected virtual void Dispose(bool disposing) return; } - if (Logger.IsEnabled(LogLevel.Debug)) - { - Logger.LogDebug(Messages.ShuttingDownPersistence); - } + MongoPersistenceEngineLogMessages.ShuttingDownPersistence(Logger); IsDisposed = true; } @@ -746,16 +707,13 @@ private void UpdateStreamHeadInBackgroundThread(string bucketId, string streamId } catch (OutOfMemoryException ex) { - Logger.LogError(ex, "OutOfMemoryException:"); + MongoPersistenceEngineLogMessages.OutOfMemoryException(Logger, ex); throw; } catch (Exception ex) { //It is safe to ignore transient exception updating stream head. - if (Logger.IsEnabled(LogLevel.Warning)) - { - Logger.LogWarning(ex, "Ignored Exception '{exception}' when upserting the stream head Bucket Id [{id}] StreamId[{streamId}].\n", ex.GetType().Name, bucketId, streamId); - } + MongoPersistenceEngineLogMessages.IgnoredStreamHeadUpsertException(Logger, bucketId, streamId, ex); } }); } @@ -790,15 +748,12 @@ protected virtual void TryMongo(Action callback) } catch (MongoConnectionException e) { - if (Logger.IsEnabled(LogLevel.Warning)) - { - Logger.LogWarning(e, Messages.StorageUnavailable); - } + MongoPersistenceEngineLogMessages.StorageUnavailable(Logger, e); throw new StorageUnavailableException(e.Message, e); } catch (MongoException e) { - Logger.LogError(e, Messages.StorageThrewException, e.GetType(), e.ToString()); + MongoPersistenceEngineLogMessages.StorageThrewException(Logger, e); throw new StorageException(e.Message, e); } } diff --git a/src/NEventStore.Persistence.MongoDB/MongoPersistenceWireup.cs b/src/NEventStore.Persistence.MongoDB/MongoPersistenceWireup.cs index 564ea5e..0c205fd 100644 --- a/src/NEventStore.Persistence.MongoDB/MongoPersistenceWireup.cs +++ b/src/NEventStore.Persistence.MongoDB/MongoPersistenceWireup.cs @@ -1,4 +1,3 @@ -using System; using Microsoft.Extensions.Logging; using NEventStore.Logging; using NEventStore.Persistence.MongoDB; @@ -11,23 +10,29 @@ namespace NEventStore /// /// Represents the persistence wire-up for MongoDB. /// - public class MongoPersistenceWireup : PersistenceWireup + public partial class MongoPersistenceWireup : PersistenceWireup { private static readonly ILogger Logger = LogFactory.BuildLogger(typeof(MongoPersistenceWireup)); + [LoggerMessage(EventId = 1100, Level = LogLevel.Debug, Message = "Configuring Mongo persistence engine.")] + private static partial void ConfiguringMongoPersistenceEngineMessage(ILogger logger); + + [LoggerMessage(EventId = 1101, Level = LogLevel.Warning, Message = "MongoDB does not participate in transactions using TransactionScope.")] + private static partial void TransactionScopeWarningMessage(ILogger logger); + /// /// Initializes a new instance of the class. /// public MongoPersistenceWireup(Wireup inner, Func connectionStringProvider, IDocumentSerializer serializer, MongoPersistenceOptions? persistenceOptions) : base(inner) { - Logger.LogDebug("Configuring Mongo persistence engine."); + ConfiguringMongoPersistenceEngineMessage(Logger); /* Transaction will be handled differently by each driver var options = Container.Resolve(); if (options != TransactionScopeOption.Suppress) { - Logger.LogWarning("MongoDB does not participate in transactions using TransactionScope."); + TransactionScopeWarningMessage(Logger); } */ diff --git a/src/NEventStore.Persistence.MongoDB/MongoPersistenceWireupExtensions.cs b/src/NEventStore.Persistence.MongoDB/MongoPersistenceWireupExtensions.cs index a30aec2..34deb59 100644 --- a/src/NEventStore.Persistence.MongoDB/MongoPersistenceWireupExtensions.cs +++ b/src/NEventStore.Persistence.MongoDB/MongoPersistenceWireupExtensions.cs @@ -1,6 +1,7 @@ #if NET472_OR_GREATER using System.Configuration; #endif +using System.Globalization; using NEventStore.Persistence.MongoDB; using NEventStore.Serialization; @@ -24,7 +25,7 @@ public static PersistenceWireup UsingMongoPersistence(this Wireup wireup, string return new MongoPersistenceWireup(wireup, () => { var connectionStringSettings = ConfigurationManager.ConnectionStrings[connectionName] - ?? throw new NEventStore.Persistence.MongoDB.ConfigurationException(Messages.ConnectionNotFound.FormatWith(connectionName)); + ?? throw new NEventStore.Persistence.MongoDB.ConfigurationException(string.Format(CultureInfo.InvariantCulture, MongoPersistenceEngineLogMessages.ConnectionNotFoundTemplate, connectionName)); return connectionStringSettings.ConnectionString; }, serializer, options); } @@ -38,7 +39,7 @@ public static PersistenceWireup UsingMongoPersistence(this Wireup wireup, string return new MongoPersistenceWireup(wireup, () => { if (string.IsNullOrWhiteSpace(connectionString)) - throw new NEventStore.Persistence.MongoDB.ConfigurationException(Messages.ConnectionNotFound.FormatWith(connectionString)); + throw new NEventStore.Persistence.MongoDB.ConfigurationException(string.Format(CultureInfo.InvariantCulture, MongoPersistenceEngineLogMessages.ConnectionNotFoundTemplate, connectionString)); return connectionString; }, serializer, options); diff --git a/src/NEventStore.Persistence.MongoDB/NEventStore.Persistence.MongoDB.Core.csproj b/src/NEventStore.Persistence.MongoDB/NEventStore.Persistence.MongoDB.Core.csproj index 2092157..34e8373 100644 --- a/src/NEventStore.Persistence.MongoDB/NEventStore.Persistence.MongoDB.Core.csproj +++ b/src/NEventStore.Persistence.MongoDB/NEventStore.Persistence.MongoDB.Core.csproj @@ -1,7 +1,7 @@  - net6.0;netstandard2.1;net472 + netstandard2.1;net472;net6.0;$(ModernTargetFrameworks) false NEventStore.Persistence.MongoDB NEventStore.Persistence.MongoDB @@ -41,7 +41,7 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive @@ -55,20 +55,4 @@ - - - True - True - Messages.resx - - - - - - ResXFileCodeGenerator - Messages.Designer.cs - NEventStore.Persistence.MongoDB - - - \ No newline at end of file