feat: parity with official Firecrawl .NET SDK + v2 endpoint coverage

Tier 1 — SDK-level fixes for parity with firecrawl/firecrawl dot-net-sdk:
- Fix terminal-state polling bug: WaitJobAsync/WaitBatchAsync now treat
  "cancelled" as terminal instead of spinning until timeout.
- Typed exception hierarchy (FirecrawlAuthenticationException 401,
  FirecrawlPaymentRequiredException 402, FirecrawlRateLimitException 429
  with Retry-After parsing, FirecrawlServerException 5xx) wired via
  ProcessResponse partial on every sub-client.
- FirecrawlClient.FromEnvironment() reads FIRECRAWL_API_KEY / FIRECRAWL_API_URL.
- IdempotencyKeyExtensions adds x-idempotency-key header overloads for
  CrawlUrlsAsync and batch ScrapeAndExtractFromUrlsAsync.
- Auto-pagination: CrawlingClient.PaginateAsync / WaitJobAndPaginateAsync
  and ScrapingClient.PaginateBatchAsync / WaitBatchAndPaginateAsync walk
  the next-URL chain with same-origin validation to prevent API-key
  exfiltration via hostile pagination pointers.

Tier 2 — hand-written v2 endpoint layer (FirecrawlClient.V2):
- Firecrawl has not published a v2 OpenAPI spec, so this layer is manual.
- POST /v2/parse (multipart file upload + ParseFile.FromPath/FromBytes
  with MIME-type guessing).
- GET /v2/concurrency-check.
- Monitor CRUD: POST/GET/PATCH/DELETE /v2/monitor + monitor/{id}/run +
  /checks + /checks/{id} (with auto-pagination of MonitorCheckDetail.Next).
- GET /v2/team/credit-usage/historical + /v2/team/token-usage/historical.
- Dedicated V2SourceGenerationContext for AOT-safe System.Text.Json.

Tier 3 — filed 6 generator improvement requests upstream at tryAGI/AutoSDK:
- #332 typed HTTP exception hierarchy
- #333 idempotency-key parameter on POST operations
- #334 extend --generate-pageable-helpers to follow absolute 'next' URLs
- #335 built-in retry handler with exponential backoff
- #336 first-class multipart file-upload helpers
- #337 polymorphic-array discriminator helpers (oneOf{string,object})

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

Author: HavenDV

src/libs/Firecrawl/CrawlClient.Paginate.cs

@@ -0,0 +1,67 @@
+namespace Firecrawl;
+
+public partial class CrawlingClient
+{
+    /// <summary>
+    /// Walks the <c>next</c>-URL chain on a crawl status response, appending
+    /// every page of <see cref="CrawlStatusResponseObj.Data"/> into the first
+    /// response and clearing <see cref="CrawlStatusResponseObj.Next"/> when
+    /// done. The supplied <paramref name="response"/> is mutated and returned.
+    /// </summary>
+    /// <remarks>
+    /// Firecrawl's <c>next</c> pointers are absolute URLs; we refuse to
+    /// forward the <c>Authorization</c> header across origins to avoid leaking
+    /// the API key.
+    /// </remarks>
+    public async Task<CrawlStatusResponseObj> PaginateAsync(
+        CrawlStatusResponseObj response,
+        CancellationToken cancellationToken = default)
+    {
+        ArgumentNullException.ThrowIfNull(response);
+
+        response.Data ??= new List<CrawlStatusResponseObjDataItem>();
+        var next = response.Next;
+
+        while (!string.IsNullOrEmpty(next))
+        {
+            cancellationToken.ThrowIfCancellationRequested();
+
+            var content = await PaginationHelper
+                .FetchNextPageJsonAsync(HttpClient, next, cancellationToken)
+                .ConfigureAwait(false);
+
+            var page = CrawlStatusResponseObj.FromJson(content, JsonSerializerContext)
+                ?? throw new InvalidOperationException("Pagination response deserialization returned null.");
+
+            if (page.Data is { Count: > 0 })
+            {
+                foreach (var item in page.Data)
+                {
+                    response.Data.Add(item);
+                }
+            }
+
+            next = page.Next;
+        }
+
+        response.Next = null;
+        return response;
+    }
+
+    /// <summary>
+    /// Convenience helper: waits for the job to finish and then paginates the
+    /// full result set in one call.
+    /// </summary>
+    public async Task<CrawlStatusResponseObj> WaitJobAndPaginateAsync(
+        string jobId,
+        TimeSpan? pollingInterval = null,
+        IProgress<CrawlStatusResponseObj>? progress = null,
+        TimeSpan? timeout = null,
+        CancellationToken cancellationToken = default)
+    {
+        var job = await WaitJobAsync(jobId, pollingInterval, progress, timeout, cancellationToken)
+            .ConfigureAwait(false);
+
+        return await PaginateAsync(job, cancellationToken).ConfigureAwait(false);
+    }
+}

src/libs/Firecrawl/CrawlClient.WaitJob.cs

@@ -30,7 +30,7 @@ public Task<CrawlStatusResponseObj> WaitJobAsync(
     {
         return PollingHelper.PollUntilAsync(
             fetchStatus: ct => GetCrawlStatusAsync(id: jobId, cancellationToken: ct),
-            isComplete: r => r.Status is "completed" or "failed",
+            isComplete: r => r.Status is "completed" or "failed" or "cancelled",
             jobDescription: $"Crawl job {jobId}",
             pollingInterval: pollingInterval,
             progress: progress,

src/libs/Firecrawl/Firecrawl.csproj

@@ -3,10 +3,26 @@
   <PropertyGroup>
     <TargetFramework>net10.0</TargetFramework>
   </PropertyGroup>
-  
+
   <PropertyGroup Label="Nuget">
     <Description>Generated C# SDK based on Firecrawl OpenAPI specification.</Description>
     <PackageTags>api;client;sdk;dotnet;swagger;openapi;specification;generated;nswag</PackageTags>
   </PropertyGroup>
 
+  <PropertyGroup Label="HandWrittenSuppressions">
+    <!--
+      Rules below are relaxed for hand-written DTO/extension code in this project.
+      Generated files in Generated/ already comply via AutoSDK's templates.
+        CS1591 — missing XML docs on public members of hand-written DTOs.
+        CA1032 — extra exception constructors not used by our throw sites.
+        CA1056 — Url string properties (Firecrawl returns plain strings, not Uri).
+        CA1308 — ToLowerInvariant used for file-extension matching (intentional).
+        CA1819 — byte[] property on ParseFile (necessary for multipart upload).
+        CA1822 — partial-method overrides that don't touch instance state.
+        CA2000 — StringContent/ByteArrayContent ownership transferred to MultipartFormDataContent.
+        CA2227 — DTO collection setters required for System.Text.Json source-gen round-trip.
+    -->
+    <NoWarn>$(NoWarn);CS1591;CA1032;CA1056;CA1308;CA1819;CA1822;CA2000;CA2227</NoWarn>
+  </PropertyGroup>
+
 </Project>

src/libs/Firecrawl/FirecrawlClient.Environment.cs

@@ -0,0 +1,47 @@
+namespace Firecrawl;
+
+public sealed partial class FirecrawlClient
+{
+    /// <summary>
+    /// Environment variable read by <see cref="FromEnvironment"/> for the API key.
+    /// </summary>
+    public const string ApiKeyEnvironmentVariable = "FIRECRAWL_API_KEY";
+
+    /// <summary>
+    /// Environment variable read by <see cref="FromEnvironment"/> for the base URL.
+    /// </summary>
+    public const string ApiUrlEnvironmentVariable = "FIRECRAWL_API_URL";
+
+    /// <summary>
+    /// Creates a <see cref="FirecrawlClient"/> from the
+    /// <c>FIRECRAWL_API_KEY</c> environment variable.
+    /// Falls back to <c>FIRECRAWL_API_URL</c> for the base URL when set.
+    /// </summary>
+    /// <exception cref="InvalidOperationException">
+    /// Thrown when <c>FIRECRAWL_API_KEY</c> is not set or empty.
+    /// </exception>
+    public static FirecrawlClient FromEnvironment(
+        System.Net.Http.HttpClient? httpClient = null,
+        bool disposeHttpClient = true)
+    {
+        var apiKey = Environment.GetEnvironmentVariable(ApiKeyEnvironmentVariable);
+        if (string.IsNullOrWhiteSpace(apiKey))
+        {
+            throw new InvalidOperationException(
+                $"API key not found. Set the {ApiKeyEnvironmentVariable} environment variable " +
+                "or use a constructor overload that accepts an apiKey argument.");
+        }
+
+        var baseUrlOverride = Environment.GetEnvironmentVariable(ApiUrlEnvironmentVariable);
+        var baseUri = string.IsNullOrWhiteSpace(baseUrlOverride)
+            ? null
+            : new Uri(baseUrlOverride, UriKind.Absolute);
+
+        return new FirecrawlClient(
+            apiKey: apiKey,
+            httpClient: httpClient,
+            baseUri: baseUri,
+            authorizations: null,
+            disposeHttpClient: disposeHttpClient);
+    }
+}

src/libs/Firecrawl/FirecrawlClient.Exceptions.cs

@@ -0,0 +1,57 @@
+using System.Net.Http;
+
+namespace Firecrawl;
+
+public sealed partial class FirecrawlClient
+{
+    partial void ProcessResponse(HttpClient client, HttpResponseMessage response)
+        => FirecrawlExceptionMapper.ThrowTypedFirecrawlException(response);
+}
+
+public partial class BillingClient
+{
+    partial void ProcessResponse(HttpClient client, HttpResponseMessage response)
+        => FirecrawlExceptionMapper.ThrowTypedFirecrawlException(response);
+}
+
+public partial class CrawlingClient
+{
+    partial void ProcessResponse(HttpClient client, HttpResponseMessage response)
+        => FirecrawlExceptionMapper.ThrowTypedFirecrawlException(response);
+}
+
+public partial class ExtractionClient
+{
+    partial void ProcessResponse(HttpClient client, HttpResponseMessage response)
+        => FirecrawlExceptionMapper.ThrowTypedFirecrawlException(response);
+}
+
+public partial class LLMsTxtClient
+{
+    partial void ProcessResponse(HttpClient client, HttpResponseMessage response)
+        => FirecrawlExceptionMapper.ThrowTypedFirecrawlException(response);
+}
+
+public partial class MappingClient
+{
+    partial void ProcessResponse(HttpClient client, HttpResponseMessage response)
+        => FirecrawlExceptionMapper.ThrowTypedFirecrawlException(response);
+}
+
+public partial class ResearchClient
+{
+    partial void ProcessResponse(HttpClient client, HttpResponseMessage response)
+        => FirecrawlExceptionMapper.ThrowTypedFirecrawlException(response);
+}
+
+public partial class ScrapingClient
+{
+    partial void ProcessResponse(HttpClient client, HttpResponseMessage response)
+        => FirecrawlExceptionMapper.ThrowTypedFirecrawlException(response);
+}
+
+public partial class SearchClient
+{
+    partial void ProcessResponse(HttpClient client, HttpResponseMessage response)
+        => FirecrawlExceptionMapper.ThrowTypedFirecrawlException(response);
+}

src/libs/Firecrawl/FirecrawlClient.V2.cs

@@ -0,0 +1,13 @@
+namespace Firecrawl;
+
+public sealed partial class FirecrawlClient
+{
+    /// <summary>
+    /// Entry point for endpoints exposed only by the Firecrawl v2 API
+    /// (parse, monitor, concurrency-check, historical usage).
+    /// The v2 surface is hand-written because Firecrawl has not yet
+    /// published an OpenAPI definition for v2; the generated client on this
+    /// class covers v1 only.
+    /// </summary>
+    public V2.V2Client V2 => new(HttpClient);
+}

src/libs/Firecrawl/FirecrawlExceptionMapper.cs

@@ -0,0 +1,56 @@
+using System.Net.Http;
+
+namespace Firecrawl;
+
+internal static class FirecrawlExceptionMapper
+{
+    /// <summary>
+    /// Promotes 401/402/429/5xx HTTP responses into typed
+    /// <see cref="FirecrawlAuthenticationException"/>,
+    /// <see cref="FirecrawlPaymentRequiredException"/>,
+    /// <see cref="FirecrawlRateLimitException"/>, and
+    /// <see cref="FirecrawlServerException"/> so callers can catch by intent
+    /// rather than branching on <see cref="ApiException.StatusCode"/>.
+    /// </summary>
+    internal static void ThrowTypedFirecrawlException(HttpResponseMessage response)
+    {
+        if (response.IsSuccessStatusCode)
+            return;
+
+        var status = response.StatusCode;
+        var reason = response.ReasonPhrase ?? status.ToString();
+
+        switch ((int)status)
+        {
+            case 401:
+                throw new FirecrawlAuthenticationException(reason);
+            case 402:
+                throw new FirecrawlPaymentRequiredException(reason);
+            case 429:
+                throw new FirecrawlRateLimitException(reason)
+                {
+                    RetryAfter = ParseRetryAfter(response),
+                };
+            case >= 500 and < 600:
+                throw new FirecrawlServerException(reason, status);
+        }
+    }
+
+    private static TimeSpan? ParseRetryAfter(HttpResponseMessage response)
+    {
+        var header = response.Headers.RetryAfter;
+        if (header is null)
+            return null;
+
+        if (header.Delta.HasValue)
+            return header.Delta.Value;
+
+        if (header.Date is { } date)
+        {
+            var remaining = date - DateTimeOffset.UtcNow;
+            return remaining > TimeSpan.Zero ? remaining : TimeSpan.Zero;
+        }
+
+        return null;
+    }
+}

src/libs/Firecrawl/FirecrawlExceptions.cs

@@ -0,0 +1,69 @@
+using System.Net;
+
+namespace Firecrawl;
+
+/// <summary>
+/// Thrown for HTTP 401 Unauthorized responses (invalid or missing API key).
+/// </summary>
+[Serializable]
+public class FirecrawlAuthenticationException : ApiException
+{
+    public FirecrawlAuthenticationException() { }
+
+    public FirecrawlAuthenticationException(string message)
+        : base(message, HttpStatusCode.Unauthorized) { }
+
+    public FirecrawlAuthenticationException(string message, Exception? innerException)
+        : base(message, innerException, HttpStatusCode.Unauthorized) { }
+}
+
+/// <summary>
+/// Thrown for HTTP 429 Too Many Requests responses.
+/// </summary>
+[Serializable]
+public class FirecrawlRateLimitException : ApiException
+{
+    /// <summary>
+    /// Value of the <c>Retry-After</c> response header, if present.
+    /// Delta-seconds or HTTP-date depending on the server.
+    /// </summary>
+    public TimeSpan? RetryAfter { get; init; }
+
+    public FirecrawlRateLimitException() { }
+
+    public FirecrawlRateLimitException(string message)
+        : base(message, HttpStatusCode.TooManyRequests) { }
+
+    public FirecrawlRateLimitException(string message, Exception? innerException)
+        : base(message, innerException, HttpStatusCode.TooManyRequests) { }
+}
+
+/// <summary>
+/// Thrown for HTTP 402 Payment Required responses (insufficient credits).
+/// </summary>
+[Serializable]
+public class FirecrawlPaymentRequiredException : ApiException
+{
+    public FirecrawlPaymentRequiredException() { }
+
+    public FirecrawlPaymentRequiredException(string message)
+        : base(message, HttpStatusCode.PaymentRequired) { }
+
+    public FirecrawlPaymentRequiredException(string message, Exception? innerException)
+        : base(message, innerException, HttpStatusCode.PaymentRequired) { }
+}
+
+/// <summary>
+/// Thrown for HTTP 5xx Server Error responses.
+/// </summary>
+[Serializable]
+public class FirecrawlServerException : ApiException
+{
+    public FirecrawlServerException() { }
+
+    public FirecrawlServerException(string message, HttpStatusCode statusCode)
+        : base(message, statusCode) { }
+
+    public FirecrawlServerException(string message, Exception? innerException, HttpStatusCode statusCode)
+        : base(message, innerException, statusCode) { }
+}