fix: 🚑️ include issues and optimized query processing

CLAUDE.md

@@ -48,6 +48,11 @@ dotnet csharpier . --check
 ### Documentation
 Documentation is built using DocFX and deployed to GitHub Pages. The documentation source is in `/docs/` and the built site goes to `/docs/_site/`.
 
+### Debugging
+Enable detailed logging for query processing and troubleshooting:
+- Set `"JsonApiToolkit": "Debug"` in appsettings.json
+- See `/docs/docs/debugging.md` for comprehensive debugging guide
+
 ## Architecture
 
 ### Core Components
@@ -66,7 +71,12 @@ Documentation is built using DocFX and deployed to GitHub Pages. The documentati
 3. **Query Processing Pipeline** (`Extensions/Querying/`)
    - `JsonApiQueryParser`: Parses JSON:API query parameters
    - `FilterExpressionBuilder`: Builds LINQ expressions from filters
-   - `QueryableExtensions`: Extension methods for applying filters, sorting, pagination
+   - `FilterHandler`: Applies filter expressions to queryables
+   - `SortingHandler`: Applies sorting to queryables
+   - `PaginationHandler`: Applies pagination to queryables
+   - `QueryHelpers`: Helper methods for property name mapping and type conversion
+   - `IncludeFilterParser`: Separates filters targeting included resources from main entity filters
+   - `FilteredIncludeBuilder`: Applies filtered includes using EF Core's filtered Include functionality
 
 4. **Models** (`Models/`)
    - Document structures: `JsonApiDocument<T>`, `JsonApiCollectionDocument<T>`
@@ -79,19 +89,23 @@ Documentation is built using DocFX and deployed to GitHub Pages. The documentati
 
 6. **Validation** (`Validation/`)
    - `IncludePatternValidator`: Validates include patterns with wildcard support
+   - `IncludeValidator`: Validates include paths against entity relationships
+   - `IncludePattern`: Model representing validated include patterns
 
-7. **Include Filtering** (`Extensions/Querying/`)
-   - `IncludeFilterParser`: Separates filters targeting included resources from main entity filters
-   - `FilteredIncludeBuilder`: Applies filtered includes using EF Core's filtered Include functionality
-   - Enables filtering on relationships (e.g., `filter[author.name]=John` with `include=author`)
+7. **Services** (`Services/`)
+   - `IJsonApiQueryParser`: Interface for query parameter parsing
+   - `JsonApiQueryParserService`: Service implementation for parsing JSON:API query strings
+
+8. **Helpers** (`Helpers/`)
+   - `EfIncludePathHelper`: Utilities for building EF Core Include expressions
 
 ### Key Patterns
 
 - **Convention-based mapping**: Properties are automatically mapped from C# PascalCase to JSON camelCase
 - **Query parameter parsing**: Standard JSON:API query syntax (`filter[field]=value`, `sort=field,-field2`, `page[number]=1&page[size]=10`, `include=relationship`)
 - **Async-first**: Main controller method `JsonApiQueryAsync()` is async and works with `IQueryable<T>`
 - **Entity Framework integration**: Uses EF Core's `Include()` and query building capabilities
-- **Filter expressions**: Complex filtering with operators (eq, ne, gt, lt, contains, etc.), logical grouping, enum support, and filtering on included resources
+- **Filter expressions**: Complex filtering with operators (eq, ne, gt, lt, contains, etc.), logical grouping, enum support, and filtering on included resources via dot notation (e.g., `filter[author.name]=John` with `include=author`)
 - **JSON column detection**: Collections and complex objects without ID properties are automatically mapped as JSON attributes instead of relationships (useful for EF Core owned entities stored as JSON columns)
 - **Pagination safety**: Invalid page numbers are automatically clamped to valid ranges (page 1 for negative/zero, last page for overflow)
 - **Include whitelisting**: Use `AllowedIncludesAttribute` on controller actions to restrict which relationships can be included, preventing unauthorized data exposure
@@ -148,6 +162,7 @@ Tests are organized by component:
 - Entity types should have an `Id` property (auto-detected by `EntityMapper.GetIdProperty()`)
 - Use `QueryParameters queryParams = GetJsonApiQueryParameters()` to access parsed query parameters
 - For manual mapping, use `JsonApiMapper.ToDocument()` or `ToCollectionDocument()`
+- Enable debug logging with `"JsonApiToolkit": "Debug"` in appsettings.json for detailed query processing insights
 
 ## Package Publication
 

JsonApiToolkit.Tests/Controllers/JsonApiControllerTests.cs

@@ -2,9 +2,13 @@
 using JsonApiToolkit.Models.Documents;
 using JsonApiToolkit.Models.Errors;
 using JsonApiToolkit.Models.Resources;
+using JsonApiToolkit.Services;
 using JsonApiToolkit.Tests.Models;
 using Microsoft.AspNetCore.Http;
 using Microsoft.AspNetCore.Mvc;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging;
+using Moq;
 
 namespace JsonApiToolkit.Tests.Controllers;
 
@@ -42,9 +46,25 @@ public class JsonApiControllerTests
 
     public JsonApiControllerTests()
     {
+        var services = new ServiceCollection();
+        services.AddLogging();
+        services.AddScoped<IJsonApiQueryParser>(provider =>
+        {
+            var mock = new Mock<IJsonApiQueryParser>();
+            mock.Setup(x => x.Parse(It.IsAny<Microsoft.AspNetCore.Http.HttpRequest>()))
+                .Returns(
+                    new JsonApiToolkit.Models.Querying.QueryParameters
+                    {
+                        Include = new List<string>(),
+                    }
+                );
+            return mock.Object;
+        });
+
+        var serviceProvider = services.BuildServiceProvider();
         _controller = new TestJsonApiController();
 
-        var httpContext = new DefaultHttpContext();
+        var httpContext = new DefaultHttpContext { RequestServices = serviceProvider };
         httpContext.Request.Scheme = "https";
         httpContext.Request.Host = new HostString("api.example.com");
         httpContext.Request.Path = "/test-entities";

JsonApiToolkit.Tests/Extensions/IncludesWithPaginationTests.cs

@@ -0,0 +1,258 @@
+using JsonApiToolkit.Extensions;
+using Microsoft.EntityFrameworkCore;
+using Xunit;
+
+namespace JsonApiToolkit.Tests.Extensions;
+
+/// <summary>
+/// Tests for EF Core Include behavior with pagination to catch split query issues.
+/// </summary>
+public class IncludesWithPaginationTests
+{
+    private class TestEntity
+    {
+        public int Id { get; set; }
+        public string Name { get; set; } = string.Empty;
+        public List<RelatedEntity> Related { get; set; } = new();
+    }
+
+    private class RelatedEntity
+    {
+        public int Id { get; set; }
+        public string Value { get; set; } = string.Empty;
+        public int TestEntityId { get; set; }
+        public TestEntity? TestEntity { get; set; }
+    }
+
+    private class TestDbContext : DbContext
+    {
+        public DbSet<TestEntity> TestEntities { get; set; } = null!;
+        public DbSet<RelatedEntity> RelatedEntities { get; set; } = null!;
+
+        public TestDbContext(DbContextOptions<TestDbContext> options)
+            : base(options) { }
+
+        protected override void OnModelCreating(ModelBuilder modelBuilder)
+        {
+            modelBuilder.Entity<TestEntity>().HasMany(e => e.Related).WithOne(r => r.TestEntity);
+        }
+    }
+
+    private static TestDbContext CreateInMemoryContext()
+    {
+        var options = new DbContextOptionsBuilder<TestDbContext>()
+            .UseInMemoryDatabase(Guid.NewGuid().ToString())
+            .Options;
+
+        var context = new TestDbContext(options);
+
+        // Create a large dataset similar to the user's scenario
+        for (int i = 1; i <= 100; i++)
+        {
+            var entity = new TestEntity
+            {
+                Id = i,
+                Name = $"Entity {i}",
+                Related = new(),
+            };
+
+            // Add 2-5 related entities per main entity
+            var relatedCount = (i % 4) + 2;
+            for (int j = 1; j <= relatedCount; j++)
+            {
+                entity.Related.Add(
+                    new RelatedEntity
+                    {
+                        Id = i * 100 + j,
+                        Value = $"Related {i}-{j}",
+                        TestEntityId = i,
+                    }
+                );
+            }
+
+            context.TestEntities.Add(entity);
+        }
+
+        context.SaveChanges();
+        return context;
+    }
+
+    [Theory]
+    [InlineData(1)]
+    [InlineData(5)]
+    [InlineData(9)]
+    [InlineData(10)]
+    [InlineData(20)]
+    [InlineData(50)]
+    public async Task ApplyIncludesSingleQuery_WithPagination_LoadsAllRelatedEntitiesAsync(
+        int pageSize
+    )
+    {
+        // Arrange
+        using var context = CreateInMemoryContext();
+
+        // Act - Apply includes with AsSingleQuery and pagination
+        var query = context
+            .TestEntities.OrderBy(e => e.Id)
+            .ApplyIncludesSingleQuery(new List<string> { "Related" })
+            .Skip(0)
+            .Take(pageSize);
+
+        var results = await query.ToListAsync();
+
+        // Assert
+        Assert.NotEmpty(results);
+        Assert.Equal(pageSize, results.Count);
+
+        // Verify that ALL entities have their related entities loaded
+        foreach (var entity in results)
+        {
+            Assert.NotNull(entity.Related);
+            Assert.NotEmpty(entity.Related);
+            Assert.InRange(entity.Related.Count, 2, 5); // We created 2-5 related entities per entity
+        }
+    }
+
+    [Theory]
+    [InlineData(1)]
+    [InlineData(5)]
+    [InlineData(9)]
+    public async Task ApplyIncludes_WithPagination_MayFailOnSmallPageSizesAsync(int pageSize)
+    {
+        // Arrange
+        using var context = CreateInMemoryContext();
+
+        // Act - Regular includes (may exhibit split query issues)
+        var query = context
+            .TestEntities.OrderBy(e => e.Id)
+            .ApplyIncludes(new List<string> { "Related" })
+            .Skip(0)
+            .Take(pageSize);
+
+        var results = await query.ToListAsync();
+
+        // Assert - Just verify we got results, may or may not have includes loaded
+        Assert.NotEmpty(results);
+        Assert.Equal(pageSize, results.Count);
+
+        // Note: This test documents the issue - with regular ApplyIncludes,
+        // related entities may not be loaded consistently across page sizes
+    }
+
+    [Fact]
+    public async Task ApplyIncludesSingleQuery_WithLargeDatasetAndPaginationAtFirstPage_LoadsCorrectEntitiesAsync()
+    {
+        // Arrange
+        using var context = CreateInMemoryContext();
+
+        // Act - Get first page with page size 1 (the failing scenario from user's issue)
+        var results = await context
+            .TestEntities.OrderBy(e => e.Id)
+            .ApplyIncludesSingleQuery(new List<string> { "Related" })
+            .Skip(0)
+            .Take(1)
+            .ToListAsync();
+
+        // Assert
+        var entity = Assert.Single(results);
+        Assert.Equal(1, entity.Id);
+        Assert.NotEmpty(entity.Related);
+        Assert.All(entity.Related, r => Assert.Equal(1, r.TestEntityId));
+    }
+
+    [Fact]
+    public async Task ApplyIncludesSingleQuery_WithLargeDatasetAndPaginationAtMiddlePage_LoadsCorrectEntitiesAsync()
+    {
+        // Arrange
+        using var context = CreateInMemoryContext();
+
+        // Act - Get middle page (page 50) with page size 1
+        var results = await context
+            .TestEntities.OrderBy(e => e.Id)
+            .ApplyIncludesSingleQuery(new List<string> { "Related" })
+            .Skip(49)
+            .Take(1)
+            .ToListAsync();
+
+        // Assert
+        var entity = Assert.Single(results);
+        Assert.Equal(50, entity.Id);
+        Assert.NotEmpty(entity.Related);
+        Assert.All(entity.Related, r => Assert.Equal(50, r.TestEntityId));
+    }
+
+    [Fact]
+    public async Task ApplyIncludesSingleQuery_WithMultiplePagesSmallPageSize_EachPageHasCorrectIncludesAsync()
+    {
+        // Arrange
+        using var context = CreateInMemoryContext();
+        const int pageSize = 3;
+        const int totalPages = 5;
+
+        // Act & Assert - Verify each page has correct includes
+        for (int page = 0; page < totalPages; page++)
+        {
+            var results = await context
+                .TestEntities.OrderBy(e => e.Id)
+                .ApplyIncludesSingleQuery(new List<string> { "Related" })
+                .Skip(page * pageSize)
+                .Take(pageSize)
+                .ToListAsync();
+
+            Assert.Equal(pageSize, results.Count);
+
+            foreach (var entity in results)
+            {
+                Assert.NotEmpty(entity.Related);
+                // Verify the related entities belong to this entity
+                Assert.All(entity.Related, r => Assert.Equal(entity.Id, r.TestEntityId));
+            }
+        }
+    }
+
+    [Fact]
+    public async Task ApplyIncludesSingleQuery_WithNullIncludePaths_ReturnsQueryUnmodifiedAsync()
+    {
+        // Arrange
+        using var context = CreateInMemoryContext();
+
+        // Act
+        var query = context.TestEntities.ApplyIncludesSingleQuery(null);
+        var results = await query.ToListAsync();
+
+        // Assert
+        Assert.NotEmpty(results);
+    }
+
+    [Fact]
+    public async Task ApplyIncludesSingleQuery_WithEmptyIncludePaths_ReturnsQueryUnmodifiedAsync()
+    {
+        // Arrange
+        using var context = CreateInMemoryContext();
+
+        // Act
+        var query = context.TestEntities.ApplyIncludesSingleQuery(new List<string>());
+        var results = await query.ToListAsync();
+
+        // Assert
+        Assert.NotEmpty(results);
+    }
+
+    [Fact]
+    public async Task ApplyIncludesSingleQuery_WithoutPagination_WorksCorrectlyAsync()
+    {
+        // Arrange
+        using var context = CreateInMemoryContext();
+
+        // Act
+        var results = await context
+            .TestEntities.OrderBy(e => e.Id)
+            .ApplyIncludesSingleQuery(new List<string> { "Related" })
+            .Take(10)
+            .ToListAsync();
+
+        // Assert
+        Assert.Equal(10, results.Count);
+        Assert.All(results, e => Assert.NotEmpty(e.Related));
+    }
+}

JsonApiToolkit.Tests/Extensions/QueryHelpersTests.cs

@@ -76,7 +76,7 @@ public void ConvertToPropertyType_WithInvalidInt_ThrowsFormatException()
         );
 
         Assert.Contains(
-            "Failed to convert 'not-a-number' to type 'System.Int32'",
+            "Failed to convert filter value 'not-a-number' to type 'System.Int32'",
             exception.Message
         );
     }

JsonApiToolkit.Tests/Integration/AllowedIncludesIntegrationTests.cs

@@ -4,13 +4,15 @@
 using JsonApiToolkit.Controllers;
 using JsonApiToolkit.Extensions;
 using JsonApiToolkit.Models.Errors;
+using JsonApiToolkit.Services;
 using Microsoft.AspNetCore.Builder;
 using Microsoft.AspNetCore.Hosting;
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.AspNetCore.TestHost;
 using Microsoft.EntityFrameworkCore;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
 
 namespace JsonApiToolkit.Tests.Integration;