feat: improved wql parrser and minor bugs

Author: ReneSchwarzer

src/WebExpress.WebIndex.Test/TestWqlExpressionNodeFilterFunctionConstant.cs

@@ -1,4 +1,5 @@
-using WebExpress.WebIndex.Wql.Function;
+using WebExpress.WebIndex.Wql;
+using WebExpress.WebIndex.Wql.Function;
 
 namespace WebExpress.WebIndex.Test
 {
@@ -8,6 +9,11 @@ namespace WebExpress.WebIndex.Test
     /// <typeparam name="UnitTestIndexTestDocumentA">The type of the index item.</typeparam>
     internal class TestWqlExpressionNodeFilterFunctionConstant<TIndexItem> : IWqlExpressionNodeFilterFunction<TIndexItem> where TIndexItem : IIndexItem
     {
+        /// <summary>
+        /// Returns the tokens associated with this syntax tree node.
+        /// </summary>
+        public IEnumerable<IWqlToken> Tokens { get; internal set; }
+
         // <summary>
         // Returns the name of the function.
         // </summary>

src/WebExpress.WebIndex.Test/WQL/UnitTestWqlWordSearchA.cs

@@ -1,4 +1,6 @@
-using WebExpress.WebIndex.Test.Fixture;
+using WebExpress.WebIndex.Test.Document;
+using WebExpress.WebIndex.Test.Fixture;
+using WebExpress.WebIndex.Wql;
 using Xunit.Abstractions;
 
 namespace WebExpress.WebIndex.Test.WQL
@@ -198,5 +200,59 @@ public void MultipleWords()
             Assert.Null(wql.Order);
             Assert.Null(wql.Partitioning);
         }
+
+        /// <summary>
+        /// Verifies that the abstract syntax tree (AST) is generated.
+        /// </summary>
+        [Fact]
+        public void AbstracrSyntaxTree()
+        {
+            // arrange
+            var wql = Fixture.ExecuteWql("text~'Helena'");
+
+            // act
+            var ast = wql.AbstractSyntaxTree;
+
+            // validation
+            Assert.NotNull(ast);
+        }
+
+        /// <summary>
+        /// Performs the incremental lookahead analysis.
+        /// </summary>
+        [Theory]
+        [InlineData("", true, 0, WqlExpressionType.None, WqlExpressionType.Attribute)]
+        [InlineData("text", false, 1, WqlExpressionType.Attribute, WqlExpressionType.Operator)]
+        [InlineData("text #", false, 2, WqlExpressionType.Operator, WqlExpressionType.Operator)]
+        [InlineData("text ~", false, 2, WqlExpressionType.Operator, WqlExpressionType.Parameter)]
+        [InlineData("text ~ (", false, 2, WqlExpressionType.Operator, WqlExpressionType.Parameter)]
+        [InlineData("text ~ )", false, 2, WqlExpressionType.Operator, WqlExpressionType.Parameter)]
+        [InlineData("text ~ and", false, 2, WqlExpressionType.Operator, WqlExpressionType.Parameter)]
+        [InlineData("text ~ or", false, 2, WqlExpressionType.Operator, WqlExpressionType.Parameter)]
+        [InlineData("text ~ &", false, 2, WqlExpressionType.Operator, WqlExpressionType.Parameter)]
+        [InlineData("text ~ ||", false, 2, WqlExpressionType.Operator, WqlExpressionType.Parameter)]
+        [InlineData("text ~ '", false, 3, WqlExpressionType.Parameter, WqlExpressionType.Parameter)]
+        [InlineData("text ~ \"", false, 3, WqlExpressionType.Parameter, WqlExpressionType.Parameter)]
+        [InlineData("text ~ Helena", true, 3, WqlExpressionType.Parameter, WqlExpressionType.LogicalOperator)]
+        [InlineData("text ~ 'Helena'", true, 3, WqlExpressionType.Parameter, WqlExpressionType.LogicalOperator)]
+        [InlineData("text ~ \"Helena\"", true, 3, WqlExpressionType.Parameter, WqlExpressionType.LogicalOperator)]
+        [InlineData("text ~ day(", false, 3, WqlExpressionType.Parameter, WqlExpressionType.LogicalOperator)]
+        [InlineData("text ~ day()", true, 3, WqlExpressionType.Parameter, WqlExpressionType.LogicalOperator)]
+        [InlineData("text ~ Helena and", false, 4, WqlExpressionType.LogicalOperator, WqlExpressionType.Attribute)]
+        public void Analyze(string wql, bool valid, int count, WqlExpressionType type, params WqlExpressionType[] expectedNextTokens)
+        {
+            // arrange
+            var parser = new WqlParser<UnitTestIndexTestDocumentA>();
+
+            // act
+            var ila = parser.Analyze(wql);
+
+            // validation
+            Assert.NotNull(ila);
+            Assert.True(ila.IsValidSoFar == valid);
+            Assert.Equal(type, ila.LastExpressionType);
+            Assert.Equal(count, ila.Items.Count());
+            Assert.Contains(expectedNextTokens, t => ila.ExpectedNextTokens.Contains(t));
+        }
     }
 }

src/WebExpress.WebIndex/Internationalization/de

@@ -1,2 +1,16 @@
 # Deutsch
-indexmanager.initialization=Der Index-Manager wurde initialisiert.
+indexmanager.initialization=Der Index-Manager wurde initialisiert.
+
+wql.unexpected_token=Unerwartetes Token.
+wql.condition_unknown=Unbekannte Bedingung.
+wql.expected_operator=Es wird ein Operator erwartet.
+wql.expected_binary_or_set_condition=Es wird eine binäre oder Mengenbedingung erwartet.
+wql.expected_logicaloperator=Es wird ein logischer Operator erwartet.
+wql.attribute_unknown=Unbekanntes Attribut.
+wql.expected_terminated_string_token=Es wird ein String-Token mit einem abschließenden Anführungszeichen erwartet.
+wql.expected_similarity=Es wird eine Ähnlichkeitsbedingung erwartet.
+wql.expected_distance=Es wird eine Distanzbedingung erwartet.
+wql.function_unknown=Unbekannte Funktion.
+wql.expected_skip_or_take=Es wird eine Skip- oder Take-Bedingung erwartet.
+wql.expected_token=Es wird ein Token erwartet.
+wql.expected_token_matching=Das Token entspricht nicht dem erwarteten Muster oder Format.

src/WebExpress.WebIndex/Internationalization/en

@@ -1,2 +1,16 @@
 # Englisch
-indexmanager.initialization=The index manager has been initialized.
+indexmanager.initialization=The index manager has been initialized.
+
+wql.unexpected_token=Unexpected token.
+wql.condition_unknown=Unknown condition.
+wql.expected_operator=An operator is expected.
+wql.expected_binary_or_set_condition=A binary or set condition is expected.
+wql.expected_logicaloperator=A logical operator is expected.
+wql.attribute_unknown=Unknown attribute.
+wql.expected_terminated_string_token=A string token with a terminating quotation mark is expected.
+wql.expected_similarity=A similarity condition is expected.
+wql.expected_distance=A distance condition is expected.
+wql.function_unknown=Unknown function.
+wql.expected_skip_or_take=A skip or take condition is expected.
+wql.expected_token=A token is expected.
+wql.expected_token_matching=The token does not match the expected pattern or format.

src/WebExpress.WebIndex/Wql/IWqlLookahead.cs

@@ -0,0 +1,33 @@
+using System.Collections.Generic;
+
+namespace WebExpress.WebIndex.Wql
+{
+    /// <summary>
+    /// Defines the contract for incremental lookahead analysis (ILA) in WQL queries.
+    /// </summary>
+    public interface IWqlLookahead
+    {
+        /// <summary>
+        /// Returns the list of lookahead items representing each token that was
+        /// successfully processed, including metadata such as token type.
+        /// </summary>
+        IEnumerable<IWqlLookaheadToken> Items { get; }
+
+        /// <summary>
+        /// Indicates whether the input is syntactically valid up to the current
+        /// position.
+        /// </summary>
+        bool IsValidSoFar { get; }
+
+        /// <summary>
+        /// Returns the type of the last token of the WQL expression.
+        /// </summary>
+        WqlExpressionType LastExpressionType { get; }
+
+        /// <summary>
+        /// Returns the set of tokens that would be syntactically valid at the
+        /// current position (lookahead).
+        /// </summary>
+        IEnumerable<WqlExpressionType> ExpectedNextTokens { get; }
+    }
+}

src/WebExpress.WebIndex/Wql/IWqlLookaheadToken.cs

@@ -0,0 +1,33 @@
+using System.Collections.Generic;
+
+namespace WebExpress.WebIndex.Wql
+{
+    /// <summary>
+    /// Defines an interface for a token that supports lookahead functionality during
+    /// WQL parsing.
+    /// </summary>
+    public interface IWqlLookaheadToken
+    {
+        /// <summary>
+        /// Returns the original token as produced by the tokenizer.
+        /// </summary>
+        IWqlToken Token { get; }
+
+        /// <summary>
+        /// Returns the type of the WQL expression represented by this instance.
+        /// </summary>
+        WqlExpressionType ExpreesionType { get; }
+
+        /// <summary>
+        /// Returns the set of tokens that would be syntactically valid at the
+        /// current position (lookahead).
+        /// </summary>
+        IEnumerable<WqlExpressionType> ExpectedNextTokens { get; }
+
+        /// <summary>
+        /// Indicates whether the input is syntactically valid up to the current
+        /// position.
+        /// </summary>
+        bool IsValid { get; }
+    }
+}

src/WebExpress.WebIndex/Wql/IWqlStatement.cs

@@ -1,5 +1,4 @@
-using System.Collections.Generic;
-using System.Globalization;
+using System.Globalization;
 using System.Linq;
 using WebExpress.WebIndex.Queries;
 
@@ -50,7 +49,7 @@ public interface IWqlStatement<TIndexItem>
         /// <summary>
         /// Returns the syntax tree of the wql query.
         /// </summary>
-        IEnumerable<IWqlExpressionNode<TIndexItem>> AbstractSyntaxTree { get; }
+        IWqlSyntaxTree<TIndexItem> AbstractSyntaxTree { get; }
 
         /// <summary>
         /// Applies the filter to the index.

src/WebExpress.WebIndex/Wql/IWqlSyntaxTree.cs

@@ -0,0 +1,25 @@
+namespace WebExpress.WebIndex.Wql
+{
+    /// <summary>
+    /// Represents the syntax tree for a WQL query and provides access to the root nodes.
+    /// </summary>
+    /// <typeparam name="TIndexItem">The index item type for the syntax tree nodes.</typeparam>
+    public interface IWqlSyntaxTree<TIndexItem>
+        where TIndexItem : IIndexItem
+    {
+        /// <summary>
+        /// Returns the filter node of the syntax tree.
+        /// </summary>
+        IWqlExpressionNode<TIndexItem> Filter { get; }
+
+        /// <summary>
+        /// Returns the order node of the syntax tree.
+        /// </summary>
+        IWqlExpressionNode<TIndexItem> Order { get; }
+
+        /// <summary>
+        /// Returns the partitioning node of the syntax tree.
+        /// </summary>
+        IWqlExpressionNode<TIndexItem> Partitioning { get; }
+    }
+}

src/WebExpress.WebIndex/Wql/IWqlToken.cs

@@ -0,0 +1,29 @@
+namespace WebExpress.WebIndex.Wql
+{
+    /// <summary>
+    /// Represents a token of the wql syntax.
+    /// </summary>
+    public interface IWqlToken
+    {
+        /// <summary>
+        /// Returns the starting position of the token in the raw statement.
+        /// </summary>
+        int Offset { get; }
+
+        /// <summary>
+        /// Returns the length of the token (start + length = end) in the raw statement.
+        /// </summary>
+        int Length { get; }
+
+        /// <summary>
+        /// Checks if the token is empty.
+        /// </summary>
+        /// <returns>True if no value is stored, false otherwise.</returns>
+        bool IsEmpty { get; }
+
+        /// <summary>
+        /// Returns the token value.
+        /// </summary>
+        string Value { get; }
+    }
+}

src/WebExpress.WebIndex/Wql/WqlExpressionNodeAttribute.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Generic;
 using System.Linq.Expressions;
 using System.Reflection;
 
@@ -11,6 +12,11 @@ namespace WebExpress.WebIndex.Wql
     public class WqlExpressionNodeAttribute<TIndexItem> : IWqlExpressionNode<TIndexItem>
         where TIndexItem : IIndexItem
     {
+        /// <summary>
+        /// Returns the tokens associated with this syntax tree node.
+        /// </summary>
+        public IEnumerable<IWqlToken> Tokens { get; internal set; }
+
         /// <summary>
         /// Returns the name of the attribute.
         /// </summary>
@@ -47,11 +53,11 @@ internal WqlExpressionNodeAttribute()
         /// <exception cref="InvalidOperationException"> 
         /// Thrown when no <see cref="PropertyInfo"/> is assigned to this attribute node. 
         /// </exception>
-        public Expression ToExpression(ParameterExpression param) 
-        { 
-            if (Property is null) 
+        public Expression ToExpression(ParameterExpression param)
+        {
+            if (Property is null)
             {
-                throw new InvalidOperationException( $"Attribute '{Name}' has no PropertyInfo assigned."); 
+                throw new InvalidOperationException($"Attribute '{Name}' has no PropertyInfo assigned.");
             }
 
             return Expression.Property(param, Property);