Adapts SM Repo QL

Author: FriedJannik

basyx.aasrepository/basyx.aasrepository-feature-search/README.md

@@ -47,6 +47,7 @@
 ## Risks
 
 * Errors can be catched in the decorated functions (transactional)
+* Security needs to be considered -> Just query id of elements and then go to the MongoDB
 
 
 ## TODOS
@@ -61,7 +62,10 @@
 * [x] Configurable Index Names
 * [x] Implement Casting Operators
 * [x] Unit Test
-* [x] !Priority! Duplicate search module for the missing components 
-* [ ] In SearchSubmodelRepository, ensure that the index does not disable certain fields in ensureIndexExists
+* [x] !Priority! Duplicate search module for the missing components
 * [ ] Integration Tests
-* [ ] Validate Expected Queries (unformatted ones) -> Depends on other components to utilize the AASQL
+* [ ] Validate Expected Queries (unformatted ones) -> Depends on other components to utilize the AASQL
+* [ ] Make the following SME Props queryable: value, valueType, semanticId, idShort
+* [ ] Make other Components (AAS,CD) also fetch Data from MongoDB not ES
+* [ ] Note down that comparison operators don't work for SME filtering in SM Repo without idShortPath
+* [ ] Handle Search Queries with SML (Indices)

basyx.common/basyx.querycore/src/main/java/org/eclipse/digitaltwin/basyx/querycore/query/converter/ElasticSearchRequestBuilder.java

@@ -69,68 +69,5 @@ public SearchRequest buildSearchRequest(AASQuery customQuery, String indexName,
 
         return searchBuilder.build();
     }
-    
-    /**
-     * Builds ElasticSearch SearchRequest with custom source includes
-     * 
-     * @param customQuery The custom query to convert
-     * @param indexName The ElasticSearch index name to search
-     * @param sourceIncludes List of fields to include in the response
-     * @return ElasticSearch SearchRequest
-     */
-    public SearchRequest buildSearchRequestWithSources(AASQuery customQuery, String indexName, List<String> sourceIncludes) {
-        co.elastic.clients.elasticsearch._types.query_dsl.Query esQuery = queryConverter.convert(customQuery);
-        
-        SearchRequest.Builder searchBuilder = new SearchRequest.Builder()
-            .index(indexName)
-            .query(esQuery);
-        
-        if (sourceIncludes != null && !sourceIncludes.isEmpty()) {
-            searchBuilder.source(SourceConfig.of(s -> s
-                .filter(SourceFilter.of(f -> f
-                    .includes(sourceIncludes)
-                ))
-            ));
-        }
-        
-        return searchBuilder.build();
-    }
-    
-    /**
-     * Builds ElasticSearch SearchRequest with custom source includes and excludes
-     * 
-     * @param customQuery The custom query to convert
-     * @param indexName The ElasticSearch index name to search
-     * @param sourceIncludes List of fields to include in the response
-     * @param sourceExcludes List of fields to exclude from the response
-     * @return ElasticSearch SearchRequest
-     */
-    public SearchRequest buildSearchRequestWithSourceFilters(AASQuery customQuery, String indexName,
-                                                             List<String> sourceIncludes, List<String> sourceExcludes) {
-        co.elastic.clients.elasticsearch._types.query_dsl.Query esQuery = queryConverter.convert(customQuery);
-        
-        SearchRequest.Builder searchBuilder = new SearchRequest.Builder()
-            .index(indexName)
-            .query(esQuery);
-        
-        if ((sourceIncludes != null && !sourceIncludes.isEmpty()) || 
-            (sourceExcludes != null && !sourceExcludes.isEmpty())) {
-            
-            SourceFilter.Builder filterBuilder = new SourceFilter.Builder();
-            
-            if (sourceIncludes != null && !sourceIncludes.isEmpty()) {
-                filterBuilder.includes(sourceIncludes);
-            }
-            
-            if (sourceExcludes != null && !sourceExcludes.isEmpty()) {
-                filterBuilder.excludes(sourceExcludes);
-            }
-            
-            searchBuilder.source(SourceConfig.of(s -> s
-                .filter(filterBuilder.build())
-            ));
-        }
-        
-        return searchBuilder.build();
-    }
-}
+
+}

basyx.common/basyx.querycore/src/main/java/org/eclipse/digitaltwin/basyx/querycore/query/converter/ValueConverter.java

@@ -396,9 +396,19 @@ private String convertModelFieldToElasticField(String modelField) {
         } else if (modelField.startsWith("$sm#")) {
             result = modelField.replace("$sm#", "");
         } else if (modelField.startsWith("$sme")) {
+            boolean hasIdShortPath = !result.startsWith("$sme#");
+            String idShortPath = "";
+            if (hasIdShortPath) {
+                int indexOfRaute = result.indexOf("#");
+                idShortPath = result.substring(5, indexOfRaute);
+            }
             result = modelField.replaceFirst("\\$sme(?:\\.[^#]*)?#", "");
             // Mark as SME field for wildcard handling
-            result = "SME_WILDCARD:" + result;
+            if (!hasIdShortPath) {
+                result = "SME_WILDCARD:" + result;
+            } else {
+                result = "submodelElements." + idShortPath+ "." + result;
+            }
         } else if (modelField.startsWith("$cd#")) {
             result = modelField.replace("$cd#", "");
         } else if (modelField.startsWith("$aasdesc#")) {
@@ -668,7 +678,7 @@ private Query createSmeWildcardQuery(String wildcardField, String value) {
 
         // Create a wildcard pattern that matches the field at any nesting level
         // Pattern: submodelElements.*{fieldName}:{value} OR submodelElements.*.smcChildren.*{fieldName}:{value}
-        String queryPattern = "submodelElements.*."+searchField;
+        String queryPattern = "*"+searchField;
 
         return QueryBuilders.queryString(q -> q
                 .query(value)
@@ -682,25 +692,20 @@ private Query createSmeWildcardQuery(String wildcardField, String value) {
     private Query createSmeWildcardStringQuery(String wildcardField, String value, String operation) {
         String fieldName = extractSmeFieldName(wildcardField);
 
-        // Add .keyword suffix for string fields that need exact matching
         String searchField = fieldName;
-        if (isStringField(fieldName)) {
-            searchField = fieldName + ".keyword";
-        }
-        
+
         String searchValue;
         switch (operation) {
             case "contains":
-                searchValue = "*" + escapeQueryString(value) + "*";
+                searchValue = "*" + value + "*";
                 break;
             case "starts-with":
-                searchValue = escapeQueryString(value) + "*";
+                searchValue = value + "*";
                 break;
             case "ends-with":
-                searchValue = "*" + escapeQueryString(value);
+                searchValue = "*" + value;
                 break;
             case "regex":
-                // For regex, use the value as-is (queryString supports regex)
                 searchValue = value;
                 break;
             default:
@@ -709,10 +714,10 @@ private Query createSmeWildcardStringQuery(String wildcardField, String value, S
         }
 
         // Create a wildcard pattern that matches the field at any nesting level
-        String queryPattern = "submodelElements.*."+searchField;
+        String queryPattern = "*"+searchField;
 
         return QueryBuilders.queryString(q -> q
-                .query(value)
+                .query(searchValue)
                 .fields(queryPattern)
         );
     }
@@ -723,12 +728,8 @@ private Query createSmeWildcardStringQuery(String wildcardField, String value, S
     private Query createSmeWildcardRangeQuery(String wildcardField, Object value, String operator) {
         String fieldName = extractSmeFieldName(wildcardField);
 
-        // Add .keyword suffix for string fields that need exact matching
         String searchField = fieldName;
-        if (isStringField(fieldName)) {
-            searchField = fieldName + ".keyword";
-        }
-        
+
         String rangeOperator;
         switch (operator) {
             case "gt": rangeOperator = ">"; break;
@@ -739,11 +740,13 @@ private Query createSmeWildcardRangeQuery(String wildcardField, Object value, St
         }
 
         // Create a wildcard pattern that matches the field at any nesting level with range comparison
-        String queryPattern = "submodelElements.*."+searchField;
+        String queryPattern = "*"+searchField;
+
+        // Construct the range query string with proper syntax: fieldPattern:(>=value)
+        String rangeQuery = queryPattern + ":(" + rangeOperator + value.toString() + ")";
 
         return QueryBuilders.queryString(q -> q
-                .query(value.toString())
-                .fields(queryPattern)
+                .query(rangeQuery)
         );
     }
 

basyx.common/basyx.querycore/src/main/java/org/eclipse/digitaltwin/basyx/querycore/query/executor/ESQueryExecutor.java

@@ -121,9 +121,8 @@ private Object filterEmptyArrays(Object obj) {
     }
 
     private QueryResponse getQueryResponse(AASQuery query, List<Object> objectHits, String nextCursor) {
-        QueryResult queryResult = new QueryResult(objectHits);
         QueryPaging queryPaging = new QueryPaging(nextCursor, getResultType(query));
-        QueryResponse queryResponse = new QueryResponse(queryPaging, queryResult);
+        QueryResponse queryResponse = new QueryResponse(queryPaging, objectHits);
         return queryResponse;
     }
 

basyx.common/basyx.querycore/src/main/java/org/eclipse/digitaltwin/basyx/querycore/query/model/QueryResponse.java

@@ -1,11 +1,13 @@
 package org.eclipse.digitaltwin.basyx.querycore.query.model;
 
+import java.util.List;
+
 public class QueryResponse {
 
         public QueryPaging paging_metadata;
-        public QueryResult result;
+        public List<Object> result;
 
-        public QueryResponse(QueryPaging paging_metadata, QueryResult result) {
+        public QueryResponse(QueryPaging paging_metadata, List<Object> result) {
             this.paging_metadata = paging_metadata;
             this.result = result;
         }

basyx.common/basyx.querycore/src/test/java/AASQueryConverterTest.java

@@ -52,6 +52,8 @@ public class AASQueryConverterTest {
         put("match_eq_eq_field_strval.json", "expected_match_eq_eq_field_strval.json");
         put("match_specific_asset_ids.json", "expected_match_specific_asset_ids.json");
         put("regex_field_strval.json", "expected_regex_field_strval.json");
+        put("starts-with_field_strval.json", "expected_starts-with_field_strval.json");
+        put("ends-with_field_strval.json", "expected_ends-with_field_strval.json");
     }};
 
     @Test

basyx.common/basyx.querycore/src/test/resources/input/ends-with_field_strval.json

@@ -0,0 +1,8 @@
+{
+  "$condition": {
+    "$starts-with": [
+      { "$field": "$aas#assetInformation.assetKind" },
+      { "$strVal": "NST" }
+    ]
+  }
+}

basyx.common/basyx.querycore/src/test/resources/input/starts-with_field_strval.json

@@ -0,0 +1,8 @@
+{
+  "$condition": {
+    "$starts-with": [
+      { "$field": "$aas#assetInformation.assetKind" },
+      { "$strVal": "NST" }
+    ]
+  }
+}

basyx.common/basyx.querycore/src/test/resources/output/expected_ends-with_field_strval.json

@@ -0,0 +1 @@
+{"wildcard":{"assetInformation.assetKind.keyword":{"value":"*NST"}}}

basyx.common/basyx.querycore/src/test/resources/output/expected_starts-with_field_strval.json

@@ -0,0 +1 @@
+{"wildcard":{"assetInformation.assetKind.keyword":{"value":"NST*"}}}