Skip to content

Commit 6ee8ddf

Browse files
mjabascal10mjabascal10
authored
Merge pull request #1489 from utmstack/backlog/add-sql-query-support-to-logexplorer-via-opensearch
Backlog/add sql query support to logexplorer via opensearch
2 parents b68689e + 78de7fb commit 6ee8ddf

21 files changed

Lines changed: 901 additions & 52 deletions

File tree

‎backend/pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -278,7 +278,7 @@
278278
<dependency>
279279
<groupId>com.utmstack</groupId>
280280
<artifactId>opensearch-connector</artifactId>
281-
<version>1.0.2</version>
281+
<version>1.0.3</version>
282282
</dependency>
283283
<dependency>
284284
<groupId>jakarta.json</groupId>

‎backend/src/main/java/com/park/utmstack/service/dto/elastic/SqlSearchDto.java‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
package com.park.utmstack.service.dto.elastic;
2+
3+
import com.park.utmstack.validation.elasticsearch.SqlSelectOnly;
4+
import lombok.AllArgsConstructor;
5+
import lombok.Data;
6+
import lombok.NoArgsConstructor;
7+
8+
import javax.validation.constraints.NotNull;
9+
10+
@Data
11+
@AllArgsConstructor
12+
@NoArgsConstructor
13+
public class SqlSearchDto {
14+
15+
@SqlSelectOnly
16+
private String query;
17+
18+
/**
19+
* Specifies the maximum number of results to fetch per query execution.
20+
* Acceptable values are positive integers; if null or not set, the default fetch size will be used.
21+
* This parameter can be used to limit the number of records returned by the SQL query.
22+
*/
23+
private Integer fetchSize;
24+
}

‎backend/src/main/java/com/park/utmstack/service/elasticsearch/ElasticsearchService.java‎

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@
55
import com.park.utmstack.domain.UtmSpaceNotificationControl;
66
import com.park.utmstack.domain.application_events.enums.ApplicationEventType;
77
import com.park.utmstack.domain.chart_builder.types.query.FilterType;
8-
import com.park.utmstack.domain.chart_builder.types.query.OperatorType;
98
import com.park.utmstack.domain.index_pattern.enums.SystemIndexPattern;
109
import com.park.utmstack.repository.UserRepository;
1110
import com.park.utmstack.service.MailService;
@@ -19,11 +18,10 @@
1918
import com.utmstack.opensearch_connector.exceptions.OpenSearchException;
2019
import com.utmstack.opensearch_connector.types.ElasticCluster;
2120
import com.utmstack.opensearch_connector.types.IndexSort;
22-
import org.elasticsearch.index.query.BoolQueryBuilder;
23-
import org.opensearch.client.json.JsonData;
21+
import com.utmstack.opensearch_connector.types.SearchSqlResponse;
22+
import com.utmstack.opensearch_connector.types.SqlQueryRequest;
2423
import org.opensearch.client.opensearch._types.SortOrder;
2524
import org.opensearch.client.opensearch._types.query_dsl.Query;
26-
import org.opensearch.client.opensearch.cat.CountResponse;
2725
import org.opensearch.client.opensearch.cat.indices.IndicesRecord;
2826
import org.opensearch.client.opensearch.core.*;
2927
import org.slf4j.Logger;
@@ -396,4 +394,13 @@ private SearchRequest buildQuery(String pattern, List<FilterType> filters, Integ
396394
throw new UtmElasticsearchException(ctx + ": " + e.getMessage());
397395
}
398396
}
399-
}
397+
398+
public <T> SearchSqlResponse<T> searchBySql(SqlQueryRequest request, Class<T> responseType) {
399+
final String ctx = CLASSNAME + ".searchBySql";
400+
try {
401+
return client.getClient().searchBySqlQuery(request, responseType);
402+
} catch (Exception e) {
403+
throw new RuntimeException(ctx + ": " + e.getMessage());
404+
}
405+
}
406+
}

‎backend/src/main/java/com/park/utmstack/util/elastic/SqlPaginationUtil.java‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
package com.park.utmstack.util.elastic;
2+
3+
import org.springframework.data.domain.Pageable;
4+
5+
public class SqlPaginationUtil {
6+
7+
public static String applyPagination(String query, Pageable pageable) {
8+
String upper = query.toUpperCase();
9+
10+
boolean hasLimit = upper.contains("LIMIT");
11+
boolean hasOffset = upper.contains("OFFSET");
12+
13+
if (hasLimit && hasOffset) {
14+
return query;
15+
} else if (hasLimit && !hasOffset) {
16+
int offset = pageable.getPageNumber() * pageable.getPageSize();
17+
return query + " OFFSET " + offset;
18+
} else if (!hasLimit && hasOffset) {
19+
int pageSize = pageable.getPageSize();
20+
return query + " LIMIT " + pageSize;
21+
} else {
22+
int pageSize = pageable.getPageSize();
23+
int offset = (pageable.getPageNumber() - 1) * pageSize;
24+
return query + " LIMIT " + pageSize + " OFFSET " + offset;
25+
}
26+
}
27+
}
28+

‎backend/src/main/java/com/park/utmstack/validation/elasticsearch/SqlSelectOnly.java‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
package com.park.utmstack.validation.elasticsearch;
2+
3+
import javax.validation.Constraint;
4+
import javax.validation.Payload;
5+
import java.lang.annotation.Documented;
6+
import java.lang.annotation.Retention;
7+
import java.lang.annotation.Target;
8+
9+
import static java.lang.annotation.ElementType.*;
10+
import static java.lang.annotation.RetentionPolicy.RUNTIME;
11+
12+
@Documented
13+
@Constraint(validatedBy = SqlSelectOnlyValidator.class)
14+
@Target({ FIELD, PARAMETER })
15+
@Retention(RUNTIME)
16+
public @interface SqlSelectOnly {
17+
String message() default "Only SELECT queries are allowed";
18+
Class<?>[] groups() default {};
19+
Class<? extends Payload>[] payload() default {};
20+
}

‎backend/src/main/java/com/park/utmstack/validation/elasticsearch/SqlSelectOnlyValidator.java‎

Lines changed: 171 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,171 @@
1+
package com.park.utmstack.validation.elasticsearch;
2+
3+
import javax.validation.ConstraintValidator;
4+
import javax.validation.ConstraintValidatorContext;
5+
import java.util.HashSet;
6+
import java.util.regex.Matcher;
7+
import java.util.regex.Pattern;
8+
import java.util.Set;
9+
10+
public class SqlSelectOnlyValidator implements ConstraintValidator<SqlSelectOnly, String> {
11+
12+
private static final Pattern START_PATTERN =
13+
Pattern.compile("(?is)^\\s*select\\b.*", Pattern.DOTALL);
14+
15+
private static final Pattern FORBIDDEN_PATTERN =
16+
Pattern.compile("(?is)\\b(insert|update|delete|drop|alter|create|replace|truncate|" +
17+
"merge|grant|revoke|exec|execute|commit|rollback|into)\\b");
18+
19+
private static final Pattern COMMENT_PATTERN =
20+
Pattern.compile("(?s)(--.*?$|/\\*.*?\\*/)", Pattern.MULTILINE);
21+
22+
private static final Set<String> ALLOWED_FUNCTIONS = Set.of(
23+
"COUNT", "AVG", "MIN", "MAX", "SUM"
24+
);
25+
26+
private static final Set<String> KEYWORDS_TO_IGNORE = Set.of(
27+
"AS"
28+
);
29+
30+
@Override
31+
public boolean isValid(String value, ConstraintValidatorContext context) {
32+
if (value == null || value.trim().isEmpty()) {
33+
return true;
34+
}
35+
36+
String query = value.trim().replaceAll(";+$", "").trim();
37+
String upper = query.toUpperCase();
38+
39+
if (!START_PATTERN.matcher(query).matches()) {
40+
return addConstraintViolation(context, "Query must start with SELECT.");
41+
}
42+
43+
if (FORBIDDEN_PATTERN.matcher(query).find()) {
44+
return addConstraintViolation(context, "Query contains forbidden SQL keywords.");
45+
}
46+
47+
if (COMMENT_PATTERN.matcher(query).find()) {
48+
return addConstraintViolation(context, "Query must not contain SQL comments (-- or /* */).");
49+
}
50+
51+
if (query.contains(";")) {
52+
return addConstraintViolation(context, "Query must not contain internal semicolons.");
53+
}
54+
55+
if (!isBalancedQuotes(query)) {
56+
return addConstraintViolation(context, "Quotes are not balanced.");
57+
}
58+
59+
if (!isBalancedParentheses(query)) {
60+
return addConstraintViolation(context, "Parentheses are not balanced.");
61+
}
62+
63+
if (query.toUpperCase().matches("(?i).*FROM\\s+(GROUP|WHERE|ORDER|$).*")) {
64+
return addConstraintViolation(context, "FROM clause must contain a valid index or pattern.");
65+
}
66+
67+
if (hasMisplacedCommas(query)) {
68+
return addConstraintViolation(context, "Query contains misplaced commas.");
69+
}
70+
71+
if (hasSubqueryWithoutAlias(query)) {
72+
return addConstraintViolation(context, "Subquery in FROM must have an alias.");
73+
}
74+
75+
for (String func : extractFunctions(upper)) {
76+
if (!ALLOWED_FUNCTIONS.contains(func)) {
77+
return addConstraintViolation(context, "Unsupported SQL function: " + func + ".");
78+
}
79+
}
80+
81+
if (upper.contains("HAVING") && !upper.contains("GROUP BY")) {
82+
return addConstraintViolation(context, "HAVING clause requires GROUP BY.");
83+
}
84+
85+
return true;
86+
}
87+
88+
private boolean addConstraintViolation(ConstraintValidatorContext context, String msg) {
89+
context.disableDefaultConstraintViolation();
90+
context.buildConstraintViolationWithTemplate(msg).addConstraintViolation();
91+
return false;
92+
}
93+
94+
private boolean isBalancedParentheses(String query) {
95+
int count = 0;
96+
for (char c : query.toCharArray()) {
97+
if (c == '(') count++;
98+
else if (c == ')') count--;
99+
if (count < 0) return false;
100+
}
101+
return count == 0;
102+
}
103+
104+
private boolean isBalancedQuotes(String query) {
105+
int sq = 0;
106+
int dq = 0;
107+
boolean escaped = false;
108+
109+
for (char c : query.toCharArray()) {
110+
if (escaped) { escaped = false; continue; }
111+
if (c == '\\') { escaped = true; continue; }
112+
if (c == '\'') sq++;
113+
else if (c == '"') dq++;
114+
}
115+
116+
return (sq % 2 == 0) && (dq % 2 == 0);
117+
}
118+
119+
private Set<String> extractFunctions(String upperQuery) {
120+
Pattern funcPattern = Pattern.compile("\\b(COUNT|AVG|MIN|MAX|SUM)\\s*\\(");
121+
Matcher matcher = funcPattern.matcher(upperQuery);
122+
123+
Set<String> funcs = new HashSet<>();
124+
while (matcher.find()) {
125+
String func = matcher.group(1);
126+
funcs.add(func);
127+
}
128+
return funcs;
129+
}
130+
131+
private boolean hasMisplacedCommas(String query) {
132+
String upperQuery = query.toUpperCase();
133+
134+
if (upperQuery.startsWith("SELECT ,") || upperQuery.contains(",,")) {
135+
return true;
136+
}
137+
138+
if (upperQuery.matches(".*\\,\\s*FROM.*")) {
139+
return true;
140+
}
141+
142+
String selectPart = query.replaceAll("(?i)^SELECT\\s+", "")
143+
.replaceAll("(?i)\\s+FROM.*", "")
144+
.trim();
145+
146+
if (selectPart.startsWith(",") || selectPart.endsWith(",")) {
147+
return true;
148+
}
149+
150+
String[] fields = selectPart.split(",");
151+
for (String f : fields) {
152+
if (f.trim().isEmpty()) {
153+
return true;
154+
}
155+
}
156+
157+
return false;
158+
}
159+
160+
private boolean hasSubqueryWithoutAlias(String query) {
161+
Pattern subqueryPattern = Pattern.compile("(?i)FROM\\s*\\([^)]*\\)");
162+
Matcher subqueryMatcher = subqueryPattern.matcher(query);
163+
if (!subqueryMatcher.find()) {
164+
return false;
165+
}
166+
167+
Pattern aliasPattern = Pattern.compile("(?i)FROM\\s*\\([^)]*\\)\\s+(AS\\s+\\w+|\\w+)");
168+
Matcher aliasMatcher = aliasPattern.matcher(query);
169+
return !aliasMatcher.find();
170+
}
171+
}

‎backend/src/main/java/com/park/utmstack/web/rest/elasticsearch/ElasticsearchResource.java‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
import com.park.utmstack.domain.chart_builder.types.query.OperatorType;
66
import com.park.utmstack.domain.shared_types.CsvExportingParams;
77
import com.park.utmstack.service.application_events.ApplicationEventService;
8+
import com.park.utmstack.service.dto.elastic.SqlSearchDto;
89
import com.park.utmstack.service.elasticsearch.ElasticsearchService;
910
import com.park.utmstack.service.elasticsearch.processor.SearchProcessorRegistry;
1011
import com.park.utmstack.service.elasticsearch.processor.SearchResultProcessor;
@@ -13,10 +14,13 @@
1314
import com.park.utmstack.util.ResponseUtil;
1415
import com.park.utmstack.util.chart_builder.IndexPropertyType;
1516
import com.park.utmstack.util.chart_builder.IndexType;
17+
import com.park.utmstack.util.elastic.SqlPaginationUtil;
1618
import com.park.utmstack.util.exceptions.OpenSearchIndexNotFoundException;
1719
import com.park.utmstack.web.rest.util.HeaderUtil;
1820
import com.park.utmstack.web.rest.util.PaginationUtil;
1921
import com.utmstack.opensearch_connector.types.ElasticCluster;
22+
import com.utmstack.opensearch_connector.types.SearchSqlResponse;
23+
import com.utmstack.opensearch_connector.types.SqlQueryRequest;
2024
import lombok.RequiredArgsConstructor;
2125
import org.opensearch.client.opensearch.cat.indices.IndicesRecord;
2226
import org.opensearch.client.opensearch.core.SearchResponse;
@@ -210,6 +214,40 @@ public ResponseEntity<Void> searchToCsv(@RequestBody @Valid CsvExportingParams p
210214
}
211215
}
212216

217+
@PostMapping("/search/sql")
218+
public ResponseEntity<List<Map>> searchBySql(@RequestBody @Valid SqlSearchDto request,
219+
Pageable pageable) {
220+
final String ctx = CLASSNAME + ".searchBySql";
221+
try {
222+
String sanitizedQuery = request.getQuery()
223+
.trim()
224+
.replaceAll(";+$", "")
225+
.trim();
226+
227+
String sqlQuery = SqlPaginationUtil.applyPagination(sanitizedQuery, pageable);
228+
229+
SearchSqlResponse<Map> response = elasticsearchService
230+
.searchBySql(new SqlQueryRequest(sqlQuery, null), Map.class);
231+
232+
String countQuery = "SELECT COUNT(*) FROM (" + sanitizedQuery + ") AS total_count";
233+
SearchSqlResponse<Map> countResponse = elasticsearchService
234+
.searchBySql(new SqlQueryRequest(countQuery, null), Map.class);
235+
236+
String countString = countResponse.getData().get(0).get("COUNT(*)").toString();
237+
int totalElements = (int) Double.parseDouble(countString);
238+
239+
HttpHeaders headers = UtilPagination.generatePaginationHttpHeaders((long) Math.min(totalElements, 10000),
240+
pageable.getPageNumber(), pageable.getPageSize(), "/api/elasticsearch/search");
241+
242+
return ResponseEntity.ok().headers(headers).body(response.getData());
243+
} catch (Exception e) {
244+
String msg = ctx + ": " + e.getMessage();
245+
log.error(msg);
246+
applicationEventService.createEvent(msg, ApplicationEventType.ERROR);
247+
return ResponseUtil.buildErrorResponse(HttpStatus.INTERNAL_SERVER_ERROR, msg);
248+
}
249+
}
250+
213251
@PostMapping("/generic-search")
214252
public ResponseEntity<List<Map>> genericSearch(@Valid @RequestBody GenericSearchBody body, Pageable pageable) {
215253
final String ctx = CLASSNAME + ".genericSearch";

‎frontend/angular.json‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,12 @@
2424
"tsConfig": "src/tsconfig.app.json",
2525
"assets": [
2626
"src/favicon.ico",
27-
"src/assets"
27+
"src/assets",
28+
{
29+
"glob": "**/*",
30+
"input": "node_modules/monaco-editor/min/vs",
31+
"output": "/assets/monaco/vs"
32+
}
2833
],
2934
"styles": [
3035
"node_modules/bootstrap/dist/css/bootstrap.min.css",

‎frontend/package-lock.json‎

Lines changed: 13 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)