Merge pull request #1606 from utmstack/backlog/extend-visualization-creation-flow-to-include-SQL

Backlog/extend visualization creation flow to include sql

Author: mjabascal10

backend/src/main/java/com/park/utmstack/domain/chart_builder/UtmVisualization.java

@@ -69,7 +69,6 @@ public class UtmVisualization implements Serializable {
     @Column(name = "system_owner")
     private Boolean systemOwner;
 
-    @NotNull
     @Column(name = "id_pattern")
     private Long idPattern;
 
@@ -104,7 +103,10 @@ public class UtmVisualization implements Serializable {
     @JsonDeserialize
     private AggregationType aggregationType;
 
-    @ManyToOne(fetch = FetchType.EAGER)
+    @Column(name = "sql_query", nullable = true)
+    private String sqlQuery;
+
+    @ManyToOne(fetch = FetchType.EAGER, optional = true)
     @JoinColumn(name = "id_pattern", referencedColumnName = "id", insertable = false, updatable = false)
     private UtmIndexPattern pattern;
 
@@ -196,6 +198,10 @@ public void setQuery(String query) {
     }
 
     public List<FilterType> getFilterType() throws UtmSerializationException {
+        if (_filters == null) {
+            return null;
+        }
+
         filterType = UtilSerializer.jsonDeserializeList(FilterType.class, _filters);
         return filterType;
     }
@@ -225,6 +231,14 @@ public void setAggregationType(AggregationType aggregationType) throws UtmSerial
         this.aggregationType = aggregationType;
     }
 
+    public String getSqlQuery() {
+        return sqlQuery;
+    }
+
+    public void setSqlQuery(String sqlQuery) {
+        this.sqlQuery = sqlQuery;
+    }
+
     public String getChartAction() {
         return chartAction;
     }

backend/src/main/java/com/park/utmstack/service/dto/elastic/SqlSearchDto.java

@@ -5,8 +5,6 @@
 import lombok.Data;
 import lombok.NoArgsConstructor;
 
-import javax.validation.constraints.NotNull;
-
 @Data
 @AllArgsConstructor
 @NoArgsConstructor

backend/src/main/java/com/park/utmstack/service/dto/visualization/UtmVisualizationDto.java

@@ -0,0 +1,41 @@
+package com.park.utmstack.service.dto.visualization;
+
+import com.park.utmstack.domain.chart_builder.types.ChartType;
+import com.park.utmstack.domain.chart_builder.types.aggregation.AggregationType;
+import com.park.utmstack.domain.chart_builder.types.query.FilterType;
+import com.park.utmstack.domain.index_pattern.UtmIndexPattern;
+import com.park.utmstack.service.dto.visualization.enums.QueryLanguageEnum;
+import com.park.utmstack.validation.elasticsearch.SqlSelectOnly;
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+import java.time.Instant;
+import java.util.List;
+
+@Data
+@NoArgsConstructor
+@AllArgsConstructor
+public class UtmVisualizationDto {
+    private Long id;
+    private String name;
+    private String description;
+    private String eventType;
+    private Instant createdDate;
+    private Instant modifiedDate;
+    private String userCreated;
+    private String userModified;
+    private String chartConfig;
+    private String chartAction;
+    private Boolean systemOwner;
+    private Long idPattern;
+    private UtmIndexPattern pattern;
+    private ChartType chartType;
+    private String query;
+    @SqlSelectOnly
+    private String sqlQuery;
+    private List<FilterType> filterType;
+    private AggregationType aggregationType;
+    private QueryLanguageEnum queryLanguage;
+}
+

backend/src/main/java/com/park/utmstack/service/dto/visualization/enums/QueryLanguageEnum.java

@@ -0,0 +1,6 @@
+package com.park.utmstack.service.dto.visualization.enums;
+
+public enum QueryLanguageEnum {
+    DSL,
+    SQL
+}

backend/src/main/java/com/park/utmstack/service/dto/visualization/mapper/UtmVisualizationMapper.java

@@ -0,0 +1,16 @@
+package com.park.utmstack.service.dto.visualization.mapper;
+
+import com.park.utmstack.domain.chart_builder.UtmVisualization;
+import com.park.utmstack.service.dto.visualization.UtmVisualizationDto;
+import com.park.utmstack.util.exceptions.UtmSerializationException;
+import org.mapstruct.Mapper;
+import org.mapstruct.factory.Mappers;
+
+@Mapper(componentModel = "spring")
+public interface UtmVisualizationMapper {
+    UtmVisualizationMapper INSTANCE = Mappers.getMapper(UtmVisualizationMapper.class);
+
+    UtmVisualizationDto toDto(UtmVisualization entity) throws UtmSerializationException;
+
+    UtmVisualization toEntity(UtmVisualizationDto dto) throws UtmSerializationException;
+}

backend/src/main/java/com/park/utmstack/util/chart_builder/elasticsearch_dsl/responses/ResponseParser.java

@@ -2,10 +2,16 @@
 
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.park.utmstack.domain.chart_builder.UtmVisualization;
+import com.utmstack.opensearch_connector.types.SearchSqlResponse;
 import org.opensearch.client.opensearch.core.SearchResponse;
 
 import java.util.List;
+import java.util.Map;
 
 public interface ResponseParser<T> {
     List<T> parse(UtmVisualization visualization, SearchResponse<ObjectNode> result);
+
+    default List<T> parse(UtmVisualization visualization, SearchSqlResponse<Map> result) {
+        return null;
+    }
 }

backend/src/main/java/com/park/utmstack/util/chart_builder/elasticsearch_dsl/responses/impl/bar_chart/ResponseParserForBarChart.java

@@ -10,6 +10,7 @@
 import com.utmstack.opensearch_connector.parsers.DateHistogramAggregateParser;
 import com.utmstack.opensearch_connector.parsers.TermAggregateParser;
 import com.utmstack.opensearch_connector.types.BucketAggregation;
+import com.utmstack.opensearch_connector.types.SearchSqlResponse;
 import org.opensearch.client.opensearch._types.aggregations.*;
 import org.opensearch.client.opensearch.core.SearchResponse;
 import org.springframework.util.CollectionUtils;
@@ -201,4 +202,49 @@ private void parseMetric(SearchResponse<?> result, List<Metric> metrics) {
             throw new RuntimeException(ctx + ": " + e.getLocalizedMessage());
         }
     }
+
+    @Override
+    public List<BarChartResult> parse(UtmVisualization visualization, SearchSqlResponse<Map> result) {
+        final String ctx = CLASSNAME + ".parse(SearchSqlResponse)";
+        try {
+            BarChartResult retValue = new BarChartResult();
+
+            BarChartResult.Serie serie = new BarChartResult.Serie();
+            serie.setMetricId("1");
+
+            for (Object rowObj : result.getData()) {
+                if (!(rowObj instanceof Map)) {
+                    continue;
+                }
+                Map<String, Object> row = (Map<String, Object>) rowObj;
+
+                String category = null;
+                Double value = 0.0;
+
+                for (Map.Entry<String, Object> entry : row.entrySet()) {
+                    Object val = entry.getValue();
+                    if (val instanceof Number) {
+                        value = ((Number) val).doubleValue();
+                        if (serie.getName() == null) {
+                            serie.setName(entry.getKey() != null ? entry.getKey()  : "metric");
+                        }
+                    } else {
+                        category = val != null ? val.toString() : "UNKNOWN";
+                    }
+                }
+
+                if (category != null) {
+                    retValue.addCategory(category);
+                    serie.addData(value);
+                }
+            }
+
+            retValue.addSerie(serie);
+
+            return Collections.singletonList(retValue);
+        } catch (Exception e) {
+            throw new RuntimeException(ctx + ": " + e.getMessage());
+        }
+    }
+
 }

backend/src/main/java/com/park/utmstack/util/chart_builder/elasticsearch_dsl/responses/impl/coordinate_map/ResponseParserForCoordinateMapChart.java

@@ -11,14 +11,17 @@
 import com.park.utmstack.util.exceptions.UtmIpInfoException;
 import com.utmstack.opensearch_connector.parsers.TermAggregateParser;
 import com.utmstack.opensearch_connector.types.BucketAggregation;
+import com.utmstack.opensearch_connector.types.SearchSqlResponse;
 import org.opensearch.client.opensearch.core.SearchResponse;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.stereotype.Component;
+import org.springframework.util.Assert;
 import org.springframework.util.StringUtils;
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Map;
 import java.util.stream.Collectors;
 
 @Component
@@ -110,5 +113,58 @@ public static boolean isValidIPv6(String ip) {
         return ip.matches(regex);
     }
 
+    @Override
+    public List<CoordinateMapChartResult> parse(UtmVisualization visualization, SearchSqlResponse<Map> result) {
+        final String ctx = CLASSNAME + ".parse(SearchSqlResponse)";
+        List<CoordinateMapChartResult> retValue = new ArrayList<>();
+
+        try {
+            Assert.notNull(visualization, "Param visualization must not be null");
+
+            for (Object rowObj : result.getData()) {
+                if (!(rowObj instanceof Map)) continue;
+                Map<String, Object> row = (Map<String, Object>) rowObj;
+
+                String ip = null;
+                Double metricValue = null;
+
+                for (Map.Entry<String, Object> entry : row.entrySet()) {
+                    Object val = entry.getValue();
+                    if (val == null) continue;
+
+                    String strVal = val.toString();
+                    if (ip == null && isValidIP(strVal)) {
+                        ip = strVal;
+                    } else if (metricValue == null && val instanceof Number) {
+                        metricValue = ((Number) val).doubleValue();
+                    }
+                }
 
+                if (ip == null || metricValue == null) continue;
+
+                GeoIp ipInfo;
+                try {
+                    ipInfo = ipInfoService.getIpInfo(ip);
+                    if (ipInfo == null) continue;
+                } catch (UtmIpInfoException e) {
+                    log.error(e.getMessage());
+                    continue;
+                }
+
+                CoordinateMapChartResult chartResult = new CoordinateMapChartResult();
+                chartResult.setName(ip);
+                chartResult.setValue(new Double[] {
+                        ipInfo.getLatitude(),
+                        ipInfo.getLongitude(),
+                        metricValue
+                });
+
+                retValue.add(chartResult);
+            }
+
+            return retValue;
+        } catch (Exception e) {
+            throw new RuntimeException(ctx + ": " + e.getMessage(), e);
+        }
+    }
 }

backend/src/main/java/com/park/utmstack/util/chart_builder/elasticsearch_dsl/responses/impl/gauge_goal_chart/ResponseParserForGaugeGoalChart.java

@@ -9,6 +9,7 @@
 import com.park.utmstack.util.chart_builder.elasticsearch_dsl.responses.ResponseParser;
 import com.utmstack.opensearch_connector.parsers.TermAggregateParser;
 import com.utmstack.opensearch_connector.types.BucketAggregation;
+import com.utmstack.opensearch_connector.types.SearchSqlResponse;
 import org.opensearch.client.opensearch._types.aggregations.*;
 import org.opensearch.client.opensearch.core.SearchResponse;
 import org.springframework.util.Assert;
@@ -111,4 +112,51 @@ private List<GaugeGoalChartResult> parseTermAggregation(Map<String, Aggregate> r
         }
         return rturn;
     }
+
+    @Override
+    public List<GaugeGoalChartResult> parse(UtmVisualization visualization, SearchSqlResponse<Map> result) {
+        final String ctx = CLASSNAME + ".parse(SearchSqlResponse)";
+        try {
+            Assert.notNull(visualization, "Param visualization must not be null");
+
+            List<GaugeGoalChartResult> results = new ArrayList<>();
+
+            for (Object rowObj : result.getData()) {
+                if (!(rowObj instanceof Map)) continue;
+                Map<String, Object> row = (Map<String, Object>) rowObj;
+
+                String bucketKey = null;
+                String bucketId = null;
+
+                Double metricValue = null;
+                String metricId = null;
+
+                for (Map.Entry<String, Object> entry : row.entrySet()) {
+                    String key = entry.getKey();
+                    Object val = entry.getValue();
+
+                    if (val == null) continue;
+
+                    if (val instanceof Number) {
+                        metricValue = ((Number) val).doubleValue();
+                        metricId = key;
+                    } else {
+                        bucketKey = val.toString();
+                        bucketId = key;
+                    }
+                }
+
+                if (metricValue == null) metricValue = 0.0;
+                if (metricId == null) metricId = "metric";
+                if (bucketKey == null) bucketKey = "UNKNOWN";
+                if (bucketId == null) bucketId = "bucket";
+
+                results.add(new GaugeGoalChartResult(metricId, metricValue, bucketKey, bucketId));
+            }
+
+            return results;
+        } catch (Exception e) {
+            throw new RuntimeException(ctx + ": " + e.getMessage(), e);
+        }
+    }
 }