diff --git a/Convert/build.gradle b/Convert/build.gradle index 35b4636..b88b1a1 100644 --- a/Convert/build.gradle +++ b/Convert/build.gradle @@ -33,7 +33,13 @@ dependencies compile 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.9.2' compile 'com.squareup.okhttp:okhttp:2.7.5' compile 'com.google.code.gson:gson:2.6.2' - compile project(':wavefront-java-sdk') + compile 'org.threeten:threetenbp:1.4.0' + compile group: 'io.swagger.core.v3', name: 'swagger-annotations', version: '2.1.0' + compile group: 'io.swagger.codegen.v3', name: 'swagger-codegen-generators', version: '1.0.14' + compile group: 'javax.annotation', name: 'javax.annotation-api', version: '1.2' + compile group: 'com.squareup.okhttp', name: 'logging-interceptor', version: '2.7.5' + compile group: 'io.gsonfire', name: 'gson-fire', version: '1.8.3' + compile group: 'com.wavefront', name: 'wavefront-sdk-java', version: '1.15' compile 'junit:junit:4.12' } diff --git a/Convert/build.sh b/Convert/build.sh new file mode 100755 index 0000000..5eeaacc --- /dev/null +++ b/Convert/build.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +pushd swagger +./generate.sh +popd + +mv swagger/src/main src/main +mv swagger/src/test src/test diff --git a/Convert/datadog-name-processor.yaml b/Convert/datadog-name-processor.yaml index dffdc2b..a535cb8 100644 --- a/Convert/datadog-name-processor.yaml +++ b/Convert/datadog-name-processor.yaml @@ -25,3 +25,21 @@ rules: search : 'mysql.galera.wsrep_cluster_size' replace : 'mysql.galera.wsrep_cluster_size' + - rule : datadog.nozzle goes away + match : '^datadog\.nozzle\..*$' + search : '^datadog\.nozzle\.(.*)$' + replace : 'pws-prod-test.$1' + + - rule : CapacityRemaining gets MiB + match : '^.*\.Capacity[a-zA-Z]*Memory$' + search : '^(.*\.Capacity[a-zA-Z]*Memory)$' + replace : '$1.MiB' + - rule : system.healthy gets ".b" + match : '^.*\.system.healthy$' + search : '^(.*\.system.healthy)$' + replace : '$1.b' + +# - rule : origin goes away +# match : '^.*origin.*$' +# search : '^.*origin.*$' +# replace : 'found.origin' diff --git a/Convert/datadog.properties b/Convert/datadog.properties index 027822f..6cb72d1 100644 --- a/Convert/datadog.properties +++ b/Convert/datadog.properties @@ -4,9 +4,9 @@ datadog.underscoreReplace=. datadog.dropTags=role,stackname,stack -convert.converter=com.wavefront.labs.convert.converter.datadog.DatadogConverter -convert.writer=com.wavefront.labs.convert.writer.FolderWriter +convert.converter=com.wavefront.labs.convert.converter.datadog.DatadogConverter2 convert.expressionBuilder=com.wavefront.labs.convert.converter.datadog.DatadogExpressionBuilder +convert.writer=com.wavefront.labs.convert.writer.FolderWriter convert.name.processor.file=datadog-name-processor.yaml convert.writer.tags=wf-converted diff --git a/Convert/src/com/wavefront/labs/convert/Convert.java b/Convert/src/com/wavefront/labs/convert/Convert.java index 34fa5af..e233772 100644 --- a/Convert/src/com/wavefront/labs/convert/Convert.java +++ b/Convert/src/com/wavefront/labs/convert/Convert.java @@ -7,6 +7,7 @@ import java.io.File; import java.io.FileReader; import java.io.IOException; +import java.lang.reflect.InvocationTargetException; import java.nio.file.Files; import java.nio.file.Path; import java.util.Arrays; @@ -38,7 +39,7 @@ public void start(String[] args) { doWrite(models); - } catch (IOException | InstantiationException | ClassNotFoundException | IllegalAccessException e) { + } catch (IOException | InstantiationException | ClassNotFoundException | IllegalAccessException | NoSuchMethodException | InvocationTargetException e) { logger.error("Fatal error in start.", e); } @@ -46,10 +47,10 @@ public void start(String[] args) { logger.error(com.wavefront.labs.convert.utils.Tracker.map); } - private List doConvert(String[] args) throws ClassNotFoundException, IllegalAccessException, InstantiationException, IOException { + private List doConvert(String[] args) throws ClassNotFoundException, IllegalAccessException, InstantiationException, IOException, NoSuchMethodException, InvocationTargetException { logger.info("Start Conversion"); - Converter converter = (Converter) Class.forName(properties.getProperty("convert.converter")).newInstance(); + Converter converter = (Converter) Class.forName(properties.getProperty("convert.converter")).getDeclaredConstructor().newInstance(); converter.init(properties); String filename = null; @@ -83,11 +84,11 @@ private List doConvert(String[] args) throws ClassNotFoundException, IllegalAcce return converter.convert(); } - private void doWrite(List models) throws ClassNotFoundException, IllegalAccessException, InstantiationException { + private void doWrite(List models) throws ClassNotFoundException, IllegalAccessException, InstantiationException, NoSuchMethodException, InvocationTargetException { logger.info("Start Writing"); String generatorName = properties.getProperty("convert.writer", "com.wavefront.labs.convert.writer.WavefrontPublisher"); - Writer writer = (Writer) Class.forName(generatorName).newInstance(); + Writer writer = (Writer) Class.forName(generatorName).getDeclaredConstructor().newInstance(); writer.init(properties); // tags can be separated by whitespace, comma, or semi-colon diff --git a/Convert/src/com/wavefront/labs/convert/DefaultExpressionBuilder.java b/Convert/src/com/wavefront/labs/convert/DefaultExpressionBuilder.java index 853b494..ee04fb6 100644 --- a/Convert/src/com/wavefront/labs/convert/DefaultExpressionBuilder.java +++ b/Convert/src/com/wavefront/labs/convert/DefaultExpressionBuilder.java @@ -20,7 +20,7 @@ public class DefaultExpressionBuilder implements ExpressionBuilder { protected Properties properties; - private NameProcessor nameProcessor; + protected NameProcessor nameProcessor; @Override public void init(Properties properties) { diff --git a/Convert/src/com/wavefront/labs/convert/converter/datadog/DatadogExpressionBuilder.java b/Convert/src/com/wavefront/labs/convert/converter/datadog/DatadogExpressionBuilder.java index cc5a017..fa99322 100644 --- a/Convert/src/com/wavefront/labs/convert/converter/datadog/DatadogExpressionBuilder.java +++ b/Convert/src/com/wavefront/labs/convert/converter/datadog/DatadogExpressionBuilder.java @@ -24,7 +24,9 @@ public class DatadogExpressionBuilder extends DefaultExpressionBuilder { private static final Pattern expressionListPattern = Pattern.compile("\\{.*?\\}|(\\+|-|\\*|\\/|;)"); private static final Pattern topConveniencePattern = Pattern.compile("^(top|bottom)(5|10|15|20)_?(mean|min|max|last|area|l2norm|norm)?$"); - private static final Pattern operatorNumberPattern = Pattern.compile("(\\+|-|\\*|/)|(-?\\d+([.]\\d+)?)"); + +// NOTE: Including '(',')' as operators so they get added back to the query after parsing + private static final Pattern operatorNumberPattern = Pattern.compile("(\\(|\\)|\\+|-|\\*|/)|(-?\\d+([.]\\d+)?)"); private static final HashMap> functionMap = new HashMap(); @@ -91,10 +93,23 @@ public void init(Properties properties) { variablesMap = new HashMap(); } - @Override - public String buildMetricName(String orig) { + + public String buildMetricName(DatadogQuery datadogQuery) { + String orig = datadogQuery.getMetric(); + List scopes = datadogQuery.getScopes(); + if (scopes != null && scopes.size() > 0) { + for (String scope : scopes) { + String[] scopeParts = scope.split(":"); + if (scopeParts.length > 1 && scopeParts[0].equals("origin")) { + String origin = scopeParts[1]; + String matcher = "^datadog\\.nozzle\\.(.*)$"; + String replace = "datadog.nozzle." + origin + ".$1"; + orig = orig.replaceAll(matcher,replace); + } + } + } String metricName = buildName(orig, "metric"); - metricName = metricName.replaceAll("_", underscoreReplace); + //metricName = metricName.replaceAll("_", underscoreReplace); return super.buildMetricName(metricName); } @@ -188,7 +203,7 @@ private String makeMetricQuery(DatadogQuery datadogQuery) { String query = datadogQuery.getNumeral(); if (datadogQuery.getMetric() != null && !"".equals(datadogQuery.getMetric())) { - query = "ts(\"" + buildMetricName(datadogQuery.getMetric()) + "\""; + query = "ts(\"" + buildMetricName(datadogQuery) + "\""; List scopes = datadogQuery.getScopes(); if (scopes != null && scopes.size() > 0) { @@ -261,12 +276,31 @@ private ArrayList createQueryList(String expression) { ArrayList expressionList = new ArrayList(); Matcher matcher = expressionListPattern.matcher(expression); + Pattern lParenMatcher = Pattern.compile("^\\("); + Pattern rParenMatcher = Pattern.compile("\\)$"); int lastPos = 0; while (matcher.find()) { if (matcher.group(1) != null) { - expressionList.add(expression.substring(lastPos, matcher.start()).trim()); + String subexpression = expression.substring(lastPos, matcher.start()).trim(); + + if (lParenMatcher.matcher(subexpression).find()) { +// Strip left parenthesis off expression, to be added back later (similar to how operators are handled) + expressionList.add("("); + expressionList.add(subexpression.substring(1,subexpression.length())); + } + else if (rParenMatcher.matcher(subexpression).find()) { +// Strip right parenthesis off expression, to be added back later (similar to how operators are handled) + expressionList.add(subexpression.substring(0, subexpression.length() - 1)); + expressionList.add(")"); + + } + else { + expressionList.add(subexpression); + } + expressionList.add(matcher.group(1)); + lastPos = matcher.end(); } } diff --git a/Convert/src/com/wavefront/labs/convert/converter/datadog/DatadogTimeboardConverter.java b/Convert/src/com/wavefront/labs/convert/converter/datadog/DatadogTimeboardConverter.java index 90bb741..9a86703 100644 --- a/Convert/src/com/wavefront/labs/convert/converter/datadog/DatadogTimeboardConverter.java +++ b/Convert/src/com/wavefront/labs/convert/converter/datadog/DatadogTimeboardConverter.java @@ -208,7 +208,6 @@ private DashboardParameterValue createDashboardParameter(Variable variable) { //dashboardParameterValue.setDynamicFieldType(DashboardParameterValue.DynamicFieldTypeEnum.TAG_KEY); //dashboardParameterValue.setTagKey(variable.getValue()); //dashboardParameterValue.setQueryValue("ts(query.filter)"); - dashboardParameterValue.setValue(""); return dashboardParameterValue; } diff --git a/Convert/src/com/wavefront/labs/convert/converter/datadog/DatadogTimeboardConverter2.java b/Convert/src/com/wavefront/labs/convert/converter/datadog/DatadogTimeboardConverter2.java index c2dd8c3..22def53 100644 --- a/Convert/src/com/wavefront/labs/convert/converter/datadog/DatadogTimeboardConverter2.java +++ b/Convert/src/com/wavefront/labs/convert/converter/datadog/DatadogTimeboardConverter2.java @@ -211,7 +211,6 @@ private DashboardParameterValue createDashboardParameter(Variable variable) { //dashboardParameterValue.setDynamicFieldType(DashboardParameterValue.DynamicFieldTypeEnum.TAG_KEY); //dashboardParameterValue.setTagKey(variable.getValue()); //dashboardParameterValue.setQueryValue("ts(query.filter)"); - dashboardParameterValue.setValue(""); return dashboardParameterValue; } diff --git a/Convert/src/com/wavefront/labs/convert/converter/datadog/models/DatadogGraphDefinition2.java b/Convert/src/com/wavefront/labs/convert/converter/datadog/models/DatadogGraphDefinition2.java index 4165af0..e05aaab 100644 --- a/Convert/src/com/wavefront/labs/convert/converter/datadog/models/DatadogGraphDefinition2.java +++ b/Convert/src/com/wavefront/labs/convert/converter/datadog/models/DatadogGraphDefinition2.java @@ -4,7 +4,7 @@ import java.util.List; -@JsonIgnoreProperties({"autoscale", "status", "markers", "style", "group", "no_metric_hosts", "scope", "no_group_hosts", "notes", "show_legend", "legend_size", "text_align", "custom_unit", "size_by", "tick_pos", "color_by_groups", "node_type", "show_tick", "compare_to", "xaxis", "color_by", "events", "tick_edge", "q", "group_by", "content", "font_size", "background_color", "precision", "layout_type", "title_size", "title_align", "color", "viz_type", "url", "time", "text", "alert_id", "tags_execution", "sizing", "query", "sort", "tags", "event_size", "margin", "count", "check", "hide_zero_counts", "grouping", "unit", "start", "color_preference", "display_format"}) +@JsonIgnoreProperties({"show_last_triggered","autoscale", "status", "markers", "style", "group", "no_metric_hosts", "scope", "no_group_hosts", "notes", "show_legend", "legend_size", "text_align", "custom_unit", "size_by", "tick_pos", "color_by_groups", "node_type", "show_tick", "compare_to", "xaxis", "color_by", "events", "tick_edge", "q", "group_by", "content", "font_size", "background_color", "precision", "layout_type", "title_size", "title_align", "color", "viz_type", "url", "time", "text", "alert_id", "tags_execution", "sizing", "query", "sort", "tags", "event_size", "margin", "count", "check", "hide_zero_counts", "grouping", "unit", "start", "color_preference", "display_format"}) public class DatadogGraphDefinition2 { private String type; diff --git a/Convert/src/com/wavefront/labs/convert/converter/grafana/GrafanaConverter.java b/Convert/src/com/wavefront/labs/convert/converter/grafana/GrafanaConverter.java index b95c6df..e3da80c 100644 --- a/Convert/src/com/wavefront/labs/convert/converter/grafana/GrafanaConverter.java +++ b/Convert/src/com/wavefront/labs/convert/converter/grafana/GrafanaConverter.java @@ -17,6 +17,7 @@ import java.io.File; import java.io.IOException; +import java.lang.reflect.InvocationTargetException; import java.util.ArrayList; import java.util.Comparator; import java.util.List; @@ -38,8 +39,8 @@ public void init(Properties properties) { String expressionBuilderClass = properties.getProperty("convert.expressionBuilder", ""); if (!expressionBuilderClass.equals("")) { try { - expressionBuilder = (ExpressionBuilder) Class.forName(expressionBuilderClass).newInstance(); - } catch (IllegalAccessException | InstantiationException | ClassNotFoundException e) { + expressionBuilder = (ExpressionBuilder) Class.forName(expressionBuilderClass).getDeclaredConstructor().newInstance(); + } catch (IllegalAccessException | InstantiationException | ClassNotFoundException | NoSuchMethodException | InvocationTargetException e) { logger.error("Could not create instance of: " + expressionBuilderClass, e); expressionBuilder = new SimpleExpressionBuilder(); } diff --git a/Convert/src/com/wavefront/labs/convert/converter/grafana/GrafanaConverterHelper.java b/Convert/src/com/wavefront/labs/convert/converter/grafana/GrafanaConverterHelper.java index d0d6131..032b462 100644 --- a/Convert/src/com/wavefront/labs/convert/converter/grafana/GrafanaConverterHelper.java +++ b/Convert/src/com/wavefront/labs/convert/converter/grafana/GrafanaConverterHelper.java @@ -262,7 +262,17 @@ private List fromTargets(List targets) { chartSourceQuery.setDisabled(target.isHide()); chartSourceQuery.setName(target.getRefId()); - String query = target.getTargetFull() != null && !target.getTargetFull().equals("") ? target.getTargetFull() : target.getTarget(); + String query; + if (target.getExpr() != null && !target.getExpr().equals("")) { + query = target.getExpr(); + } else if (target.getTargetFull() != null && !target.getTargetFull().equals("")) { + query = target.getTargetFull(); + } else if (target.getTarget() != null && !target.getTarget().equals("")){ + query = target.getTarget(); + } else { + continue; + } + chartSourceQuery.setQuery(expressionBuilder.buildExpression(query)); chartSourceQueries.add(chartSourceQuery); diff --git a/Convert/src/com/wavefront/labs/convert/converter/grafana/model/GrafanaPanelTarget.java b/Convert/src/com/wavefront/labs/convert/converter/grafana/model/GrafanaPanelTarget.java index 4befcf3..ff1f9f7 100644 --- a/Convert/src/com/wavefront/labs/convert/converter/grafana/model/GrafanaPanelTarget.java +++ b/Convert/src/com/wavefront/labs/convert/converter/grafana/model/GrafanaPanelTarget.java @@ -10,6 +10,7 @@ public class GrafanaPanelTarget { private String refId; private String target; private String targetFull; + private String expr; private boolean textEditor; public boolean isHide() { @@ -52,6 +53,14 @@ public void setTargetFull(String targetFull) { this.targetFull = targetFull; } + public String getExpr() { + return expr; + } + + public void setExpr(String expr) { + this.expr = expr; + } + public boolean isTextEditor() { return textEditor; } diff --git a/Convert/src/com/wavefront/labs/convert/converter/rrd/models/Def.java b/Convert/src/com/wavefront/labs/convert/converter/rrd/models/Def.java index 0f77315..bc96476 100644 --- a/Convert/src/com/wavefront/labs/convert/converter/rrd/models/Def.java +++ b/Convert/src/com/wavefront/labs/convert/converter/rrd/models/Def.java @@ -6,6 +6,8 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import java.lang.reflect.InvocationTargetException; + public class Def extends Definition { private static final Logger logger = LogManager.getLogger(Def.class); @@ -35,8 +37,8 @@ public String calculate(RRDContext context) { String expressionBuilderClass = context.getProperties().getProperty("convert.expressionBuilder", ""); if (!expressionBuilderClass.equals("")) { try { - expressionBuilder = (ExpressionBuilder) Class.forName(expressionBuilderClass).newInstance(); - } catch (IllegalAccessException | InstantiationException | ClassNotFoundException e) { + expressionBuilder = (ExpressionBuilder) Class.forName(expressionBuilderClass).getDeclaredConstructor().newInstance(); + } catch (IllegalAccessException | InstantiationException | ClassNotFoundException | NoSuchMethodException | InvocationTargetException e) { logger.error("Could not create instance of: " + expressionBuilderClass, e); expressionBuilder = new RRDExpressionBuilder(); } diff --git a/Convert/src/com/wavefront/labs/convert/converter/rrd/models/VDef.java b/Convert/src/com/wavefront/labs/convert/converter/rrd/models/VDef.java index 1d178cb..acc9910 100644 --- a/Convert/src/com/wavefront/labs/convert/converter/rrd/models/VDef.java +++ b/Convert/src/com/wavefront/labs/convert/converter/rrd/models/VDef.java @@ -12,6 +12,7 @@ public class VDef extends Definition { private static HashMap, String>> functionMap = new HashMap(); + static { functionMap.put("MAXIMUM", NotSupported::warning); functionMap.put("MINIMUM", NotSupported::warning); @@ -27,7 +28,7 @@ public class VDef extends Definition { functionMap.put("LSLCORREL", NotSupported::warning); } - public VDef (String line) { + public VDef(String line) { super(line); }