[not meant to merge directly] PWS Platform changes to conversions

Convert/build.gradle

@@ -33,7 +33,13 @@ dependencies
     compile 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.9.2'
     compile 'com.squareup.okhttp:okhttp:2.7.5'
     compile 'com.google.code.gson:gson:2.6.2'
-    compile project(':wavefront-java-sdk')
+    compile 'org.threeten:threetenbp:1.4.0'
+    compile group: 'io.swagger.core.v3', name: 'swagger-annotations', version: '2.1.0'
+    compile group: 'io.swagger.codegen.v3', name: 'swagger-codegen-generators', version: '1.0.14'
+    compile group: 'javax.annotation', name: 'javax.annotation-api', version: '1.2'
+    compile group: 'com.squareup.okhttp', name: 'logging-interceptor', version: '2.7.5'
+    compile group: 'io.gsonfire', name: 'gson-fire', version: '1.8.3'
+    compile group: 'com.wavefront', name: 'wavefront-sdk-java', version: '1.15'
     compile 'junit:junit:4.12'
 }
 

Convert/build.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+pushd swagger
+./generate.sh
+popd
+
+mv swagger/src/main src/main
+mv swagger/src/test src/test

Convert/datadog-name-processor.yaml

@@ -25,3 +25,21 @@ rules:
       search  : 'mysql.galera.wsrep_cluster_size'
       replace : 'mysql.galera.wsrep_cluster_size'
 
+    - rule    : datadog.nozzle goes away
+      match   : '^datadog\.nozzle\..*$'
+      search  : '^datadog\.nozzle\.(.*)$'
+      replace  : 'pws-prod-test.$1'
+
+    - rule    : CapacityRemaining gets MiB
+      match   : '^.*\.Capacity[a-zA-Z]*Memory$'
+      search  : '^(.*\.Capacity[a-zA-Z]*Memory)$'
+      replace  : '$1.MiB'
+    - rule    : system.healthy gets ".b"
+      match   : '^.*\.system.healthy$'
+      search  : '^(.*\.system.healthy)$'
+      replace  : '$1.b'
+
+#    - rule    : origin goes away
+#      match   : '^.*origin.*$'
+#      search   : '^.*origin.*$'
+#      replace  : 'found.origin'

Convert/datadog.properties

@@ -4,9 +4,9 @@
 datadog.underscoreReplace=.
 datadog.dropTags=role,stackname,stack
 
-convert.converter=com.wavefront.labs.convert.converter.datadog.DatadogConverter
-convert.writer=com.wavefront.labs.convert.writer.FolderWriter
+convert.converter=com.wavefront.labs.convert.converter.datadog.DatadogConverter2
 convert.expressionBuilder=com.wavefront.labs.convert.converter.datadog.DatadogExpressionBuilder
+convert.writer=com.wavefront.labs.convert.writer.FolderWriter
 convert.name.processor.file=datadog-name-processor.yaml
 
 convert.writer.tags=wf-converted

Convert/src/com/wavefront/labs/convert/Convert.java

@@ -7,6 +7,7 @@
 import java.io.File;
 import java.io.FileReader;
 import java.io.IOException;
+import java.lang.reflect.InvocationTargetException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.Arrays;
@@ -38,18 +39,18 @@ public void start(String[] args) {
 
 			doWrite(models);
 
-		} catch (IOException | InstantiationException | ClassNotFoundException | IllegalAccessException e) {
+		} catch (IOException | InstantiationException | ClassNotFoundException | IllegalAccessException | NoSuchMethodException | InvocationTargetException e) {
 			logger.error("Fatal error in start.", e);
 		}
 
 		logger.info("Convert to Wavefront finished!");
 		logger.error(com.wavefront.labs.convert.utils.Tracker.map);
 	}
 
-	private List doConvert(String[] args) throws ClassNotFoundException, IllegalAccessException, InstantiationException, IOException {
+	private List doConvert(String[] args) throws ClassNotFoundException, IllegalAccessException, InstantiationException, IOException, NoSuchMethodException, InvocationTargetException {
 		logger.info("Start Conversion");
 
-		Converter converter = (Converter) Class.forName(properties.getProperty("convert.converter")).newInstance();
+		Converter converter = (Converter) Class.forName(properties.getProperty("convert.converter")).getDeclaredConstructor().newInstance();
 		converter.init(properties);
 
 		String filename = null;
@@ -83,11 +84,11 @@ private List doConvert(String[] args) throws ClassNotFoundException, IllegalAcce
 		return converter.convert();
 	}
 
-	private void doWrite(List models) throws ClassNotFoundException, IllegalAccessException, InstantiationException {
+	private void doWrite(List models) throws ClassNotFoundException, IllegalAccessException, InstantiationException, NoSuchMethodException, InvocationTargetException {
 		logger.info("Start Writing");
 
 		String generatorName = properties.getProperty("convert.writer", "com.wavefront.labs.convert.writer.WavefrontPublisher");
-		Writer writer = (Writer) Class.forName(generatorName).newInstance();
+		Writer writer = (Writer) Class.forName(generatorName).getDeclaredConstructor().newInstance();
 		writer.init(properties);
 
 		// tags can be separated by whitespace, comma, or semi-colon

Convert/src/com/wavefront/labs/convert/DefaultExpressionBuilder.java

@@ -20,7 +20,7 @@ public class DefaultExpressionBuilder implements ExpressionBuilder {
 
 	protected Properties properties;
 
-	private NameProcessor nameProcessor;
+	protected NameProcessor nameProcessor;
 
 	@Override
 	public void init(Properties properties) {

Convert/src/com/wavefront/labs/convert/converter/datadog/DatadogExpressionBuilder.java

@@ -24,7 +24,9 @@ public class DatadogExpressionBuilder extends DefaultExpressionBuilder {
 
 	private static final Pattern expressionListPattern = Pattern.compile("\\{.*?\\}|(\\+|-|\\*|\\/|;)");
 	private static final Pattern topConveniencePattern = Pattern.compile("^(top|bottom)(5|10|15|20)_?(mean|min|max|last|area|l2norm|norm)?$");
-	private static final Pattern operatorNumberPattern = Pattern.compile("(\\+|-|\\*|/)|(-?\\d+([.]\\d+)?)");
+
+//	NOTE: Including '(',')' as operators so they get added back to the query after parsing
+	private static final Pattern operatorNumberPattern = Pattern.compile("(\\(|\\)|\\+|-|\\*|/)|(-?\\d+([.]\\d+)?)");
 
 	private static final HashMap<String, Function<DatadogFunction, String>> functionMap = new HashMap();
 
@@ -91,10 +93,23 @@ public void init(Properties properties) {
 		variablesMap = new HashMap();
 	}
 
-	@Override
-	public String buildMetricName(String orig) {
+
+	public String buildMetricName(DatadogQuery datadogQuery) {
+		String orig = datadogQuery.getMetric();
+		List<String> scopes = datadogQuery.getScopes();
+		if (scopes != null && scopes.size() > 0) {
+			for (String scope : scopes) {
+				String[] scopeParts = scope.split(":");
+				if (scopeParts.length > 1 && scopeParts[0].equals("origin")) {
+					String origin = scopeParts[1];
+					String matcher = "^datadog\\.nozzle\\.(.*)$";
+					String replace = "datadog.nozzle." + origin + ".$1";
+					orig = orig.replaceAll(matcher,replace);
+				}
+			}
+		}
 		String metricName = buildName(orig, "metric");
-		metricName = metricName.replaceAll("_", underscoreReplace);
+		//metricName = metricName.replaceAll("_", underscoreReplace);
 		return super.buildMetricName(metricName);
 	}
 
@@ -188,7 +203,7 @@ private String makeMetricQuery(DatadogQuery datadogQuery) {
 
 		String query = datadogQuery.getNumeral();
 		if (datadogQuery.getMetric() != null && !"".equals(datadogQuery.getMetric())) {
-			query = "ts(\"" + buildMetricName(datadogQuery.getMetric()) + "\"";
+			query = "ts(\"" + buildMetricName(datadogQuery) + "\"";
 
 			List<String> scopes = datadogQuery.getScopes();
 			if (scopes != null && scopes.size() > 0) {
@@ -261,12 +276,31 @@ private ArrayList<String> createQueryList(String expression) {
 		ArrayList<String> expressionList = new ArrayList();
 
 		Matcher matcher = expressionListPattern.matcher(expression);
+		Pattern lParenMatcher = Pattern.compile("^\\(");
+		Pattern rParenMatcher = Pattern.compile("\\)$");
 
 		int lastPos = 0;
 		while (matcher.find()) {
 			if (matcher.group(1) != null) {
-				expressionList.add(expression.substring(lastPos, matcher.start()).trim());
+			    String subexpression = expression.substring(lastPos, matcher.start()).trim();
+
+			    if (lParenMatcher.matcher(subexpression).find()) {
+//			        Strip left parenthesis off expression, to be added back later (similar to how operators are handled)
+					expressionList.add("(");
+					expressionList.add(subexpression.substring(1,subexpression.length()));
+				}
+				else if (rParenMatcher.matcher(subexpression).find()) {
+//			        Strip right parenthesis off expression, to be added back later (similar to how operators are handled)
+					expressionList.add(subexpression.substring(0, subexpression.length() - 1));
+					expressionList.add(")");
+
+				}
+				else {
+			    	expressionList.add(subexpression);
+				}
+
 				expressionList.add(matcher.group(1));
+
 				lastPos = matcher.end();
 			}
 		}

Convert/src/com/wavefront/labs/convert/converter/datadog/DatadogTimeboardConverter.java

@@ -208,7 +208,6 @@ private DashboardParameterValue createDashboardParameter(Variable variable) {
 		//dashboardParameterValue.setDynamicFieldType(DashboardParameterValue.DynamicFieldTypeEnum.TAG_KEY);
 		//dashboardParameterValue.setTagKey(variable.getValue());
 		//dashboardParameterValue.setQueryValue("ts(query.filter)");
-		dashboardParameterValue.setValue("");
 
 		return dashboardParameterValue;
 	}

Convert/src/com/wavefront/labs/convert/converter/datadog/DatadogTimeboardConverter2.java

@@ -211,7 +211,6 @@ private DashboardParameterValue createDashboardParameter(Variable variable) {
 		//dashboardParameterValue.setDynamicFieldType(DashboardParameterValue.DynamicFieldTypeEnum.TAG_KEY);
 		//dashboardParameterValue.setTagKey(variable.getValue());
 		//dashboardParameterValue.setQueryValue("ts(query.filter)");
-		dashboardParameterValue.setValue("");
 
 		return dashboardParameterValue;
 	}

Convert/src/com/wavefront/labs/convert/converter/datadog/models/DatadogGraphDefinition2.java

@@ -4,7 +4,7 @@
 
 import java.util.List;
 
-@JsonIgnoreProperties({"autoscale", "status", "markers", "style", "group", "no_metric_hosts", "scope", "no_group_hosts", "notes", "show_legend", "legend_size", "text_align", "custom_unit", "size_by", "tick_pos", "color_by_groups", "node_type", "show_tick", "compare_to", "xaxis", "color_by", "events", "tick_edge", "q", "group_by", "content", "font_size", "background_color", "precision", "layout_type", "title_size", "title_align", "color", "viz_type", "url", "time", "text", "alert_id", "tags_execution", "sizing", "query", "sort", "tags", "event_size", "margin", "count", "check", "hide_zero_counts", "grouping", "unit", "start", "color_preference", "display_format"})
+@JsonIgnoreProperties({"show_last_triggered","autoscale", "status", "markers", "style", "group", "no_metric_hosts", "scope", "no_group_hosts", "notes", "show_legend", "legend_size", "text_align", "custom_unit", "size_by", "tick_pos", "color_by_groups", "node_type", "show_tick", "compare_to", "xaxis", "color_by", "events", "tick_edge", "q", "group_by", "content", "font_size", "background_color", "precision", "layout_type", "title_size", "title_align", "color", "viz_type", "url", "time", "text", "alert_id", "tags_execution", "sizing", "query", "sort", "tags", "event_size", "margin", "count", "check", "hide_zero_counts", "grouping", "unit", "start", "color_preference", "display_format"})
 public class DatadogGraphDefinition2 {
 
 	private String type;

Convert/src/com/wavefront/labs/convert/converter/grafana/GrafanaConverter.java

@@ -17,6 +17,7 @@
 
 import java.io.File;
 import java.io.IOException;
+import java.lang.reflect.InvocationTargetException;
 import java.util.ArrayList;
 import java.util.Comparator;
 import java.util.List;
@@ -38,8 +39,8 @@ public void init(Properties properties) {
 		String expressionBuilderClass = properties.getProperty("convert.expressionBuilder", "");
 		if (!expressionBuilderClass.equals("")) {
 			try {
-				expressionBuilder = (ExpressionBuilder) Class.forName(expressionBuilderClass).newInstance();
-			} catch (IllegalAccessException | InstantiationException | ClassNotFoundException e) {
+				expressionBuilder = (ExpressionBuilder) Class.forName(expressionBuilderClass).getDeclaredConstructor().newInstance();
+			} catch (IllegalAccessException | InstantiationException | ClassNotFoundException | NoSuchMethodException | InvocationTargetException e) {
 				logger.error("Could not create instance of: " + expressionBuilderClass, e);
 				expressionBuilder = new SimpleExpressionBuilder();
 			}

Convert/src/com/wavefront/labs/convert/converter/grafana/GrafanaConverterHelper.java

@@ -262,7 +262,17 @@ private List<ChartSourceQuery> fromTargets(List<GrafanaPanelTarget> targets) {
 				chartSourceQuery.setDisabled(target.isHide());
 				chartSourceQuery.setName(target.getRefId());
 
-				String query = target.getTargetFull() != null && !target.getTargetFull().equals("") ? target.getTargetFull() : target.getTarget();
+				String query;
+				if (target.getExpr() != null && !target.getExpr().equals("")) {
+					query = target.getExpr();
+				} else if (target.getTargetFull() != null && !target.getTargetFull().equals("")) {
+					query = target.getTargetFull();
+				} else if (target.getTarget() != null && !target.getTarget().equals("")){
+					query = target.getTarget();
+				} else {
+					continue;
+				}
+
 				chartSourceQuery.setQuery(expressionBuilder.buildExpression(query));
 
 				chartSourceQueries.add(chartSourceQuery);

Convert/src/com/wavefront/labs/convert/converter/grafana/model/GrafanaPanelTarget.java

@@ -10,6 +10,7 @@ public class GrafanaPanelTarget {
 	private String refId;
 	private String target;
 	private String targetFull;
+	private String expr;
 	private boolean textEditor;
 
 	public boolean isHide() {
@@ -52,6 +53,14 @@ public void setTargetFull(String targetFull) {
 		this.targetFull = targetFull;
 	}
 
+	public String getExpr() {
+		return expr;
+	}
+
+	public void setExpr(String expr) {
+		this.expr = expr;
+	}
+
 	public boolean isTextEditor() {
 		return textEditor;
 	}

Convert/src/com/wavefront/labs/convert/converter/rrd/models/Def.java

@@ -6,6 +6,8 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
+import java.lang.reflect.InvocationTargetException;
+
 public class Def extends Definition {
 	private static final Logger logger = LogManager.getLogger(Def.class);
 
@@ -35,8 +37,8 @@ public String calculate(RRDContext context) {
 		String expressionBuilderClass = context.getProperties().getProperty("convert.expressionBuilder", "");
 		if (!expressionBuilderClass.equals("")) {
 			try {
-				expressionBuilder = (ExpressionBuilder) Class.forName(expressionBuilderClass).newInstance();
-			} catch (IllegalAccessException | InstantiationException | ClassNotFoundException e) {
+				expressionBuilder = (ExpressionBuilder) Class.forName(expressionBuilderClass).getDeclaredConstructor().newInstance();
+			} catch (IllegalAccessException | InstantiationException | ClassNotFoundException | NoSuchMethodException | InvocationTargetException e) {
 				logger.error("Could not create instance of: " + expressionBuilderClass, e);
 				expressionBuilder = new RRDExpressionBuilder();
 			}

Convert/src/com/wavefront/labs/convert/converter/rrd/models/VDef.java

@@ -12,6 +12,7 @@
 public class VDef extends Definition {
 
 	private static HashMap<String, Function<Deque<String>, String>> functionMap = new HashMap();
+
 	static {
 		functionMap.put("MAXIMUM", NotSupported::warning);
 		functionMap.put("MINIMUM", NotSupported::warning);
@@ -27,7 +28,7 @@ public class VDef extends Definition {
 		functionMap.put("LSLCORREL", NotSupported::warning);
 	}
 
-	public VDef (String line) {
+	public VDef(String line) {
 		super(line);
 	}