scripts, interval, collectd, try to build grafana plugin

analytics/pom.xml

@@ -4,7 +4,7 @@
   <parent>
     <groupId>timely</groupId>
     <artifactId>timely</artifactId>
-    <version>2.1.2</version>
+    <version>2.1.4-SNAPSHOT</version>
   </parent>
   <artifactId>timely-analytics</artifactId>
   <!--

balancer/bin/timely-balancer.sh

@@ -4,7 +4,6 @@ if [[ $(uname) == "Darwin" ]]; then
         THIS_SCRIPT=$(python -c 'import os,sys; print os.path.realpath(sys.argv[1])' "$0")
 else
         THIS_SCRIPT=$(readlink -f "$0")
-        TCNATIVE_SUFFIX="so"
         HW=`uname -m`
         OS_PATH=linux-${HW}
 fi
@@ -56,14 +55,14 @@ if [ "$PID" == "" ]; then
 
     if [[ -n "$OS_PATH" ]]; then
       pushd "${BASE_DIR}/bin" || exit
-      "${JAVA_HOME}"/bin/jar xf "${LIB_DIR}"/netty-tcnative-boringssl-static-*.Final-"${OS_PATH}".jar META-INF/native/libnetty_tcnative_linux_"${HW}".so
-      "${JAVA_HOME}"/bin/jar xf "${LIB_DIR}"/netty-transport-native-epoll-*-"${OS_PATH}".jar META-INF/native/libnetty_transport_native_epoll_"${HW}".so
+      "${JAVA_HOME}"/bin/jar xf "${LIB_DIR}"/netty-tcnative-boringssl-static-*.Final-"${OS_PATH}".jar ${NATIVE_DIR}/libnetty_tcnative_linux_"${HW}".so
+      "${JAVA_HOME}"/bin/jar xf "${LIB_DIR}"/netty-transport-native-epoll-*-"${OS_PATH}".jar ${NATIVE_DIR}/libnetty_transport_native_epoll_"${HW}".so
       popd || exit
       JVM_ARGS="${JVM_ARGS} -Djava.library.path=${NATIVE_DIR}"
     fi
 
-    echo "${JAVA_HOME}/bin/java ${JVM_ARGS} -jar ${THIS_DIR}/timely-balancer-*-exec.jar --instance=${INSTANCE}"
-    nohup "${JAVA_HOME}"/bin/java ${JVM_ARGS} -jar "${THIS_DIR}"/timely-balancer-*-exec.jar --instance="${INSTANCE}" >> "${LOG_DIR}/timely-balancer${INSTANCE}.out" 2>&1 &
+    echo "${JAVA_HOME}/bin/java ${JVM_ARGS} -jar ${BIN_DIR}/timely-balancer-*-exec.jar --instance=${INSTANCE}"
+    nohup "${JAVA_HOME}"/bin/java ${JVM_ARGS} -jar "${BIN_DIR}"/timely-balancer-*-exec.jar --instance="${INSTANCE}" >> "${LOG_DIR}/timely-balancer${INSTANCE}.out" 2>&1 &
 else
     echo "timely-balancer --instance=${INSTANCE} already running with pid ${PID}"
 fi

balancer/pom.xml

@@ -4,7 +4,7 @@
   <parent>
     <groupId>timely</groupId>
     <artifactId>timely</artifactId>
-    <version>2.1.2</version>
+    <version>2.1.4-SNAPSHOT</version>
   </parent>
   <artifactId>timely-balancer</artifactId>
   <description>Timely Balancer</description>

client/pom.xml

@@ -4,13 +4,13 @@
   <parent>
     <groupId>timely</groupId>
     <artifactId>timely</artifactId>
-    <version>2.1.2</version>
+    <version>2.1.4-SNAPSHOT</version>
   </parent>
   <artifactId>timely-client</artifactId>
   <properties>
     <!-- PEP 440 compliant version gets substituted into src/main/python/template/setup.py -->
     <!-- use .dev0 instead of -SNAPSHOT -->
-    <python.client.version>2.1.2</python.client.version>
+    <python.client.version>2.1.4.dev0</python.client.version>
   </properties>
   <dependencies>
     <dependency>

collectd-nsq-plugin/pom.xml

@@ -4,7 +4,7 @@
   <parent>
     <groupId>timely</groupId>
     <artifactId>timely</artifactId>
-    <version>2.1.2</version>
+    <version>2.1.4-SNAPSHOT</version>
   </parent>
   <artifactId>collectd-nsq-plugin</artifactId>
   <dependencies>

collectd-plugin-parent/pom.xml

@@ -4,7 +4,7 @@
   <parent>
     <groupId>timely</groupId>
     <artifactId>timely</artifactId>
-    <version>2.1.2</version>
+    <version>2.1.4-SNAPSHOT</version>
   </parent>
   <artifactId>collectd-plugin-parent</artifactId>
   <dependencies>

collectd-plugin-parent/src/main/java/timely/collectd/plugin/CollectDPluginParent.java

@@ -2,51 +2,58 @@ package timely.collectd.plugin;
 
 import java.io.OutputStream;
 import java.text.MessageFormat;
-import java.util.HashSet;
-import java.util.Set;
+import java.util.Arrays;
+import java.util.Map;
+import java.util.TreeMap;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
+import java.util.stream.Collectors;
 
 import org.collectd.api.Collectd;
-import org.collectd.api.DataSource;
 import org.collectd.api.OConfigItem;
 import org.collectd.api.ValueList;
 
 public abstract class CollectDPluginParent {
 
-    private static final String PUT = "put {0} {1} {2}{3}\n";
-    private static final Pattern HADOOP_STATSD_PATTERN = Pattern.compile("([\\w-_]+)\\.([\\w-_]+)\\.([\\w-_]+)\\.([\\w-_]+)");
-    private static final String INSTANCE = " instance=";
-    private static final String NAME = " name=";
-    private static final String SAMPLE = " sample=";
-    private static final String CODE = " code=";
+    private static final String PUT = "put {0} {1} {2} {3}";
+    private static final String INSTANCE = "instance";
+    private static final String NAME = "name";
+    private static final String SAMPLE_TYPE = "sampleType";
+    private static final String CODE = "code";
     private static final String PERIOD = ".";
-    private static final String SAMPLE_TYPE = " sampleType=";
     private static final String COUNTER = "COUNTER";
     private static final String GAUGE = "GAUGE";
     private static final String DERIVE = "DERIVE";
     private static final String ABSOLUTE = "ABSOLUTE";
-    private static final Pattern NSQ_PATTERN1 = Pattern.compile("([\\w-_]+)\\.([\\w-_]+)\\.([\\w-_]+)");
-    private static final Pattern NSQ_PATTERN2 = Pattern.compile("([\\w-_]+)\\.([\\w-_]+)\\.([\\w-_]+)\\.([\\w-_]+)");
-    private static final Pattern NSQ_PATTERN3 = Pattern.compile("([\\w-_]+)\\.([\\w-_]+)\\.([\\w-_]+)\\.([\\w-_]+)\\.([\\w-_#]+)\\.([\\w-_]+)");
+    private static final Pattern STATSD_PATTERN_3_GROUPS = Pattern.compile("([\\w-_]+)\\.([\\w-_]+)\\.([\\w-_]+)");
+    private static final Pattern STATSD_PATTERN_4_GROUPS = Pattern.compile("([\\w-_]+)\\.([\\w-_]+)\\.([\\w-_]+)\\.([\\w-_]+)");
+    private static final Pattern STATSD_PATTERN_6_GROUPS = Pattern.compile("([\\w-_]+)\\.([\\w-_]+)\\.([\\w-_]+)\\.([\\w-_]+)\\.([\\w-_#]+)\\.([\\w-_]+)");
     private static final Pattern ETHSTAT_PATTERN = Pattern.compile("([\\w-_]+)_queue_([\\w-_]+)_([\\w-_]+)");
     private static final String STATSD_PREFIX = "statsd";
     private static final String NSQ_PREFIX = STATSD_PREFIX + ".nsq.";
     private static final Pattern HAPROXY_PATTERN = Pattern.compile("\\[([\\w-_=]+),([\\w-_=]+)\\]");
 
     private final SMARTCodeMapping smart = new SMARTCodeMapping();
-
-    protected Set<String> addlTags = new HashSet<>();
+    protected Map<String,String> addlTags = new TreeMap<>();
+    private boolean debug = false;
 
     public int config(OConfigItem config) {
         for (OConfigItem child : config.getChildren()) {
             switch (child.getKey()) {
                 case "Tags":
                 case "tags":
-                    String[] additionalTags = child.getValues().get(0).getString().split(",");
-                    for (String t : additionalTags) {
-                        addlTags.add(" " + t.trim());
-                    }
+                    String tagValue = child.getValues().get(0).getString();
+                    Arrays.stream(tagValue.split(",")).map(String::trim).forEach(p -> {
+                        String[] pair = p.split("=");
+                        if (pair.length == 2) {
+                            addTag(addlTags, pair[0], pair[1]);
+                        }
+                    });
+                    break;
+                case "Debug":
+                case "debug":
+                    debug = child.getValues().get(0).getBoolean();
+                    break;
                 default:
             }
         }
@@ -55,196 +62,256 @@ public abstract class CollectDPluginParent {
 
     public abstract void write(String metric, OutputStream out);
 
+    protected void addTag(Map<String,String> m, String k, String v) {
+        if (k != null && !k.isBlank() && v != null && !v.isBlank()) {
+            m.put(k.trim(), v.trim());
+        }
+    }
+
+    protected void addTag(Map<String,String> m, String kv) {
+        if (kv != null && !kv.isBlank()) {
+            String[] parts = kv.split("=");
+            if (parts.length == 2 && !parts[0].isBlank() && !parts[1].isBlank()) {
+                m.put(parts[0].trim(), parts[1].trim());
+            }
+        }
+    }
+
     public void process(ValueList vl, OutputStream out) {
+        try {
+            processInternal(vl, out);
+        } catch (Exception e) {
+            Collectd.logError(e.getMessage());
+            e.printStackTrace(System.err);
+        }
+    }
+
+    private void processInternal(ValueList vl, OutputStream out) {
         debugValueList(vl);
 
         StringBuilder metric = new StringBuilder();
         Long timestamp = vl.getTime();
-        StringBuilder tags = new StringBuilder();
+        Map<String,String> tagMap = new TreeMap<>();
         String host = vl.getHost();
         int idx = host.indexOf(PERIOD);
         if (-1 != idx) {
-            tags.append(" host=").append(host.substring(0, idx));
+            addTag(tagMap, "host", host.substring(0, idx));
         } else {
-            tags.append(" host=").append(host);
+            addTag(tagMap, "host", host);
         }
         int nIdx = host.indexOf('n');
         if (-1 != nIdx) {
-            tags.append(" rack=").append(host.substring(0, nIdx));
+            addTag(tagMap, "rack", host.substring(0, nIdx));
         }
-        for (String tag : addlTags) {
-            tags.append(tag);
-        }
-        if (vl.getPlugin().equals(STATSD_PREFIX)) {
-            Matcher m = HADOOP_STATSD_PATTERN.matcher(vl.getTypeInstance());
-            Matcher n1 = NSQ_PATTERN1.matcher(vl.getTypeInstance());
-            Matcher n2 = NSQ_PATTERN2.matcher(vl.getTypeInstance());
-            Matcher n3 = NSQ_PATTERN3.matcher(vl.getTypeInstance());
-            String instance = null;
-            if (m.matches() && !vl.getTypeInstance().startsWith("nsq")) {
-                // Here we are processing the statsd metrics coming from the
-                // Hadoop Metrics2 StatsDSink without the host name.
-                // The format of metric is:
-                // serviceName.contextName.recordName.metricName. The recordName
-                // is typically duplicative and is dropped here. The serviceName
-                // is used as the instance.
-                metric.append(STATSD_PREFIX).append(PERIOD).append(m.group(2)).append(PERIOD).append(m.group(4));
-                instance = m.group(1);
-            } else if (n1.matches() && vl.getTypeInstance().startsWith("nsq")) {
-                metric.append(NSQ_PREFIX).append(n1.group(2)).append(PERIOD).append(n1.group(3));
-            } else if (n2.matches() && vl.getTypeInstance().startsWith("nsq")) {
-                metric.append(NSQ_PREFIX).append(n2.group(2)).append(PERIOD).append(n2.group(4));
-                instance = n2.group(3);
-            } else if (n3.matches() && vl.getTypeInstance().startsWith("nsq")) {
-                metric.append(NSQ_PREFIX).append(n3.group(4)).append(PERIOD).append(n3.group(6));
-                instance = n3.group(5);
-            } else {
-                // Handle StatsD metrics of unknown formats. If there is a
-                // period in the metric name, use everything up to that as
-                // the instance.
-                int period = vl.getTypeInstance().indexOf('.');
-                if (-1 == period) {
-                    metric.append(STATSD_PREFIX).append(PERIOD).append(vl.getTypeInstance());
+        tagMap.putAll(addlTags);
+
+        String plugin = vl.getPlugin();
+        String pluginInstance = vl.getPluginInstance();
+        String type = vl.getType();
+        String typeInstance = vl.getTypeInstance();
+
+        switch (plugin) {
+            case STATSD_PREFIX:
+                Matcher statsd_3_groups = STATSD_PATTERN_3_GROUPS.matcher(typeInstance);
+                Matcher statsd_4_groups = STATSD_PATTERN_4_GROUPS.matcher(typeInstance);
+                Matcher statsd_6_groups = STATSD_PATTERN_6_GROUPS.matcher(typeInstance);
+                String instance = null;
+                if (!typeInstance.startsWith("nsq")) {
+                    String[] parts = typeInstance.split("\\.");
+                    if (parts.length % 2 == 1 && parts.length >= 3) {
+                        switch(parts[0]) {
+                            case "dwingest":
+                                if (parts.length >= 4) {
+                                    metric.append(STATSD_PREFIX).append(PERIOD).append(parts[1]).append(PERIOD).append(parts[3]);
+                                    instance = parts[0];
+                                }
+                                break;
+                            case "dwquery":
+                                if (parts.length >= 4) {
+                                    metric.append(STATSD_PREFIX).append(PERIOD).append(parts[1]).append(PERIOD).append(parts[3]);
+                                    addTag(tagMap, "queryId", parts[0]);
+                                }
+                                break;
+                            default:
+                                // EtsyStatsD format -- metric.(tagName.tagValue)*
+                                metric.append(STATSD_PREFIX).append(PERIOD).append(parts[0]);
+                                for (int x = 1; x < parts.length; x += 2) {
+                                    addTag(tagMap, parts[x], parts[x + 1]);
+                                }
+                            }
+                        }
+                    } else if (statsd_4_groups.matches()) {
+                        // Here we are processing the statsd metrics coming from the Hadoop Metrics2 StatsDSink without the host name.
+                        // The format of metric is: serviceName.contextName.recordName.metricName. The recordName is typically duplicative
+                        // and is dropped here. The serviceName is used as the instance.
+                        metric.append(STATSD_PREFIX).append(PERIOD).append(statsd_4_groups.group(2)).append(PERIOD).append(statsd_4_groups.group(4));
+                        instance = statsd_4_groups.group(1);
+                    }
+                } else if (statsd_3_groups.matches()) {
+                    metric.append(NSQ_PREFIX).append(statsd_3_groups.group(2)).append(PERIOD).append(statsd_3_groups.group(3));
+                } else if (statsd_4_groups.matches()) {
+                    metric.append(NSQ_PREFIX).append(statsd_4_groups.group(2)).append(PERIOD).append(statsd_4_groups.group(4));
+                    instance = statsd_4_groups.group(3);
+                } else if (statsd_6_groups.matches()) {
+                    metric.append(NSQ_PREFIX).append(statsd_6_groups.group(4)).append(PERIOD).append(statsd_6_groups.group(6));
+                    instance = statsd_6_groups.group(5);
                 } else {
-                    instance = vl.getTypeInstance().substring(0, period);
-                    metric.append(STATSD_PREFIX).append(PERIOD).append(vl.getTypeInstance().substring(period + 1));
+                    // Handle StatsD metrics of unknown formats. If there is a
+                    // period in the metric name, use everything up to that as
+                    // the instance.
+                    int period = typeInstance.indexOf('.');
+                    if (-1 == period) {
+                        metric.append(STATSD_PREFIX).append(PERIOD).append(typeInstance);
+                    } else {
+                        instance = typeInstance.substring(0, period);
+                        metric.append(STATSD_PREFIX).append(PERIOD).append(typeInstance.substring(period + 1));
+                    }
                 }
-            }
-            timestamp = vl.getTime();
-            if (null != instance) {
-                tags.append(INSTANCE).append(instance);
-            }
-        } else if (vl.getPlugin().equals("ethstat")) {
-            metric.append("sys.ethstat.");
-            if (vl.getTypeInstance().contains("queue")) {
-                Matcher m = ETHSTAT_PATTERN.matcher(vl.getTypeInstance());
-                if (m.matches()) {
-                    metric.append(m.group(1)).append("_").append(m.group(3));
-                    tags.append(" queue=").append(m.group(2));
+                if (null != instance) {
+                    addTag(tagMap, INSTANCE, instance);
+                }
+                break;
+            case "ethstat":
+                metric.append("sys.ethstat.");
+                if (typeInstance.contains("queue")) {
+                    Matcher ethstat_m = ETHSTAT_PATTERN.matcher(typeInstance);
+                    if (ethstat_m.matches()) {
+                        metric.append(ethstat_m.group(1)).append("_").append(ethstat_m.group(3));
+                        addTag(tagMap, "queue", ethstat_m.group(2));
+                    } else {
+                        metric.append(typeInstance);
+                    }
                 } else {
-                    metric.append(vl.getTypeInstance());
+                    metric.append(typeInstance);
                 }
-            } else {
-                metric.append(vl.getTypeInstance());
-            }
-            tags.append(INSTANCE).append(vl.getPluginInstance());
-        } else if (vl.getPlugin().equals("hddtemp")) {
-            metric.append("sys.hddtemp.").append(vl.getType());
-            tags.append(INSTANCE).append(vl.getTypeInstance());
-        } else if (vl.getPlugin().equals("smart")) {
-            int code = -1;
-            String name = null;
-            if (vl.getTypeInstance().startsWith("attribute-")) {
-                int hyphen = vl.getTypeInstance().indexOf('-');
-                code = Integer.parseInt(vl.getTypeInstance().substring(hyphen + 1));
-                name = smart.get(code);
-            }
-            if (code == -1) {
-                if (notEmpty(vl.getTypeInstance())) {
-                    metric.append("sys.smart.").append(vl.getTypeInstance());
+                addTag(tagMap, INSTANCE, pluginInstance);
+                break;
+            case "hddtemp":
+                metric.append("sys.hddtemp.").append(type);
+                addTag(tagMap, INSTANCE, typeInstance);
+                break;
+            case "smart":
+                int code = -1;
+                String name = null;
+                if (typeInstance.startsWith("attribute-")) {
+                    int hyphen = typeInstance.indexOf('-');
+                    code = Integer.parseInt(typeInstance.substring(hyphen + 1));
+                    name = smart.get(code);
+                }
+                if (code == -1) {
+                    if (notEmpty(typeInstance)) {
+                        metric.append("sys.smart.").append(typeInstance);
+                    } else {
+                        metric.append("sys.smart.").append(type);
+                    }
                 } else {
-                    metric.append("sys.smart.").append(vl.getType());
+                    metric.append("sys.smart.").append(name);
+                    addTag(tagMap, CODE, Integer.toString(code));
                 }
-            } else {
-                metric.append("sys.smart.").append(name);
-                tags.append(CODE).append(code);
-            }
-            tags.append(INSTANCE).append(vl.getPluginInstance());
-        } else if (vl.getPlugin().equals("sensors")) {
-            String instance = "";
-            if (vl.getTypeInstance().startsWith("temp")) {
-                instance = vl.getTypeInstance().substring(4);
-            }
-            metric.append("sys.sensors.").append(vl.getType()).append(PERIOD).append(vl.getPluginInstance());
-            tags.append(INSTANCE).append(instance);
-        } else if (vl.getPlugin().equals("haproxy")) {
-            metric.append("sys.haproxy.").append(vl.getTypeInstance());
-            Matcher m = HAPROXY_PATTERN.matcher(vl.getPluginInstance());
-            if (m.matches()) {
-                tags.append(" ").append(m.group(1));
-                tags.append(" ").append(m.group(2));
-            }
-        } else if (vl.getPlugin().equals("ipmi")) {
-            metric.append("sys.ipmi.").append(vl.getType());
-            tags.append(INSTANCE).append(vl.getTypeInstance().replaceAll(" ", "_"));
-        } else if (vl.getPlugin().equals("snmp")) {
-            metric.append("sys.snmp.").append(vl.getType());
-            tags.append(INSTANCE).append(vl.getTypeInstance().replaceAll(" ", "_"));
-        } else if (vl.getPlugin().equals("GenericJMX")) {
-            metric.append("sys.").append(vl.getPlugin()).append(PERIOD).append(vl.getType()).append(PERIOD).append(vl.getTypeInstance());
-            String[] pluginInstanceSplit = vl.getPluginInstance().split("-");
-            if (pluginInstanceSplit.length > 0) {
-                tags.append(INSTANCE).append(pluginInstanceSplit[0].replaceAll(" ", "_"));
-            }
-            if (pluginInstanceSplit.length > 1) {
-                tags.append(NAME).append(pluginInstanceSplit[1].replaceAll(" ", "_"));
-            }
-        } else if (notEmpty(vl.getTypeInstance()) && notEmpty(vl.getType()) && notEmpty(vl.getPluginInstance()) && notEmpty(vl.getPlugin())) {
-            metric.append("sys.").append(vl.getPlugin()).append(PERIOD).append(vl.getType()).append(PERIOD).append(vl.getTypeInstance());
-            tags.append(INSTANCE).append(vl.getPluginInstance().replaceAll(" ", "_"));
-        } else if (notEmpty(vl.getTypeInstance()) && notEmpty(vl.getType()) && notEmpty(vl.getPlugin())) {
-            metric.append("sys.").append(vl.getPlugin()).append(PERIOD).append(vl.getType()).append(PERIOD).append(vl.getTypeInstance());
-        } else if (notEmpty(vl.getType()) && notEmpty(vl.getPluginInstance()) && notEmpty(vl.getPlugin())) {
-            metric.append("sys.").append(vl.getPlugin()).append(PERIOD).append(vl.getType());
-            tags.append(INSTANCE).append(vl.getPluginInstance().replaceAll(" ", "_"));
-        } else if (notEmpty(vl.getType()) && notEmpty(vl.getPlugin())) {
-            metric.append("sys.").append(vl.getPlugin()).append(PERIOD).append(vl.getType());
-        } else {
-            Collectd.logWarning("Unhandled metric: " + vl.toString());
-            return;
+                addTag(tagMap, INSTANCE, pluginInstance);
+                break;
+            case "sensors":
+                if (typeInstance.startsWith("temp")) {
+                    addTag(tagMap, INSTANCE, typeInstance.substring(4));
+                }
+                metric.append("sys.sensors.").append(type).append(PERIOD).append(pluginInstance);
+                break;
+            case "haproxy":
+                metric.append("sys.haproxy.").append(typeInstance);
+                Matcher hadoop_m2 = HAPROXY_PATTERN.matcher(pluginInstance);
+                if (hadoop_m2.matches()) {
+                    addTag(tagMap, hadoop_m2.group(1));
+                    addTag(tagMap, hadoop_m2.group(2));
+                }
+                break;
+            case "ipmi":
+            case "snmp":
+                metric.append("sys.").append(plugin).append(PERIOD).append(type);
+                addTag(tagMap, INSTANCE, typeInstance.replaceAll(" ", "_"));
+                break;
+            case "GenericJMX":
+                metric.append("sys.").append(plugin).append(PERIOD).append(type).append(PERIOD).append(typeInstance);
+                String[] pluginInstanceSplit = pluginInstance.split("-");
+                if (pluginInstanceSplit.length > 0) {
+                    addTag(tagMap, INSTANCE, pluginInstanceSplit[0].replaceAll(" ", "_"));
+                }
+                if (pluginInstanceSplit.length > 1) {
+                    addTag(tagMap, NAME, pluginInstanceSplit[1].replaceAll(" ", "_"));
+                }
+            default:
+                if (notEmpty(type) && notEmpty(typeInstance) && notEmpty(plugin) && notEmpty(pluginInstance)) {
+                    metric.append("sys.").append(plugin).append(PERIOD).append(type).append(PERIOD).append(typeInstance);
+                    addTag(tagMap, INSTANCE, pluginInstance.replaceAll(" ", "_"));
+                } else if (notEmpty(type) && notEmpty(typeInstance) && notEmpty(plugin)) {
+                    metric.append("sys.").append(plugin).append(PERIOD).append(type).append(PERIOD).append(typeInstance);
+                } else if (notEmpty(type) && notEmpty(plugin) && notEmpty(pluginInstance)) {
+                    metric.append("sys.").append(plugin).append(PERIOD).append(type);
+                    addTag(tagMap, INSTANCE, pluginInstance.replaceAll(" ", "_"));
+                } else if (notEmpty(type) && notEmpty(plugin)) {
+                    metric.append("sys.").append(plugin).append(PERIOD).append(type);
+                } else {
+                    Collectd.logWarning("Unhandled metric: " + vl);
+                    return;
+                }
+                break;
         }
         final String metricName = metric.toString().replaceAll(" ", "_");
         for (int i = 0; i < vl.getValues().size(); i++) {
-            StringBuilder tagsWithSample = new StringBuilder(tags.toString());
-            String sampleName = vl.getDataSet().getDataSources().get(i).getName();
-            int type = vl.getDataSet().getDataSources().get(i).getType();
-            String sampleType = convertType(type);
-            if (null != sampleName) {
-                tagsWithSample.append(SAMPLE).append(sampleName);
-            }
+            int dataSourceSampleType = vl.getDataSet().getDataSources().get(i).getType();
+            String sampleType = convertType(dataSourceSampleType);
             if (null != sampleType) {
-                tagsWithSample.append(SAMPLE_TYPE).append(sampleType);
+                addTag(tagMap, SAMPLE_TYPE, sampleType);
             }
             Double value = vl.getValues().get(i).doubleValue();
-            String datapoint = MessageFormat.format(PUT, metricName, timestamp.toString(), value.toString(), tagsWithSample.toString());
-            Collectd.logDebug("Writing: " + datapoint);
-            write(datapoint, out);
+            String tagString = tagMap.entrySet().stream().map(e -> e.getKey() + "=" + e.getValue()).collect(Collectors.joining(" "));
+            String datapoint = MessageFormat.format(PUT, metricName, timestamp.toString(), value.toString(), tagString);
+            logDebug(String.format("Writing: %s", datapoint));
+            write(datapoint + "\n", out);
+        }
+    }
+
+    private void logDebug(String s) {
+        // collectd must be compiled with debug enabled in order for log level debug to work
+        if (debug) {
+            Collectd.logInfo(s);
+        } else {
+            Collectd.logDebug(s);
         }
     }
 
     private void debugValueList(ValueList vl) {
-        Collectd.logDebug("Input: " + vl.toString());
-        Collectd.logDebug("Plugin: " + vl.getPlugin());
-        Collectd.logDebug("PluginInstance: " + vl.getPluginInstance());
-        Collectd.logDebug("Type: " + vl.getType());
-        Collectd.logDebug("TypeInstance: " + vl.getTypeInstance());
-        for (int i = 0; i < vl.getValues().size(); i++) {
-            Number value = vl.getValues().get(i);
-            DataSource ds = vl.getDataSet().getDataSources().get(i);
-            Collectd.logDebug(convertType(ds.getType()) + " " + ds.getName() + " = " + value);
+        try {
+            StringBuilder sb = new StringBuilder();
+            sb.append(String.format("Plugin:%s", vl.getPlugin()));
+            if (vl.getPluginInstance() != null && !vl.getPluginInstance().isBlank()) {
+                sb.append(String.format(" PluginInstance:%s", vl.getPluginInstance()));
+            }
+            sb.append(String.format(" Type:%s", vl.getType()));
+            sb.append(String.format(" TypeInstance:%s", vl.getTypeInstance()));
+            for (int i = 0; i < vl.getValues().size(); i++) {
+                Number value = vl.getValues().get(i);
+                sb.append(String.format(" Value:%s", value.toString()));
+            }
+            logDebug(sb.toString());
+        } catch (Exception e) {
+            e.printStackTrace();
+            logDebug(e.getMessage());
         }
     }
 
     private String convertType(int type) {
-        String result = null;
         switch (type) {
             case 0:
-                result = COUNTER;
-                break;
+                return COUNTER;
             case 1:
-                result = GAUGE;
-                break;
+                return GAUGE;
             case 2:
-                result = DERIVE;
-                break;
+                return DERIVE;
             case 3:
-                result = ABSOLUTE;
-                break;
+                return ABSOLUTE;
             default:
-                result = GAUGE;
-                break;
+                return GAUGE;
         }
-        return result;
     }
 
     private boolean notEmpty(String arg) {

collectd-plugin-parent/src/test/java/timely/collectd/plugin/PluginParentTest.java

@@ -26,7 +26,7 @@ public class PluginParentTest {
     public class TestPlugin extends CollectDPluginParent {
 
         public TestPlugin() {
-            super.addlTags.add(" addl1=foo");
+            super.addlTags.put("addl1", "foo");
         }
 
         @Override

collectd-timely-plugin/pom.xml

@@ -4,7 +4,7 @@
   <parent>
     <groupId>timely</groupId>
     <artifactId>timely</artifactId>
-    <version>2.1.2</version>
+    <version>2.1.4-SNAPSHOT</version>
   </parent>
   <artifactId>collectd-timely-plugin</artifactId>
   <dependencies>

common/pom.xml

@@ -4,7 +4,7 @@
   <parent>
     <groupId>timely</groupId>
     <artifactId>timely</artifactId>
-    <version>2.1.2</version>
+    <version>2.1.4-SNAPSHOT</version>
   </parent>
   <artifactId>timely-common</artifactId>
   <dependencies>

common/src/main/java/timely/api/request/timeseries/QueryRequest.java

@@ -32,6 +32,7 @@ public class QueryRequest extends AuthenticatedRequest implements HttpGetRequest
         private boolean counter = false;
         private long counterMax = 0;
         private long resetValue = 0;
+        private Optional<String> interval = Optional.empty();
 
         public RateOption() {}
 
@@ -84,12 +85,21 @@ public class QueryRequest extends AuthenticatedRequest implements HttpGetRequest
             this.resetValue = resetValue;
         }
 
+        public Optional<String> getInterval() {
+            return interval;
+        }
+
+        public void setInterval(Optional<String> interval) {
+            this.interval = interval;
+        }
+
         @Override
         public String toString() {
             ToStringBuilder tsb = new ToStringBuilder(this);
             tsb.append("counter", counter);
             tsb.append("counterMax", counterMax);
             tsb.append("resetValue", resetValue);
+            tsb.append("interval", interval);
             return tsb.toString();
         }
 
@@ -99,6 +109,7 @@ public class QueryRequest extends AuthenticatedRequest implements HttpGetRequest
             hcb.append(counter);
             hcb.append(counterMax);
             hcb.append(resetValue);
+            hcb.append(interval);
             return hcb.toHashCode();
         }
 
@@ -536,9 +547,12 @@ public class QueryRequest extends AuthenticatedRequest implements HttpGetRequest
                                         options.setCounter(rateOptions[x].endsWith("counter"));
                                         break;
                                     case 1:
-                                        options.setCounterMax(Long.parseLong(rateOptions[x]));
+                                        options.setInterval(Optional.of(rateOptions[x]));
                                         break;
                                     case 2:
+                                        options.setCounterMax(Long.parseLong(rateOptions[x]));
+                                        break;
+                                    case 3:
                                         options.setResetValue(Long.parseLong(rateOptions[x]));
                                         break;
                                     default:

grafana/pom.xml

@@ -4,7 +4,7 @@
   <parent>
     <groupId>timely</groupId>
     <artifactId>timely</artifactId>
-    <version>2.1.2</version>
+    <version>2.1.4-SNAPSHOT</version>
   </parent>
   <artifactId>timely-app</artifactId>
   <packaging>pom</packaging>

grafana/timely-app/dist/plugin.json

@@ -28,8 +28,8 @@
       }
     ],
     "screenshots": [],
-    "version": "2.1.2",
-    "updated": "2024-03-06"
+    "version": "2.1.4-SNAPSHOT",
+    "updated": "2024-03-13"
   },
   "dependencies": {
     "classnames": "2.2.6",

grafana/timely-app/go.mod

@@ -1,11 +1,96 @@
 module github.com/NationalSecurityAgency/timely-datasource
 
-go 1.14
+go 1.21
+
+toolchain go1.22.1
 
 require (
-	github.com/grafana/grafana-plugin-sdk-go v0.111.0
-	github.com/hashicorp/go-hclog v0.16.1
-	github.com/mattn/go-colorable v0.1.6 // indirect
-	github.com/prometheus/common v0.29.0 // indirect
-	golang.org/x/net v0.17.0 // indirect
+	github.com/grafana/grafana-plugin-sdk-go v0.218.0
+	github.com/hashicorp/go-hclog v1.6.2
+)
+
+require (
+	github.com/BurntSushi/toml v1.3.2 // indirect
+	github.com/apache/arrow/go/v15 v15.0.0 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/cheekybits/genny v1.0.0 // indirect
+	github.com/chromedp/cdproto v0.0.0-20220208224320-6efb837e6bc2 // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
+	github.com/elazarl/goproxy v0.0.0-20230731152917-f99041a5c027 // indirect
+	github.com/fatih/color v1.15.0 // indirect
+	github.com/fsnotify/fsnotify v1.4.9 // indirect
+	github.com/getkin/kin-openapi v0.120.0 // indirect
+	github.com/go-logr/logr v1.4.1 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/go-openapi/jsonpointer v0.19.6 // indirect
+	github.com/go-openapi/swag v0.22.4 // indirect
+	github.com/goccy/go-json v0.10.2 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/google/flatbuffers v23.5.26+incompatible // indirect
+	github.com/google/go-cmp v0.6.0 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/gorilla/mux v1.8.0 // indirect
+	github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // indirect
+	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.0 // indirect
+	github.com/hashicorp/go-plugin v1.6.0 // indirect
+	github.com/hashicorp/yamux v0.1.1 // indirect
+	github.com/invopop/yaml v0.2.0 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/klauspost/compress v1.16.7 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.5 // indirect
+	github.com/magefile/mage v1.15.0 // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/mattetti/filebuffer v1.0.1 // indirect
+	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-isatty v0.0.19 // indirect
+	github.com/mattn/go-runewidth v0.0.9 // indirect
+	github.com/mitchellh/go-testing-interface v1.14.1 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect
+	github.com/oklog/run v1.1.0 // indirect
+	github.com/olekukonko/tablewriter v0.0.5 // indirect
+	github.com/perimeterx/marshmallow v1.1.5 // indirect
+	github.com/pierrec/lz4/v4 v4.1.18 // indirect
+	github.com/prometheus/client_golang v1.18.0 // indirect
+	github.com/prometheus/client_model v0.5.0 // indirect
+	github.com/prometheus/common v0.46.0 // indirect
+	github.com/prometheus/procfs v0.12.0 // indirect
+	github.com/russross/blackfriday/v2 v2.1.0 // indirect
+	github.com/smartystreets/goconvey v1.6.4 // indirect
+	github.com/unknwon/bra v0.0.0-20200517080246-1e3013ecaff8 // indirect
+	github.com/unknwon/com v1.0.1 // indirect
+	github.com/unknwon/log v0.0.0-20150304194804-e617c87089d3 // indirect
+	github.com/urfave/cli v1.22.14 // indirect
+	github.com/zeebo/xxh3 v1.0.2 // indirect
+	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.49.0 // indirect
+	go.opentelemetry.io/contrib/propagators/jaeger v1.22.0 // indirect
+	go.opentelemetry.io/contrib/samplers/jaegerremote v0.18.0 // indirect
+	go.opentelemetry.io/otel v1.24.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.24.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.24.0 // indirect
+	go.opentelemetry.io/otel/metric v1.24.0 // indirect
+	go.opentelemetry.io/otel/sdk v1.24.0 // indirect
+	go.opentelemetry.io/otel/trace v1.24.0 // indirect
+	go.opentelemetry.io/proto/otlp v1.1.0 // indirect
+	golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect
+	golang.org/x/mod v0.13.0 // indirect
+	golang.org/x/net v0.22.0 // indirect
+	golang.org/x/sys v0.18.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
+	golang.org/x/tools v0.14.0 // indirect
+	golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
+	google.golang.org/genproto v0.0.0-20240123012728-ef4313101c80 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20240123012728-ef4313101c80 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80 // indirect
+	google.golang.org/grpc v1.62.1 // indirect
+	google.golang.org/protobuf v1.33.0 // indirect
+	gopkg.in/fsnotify/fsnotify.v1 v1.4.7 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

grafana/timely-app/package.json

@@ -1,6 +1,6 @@
 {
   "name": "timely-datasource",
-  "version": "2.1.2",
+  "version": "2.1.4-SNAPSHOT",
   "description": "Timely",
   "backend": true,
   "alerting": true,

grafana/timely-app/pkg/timely.go

@@ -398,6 +398,13 @@ func (td *TimelyDatasource) convertTimelyQuery(timelyQueryForm *TimelyQueryForm)
 		timelyQuery.RateOptions.Counter = timelyQueryForm.IsCounter
 	}
 
+    if len(timelyQueryForm.RateInterval) > 0 {
+        rateInterval, error := strconv.ParseInt(timelyQueryForm.RateInterval, 10, 32)
+        if error == nil {
+            timelyQuery.RateOptions.Interval = int32(rateInterval)
+        }
+    }
+
 	if len(timelyQueryForm.CounterMax) > 0 {
 		counterMax, error := strconv.ParseInt(timelyQueryForm.CounterMax, 10, 32)
 		if error == nil {

grafana/timely-app/pkg/types.go

@@ -11,6 +11,7 @@ type TimelyQueryForm struct {
     Tags                 map[string]string `json:"tags"`
     Filters              []Filter          `json:"filters"`
     ShouldComputeRate    bool              `json:"shouldComputeRate"`
+    RateInterval         string            `json:"rateInterval"`
     IsCounter            bool              `json:"isCounter"`
     CounterMax           string            `json:"counterMax"`
     CounterResetValue    string            `json:"counterResetValue"`
@@ -34,6 +35,7 @@ type TimelyQueryServer struct {
     Tags                 map[string]string `json:"tags"`
     Filters              []Filter          `json:"filters"`
     ShouldComputeRate    bool              `json:"shouldComputeRate"`
+    RateInterval         string            `json:"rateInterval"`
     IsCounter            bool              `json:"isCounter"`
     CounterMax           string            `json:"counterMax"`
     CounterResetValue    string            `json:"counterResetValue"`
@@ -50,6 +52,7 @@ type Filter struct {
 }
 
 type RateOptions struct {
+    Interval   int32 `json:"interval"`
     Counter    bool  `json:"counter"`
     CounterMax int32 `json:"counterMax"`
     ResetValue int32 `json:"resetValue"`
@@ -96,4 +99,4 @@ type TimelyErrorResponse struct {
     ResponseCode                    int32 `json:responseCode`
     Message                         string `json:message`
     DetailMessage                   string `json:detailMessage`
-}
+}

grafana/timely-app/src/QueryEditor.tsx

@@ -37,6 +37,7 @@ export class QueryEditor extends Component<Props, State> {
       downsampleInterval: '1m',
       downsampleAggregator: 'avg',
       shouldComputeRate: false,
+      rateInterval: '1m',
       isCounter: false,
       tags: {},
     };
@@ -123,6 +124,15 @@ export class QueryEditor extends Component<Props, State> {
     });
   };
 
+  onRateIntervalChange = (option: React.FormEvent<HTMLInputElement>) => {
+    const value = e.target.value;
+    this.setState((state, props) => {
+      var query = Object.assign(state.query, { rateInterval: value });
+      this.updateQuery(query);
+      return { query: query };
+    });
+  };
+
   onCounterChange = (option: React.FormEvent<HTMLInputElement>) => {
     const checked = option.currentTarget.checked;
     this.setState((state, props) => {
@@ -247,7 +257,7 @@ export class QueryEditor extends Component<Props, State> {
     const { metric, aggregator, alias } = this.state.query;
     const { downsampleInterval, downsampleAggregator, disableDownsampling } = this.state.query;
     const { tags } = this.state.query;
-    const { shouldComputeRate, isCounter, counterResetValue, counterMax } = this.state.query;
+    const { shouldComputeRate, rateInterval, isCounter, counterResetValue, counterMax } = this.state.query;
     const aggregatorOptions: Array<{}> = _.map(this.state.aggregatorTypes, (value: string) => ({
       label: value,
       value: value,
@@ -345,7 +355,7 @@ export class QueryEditor extends Component<Props, State> {
               spellCheck={false}
               value={downsampleInterval || ''}
               onChange={this.onDownsampleChange}
-              placeholder={'interval'}
+              placeholder={'downsample interval'}
             />
           </div>
           <div className="gf-form timely">
@@ -390,6 +400,13 @@ export class QueryEditor extends Component<Props, State> {
               onChange={this.onRateChange}
             />
             <div hidden={!shouldComputeRate}>
+              <Input
+                  className={'min-width-6 flex-grow-1'}
+                  spellCheck={false}
+                  value={rateInterval || ''}
+                  onChange={this.onRateIntervalChange}
+                  placeholder={'rate interval'}
+              />
               <Switch
                 label={'Counter'}
                 labelClass="width-10"

grafana/timely-app/src/types.ts

@@ -9,6 +9,7 @@ export interface TimelyQueryForm extends DataQuery {
   downsampleAggregator?: string;
   tags: { [key: string]: string };
   shouldComputeRate?: boolean;
+  rateInterval?: string;
   isCounter?: boolean;
   counterMax?: string;
   counterResetValue?: string;

pom.xml

@@ -3,7 +3,7 @@
   <modelVersion>4.0.0</modelVersion>
   <groupId>timely</groupId>
   <artifactId>timely</artifactId>
-  <version>2.1.2</version>
+  <version>2.1.4-SNAPSHOT</version>
   <packaging>pom</packaging>
   <name>Timely</name>
   <description>A secure and scalable time series database based on Apache Accumulo</description>

server/bin/timely-server.sh

@@ -4,7 +4,6 @@ if [[ $(uname) == "Darwin" ]]; then
         THIS_SCRIPT=$(python -c 'import os,sys; print os.path.realpath(sys.argv[1])' "$0")
 else
         THIS_SCRIPT=$(readlink -f "$0")
-        TCNATIVE_SUFFIX="so"
         HW=`uname -m`
         OS_PATH=linux-${HW}
 fi
@@ -56,14 +55,14 @@ if [ "$PID" == "" ]; then
 
     if [[ -n "$OS_PATH" ]]; then
       pushd "${BASE_DIR}/bin" || exit
-      "${JAVA_HOME}"/bin/jar xf "${LIB_DIR}"/netty-tcnative-boringssl-static-*-"${OS_PATH}".jar META-INF/native/libnetty_tcnative_linux_"${HW}".so
-      "${JAVA_HOME}"/bin/jar xf "${LIB_DIR}"/netty-transport-native-epoll-*-"${OS_PATH}".jar META-INF/native/libnetty_transport_native_epoll_"${HW}".so
+      "${JAVA_HOME}"/bin/jar xf "${LIB_DIR}"/netty-tcnative-boringssl-static-*-"${OS_PATH}".jar ${NATIVE_DIR}/libnetty_tcnative_linux_"${HW}".so
+      "${JAVA_HOME}"/bin/jar xf "${LIB_DIR}"/netty-transport-native-epoll-*-"${OS_PATH}".jar ${NATIVE_DIR}/libnetty_transport_native_epoll_"${HW}".so
       popd || exit
       JVM_ARGS="${JVM_ARGS} -Djava.library.path=${NATIVE_DIR}"
     fi
 
-    echo "${JAVA_HOME}/bin/java ${JVM_ARGS} -jar ${THIS_DIR}/timely-server*-exec.jar timelyserver --instance=${INSTANCE}"
-    nohup "${JAVA_HOME}"/bin/java ${JVM_ARGS} -jar "${THIS_DIR}"/timely-server-*-exec.jar timelyserver --instance="${INSTANCE}" >> "${LOG_DIR}/timely-server${INSTANCE}.out" 2>&1 &
+    echo "${JAVA_HOME}/bin/java ${JVM_ARGS} -jar ${BIN_DIR}/timely-server*-exec.jar timelyserver --instance=${INSTANCE}"
+    nohup "${JAVA_HOME}"/bin/java ${JVM_ARGS} -jar "${BIN_DIR}"/timely-server-*-exec.jar timelyserver --instance="${INSTANCE}" >> "${LOG_DIR}/timely-server${INSTANCE}.out" 2>&1 &
 else
     echo "timely-server --instance=${INSTANCE} already running with pid ${PID}"
 fi

server/bin/util/get-metrics.sh

@@ -1,11 +1,15 @@
 #!/bin/bash
 
 if [[ `uname` == "Darwin" ]]; then
-        THIS_SCRIPT=`python -c 'import os,sys; print os.path.realpath(sys.argv[1])' $0`
+    THIS_SCRIPT=`python -c 'import os,sys; print os.path.realpath(sys.argv[1])' $0`
 else
-        THIS_SCRIPT=`readlink -f $0`
+    THIS_SCRIPT=`readlink -f $0`
+    HW=`uname -m`
+    OS_PATH=linux-${HW}
 fi
 
+export INSTANCE="1"
+
 THIS_DIR="${THIS_SCRIPT%/*}"
 BASE_DIR=$(cd $THIS_DIR/../.. && pwd)
 LIB_DIR="${BASE_DIR}/lib"
@@ -21,5 +25,13 @@ JVM_ARGS="-Xmx256m -Xms128m"
 JVM_ARGS="${JVM_ARGS} -Dlogging.config=${CONF_DIR}/log4j2-console.yml"
 JVM_ARGS="${JVM_ARGS} -Dlog4j2.contextSelector=org.apache.logging.log4j.core.async.AsyncLoggerContextSelector"
 
-echo "$JAVA_HOME/bin/java ${JVM_ARGS} -jar ${THIS_DIR}/timely-server*-exec.jar metrics"
-$JAVA_HOME/bin/java ${JVM_ARGS} -jar ${THIS_DIR}/timely-server-*-exec.jar metrics
+if [[ -n "$OS_PATH" ]]; then
+    pushd "${BASE_DIR}/bin" || exit
+    "${JAVA_HOME}"/bin/jar xf "${LIB_DIR}"/netty-tcnative-boringssl-static-*.Final-"${OS_PATH}".jar ${NATIVE_DIR}/libnetty_tcnative_linux_"${HW}".so
+    "${JAVA_HOME}"/bin/jar xf "${LIB_DIR}"/netty-transport-native-epoll-*-"${OS_PATH}".jar ${NATIVE_DIR}/libnetty_transport_native_epoll_"${HW}".so
+    popd || exit
+    JVM_ARGS="${JVM_ARGS} -Djava.library.path=${NATIVE_DIR}"
+fi
+
+echo "$JAVA_HOME/bin/java ${JVM_ARGS} -jar ${BIN_DIR}/timely-server*-exec.jar metrics"
+$JAVA_HOME/bin/java ${JVM_ARGS} -jar ${BIN_DIR}/timely-server-*-exec.jar metrics

server/bin/util/get-tablet-summary.sh

@@ -1,11 +1,15 @@
 #!/bin/bash
 
 if [[ `uname` == "Darwin" ]]; then
-        THIS_SCRIPT=`python -c 'import os,sys; print os.path.realpath(sys.argv[1])' $0`
+    THIS_SCRIPT=`python -c 'import os,sys; print os.path.realpath(sys.argv[1])' $0`
 else
-        THIS_SCRIPT=`readlink -f $0`
+    THIS_SCRIPT=`readlink -f $0`
+    HW=`uname -m`
+    OS_PATH=linux-${HW}
 fi
 
+export INSTANCE="1"
+
 THIS_DIR="${THIS_SCRIPT%/*}"
 BASE_DIR=$(cd $THIS_DIR/../.. && pwd)
 LIB_DIR="${BASE_DIR}/lib"
@@ -21,5 +25,13 @@ JVM_ARGS="-Xmx1024m -Xms128m"
 JVM_ARGS="${JVM_ARGS} -Dlogging.config=${CONF_DIR}/log4j2-console.yml"
 JVM_ARGS="${JVM_ARGS} -Dlog4j2.contextSelector=org.apache.logging.log4j.core.async.AsyncLoggerContextSelector"
 
-echo "$JAVA_HOME/bin/java ${JVM_ARGS} -jar ${THIS_DIR}/timely-server*-exec.jar tablet --spring.profiles.active=timely,tablet"
-$JAVA_HOME/bin/java ${JVM_ARGS} -jar ${THIS_DIR}/timely-server-*-exec.jar tablet --spring.profiles.active=timely,tablet
+if [[ -n "$OS_PATH" ]]; then
+    pushd "${BASE_DIR}/bin" || exit
+    "${JAVA_HOME}"/bin/jar xf "${LIB_DIR}"/netty-tcnative-boringssl-static-*.Final-"${OS_PATH}".jar ${NATIVE_DIR}/libnetty_tcnative_linux_"${HW}".so
+    "${JAVA_HOME}"/bin/jar xf "${LIB_DIR}"/netty-transport-native-epoll-*-"${OS_PATH}".jar ${NATIVE_DIR}/libnetty_transport_native_epoll_"${HW}".so
+    popd || exit
+    JVM_ARGS="${JVM_ARGS} -Djava.library.path=${NATIVE_DIR}"
+fi
+
+echo "$JAVA_HOME/bin/java ${JVM_ARGS} -jar ${BIN_DIR}/timely-server*-exec.jar tablet --spring.profiles.active=timely,tablet"
+$JAVA_HOME/bin/java ${JVM_ARGS} -jar ${BIN_DIR}/timely-server-*-exec.jar tablet --spring.profiles.active=timely,tablet

server/pom.xml

@@ -4,7 +4,7 @@
   <parent>
     <groupId>timely</groupId>
     <artifactId>timely</artifactId>
-    <version>2.1.2</version>
+    <version>2.1.4-SNAPSHOT</version>
   </parent>
   <artifactId>timely-server</artifactId>
   <description>Timely Server</description>

server/src/main/java/timely/server/sample/iterators/RateIterator.java

@@ -1,5 +1,7 @@
 package timely.server.sample.iterators;
 
+import static org.apache.accumulo.core.conf.ConfigurationTypeHelper.getTimeInMillis;
+
 import java.io.IOException;
 import java.util.HashMap;
 import java.util.Iterator;
@@ -20,12 +22,14 @@ import timely.api.request.timeseries.QueryRequest;
 public class RateIterator extends TimeSeriesGroupingIterator {
 
     private static final Logger log = LoggerFactory.getLogger(RateIterator.class);
-    private static final String COUNTER_MAX = "rate.counter.max";
-    private static final String RATE_RESET_VALUE = "rate.reset.value";
+    public static final String COUNTER_MAX = "rate.counter.max";
+    public static final String RATE_RESET_VALUE = "rate.reset.value";
+    public static final String RATE_INTERVAL = "rate.interval";
 
     private boolean isCounter = false;
     private long maxCounter = 0;
     private long resetValue = 0;
+    private long interval = 1;
 
     public static void setRateOptions(IteratorSetting is, QueryRequest.RateOption options) {
         if (options != null && options.isCounter()) {
@@ -33,6 +37,11 @@ public class RateIterator extends TimeSeriesGroupingIterator {
             is.addOption(COUNTER_MAX, Long.toString(options.getCounterMax()));
             is.addOption(RATE_RESET_VALUE, Long.toString(options.getResetValue()));
         }
+        if (options != null && options.getInterval().isPresent()) {
+            is.addOption(RATE_INTERVAL, Long.toString(getTimeInMillis(options.getInterval().get())));
+        } else {
+            is.addOption(RATE_INTERVAL, "1");
+        }
     }
 
     @Override
@@ -41,8 +50,11 @@ public class RateIterator extends TimeSeriesGroupingIterator {
             this.isCounter = true;
             this.maxCounter = Long.parseLong(options.get(COUNTER_MAX));
             this.resetValue = Long.parseLong(options.get(RATE_RESET_VALUE));
-            log.trace("Setting rate counter options cm:{}, rv:{}", this.maxCounter, this.resetValue);
         }
+        if (options.containsKey(RATE_INTERVAL)) {
+            this.interval = Long.parseLong(options.get(RATE_INTERVAL));
+        }
+        log.trace("Setting rate options interval:{} isCounter:{} cm:{}, rv:{}", this.interval, this.isCounter, this.maxCounter, this.resetValue);
         Map<String,String> opts = new HashMap<>(options);
         opts.put(FILTER, "-1,1");
         super.init(source, opts, env);
@@ -76,7 +88,9 @@ public class RateIterator extends TimeSeriesGroupingIterator {
         if (timeDiff == 0) {
             return 0.0D;
         }
-        Double result = ((secondVal - firstVal) / timeDiff);
+        // (secondVal - firstVal) / timeDiff will be value per millisecond, so multiply by interval to get
+        // value per interval. Example 60000 milliseconds in 1m, so multiply by 60000 milliseconds
+        Double result = ((secondVal - firstVal) / timeDiff * interval);
         log.trace("compute - result: {}", result);
 
         if (isCounter && resetValue > 0 && result > resetValue) {

server/src/main/java/timely/server/store/DataStore.java

@@ -700,6 +700,12 @@ public class DataStore {
                             if (query.isRate()) {
                                 log.trace("Adding rate iterator");
                                 IteratorSetting rate = new IteratorSetting(499, RateIterator.class);
+                                RateOption rateOptions = query.getRateOptions();
+                                // if there is no rate interval set, then use the downsample value
+                                // so that the result is the change per downsample period
+                                if (rateOptions != null && !rateOptions.getInterval().isPresent()) {
+                                    rateOptions.setInterval(Optional.of(downsample + "ms"));
+                                }
                                 RateIterator.setRateOptions(rate, query.getRateOptions());
                                 scanner.addScanIterator(rate);
                             }

server/src/main/java/timely/server/store/cache/DataStoreCache.java

@@ -14,6 +14,7 @@ import java.util.Iterator;
 import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
 import java.util.TreeSet;
 import java.util.concurrent.Executors;
@@ -765,7 +766,13 @@ public class DataStoreCache {
             if (subQuery.isRate()) {
                 log.trace("Adding rate iterator");
                 IteratorSetting rate = new IteratorSetting(499, RateIterator.class);
-                RateIterator.setRateOptions(rate, subQuery.getRateOptions());
+                QueryRequest.RateOption rateOptions = subQuery.getRateOptions();
+                // if there is no rate interval set, then use the downsample value
+                // so that the result is the change per downsample period
+                if (rateOptions != null && !rateOptions.getInterval().isPresent()) {
+                    rateOptions.setInterval(Optional.of(downsamplePeriod + "ms"));
+                }
+                RateIterator.setRateOptions(rate, rateOptions);
                 RateIterator rateIterator = new RateIterator();
                 rateIterator.init(itr, rate.getOptions(), null);
                 itr = rateIterator;

server/src/test/java/timely/server/sample/iterators/RateIteratorTest.java

@@ -7,6 +7,7 @@ import static org.junit.Assert.assertTrue;
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Optional;
 import java.util.Random;
 import java.util.TreeMap;
 
@@ -46,14 +47,26 @@ public class RateIteratorTest extends IteratorTestBase {
 
     @Test
     public void testConstantTimeRate() throws Exception {
+        testInterval(null, 0.001D);
+        testInterval("1ms", 0.001D);
+        testInterval("1s", 1.0D);
+        testInterval("60s", 60.0D);
+    }
+
+    private void testInterval(String interval, double result) throws Exception {
         SortedMapIterator source = new SortedMapIterator(table);
         RateIterator iter = new RateIterator();
         IteratorSetting settings = new IteratorSetting(100, RateIterator.class);
+        QueryRequest.RateOption rateOption = new QueryRequest.RateOption();
+        if (interval != null) {
+            rateOption.setInterval(Optional.of(interval));
+        }
+        RateIterator.setRateOptions(settings, rateOption);
         iter.init(source, settings.getOptions(), SCAN_IE);
         iter.seek(new Range(), EMPTY_COL_FAMS, true);
         for (int i = 0; i < 99; i++) {
             assertTrue(iter.hasTop());
-            assertEquals(0.001D, MetricAdapter.decodeValue(iter.getTopValue().get()), 0.0D);
+            assertEquals(result, MetricAdapter.decodeValue(iter.getTopValue().get()), 0.0D);
             iter.next();
         }
         assertFalse(iter.hasTop());

test/pom.xml

@@ -4,7 +4,7 @@
   <parent>
     <groupId>timely</groupId>
     <artifactId>timely</artifactId>
-    <version>2.1.2</version>
+    <version>2.1.4-SNAPSHOT</version>
   </parent>
   <artifactId>timely-test</artifactId>
   <properties>