scripts, interval, collectd, try to build grafana plugin

analytics/pom.xml

@@ -4,7 +4,7 @@
   <parent>
     <groupId>timely</groupId>
     <artifactId>timely</artifactId>
-    <version>2.1.0</version>
+    <version>2.1.4-SNAPSHOT</version>
   </parent>
   <artifactId>timely-analytics</artifactId>
   <!--

balancer/bin/timely-balancer-env.sh

@@ -25,6 +25,3 @@ AUTHORIZATION_CERTIFICATE_TYPE=
 AUTHORIZATION_CERTIFICATE_FILE=
 ANONYMOUS_HTTP=true
 ANONYMOUS_WEBSOCKET=true
-
-REQUIRED_ROLES=
-REQUIRED_AUTHS=

balancer/bin/timely-balancer.sh

@@ -2,10 +2,10 @@
 
 if [[ $(uname) == "Darwin" ]]; then
         THIS_SCRIPT=$(python -c 'import os,sys; print os.path.realpath(sys.argv[1])' "$0")
-        TCNATIVE_SUFFIX="jnilib"
 else
         THIS_SCRIPT=$(readlink -f "$0")
-        TCNATIVE_SUFFIX="so"
+        HW=`uname -m`
+        OS_PATH=linux-${HW}
 fi
 
 if [ -z "$1" ]; then
@@ -17,7 +17,6 @@ fi
 PID=$(jps -m | grep -E 'timely-balancer-.*-exec.jar' | grep "instance=${INSTANCE}" | awk '{print $1}')
 
 if [ "$PID" == "" ]; then
-
     THIS_DIR="${THIS_SCRIPT%/*}"
     BASE_DIR=$(cd "$THIS_DIR"/.. && pwd)
     TMP_DIR="${BASE_DIR}/tmp"
@@ -47,20 +46,23 @@ if [ "$PID" == "" ]; then
       mkdir "${LOG_DIR}"
     fi
 
-    pushd "${BASE_DIR}/bin" || exit
-    "${JAVA_HOME}/bin/jar xf ${LIB_DIR}/netty-tcnative-boringssl-static*.jar META-INF/native/libnetty_tcnative_linux_x86_64.${TCNATIVE_SUFFIX}"
-    "${JAVA_HOME}/bin/jar xf ${LIB_DIR}/netty-all*.jar META-INF/native/libnetty_transport_native_epoll_x86_64.${TCNATIVE_SUFFIX}"
-    popd || exit
-
     JVM_ARGS="-Xmx2G -Xms2G -XX:NewSize=1G -XX:MaxNewSize=1G"
-    JVM_ARGS="${JVM_ARGS} -Djava.library.path=${NATIVE_DIR}"
     JVM_ARGS="${JVM_ARGS} -Dlogging.config=${CONF_DIR}/log4j2.yml"
+    JVM_ARGS="${JVM_ARGS} -Dlog4j2.contextSelector=org.apache.logging.log4j.core.async.AsyncLoggerContextSelector"
     JVM_ARGS="${JVM_ARGS} -XX:+UseG1GC -XX:+UseStringDeduplication"
     JVM_ARGS="${JVM_ARGS} -Djava.net.preferIPv4Stack=true"
     JVM_ARGS="${JVM_ARGS} -XX:+UseNUMA"
 
-    echo "${JAVA_HOME}/bin/java ${JVM_ARGS} -jar ${THIS_DIR}/timely-balancer-*-exec.jar --instance=${INSTANCE}"
-    nohup "${JAVA_HOME}"/bin/java ${JVM_ARGS} -jar "${THIS_DIR}"/timely-balancer-*-exec.jar --instance="${INSTANCE}" >> "${LOG_DIR}/timely-balancer${INSTANCE}.out" 2>&1 &
+    if [[ -n "$OS_PATH" ]]; then
+      pushd "${BASE_DIR}/bin" || exit
+      "${JAVA_HOME}"/bin/jar xf "${LIB_DIR}"/netty-tcnative-boringssl-static-*.Final-"${OS_PATH}".jar ${NATIVE_DIR}/libnetty_tcnative_linux_"${HW}".so
+      "${JAVA_HOME}"/bin/jar xf "${LIB_DIR}"/netty-transport-native-epoll-*-"${OS_PATH}".jar ${NATIVE_DIR}/libnetty_transport_native_epoll_"${HW}".so
+      popd || exit
+      JVM_ARGS="${JVM_ARGS} -Djava.library.path=${NATIVE_DIR}"
+    fi
+
+    echo "${JAVA_HOME}/bin/java ${JVM_ARGS} -jar ${BIN_DIR}/timely-balancer-*-exec.jar --instance=${INSTANCE}"
+    nohup "${JAVA_HOME}"/bin/java ${JVM_ARGS} -jar "${BIN_DIR}"/timely-balancer-*-exec.jar --instance="${INSTANCE}" >> "${LOG_DIR}/timely-balancer${INSTANCE}.out" 2>&1 &
 else
     echo "timely-balancer --instance=${INSTANCE} already running with pid ${PID}"
 fi

balancer/conf/log4j2.yml

@@ -9,7 +9,7 @@ Configuration:
       - name: PID
         value: "????"
       - name: LOG_PATTERN
-        value: "%clr{%d{yyyy-MM-dd HH:mm:ss.SSS}}{faint} %clr{%5p} %clr{${sys:PID}}{magenta} %clr{---}{faint} %clr{[%15.15t]}{faint} %clr{%-40.40c{1.}}{cyan} %clr{:}{faint} %m%n%wEx"
+        value: "%d{ISO8601} %-5p [%-40.40t] [%-40.40c{1.}] %m%n%wEx"
 
   Appenders:
     RollingFile:

balancer/conf/timely-balancer.yml

@@ -8,8 +8,6 @@ timely:
     cache-expiration-minutes: 60
     cache-refresh-minutes: 5
     authorization-url: ${AUTHORIZATION_URL}
-    required-roles: ${REQUIRED_ROLES}
-    required-auths: ${REQUIRED_AUTHS}
     allow-anonymous-http-access: ${ANONYMOUS_HTTP:true}
     allow-anonymous-ws-access: ${ANONYMOUS_WEBSOCKET:true}
     server-ssl:

balancer/pom.xml

@@ -4,7 +4,7 @@
   <parent>
     <groupId>timely</groupId>
     <artifactId>timely</artifactId>
-    <version>2.1.0</version>
+    <version>2.1.4-SNAPSHOT</version>
   </parent>
   <artifactId>timely-balancer</artifactId>
   <description>Timely Balancer</description>
@@ -44,10 +44,6 @@
       <groupId>io.github.classgraph</groupId>
       <artifactId>classgraph</artifactId>
     </dependency>
-    <dependency>
-      <groupId>io.netty</groupId>
-      <artifactId>netty-all</artifactId>
-    </dependency>
     <dependency>
       <groupId>net.sourceforge.javacsv</groupId>
       <artifactId>javacsv</artifactId>
@@ -162,6 +158,25 @@
     <dependency>
       <groupId>io.netty</groupId>
       <artifactId>netty-tcnative-boringssl-static</artifactId>
+      <classifier>linux-aarch_64</classifier>
+      <scope>runtime</scope>
+    </dependency>
+    <dependency>
+      <groupId>io.netty</groupId>
+      <artifactId>netty-tcnative-boringssl-static</artifactId>
+      <classifier>linux-x86_64</classifier>
+      <scope>runtime</scope>
+    </dependency>
+    <dependency>
+      <groupId>io.netty</groupId>
+      <artifactId>netty-transport-native-epoll</artifactId>
+      <classifier>linux-aarch_64</classifier>
+      <scope>runtime</scope>
+    </dependency>
+    <dependency>
+      <groupId>io.netty</groupId>
+      <artifactId>netty-transport-native-epoll</artifactId>
+      <classifier>linux-x86_64</classifier>
       <scope>runtime</scope>
     </dependency>
     <dependency>

balancer/src/main/dist/assembly.xml

@@ -19,8 +19,10 @@
       <directory>${project.build.directory}/lib</directory>
       <outputDirectory>lib</outputDirectory>
       <includes>
-        <include>netty-tcnative-boringssl-static*.jar</include>
-        <include>netty-all*.jar</include>
+        <include>netty-tcnative-boringssl-static*linux-x86_64.jar</include>
+        <include>netty-tcnative-boringssl-static*linux-aarch_64.jar</include>
+        <include>netty-transport-native-epoll*linux-x86_64.jar</include>
+        <include>netty-transport-native-epoll*linux-aarch_64.jar</include>
       </includes>
     </fileSet>
     <fileSet>
@@ -52,4 +54,4 @@
 	  </includes>
 	</fileSet>
   </fileSets>
-</assembly>
+</assembly>

client/pom.xml

@@ -4,13 +4,13 @@
   <parent>
     <groupId>timely</groupId>
     <artifactId>timely</artifactId>
-    <version>2.1.0</version>
+    <version>2.1.4-SNAPSHOT</version>
   </parent>
   <artifactId>timely-client</artifactId>
   <properties>
     <!-- PEP 440 compliant version gets substituted into src/main/python/template/setup.py -->
     <!-- use .dev0 instead of -SNAPSHOT -->
-    <python.client.version>2.1.0</python.client.version>
+    <python.client.version>2.1.4.dev0</python.client.version>
   </properties>
   <dependencies>
     <dependency>

collectd-nsq-plugin/pom.xml

@@ -4,7 +4,7 @@
   <parent>
     <groupId>timely</groupId>
     <artifactId>timely</artifactId>
-    <version>2.1.0</version>
+    <version>2.1.4-SNAPSHOT</version>
   </parent>
   <artifactId>collectd-nsq-plugin</artifactId>
   <dependencies>
@@ -20,9 +20,14 @@
       <groupId>org.apache.httpcomponents</groupId>
       <artifactId>httpclient</artifactId>
     </dependency>
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>jcl-over-slf4j</artifactId>
+    </dependency>
     <dependency>
       <groupId>timely</groupId>
       <artifactId>collectd-plugin-parent</artifactId>
+      <version>${project.version}</version>
     </dependency>
   </dependencies>
   <build>

collectd-plugin-parent/pom.xml

@@ -4,7 +4,7 @@
   <parent>
     <groupId>timely</groupId>
     <artifactId>timely</artifactId>
-    <version>2.1.0</version>
+    <version>2.1.4-SNAPSHOT</version>
   </parent>
   <artifactId>collectd-plugin-parent</artifactId>
   <dependencies>

collectd-plugin-parent/src/main/java/timely/collectd/plugin/CollectDPluginParent.java

@@ -2,51 +2,58 @@ package timely.collectd.plugin;
 
 import java.io.OutputStream;
 import java.text.MessageFormat;
-import java.util.HashSet;
-import java.util.Set;
+import java.util.Arrays;
+import java.util.Map;
+import java.util.TreeMap;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
+import java.util.stream.Collectors;
 
 import org.collectd.api.Collectd;
-import org.collectd.api.DataSource;
 import org.collectd.api.OConfigItem;
 import org.collectd.api.ValueList;
 
 public abstract class CollectDPluginParent {
 
-    private static final String PUT = "put {0} {1} {2}{3}\n";
-    private static final Pattern HADOOP_STATSD_PATTERN = Pattern.compile("([\\w-_]+)\\.([\\w-_]+)\\.([\\w-_]+)\\.([\\w-_]+)");
-    private static final String INSTANCE = " instance=";
-    private static final String NAME = " name=";
-    private static final String SAMPLE = " sample=";
-    private static final String CODE = " code=";
+    private static final String PUT = "put {0} {1} {2} {3}";
+    private static final String INSTANCE = "instance";
+    private static final String NAME = "name";
+    private static final String SAMPLE_TYPE = "sampleType";
+    private static final String CODE = "code";
     private static final String PERIOD = ".";
-    private static final String SAMPLE_TYPE = " sampleType=";
     private static final String COUNTER = "COUNTER";
     private static final String GAUGE = "GAUGE";
     private static final String DERIVE = "DERIVE";
     private static final String ABSOLUTE = "ABSOLUTE";
-    private static final Pattern NSQ_PATTERN1 = Pattern.compile("([\\w-_]+)\\.([\\w-_]+)\\.([\\w-_]+)");
-    private static final Pattern NSQ_PATTERN2 = Pattern.compile("([\\w-_]+)\\.([\\w-_]+)\\.([\\w-_]+)\\.([\\w-_]+)");
-    private static final Pattern NSQ_PATTERN3 = Pattern.compile("([\\w-_]+)\\.([\\w-_]+)\\.([\\w-_]+)\\.([\\w-_]+)\\.([\\w-_#]+)\\.([\\w-_]+)");
+    private static final Pattern STATSD_PATTERN_3_GROUPS = Pattern.compile("([\\w-_]+)\\.([\\w-_]+)\\.([\\w-_]+)");
+    private static final Pattern STATSD_PATTERN_4_GROUPS = Pattern.compile("([\\w-_]+)\\.([\\w-_]+)\\.([\\w-_]+)\\.([\\w-_]+)");
+    private static final Pattern STATSD_PATTERN_6_GROUPS = Pattern.compile("([\\w-_]+)\\.([\\w-_]+)\\.([\\w-_]+)\\.([\\w-_]+)\\.([\\w-_#]+)\\.([\\w-_]+)");
     private static final Pattern ETHSTAT_PATTERN = Pattern.compile("([\\w-_]+)_queue_([\\w-_]+)_([\\w-_]+)");
     private static final String STATSD_PREFIX = "statsd";
     private static final String NSQ_PREFIX = STATSD_PREFIX + ".nsq.";
     private static final Pattern HAPROXY_PATTERN = Pattern.compile("\\[([\\w-_=]+),([\\w-_=]+)\\]");
 
     private final SMARTCodeMapping smart = new SMARTCodeMapping();
-
-    protected Set<String> addlTags = new HashSet<>();
+    protected Map<String,String> addlTags = new TreeMap<>();
+    private boolean debug = false;
 
     public int config(OConfigItem config) {
         for (OConfigItem child : config.getChildren()) {
             switch (child.getKey()) {
                 case "Tags":
                 case "tags":
-                    String[] additionalTags = child.getValues().get(0).getString().split(",");
-                    for (String t : additionalTags) {
-                        addlTags.add(" " + t.trim());
-                    }
+                    String tagValue = child.getValues().get(0).getString();
+                    Arrays.stream(tagValue.split(",")).map(String::trim).forEach(p -> {
+                        String[] pair = p.split("=");
+                        if (pair.length == 2) {
+                            addTag(addlTags, pair[0], pair[1]);
+                        }
+                    });
+                    break;
+                case "Debug":
+                case "debug":
+                    debug = child.getValues().get(0).getBoolean();
+                    break;
                 default:
             }
         }
@@ -55,196 +62,256 @@ public abstract class CollectDPluginParent {
 
     public abstract void write(String metric, OutputStream out);
 
+    protected void addTag(Map<String,String> m, String k, String v) {
+        if (k != null && !k.isBlank() && v != null && !v.isBlank()) {
+            m.put(k.trim(), v.trim());
+        }
+    }
+
+    protected void addTag(Map<String,String> m, String kv) {
+        if (kv != null && !kv.isBlank()) {
+            String[] parts = kv.split("=");
+            if (parts.length == 2 && !parts[0].isBlank() && !parts[1].isBlank()) {
+                m.put(parts[0].trim(), parts[1].trim());
+            }
+        }
+    }
+
     public void process(ValueList vl, OutputStream out) {
+        try {
+            processInternal(vl, out);
+        } catch (Exception e) {
+            Collectd.logError(e.getMessage());
+            e.printStackTrace(System.err);
+        }
+    }
+
+    private void processInternal(ValueList vl, OutputStream out) {
         debugValueList(vl);
 
         StringBuilder metric = new StringBuilder();
         Long timestamp = vl.getTime();
-        StringBuilder tags = new StringBuilder();
+        Map<String,String> tagMap = new TreeMap<>();
         String host = vl.getHost();
         int idx = host.indexOf(PERIOD);
         if (-1 != idx) {
-            tags.append(" host=").append(host.substring(0, idx));
+            addTag(tagMap, "host", host.substring(0, idx));
         } else {
-            tags.append(" host=").append(host);
+            addTag(tagMap, "host", host);
         }
         int nIdx = host.indexOf('n');
         if (-1 != nIdx) {
-            tags.append(" rack=").append(host.substring(0, nIdx));
+            addTag(tagMap, "rack", host.substring(0, nIdx));
         }
-        for (String tag : addlTags) {
-            tags.append(tag);
-        }
-        if (vl.getPlugin().equals(STATSD_PREFIX)) {
-            Matcher m = HADOOP_STATSD_PATTERN.matcher(vl.getTypeInstance());
-            Matcher n1 = NSQ_PATTERN1.matcher(vl.getTypeInstance());
-            Matcher n2 = NSQ_PATTERN2.matcher(vl.getTypeInstance());
-            Matcher n3 = NSQ_PATTERN3.matcher(vl.getTypeInstance());
-            String instance = null;
-            if (m.matches() && !vl.getTypeInstance().startsWith("nsq")) {
-                // Here we are processing the statsd metrics coming from the
-                // Hadoop Metrics2 StatsDSink without the host name.
-                // The format of metric is:
-                // serviceName.contextName.recordName.metricName. The recordName
-                // is typically duplicative and is dropped here. The serviceName
-                // is used as the instance.
-                metric.append(STATSD_PREFIX).append(PERIOD).append(m.group(2)).append(PERIOD).append(m.group(4));
-                instance = m.group(1);
-            } else if (n1.matches() && vl.getTypeInstance().startsWith("nsq")) {
-                metric.append(NSQ_PREFIX).append(n1.group(2)).append(PERIOD).append(n1.group(3));
-            } else if (n2.matches() && vl.getTypeInstance().startsWith("nsq")) {
-                metric.append(NSQ_PREFIX).append(n2.group(2)).append(PERIOD).append(n2.group(4));
-                instance = n2.group(3);
-            } else if (n3.matches() && vl.getTypeInstance().startsWith("nsq")) {
-                metric.append(NSQ_PREFIX).append(n3.group(4)).append(PERIOD).append(n3.group(6));
-                instance = n3.group(5);
-            } else {
-                // Handle StatsD metrics of unknown formats. If there is a
-                // period in the metric name, use everything up to that as
-                // the instance.
-                int period = vl.getTypeInstance().indexOf('.');
-                if (-1 == period) {
-                    metric.append(STATSD_PREFIX).append(PERIOD).append(vl.getTypeInstance());
+        tagMap.putAll(addlTags);
+
+        String plugin = vl.getPlugin();
+        String pluginInstance = vl.getPluginInstance();
+        String type = vl.getType();
+        String typeInstance = vl.getTypeInstance();
+
+        switch (plugin) {
+            case STATSD_PREFIX:
+                Matcher statsd_3_groups = STATSD_PATTERN_3_GROUPS.matcher(typeInstance);
+                Matcher statsd_4_groups = STATSD_PATTERN_4_GROUPS.matcher(typeInstance);
+                Matcher statsd_6_groups = STATSD_PATTERN_6_GROUPS.matcher(typeInstance);
+                String instance = null;
+                if (!typeInstance.startsWith("nsq")) {
+                    String[] parts = typeInstance.split("\\.");
+                    if (parts.length % 2 == 1 && parts.length >= 3) {
+                        switch(parts[0]) {
+                            case "dwingest":
+                                if (parts.length >= 4) {
+                                    metric.append(STATSD_PREFIX).append(PERIOD).append(parts[1]).append(PERIOD).append(parts[3]);
+                                    instance = parts[0];
+                                }
+                                break;
+                            case "dwquery":
+                                if (parts.length >= 4) {
+                                    metric.append(STATSD_PREFIX).append(PERIOD).append(parts[1]).append(PERIOD).append(parts[3]);
+                                    addTag(tagMap, "queryId", parts[0]);
+                                }
+                                break;
+                            default:
+                                // EtsyStatsD format -- metric.(tagName.tagValue)*
+                                metric.append(STATSD_PREFIX).append(PERIOD).append(parts[0]);
+                                for (int x = 1; x < parts.length; x += 2) {
+                                    addTag(tagMap, parts[x], parts[x + 1]);
+                                }
+                            }
+                        }
+                    } else if (statsd_4_groups.matches()) {
+                        // Here we are processing the statsd metrics coming from the Hadoop Metrics2 StatsDSink without the host name.
+                        // The format of metric is: serviceName.contextName.recordName.metricName. The recordName is typically duplicative
+                        // and is dropped here. The serviceName is used as the instance.
+                        metric.append(STATSD_PREFIX).append(PERIOD).append(statsd_4_groups.group(2)).append(PERIOD).append(statsd_4_groups.group(4));
+                        instance = statsd_4_groups.group(1);
+                    }
+                } else if (statsd_3_groups.matches()) {
+                    metric.append(NSQ_PREFIX).append(statsd_3_groups.group(2)).append(PERIOD).append(statsd_3_groups.group(3));
+                } else if (statsd_4_groups.matches()) {
+                    metric.append(NSQ_PREFIX).append(statsd_4_groups.group(2)).append(PERIOD).append(statsd_4_groups.group(4));
+                    instance = statsd_4_groups.group(3);
+                } else if (statsd_6_groups.matches()) {
+                    metric.append(NSQ_PREFIX).append(statsd_6_groups.group(4)).append(PERIOD).append(statsd_6_groups.group(6));
+                    instance = statsd_6_groups.group(5);
                 } else {
-                    instance = vl.getTypeInstance().substring(0, period);
-                    metric.append(STATSD_PREFIX).append(PERIOD).append(vl.getTypeInstance().substring(period + 1));
+                    // Handle StatsD metrics of unknown formats. If there is a
+                    // period in the metric name, use everything up to that as
+                    // the instance.
+                    int period = typeInstance.indexOf('.');
+                    if (-1 == period) {
+                        metric.append(STATSD_PREFIX).append(PERIOD).append(typeInstance);
+                    } else {
+                        instance = typeInstance.substring(0, period);
+                        metric.append(STATSD_PREFIX).append(PERIOD).append(typeInstance.substring(period + 1));
+                    }
                 }
-            }
-            timestamp = vl.getTime();
-            if (null != instance) {
-                tags.append(INSTANCE).append(instance);
-            }
-        } else if (vl.getPlugin().equals("ethstat")) {
-            metric.append("sys.ethstat.");
-            if (vl.getTypeInstance().contains("queue")) {
-                Matcher m = ETHSTAT_PATTERN.matcher(vl.getTypeInstance());
-                if (m.matches()) {
-                    metric.append(m.group(1)).append("_").append(m.group(3));
-                    tags.append(" queue=").append(m.group(2));
+                if (null != instance) {
+                    addTag(tagMap, INSTANCE, instance);
+                }
+                break;
+            case "ethstat":
+                metric.append("sys.ethstat.");
+                if (typeInstance.contains("queue")) {
+                    Matcher ethstat_m = ETHSTAT_PATTERN.matcher(typeInstance);
+                    if (ethstat_m.matches()) {
+                        metric.append(ethstat_m.group(1)).append("_").append(ethstat_m.group(3));
+                        addTag(tagMap, "queue", ethstat_m.group(2));
+                    } else {
+                        metric.append(typeInstance);
+                    }
                 } else {
-                    metric.append(vl.getTypeInstance());
+                    metric.append(typeInstance);
                 }
-            } else {
-                metric.append(vl.getTypeInstance());
-            }
-            tags.append(INSTANCE).append(vl.getPluginInstance());
-        } else if (vl.getPlugin().equals("hddtemp")) {
-            metric.append("sys.hddtemp.").append(vl.getType());
-            tags.append(INSTANCE).append(vl.getTypeInstance());
-        } else if (vl.getPlugin().equals("smart")) {
-            int code = -1;
-            String name = null;
-            if (vl.getTypeInstance().startsWith("attribute-")) {
-                int hyphen = vl.getTypeInstance().indexOf('-');
-                code = Integer.parseInt(vl.getTypeInstance().substring(hyphen + 1));
-                name = smart.get(code);
-            }
-            if (code == -1) {
-                if (notEmpty(vl.getTypeInstance())) {
-                    metric.append("sys.smart.").append(vl.getTypeInstance());
+                addTag(tagMap, INSTANCE, pluginInstance);
+                break;
+            case "hddtemp":
+                metric.append("sys.hddtemp.").append(type);
+                addTag(tagMap, INSTANCE, typeInstance);
+                break;
+            case "smart":
+                int code = -1;
+                String name = null;
+                if (typeInstance.startsWith("attribute-")) {
+                    int hyphen = typeInstance.indexOf('-');
+                    code = Integer.parseInt(typeInstance.substring(hyphen + 1));
+                    name = smart.get(code);
+                }
+                if (code == -1) {
+                    if (notEmpty(typeInstance)) {
+                        metric.append("sys.smart.").append(typeInstance);
+                    } else {
+                        metric.append("sys.smart.").append(type);
+                    }
                 } else {
-                    metric.append("sys.smart.").append(vl.getType());
+                    metric.append("sys.smart.").append(name);
+                    addTag(tagMap, CODE, Integer.toString(code));
                 }
-            } else {
-                metric.append("sys.smart.").append(name);
-                tags.append(CODE).append(code);
-            }
-            tags.append(INSTANCE).append(vl.getPluginInstance());
-        } else if (vl.getPlugin().equals("sensors")) {
-            String instance = "";
-            if (vl.getTypeInstance().startsWith("temp")) {
-                instance = vl.getTypeInstance().substring(4);
-            }
-            metric.append("sys.sensors.").append(vl.getType()).append(PERIOD).append(vl.getPluginInstance());
-            tags.append(INSTANCE).append(instance);
-        } else if (vl.getPlugin().equals("haproxy")) {
-            metric.append("sys.haproxy.").append(vl.getTypeInstance());
-            Matcher m = HAPROXY_PATTERN.matcher(vl.getPluginInstance());
-            if (m.matches()) {
-                tags.append(" ").append(m.group(1));
-                tags.append(" ").append(m.group(2));
-            }
-        } else if (vl.getPlugin().equals("ipmi")) {
-            metric.append("sys.ipmi.").append(vl.getType());
-            tags.append(INSTANCE).append(vl.getTypeInstance().replaceAll(" ", "_"));
-        } else if (vl.getPlugin().equals("snmp")) {
-            metric.append("sys.snmp.").append(vl.getType());
-            tags.append(INSTANCE).append(vl.getTypeInstance().replaceAll(" ", "_"));
-        } else if (vl.getPlugin().equals("GenericJMX")) {
-            metric.append("sys.").append(vl.getPlugin()).append(PERIOD).append(vl.getType()).append(PERIOD).append(vl.getTypeInstance());
-            String[] pluginInstanceSplit = vl.getPluginInstance().split("-");
-            if (pluginInstanceSplit.length > 0) {
-                tags.append(INSTANCE).append(pluginInstanceSplit[0].replaceAll(" ", "_"));
-            }
-            if (pluginInstanceSplit.length > 1) {
-                tags.append(NAME).append(pluginInstanceSplit[1].replaceAll(" ", "_"));
-            }
-        } else if (notEmpty(vl.getTypeInstance()) && notEmpty(vl.getType()) && notEmpty(vl.getPluginInstance()) && notEmpty(vl.getPlugin())) {
-            metric.append("sys.").append(vl.getPlugin()).append(PERIOD).append(vl.getType()).append(PERIOD).append(vl.getTypeInstance());
-            tags.append(INSTANCE).append(vl.getPluginInstance().replaceAll(" ", "_"));
-        } else if (notEmpty(vl.getTypeInstance()) && notEmpty(vl.getType()) && notEmpty(vl.getPlugin())) {
-            metric.append("sys.").append(vl.getPlugin()).append(PERIOD).append(vl.getType()).append(PERIOD).append(vl.getTypeInstance());
-        } else if (notEmpty(vl.getType()) && notEmpty(vl.getPluginInstance()) && notEmpty(vl.getPlugin())) {
-            metric.append("sys.").append(vl.getPlugin()).append(PERIOD).append(vl.getType());
-            tags.append(INSTANCE).append(vl.getPluginInstance().replaceAll(" ", "_"));
-        } else if (notEmpty(vl.getType()) && notEmpty(vl.getPlugin())) {
-            metric.append("sys.").append(vl.getPlugin()).append(PERIOD).append(vl.getType());
-        } else {
-            Collectd.logWarning("Unhandled metric: " + vl.toString());
-            return;
+                addTag(tagMap, INSTANCE, pluginInstance);
+                break;
+            case "sensors":
+                if (typeInstance.startsWith("temp")) {
+                    addTag(tagMap, INSTANCE, typeInstance.substring(4));
+                }
+                metric.append("sys.sensors.").append(type).append(PERIOD).append(pluginInstance);
+                break;
+            case "haproxy":
+                metric.append("sys.haproxy.").append(typeInstance);
+                Matcher hadoop_m2 = HAPROXY_PATTERN.matcher(pluginInstance);
+                if (hadoop_m2.matches()) {
+                    addTag(tagMap, hadoop_m2.group(1));
+                    addTag(tagMap, hadoop_m2.group(2));
+                }
+                break;
+            case "ipmi":
+            case "snmp":
+                metric.append("sys.").append(plugin).append(PERIOD).append(type);
+                addTag(tagMap, INSTANCE, typeInstance.replaceAll(" ", "_"));
+                break;
+            case "GenericJMX":
+                metric.append("sys.").append(plugin).append(PERIOD).append(type).append(PERIOD).append(typeInstance);
+                String[] pluginInstanceSplit = pluginInstance.split("-");
+                if (pluginInstanceSplit.length > 0) {
+                    addTag(tagMap, INSTANCE, pluginInstanceSplit[0].replaceAll(" ", "_"));
+                }
+                if (pluginInstanceSplit.length > 1) {
+                    addTag(tagMap, NAME, pluginInstanceSplit[1].replaceAll(" ", "_"));
+                }
+            default:
+                if (notEmpty(type) && notEmpty(typeInstance) && notEmpty(plugin) && notEmpty(pluginInstance)) {
+                    metric.append("sys.").append(plugin).append(PERIOD).append(type).append(PERIOD).append(typeInstance);
+                    addTag(tagMap, INSTANCE, pluginInstance.replaceAll(" ", "_"));
+                } else if (notEmpty(type) && notEmpty(typeInstance) && notEmpty(plugin)) {
+                    metric.append("sys.").append(plugin).append(PERIOD).append(type).append(PERIOD).append(typeInstance);
+                } else if (notEmpty(type) && notEmpty(plugin) && notEmpty(pluginInstance)) {
+                    metric.append("sys.").append(plugin).append(PERIOD).append(type);
+                    addTag(tagMap, INSTANCE, pluginInstance.replaceAll(" ", "_"));
+                } else if (notEmpty(type) && notEmpty(plugin)) {
+                    metric.append("sys.").append(plugin).append(PERIOD).append(type);
+                } else {
+                    Collectd.logWarning("Unhandled metric: " + vl);
+                    return;
+                }
+                break;
         }
         final String metricName = metric.toString().replaceAll(" ", "_");
         for (int i = 0; i < vl.getValues().size(); i++) {
-            StringBuilder tagsWithSample = new StringBuilder(tags.toString());
-            String sampleName = vl.getDataSet().getDataSources().get(i).getName();
-            int type = vl.getDataSet().getDataSources().get(i).getType();
-            String sampleType = convertType(type);
-            if (null != sampleName) {
-                tagsWithSample.append(SAMPLE).append(sampleName);
-            }
+            int dataSourceSampleType = vl.getDataSet().getDataSources().get(i).getType();
+            String sampleType = convertType(dataSourceSampleType);
             if (null != sampleType) {
-                tagsWithSample.append(SAMPLE_TYPE).append(sampleType);
+                addTag(tagMap, SAMPLE_TYPE, sampleType);
             }
             Double value = vl.getValues().get(i).doubleValue();
-            String datapoint = MessageFormat.format(PUT, metricName, timestamp.toString(), value.toString(), tagsWithSample.toString());
-            Collectd.logDebug("Writing: " + datapoint);
-            write(datapoint, out);
+            String tagString = tagMap.entrySet().stream().map(e -> e.getKey() + "=" + e.getValue()).collect(Collectors.joining(" "));
+            String datapoint = MessageFormat.format(PUT, metricName, timestamp.toString(), value.toString(), tagString);
+            logDebug(String.format("Writing: %s", datapoint));
+            write(datapoint + "\n", out);
+        }
+    }
+
+    private void logDebug(String s) {
+        // collectd must be compiled with debug enabled in order for log level debug to work
+        if (debug) {
+            Collectd.logInfo(s);
+        } else {
+            Collectd.logDebug(s);
         }
     }
 
     private void debugValueList(ValueList vl) {
-        Collectd.logDebug("Input: " + vl.toString());
-        Collectd.logDebug("Plugin: " + vl.getPlugin());
-        Collectd.logDebug("PluginInstance: " + vl.getPluginInstance());
-        Collectd.logDebug("Type: " + vl.getType());
-        Collectd.logDebug("TypeInstance: " + vl.getTypeInstance());
-        for (int i = 0; i < vl.getValues().size(); i++) {
-            Number value = vl.getValues().get(i);
-            DataSource ds = vl.getDataSet().getDataSources().get(i);
-            Collectd.logDebug(convertType(ds.getType()) + " " + ds.getName() + " = " + value);
+        try {
+            StringBuilder sb = new StringBuilder();
+            sb.append(String.format("Plugin:%s", vl.getPlugin()));
+            if (vl.getPluginInstance() != null && !vl.getPluginInstance().isBlank()) {
+                sb.append(String.format(" PluginInstance:%s", vl.getPluginInstance()));
+            }
+            sb.append(String.format(" Type:%s", vl.getType()));
+            sb.append(String.format(" TypeInstance:%s", vl.getTypeInstance()));
+            for (int i = 0; i < vl.getValues().size(); i++) {
+                Number value = vl.getValues().get(i);
+                sb.append(String.format(" Value:%s", value.toString()));
+            }
+            logDebug(sb.toString());
+        } catch (Exception e) {
+            e.printStackTrace();
+            logDebug(e.getMessage());
         }
     }
 
     private String convertType(int type) {
-        String result = null;
         switch (type) {
             case 0:
-                result = COUNTER;
-                break;
+                return COUNTER;
             case 1:
-                result = GAUGE;
-                break;
+                return GAUGE;
             case 2:
-                result = DERIVE;
-                break;
+                return DERIVE;
             case 3:
-                result = ABSOLUTE;
-                break;
+                return ABSOLUTE;
             default:
-                result = GAUGE;
-                break;
+                return GAUGE;
         }
-        return result;
     }
 
     private boolean notEmpty(String arg) {

collectd-plugin-parent/src/test/java/timely/collectd/plugin/PluginParentTest.java

@@ -26,7 +26,7 @@ public class PluginParentTest {
     public class TestPlugin extends CollectDPluginParent {
 
         public TestPlugin() {
-            super.addlTags.add(" addl1=foo");
+            super.addlTags.put("addl1", "foo");
         }
 
         @Override

collectd-timely-plugin/pom.xml

@@ -4,7 +4,7 @@
   <parent>
     <groupId>timely</groupId>
     <artifactId>timely</artifactId>
-    <version>2.1.0</version>
+    <version>2.1.4-SNAPSHOT</version>
   </parent>
   <artifactId>collectd-timely-plugin</artifactId>
   <dependencies>

common/pom.xml

@@ -4,7 +4,7 @@
   <parent>
     <groupId>timely</groupId>
     <artifactId>timely</artifactId>
-    <version>2.1.0</version>
+    <version>2.1.4-SNAPSHOT</version>
   </parent>
   <artifactId>timely-common</artifactId>
   <dependencies>
@@ -72,10 +72,6 @@
       <groupId>io.jsonwebtoken</groupId>
       <artifactId>jjwt-jackson</artifactId>
     </dependency>
-    <dependency>
-      <groupId>io.netty</groupId>
-      <artifactId>netty-all</artifactId>
-    </dependency>
     <dependency>
       <groupId>io.projectreactor.netty</groupId>
       <artifactId>reactor-netty</artifactId>

common/src/main/java/timely/api/request/timeseries/QueryRequest.java

@@ -32,6 +32,7 @@ public class QueryRequest extends AuthenticatedRequest implements HttpGetRequest
         private boolean counter = false;
         private long counterMax = 0;
         private long resetValue = 0;
+        private Optional<String> interval = Optional.empty();
 
         public RateOption() {}
 
@@ -84,12 +85,21 @@ public class QueryRequest extends AuthenticatedRequest implements HttpGetRequest
             this.resetValue = resetValue;
         }
 
+        public Optional<String> getInterval() {
+            return interval;
+        }
+
+        public void setInterval(Optional<String> interval) {
+            this.interval = interval;
+        }
+
         @Override
         public String toString() {
             ToStringBuilder tsb = new ToStringBuilder(this);
             tsb.append("counter", counter);
             tsb.append("counterMax", counterMax);
             tsb.append("resetValue", resetValue);
+            tsb.append("interval", interval);
             return tsb.toString();
         }
 
@@ -99,6 +109,7 @@ public class QueryRequest extends AuthenticatedRequest implements HttpGetRequest
             hcb.append(counter);
             hcb.append(counterMax);
             hcb.append(resetValue);
+            hcb.append(interval);
             return hcb.toHashCode();
         }
 
@@ -536,9 +547,12 @@ public class QueryRequest extends AuthenticatedRequest implements HttpGetRequest
                                         options.setCounter(rateOptions[x].endsWith("counter"));
                                         break;
                                     case 1:
-                                        options.setCounterMax(Long.parseLong(rateOptions[x]));
+                                        options.setInterval(Optional.of(rateOptions[x]));
                                         break;
                                     case 2:
+                                        options.setCounterMax(Long.parseLong(rateOptions[x]));
+                                        break;
+                                    case 3:
                                         options.setResetValue(Long.parseLong(rateOptions[x]));
                                         break;
                                     default:

common/src/main/java/timely/auth/RemoteAuthenticationProvider.java

@@ -37,6 +37,9 @@ public class RemoteAuthenticationProvider implements AuthenticationProvider {
     public static final String ISSUERS_HEADER = "X-ProxiedIssuersChain";
 
     public RemoteAuthenticationProvider(WebClient.Builder builder, SecurityProperties securityProperties) {
+        log.debug("creating RemoteAuthenticationProvider with authorizationUrl:{} anonymousHttp:{} anonymousWs:{} requiredRoles:{} requiredAuths{}",
+                        securityProperties.getAuthorizationUrl(), securityProperties.isAllowAnonymousHttpAccess(),
+                        securityProperties.isAllowAnonymousWsAccess(), securityProperties.getRequiredRoles(), securityProperties.getRequiredAuths());
         this.webClient = builder.baseUrl(securityProperties.getAuthorizationUrl()).build();
     }
 

common/src/main/java/timely/auth/TimelyUser.java

@@ -3,7 +3,7 @@ package timely.auth;
 import java.io.Serializable;
 import java.util.Collection;
 import java.util.Collections;
-import java.util.LinkedHashSet;
+import java.util.TreeSet;
 
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonIgnore;
@@ -51,10 +51,10 @@ public class TimelyUser implements Serializable {
         this.userType = DnUtils.isServerDN(subjectDn) ? UserType.SERVER : UserType.USER;
         this.email = null;
         this.login = ProxiedEntityUtils.getShortName(dn.subjectDN());
-        this.auths = auths == null ? Collections.emptyList() : new LinkedHashSet<>(auths);
-        this.unmodifiableAuths = Collections.unmodifiableCollection(this.auths);
-        this.roles = roles == null ? Collections.emptyList() : new LinkedHashSet<>(roles);
-        this.unmodifiableRoles = Collections.unmodifiableCollection(this.roles);
+        this.auths = auths == null ? Collections.emptySortedSet() : new TreeSet<>(auths);
+        this.unmodifiableAuths = Collections.unmodifiableSortedSet(new TreeSet<>(this.auths));
+        this.roles = roles == null ? Collections.emptySortedSet() : new TreeSet<>(roles);
+        this.unmodifiableRoles = Collections.unmodifiableSortedSet(new TreeSet(this.roles));
         this.roleToAuthMapping = null;
         this.creationTime = System.currentTimeMillis();
         this.expirationTime = -1L;
@@ -89,10 +89,10 @@ public class TimelyUser implements Serializable {
         this.email = email;
         this.dn = dn;
         this.userType = userType;
-        this.auths = auths == null ? Collections.emptyList() : new LinkedHashSet<>(auths);
-        this.unmodifiableAuths = Collections.unmodifiableCollection(this.auths);
-        this.roles = roles == null ? Collections.emptyList() : new LinkedHashSet<>(roles);
-        this.unmodifiableRoles = Collections.unmodifiableCollection(this.roles);
+        this.auths = auths == null ? Collections.emptySortedSet() : new TreeSet<>(auths);
+        this.unmodifiableAuths = Collections.unmodifiableSortedSet(new TreeSet<>(this.auths));
+        this.roles = roles == null ? Collections.emptySortedSet() : new TreeSet<>(roles);
+        this.unmodifiableRoles = Collections.unmodifiableSortedSet(new TreeSet<>(this.roles));
         this.roleToAuthMapping = roleToAuthMapping == null ? LinkedHashMultimap.create() : LinkedHashMultimap.create(roleToAuthMapping);
         this.creationTime = creationTime;
         this.expirationTime = expirationTime;

common/src/main/java/timely/common/component/AuthenticationService.java

@@ -205,10 +205,9 @@ public class AuthenticationService {
             String message = "";
             if (!missingRoles.isEmpty() && !missingAuths.isEmpty()) {
                 message = "User:" + primaryUser.getName() + " is missing role(s):" + missingRoles + " and auth(s):" + missingAuths;
-
             } else if (!missingRoles.isEmpty()) {
                 message = "User:" + primaryUser.getName() + " is missing role(s):" + missingRoles;
-            } else if (missingAuths.isEmpty()) {
+            } else if (!missingAuths.isEmpty()) {
                 message = "User:" + primaryUser.getName() + " is missing auths(s):" + missingAuths;
             }
             log.debug(message);

common/src/main/java/timely/common/component/ClientCertRestTemplateCustomizer.java

@@ -1,63 +0,0 @@
-package timely.common.component;
-
-import javax.net.ssl.SSLContext;
-
-import org.apache.http.client.HttpClient;
-import org.apache.http.conn.ssl.NoopHostnameVerifier;
-import org.apache.http.impl.client.HttpClientBuilder;
-import org.apache.http.impl.client.HttpClients;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Qualifier;
-import org.springframework.boot.autoconfigure.condition.ConditionalOnBean;
-import org.springframework.boot.web.client.RestTemplateCustomizer;
-import org.springframework.core.annotation.Order;
-import org.springframework.http.client.ClientHttpRequestFactory;
-import org.springframework.http.client.HttpComponentsClientHttpRequestFactory;
-import org.springframework.stereotype.Component;
-import org.springframework.web.client.RestTemplate;
-
-import timely.common.configuration.RestClientProperties;
-
-/**
- * Customizes Spring {@link RestTemplate} instances by using our configured SSL certificate to present a client certificate whenever asked by remote services.
- */
-@Component
-@Order(100) // execute this after standard customizers so we're sure to overwrite the client
-            // request factory
-@ConditionalOnBean(name = "outboundJDKSslContext")
-public class ClientCertRestTemplateCustomizer implements RestTemplateCustomizer {
-
-    private final SSLContext sslContext;
-    private final int maxConnectionsTotal;
-    private final int maxConnectionsPerRoute;
-
-    @Autowired
-    public ClientCertRestTemplateCustomizer(@Qualifier("outboundJDKSslContext") SSLContext sslContext, RestClientProperties restClientProperties) {
-        this.sslContext = sslContext;
-        this.maxConnectionsTotal = restClientProperties.getMaxConnectionsTotal();
-        this.maxConnectionsPerRoute = restClientProperties.getMaxConnectionsPerRoute();
-    }
-
-    @Override
-    public void customize(RestTemplate restTemplate) {
-        restTemplate.setRequestFactory(clientHttpRequestFactory());
-    }
-
-    protected ClientHttpRequestFactory clientHttpRequestFactory() {
-        HttpClient httpClient = customizeHttpClient(HttpClients.custom(), sslContext).build();
-        return new HttpComponentsClientHttpRequestFactory(httpClient);
-    }
-
-    protected HttpClientBuilder customizeHttpClient(HttpClientBuilder httpClientBuilder, SSLContext sslContext) {
-        if (sslContext != null) {
-            httpClientBuilder.setSSLContext(sslContext);
-        }
-        httpClientBuilder.setMaxConnTotal(maxConnectionsTotal);
-        httpClientBuilder.setMaxConnPerRoute(maxConnectionsPerRoute);
-        // TODO: We're allowing all hosts, since the cert presented by the service we're
-        // calling likely won't match its hostname (e.g., a docker host name)
-        // Instead, we could list the expected cert as a property (or use our server
-        // cert), and verify that the presented name matches.
-        return httpClientBuilder.setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE);
-    }
-}

common/src/main/java/timely/common/configuration/ClientCertWebClientCustomizer.java

@@ -1,31 +0,0 @@
-package timely.common.configuration;
-
-import org.springframework.beans.factory.annotation.Qualifier;
-import org.springframework.boot.autoconfigure.condition.ConditionalOnBean;
-import org.springframework.boot.web.reactive.function.client.WebClientCustomizer;
-import org.springframework.context.annotation.Bean;
-import org.springframework.context.annotation.Configuration;
-import org.springframework.core.annotation.Order;
-import org.springframework.http.client.reactive.ReactorClientHttpConnector;
-
-import io.netty.handler.ssl.SslContext;
-import reactor.netty.http.client.HttpClient;
-
-/**
- * Customizes the Spring-provided {@link org.springframework.web.reactive.function.client.WebClient.Builder} with an {@link SslContext} that will provide a
- * client certificate to the remote server if one is requested.
- */
-@Configuration
-@Order(100) // execute this after standard customizers to ensure that we overwrite the client connector
-@ConditionalOnBean(name = "outboundNettySslContext")
-public class ClientCertWebClientCustomizer {
-
-    @Bean
-    public WebClientCustomizer webClientCustomizer(@Qualifier("outboundNettySslContext") SslContext sslContext) {
-        return webClientBuilder -> {
-            HttpClient httpClient = HttpClient.create().secure(sslContextSpec -> sslContextSpec.sslContext(sslContext));
-            ReactorClientHttpConnector clientHttpConnector = new ReactorClientHttpConnector(httpClient);
-            webClientBuilder.clientConnector(clientHttpConnector);
-        };
-    }
-}

common/src/main/java/timely/common/configuration/SecurityConfiguration.java

@@ -9,9 +9,12 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
 import org.springframework.boot.context.properties.EnableConfigurationProperties;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
+import org.springframework.http.client.reactive.ReactorClientHttpConnector;
 import org.springframework.security.web.authentication.preauth.PreAuthenticatedAuthenticationProvider;
 import org.springframework.web.reactive.function.client.WebClient;
 
+import io.netty.handler.ssl.SslContext;
+import reactor.netty.http.client.HttpClient;
 import timely.auth.FileUserDetailsService;
 import timely.auth.RemoteAuthenticationProvider;
 import timely.auth.SubjectIssuerDNPair;
@@ -30,7 +33,11 @@ public class SecurityConfiguration {
 
     @Bean
     @ConditionalOnExpression("T(org.apache.commons.lang3.StringUtils).isNotEmpty('${timely.security.authorization-url:}')")
-    public RemoteAuthenticationProvider remoteAuthenticationProvider(SecurityProperties securityProperties, WebClient.Builder builder) {
+    public RemoteAuthenticationProvider remoteAuthenticationProvider(SecurityProperties securityProperties, WebClient.Builder builder,
+                    @Qualifier("outboundNettySslContext") SslContext sslContext) {
+        HttpClient httpClient = HttpClient.create().secure(sslContextSpec -> sslContextSpec.sslContext(sslContext));
+        ReactorClientHttpConnector clientHttpConnector = new ReactorClientHttpConnector(httpClient);
+        builder.clientConnector(clientHttpConnector);
         return new RemoteAuthenticationProvider(builder, securityProperties);
     }
 

common/src/main/java/timely/common/configuration/SecurityProperties.java

@@ -1,14 +1,10 @@
 package timely.common.configuration;
 
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.List;
-import java.util.Objects;
-import java.util.stream.Collectors;
 
 import javax.validation.Valid;
 
-import org.apache.commons.lang3.StringUtils;
 import org.springframework.boot.context.properties.ConfigurationProperties;
 import org.springframework.boot.context.properties.NestedConfigurationProperty;
 import org.springframework.cloud.context.config.annotation.RefreshScope;
@@ -106,13 +102,6 @@ public class SecurityProperties {
         this.requiredAuths = requiredAuths;
     }
 
-    public void setRequiredAuths(String requiredAuths) {
-        if (StringUtils.isNotBlank(requiredAuths)) {
-            this.requiredAuths = Arrays.stream(StringUtils.split(requiredAuths)).filter(Objects::nonNull).filter(s -> StringUtils.isNotBlank(s))
-                            .map(String::trim).collect(Collectors.toList());
-        }
-    }
-
     public List<String> getRequiredRoles() {
         return requiredRoles;
     }
@@ -121,10 +110,4 @@ public class SecurityProperties {
         this.requiredRoles = requiredRoles;
     }
 
-    public void setRequiredRoles(String requiredRoles) {
-        if (StringUtils.isNotBlank(requiredRoles)) {
-            this.requiredRoles = Arrays.stream(StringUtils.split(requiredRoles)).filter(Objects::nonNull).filter(s -> StringUtils.isNotBlank(s))
-                            .map(String::trim).collect(Collectors.toList());
-        }
-    }
 }

common/src/main/java/timely/common/configuration/SslClientConfig.java

@@ -2,6 +2,8 @@ package timely.common.configuration;
 
 import javax.net.ssl.SSLContext;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
 import org.springframework.boot.context.properties.EnableConfigurationProperties;
@@ -18,9 +20,13 @@ import io.netty.handler.ssl.SslContext;
 @ConditionalOnExpression("!T(org.springframework.util.StringUtils).isEmpty('${timely.security.client-ssl.key-store-file:}')")
 public class SslClientConfig {
 
+    private static final Logger log = LoggerFactory.getLogger(SslClientConfig.class);
+
     @Bean
     @Qualifier("outboundJDKSslContext")
     public SSLContext outboundJDKSslContext(SslClientProperties sslClientProperties) throws Exception {
+        log.debug("Creating outboundJDKSslContext using keyStoreFile:{} trustStoreFile:{}", sslClientProperties.getKeyStoreFile(),
+                        sslClientProperties.getTrustStoreFile());
         SSLContext sslContext = SSLContext.getInstance("TLS");
         sslContext.init(SslHelper.getKeyManagerFactory(sslClientProperties).getKeyManagers(),
                         SslHelper.getTrustManagerFactory(sslClientProperties).getTrustManagers(), null);
@@ -30,6 +36,8 @@ public class SslClientConfig {
     @Bean
     @Qualifier("outboundNettySslContext")
     public SslContext outboundNettySslContext(SslClientProperties sslClientProperties) throws Exception {
+        log.debug("Creating outboundNettySslContext using keyStoreFile:{} trustStoreFile:{}", sslClientProperties.getKeyStoreFile(),
+                        sslClientProperties.getTrustStoreFile());
         return SslHelper.getSslContext(sslClientProperties);
     }
 }

common/src/main/java/timely/common/configuration/SslClientProperties.java

@@ -7,8 +7,7 @@ import org.springframework.cloud.context.config.annotation.RefreshScope;
 @ConfigurationProperties(prefix = "timely.security.client-ssl")
 public class SslClientProperties implements SslProperties {
 
-    private String[] ciphers = new String[] {"TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256", "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA", "TLS_RSA_WITH_AES_128_GCM_SHA256",
-            "TLS_RSA_WITH_AES_128_CBC_SHA", "SSL_RSA_WITH_3DES_EDE_CBC_SHA"};
+    private String[] ciphers;
     private String[] enabledProtocols;
     private String keyStoreType;
     private String keyStoreFile;

common/src/main/java/timely/common/configuration/SslServerConfig.java

@@ -39,9 +39,11 @@ public class SslServerConfig {
         Boolean generate = sslServerProperties.isUseGeneratedKeypair();
         SslContextBuilder sslContextBuilder;
         if (generate) {
-            log.debug("Using generated self signed server certificate");
+            log.debug("Creating nettySslContext using generated self signed server certificate");
             sslContextBuilder = SslContextBuilder.forServer(ssc.certificate(), ssc.privateKey());
         } else {
+            log.debug("Creating nettySslContext using keyStoreFile:{} trustStoreFile:{}", sslServerProperties.getKeyStoreFile(),
+                            sslServerProperties.getTrustStoreFile());
             sslContextBuilder = SslHelper.getSslContextBuilder(sslServerProperties);
         }
 

grafana/pom.xml

@@ -4,7 +4,7 @@
   <parent>
     <groupId>timely</groupId>
     <artifactId>timely</artifactId>
-    <version>2.1.0</version>
+    <version>2.1.4-SNAPSHOT</version>
   </parent>
   <artifactId>timely-app</artifactId>
   <packaging>pom</packaging>

grafana/timely-app/dist/plugin.json

@@ -28,8 +28,8 @@
       }
     ],
     "screenshots": [],
-    "version": "2.1.0",
-    "updated": "2024-02-26"
+    "version": "2.1.4-SNAPSHOT",
+    "updated": "2024-03-13"
   },
   "dependencies": {
     "classnames": "2.2.6",

grafana/timely-app/go.mod

@@ -1,11 +1,96 @@
 module github.com/NationalSecurityAgency/timely-datasource
 
-go 1.14
+go 1.21
+
+toolchain go1.22.1
 
 require (
-	github.com/grafana/grafana-plugin-sdk-go v0.111.0
-	github.com/hashicorp/go-hclog v0.16.1
-	github.com/mattn/go-colorable v0.1.6 // indirect
-	github.com/prometheus/common v0.29.0 // indirect
-	golang.org/x/net v0.17.0 // indirect
+	github.com/grafana/grafana-plugin-sdk-go v0.218.0
+	github.com/hashicorp/go-hclog v1.6.2
+)
+
+require (
+	github.com/BurntSushi/toml v1.3.2 // indirect
+	github.com/apache/arrow/go/v15 v15.0.0 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/cheekybits/genny v1.0.0 // indirect
+	github.com/chromedp/cdproto v0.0.0-20220208224320-6efb837e6bc2 // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
+	github.com/elazarl/goproxy v0.0.0-20230731152917-f99041a5c027 // indirect
+	github.com/fatih/color v1.15.0 // indirect
+	github.com/fsnotify/fsnotify v1.4.9 // indirect
+	github.com/getkin/kin-openapi v0.120.0 // indirect
+	github.com/go-logr/logr v1.4.1 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/go-openapi/jsonpointer v0.19.6 // indirect
+	github.com/go-openapi/swag v0.22.4 // indirect
+	github.com/goccy/go-json v0.10.2 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/google/flatbuffers v23.5.26+incompatible // indirect
+	github.com/google/go-cmp v0.6.0 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/gorilla/mux v1.8.0 // indirect
+	github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // indirect
+	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.0 // indirect
+	github.com/hashicorp/go-plugin v1.6.0 // indirect
+	github.com/hashicorp/yamux v0.1.1 // indirect
+	github.com/invopop/yaml v0.2.0 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/klauspost/compress v1.16.7 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.5 // indirect
+	github.com/magefile/mage v1.15.0 // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/mattetti/filebuffer v1.0.1 // indirect
+	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-isatty v0.0.19 // indirect
+	github.com/mattn/go-runewidth v0.0.9 // indirect
+	github.com/mitchellh/go-testing-interface v1.14.1 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect
+	github.com/oklog/run v1.1.0 // indirect
+	github.com/olekukonko/tablewriter v0.0.5 // indirect
+	github.com/perimeterx/marshmallow v1.1.5 // indirect
+	github.com/pierrec/lz4/v4 v4.1.18 // indirect
+	github.com/prometheus/client_golang v1.18.0 // indirect
+	github.com/prometheus/client_model v0.5.0 // indirect
+	github.com/prometheus/common v0.46.0 // indirect
+	github.com/prometheus/procfs v0.12.0 // indirect
+	github.com/russross/blackfriday/v2 v2.1.0 // indirect
+	github.com/smartystreets/goconvey v1.6.4 // indirect
+	github.com/unknwon/bra v0.0.0-20200517080246-1e3013ecaff8 // indirect
+	github.com/unknwon/com v1.0.1 // indirect
+	github.com/unknwon/log v0.0.0-20150304194804-e617c87089d3 // indirect
+	github.com/urfave/cli v1.22.14 // indirect
+	github.com/zeebo/xxh3 v1.0.2 // indirect
+	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.49.0 // indirect
+	go.opentelemetry.io/contrib/propagators/jaeger v1.22.0 // indirect
+	go.opentelemetry.io/contrib/samplers/jaegerremote v0.18.0 // indirect
+	go.opentelemetry.io/otel v1.24.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.24.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.24.0 // indirect
+	go.opentelemetry.io/otel/metric v1.24.0 // indirect
+	go.opentelemetry.io/otel/sdk v1.24.0 // indirect
+	go.opentelemetry.io/otel/trace v1.24.0 // indirect
+	go.opentelemetry.io/proto/otlp v1.1.0 // indirect
+	golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect
+	golang.org/x/mod v0.13.0 // indirect
+	golang.org/x/net v0.22.0 // indirect
+	golang.org/x/sys v0.18.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
+	golang.org/x/tools v0.14.0 // indirect
+	golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
+	google.golang.org/genproto v0.0.0-20240123012728-ef4313101c80 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20240123012728-ef4313101c80 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80 // indirect
+	google.golang.org/grpc v1.62.1 // indirect
+	google.golang.org/protobuf v1.33.0 // indirect
+	gopkg.in/fsnotify/fsnotify.v1 v1.4.7 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

grafana/timely-app/package.json

@@ -1,6 +1,6 @@
 {
   "name": "timely-datasource",
-  "version": "2.1.0",
+  "version": "2.1.4-SNAPSHOT",
   "description": "Timely",
   "backend": true,
   "alerting": true,

grafana/timely-app/pkg/timely.go

@@ -398,6 +398,13 @@ func (td *TimelyDatasource) convertTimelyQuery(timelyQueryForm *TimelyQueryForm)
 		timelyQuery.RateOptions.Counter = timelyQueryForm.IsCounter
 	}
 
+    if len(timelyQueryForm.RateInterval) > 0 {
+        rateInterval, error := strconv.ParseInt(timelyQueryForm.RateInterval, 10, 32)
+        if error == nil {
+            timelyQuery.RateOptions.Interval = int32(rateInterval)
+        }
+    }
+
 	if len(timelyQueryForm.CounterMax) > 0 {
 		counterMax, error := strconv.ParseInt(timelyQueryForm.CounterMax, 10, 32)
 		if error == nil {

grafana/timely-app/pkg/types.go

@@ -11,6 +11,7 @@ type TimelyQueryForm struct {
     Tags                 map[string]string `json:"tags"`
     Filters              []Filter          `json:"filters"`
     ShouldComputeRate    bool              `json:"shouldComputeRate"`
+    RateInterval         string            `json:"rateInterval"`
     IsCounter            bool              `json:"isCounter"`
     CounterMax           string            `json:"counterMax"`
     CounterResetValue    string            `json:"counterResetValue"`
@@ -34,6 +35,7 @@ type TimelyQueryServer struct {
     Tags                 map[string]string `json:"tags"`
     Filters              []Filter          `json:"filters"`
     ShouldComputeRate    bool              `json:"shouldComputeRate"`
+    RateInterval         string            `json:"rateInterval"`
     IsCounter            bool              `json:"isCounter"`
     CounterMax           string            `json:"counterMax"`
     CounterResetValue    string            `json:"counterResetValue"`
@@ -50,6 +52,7 @@ type Filter struct {
 }
 
 type RateOptions struct {
+    Interval   int32 `json:"interval"`
     Counter    bool  `json:"counter"`
     CounterMax int32 `json:"counterMax"`
     ResetValue int32 `json:"resetValue"`
@@ -96,4 +99,4 @@ type TimelyErrorResponse struct {
     ResponseCode                    int32 `json:responseCode`
     Message                         string `json:message`
     DetailMessage                   string `json:detailMessage`
-}
+}

grafana/timely-app/src/QueryEditor.tsx

@@ -37,6 +37,7 @@ export class QueryEditor extends Component<Props, State> {
       downsampleInterval: '1m',
       downsampleAggregator: 'avg',
       shouldComputeRate: false,
+      rateInterval: '1m',
       isCounter: false,
       tags: {},
     };
@@ -123,6 +124,15 @@ export class QueryEditor extends Component<Props, State> {
     });
   };
 
+  onRateIntervalChange = (option: React.FormEvent<HTMLInputElement>) => {
+    const value = e.target.value;
+    this.setState((state, props) => {
+      var query = Object.assign(state.query, { rateInterval: value });
+      this.updateQuery(query);
+      return { query: query };
+    });
+  };
+
   onCounterChange = (option: React.FormEvent<HTMLInputElement>) => {
     const checked = option.currentTarget.checked;
     this.setState((state, props) => {
@@ -247,7 +257,7 @@ export class QueryEditor extends Component<Props, State> {
     const { metric, aggregator, alias } = this.state.query;
     const { downsampleInterval, downsampleAggregator, disableDownsampling } = this.state.query;
     const { tags } = this.state.query;
-    const { shouldComputeRate, isCounter, counterResetValue, counterMax } = this.state.query;
+    const { shouldComputeRate, rateInterval, isCounter, counterResetValue, counterMax } = this.state.query;
     const aggregatorOptions: Array<{}> = _.map(this.state.aggregatorTypes, (value: string) => ({
       label: value,
       value: value,
@@ -345,7 +355,7 @@ export class QueryEditor extends Component<Props, State> {
               spellCheck={false}
               value={downsampleInterval || ''}
               onChange={this.onDownsampleChange}
-              placeholder={'interval'}
+              placeholder={'downsample interval'}
             />
           </div>
           <div className="gf-form timely">
@@ -390,6 +400,13 @@ export class QueryEditor extends Component<Props, State> {
               onChange={this.onRateChange}
             />
             <div hidden={!shouldComputeRate}>
+              <Input
+                  className={'min-width-6 flex-grow-1'}
+                  spellCheck={false}
+                  value={rateInterval || ''}
+                  onChange={this.onRateIntervalChange}
+                  placeholder={'rate interval'}
+              />
               <Switch
                 label={'Counter'}
                 labelClass="width-10"

grafana/timely-app/src/types.ts

@@ -9,6 +9,7 @@ export interface TimelyQueryForm extends DataQuery {
   downsampleAggregator?: string;
   tags: { [key: string]: string };
   shouldComputeRate?: boolean;
+  rateInterval?: string;
   isCounter?: boolean;
   counterMax?: string;
   counterResetValue?: string;

pom.xml

@@ -3,7 +3,7 @@
   <modelVersion>4.0.0</modelVersion>
   <groupId>timely</groupId>
   <artifactId>timely</artifactId>
-  <version>2.1.0</version>
+  <version>2.1.4-SNAPSHOT</version>
   <packaging>pom</packaging>
   <name>Timely</name>
   <description>A secure and scalable time series database based on Apache Accumulo</description>
@@ -237,13 +237,13 @@
       </dependency>
       <dependency>
         <groupId>io.netty</groupId>
-        <artifactId>netty-all</artifactId>
-        <version>${version.netty}</version>
+        <artifactId>netty-tcnative-boringssl-static</artifactId>
+        <version>${version.netty-tcnative}</version>
       </dependency>
       <dependency>
         <groupId>io.netty</groupId>
-        <artifactId>netty-tcnative-boringssl-static</artifactId>
-        <version>${version.netty-tcnative}</version>
+        <artifactId>netty-transport-native-epoll</artifactId>
+        <version>${version.netty}</version>
       </dependency>
       <dependency>
         <groupId>io.projectreactor.netty</groupId>
@@ -592,6 +592,11 @@
         <artifactId>jsoup</artifactId>
         <version>${version.jsoup}</version>
       </dependency>
+      <dependency>
+        <groupId>org.slf4j</groupId>
+        <artifactId>jcl-over-slf4j</artifactId>
+        <version>${version.slf4j}</version>
+      </dependency>
       <dependency>
         <groupId>org.slf4j</groupId>
         <artifactId>slf4j-api</artifactId>

server/bin/create-timely-profiles.sh

@@ -1,40 +0,0 @@
-#!/bin/bash
-
-if [[ `uname` == "Darwin" ]]; then
-    THIS_SCRIPT=`python -c 'import os,sys; print os.path.realpath(sys.argv[1])' $0`
-else
-    THIS_SCRIPT=`readlink -f $0`
-fi
-
-THIS_DIR="${THIS_SCRIPT%/*}"
-BASE_DIR=$(cd $THIS_DIR/.. && pwd)
-BIN_DIR=${BASE_DIR}/bin
-CONF_DIR=${BASE_DIR}/conf
-
-. ${BIN_DIR}/timely-server-env.sh
-
-for i in $(seq 1 $NUM_PROFILES); do
-    if [ "$NUM_PROFILES" -eq "1" ]; then
-        FILENAME=timely.yml
-    else
-        FILENAME=timely-${i}.yml
-    fi
-
-    echo "creating configuration file ${CONF_DIR}/${FILENAME}"
-    cp ${CONF_DIR}/timely-template.yml ${CONF_DIR}/${FILENAME}
-#    if [ "$NUM_PROFILES" -eq "1" ]; then
-#        sed -i "s/\${INSTANCE}//g" ${CONF_DIR}/${FILENAME}
-#    else
-#        sed -i "s/\${INSTANCE}/${i}/g" ${CONF_DIR}/${FILENAME}
-#    fi
-    MULTIPLE=$(($i - 1))
-    TCP_PORT=$(($TCP_PORT_START + $MULTIPLE * $PORT_INCREMENT))
-    sed -i "s/\${TCP_PORT}/${TCP_PORT}/g" ${CONF_DIR}/${FILENAME}
-    HTTP_PORT=$(($HTTP_PORT_START + $MULTIPLE * $PORT_INCREMENT))
-    sed -i "s/\${HTTP_PORT}/${HTTP_PORT}/g" ${CONF_DIR}/${FILENAME}
-    WEBSOCKET_PORT=$(($WEBSOCKET_PORT_START + $MULTIPLE * $PORT_INCREMENT))
-    sed -i "s/\${WEBSOCKET_PORT}/${WEBSOCKET_PORT}/g" ${CONF_DIR}/${FILENAME}
-    UDP_PORT=$(($UDP_PORT_START + $MULTIPLE * $PORT_INCREMENT))
-    sed -i "s/\${UDP_PORT}/${UDP_PORT}/g" ${CONF_DIR}/${FILENAME}
-done
-

server/bin/test/load-test-ingest.sh

@@ -19,6 +19,7 @@ set +a
 
 JVM_ARGS="-Xmx1G -Xms1G"
 JVM_ARGS="${JVM_ARGS} -Dlogging.config=${CONF_DIR}/log4j2-console.yml"
+JVM_ARGS="${JVM_ARGS} -Dlog4j2.contextSelector=org.apache.logging.log4j.core.async.AsyncLoggerContextSelector"
 
 echo "$JAVA_HOME/bin/java ${JVM_ARGS} -jar ${THIS_DIR}/timely-server*-exec.jar testingest --spring.profiles.active=timely,loadtest"
 $JAVA_HOME/bin/java ${JVM_ARGS} -jar ${THIS_DIR}/timely-server-*-exec.jar testingest --spring.profiles.active=timely,loadtest

server/bin/test/load-test-query.sh

@@ -19,6 +19,7 @@ set +a
 
 JVM_ARGS="-Xmx1G -Xms1G"
 JVM_ARGS="${JVM_ARGS} -Dlogging.config=${CONF_DIR}/log4j2-console.yml"
+JVM_ARGS="${JVM_ARGS} -Dlog4j2.contextSelector=org.apache.logging.log4j.core.async.AsyncLoggerContextSelector"
 
 echo "$JAVA_HOME/bin/java ${JVM_ARGS} -jar ${THIS_DIR}/timely-server*-exec.jar testquery --spring.profiles.active=timely,loadtest"
 $JAVA_HOME/bin/java ${JVM_ARGS} -jar ${THIS_DIR}/timely-server-*-exec.jar testquery --spring.profiles.active=timely,loadtest

server/bin/timely-server.sh

@@ -2,10 +2,10 @@
 
 if [[ $(uname) == "Darwin" ]]; then
         THIS_SCRIPT=$(python -c 'import os,sys; print os.path.realpath(sys.argv[1])' "$0")
-        TCNATIVE_SUFFIX="jnilib"
 else
         THIS_SCRIPT=$(readlink -f "$0")
-        TCNATIVE_SUFFIX="so"
+        HW=`uname -m`
+        OS_PATH=linux-${HW}
 fi
 
 if [ -z "$1" ]; then
@@ -17,7 +17,6 @@ fi
 PID=$(jps -m | grep -E 'timely-server-.*-exec.jar timelyserver' | grep "instance=${INSTANCE}" | awk '{print $1}')
 
 if [ "$PID" == "" ]; then
-
     THIS_DIR="${THIS_SCRIPT%/*}"
     BASE_DIR=$(cd "$THIS_DIR"/.. && pwd)
     TMP_DIR="${BASE_DIR}/tmp"
@@ -47,20 +46,23 @@ if [ "$PID" == "" ]; then
       mkdir "${LOG_DIR}"
     fi
 
-    pushd "${BASE_DIR}/bin" || exit
-    "${JAVA_HOME}/bin/jar xf ${LIB_DIR}/netty-tcnative-boringssl-static*.jar META-INF/native/libnetty_tcnative_linux_x86_64.${TCNATIVE_SUFFIX}"
-    "${JAVA_HOME}/bin/jar xf ${LIB_DIR}/netty-all*.jar META-INF/native/libnetty_transport_native_epoll_x86_64.${TCNATIVE_SUFFIX}"
-    popd || exit
-
     JVM_ARGS="-Xmx4G -Xms4G -XX:NewSize=1G -XX:MaxNewSize=1G"
-    JVM_ARGS="${JVM_ARGS} -Djava.library.path=${NATIVE_DIR}"
     JVM_ARGS="${JVM_ARGS} -Dlogging.config=${CONF_DIR}/log4j2.yml"
+    JVM_ARGS="${JVM_ARGS} -Dlog4j2.contextSelector=org.apache.logging.log4j.core.async.AsyncLoggerContextSelector"
     JVM_ARGS="${JVM_ARGS} -XX:+UseG1GC -XX:+UseStringDeduplication"
     JVM_ARGS="${JVM_ARGS} -Djava.net.preferIPv4Stack=true"
     JVM_ARGS="${JVM_ARGS} -XX:+UseNUMA"
 
-    echo "${JAVA_HOME}/bin/java ${JVM_ARGS} -jar ${THIS_DIR}/timely-server*-exec.jar timelyserver --instance=${INSTANCE}"
-    nohup "${JAVA_HOME}"/bin/java ${JVM_ARGS} -jar "${THIS_DIR}"/timely-server-*-exec.jar timelyserver --instance="${INSTANCE}" >> "${LOG_DIR}/timely-server${INSTANCE}.out" 2>&1 &
+    if [[ -n "$OS_PATH" ]]; then
+      pushd "${BASE_DIR}/bin" || exit
+      "${JAVA_HOME}"/bin/jar xf "${LIB_DIR}"/netty-tcnative-boringssl-static-*-"${OS_PATH}".jar ${NATIVE_DIR}/libnetty_tcnative_linux_"${HW}".so
+      "${JAVA_HOME}"/bin/jar xf "${LIB_DIR}"/netty-transport-native-epoll-*-"${OS_PATH}".jar ${NATIVE_DIR}/libnetty_transport_native_epoll_"${HW}".so
+      popd || exit
+      JVM_ARGS="${JVM_ARGS} -Djava.library.path=${NATIVE_DIR}"
+    fi
+
+    echo "${JAVA_HOME}/bin/java ${JVM_ARGS} -jar ${BIN_DIR}/timely-server*-exec.jar timelyserver --instance=${INSTANCE}"
+    nohup "${JAVA_HOME}"/bin/java ${JVM_ARGS} -jar "${BIN_DIR}"/timely-server-*-exec.jar timelyserver --instance="${INSTANCE}" >> "${LOG_DIR}/timely-server${INSTANCE}.out" 2>&1 &
 else
     echo "timely-server --instance=${INSTANCE} already running with pid ${PID}"
 fi

server/bin/util/get-metrics.sh

@@ -1,11 +1,15 @@
 #!/bin/bash
 
 if [[ `uname` == "Darwin" ]]; then
-        THIS_SCRIPT=`python -c 'import os,sys; print os.path.realpath(sys.argv[1])' $0`
+    THIS_SCRIPT=`python -c 'import os,sys; print os.path.realpath(sys.argv[1])' $0`
 else
-        THIS_SCRIPT=`readlink -f $0`
+    THIS_SCRIPT=`readlink -f $0`
+    HW=`uname -m`
+    OS_PATH=linux-${HW}
 fi
 
+export INSTANCE="1"
+
 THIS_DIR="${THIS_SCRIPT%/*}"
 BASE_DIR=$(cd $THIS_DIR/../.. && pwd)
 LIB_DIR="${BASE_DIR}/lib"
@@ -19,6 +23,15 @@ set +a
 
 JVM_ARGS="-Xmx256m -Xms128m"
 JVM_ARGS="${JVM_ARGS} -Dlogging.config=${CONF_DIR}/log4j2-console.yml"
+JVM_ARGS="${JVM_ARGS} -Dlog4j2.contextSelector=org.apache.logging.log4j.core.async.AsyncLoggerContextSelector"
 
-echo "$JAVA_HOME/bin/java ${JVM_ARGS} -jar ${THIS_DIR}/timely-server*-exec.jar metrics"
-$JAVA_HOME/bin/java ${JVM_ARGS} -jar ${THIS_DIR}/timely-server-*-exec.jar metrics
+if [[ -n "$OS_PATH" ]]; then
+    pushd "${BASE_DIR}/bin" || exit
+    "${JAVA_HOME}"/bin/jar xf "${LIB_DIR}"/netty-tcnative-boringssl-static-*.Final-"${OS_PATH}".jar ${NATIVE_DIR}/libnetty_tcnative_linux_"${HW}".so
+    "${JAVA_HOME}"/bin/jar xf "${LIB_DIR}"/netty-transport-native-epoll-*-"${OS_PATH}".jar ${NATIVE_DIR}/libnetty_transport_native_epoll_"${HW}".so
+    popd || exit
+    JVM_ARGS="${JVM_ARGS} -Djava.library.path=${NATIVE_DIR}"
+fi
+
+echo "$JAVA_HOME/bin/java ${JVM_ARGS} -jar ${BIN_DIR}/timely-server*-exec.jar metrics"
+$JAVA_HOME/bin/java ${JVM_ARGS} -jar ${BIN_DIR}/timely-server-*-exec.jar metrics

server/bin/util/get-tablet-summary.sh

@@ -1,11 +1,15 @@
 #!/bin/bash
 
 if [[ `uname` == "Darwin" ]]; then
-        THIS_SCRIPT=`python -c 'import os,sys; print os.path.realpath(sys.argv[1])' $0`
+    THIS_SCRIPT=`python -c 'import os,sys; print os.path.realpath(sys.argv[1])' $0`
 else
-        THIS_SCRIPT=`readlink -f $0`
+    THIS_SCRIPT=`readlink -f $0`
+    HW=`uname -m`
+    OS_PATH=linux-${HW}
 fi
 
+export INSTANCE="1"
+
 THIS_DIR="${THIS_SCRIPT%/*}"
 BASE_DIR=$(cd $THIS_DIR/../.. && pwd)
 LIB_DIR="${BASE_DIR}/lib"
@@ -19,6 +23,15 @@ set +a
 
 JVM_ARGS="-Xmx1024m -Xms128m"
 JVM_ARGS="${JVM_ARGS} -Dlogging.config=${CONF_DIR}/log4j2-console.yml"
+JVM_ARGS="${JVM_ARGS} -Dlog4j2.contextSelector=org.apache.logging.log4j.core.async.AsyncLoggerContextSelector"
 
-echo "$JAVA_HOME/bin/java ${JVM_ARGS} -jar ${THIS_DIR}/timely-server*-exec.jar tablet --spring.profiles.active=timely,tablet"
-$JAVA_HOME/bin/java ${JVM_ARGS} -jar ${THIS_DIR}/timely-server-*-exec.jar tablet --spring.profiles.active=timely,tablet
+if [[ -n "$OS_PATH" ]]; then
+    pushd "${BASE_DIR}/bin" || exit
+    "${JAVA_HOME}"/bin/jar xf "${LIB_DIR}"/netty-tcnative-boringssl-static-*.Final-"${OS_PATH}".jar ${NATIVE_DIR}/libnetty_tcnative_linux_"${HW}".so
+    "${JAVA_HOME}"/bin/jar xf "${LIB_DIR}"/netty-transport-native-epoll-*-"${OS_PATH}".jar ${NATIVE_DIR}/libnetty_transport_native_epoll_"${HW}".so
+    popd || exit
+    JVM_ARGS="${JVM_ARGS} -Djava.library.path=${NATIVE_DIR}"
+fi
+
+echo "$JAVA_HOME/bin/java ${JVM_ARGS} -jar ${BIN_DIR}/timely-server*-exec.jar tablet --spring.profiles.active=timely,tablet"
+$JAVA_HOME/bin/java ${JVM_ARGS} -jar ${BIN_DIR}/timely-server-*-exec.jar tablet --spring.profiles.active=timely,tablet

server/conf/log4j2-console.yml

@@ -9,7 +9,7 @@ Configuration:
       - name: PID
         value: "????"
       - name: LOG_PATTERN
-        value: "%clr{%d{yyyy-MM-dd HH:mm:ss.SSS}}{faint} %clr{%5p} %clr{${sys:PID}}{magenta} %clr{---}{faint} %clr{[%15.15t]}{faint} %clr{%-40.40c{1.}}{cyan} %clr{:}{faint} %m%n%wEx"
+        value: "%d{ISO8601} %-5p [%-40.40t] [%-40.40c{1.}] %m%n%wEx"
 
   Appenders:
     Console:

server/conf/log4j2.yml

@@ -9,7 +9,7 @@ Configuration:
       - name: PID
         value: "????"
       - name: LOG_PATTERN
-        value: "%clr{%d{yyyy-MM-dd HH:mm:ss.SSS}}{faint} %clr{%5p} %clr{${sys:PID}}{magenta} %clr{---}{faint} %clr{[%15.15t]}{faint} %clr{%-40.40c{1.}}{cyan} %clr{:}{faint} %m%n%wEx"
+        value: "%d{ISO8601} %-5p [%-40.40t] [%-40.40c{1.}] %m%n%wEx"
 
   Appenders:
     RollingFile:

server/pom.xml

@@ -4,7 +4,7 @@
   <parent>
     <groupId>timely</groupId>
     <artifactId>timely</artifactId>
-    <version>2.1.0</version>
+    <version>2.1.4-SNAPSHOT</version>
   </parent>
   <artifactId>timely-server</artifactId>
   <description>Timely Server</description>
@@ -68,10 +68,6 @@
       <groupId>io.micrometer</groupId>
       <artifactId>micrometer-core</artifactId>
     </dependency>
-    <dependency>
-      <groupId>io.netty</groupId>
-      <artifactId>netty-all</artifactId>
-    </dependency>
     <dependency>
       <groupId>io.projectreactor.netty</groupId>
       <artifactId>reactor-netty</artifactId>
@@ -192,6 +188,25 @@
     <dependency>
       <groupId>io.netty</groupId>
       <artifactId>netty-tcnative-boringssl-static</artifactId>
+      <classifier>linux-aarch_64</classifier>
+      <scope>runtime</scope>
+    </dependency>
+    <dependency>
+      <groupId>io.netty</groupId>
+      <artifactId>netty-tcnative-boringssl-static</artifactId>
+      <classifier>linux-x86_64</classifier>
+      <scope>runtime</scope>
+    </dependency>
+    <dependency>
+      <groupId>io.netty</groupId>
+      <artifactId>netty-transport-native-epoll</artifactId>
+      <classifier>linux-aarch_64</classifier>
+      <scope>runtime</scope>
+    </dependency>
+    <dependency>
+      <groupId>io.netty</groupId>
+      <artifactId>netty-transport-native-epoll</artifactId>
+      <classifier>linux-x86_64</classifier>
       <scope>runtime</scope>
     </dependency>
     <dependency>

server/src/main/dist/assembly.xml

@@ -19,8 +19,10 @@
       <directory>${project.build.directory}/lib</directory>
       <outputDirectory>lib</outputDirectory>
       <includes>
-        <include>netty-tcnative-boringssl-static*.jar</include>
-        <include>netty-all*.jar</include>
+        <include>netty-tcnative-boringssl-static*linux-x86_64.jar</include>
+        <include>netty-tcnative-boringssl-static*linux-aarch_64.jar</include>
+        <include>netty-transport-native-epoll*linux-x86_64.jar</include>
+        <include>netty-transport-native-epoll*linux-aarch_64.jar</include>
       </includes>
     </fileSet>
     <fileSet>

server/src/main/java/timely/server/sample/iterators/RateIterator.java

@@ -1,5 +1,7 @@
 package timely.server.sample.iterators;
 
+import static org.apache.accumulo.core.conf.ConfigurationTypeHelper.getTimeInMillis;
+
 import java.io.IOException;
 import java.util.HashMap;
 import java.util.Iterator;
@@ -20,12 +22,14 @@ import timely.api.request.timeseries.QueryRequest;
 public class RateIterator extends TimeSeriesGroupingIterator {
 
     private static final Logger log = LoggerFactory.getLogger(RateIterator.class);
-    private static final String COUNTER_MAX = "rate.counter.max";
-    private static final String RATE_RESET_VALUE = "rate.reset.value";
+    public static final String COUNTER_MAX = "rate.counter.max";
+    public static final String RATE_RESET_VALUE = "rate.reset.value";
+    public static final String RATE_INTERVAL = "rate.interval";
 
     private boolean isCounter = false;
     private long maxCounter = 0;
     private long resetValue = 0;
+    private long interval = 1;
 
     public static void setRateOptions(IteratorSetting is, QueryRequest.RateOption options) {
         if (options != null && options.isCounter()) {
@@ -33,6 +37,11 @@ public class RateIterator extends TimeSeriesGroupingIterator {
             is.addOption(COUNTER_MAX, Long.toString(options.getCounterMax()));
             is.addOption(RATE_RESET_VALUE, Long.toString(options.getResetValue()));
         }
+        if (options != null && options.getInterval().isPresent()) {
+            is.addOption(RATE_INTERVAL, Long.toString(getTimeInMillis(options.getInterval().get())));
+        } else {
+            is.addOption(RATE_INTERVAL, "1");
+        }
     }
 
     @Override
@@ -41,8 +50,11 @@ public class RateIterator extends TimeSeriesGroupingIterator {
             this.isCounter = true;
             this.maxCounter = Long.parseLong(options.get(COUNTER_MAX));
             this.resetValue = Long.parseLong(options.get(RATE_RESET_VALUE));
-            log.trace("Setting rate counter options cm:{}, rv:{}", this.maxCounter, this.resetValue);
         }
+        if (options.containsKey(RATE_INTERVAL)) {
+            this.interval = Long.parseLong(options.get(RATE_INTERVAL));
+        }
+        log.trace("Setting rate options interval:{} isCounter:{} cm:{}, rv:{}", this.interval, this.isCounter, this.maxCounter, this.resetValue);
         Map<String,String> opts = new HashMap<>(options);
         opts.put(FILTER, "-1,1");
         super.init(source, opts, env);
@@ -76,7 +88,9 @@ public class RateIterator extends TimeSeriesGroupingIterator {
         if (timeDiff == 0) {
             return 0.0D;
         }
-        Double result = ((secondVal - firstVal) / timeDiff);
+        // (secondVal - firstVal) / timeDiff will be value per millisecond, so multiply by interval to get
+        // value per interval. Example 60000 milliseconds in 1m, so multiply by 60000 milliseconds
+        Double result = ((secondVal - firstVal) / timeDiff * interval);
         log.trace("compute - result: {}", result);
 
         if (isCounter && resetValue > 0 && result > resetValue) {

server/src/main/java/timely/server/store/DataStore.java

@@ -700,6 +700,12 @@ public class DataStore {
                             if (query.isRate()) {
                                 log.trace("Adding rate iterator");
                                 IteratorSetting rate = new IteratorSetting(499, RateIterator.class);
+                                RateOption rateOptions = query.getRateOptions();
+                                // if there is no rate interval set, then use the downsample value
+                                // so that the result is the change per downsample period
+                                if (rateOptions != null && !rateOptions.getInterval().isPresent()) {
+                                    rateOptions.setInterval(Optional.of(downsample + "ms"));
+                                }
                                 RateIterator.setRateOptions(rate, query.getRateOptions());
                                 scanner.addScanIterator(rate);
                             }

server/src/main/java/timely/server/store/cache/DataStoreCache.java

@@ -14,6 +14,7 @@ import java.util.Iterator;
 import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
 import java.util.TreeSet;
 import java.util.concurrent.Executors;
@@ -765,7 +766,13 @@ public class DataStoreCache {
             if (subQuery.isRate()) {
                 log.trace("Adding rate iterator");
                 IteratorSetting rate = new IteratorSetting(499, RateIterator.class);
-                RateIterator.setRateOptions(rate, subQuery.getRateOptions());
+                QueryRequest.RateOption rateOptions = subQuery.getRateOptions();
+                // if there is no rate interval set, then use the downsample value
+                // so that the result is the change per downsample period
+                if (rateOptions != null && !rateOptions.getInterval().isPresent()) {
+                    rateOptions.setInterval(Optional.of(downsamplePeriod + "ms"));
+                }
+                RateIterator.setRateOptions(rate, rateOptions);
                 RateIterator rateIterator = new RateIterator();
                 rateIterator.init(itr, rate.getOptions(), null);
                 itr = rateIterator;

server/src/test/java/timely/server/sample/iterators/RateIteratorTest.java

@@ -7,6 +7,7 @@ import static org.junit.Assert.assertTrue;
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Optional;
 import java.util.Random;
 import java.util.TreeMap;
 
@@ -46,14 +47,26 @@ public class RateIteratorTest extends IteratorTestBase {
 
     @Test
     public void testConstantTimeRate() throws Exception {
+        testInterval(null, 0.001D);
+        testInterval("1ms", 0.001D);
+        testInterval("1s", 1.0D);
+        testInterval("60s", 60.0D);
+    }
+
+    private void testInterval(String interval, double result) throws Exception {
         SortedMapIterator source = new SortedMapIterator(table);
         RateIterator iter = new RateIterator();
         IteratorSetting settings = new IteratorSetting(100, RateIterator.class);
+        QueryRequest.RateOption rateOption = new QueryRequest.RateOption();
+        if (interval != null) {
+            rateOption.setInterval(Optional.of(interval));
+        }
+        RateIterator.setRateOptions(settings, rateOption);
         iter.init(source, settings.getOptions(), SCAN_IE);
         iter.seek(new Range(), EMPTY_COL_FAMS, true);
         for (int i = 0; i < 99; i++) {
             assertTrue(iter.hasTop());
-            assertEquals(0.001D, MetricAdapter.decodeValue(iter.getTopValue().get()), 0.0D);
+            assertEquals(result, MetricAdapter.decodeValue(iter.getTopValue().get()), 0.0D);
             iter.next();
         }
         assertFalse(iter.hasTop());

test/pom.xml

@@ -4,7 +4,7 @@
   <parent>
     <groupId>timely</groupId>
     <artifactId>timely</artifactId>
-    <version>2.1.0</version>
+    <version>2.1.4-SNAPSHOT</version>
   </parent>
   <artifactId>timely-test</artifactId>
   <properties>