Release version 2.1.5

analytics/pom.xml

@@ -4,7 +4,7 @@
   <parent>
     <groupId>timely</groupId>
     <artifactId>timely</artifactId>
-    <version>2.1.4</version>
+    <version>2.1.5</version>
   </parent>
   <artifactId>timely-analytics</artifactId>
   <!--

balancer/bin/timely-balancer.sh

@@ -1,17 +1,17 @@
 #!/bin/bash
 
 if [[ $(uname) == "Darwin" ]]; then
-        THIS_SCRIPT=$(python -c 'import os,sys; print os.path.realpath(sys.argv[1])' "$0")
+    THIS_SCRIPT=$(python -c 'import os,sys; print os.path.realpath(sys.argv[1])' "$0")
 else
-        THIS_SCRIPT=$(readlink -f "$0")
-        HW=`uname -m`
-        OS_PATH=linux-${HW}
+    THIS_SCRIPT=$(readlink -f "$0")
+    HW=`uname -m`
+    OS_PATH=linux-${HW}
 fi
 
 if [ -z "$1" ]; then
-  export INSTANCE="1"
+    export INSTANCE="1"
 else
-  export INSTANCE="$1"
+    export INSTANCE="$1"
 fi
 
 PID=$(jps -m | grep -E 'timely-balancer-.*-exec.jar' | grep "instance=${INSTANCE}" | awk '{print $1}')
@@ -19,7 +19,6 @@ PID=$(jps -m | grep -E 'timely-balancer-.*-exec.jar' | grep "instance=${INSTANCE
 if [ "$PID" == "" ]; then
     THIS_DIR="${THIS_SCRIPT%/*}"
     BASE_DIR=$(cd "$THIS_DIR"/.. && pwd)
-    TMP_DIR="${BASE_DIR}/tmp"
     LIB_DIR="${BASE_DIR}/lib"
     BIN_DIR="${BASE_DIR}/bin"
     export CONF_DIR="${BASE_DIR}/conf"
@@ -32,18 +31,12 @@ if [ "$PID" == "" ]; then
     # use either a value from timely-balancer-env.sh or the default
     export LOG_DIR="${TIMELY_LOG_DIR:-${BASE_DIR}/logs}"
 
-    if [[ -e ${TMP_DIR} ]]; then
-      rm -rf "${TMP_DIR}"
+    if [[ ! -e ${NATIVE_DIR} ]]; then
+        mkdir -p "${NATIVE_DIR}"
     fi
-    mkdir "${TMP_DIR}"
-
-    if [[ -e ${NATIVE_DIR} ]]; then
-      rm -rf "${NATIVE_DIR}"
-    fi
-    mkdir -p "${NATIVE_DIR}"
 
     if [[ ! -e ${LOG_DIR} ]]; then
-      mkdir "${LOG_DIR}"
+        mkdir "${LOG_DIR}"
     fi
 
     JVM_ARGS="-Xmx2G -Xms2G -XX:NewSize=1G -XX:MaxNewSize=1G"
@@ -54,11 +47,13 @@ if [ "$PID" == "" ]; then
     JVM_ARGS="${JVM_ARGS} -XX:+UseNUMA"
 
     # for linux varieties of OS, extract native libraries if not already present
-    if [[ -n "$OS_PATH" && (! -d ${NATIVE_DIR} || -z "$(ls -A ${NATIVE_DIR})") ]]; then
-        pushd "${BASE_DIR}/bin" || exit
-        "${JAVA_HOME}"/bin/jar xf "${LIB_DIR}"/netty-tcnative-boringssl-static-*.Final-"${OS_PATH}".jar ${NATIVE_DIR}/libnetty_tcnative_linux_"${HW}".so
-        "${JAVA_HOME}"/bin/jar xf "${LIB_DIR}"/netty-transport-native-epoll-*-"${OS_PATH}".jar ${NATIVE_DIR}/libnetty_transport_native_epoll_"${HW}".so
-        popd || exit
+    if [[ -n "$OS_PATH" ]]; then
+        if [[ ! -d ${NATIVE_DIR} || -z "$(ls -A ${NATIVE_DIR})" ]]; then
+            pushd "${BASE_DIR}/bin" || exit
+            "${JAVA_HOME}"/bin/jar xf "${LIB_DIR}"/netty-tcnative-boringssl-static-*.Final-"${OS_PATH}".jar META-INF/native/libnetty_tcnative_linux_"${HW}".so
+            "${JAVA_HOME}"/bin/jar xf "${LIB_DIR}"/netty-transport-native-epoll-*-"${OS_PATH}".jar META-INF/native/libnetty_transport_native_epoll_"${HW}".so
+            popd || exit
+        fi
         JVM_ARGS="${JVM_ARGS} -Djava.library.path=${NATIVE_DIR}"
     fi
 

balancer/pom.xml

@@ -4,7 +4,7 @@
   <parent>
     <groupId>timely</groupId>
     <artifactId>timely</artifactId>
-    <version>2.1.4</version>
+    <version>2.1.5</version>
   </parent>
   <artifactId>timely-balancer</artifactId>
   <description>Timely Balancer</description>

balancer/src/main/java/timely/balancer/MetricResolver.java

@@ -91,17 +91,21 @@ public class MetricResolver {
     final protected BalancerProperties balancerProperties;
     final protected CacheProperties cacheProperties;
     final protected HealthChecker healthChecker;
+    final protected Path assignmentFile;
+    final protected FileSystem fs;
 
     private enum BalanceType {
         HIGH_LOW, HIGH_AVG, AVG_LOW
     }
 
     public MetricResolver(CuratorFramework curatorFramework, BalancerProperties balancerProperties, CacheProperties cacheProperties,
-                    HealthChecker healthChecker) {
+                    HealthChecker healthChecker) throws Exception {
         this.curatorFramework = curatorFramework;
         this.balancerProperties = balancerProperties;
         this.cacheProperties = cacheProperties;
         this.healthChecker = healthChecker;
+        this.assignmentFile = new Path(balancerProperties.getAssignmentFile());
+        this.fs = getFileSystem(this.assignmentFile);
     }
 
     public void start() {
@@ -916,10 +920,8 @@ public class MetricResolver {
 
     protected void createAssignmentFile() {
         try {
-            log.info("Creating assignment file: " + balancerProperties.getAssignmentFile());
-            Path assignmentFile = new Path(balancerProperties.getAssignmentFile());
-            FileSystem fs = getFileSystem(assignmentFile);
-            fs.create(assignmentFile);
+            log.info("Creating assignment file: " + this.assignmentFile);
+            this.fs.create(this.assignmentFile);
         } catch (Exception e) {
             log.error(e.getMessage(), e);
         }
@@ -927,9 +929,7 @@ public class MetricResolver {
 
     protected long getLastWrittenToHdfsTimestamp() {
         try {
-            Path assignmentFile = new Path(balancerProperties.getAssignmentFile());
-            FileSystem fs = getFileSystem(assignmentFile);
-            FileStatus fileStatus = fs.getFileStatus(assignmentFile);
+            FileStatus fileStatus = this.fs.getFileStatus(this.assignmentFile);
             return fileStatus.getModificationTime();
         } catch (FileNotFoundException e) {
             createAssignmentFile();
@@ -960,16 +960,15 @@ public class MetricResolver {
         // proceed with reading from HDFS
 
         Map<String,TimelyBalancedHost> assignedMetricToHostMap = new TreeMap<>();
+        CsvReader reader = null;
         try {
             boolean acquired = false;
             while (!acquired) {
                 acquired = assignmentsIPRWLock.readLock().acquire(60, TimeUnit.SECONDS);
             }
             balancerLock.writeLock().lock();
-            Path assignmentFile = new Path(balancerProperties.getAssignmentFile());
-            FileSystem fs = getFileSystem(assignmentFile);
-            FSDataInputStream iStream = fs.open(assignmentFile);
-            CsvReader reader = new CsvReader(iStream, ',', Charset.forName("UTF-8"));
+            FSDataInputStream iStream = this.fs.open(this.assignmentFile);
+            reader = new CsvReader(iStream, ',', Charset.forName("UTF-8"));
             reader.setUseTextQualifier(false);
 
             // skip the headers
@@ -1031,6 +1030,9 @@ public class MetricResolver {
         } catch (Exception e) {
             log.error(e.getMessage(), e);
         } finally {
+            if (reader != null) {
+                reader.close();
+            }
             balancerLock.writeLock().unlock();
             try {
                 assignmentsIPRWLock.readLock().release();
@@ -1052,12 +1054,10 @@ public class MetricResolver {
             nonCachedMetricsLocalLock.readLock().lock();
             try {
                 if (!metricToHostMap.isEmpty()) {
-                    Path assignmentFile = new Path(balancerProperties.getAssignmentFile());
-                    FileSystem fs = getFileSystem(assignmentFile);
-                    if (!fs.exists(assignmentFile.getParent())) {
-                        fs.mkdirs(assignmentFile.getParent());
+                    if (!this.fs.exists(this.assignmentFile.getParent())) {
+                        this.fs.mkdirs(this.assignmentFile.getParent());
                     }
-                    FSDataOutputStream oStream = fs.create(assignmentFile, true);
+                    FSDataOutputStream oStream = this.fs.create(this.assignmentFile, true);
                     writer = new CsvWriter(oStream, ',', Charset.forName("UTF-8"));
                     writer.setUseTextQualifier(false);
                     writer.write("metric");
@@ -1080,7 +1080,7 @@ public class MetricResolver {
                     if (!assignmentsLastUpdatedInHdfs.get().succeeded()) {
                         assignmentsLastUpdatedInHdfs.forceSet(now);
                     }
-                    log.debug("Wrote {} assignments to hdfs lastHdfsUpdate = lastLocalUpdate ({})", metricToHostMap.size(),
+                    log.info("Wrote {} assignments to hdfs lastHdfsUpdate = lastLocalUpdate ({})", metricToHostMap.size(),
                                     new Date(assignmentsLastUpdatedLocal.get()));
                     // remove metric from metricMap (ArrivalRate) if not being cached
                     metricMap.entrySet().removeIf(e -> !metricToHostMap.containsKey(e.getKey()));

balancer/src/main/java/timely/balancer/configuration/BalancerConfiguration.java

@@ -50,7 +50,7 @@ public class BalancerConfiguration {
     @Bean
     @ConditionalOnMissingBean
     public MetricResolver metricResolver(CuratorFramework curatorFramework, BalancerProperties balancerProperties, CacheProperties cacheProperties,
-                    HealthChecker healthChecker) {
+                    HealthChecker healthChecker) throws Exception {
         MetricResolver metricResolver = new MetricResolver(curatorFramework, balancerProperties, cacheProperties, healthChecker);
         metricResolver.start();
         return metricResolver;

balancer/src/test/java/timely/balancer/configuration/TestBalancerConfiguration.java

@@ -48,7 +48,7 @@ public class TestBalancerConfiguration {
 
     @Bean
     public TestMetricResolver metricResolver(CuratorFramework curatorFramework, BalancerProperties balancerProperties, CacheProperties cacheProperties,
-                    HealthChecker healthChecker) {
+                    HealthChecker healthChecker) throws Exception {
         TestMetricResolver metricResolver = new TestMetricResolver(curatorFramework, balancerProperties, cacheProperties, healthChecker);
         metricResolver.start();
         return metricResolver;

balancer/src/test/java/timely/balancer/test/TestMetricResolver.java

@@ -20,7 +20,7 @@ public class TestMetricResolver extends MetricResolver {
     private static final Logger log = LoggerFactory.getLogger(TestMetricResolver.class);
 
     public TestMetricResolver(CuratorFramework curatorFramework, BalancerProperties balancerProperties, CacheProperties cacheProperties,
-                    HealthChecker healthChecker) {
+                    HealthChecker healthChecker) throws Exception {
         super(curatorFramework, balancerProperties, cacheProperties, healthChecker);
     }
 

client/pom.xml

@@ -4,13 +4,13 @@
   <parent>
     <groupId>timely</groupId>
     <artifactId>timely</artifactId>
-    <version>2.1.4</version>
+    <version>2.1.5</version>
   </parent>
   <artifactId>timely-client</artifactId>
   <properties>
     <!-- PEP 440 compliant version gets substituted into src/main/python/template/setup.py -->
     <!-- use .dev0 instead of -SNAPSHOT -->
-    <python.client.version>2.1.4</python.client.version>
+    <python.client.version>2.1.5</python.client.version>
   </properties>
   <dependencies>
     <dependency>

collectd-nsq-plugin/pom.xml

@@ -4,7 +4,7 @@
   <parent>
     <groupId>timely</groupId>
     <artifactId>timely</artifactId>
-    <version>2.1.4</version>
+    <version>2.1.5</version>
   </parent>
   <artifactId>collectd-nsq-plugin</artifactId>
   <dependencies>

collectd-plugin-parent/pom.xml

@@ -4,7 +4,7 @@
   <parent>
     <groupId>timely</groupId>
     <artifactId>timely</artifactId>
-    <version>2.1.4</version>
+    <version>2.1.5</version>
   </parent>
   <artifactId>collectd-plugin-parent</artifactId>
   <dependencies>

collectd-plugin-parent/src/main/java/timely/collectd/plugin/CollectDPluginParent.java

@@ -3,6 +3,7 @@ package timely.collectd.plugin;
 import java.io.OutputStream;
 import java.text.MessageFormat;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.Map;
 import java.util.TreeMap;
 import java.util.regex.Matcher;
@@ -25,6 +26,9 @@ public abstract class CollectDPluginParent {
     private static final String GAUGE = "GAUGE";
     private static final String DERIVE = "DERIVE";
     private static final String ABSOLUTE = "ABSOLUTE";
+    private static final Collection<String> HADOOP_CONTEXTS = Arrays.asList("NameNode", "DataNode", "JobManager", "NodeManager", "JobHistoryServer",
+                    "ResourceManager", "MRAppMaster");
+    private static final Pattern HADOOP_PATTERN_4_GROUPS = Pattern.compile("([\\w-_]+)\\.([\\w-_]+)\\.([\\w-_]+)\\.([\\w-_]+)");
     private static final Pattern STATSD_PATTERN_3_GROUPS = Pattern.compile("([\\w-_]+)\\.([\\w-_]+)\\.([\\w-_]+)");
     private static final Pattern STATSD_PATTERN_4_GROUPS = Pattern.compile("([\\w-_]+)\\.([\\w-_]+)\\.([\\w-_]+)\\.([\\w-_]+)");
     private static final Pattern STATSD_PATTERN_6_GROUPS = Pattern.compile("([\\w-_]+)\\.([\\w-_]+)\\.([\\w-_]+)\\.([\\w-_]+)\\.([\\w-_#]+)\\.([\\w-_]+)");
@@ -64,6 +68,8 @@ public abstract class CollectDPluginParent {
 
     protected void addTag(Map<String,String> m, String k, String v) {
         if (k != null && !k.isBlank() && v != null && !v.isBlank()) {
+            k = k.replaceAll("=", ":");
+            v = v.replaceAll("=", ":");
             m.put(k.trim(), v.trim());
         }
     }
@@ -71,8 +77,8 @@ public abstract class CollectDPluginParent {
     protected void addTag(Map<String,String> m, String kv) {
         if (kv != null && !kv.isBlank()) {
             String[] parts = kv.split("=");
-            if (parts.length == 2 && !parts[0].isBlank() && !parts[1].isBlank()) {
-                m.put(parts[0].trim(), parts[1].trim());
+            if (parts.length == 2) {
+                addTag(m, parts[0], parts[1]);
             }
         }
     }
@@ -99,9 +105,10 @@ public abstract class CollectDPluginParent {
         } else {
             addTag(tagMap, "host", host);
         }
-        int nIdx = host.lastIndexOf('n');
+        String[] hostSplit = host.split("\\.");
+        int nIdx = hostSplit[0].lastIndexOf('n');
         if (-1 != nIdx) {
-            addTag(tagMap, "rack", host.substring(0, nIdx));
+            addTag(tagMap, "rack", hostSplit[0].substring(0, nIdx));
         }
         tagMap.putAll(addlTags);
 
@@ -112,23 +119,53 @@ public abstract class CollectDPluginParent {
 
         switch (plugin) {
             case STATSD_PREFIX:
+                Matcher hadoop_4_groups = HADOOP_PATTERN_4_GROUPS.matcher(typeInstance);
                 Matcher statsd_3_groups = STATSD_PATTERN_3_GROUPS.matcher(typeInstance);
                 Matcher statsd_4_groups = STATSD_PATTERN_4_GROUPS.matcher(typeInstance);
                 Matcher statsd_6_groups = STATSD_PATTERN_6_GROUPS.matcher(typeInstance);
                 String instance = null;
                 if (!typeInstance.startsWith("nsq")) {
                     String[] parts = typeInstance.split("\\.");
-                    if (parts.length % 2 == 1) {
-                        if (parts[0].equals("dwquery") && parts.length >= 4) {
-                            metric.append(STATSD_PREFIX).append(PERIOD).append(parts[1]).append(PERIOD).append(parts[3]);
-                            addTag(tagMap, "queryId", parts[0]);
-                        } else {
-                            // EtsyStatsD format -- metric.(tagName.tagValue)*
-                            metric.append(STATSD_PREFIX).append(PERIOD).append(parts[0]);
-                            for (int x = 1; x < parts.length; x += 2) {
-                                addTag(tagMap, parts[x], parts[x + 1]);
+                    if (parts.length >= 4 && (HADOOP_CONTEXTS.contains(parts[0]) || hadoop_4_groups.matches())) {
+                        // Here we are processing the statsd metrics coming from the Hadoop Metrics2 StatsDSink without the host name.
+                        // The format of metric is: serviceName.contextName.recordName.metricName
+                        // The serviceName is used as the instance.
+                        instance = parts[0];
+                        String contextName = parts[1];
+                        int firstPeriod = typeInstance.indexOf(".");
+                        int lastPeriod = typeInstance.lastIndexOf(".");
+                        String metricName = typeInstance.substring(lastPeriod + 1);
+                        String recordName = typeInstance.substring(firstPeriod + 1, lastPeriod);
+                        // the recordName may contain useful tags such as op=<operation> and user<user>
+                        // in addition to a possibly duplicative recordName
+                        String[] recordNameSplits = recordName.split("\\.");
+                        metric.append(STATSD_PREFIX).append(PERIOD).append(contextName).append(PERIOD);
+                        for (String split : recordNameSplits) {
+                            // avoid duplication with either the contextName or instance
+                            if (!split.equalsIgnoreCase(contextName) && !split.equalsIgnoreCase(instance)) {
+                                if (split.startsWith("NNTopUserOpCounts") && split.contains("windowMs=")) {
+                                    metric.append("NNTopUserOpCounts").append(PERIOD);
+                                    addTag(tagMap, split.substring(split.indexOf("windowMs=")));
+                                } else if (split.contains("=")) {
+                                    addTag(tagMap, split);
+                                } else {
+                                    addTag(tagMap, "record", split);
+                                }
                             }
                         }
+                        metric.append(metricName);
+                    } else if (parts.length >= 2 && parts[1].equals("dwquery")) {
+                        // These come from Datawave's QueryIterator metrics - the first split part is the queryId
+                        // Use statsd plus the rest of the typeInstance after the first period as the metric name
+                        int x = typeInstance.indexOf(".");
+                        metric.append(STATSD_PREFIX).append(PERIOD).append(typeInstance.substring(x + 1));
+                        addTag(tagMap, "queryId", parts[0]);
+                    } else if (parts.length % 2 == 1) {
+                        // EtsyStatsD format -- metric.(tagName.tagValue)*
+                        metric.append(STATSD_PREFIX).append(PERIOD).append(parts[0]);
+                        for (int x = 1; x < parts.length; x += 2) {
+                            addTag(tagMap, parts[x], parts[x + 1]);
+                        }
                     } else if (statsd_4_groups.matches()) {
                         // Here we are processing the statsd metrics coming from the Hadoop Metrics2 StatsDSink without the host name.
                         // The format of metric is: serviceName.contextName.recordName.metricName. The recordName is typically duplicative
@@ -218,6 +255,12 @@ public abstract class CollectDPluginParent {
                 metric.append("sys.").append(plugin).append(PERIOD).append(type);
                 addTag(tagMap, INSTANCE, typeInstance.replaceAll(" ", "_"));
                 break;
+            case "load":
+                metric.append("sys.load.").append(plugin).append(PERIOD).append(type);
+                // plugin returns 1 min, 5 min, 15 min averages, but Timely can
+                // downsample on its own so we only use the first (1 min) value
+                vl.setValues(Arrays.asList(vl.getValues().get(0)));
+                break;
             case "GenericJMX":
                 metric.append("sys.").append(plugin).append(PERIOD).append(type).append(PERIOD).append(typeInstance);
                 String[] pluginInstanceSplit = pluginInstance.split("-");
@@ -227,6 +270,7 @@ public abstract class CollectDPluginParent {
                 if (pluginInstanceSplit.length > 1) {
                     addTag(tagMap, NAME, pluginInstanceSplit[1].replaceAll(" ", "_"));
                 }
+                break;
             default:
                 if (notEmpty(type) && notEmpty(typeInstance) && notEmpty(plugin) && notEmpty(pluginInstance)) {
                     metric.append("sys.").append(plugin).append(PERIOD).append(type).append(PERIOD).append(typeInstance);
@@ -253,9 +297,15 @@ public abstract class CollectDPluginParent {
             }
             Double value = vl.getValues().get(i).doubleValue();
             String tagString = tagMap.entrySet().stream().map(e -> e.getKey() + "=" + e.getValue()).collect(Collectors.joining(" "));
-            String datapoint = MessageFormat.format(PUT, metricName, timestamp.toString(), value.toString(), tagString);
-            logDebug(String.format("Writing: %s", datapoint));
-            write(datapoint + "\n", out);
+
+            if (metricName.isBlank() || timestamp == null || value == null) {
+                logDebug(String.format("Not writing unhandled metric: plugin:%s pluginInstance:%s type:%s typeInstance:%s", plugin, pluginInstance, type,
+                                typeInstance));
+            } else {
+                String datapoint = MessageFormat.format(PUT, metricName, timestamp.toString(), value.toString(), tagString);
+                logDebug(String.format("Writing: %s", datapoint));
+                write(datapoint + "\n", out);
+            }
         }
     }
 
@@ -306,5 +356,4 @@ public abstract class CollectDPluginParent {
     private boolean notEmpty(String arg) {
         return (null != arg) && !(arg.equals(""));
     }
-
 }

collectd-timely-plugin/pom.xml

@@ -4,7 +4,7 @@
   <parent>
     <groupId>timely</groupId>
     <artifactId>timely</artifactId>
-    <version>2.1.4</version>
+    <version>2.1.5</version>
   </parent>
   <artifactId>collectd-timely-plugin</artifactId>
   <dependencies>

common/pom.xml

@@ -4,7 +4,7 @@
   <parent>
     <groupId>timely</groupId>
     <artifactId>timely</artifactId>
-    <version>2.1.4</version>
+    <version>2.1.5</version>
   </parent>
   <artifactId>timely-common</artifactId>
   <dependencies>

grafana/pom.xml

@@ -4,7 +4,7 @@
   <parent>
     <groupId>timely</groupId>
     <artifactId>timely</artifactId>
-    <version>2.1.4</version>
+    <version>2.1.5</version>
   </parent>
   <artifactId>timely-app</artifactId>
   <packaging>pom</packaging>

grafana/timely-app/dist/plugin.json

@@ -28,8 +28,8 @@
       }
     ],
     "screenshots": [],
-    "version": "2.1.4",
-    "updated": "2024-03-29"
+    "version": "2.1.5",
+    "updated": "2024-04-10"
   },
   "dependencies": {
     "classnames": "2.2.6",

grafana/timely-app/package.json

@@ -1,6 +1,6 @@
 {
   "name": "timely-datasource",
-  "version": "2.1.4",
+  "version": "2.1.5",
   "description": "Timely",
   "backend": true,
   "alerting": true,

pom.xml

@@ -3,7 +3,7 @@
   <modelVersion>4.0.0</modelVersion>
   <groupId>timely</groupId>
   <artifactId>timely</artifactId>
-  <version>2.1.4</version>
+  <version>2.1.5</version>
   <packaging>pom</packaging>
   <name>Timely</name>
   <description>A secure and scalable time series database based on Apache Accumulo</description>
@@ -845,6 +845,45 @@
           <groupId>org.apache.maven.plugins</groupId>
           <artifactId>maven-dependency-plugin</artifactId>
           <version>${version.maven-dependency-plugin}</version>
+          <dependencies>
+            <dependency>
+              <groupId>org.apache.commons</groupId>
+              <artifactId>commons-lang3</artifactId>
+              <version>${version.commons-lang3}</version>
+            </dependency>
+            <dependency>
+              <groupId>org.apache.logging.log4j</groupId>
+              <artifactId>log4j-1.2-api</artifactId>
+              <version>${version.log4j2}</version>
+            </dependency>
+            <dependency>
+              <groupId>org.apache.logging.log4j</groupId>
+              <artifactId>log4j-core</artifactId>
+              <version>${version.log4j2}</version>
+            </dependency>
+            <dependency>
+              <groupId>org.apache.maven.doxia</groupId>
+              <artifactId>doxia-site-renderer</artifactId>
+              <version>1.4</version>
+              <exclusions>
+                <exclusion>
+                  <groupId>log4j</groupId>
+                  <artifactId>log4j</artifactId>
+                </exclusion>
+              </exclusions>
+            </dependency>
+            <dependency>
+              <groupId>org.apache.maven.reporting</groupId>
+              <artifactId>maven-reporting-impl</artifactId>
+              <version>3.0.0</version>
+              <exclusions>
+                <exclusion>
+                  <groupId>log4j</groupId>
+                  <artifactId>log4j</artifactId>
+                </exclusion>
+              </exclusions>
+            </dependency>
+          </dependencies>
         </plugin>
         <plugin>
           <groupId>org.apache.maven.plugins</groupId>
@@ -969,6 +1008,29 @@
             <threshold>Medium</threshold>
             <xmlOutput>true</xmlOutput>
           </configuration>
+          <dependencies>
+            <dependency>
+              <groupId>org.apache.logging.log4j</groupId>
+              <artifactId>log4j-1.2-api</artifactId>
+              <version>${version.log4j2}</version>
+            </dependency>
+            <dependency>
+              <groupId>org.apache.logging.log4j</groupId>
+              <artifactId>log4j-core</artifactId>
+              <version>${version.log4j2}</version>
+            </dependency>
+            <dependency>
+              <groupId>org.apache.maven.reporting</groupId>
+              <artifactId>maven-reporting-impl</artifactId>
+              <version>3.0.0</version>
+              <exclusions>
+                <exclusion>
+                  <groupId>log4j</groupId>
+                  <artifactId>log4j</artifactId>
+                </exclusion>
+              </exclusions>
+            </dependency>
+          </dependencies>
         </plugin>
         <plugin>
           <groupId>org.jacoco</groupId>

server/bin/timely-server.sh

@@ -1,17 +1,17 @@
 #!/bin/bash
 
 if [[ $(uname) == "Darwin" ]]; then
-        THIS_SCRIPT=$(python -c 'import os,sys; print os.path.realpath(sys.argv[1])' "$0")
+    THIS_SCRIPT=$(python -c 'import os,sys; print os.path.realpath(sys.argv[1])' "$0")
 else
-        THIS_SCRIPT=$(readlink -f "$0")
-        HW=`uname -m`
-        OS_PATH=linux-${HW}
+    THIS_SCRIPT=$(readlink -f "$0")
+    HW=`uname -m`
+    OS_PATH=linux-${HW}
 fi
 
 if [ -z "$1" ]; then
-  export INSTANCE="1"
+    export INSTANCE="1"
 else
-  export INSTANCE="$1"
+    export INSTANCE="$1"
 fi
 
 PID=$(jps -m | grep -E 'timely-server-.*-exec.jar timelyserver' | grep "instance=${INSTANCE}" | awk '{print $1}')
@@ -19,7 +19,6 @@ PID=$(jps -m | grep -E 'timely-server-.*-exec.jar timelyserver' | grep "instance
 if [ "$PID" == "" ]; then
     THIS_DIR="${THIS_SCRIPT%/*}"
     BASE_DIR=$(cd "$THIS_DIR"/.. && pwd)
-    TMP_DIR="${BASE_DIR}/tmp"
     LIB_DIR="${BASE_DIR}/lib"
     BIN_DIR="${BASE_DIR}/bin"
     export CONF_DIR="${BASE_DIR}/conf"
@@ -32,18 +31,12 @@ if [ "$PID" == "" ]; then
     # use either a value from timely-server-env.sh or the default
     export LOG_DIR="${TIMELY_LOG_DIR:-${BASE_DIR}/logs}"
 
-    if [[ -e ${TMP_DIR} ]]; then
-      rm -rf "${TMP_DIR}"
+    if [[ ! -e ${NATIVE_DIR} ]]; then
+        mkdir -p "${NATIVE_DIR}"
     fi
-    mkdir "${TMP_DIR}"
-
-    if [[ -e ${NATIVE_DIR} ]]; then
-      rm -rf "${NATIVE_DIR}"
-    fi
-    mkdir -p "${NATIVE_DIR}"
 
     if [[ ! -e ${LOG_DIR} ]]; then
-      mkdir "${LOG_DIR}"
+        mkdir "${LOG_DIR}"
     fi
 
     JVM_ARGS="-Xmx4G -Xms4G -XX:NewSize=1G -XX:MaxNewSize=1G"
@@ -54,11 +47,13 @@ if [ "$PID" == "" ]; then
     JVM_ARGS="${JVM_ARGS} -XX:+UseNUMA"
 
     # for linux varieties of OS, extract native libraries if not already present
-    if [[ -n "$OS_PATH" && (! -d ${NATIVE_DIR} || -z "$(ls -A ${NATIVE_DIR})") ]]; then
-        pushd "${BASE_DIR}/bin" || exit
-        "${JAVA_HOME}"/bin/jar xf "${LIB_DIR}"/netty-tcnative-boringssl-static-*.Final-"${OS_PATH}".jar ${NATIVE_DIR}/libnetty_tcnative_linux_"${HW}".so
-        "${JAVA_HOME}"/bin/jar xf "${LIB_DIR}"/netty-transport-native-epoll-*-"${OS_PATH}".jar ${NATIVE_DIR}/libnetty_transport_native_epoll_"${HW}".so
-        popd || exit
+    if [[ -n "$OS_PATH" ]]; then
+        if [[ ! -d ${NATIVE_DIR} || -z "$(ls -A ${NATIVE_DIR})" ]]; then
+            pushd "${BASE_DIR}/bin" || exit
+            "${JAVA_HOME}"/bin/jar xf "${LIB_DIR}"/netty-tcnative-boringssl-static-*.Final-"${OS_PATH}".jar META-INF/native/libnetty_tcnative_linux_"${HW}".so
+            "${JAVA_HOME}"/bin/jar xf "${LIB_DIR}"/netty-transport-native-epoll-*-"${OS_PATH}".jar META-INF/native/libnetty_transport_native_epoll_"${HW}".so
+            popd || exit
+        fi
         JVM_ARGS="${JVM_ARGS} -Djava.library.path=${NATIVE_DIR}"
     fi
 

server/bin/util/get-metrics.sh

@@ -1,36 +1,50 @@
 #!/bin/bash
 
-if [[ `uname` == "Darwin" ]]; then
-    THIS_SCRIPT=`python -c 'import os,sys; print os.path.realpath(sys.argv[1])' $0`
+if [[ $(uname) == "Darwin" ]]; then
+    THIS_SCRIPT=$(python -c 'import os,sys; print os.path.realpath(sys.argv[1])' "$0")
 else
-    THIS_SCRIPT=`readlink -f $0`
+    THIS_SCRIPT=$(readlink -f "$0")
     HW=`uname -m`
     OS_PATH=linux-${HW}
 fi
 
-export INSTANCE="1"
-
 THIS_DIR="${THIS_SCRIPT%/*}"
-BASE_DIR=$(cd $THIS_DIR/../.. && pwd)
+BASE_DIR=$(cd "$THIS_DIR"/.. && pwd)
 LIB_DIR="${BASE_DIR}/lib"
 BIN_DIR="${BASE_DIR}/bin"
 export CONF_DIR="${BASE_DIR}/conf"
-export LOG_DIR="${BASE_DIR}/logs"
+NATIVE_DIR="${BIN_DIR}/META-INF/native"
 
 set -a
-. ${BIN_DIR}/timely-server-env.sh
+. "${BIN_DIR}/timely-server-env.sh"
 set +a
 
+# use either a value from timely-server-env.sh or the default
+export LOG_DIR="${TIMELY_LOG_DIR:-${BASE_DIR}/logs}"
+
+if [[ ! -e ${NATIVE_DIR} ]]; then
+    mkdir -p "${NATIVE_DIR}"
+fi
+
+if [[ ! -e ${LOG_DIR} ]]; then
+    mkdir "${LOG_DIR}"
+fi
+
 JVM_ARGS="-Xmx256m -Xms128m"
 JVM_ARGS="${JVM_ARGS} -Dlogging.config=${CONF_DIR}/log4j2-console.yml"
 JVM_ARGS="${JVM_ARGS} -Dlog4j2.contextSelector=org.apache.logging.log4j.core.async.AsyncLoggerContextSelector"
+JVM_ARGS="${JVM_ARGS} -XX:+UseG1GC -XX:+UseStringDeduplication"
+JVM_ARGS="${JVM_ARGS} -Djava.net.preferIPv4Stack=true"
+JVM_ARGS="${JVM_ARGS} -XX:+UseNUMA"
 
 # for linux varieties of OS, extract native libraries if not already present
-if [[ -n "$OS_PATH" && (! -d ${NATIVE_DIR} || -z "$(ls -A ${NATIVE_DIR})") ]]; then
-    pushd "${BASE_DIR}/bin" || exit
-    "${JAVA_HOME}"/bin/jar xf "${LIB_DIR}"/netty-tcnative-boringssl-static-*.Final-"${OS_PATH}".jar ${NATIVE_DIR}/libnetty_tcnative_linux_"${HW}".so
-    "${JAVA_HOME}"/bin/jar xf "${LIB_DIR}"/netty-transport-native-epoll-*-"${OS_PATH}".jar ${NATIVE_DIR}/libnetty_transport_native_epoll_"${HW}".so
-    popd || exit
+if [[ -n "$OS_PATH" ]]; then
+    if [[ ! -d ${NATIVE_DIR} || -z "$(ls -A ${NATIVE_DIR})" ]]; then
+        pushd "${BASE_DIR}/bin" || exit
+        "${JAVA_HOME}"/bin/jar xf "${LIB_DIR}"/netty-tcnative-boringssl-static-*.Final-"${OS_PATH}".jar META-INF/native/libnetty_tcnative_linux_"${HW}".so
+        "${JAVA_HOME}"/bin/jar xf "${LIB_DIR}"/netty-transport-native-epoll-*-"${OS_PATH}".jar META-INF/native/libnetty_transport_native_epoll_"${HW}".so
+        popd || exit
+    fi
     JVM_ARGS="${JVM_ARGS} -Djava.library.path=${NATIVE_DIR}"
 fi
 

server/bin/util/get-tablet-summary.sh

@@ -1,38 +1,52 @@
 #!/bin/bash
 
-if [[ `uname` == "Darwin" ]]; then
-    THIS_SCRIPT=`python -c 'import os,sys; print os.path.realpath(sys.argv[1])' $0`
+if [[ $(uname) == "Darwin" ]]; then
+    THIS_SCRIPT=$(python -c 'import os,sys; print os.path.realpath(sys.argv[1])' "$0")
 else
-    THIS_SCRIPT=`readlink -f $0`
+    THIS_SCRIPT=$(readlink -f "$0")
     HW=`uname -m`
     OS_PATH=linux-${HW}
 fi
 
-export INSTANCE="1"
-
 THIS_DIR="${THIS_SCRIPT%/*}"
-BASE_DIR=$(cd $THIS_DIR/../.. && pwd)
+BASE_DIR=$(cd "$THIS_DIR"/.. && pwd)
 LIB_DIR="${BASE_DIR}/lib"
 BIN_DIR="${BASE_DIR}/bin"
 export CONF_DIR="${BASE_DIR}/conf"
-export LOG_DIR="${BASE_DIR}/logs"
+NATIVE_DIR="${BIN_DIR}/META-INF/native"
 
 set -a
-. ${BIN_DIR}/timely-server-env.sh
+. "${BIN_DIR}/timely-server-env.sh"
 set +a
 
+# use either a value from timely-server-env.sh or the default
+export LOG_DIR="${TIMELY_LOG_DIR:-${BASE_DIR}/logs}"
+
+if [[ ! -e ${NATIVE_DIR} ]]; then
+    mkdir -p "${NATIVE_DIR}"
+fi
+
+if [[ ! -e ${LOG_DIR} ]]; then
+    mkdir "${LOG_DIR}"
+fi
+
 JVM_ARGS="-Xmx1024m -Xms128m"
 JVM_ARGS="${JVM_ARGS} -Dlogging.config=${CONF_DIR}/log4j2-console.yml"
 JVM_ARGS="${JVM_ARGS} -Dlog4j2.contextSelector=org.apache.logging.log4j.core.async.AsyncLoggerContextSelector"
+JVM_ARGS="${JVM_ARGS} -XX:+UseG1GC -XX:+UseStringDeduplication"
+JVM_ARGS="${JVM_ARGS} -Djava.net.preferIPv4Stack=true"
+JVM_ARGS="${JVM_ARGS} -XX:+UseNUMA"
 
 # for linux varieties of OS, extract native libraries if not already present
-if [[ -n "$OS_PATH" && (! -d ${NATIVE_DIR} || -z "$(ls -A ${NATIVE_DIR})") ]]; then
-    pushd "${BASE_DIR}/bin" || exit
-    "${JAVA_HOME}"/bin/jar xf "${LIB_DIR}"/netty-tcnative-boringssl-static-*.Final-"${OS_PATH}".jar ${NATIVE_DIR}/libnetty_tcnative_linux_"${HW}".so
-    "${JAVA_HOME}"/bin/jar xf "${LIB_DIR}"/netty-transport-native-epoll-*-"${OS_PATH}".jar ${NATIVE_DIR}/libnetty_transport_native_epoll_"${HW}".so
-    popd || exit
+if [[ -n "$OS_PATH" ]]; then
+    if [[ ! -d ${NATIVE_DIR} || -z "$(ls -A ${NATIVE_DIR})" ]]; then
+        pushd "${BASE_DIR}/bin" || exit
+        "${JAVA_HOME}"/bin/jar xf "${LIB_DIR}"/netty-tcnative-boringssl-static-*.Final-"${OS_PATH}".jar META-INF/native/libnetty_tcnative_linux_"${HW}".so
+        "${JAVA_HOME}"/bin/jar xf "${LIB_DIR}"/netty-transport-native-epoll-*-"${OS_PATH}".jar META-INF/native/libnetty_transport_native_epoll_"${HW}".so
+        popd || exit
+    fi
     JVM_ARGS="${JVM_ARGS} -Djava.library.path=${NATIVE_DIR}"
 fi
 
-echo "$JAVA_HOME/bin/java ${JVM_ARGS} -jar ${BIN_DIR}/timely-server*-exec.jar tablet --spring.profiles.active=timely,tablet"
-$JAVA_HOME/bin/java ${JVM_ARGS} -jar ${BIN_DIR}/timely-server-*-exec.jar tablet --spring.profiles.active=timely,tablet
+echo "${JAVA_HOME}/bin/java ${JVM_ARGS} -jar ${BIN_DIR}/timely-server*-exec.jar tablet --spring.profiles.active=timely,tablet"
+"${JAVA_HOME}"/bin/java ${JVM_ARGS} -jar "${BIN_DIR}"/timely-server-*-exec.jar tablet --spring.profiles.active=timely,tablet

server/pom.xml

@@ -4,7 +4,7 @@
   <parent>
     <groupId>timely</groupId>
     <artifactId>timely</artifactId>
-    <version>2.1.4</version>
+    <version>2.1.5</version>
   </parent>
   <artifactId>timely-server</artifactId>
   <description>Timely Server</description>

test/pom.xml

@@ -4,7 +4,7 @@
   <parent>
     <groupId>timely</groupId>
     <artifactId>timely</artifactId>
-    <version>2.1.4</version>
+    <version>2.1.5</version>
   </parent>
   <artifactId>timely-test</artifactId>
   <properties>