Refactor BalancedMetricResolver to allow HDFS and FILE backing, add callbacks for major events, begin working on tests

Refactor BalancedMetricResolver, allow FILE or HDFS persister, add callbacks for events, begin working on testing

Fixes

analytics/pom.xml

@@ -4,7 +4,7 @@
   <parent>
     <groupId>timely</groupId>
     <artifactId>timely</artifactId>
-    <version>1.1.1-SNAPSHOT</version>
+    <version>1.2.8-SNAPSHOT</version>
   </parent>
   <artifactId>timely-analytics</artifactId>
   <!--
@@ -68,6 +68,7 @@
     <pluginManagement>
       <plugins>
         <plugin>
+          <groupId>org.apache.maven.plugins</groupId>
           <artifactId>maven-compiler-plugin</artifactId>
           <configuration>
             <compilerId>jdt</compilerId>
@@ -76,7 +77,7 @@
             <dependency>
               <groupId>org.eclipse.tycho</groupId>
               <artifactId>tycho-compiler-jdt</artifactId>
-              <version>0.21.0</version>
+              <version>1.4.0</version>
             </dependency>
           </dependencies>
         </plugin>

analytics/src/main/java/timely/analytics/flink/SummarizationJob.java

@@ -67,7 +67,7 @@ public class SummarizationJob {
         })
         .name("Metric Windows")
         .addSink(
-    		new SocketClientSink<MetricHistogram>(jp.getTimelyHostname(), jp.getTimelyTcpPort(),
+            new SocketClientSink<MetricHistogram>(jp.getTimelyHostname(), jp.getTimelyTcpPort(),
                     new MetricHistogram(), 2, true) {
 
                 private final Logger LOG = LoggerFactory

balancer/bin/timely-balancer.sh

@@ -1,4 +1,4 @@
-#!/bin/bash -vx
+#!/bin/bash
 
 if [[ `uname` == "Darwin" ]]; then
         THIS_SCRIPT=`python -c 'import os,sys; print os.path.realpath(sys.argv[1])' $0`
@@ -27,15 +27,14 @@ fi
 mkdir -p ${NATIVE_DIR}
 
 pushd ${BASE_DIR}/bin
-$JAVA_HOME/bin/jar xf ${LIB_DIR}/netty-tcnative*.jar META-INF/native/libnetty_tcnative.${TCNATIVE_SUFFIX}
-$JAVA_HOME/bin/jar xf ${LIB_DIR}/netty-transport-native-epoll*.jar META-INF/native/libnetty_transport_native_epoll_x86_64.${TCNATIVE_SUFFIX}
+$JAVA_HOME/bin/jar xf ${LIB_DIR}/netty-tcnative-boringssl-static*.jar META-INF/native/libnetty_tcnative_linux_x86_64.${TCNATIVE_SUFFIX}
+$JAVA_HOME/bin/jar xf ${LIB_DIR}/netty-all*.jar META-INF/native/libnetty_transport_native_epoll_x86_64.${TCNATIVE_SUFFIX}
 popd
 
 export CLASSPATH="${CONF_DIR}:${LIB_DIR}/*:${HADOOP_CONF_DIR}"
-JVM_ARGS="-Xmx1G -Xms1G -Dio.netty.eventLoopThreads=${NUM_SERVER_THREADS}"
+JVM_ARGS="-Xmx1G -Xms1G -Dio.netty.eventLoopThreads=${NUM_SERVER_THREADS} -Dlog4j.configurationFile=${CONF_DIR}/log4j2-spring.xml"
 JVM_ARGS="${JVM_ARGS} -DLog4jContextSelector=org.apache.logging.log4j.core.async.AsyncLoggerContextSelector"
 JVM_ARGS="${JVM_ARGS} -Djava.library.path=${NATIVE_DIR}"
-JVM_ARGS="${JVM_ARGS} -Djava.library.path=${NATIVE_DIR}/libnetty-${TCNATIVE_SUFFIX}"
 
 #JVM_ARGS="${JVM_ARGS} -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5005"
 

balancer/conf/security.xml

@@ -11,21 +11,33 @@
 		          
 	<beans:bean id="x509PrincipalExtractor" class="org.springframework.security.web.authentication.preauth.x509.SubjectDnX509PrincipalExtractor">
 		<beans:property name="subjectDnRegex" value="CN=(.*?)(?:,|$)"/>
-	</beans:bean>	
+	</beans:bean>
 
-    <beans:bean id="timelyUserDetailsService" class="timely.auth.UserDetailsService">
-    	<beans:property name="users">
-    		<util:map key-type="java.lang.String">
-    			<beans:entry key="CN=example.com">
-   					<util:list>
-   						<beans:value>D</beans:value>
-   						<beans:value>E</beans:value>
-   						<beans:value>F</beans:value>
-   					</util:list>
-    			</beans:entry>
-    		</util:map>
-    	</beans:property>
-    </beans:bean>
+	<beans:bean id="timelyUserDetailsService" class="timely.auth.UserDetailsService">
+		<beans:property name="users">
+			<util:list>
+				<beans:bean class="timely.auth.TimelyUser">
+					<beans:constructor-arg name="subjectDn">
+						<beans:value>CN=example.com</beans:value>
+					</beans:constructor-arg>
+					<beans:constructor-arg name="auths">
+						<util:list>
+							<beans:value>D</beans:value>
+							<beans:value>E</beans:value>
+							<beans:value>F</beans:value>
+						</util:list>
+					</beans:constructor-arg>
+					<beans:constructor-arg name="roles">
+						<util:list>
+							<beans:value>G</beans:value>
+							<beans:value>H</beans:value>
+							<beans:value>I</beans:value>
+						</util:list>
+					</beans:constructor-arg>
+				</beans:bean>
+			</util:list>
+		</beans:property>
+	</beans:bean>
     
     <beans:bean id="x509AuthorizationProvider" class="org.springframework.security.web.authentication.preauth.PreAuthenticatedAuthenticationProvider">
     	<beans:property name="preAuthenticatedUserDetailsService" ref="timelyUserDetailsService" />
@@ -34,9 +46,12 @@
     <user-service id="usernamePasswordService">
     	<user name="test" password="test1" authorities="A,B,C" />
     </user-service>
-    
-    <beans:bean id="UserNamePasswordAuthenticationProvider" class="org.springframework.security.authentication.dao.DaoAuthenticationProvider">
+
+	<beans:bean id="noOpPasswordEncoder" class="org.springframework.security.crypto.password.NoOpPasswordEncoder"/>
+
+	<beans:bean id="UserNamePasswordAuthenticationProvider" class="org.springframework.security.authentication.dao.DaoAuthenticationProvider">
 		<beans:property name="userDetailsService" ref="usernamePasswordService"/>
+		<beans:property name="passwordEncoder" ref="noOpPasswordEncoder"/>
 	</beans:bean>
 	    
 	<authentication-manager alias="authenticationManager">
@@ -45,5 +60,12 @@
 		<!-- Basic authorization provider, validates provided username and password -->
 		<authentication-provider ref="UserNamePasswordAuthenticationProvider" />
 	</authentication-manager>
-	
+
+	<util:list id="requiredRoles" value-type="java.lang.String">
+		<beans:value></beans:value>
+	</util:list>
+
+	<util:list id="requiredAuths" value-type="java.lang.String">
+		<beans:value></beans:value>
+	</util:list>
 </beans:beans>

balancer/conf/timely-balancer.yml

@@ -8,6 +8,8 @@ timely-balancer:
   assignmentFile: /timely/balancer/assignments.csv
   security:
     session-max-age: 86400
+    cache-expiration-minutes: 60
+    cache-refresh-minutes: 5
     allow-anonymous-http-access: true
     allow-anonymous-ws-access: true
     serverSsl:
@@ -109,6 +111,7 @@ timely-balancer:
     flush-interval-seconds: 30
     scanner-read-ahead: 1
     subscription-batch-size: 5000
+    incomingBufferSize: 50000000
     wsClientPool:
       minIdlePerKey: 0
       maxIdlePerKey: 40

balancer/pom.xml

@@ -4,7 +4,7 @@
   <parent>
     <groupId>timely</groupId>
     <artifactId>timely</artifactId>
-    <version>1.1.1-SNAPSHOT</version>
+    <version>1.2.8-SNAPSHOT</version>
   </parent>
   <artifactId>timely-balancer</artifactId>
   <dependencies>
@@ -108,6 +108,10 @@
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-minicluster</artifactId>
     </dependency>
+    <dependency>
+      <groupId>org.glassfish.tyrus</groupId>
+      <artifactId>tyrus-client</artifactId>
+    </dependency>
     <dependency>
       <groupId>org.slf4j</groupId>
       <artifactId>slf4j-api</artifactId>
@@ -161,9 +165,7 @@
     </dependency>
     <dependency>
       <groupId>io.netty</groupId>
-      <artifactId>netty-tcnative</artifactId>
-      <version>${version.netty-tcnative}</version>
-      <classifier>${os.detected.classifier}</classifier>
+      <artifactId>netty-tcnative-boringssl-static</artifactId>
       <scope>runtime</scope>
     </dependency>
     <dependency>
@@ -247,9 +249,8 @@
               <artifactItems>
                 <artifactItem>
                   <groupId>io.netty</groupId>
-                  <artifactId>netty-tcnative</artifactId>
+                  <artifactId>netty-tcnative-boringssl-static</artifactId>
                   <version>${version.netty-tcnative}</version>
-                  <classifier>${os.detected.classifier}</classifier>
                   <type>jar</type>
                   <overWrite>false</overWrite>
                   <outputDirectory>${project.build.directory}/native</outputDirectory>
@@ -291,37 +292,4 @@
       </extension>
     </extensions>
   </build>
-  <profiles>
-    <profile>
-      <id>native-epoll-linux</id>
-      <activation>
-        <os>
-          <family>!mac</family>
-        </os>
-      </activation>
-      <dependencies>
-        <dependency>
-          <groupId>io.netty</groupId>
-          <artifactId>netty-transport-native-epoll</artifactId>
-          <version>${version.netty}</version>
-          <classifier>linux-x86_64</classifier>
-        </dependency>
-      </dependencies>
-    </profile>
-    <profile>
-      <id>native-epoll</id>
-      <activation>
-        <os>
-          <family>mac</family>
-        </os>
-      </activation>
-      <dependencies>
-        <dependency>
-          <groupId>io.netty</groupId>
-          <artifactId>netty-transport-native-epoll</artifactId>
-          <version>${version.netty}</version>
-        </dependency>
-      </dependencies>
-    </profile>
-  </profiles>
 </project>

balancer/src/main/java/timely/balancer/ArrivalRate.java

@@ -1,7 +1,7 @@
 package timely.balancer;
 
-import java.util.Timer;
-import java.util.TimerTask;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
 
 import com.google.common.util.concurrent.AtomicDouble;
@@ -20,38 +20,26 @@ public class ArrivalRate {
     private AtomicDouble longTermRate = new AtomicDouble(0);
     private AtomicLong longTermLastReset = new AtomicLong(0);
 
-    public ArrivalRate(Timer timer) {
+    public ArrivalRate(ScheduledExecutorService arrivalRateExecutor) {
         long now = System.currentTimeMillis();
         shortTermLastReset.set(now);
         mediumTermLastReset.set(now);
         longTermLastReset.set(now);
 
         // 2 minute
-        timer.schedule(new TimerTask() {
-
-            @Override
-            public void run() {
-                resetShort();
-            }
-        }, 120000, 120000);
+        arrivalRateExecutor.scheduleAtFixedRate(() -> {
+            resetShort();
+        }, 2, 2, TimeUnit.MINUTES);
 
         // 5 minute
-        timer.schedule(new TimerTask() {
-
-            @Override
-            public void run() {
-                resetMedium();
-            }
-        }, 300000, 300000);
+        arrivalRateExecutor.scheduleAtFixedRate(() -> {
+            resetMedium();
+        }, 5, 5, TimeUnit.MINUTES);
 
         // 15 minute
-        timer.schedule(new TimerTask() {
-
-            @Override
-            public void run() {
-                resetLong();
-            }
-        }, 900000, 900000);
+        arrivalRateExecutor.scheduleAtFixedRate(() -> {
+            resetLong();
+        }, 15, 15, TimeUnit.MINUTES);
     }
 
     private void resetShort() {

balancer/src/main/java/timely/balancer/Balancer.java

@@ -25,6 +25,7 @@ import io.netty.channel.ChannelHandler;
 import io.netty.channel.ChannelInitializer;
 import io.netty.channel.ChannelOption;
 import io.netty.channel.EventLoopGroup;
+import io.netty.channel.WriteBufferWaterMark;
 import io.netty.channel.epoll.EpollChannelOption;
 import io.netty.channel.epoll.EpollDatagramChannel;
 import io.netty.channel.epoll.EpollEventLoopGroup;
@@ -56,6 +57,7 @@ import io.netty.handler.stream.ChunkedWriteHandler;
 import io.netty.handler.timeout.IdleStateHandler;
 import io.netty.util.NettyRuntime;
 import io.netty.util.internal.SystemPropertyUtil;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
 import org.apache.curator.RetryPolicy;
 import org.apache.curator.framework.CuratorFramework;
 import org.apache.curator.framework.CuratorFrameworkFactory;
@@ -64,6 +66,7 @@ import org.apache.zookeeper.CreateMode;
 import org.apache.zookeeper.data.Stat;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.boot.WebApplicationType;
 import org.springframework.boot.builder.SpringApplicationBuilder;
 import org.springframework.context.ConfigurableApplicationContext;
 import timely.auth.AuthCache;
@@ -74,7 +77,7 @@ import timely.balancer.connection.http.HttpClientPool;
 import timely.balancer.connection.tcp.TcpClientPool;
 import timely.balancer.connection.udp.UdpClientPool;
 import timely.balancer.connection.ws.WsClientPool;
-import timely.balancer.healthcheck.HealthChecker;
+import timely.balancer.healthcheck.TcpHealthChecker;
 import timely.balancer.netty.http.HttpRelayHandler;
 import timely.balancer.netty.tcp.TcpRelayHandler;
 import timely.balancer.netty.udp.UdpRelayHandler;
@@ -142,8 +145,11 @@ public class Balancer {
         this.balancerConfig = balancerConf;
     }
 
-    protected static ConfigurableApplicationContext initializeConfiguration(String[] args) {
-        return new SpringApplicationBuilder(SpringBootstrap.class).web(false).run(args);
+    protected static ConfigurableApplicationContext initializeContext(String[] args) {
+        SpringApplicationBuilder builder = new SpringApplicationBuilder(SpringBootstrap.class);
+        builder.web(WebApplicationType.NONE);
+        builder.registerShutdownHook(false);
+        return builder.run(args);
     }
 
     private void shutdownHook() {
@@ -175,8 +181,9 @@ public class Balancer {
             channelFutures.add(wsChannelHandle.close());
         }
 
+        int udpChannel = 1;
         for (Channel c : udpChannelHandleList) {
-            LOG.info("Closing udpChannelHandle");
+            LOG.info("Closing udpChannelHandle #" + udpChannel++);
             channelFutures.add(c.close());
         }
 
@@ -185,14 +192,13 @@ public class Balancer {
             try {
                 f.get();
             } catch (final Exception e) {
-                LOG.error("Error while shutting down channel: {}", f.channel().config());
-                LOG.error("{}", e.getMessage());
-                e.printStackTrace();
+                LOG.error("Channel:" + f.channel().config() + " -> " + e.getMessage(), e);
             }
         });
 
-        Integer quietPeriod = balancerConfig.getServer().getShutdownQuietPeriod();
+        int quietPeriod = balancerConfig.getServer().getShutdownQuietPeriod();
         List<Future<?>> groupFutures = new ArrayList<>();
+
         if (tcpBossGroup != null) {
             LOG.info("Shutting down tcpBossGroup");
             groupFutures.add(tcpBossGroup.shutdownGracefully(quietPeriod, 10, TimeUnit.SECONDS));
@@ -237,9 +243,7 @@ public class Balancer {
             try {
                 f.get();
             } catch (final Exception e) {
-                LOG.error("Error while shutting down group: {}", f.toString());
-                LOG.error("{}", e.getMessage());
-                e.printStackTrace();
+                LOG.error("Group:" + f.toString() + " -> " + e.getMessage(), e);
             }
         });
 
@@ -250,14 +254,29 @@ public class Balancer {
             LOG.error("Error closing WebSocketRequestDecoder during shutdown", e);
         }
 
+        if (this.metricResolver != null) {
+            try {
+                LOG.info("Closing metricResolver");
+                this.metricResolver.stop();
+            } catch (Exception e) {
+                LOG.error(e.getMessage(), e);
+            }
+        }
+
+        if (curatorFramework != null) {
+            try {
+                LOG.info("Closing curatorFramework");
+                curatorFramework.close();
+            } catch (Exception e) {
+                LOG.error(e.getMessage(), e);
+            }
+        }
+
         if (applicationContext != null) {
             LOG.info("Closing applicationContext");
             applicationContext.close();
         }
 
-        if (curatorFramework != null) {
-            curatorFramework.close();
-        }
         this.shutdown = true;
         LOG.info("Server shut down.");
     }
@@ -358,12 +377,14 @@ public class Balancer {
 
         // start curator framework
         RetryPolicy retryPolicy = new RetryForever(1000);
-        curatorFramework = CuratorFrameworkFactory.newClient(balancerConfig.getZooKeeper().getServers(), 30000, 1000,
+        int timeout = Long.valueOf(AccumuloConfiguration.getTimeInMillis(balancerConfig.getZooKeeper().getTimeout()))
+                .intValue();
+        curatorFramework = CuratorFrameworkFactory.newClient(balancerConfig.getZooKeeper().getServers(), timeout, 10000,
                 retryPolicy);
         curatorFramework.start();
         ensureZkPaths(curatorFramework, zkPaths);
 
-        AuthCache.setSessionMaxAge(balancerConfig.getSecurity());
+        AuthCache.configure(balancerConfig.getSecurity());
         final boolean useEpoll = useEpoll();
         Class<? extends ServerSocketChannel> channelClass;
         Class<? extends Channel> datagramChannelClass;
@@ -396,8 +417,9 @@ public class Balancer {
         for (int x = 0; x < balancerConfig.getServer().getNumTcpPools(); x++) {
             tcpClientPools.add(new TcpClientPool(this.balancerConfig));
         }
-        HealthChecker healthChecker = new HealthChecker(this.balancerConfig, tcpClientPools.get(0));
+        TcpHealthChecker healthChecker = new TcpHealthChecker(this.balancerConfig, tcpClientPools.get(0));
         this.metricResolver = new BalancedMetricResolver(curatorFramework, this.balancerConfig, healthChecker);
+        this.metricResolver.start();
 
         final ServerBootstrap tcpServer = new ServerBootstrap();
         tcpServer.group(tcpBossGroup, tcpWorkerGroup);
@@ -429,7 +451,8 @@ public class Balancer {
         httpServer.group(httpBossGroup, httpWorkerGroup);
         httpServer.channel(channelClass);
         httpServer.handler(new LoggingHandler());
-        HttpClientPool httpClientPool = new HttpClientPool(balancerConfig, clientSSLContext);
+        HttpClientPool httpClientPool = new HttpClientPool(balancerConfig.getSecurity(), balancerConfig.getHttp(),
+                clientSSLContext);
         httpServer.childHandler(setupHttpChannel(balancerConfig, sslCtx, metricResolver, httpClientPool));
         httpServer.option(ChannelOption.ALLOCATOR, PooledByteBufAllocator.DEFAULT);
         httpServer.option(ChannelOption.SO_BACKLOG, 128);
@@ -448,10 +471,9 @@ public class Balancer {
         wsServer.option(ChannelOption.ALLOCATOR, PooledByteBufAllocator.DEFAULT);
         wsServer.option(ChannelOption.SO_BACKLOG, 128);
         wsServer.option(ChannelOption.SO_KEEPALIVE, true);
-        /* Not sure if next three lines are necessary */
+        /* Not sure if next two lines are necessary */
         wsServer.option(ChannelOption.SO_SNDBUF, 1048576);
-        wsServer.option(ChannelOption.WRITE_BUFFER_HIGH_WATER_MARK, 838860);
-        wsServer.option(ChannelOption.WRITE_BUFFER_LOW_WATER_MARK, 620145);
+        wsServer.option(ChannelOption.WRITE_BUFFER_WATER_MARK, new WriteBufferWaterMark(620145, 838860));
         wsChannelHandle = wsServer.bind(wsIp, wsPort).sync().channel();
         final String wsAddress = ((InetSocketAddress) wsChannelHandle.localAddress()).getAddress().getHostAddress();
 
@@ -476,12 +498,12 @@ public class Balancer {
 
     public static void main(String[] args) throws Exception {
 
-        Balancer.applicationContext = Balancer.initializeConfiguration(args);
-        BalancerConfiguration balancerConf = applicationContext.getBean(BalancerConfiguration.class);
+        Balancer.applicationContext = Balancer.initializeContext(args);
+        BalancerConfiguration balancerConf = Balancer.applicationContext.getBean(BalancerConfiguration.class);
 
-        Balancer b = new Balancer(balancerConf);
+        Balancer balancer = new Balancer(balancerConf);
         try {
-            b.run();
+            balancer.run();
             LATCH.await();
         } catch (final InterruptedException e) {
             LOG.info("Server shutting down.");
@@ -489,7 +511,7 @@ public class Balancer {
             LOG.error("Error running server.", e);
         } finally {
             try {
-                b.shutdown();
+                balancer.shutdown();
             } catch (Exception e) {
                 System.exit(1);
             }
@@ -509,16 +531,17 @@ public class Balancer {
                 ch.pipeline().addLast("decoder", new HttpRequestDecoder());
                 ch.pipeline().addLast("compressor", new HttpContentCompressor());
                 ch.pipeline().addLast("decompressor", new HttpContentDecompressor());
-                ch.pipeline().addLast("aggregator", new HttpObjectAggregator(8192));
+                ch.pipeline().addLast("aggregator", new HttpObjectAggregator(65536));
                 ch.pipeline().addLast("chunker", new ChunkedWriteHandler());
                 ch.pipeline().addLast("queryDecoder", new timely.netty.http.HttpRequestDecoder(
                         balancerConfig.getSecurity(), balancerConfig.getHttp()));
-                ch.pipeline().addLast("fileServer", new HttpStaticFileServerHandler());
+                ch.pipeline().addLast("fileServer", new HttpStaticFileServerHandler().setIgnoreSslHandshakeErrors(
+                        balancerConfig.getSecurity().getServerSsl().isUseGeneratedKeypair()));
                 ch.pipeline().addLast("login",
                         new X509LoginRequestHandler(balancerConfig.getSecurity(), balancerConfig.getHttp()));
-                ch.pipeline().addLast("httpRelay",
-                        new HttpRelayHandler(balancerConfig, metricResolver, httpClientPool));
-                ch.pipeline().addLast("error", new TimelyExceptionHandler());
+                ch.pipeline().addLast("httpRelay", new HttpRelayHandler(metricResolver, httpClientPool));
+                ch.pipeline().addLast("error", new TimelyExceptionHandler().setIgnoreSslHandshakeErrors(
+                        balancerConfig.getSecurity().getServerSsl().isUseGeneratedKeypair()));
             }
         };
     }
@@ -531,7 +554,7 @@ public class Balancer {
                 ch.pipeline().addLast("logger", new LoggingHandler());
                 ch.pipeline().addLast("packetDecoder", new UdpPacketToByteBuf());
                 ch.pipeline().addLast("buffer", new MetricsBufferDecoder());
-                ch.pipeline().addLast("frame", new DelimiterBasedFrameDecoder(8192, true, Delimiters.lineDelimiter()));
+                ch.pipeline().addLast("frame", new DelimiterBasedFrameDecoder(65536, true, Delimiters.lineDelimiter()));
                 ch.pipeline().addLast("putDecoder", new UdpDecoder());
                 ch.pipeline().addLast("udpRelayHandler", new UdpRelayHandler(metricResolver, udpClientPool));
             }
@@ -544,7 +567,7 @@ public class Balancer {
             @Override
             protected void initChannel(SocketChannel ch) throws Exception {
                 ch.pipeline().addLast("buffer", new MetricsBufferDecoder());
-                ch.pipeline().addLast("frame", new DelimiterBasedFrameDecoder(8192, true, Delimiters.lineDelimiter()));
+                ch.pipeline().addLast("frame", new DelimiterBasedFrameDecoder(65536, true, Delimiters.lineDelimiter()));
                 ch.pipeline().addLast("putDecoder", new TcpDecoder());
                 ch.pipeline().addLast("tcpRelayHandler", new TcpRelayHandler(metricResolver, tcpClientPools));
                 ch.pipeline().addLast("versionHandler", new TcpVersionHandler());
@@ -560,13 +583,13 @@ public class Balancer {
             protected void initChannel(SocketChannel ch) throws Exception {
                 ch.pipeline().addLast("ssl", sslCtx.newHandler(ch.alloc()));
                 ch.pipeline().addLast("httpServer", new HttpServerCodec());
-                ch.pipeline().addLast("aggregator", new HttpObjectAggregator(8192));
+                ch.pipeline().addLast("aggregator", new HttpObjectAggregator(65536));
                 ch.pipeline().addLast("sessionExtractor", new WebSocketFullRequestHandler());
 
                 ch.pipeline().addLast("idle-handler",
                         new IdleStateHandler(balancerConfig.getWebsocket().getTimeout(), 0, 0));
                 ch.pipeline().addLast("ws-protocol",
-                        new WebSocketServerProtocolHandler(WS_PATH, null, true, 65536, true));
+                        new WebSocketServerProtocolHandler(WS_PATH, null, true, 65536, false, true));
                 ch.pipeline().addLast("wsDecoder", new WebSocketRequestDecoder(balancerConfig.getSecurity()));
                 ch.pipeline().addLast("httpRelay", new WsRelayHandler(balancerConfig, metricResolver, wsClientPool));
                 ch.pipeline().addLast("error", new WSTimelyExceptionHandler());

balancer/src/main/java/timely/balancer/configuration/BalancerConfiguration.java

@@ -5,8 +5,10 @@ import javax.validation.Valid;
 import org.springframework.boot.context.properties.ConfigurationProperties;
 import org.springframework.boot.context.properties.NestedConfigurationProperty;
 import org.springframework.stereotype.Component;
+import org.springframework.validation.annotation.Validated;
 import timely.configuration.Cache;
 
+@Validated
 @Component
 @ConfigurationProperties(prefix = "timely-balancer")
 public class BalancerConfiguration {
@@ -47,6 +49,16 @@ public class BalancerConfiguration {
 
     private int serverSuccessesBeforeUp = 3;
 
+    private String metricAssignmentPersisterType = "HDFS";
+
+    private long balanceDelay = 900000;
+
+    private long balancePeriod = 900000;
+
+    private long persistDelay = 10000;
+
+    private long persistPeriod = 60000;
+
     public ZooKeeper getZooKeeper() {
         return zooKeeper;
     }
@@ -118,4 +130,44 @@ public class BalancerConfiguration {
     public void setServerSuccessesBeforeUp(int serverSuccessesBeforeUp) {
         this.serverSuccessesBeforeUp = serverSuccessesBeforeUp;
     }
+
+    public String getMetricAssignmentPersisterType() {
+        return metricAssignmentPersisterType;
+    }
+
+    public void setMetricAssignmentPersisterType(String metricAssignmentPersisterType) {
+        this.metricAssignmentPersisterType = metricAssignmentPersisterType;
+    }
+
+    public void setBalanceDelay(long balanceDelay) {
+        this.balanceDelay = balanceDelay;
+    }
+
+    public long getBalanceDelay() {
+        return balanceDelay;
+    }
+
+    public void setBalancePeriod(long balancePeriod) {
+        this.balancePeriod = balancePeriod;
+    }
+
+    public long getBalancePeriod() {
+        return balancePeriod;
+    }
+
+    public void setPersistDelay(long persistDelay) {
+        this.persistDelay = persistDelay;
+    }
+
+    public long getPersistDelay() {
+        return persistDelay;
+    }
+
+    public void setPersistPeriod(long persistPeriod) {
+        this.persistPeriod = persistPeriod;
+    }
+
+    public long getPersistPeriod() {
+        return persistPeriod;
+    }
 }

balancer/src/main/java/timely/balancer/configuration/BalancerHttp.java

@@ -3,6 +3,7 @@ package timely.balancer.configuration;
 import javax.validation.Valid;
 
 import org.apache.commons.pool2.impl.GenericKeyedObjectPoolConfig;
+import org.apache.http.impl.client.CloseableHttpClient;
 import org.springframework.boot.context.properties.NestedConfigurationProperty;
 import timely.configuration.Http;
 
@@ -10,9 +11,9 @@ public class BalancerHttp extends Http {
 
     @Valid
     @NestedConfigurationProperty
-    private GenericKeyedObjectPoolConfig httpClientPool = new GenericKeyedObjectPoolConfig();
+    private GenericKeyedObjectPoolConfig<CloseableHttpClient> httpClientPool = new GenericKeyedObjectPoolConfig<>();
 
-    public GenericKeyedObjectPoolConfig getHttpClientPool() {
+    public GenericKeyedObjectPoolConfig<CloseableHttpClient> getHttpClientPool() {
         return httpClientPool;
     }
 }

balancer/src/main/java/timely/balancer/configuration/BalancerServer.java

@@ -4,27 +4,29 @@ import javax.validation.Valid;
 
 import org.apache.commons.pool2.impl.GenericKeyedObjectPoolConfig;
 import org.springframework.boot.context.properties.NestedConfigurationProperty;
+import timely.client.tcp.TcpClient;
+import timely.client.udp.UdpClient;
 import timely.configuration.Server;
 
 public class BalancerServer extends Server {
 
     @Valid
     @NestedConfigurationProperty
-    private GenericKeyedObjectPoolConfig udpClientPool = new GenericKeyedObjectPoolConfig();
+    private GenericKeyedObjectPoolConfig<UdpClient> udpClientPool = new GenericKeyedObjectPoolConfig<>();
 
     @Valid
     @NestedConfigurationProperty
-    private GenericKeyedObjectPoolConfig tcpClientPool = new GenericKeyedObjectPoolConfig();
+    private GenericKeyedObjectPoolConfig<TcpClient> tcpClientPool = new GenericKeyedObjectPoolConfig<>();
 
     private int numTcpPools = 1;
 
     private int tcpBufferSize = -1;
 
-    public GenericKeyedObjectPoolConfig getUdpClientPool() {
+    public GenericKeyedObjectPoolConfig<UdpClient> getUdpClientPool() {
         return udpClientPool;
     }
 
-    public GenericKeyedObjectPoolConfig getTcpClientPool() {
+    public GenericKeyedObjectPoolConfig<TcpClient> getTcpClientPool() {
         return tcpClientPool;
     }
 

balancer/src/main/java/timely/balancer/configuration/BalancerWebsocket.java

@@ -4,15 +4,26 @@ import javax.validation.Valid;
 
 import org.apache.commons.pool2.impl.GenericKeyedObjectPoolConfig;
 import org.springframework.boot.context.properties.NestedConfigurationProperty;
+import timely.client.websocket.subscription.WebSocketSubscriptionClient;
 import timely.configuration.Websocket;
 
 public class BalancerWebsocket extends Websocket {
 
     @Valid
     @NestedConfigurationProperty
-    private GenericKeyedObjectPoolConfig wsClientPool = new GenericKeyedObjectPoolConfig();
+    private GenericKeyedObjectPoolConfig<WebSocketSubscriptionClient> wsClientPool = new GenericKeyedObjectPoolConfig<>();
 
-    public GenericKeyedObjectPoolConfig getWsClientPool() {
+    public GenericKeyedObjectPoolConfig<WebSocketSubscriptionClient> getWsClientPool() {
         return wsClientPool;
     }
+
+    public int incomingBufferSize = 50000000;
+
+    public int getIncomingBufferSize() {
+        return incomingBufferSize;
+    }
+
+    public void setIncomingBufferSize(int incomingBufferSize) {
+        this.incomingBufferSize = incomingBufferSize;
+    }
 }

balancer/src/main/java/timely/balancer/configuration/ClientSsl.java

@@ -8,7 +8,8 @@ public class ClientSsl {
     private String trustStoreFile;
     private String trustStoreType;
     private String trustStorePassword;
-    private boolean twoWaySsl;
+    private boolean useClientCert = true;
+    private boolean hostVerificationEnabled = true;
 
     public String getKeyFile() {
         return keyFile;
@@ -58,7 +59,19 @@ public class ClientSsl {
         this.trustStorePassword = trustStorePassword;
     }
 
-    public boolean isTwoWaySsl() {
-        return twoWaySsl;
+    public void setUseClientCert(boolean useClientCert) {
+        this.useClientCert = useClientCert;
+    }
+
+    public boolean isUseClientCert() {
+        return useClientCert;
+    }
+
+    public void setHostVerificationEnabled(boolean hostVerificationEnabled) {
+        this.hostVerificationEnabled = hostVerificationEnabled;
+    }
+
+    public boolean isHostVerificationEnabled() {
+        return hostVerificationEnabled;
     }
 }

balancer/src/main/java/timely/balancer/configuration/ZooKeeper.java

@@ -1,9 +1,14 @@
 package timely.balancer.configuration;
 
+import javax.validation.constraints.NotBlank;
+
 public class ZooKeeper {
 
     private String servers;
 
+    @NotBlank
+    private String timeout = "120s";
+
     public String getServers() {
         return servers;
     }
@@ -11,4 +16,12 @@ public class ZooKeeper {
     public void setServers(String servers) {
         this.servers = servers;
     }
+
+    public String getTimeout() {
+        return timeout;
+    }
+
+    public void setTimeout(String timeout) {
+        this.timeout = timeout;
+    }
 }

balancer/src/main/java/timely/balancer/connection/TimelyBalancedHost.java

@@ -1,7 +1,5 @@
 package timely.balancer.connection;
 
-import java.util.Timer;
-
 import org.apache.commons.lang3.builder.EqualsBuilder;
 import org.apache.commons.lang3.builder.HashCodeBuilder;
 import org.apache.commons.lang3.builder.ToStringBuilder;
@@ -13,7 +11,6 @@ import timely.balancer.configuration.BalancerConfiguration;
 public class TimelyBalancedHost {
 
     private static final Logger LOG = LoggerFactory.getLogger(TimelyBalancedHost.class);
-    static private Timer arrivalRateTimer = new Timer("ArrivalRateTimerTBH", true);
 
     private String host;
     private int tcpPort;
@@ -24,34 +21,29 @@ public class TimelyBalancedHost {
     private BalancerConfiguration balancerConfig;
     private int failures = 0;
     private int successes = 0;
-    private ArrivalRate arrivalRate = new ArrivalRate(arrivalRateTimer);
+    private ArrivalRate arrivalRate = null;
 
-    public TimelyBalancedHost() {
+    private TimelyBalancedHost() {
 
     }
 
-    public TimelyBalancedHost(String host, int tcpPort, int httpPort, int wsPort, int udpPort) {
+    private TimelyBalancedHost(String host, int tcpPort, int httpPort, int wsPort, int udpPort,
+            ArrivalRate arrivalRate) {
         this.host = host;
         this.tcpPort = tcpPort;
         this.httpPort = httpPort;
         this.wsPort = wsPort;
         this.udpPort = udpPort;
+        this.arrivalRate = arrivalRate;
     }
 
-    public TimelyBalancedHost(String host, int basePort) {
-        this.host = host;
-        this.tcpPort = basePort;
-        this.httpPort = basePort + 1;
-        this.wsPort = basePort + 2;
-        this.udpPort = basePort + 3;
-    }
-
-    static public TimelyBalancedHost of(String host, int tcpPort, int httpPort, int wsPort, int udpPort) {
-        return new TimelyBalancedHost(host, tcpPort, httpPort, wsPort, udpPort);
+    static public TimelyBalancedHost of(String host, int tcpPort, int httpPort, int wsPort, int udpPort,
+            ArrivalRate arrivalRate) {
+        return new TimelyBalancedHost(host, tcpPort, httpPort, wsPort, udpPort, arrivalRate);
     }
 
     static public TimelyBalancedHost of(String host, int basePort) {
-        return new TimelyBalancedHost(host, basePort);
+        return new TimelyBalancedHost(host, basePort, basePort + 1, basePort + 2, basePort + 3, null);
     }
 
     public void setBalancerConfig(BalancerConfiguration balancerConfig) {
@@ -141,11 +133,17 @@ public class TimelyBalancedHost {
     }
 
     public void arrived() {
-        arrivalRate.arrived();
+        if (this.arrivalRate != null) {
+            this.arrivalRate.arrived();
+        }
     }
 
     public double getArrivalRate() {
-        return arrivalRate.getRate();
+        if (this.arrivalRate == null) {
+            return 0;
+        } else {
+            return this.arrivalRate.getRate();
+        }
     }
 
     @Override
@@ -186,4 +184,11 @@ public class TimelyBalancedHost {
         builder.append(isUp);
         return builder.toString();
     }
+
+    public static String toStringShort(TimelyBalancedHost tbh) {
+        if (tbh == null) {
+            return null;
+        }
+        return tbh.host + ":" + tbh.tcpPort;
+    }
 }

balancer/src/main/java/timely/balancer/connection/http/HttpClientFactory.java

@@ -5,26 +5,25 @@ import javax.net.ssl.SSLContext;
 import org.apache.commons.pool2.KeyedPooledObjectFactory;
 import org.apache.commons.pool2.PooledObject;
 import org.apache.commons.pool2.impl.DefaultPooledObject;
-import org.apache.http.impl.client.BasicCookieStore;
 import org.apache.http.impl.client.CloseableHttpClient;
-import timely.balancer.configuration.BalancerConfiguration;
+import timely.balancer.configuration.BalancerSecurity;
 import timely.balancer.connection.TimelyBalancedHost;
 
 public class HttpClientFactory implements KeyedPooledObjectFactory<TimelyBalancedHost, CloseableHttpClient> {
 
     private final SSLContext sslContext;
-    private boolean twoWaySsl;
+    private final BalancerSecurity security;
 
-    public HttpClientFactory(BalancerConfiguration balancerConfig, SSLContext sslContext) {
+    public HttpClientFactory(BalancerSecurity security, SSLContext sslContext) {
+        this.security = security;
         this.sslContext = sslContext;
-        this.twoWaySsl = balancerConfig.getSecurity().getClientSsl().isTwoWaySsl();
     }
 
     @Override
     public PooledObject<CloseableHttpClient> makeObject(TimelyBalancedHost k) throws Exception {
-        BasicCookieStore cookieJar = new BasicCookieStore();
-        return new DefaultPooledObject<>(
-                timely.client.http.HttpClient.get(this.sslContext, cookieJar, false, twoWaySsl));
+        // disable cookie management because we are sharing connections
+        return new DefaultPooledObject<>(timely.client.http.HttpClient.get(this.sslContext, null,
+                security.getClientSsl().isHostVerificationEnabled(), security.getClientSsl().isUseClientCert()));
     }
 
     @Override

balancer/src/main/java/timely/balancer/connection/http/HttpClientPool.java

@@ -4,16 +4,13 @@ import javax.net.ssl.SSLContext;
 
 import org.apache.commons.pool2.impl.GenericKeyedObjectPool;
 import org.apache.http.impl.client.CloseableHttpClient;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import timely.balancer.configuration.BalancerConfiguration;
+import timely.balancer.configuration.BalancerHttp;
+import timely.balancer.configuration.BalancerSecurity;
 import timely.balancer.connection.TimelyBalancedHost;
 
 public class HttpClientPool extends GenericKeyedObjectPool<TimelyBalancedHost, CloseableHttpClient> {
 
-    private static final Logger LOG = LoggerFactory.getLogger(HttpClientPool.class);
-
-    public HttpClientPool(BalancerConfiguration balancerConfig, SSLContext sslContext) {
-        super(new HttpClientFactory(balancerConfig, sslContext), balancerConfig.getHttp().getHttpClientPool());
+    public HttpClientPool(BalancerSecurity balancerSecurity, BalancerHttp http, SSLContext sslContext) {
+        super(new HttpClientFactory(balancerSecurity, sslContext), http.getHttpClientPool());
     }
 }

balancer/src/main/java/timely/balancer/connection/ws/WsClientFactory.java

@@ -12,23 +12,21 @@ import timely.client.websocket.subscription.WebSocketSubscriptionClient;
 public class WsClientFactory implements KeyedPooledObjectFactory<TimelyBalancedHost, WebSocketSubscriptionClient> {
 
     private final SSLContext sslContext;
-    private final boolean doLogin;
-    private final boolean twoWaySsl;
     private BalancerConfiguration balancerConfig;
 
     public WsClientFactory(BalancerConfiguration balancerConfig, SSLContext sslContext) {
-        this.sslContext = sslContext;
-        this.doLogin = balancerConfig.isLoginRequired();
         this.balancerConfig = balancerConfig;
-        this.twoWaySsl = balancerConfig.getSecurity().getClientSsl().isTwoWaySsl();
+        this.sslContext = sslContext;
     }
 
     @Override
     public PooledObject<WebSocketSubscriptionClient> makeObject(TimelyBalancedHost k) throws Exception {
 
-        int bufferSize = balancerConfig.getWebsocket().getSubscriptionBatchSize() * 500;
+        int bufferSize = balancerConfig.getWebsocket().getIncomingBufferSize();
         WebSocketSubscriptionClient client = new WebSocketSubscriptionClient(sslContext, k.getHost(), k.getHttpPort(),
-                k.getWsPort(), twoWaySsl, doLogin, "", "", false, bufferSize);
+                k.getWsPort(), balancerConfig.getSecurity().getClientSsl().isUseClientCert(),
+                balancerConfig.isLoginRequired(), "", "",
+                balancerConfig.getSecurity().getClientSsl().isHostVerificationEnabled(), bufferSize);
         return new DefaultPooledObject<>(client);
     }
 

balancer/src/main/java/timely/balancer/healthcheck/HealthChecker.java

@@ -1,53 +1,12 @@
 package timely.balancer.healthcheck;
 
-import java.util.ArrayList;
 import java.util.List;
-import java.util.Timer;
-import java.util.TimerTask;
 
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import timely.balancer.configuration.BalancerConfiguration;
 import timely.balancer.connection.TimelyBalancedHost;
-import timely.balancer.connection.tcp.TcpClientPool;
 
-public class HealthChecker {
+public interface HealthChecker {
 
-    private static final Logger LOG = LoggerFactory.getLogger(HealthChecker.class);
-    private List<TimelyBalancedHost> timelyHosts = new ArrayList<>();
-    private Timer timer;
-    final private TCPServerHealthCheck tcpServerHealthCheck;
-    private TimerTask check = new TimerTask() {
+    public void setTimelyHosts(List<TimelyBalancedHost> timelyHosts);
 
-        @Override
-        public void run() {
-            synchronized (timelyHosts) {
-                for (TimelyBalancedHost h : timelyHosts) {
-                    try {
-                        if (tcpServerHealthCheck.isServerHealthy(h)) {
-                            h.reportSuccess();
-                        } else {
-                            h.reportFailure();
-                        }
-                    } catch (Exception e) {
-                        LOG.error(e.getMessage(), e);
-                        h.reportFailure();
-                    }
-                }
-            }
-        }
-    };
-
-    public HealthChecker(BalancerConfiguration config, TcpClientPool tcpClientPool) {
-        this.tcpServerHealthCheck = new TCPServerHealthCheck(tcpClientPool);
-        this.timer = new Timer("HealthCheckerThread", true);
-        this.timer.scheduleAtFixedRate(check, 0, config.getCheckServerHealthInterval());
-    }
-
-    public void setTimelyHosts(List<TimelyBalancedHost> timelyHosts) {
-        synchronized (timelyHosts) {
-            this.timelyHosts.clear();
-            this.timelyHosts.addAll(timelyHosts);
-        }
-    }
+    public void close() throws Exception;
 }

balancer/src/main/java/timely/balancer/healthcheck/TcpHealthChecker.java

@@ -0,0 +1,62 @@
+package timely.balancer.healthcheck;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import timely.balancer.configuration.BalancerConfiguration;
+import timely.balancer.connection.TimelyBalancedHost;
+import timely.balancer.connection.tcp.TcpClientPool;
+
+public class TcpHealthChecker implements HealthChecker {
+
+    private static final Logger LOG = LoggerFactory.getLogger(TcpHealthChecker.class);
+    private List<TimelyBalancedHost> timelyHosts = new ArrayList<>();
+    private ScheduledExecutorService executorService = Executors.newScheduledThreadPool(1);
+    final private TCPServerHealthCheck tcpServerHealthCheck;
+
+    public TcpHealthChecker(BalancerConfiguration config, TcpClientPool tcpClientPool) {
+        this.tcpServerHealthCheck = new TCPServerHealthCheck(tcpClientPool);
+        this.executorService.scheduleAtFixedRate(() -> {
+            synchronized (timelyHosts) {
+                for (TimelyBalancedHost h : timelyHosts) {
+                    try {
+                        if (tcpServerHealthCheck.isServerHealthy(h)) {
+                            h.reportSuccess();
+                        } else {
+                            h.reportFailure();
+                        }
+                    } catch (Exception e) {
+                        LOG.error(e.getMessage(), e);
+                        h.reportFailure();
+                    }
+                }
+            }
+        }, 0, config.getCheckServerHealthInterval(), TimeUnit.MILLISECONDS);
+    }
+
+    @Override
+    public void setTimelyHosts(List<TimelyBalancedHost> timelyHosts) {
+        synchronized (timelyHosts) {
+            this.timelyHosts.clear();
+            this.timelyHosts.addAll(timelyHosts);
+        }
+    }
+
+    public void close() throws Exception {
+        this.executorService.shutdown();
+        try {
+            this.executorService.awaitTermination(5, TimeUnit.SECONDS);
+        } catch (InterruptedException e) {
+
+        } finally {
+            if (!this.executorService.isTerminated()) {
+                this.executorService.shutdownNow();
+            }
+        }
+    }
+}

balancer/src/main/java/timely/balancer/netty/http/HttpRelayHandler.java

@@ -1,9 +1,8 @@
 package timely.balancer.netty.http;
 
-import static io.netty.handler.codec.http.HttpHeaders.Names.CONTENT_LENGTH;
+import static io.netty.handler.codec.http.HttpHeaderNames.CONTENT_LENGTH;
 
 import java.io.ByteArrayOutputStream;
-import java.net.URI;
 import java.net.URLEncoder;
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
@@ -41,7 +40,6 @@ import timely.api.request.timeseries.QueryRequest;
 import timely.api.response.TimelyException;
 import timely.auth.util.HttpHeaderUtils;
 import timely.auth.util.ProxiedEntityUtils;
-import timely.balancer.configuration.BalancerConfiguration;
 import timely.balancer.connection.TimelyBalancedHost;
 import timely.balancer.connection.http.HttpClientPool;
 import timely.balancer.resolver.MetricResolver;
@@ -51,13 +49,10 @@ import timely.netty.http.auth.TimelyAuthenticationToken;
 public class HttpRelayHandler extends SimpleChannelInboundHandler<HttpRequest> implements TimelyHttpHandler {
 
     private static final Logger LOG = LoggerFactory.getLogger(HttpRelayHandler.class);
-    private final BalancerConfiguration balancerConf;
     private final HttpClientPool httpClientPool;
     private MetricResolver metricResolver;
 
-    public HttpRelayHandler(BalancerConfiguration balancerConf, MetricResolver metricResolver,
-            HttpClientPool httpClientPool) {
-        this.balancerConf = balancerConf;
+    public HttpRelayHandler(MetricResolver metricResolver, HttpClientPool httpClientPool) {
         this.metricResolver = metricResolver;
         this.httpClientPool = httpClientPool;
     }
@@ -72,7 +67,7 @@ public class HttpRelayHandler extends SimpleChannelInboundHandler<HttpRequest> i
         try {
             try {
                 request = msg.getHttpRequest();
-                String originalURI = request.getUri();
+                String originalURI = request.uri();
                 originalURI = encodeURI(originalURI);
 
                 String metric = null;
@@ -91,7 +86,6 @@ public class HttpRelayHandler extends SimpleChannelInboundHandler<HttpRequest> i
                 } else {
                     k = metricResolver.getHostPortKey(null);
                 }
-                k = metricResolver.getHostPortKey(metric);
                 client = httpClientPool.borrowObject(k);
 
                 Multimap<String, String> headers = HttpHeaderUtils.toMultimap(request.headers());
@@ -107,7 +101,7 @@ public class HttpRelayHandler extends SimpleChannelInboundHandler<HttpRequest> i
 
                 List<Header> relayedHeaderList = new ArrayList<>();
                 for (Map.Entry<String, String> h : headers.entries()) {
-                    if (!h.getKey().equals(CONTENT_LENGTH)) {
+                    if (!h.getKey().equals(CONTENT_LENGTH.toString())) {
                         relayedHeaderList.add(new BasicHeader(h.getKey(), h.getValue()));
                     }
                 }
@@ -115,14 +109,14 @@ public class HttpRelayHandler extends SimpleChannelInboundHandler<HttpRequest> i
                 relayedHeaderList.toArray(headerArray);
 
                 String relayURI = "https://" + k.getHost() + ":" + k.getHttpPort() + originalURI;
-                if (request.getMethod().equals(HttpMethod.GET)) {
+                if (request.method().equals(HttpMethod.GET)) {
                     HttpUriRequest relayedRequest = new HttpGet(relayURI);
                     if (headerArray.length > 0) {
                         relayedRequest.setHeaders(headerArray);
                     }
                     relayedResponse = client.execute(relayedRequest);
 
-                } else if (request.getMethod().equals(HttpMethod.POST)) {
+                } else if (request.method().equals(HttpMethod.POST)) {
                     HttpPost relayedRequest = new HttpPost(relayURI);
 
                     String content = request.content().toString(StandardCharsets.UTF_8);
@@ -176,7 +170,7 @@ public class HttpRelayHandler extends SimpleChannelInboundHandler<HttpRequest> i
             if (client != null && k != null) {
                 httpClientPool.returnObject(k, client);
             } else {
-                LOG.error("NOT RETURNING CONNECTION! " + msg.getHttpRequest().getUri());
+                LOG.error("NOT RETURNING CONNECTION! " + msg.getHttpRequest().uri());
             }
         }
     }
@@ -208,13 +202,4 @@ public class HttpRelayHandler extends SimpleChannelInboundHandler<HttpRequest> i
         }
         return s;
     }
-
-    public static void main(String[] args) {
-
-        String s = "https://127.0.0.1:4243/api/query?start=1356998400000&end=1528398304000&m=sum:rate{false,100,0}:sys.thermal.gauge{class=aws}";
-        URI uri = URI.create(encodeURI(s));
-
-        System.out.println(uri.toString());
-    }
-
 }

balancer/src/main/java/timely/balancer/netty/ws/WsClientHandler.java

@@ -9,6 +9,7 @@ import javax.websocket.CloseReason;
 import javax.websocket.EndpointConfig;
 import javax.websocket.Session;
 
+import com.google.common.collect.HashMultimap;
 import com.google.common.collect.Multimap;
 import io.netty.channel.ChannelHandlerContext;
 import io.netty.handler.codec.http.HttpResponseStatus;
@@ -40,18 +41,20 @@ public class WsClientHandler extends ClientHandler {
 
     @Override
     public void beforeRequest(Map<String, List<String>> headers) {
-        Multimap<String, String> requestHeaders = token.getHttpHeaders();
         if (token.getClientCert() != null) {
-            ProxiedEntityUtils.addProxyHeaders(requestHeaders, token.getClientCert());
+            Multimap<String, String> proxyRequestHeaders = HashMultimap.create();
+            ProxiedEntityUtils.addProxyHeaders(proxyRequestHeaders, token.getClientCert());
+            for (String s : proxyRequestHeaders.keySet()) {
+                headers.put(s, new ArrayList<>(proxyRequestHeaders.get(s)));
+            }
         }
-        for (String s : requestHeaders.keySet()) {
-            List<String> valueList = new ArrayList<>();
-            valueList.addAll(requestHeaders.get(s));
-            if (headers.containsKey(s)) {
-                // add pre-existing values
-                valueList.addAll(headers.get(s));
+
+        Multimap<String, String> originalRequestHeaders = token.getHttpHeaders();
+        for (String s : originalRequestHeaders.keySet()) {
+            if (!headers.containsKey(s)) {
+                // add pre-existing values if key does not exist in proxyRequest
+                headers.put(s, new ArrayList<>(originalRequestHeaders.get(s)));
             }
-            headers.put(s, valueList);
         }
     }
 

balancer/src/main/java/timely/balancer/resolver/BalancedMetricResolver.java

@@ -4,10 +4,14 @@ import static timely.Server.SERVICE_DISCOVERY_PATH;
 import static timely.balancer.Balancer.ASSIGNMENTS_LAST_UPDATED_PATH;
 import static timely.balancer.Balancer.ASSIGNMENTS_LOCK_PATH;
 import static timely.balancer.Balancer.LEADER_LATCH_PATH;
+import static timely.balancer.resolver.eventing.MetricAssignedEvent.Reason.ASSIGN_FALLBACK_SEQUENTIAL;
+import static timely.balancer.resolver.eventing.MetricAssignedEvent.Reason.ASSIGN_ROUND_ROBIN;
+import static timely.balancer.resolver.eventing.MetricAssignedEvent.Reason.HOST_DOWN_ROUND_ROBIN;
+import static timely.balancer.resolver.eventing.MetricAssignedEvent.Reason.HOST_REMOVED;
+import static timely.balancer.resolver.eventing.MetricAssignedEvent.Reason.REBALANCE;
 import static timely.store.cache.DataStoreCache.NON_CACHED_METRICS;
 import static timely.store.cache.DataStoreCache.NON_CACHED_METRICS_LOCK_PATH;
 
-import java.nio.charset.Charset;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -19,21 +23,18 @@ import java.util.List;
 import java.util.Map;
 import java.util.Random;
 import java.util.Set;
-import java.util.Timer;
-import java.util.TimerTask;
 import java.util.TreeMap;
 import java.util.TreeSet;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 
-import com.csvreader.CsvReader;
-import com.csvreader.CsvWriter;
 import org.apache.commons.lang.StringUtils;
 import org.apache.commons.lang3.SerializationUtils;
 import org.apache.curator.framework.CuratorFramework;
-import org.apache.curator.framework.recipes.atomic.DistributedAtomicLong;
 import org.apache.curator.framework.recipes.atomic.DistributedAtomicValue;
 import org.apache.curator.framework.recipes.cache.TreeCache;
 import org.apache.curator.framework.recipes.cache.TreeCacheEvent;
@@ -48,11 +49,6 @@ import org.apache.curator.x.discovery.ServiceDiscovery;
 import org.apache.curator.x.discovery.ServiceDiscoveryBuilder;
 import org.apache.curator.x.discovery.ServiceInstance;
 import org.apache.curator.x.discovery.details.ServiceCacheListener;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
 import org.apache.zookeeper.CreateMode;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -61,69 +57,74 @@ import timely.balancer.ArrivalRate;
 import timely.balancer.configuration.BalancerConfiguration;
 import timely.balancer.connection.TimelyBalancedHost;
 import timely.balancer.healthcheck.HealthChecker;
+import timely.balancer.resolver.eventing.MetricAssignedCallback;
+import timely.balancer.resolver.eventing.MetricAssignedEvent;
+import timely.balancer.resolver.eventing.MetricBalanceCallback;
+import timely.balancer.resolver.eventing.MetricBalanceEvent;
+import timely.balancer.resolver.eventing.MetricHostCallback;
+import timely.balancer.resolver.eventing.MetricHostEvent;
 
 public class BalancedMetricResolver implements MetricResolver {
 
     private static final Logger LOG = LoggerFactory.getLogger(BalancedMetricResolver.class);
     private Map<String, TimelyBalancedHost> metricToHostMap = new TreeMap<>();
+    private Set<String> unassignedMetrics = new HashSet<>();
     private Map<String, ArrivalRate> metricMap = new HashMap<>();
     private ReentrantReadWriteLock balancerLock = new ReentrantReadWriteLock();
     private List<TimelyBalancedHost> serverList = new ArrayList<>();
     private Random r = new Random();
-    final private HealthChecker healthChecker;
-    private Timer timer = new Timer("RebalanceTimer", true);
-    private Timer arrivalRateTimer = new Timer("ArrivalRateTimerResolver", true);
+    private final HealthChecker healthChecker;
+    private ScheduledExecutorService assignmentExecutor = Executors.newScheduledThreadPool(2);
+    private ScheduledExecutorService arrivalRateExecutor = Executors.newScheduledThreadPool(2);
     private int roundRobinCounter = 0;
     private Set<String> nonCachedMetrics = new HashSet<>();
     private DistributedAtomicValue nonCachedMetricsIP;
     private ReentrantReadWriteLock nonCachedMetricsLocalLock = new ReentrantReadWriteLock();
     private InterProcessReadWriteLock nonCachedMetricsIPRWLock;
-    private BalancerConfiguration balancerConfig;
+    private AtomicLong assignmentsLastUpdatedLocal = new AtomicLong(0);
     private LeaderLatch leaderLatch;
     private AtomicBoolean isLeader = new AtomicBoolean(false);
     private InterProcessReadWriteLock assignmentsIPRWLock;
-    private DistributedAtomicLong assignmentsLastUpdatedInHdfs;
-    private AtomicLong assignmentsLastUpdatedLocal = new AtomicLong(0);
+    private MetricAssignmentPerister assignmentPerister;
 
-    private enum BalanceType {
-        HIGH_LOW, HIGH_AVG, AVG_LOW
+    final private BalancerConfiguration balancerConfig;
+    final private CuratorFramework curatorFramework;
+
+    private List<MetricAssignedCallback> metricAssignedCallbacks = new ArrayList<>();
+    private List<MetricBalanceCallback> metricBalanceCallbacks = new ArrayList<>();
+    private List<MetricHostCallback> metricHostCallbacks = new ArrayList<>();
+
+    public enum BalanceType {
+        HIGH_LOW, HIGH_AVG, AVG_LOW;
     }
 
     public BalancedMetricResolver(CuratorFramework curatorFramework, BalancerConfiguration balancerConfig,
             HealthChecker healthChecker) {
+        this.curatorFramework = curatorFramework;
         this.balancerConfig = balancerConfig;
         this.healthChecker = healthChecker;
+    }
 
+    public void start() {
         assignmentsIPRWLock = new InterProcessReadWriteLock(curatorFramework, ASSIGNMENTS_LOCK_PATH);
         testIPRWLock(curatorFramework, assignmentsIPRWLock, ASSIGNMENTS_LOCK_PATH);
+
+        this.assignmentPerister = new MetricAssignmentPerister(this, balancerConfig, metricToHostMap, nonCachedMetrics,
+                curatorFramework, assignmentsIPRWLock, nonCachedMetricsLocalLock, assignmentsLastUpdatedLocal,
+                balancerLock);
+
         startLeaderLatch(curatorFramework);
         startServiceListener(curatorFramework);
-        assignmentsLastUpdatedInHdfs = new DistributedAtomicLong(curatorFramework, ASSIGNMENTS_LAST_UPDATED_PATH,
-                new RetryForever(1000));
 
-        TreeCacheListener assignmentListener = new TreeCacheListener() {
-
-            @Override
-            public void childEvent(CuratorFramework curatorFramework, TreeCacheEvent event) throws Exception {
-                LOG.debug("Handling assignments event {}. assignmentsLastUpdatedInHdfs:{}", event.getType().toString(),
-                        new Date(assignmentsLastUpdatedInHdfs.get().postValue()));
-                if (event.getType().equals(TreeCacheEvent.Type.NODE_UPDATED)) {
-                    long lastLocalUpdate = assignmentsLastUpdatedLocal.get();
-                    long lastHdfsUpdate = assignmentsLastUpdatedInHdfs.get().postValue();
-                    if (lastHdfsUpdate > lastLocalUpdate) {
-                        LOG.debug("Reading assignments from hdfs lastHdfsUpdate ({}) > lastLocalUpdate ({})",
-                                new Date(lastHdfsUpdate), new Date(lastLocalUpdate));
-                        readAssignmentsFromHdfs();
-                    } else {
-                        LOG.debug("Not reading assignments from hdfs lastHdfsUpdate ({}) <= lastLocalUpdate ({})",
-                                new Date(lastHdfsUpdate), new Date(lastLocalUpdate));
-                    }
-                }
+        TreeCacheListener assignmentListener = (curatorFramework1, event) -> {
+            LOG.debug("Handling assignments event {}. assignmentsLastUpdated:{}", event.getType().toString(),
+                    new Date(assignmentPerister.getAssignmentsLastUpdatedByLeader()));
+            if (event.getType().equals(TreeCacheEvent.Type.NODE_UPDATED)) {
+                assignmentPerister.readAssignments(true);
             }
         };
 
-        try {
-            TreeCache assignmentTreeCache = new TreeCache(curatorFramework, ASSIGNMENTS_LAST_UPDATED_PATH);
+        try (TreeCache assignmentTreeCache = new TreeCache(curatorFramework, ASSIGNMENTS_LAST_UPDATED_PATH)) {
             assignmentTreeCache.getListenable().addListener(assignmentListener);
             assignmentTreeCache.start();
         } catch (Exception e) {
@@ -144,8 +145,7 @@ public class BalancedMetricResolver implements MetricResolver {
             }
         };
 
-        try {
-            TreeCache nonCachedMetricsTreeCache = new TreeCache(curatorFramework, NON_CACHED_METRICS);
+        try (TreeCache nonCachedMetricsTreeCache = new TreeCache(curatorFramework, NON_CACHED_METRICS)) {
             nonCachedMetricsTreeCache.getListenable().addListener(nonCachedMetricsListener);
             nonCachedMetricsTreeCache.start();
         } catch (Exception e) {
@@ -160,55 +160,55 @@ public class BalancedMetricResolver implements MetricResolver {
         }
         readNonCachedMetricsIP();
 
-        readAssignmentsFromHdfs();
+        assignmentPerister.readAssignments(false);
 
-        timer.schedule(new TimerTask() {
-
-            @Override
-            public void run() {
-                if (isLeader.get()) {
-                    try {
-                        balance();
-                    } catch (Exception e) {
-                        LOG.error(e.getMessage(), e);
-                    }
-                }
-            }
-        }, 900000, 900000);
-
-        timer.schedule(new TimerTask() {
-
-            @Override
-            public void run() {
+        assignmentExecutor.scheduleAtFixedRate(() -> {
+            if (isLeader.get()) {
                 try {
-                    if (isLeader.get()) {
-                        long lastLocalUpdate = assignmentsLastUpdatedLocal.get();
-                        long lastHdfsUpdate = assignmentsLastUpdatedInHdfs.get().postValue();
-                        if (lastLocalUpdate > lastHdfsUpdate) {
-                            LOG.debug("Writing assignments to hdfs lastLocalUpdate ({}) > lastHdfsUpdate ({})",
-                                    new Date(lastLocalUpdate), new Date(lastHdfsUpdate));
-                            writeAssigmentsToHdfs();
-                        }
-                    }
+                    balance();
                 } catch (Exception e) {
                     LOG.error(e.getMessage(), e);
                 }
             }
-        }, 10000, 60000);
+        }, balancerConfig.getBalanceDelay(), balancerConfig.getBalancePeriod(), TimeUnit.MILLISECONDS);
+
+        assignmentExecutor.scheduleAtFixedRate(() -> {
+            try {
+                if (isLeader.get()) {
+                    long lastLocalUpdate = assignmentsLastUpdatedLocal.get();
+                    long lastUpdateByLeader = assignmentPerister.getAssignmentsLastUpdatedByLeader();
+                    if (lastLocalUpdate > lastUpdateByLeader) {
+                        LOG.debug("Leader writing assignments lastLocalUpdate ({}) > lastUpdateByLeader ({})",
+                                new Date(lastLocalUpdate), new Date(lastUpdateByLeader));
+                        assignmentPerister.writeAssignments();
+                    } else {
+                        LOG.trace("Leader not writing assignments lastLocalUpdate ({}) <= lastUpdateByLeader ({})",
+                                new Date(lastLocalUpdate), new Date(lastUpdateByLeader));
+                    }
+                } else {
+                    assignmentPerister.readAssignments(true);
+                }
+            } catch (Exception e) {
+                LOG.error(e.getMessage(), e);
+            }
+        }, balancerConfig.getPersistDelay(), balancerConfig.getPersistPeriod(), TimeUnit.MILLISECONDS);
     }
 
     private void readNonCachedMetricsIP() {
 
         Set<String> currentNonCachedMetricsDistributed = new TreeSet<>();
         try {
-            nonCachedMetricsIPRWLock.readLock().acquire();
+            boolean acquired = false;
+            while (!acquired) {
+                acquired = nonCachedMetricsIPRWLock.readLock().acquire(60, TimeUnit.SECONDS);
+            }
             byte[] currentNonCachedMetricsDistributedBytes = nonCachedMetricsIP.get().postValue();
             if (currentNonCachedMetricsDistributedBytes != null) {
                 try {
                     currentNonCachedMetricsDistributed = SerializationUtils
                             .deserialize(currentNonCachedMetricsDistributedBytes);
                 } catch (Exception e) {
-                    LOG.error(e.getMessage());
+                    LOG.error(e.getMessage(), e);
                     currentNonCachedMetricsDistributed = new TreeSet<>();
                 }
             }
@@ -249,7 +249,7 @@ public class BalancedMetricResolver implements MetricResolver {
                 public void isLeader() {
                     LOG.info("this balancer is the leader");
                     isLeader.set(true);
-                    writeAssigmentsToHdfs();
+                    assignmentPerister.writeAssignments();
                 }
 
                 @Override
@@ -265,28 +265,29 @@ public class BalancedMetricResolver implements MetricResolver {
 
     private void startServiceListener(CuratorFramework curatorFramework) {
         try {
-            ServiceDiscovery discovery = ServiceDiscoveryBuilder.builder(ServerDetails.class).client(curatorFramework)
-                    .basePath(SERVICE_DISCOVERY_PATH).build();
+            ServiceDiscovery<ServerDetails> discovery = ServiceDiscoveryBuilder.builder(ServerDetails.class)
+                    .client(curatorFramework).basePath(SERVICE_DISCOVERY_PATH).build();
             discovery.start();
-            Collection<ServiceInstance> instances = discovery.queryForInstances("timely-server");
+            Collection<ServiceInstance<ServerDetails>> instances = discovery.queryForInstances("timely-server");
 
             balancerLock.writeLock().lock();
             try {
-                for (ServiceInstance si : instances) {
-                    ServerDetails pl = (ServerDetails) si.getPayload();
+                for (ServiceInstance<ServerDetails> si : instances) {
+                    ServerDetails pl = si.getPayload();
                     TimelyBalancedHost tbh = TimelyBalancedHost.of(pl.getHost(), pl.getTcpPort(), pl.getHttpPort(),
-                            pl.getWsPort(), pl.getUdpPort());
-                    LOG.info("adding service {} host:{} tcpPort:{} httpPort:{} wsPort:{} udpPort:{}", si.getId(),
-                            pl.getHost(), pl.getTcpPort(), pl.getHttpPort(), pl.getWsPort(), pl.getUdpPort());
+                            pl.getWsPort(), pl.getUdpPort(), new ArrivalRate(arrivalRateExecutor));
+                    LOG.info("adding service {} host:{}", si.getId(), TimelyBalancedHost.toStringShort(tbh));
                     tbh.setBalancerConfig(balancerConfig);
                     serverList.add(tbh);
+                    metricHostEvent(tbh, MetricHostEvent.ActionType.ADDED);
                 }
                 healthChecker.setTimelyHosts(serverList);
             } finally {
                 balancerLock.writeLock().unlock();
             }
 
-            final ServiceCache serviceCache = discovery.serviceCacheBuilder().name("timely-server").build();
+            final ServiceCache<ServerDetails> serviceCache = discovery.serviceCacheBuilder().name("timely-server")
+                    .build();
             ServiceCacheListener listener = new ServiceCacheListener() {
 
                 @Override
@@ -294,12 +295,13 @@ public class BalancedMetricResolver implements MetricResolver {
                     boolean rebalanceNeeded = false;
                     balancerLock.writeLock().lock();
                     try {
-                        List<ServiceInstance> instances = serviceCache.getInstances();
+                        List<ServiceInstance<ServerDetails>> instances = serviceCache.getInstances();
                         Set<TimelyBalancedHost> availableHosts = new HashSet<>();
-                        for (ServiceInstance si : instances) {
+                        for (ServiceInstance<ServerDetails> si : instances) {
                             ServerDetails pl = (ServerDetails) si.getPayload();
                             TimelyBalancedHost tbh = TimelyBalancedHost.of(pl.getHost(), pl.getTcpPort(),
-                                    pl.getHttpPort(), pl.getWsPort(), pl.getUdpPort());
+                                    pl.getHttpPort(), pl.getWsPort(), pl.getUdpPort(),
+                                    new ArrivalRate(arrivalRateExecutor));
                             tbh.setBalancerConfig(balancerConfig);
                             availableHosts.add(tbh);
                         }
@@ -313,9 +315,8 @@ public class BalancedMetricResolver implements MetricResolver {
                                 availableHosts.remove(h);
                             } else {
                                 itr.remove();
-                                LOG.info("removing service {}:{} host:{} tcpPort:{} httpPort:{} wsPort:{} udpPort:{}",
-                                        h.getHost(), h.getTcpPort(), h.getHost(), h.getTcpPort(), h.getHttpPort(),
-                                        h.getWsPort(), h.getUdpPort());
+                                metricHostEvent(h, MetricHostEvent.ActionType.REMOVED);
+                                LOG.info("removing service {}", TimelyBalancedHost.toStringShort(h));
                                 for (Map.Entry<String, TimelyBalancedHost> e : metricToHostMap.entrySet()) {
                                     if (e.getValue().equals(h)) {
                                         reassignMetrics.add(e.getKey());
@@ -327,28 +328,33 @@ public class BalancedMetricResolver implements MetricResolver {
 
                         // add new hosts that were not previously known
                         for (TimelyBalancedHost h : availableHosts) {
-                            LOG.info("adding service {}:{} host:{} tcpPort:{} httpPort:{} wsPort:{} udpPort:{}",
-                                    h.getHost(), h.getTcpPort(), h.getHost(), h.getTcpPort(), h.getHttpPort(),
-                                    h.getWsPort(), h.getUdpPort());
+                            LOG.info("adding service {}", TimelyBalancedHost.toStringShort(h));
                             serverList.add(h);
+                            metricHostEvent(h, MetricHostEvent.ActionType.ADDED);
                             rebalanceNeeded = true;
                         }
                         healthChecker.setTimelyHosts(serverList);
-
+                        if (!availableHosts.isEmpty()) {
+                            reassignMetrics.addAll(unassignedMetrics);
+                            unassignedMetrics.clear();
+                        }
                         if (isLeader.get()) {
                             for (String s : reassignMetrics) {
                                 TimelyBalancedHost h = getRoundRobinHost(null);
-                                assignMetric(s, h);
-                                LOG.debug("Assigned server removed.  Assigning {} to server {}:{}", s, h.getTcpPort());
+                                assignMetric(s, h, HOST_REMOVED);
+                                LOG.debug("Assigned server removed.  Assigning {} to server {}", s,
+                                        TimelyBalancedHost.toStringShort(h));
                             }
-                            writeAssigmentsToHdfs();
                         }
 
                     } finally {
                         balancerLock.writeLock().unlock();
                     }
-                    if (isLeader.get() && rebalanceNeeded) {
-                        balance();
+                    if (isLeader.get()) {
+                        if (rebalanceNeeded) {
+                            balance();
+                        }
+                        assignmentPerister.writeAssignments();
                     }
                 }
 
@@ -408,7 +414,7 @@ public class BalancedMetricResolver implements MetricResolver {
         return tbh;
     }
 
-    private TimelyBalancedHost getRoundRobinHost(TimelyBalancedHost notThisOne) {
+    protected TimelyBalancedHost getRoundRobinHost(TimelyBalancedHost notThisOne) {
         TimelyBalancedHost tbh = null;
         balancerLock.readLock().lock();
         try {
@@ -530,8 +536,10 @@ public class BalancedMetricResolver implements MetricResolver {
 
         int numReassigned = 0;
         if (isLeader.get()) {
+            metricBalanceEvent(MetricBalanceEvent.ProgressType.BEGIN, balanceType, 0);
             balancerLock.writeLock().lock();
             try {
+                metricBalanceEvent(MetricBalanceEvent.ProgressType.BALANCER_LOCK_ACQUIRED, balanceType, 0);
                 Map<String, ArrivalRate> tempMetricMap = new HashMap<>();
                 tempMetricMap.putAll(metricMap);
 
@@ -561,8 +569,8 @@ public class BalancedMetricResolver implements MetricResolver {
                         }
                     }
 
-                    LOG.trace("focusHost {}:{} desiredChange:{} rateSortedMetrics.size():{}", h.getHost(),
-                            h.getTcpPort(), desiredChange, rateSortedMetrics.size());
+                    LOG.trace("focusHost {} desiredChange:{} rateSortedMetrics.size():{}",
+                            TimelyBalancedHost.toStringShort(h), desiredChange, rateSortedMetrics.size());
 
                     boolean doneWithHost = false;
                     while (!doneWithHost) {
@@ -591,7 +599,7 @@ public class BalancedMetricResolver implements MetricResolver {
                                 } else {
                                     String metric = e.getValue();
                                     Double metricRate = e.getKey();
-                                    assignMetric(metric, candidateHost);
+                                    assignMetric(metric, candidateHost, REBALANCE);
                                     numReassigned++;
                                     calculatedRates.put(candidateHost, calculatedRates.get(candidateHost) + metricRate);
                                     calculatedRates.put(h, calculatedRates.get(h) - metricRate);
@@ -600,9 +608,9 @@ public class BalancedMetricResolver implements MetricResolver {
                                     itr.remove();
                                     tempMetricMap.remove(metric);
                                     LOG.info(
-                                            "rebalancing: reassigning metric:{} rate:{} from server {}:{} to {}:{} remaining delta {}",
-                                            metric, metricRate, h.getHost(), h.getTcpPort(), candidateHost.getHost(),
-                                            candidateHost.getTcpPort(), desiredChange);
+                                            "rebalancing: reassigning metric:{} rate:{} from server {} to {} remaining delta {}",
+                                            metric, metricRate, TimelyBalancedHost.toStringShort(h),
+                                            TimelyBalancedHost.toStringShort(candidateHost), desiredChange);
                                 }
                             }
                         }
@@ -625,6 +633,7 @@ public class BalancedMetricResolver implements MetricResolver {
                 }
             } finally {
                 balancerLock.writeLock().unlock();
+                metricBalanceEvent(MetricBalanceEvent.ProgressType.END, balanceType, numReassigned);
             }
         }
         return numReassigned;
@@ -642,7 +651,10 @@ public class BalancedMetricResolver implements MetricResolver {
                 }
 
                 try {
-                    nonCachedMetricsIPRWLock.writeLock().acquire();
+                    boolean acquired = false;
+                    while (!acquired) {
+                        acquired = nonCachedMetricsIPRWLock.writeLock().acquire(60, TimeUnit.SECONDS);
+                    }
                     byte[] currentNonCachedMetricsDistributedBytes = nonCachedMetricsIP.get().postValue();
                     Set<String> currentNonCachedMetricsIP;
                     if (currentNonCachedMetricsDistributedBytes == null) {
@@ -689,12 +701,12 @@ public class BalancedMetricResolver implements MetricResolver {
             try {
                 lock.writeLock().release();
             } catch (Exception e3) {
-                LOG.error(e3.getMessage());
+                LOG.error(e3.getMessage(), e3);
             }
         }
     }
 
-    private boolean shouldCache(String metricName) {
+    protected boolean shouldCache(String metricName) {
 
         if (StringUtils.isBlank(metricName)) {
             return false;
@@ -743,7 +755,7 @@ public class BalancedMetricResolver implements MetricResolver {
             if (chooseMetricSpecificHost) {
                 ArrivalRate rate = metricMap.get(metric);
                 if (rate == null) {
-                    rate = new ArrivalRate(arrivalRateTimer);
+                    rate = new ArrivalRate(arrivalRateExecutor);
                     if (!balancerLock.isWriteLockedByCurrentThread()) {
                         balancerLock.readLock().unlock();
                         balancerLock.writeLock().lock();
@@ -756,7 +768,7 @@ public class BalancedMetricResolver implements MetricResolver {
             }
 
             if (StringUtils.isBlank(metric)) {
-                tbh = getRandomHost(null);
+                tbh = getRoundRobinHost(null);
             } else {
                 tbh = metricToHostMap.get(metric);
                 if (tbh == null) {
@@ -765,17 +777,17 @@ public class BalancedMetricResolver implements MetricResolver {
                         balancerLock.readLock().unlock();
                         balancerLock.writeLock().lock();
                     }
-                    assignMetric(metric, tbh);
+                    assignMetric(metric, tbh, ASSIGN_ROUND_ROBIN);
                 } else if (!tbh.isUp()) {
                     TimelyBalancedHost oldTbh = tbh;
-                    tbh = getLeastUsedHost();
-                    LOG.debug("rebalancing from host that is down: reassigning metric {} from server {}:{} to {}:{}",
-                            metric, oldTbh.getHost(), oldTbh.getTcpPort(), tbh.getHost(), tbh.getTcpPort());
+                    tbh = getRoundRobinHost(oldTbh);
+                    LOG.debug("rebalancing from host that is down: reassigning metric {} from server {} to {}", metric,
+                            TimelyBalancedHost.toStringShort(oldTbh), TimelyBalancedHost.toStringShort(tbh));
                     if (!balancerLock.isWriteLockedByCurrentThread()) {
                         balancerLock.readLock().unlock();
                         balancerLock.writeLock().lock();
                     }
-                    assignMetric(metric, tbh);
+                    assignMetric(metric, tbh, HOST_DOWN_ROUND_ROBIN);
                 }
             }
 
@@ -792,7 +804,7 @@ public class BalancedMetricResolver implements MetricResolver {
                         balancerLock.readLock().unlock();
                         balancerLock.writeLock().lock();
                     }
-                    assignMetric(metric, tbh);
+                    assignMetric(metric, tbh, ASSIGN_FALLBACK_SEQUENTIAL);
                 }
             }
         } finally {
@@ -821,20 +833,23 @@ public class BalancedMetricResolver implements MetricResolver {
                 metric = null;
             }
 
-            if (tbh == null || !tbh.isUp()) {
-                for (int x = 0; tbh == null && x < serverList.size(); x++) {
-                    tbh = serverList.get(Math.abs(r.nextInt() & Integer.MAX_VALUE) % serverList.size());
-                    if (!tbh.isUp()) {
-                        tbh = null;
-                    }
+            if (tbh == null) {
+                tbh = getRoundRobinHost(null);
+                if (!balancerLock.isWriteLockedByCurrentThread()) {
+                    balancerLock.readLock().unlock();
+                    balancerLock.writeLock().lock();
                 }
-                if (tbh != null && StringUtils.isNotBlank(metric)) {
-                    if (!balancerLock.isWriteLockedByCurrentThread()) {
-                        balancerLock.readLock().unlock();
-                        balancerLock.writeLock().lock();
-                    }
-                    assignMetric(metric, tbh);
+                assignMetric(metric, tbh, ASSIGN_ROUND_ROBIN);
+            } else if (!tbh.isUp()) {
+                TimelyBalancedHost oldTbh = tbh;
+                tbh = getRoundRobinHost(oldTbh);
+                LOG.debug("rebalancing from host that is down: reassigning metric {} from server {} to {}", metric,
+                        TimelyBalancedHost.toStringShort(oldTbh), TimelyBalancedHost.toStringShort(tbh));
+                if (!balancerLock.isWriteLockedByCurrentThread()) {
+                    balancerLock.readLock().unlock();
+                    balancerLock.writeLock().lock();
                 }
+                assignMetric(metric, tbh, HOST_DOWN_ROUND_ROBIN);
             }
 
             // if all else fails
@@ -850,7 +865,7 @@ public class BalancedMetricResolver implements MetricResolver {
                         balancerLock.readLock().unlock();
                         balancerLock.writeLock().lock();
                     }
-                    assignMetric(metric, tbh);
+                    assignMetric(metric, tbh, ASSIGN_FALLBACK_SEQUENTIAL);
                 }
             }
         } finally {
@@ -863,7 +878,7 @@ public class BalancedMetricResolver implements MetricResolver {
         return tbh;
     }
 
-    private TimelyBalancedHost findHost(String host, int tcpPort) {
+    protected TimelyBalancedHost findHost(String host, int tcpPort) {
         TimelyBalancedHost tbh = null;
         for (TimelyBalancedHost h : serverList) {
             if (h.getHost().equals(host) && h.getTcpPort() == tcpPort) {
@@ -874,144 +889,100 @@ public class BalancedMetricResolver implements MetricResolver {
         return tbh;
     }
 
-    private void readAssignmentsFromHdfs() {
-
-        Map<String, TimelyBalancedHost> assignedMetricToHostMap = new TreeMap<>();
-        try {
-            assignmentsIPRWLock.readLock().acquire();
-            balancerLock.writeLock().lock();
-            Configuration configuration = new Configuration();
-            FileSystem fs = FileSystem.get(configuration);
-            Path assignmentFile = new Path(balancerConfig.getAssignmentFile());
-            FSDataInputStream iStream = fs.open(assignmentFile);
-
-            CsvReader reader = new CsvReader(iStream, ',', Charset.forName("UTF-8"));
-            reader.setUseTextQualifier(false);
-
-            // skip the headers
-            boolean success = true;
-            success = reader.readHeaders();
-
-            while (success) {
-                success = reader.readRecord();
-                String[] nextLine = reader.getValues();
-                if (nextLine.length >= 3) {
-                    String metric = nextLine[0];
-                    String host = nextLine[1];
-                    int tcpPort = Integer.parseInt(nextLine[2]);
-                    TimelyBalancedHost tbh = findHost(host, tcpPort);
-                    if (tbh == null) {
-                        tbh = getRoundRobinHost(null);
-                    } else {
-                        LOG.trace("Found assigment: {} to {}:{}", metric, host, tcpPort);
-                    }
-                    assignedMetricToHostMap.put(metric, tbh);
-                }
-            }
-
-            metricToHostMap.clear();
-            for (Map.Entry<String, TimelyBalancedHost> e : assignedMetricToHostMap.entrySet()) {
-                if (shouldCache(e.getKey())) {
-                    metricToHostMap.put(e.getKey(), e.getValue());
-                }
-            }
-
-            nonCachedMetricsLocalLock.readLock().lock();
-            try {
-                Iterator<Map.Entry<String, TimelyBalancedHost>> itr = metricToHostMap.entrySet().iterator();
-                while (itr.hasNext()) {
-                    Map.Entry<String, TimelyBalancedHost> e = itr.next();
-                    if (nonCachedMetrics.contains(e.getKey())) {
-                        itr.remove();
-                    }
-                }
-            } finally {
-                nonCachedMetricsLocalLock.readLock().unlock();
-            }
-            assignmentsLastUpdatedLocal.set(assignmentsLastUpdatedInHdfs.get().postValue());
-            LOG.info("Read {} assignments from hdfs lastHdfsUpdate = lastLocalUpdate ({})", metricToHostMap.size(),
-                    new Date(assignmentsLastUpdatedLocal.get()));
-        } catch (Exception e) {
-            LOG.error(e.getMessage(), e);
-        } finally {
-            balancerLock.writeLock().unlock();
-            try {
-                assignmentsIPRWLock.readLock().release();
-            } catch (Exception e) {
-                LOG.error(e.getMessage(), e);
-            }
-        }
-    }
-
-    private void writeAssigmentsToHdfs() {
-
-        CsvWriter writer = null;
-        try {
-            assignmentsIPRWLock.writeLock().acquire();
-            balancerLock.readLock().lock();
-            nonCachedMetricsLocalLock.readLock().lock();
-            try {
-                if (!metricToHostMap.isEmpty()) {
-                    Configuration configuration = new Configuration();
-                    FileSystem fs = FileSystem.get(configuration);
-                    Path assignmentFile = new Path(balancerConfig.getAssignmentFile());
-                    if (!fs.exists(assignmentFile.getParent())) {
-                        fs.mkdirs(assignmentFile.getParent());
-                    }
-                    FSDataOutputStream oStream = fs.create(assignmentFile, true);
-                    writer = new CsvWriter(oStream, ',', Charset.forName("UTF-8"));
-                    writer.setUseTextQualifier(false);
-                    writer.write("metric");
-                    writer.write("host");
-                    writer.write("tcpPort");
-                    writer.endRecord();
-                    for (Map.Entry<String, TimelyBalancedHost> e : metricToHostMap.entrySet()) {
-                        if (!nonCachedMetrics.contains(e.getKey())) {
-                            writer.write(e.getKey());
-                            writer.write(e.getValue().getHost());
-                            writer.write(Integer.toString(e.getValue().getTcpPort()));
-                            writer.endRecord();
-                            LOG.trace("Saving assigment: {} to {}:{}", e.getKey(), e.getValue().getHost(),
-                                    e.getValue().getTcpPort());
-                        }
-                    }
-
-                    long now = System.currentTimeMillis();
-                    assignmentsLastUpdatedLocal.set(now);
-                    assignmentsLastUpdatedInHdfs.trySet(now);
-                    if (!assignmentsLastUpdatedInHdfs.get().succeeded()) {
-                        assignmentsLastUpdatedInHdfs.forceSet(now);
-                    }
-                    LOG.debug("Wrote {} assignments to hdfs lastHdfsUpdate = lastLocalUpdate ({})",
-                            metricToHostMap.size(), new Date(assignmentsLastUpdatedLocal.get()));
-                }
-            } finally {
-                nonCachedMetricsLocalLock.readLock().unlock();
-                balancerLock.readLock().unlock();
-            }
-        } catch (Exception e) {
-            LOG.error(e.getMessage(), e);
-        } finally {
-            if (writer != null) {
-                writer.close();
-            }
-            try {
-                assignmentsIPRWLock.writeLock().release();
-            } catch (Exception e) {
-                LOG.error(e.getMessage(), e);
-            }
-        }
-    }
-
-    private void assignMetric(String metric, TimelyBalancedHost tbh) {
+    private void assignMetric(String metric, TimelyBalancedHost tbh, MetricAssignedEvent.Reason reason) {
         if (isLeader.get()) {
-            balancerLock.writeLock().lock();
-            try {
-                metricToHostMap.put(metric, tbh);
-                assignmentsLastUpdatedLocal.set(System.currentTimeMillis());
-            } finally {
-                balancerLock.writeLock().unlock();
+            if (StringUtils.isNotBlank(metric) && tbh != null) {
+                balancerLock.writeLock().lock();
+                try {
+                    metricAssignedEvent(metric, metricToHostMap.get(metric), tbh, reason);
+                    metricToHostMap.put(metric, tbh);
+                    assignmentsLastUpdatedLocal.set(System.currentTimeMillis());
+                } finally {
+                    balancerLock.writeLock().unlock();
+                }
+            } else {
+                unassignedMetrics.add(metric);
+                // Exception e = new IllegalStateException("Bad assignment metric:" + metric + "
+                // host:" + tbh);
+                LOG.warn("Bad assignment metric:" + metric + " host:" + tbh);
             }
         }
     }
+
+    public void registerCallback(MetricAssignedCallback callback) {
+        this.metricAssignedCallbacks.add(callback);
+    }
+
+    public void registerCallback(MetricBalanceCallback callback) {
+        this.metricBalanceCallbacks.add(callback);
+    }
+
+    public void registerCallback(MetricHostCallback callback) {
+        this.metricHostCallbacks.add(callback);
+    }
+
+    protected void metricAssignedEvent(String metric, TimelyBalancedHost losing, TimelyBalancedHost gaining,
+            MetricAssignedEvent.Reason reason) {
+        if (!metricAssignedCallbacks.isEmpty()) {
+            MetricAssignedEvent event = new MetricAssignedEvent(metric, losing, gaining, reason);
+            for (MetricAssignedCallback callback : metricAssignedCallbacks) {
+                callback.onEvent(event);
+            }
+        }
+    }
+
+    protected void metricBalanceEvent(MetricBalanceEvent.ProgressType progressType,
+            BalancedMetricResolver.BalanceType balanceType, long numReassigned) {
+        if (!metricBalanceCallbacks.isEmpty()) {
+            MetricBalanceEvent event = new MetricBalanceEvent(progressType, balanceType, numReassigned);
+            for (MetricBalanceCallback callback : metricBalanceCallbacks) {
+                callback.onEvent(event);
+            }
+        }
+    }
+
+    protected void metricHostEvent(TimelyBalancedHost host, MetricHostEvent.ActionType actionType) {
+        if (!metricHostCallbacks.isEmpty()) {
+            MetricHostEvent event = new MetricHostEvent(host, actionType);
+            for (MetricHostCallback callback : metricHostCallbacks) {
+                callback.onEvent(event);
+            }
+        }
+    }
+
+    protected void writeAssignments(Map<String, TimelyBalancedHost> metricToHostMap) {
+        assignmentPerister.writeAssignments(metricToHostMap);
+    }
+
+    @Override
+    public void stop() {
+
+        this.assignmentExecutor.shutdown();
+        try {
+            this.assignmentExecutor.awaitTermination(5, TimeUnit.SECONDS);
+        } catch (InterruptedException e) {
+
+        } finally {
+            if (!this.assignmentExecutor.isTerminated()) {
+                this.assignmentExecutor.shutdownNow();
+            }
+        }
+
+        this.arrivalRateExecutor.shutdown();
+        try {
+            this.arrivalRateExecutor.awaitTermination(5, TimeUnit.SECONDS);
+        } catch (InterruptedException e) {
+
+        } finally {
+            if (!this.arrivalRateExecutor.isTerminated()) {
+                this.arrivalRateExecutor.shutdownNow();
+            }
+        }
+
+        try {
+            this.healthChecker.close();
+        } catch (Exception e) {
+            LOG.error(e.getMessage(), e);
+        }
+    }
 }

balancer/src/main/java/timely/balancer/resolver/MetricAssignmentPerister.java

@@ -0,0 +1,320 @@
+package timely.balancer.resolver;
+
+import static timely.balancer.Balancer.ASSIGNMENTS_LAST_UPDATED_PATH;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.charset.Charset;
+import java.util.Date;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.apache.commons.lang.StringUtils;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.recipes.atomic.DistributedAtomicLong;
+import org.apache.curator.framework.recipes.locks.InterProcessReadWriteLock;
+import org.apache.curator.retry.RetryForever;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import timely.balancer.configuration.BalancerConfiguration;
+import timely.balancer.connection.TimelyBalancedHost;
+import timely.balancer.resolver.eventing.MetricAssignedEvent;
+
+public class MetricAssignmentPerister {
+
+    private final Logger LOG = LoggerFactory.getLogger(getClass());
+
+    private enum PeristerType {
+        HDFS, FILE
+    }
+
+    // passed in through constructor
+    BalancedMetricResolver balancedMetricResolver;
+    private BalancerConfiguration balancerConfig;
+    private Map<String, TimelyBalancedHost> metricToHostMap;
+    private Set<String> nonCachedMetrics;
+    private InterProcessReadWriteLock assignmentsIPRWLock;
+    private ReentrantReadWriteLock nonCachedMetricsLocalLock;
+    private ReentrantReadWriteLock balancerLock;
+    private AtomicLong assignmentsLastUpdatedLocal;
+
+    // local
+    private DistributedAtomicLong assignmentsLastUpdatedInHdfs;
+
+    public MetricAssignmentPerister(BalancedMetricResolver balancedMetricResolver, BalancerConfiguration balancerConfig,
+            Map<String, TimelyBalancedHost> metricToHostMap, Set<String> nonCachedMetrics,
+            CuratorFramework curatorFramework, InterProcessReadWriteLock assignmentsIPRWLock,
+            ReentrantReadWriteLock nonCachedMetricsLocalLock, AtomicLong assignmentsLastUpdatedLocal,
+            ReentrantReadWriteLock balancerLock) {
+
+        this.balancedMetricResolver = balancedMetricResolver;
+        this.balancerConfig = balancerConfig;
+        this.metricToHostMap = metricToHostMap;
+        this.nonCachedMetrics = nonCachedMetrics;
+        this.assignmentsIPRWLock = assignmentsIPRWLock;
+        this.nonCachedMetricsLocalLock = nonCachedMetricsLocalLock;
+        this.assignmentsLastUpdatedLocal = assignmentsLastUpdatedLocal;
+        this.balancerLock = balancerLock;
+        assignmentsLastUpdatedInHdfs = new DistributedAtomicLong(curatorFramework, ASSIGNMENTS_LAST_UPDATED_PATH,
+                new RetryForever(1000));
+    }
+
+    public long getAssignmentsLastUpdatedByLeader() throws Exception {
+        return assignmentsLastUpdatedInHdfs.get().postValue();
+    }
+
+    public void readAssignments(boolean checkIfNecessary) {
+
+        try {
+            long lastLocalUpdate = assignmentsLastUpdatedLocal.get();
+            long lastHdfsUpdate = assignmentsLastUpdatedInHdfs.get().postValue();
+            if (checkIfNecessary) {
+                if (lastHdfsUpdate <= lastLocalUpdate) {
+                    LOG.debug("Not reading assignments from storage lastStorageUpdate ({}) <= lastLocalUpdate ({})",
+                            new Date(lastHdfsUpdate), new Date(lastLocalUpdate));
+                    return;
+                }
+            }
+            LOG.debug("Reading assignments from storage lastStorageUpdate ({}) > lastLocalUpdate ({})",
+                    new Date(lastHdfsUpdate), new Date(lastLocalUpdate));
+        } catch (Exception e) {
+            LOG.error(e.getMessage(), e);
+        }
+
+        // proceed with reading from storage
+
+        try (InputStream inputStream = getInputStream(balancerConfig)) {
+            boolean acquired = false;
+            while (!acquired) {
+                acquired = assignmentsIPRWLock.readLock().acquire(60, TimeUnit.SECONDS);
+            }
+            balancerLock.writeLock().lock();
+            // get potential new list from implementation
+            Map<String, TimelyBalancedHost> assignedMetricToHostMap = readFromStream(inputStream);
+            metricToHostMap.clear();
+            for (Map.Entry<String, TimelyBalancedHost> e : assignedMetricToHostMap.entrySet()) {
+                if (StringUtils.isNotBlank(e.getKey()) && e.getValue() != null) {
+                    if (balancedMetricResolver.shouldCache(e.getKey())) {
+                        balancedMetricResolver.metricAssignedEvent(e.getKey(), null, e.getValue(),
+                                MetricAssignedEvent.Reason.ASSIGN_FILE);
+                        metricToHostMap.put(e.getKey(), e.getValue());
+                    }
+                } else {
+                    Exception e1 = new IllegalStateException(
+                            "Bad assignment metric:" + e.getKey() + " host:" + e.getValue());
+                    LOG.warn(e1.getMessage(), e1);
+                }
+            }
+            nonCachedMetricsLocalLock.readLock().lock();
+            try {
+                Iterator<Map.Entry<String, TimelyBalancedHost>> itr = metricToHostMap.entrySet().iterator();
+                while (itr.hasNext()) {
+                    Map.Entry<String, TimelyBalancedHost> e = itr.next();
+                    if (nonCachedMetrics.contains(e.getKey())) {
+                        itr.remove();
+                    }
+                }
+            } finally {
+                nonCachedMetricsLocalLock.readLock().unlock();
+            }
+            assignmentsLastUpdatedLocal.set(assignmentsLastUpdatedInHdfs.get().postValue());
+            LOG.info("Read {} assignments from storage lastStorageUpdate = lastLocalUpdate ({})",
+                    metricToHostMap.size(), new Date(assignmentsLastUpdatedLocal.get()));
+        } catch (FileNotFoundException e) {
+            createAssignmentFile();
+            metricToHostMap.clear();
+        } catch (Exception e) {
+            LOG.error(e.getMessage(), e);
+        } finally {
+            balancerLock.writeLock().unlock();
+            try {
+                assignmentsIPRWLock.readLock().release();
+            } catch (Exception e) {
+                LOG.error(e.getMessage(), e);
+            }
+        }
+    }
+
+    public void writeAssignments() {
+        writeAssignments(metricToHostMap);
+    }
+
+    protected void writeAssignments(Map<String, TimelyBalancedHost> metricToHostMap) {
+
+        try {
+            boolean acquired = false;
+            while (!acquired) {
+                acquired = assignmentsIPRWLock.writeLock().acquire(60, TimeUnit.SECONDS);
+            }
+            balancerLock.readLock().lock();
+            nonCachedMetricsLocalLock.readLock().lock();
+            try (OutputStream outputStream = getOutputStream(balancerConfig)) {
+                if (!metricToHostMap.isEmpty()) {
+                    writeToStream(outputStream);
+                    long now = System.currentTimeMillis();
+                    assignmentsLastUpdatedLocal.set(now);
+                    assignmentsLastUpdatedInHdfs.trySet(now);
+                    if (!assignmentsLastUpdatedInHdfs.get().succeeded()) {
+                        assignmentsLastUpdatedInHdfs.forceSet(now);
+                    }
+                    LOG.debug("Wrote {} assignments to storage lastStorageUpdate = lastLocalUpdate ({})",
+                            metricToHostMap.size(), new Date(assignmentsLastUpdatedLocal.get()));
+                }
+            } finally {
+                nonCachedMetricsLocalLock.readLock().unlock();
+                balancerLock.readLock().unlock();
+            }
+        } catch (Exception e) {
+            LOG.error(e.getMessage(), e);
+        } finally {
+            try {
+                assignmentsIPRWLock.writeLock().release();
+            } catch (Exception e) {
+                LOG.error(e.getMessage(), e);
+            }
+        }
+    }
+
+    public Map<String, TimelyBalancedHost> readFromStream(InputStream inputStream) throws IOException {
+        Map<String, TimelyBalancedHost> assignedMetricToHostMap = new TreeMap<>();
+        try {
+            CsvReader reader = new CsvReader(inputStream, ',', Charset.forName("UTF-8"));
+            reader.setUseTextQualifier(false);
+
+            // skip the headers
+            boolean success = true;
+            success = reader.readHeaders();
+
+            while (success) {
+                success = reader.readRecord();
+                String[] nextLine = reader.getValues();
+                if (nextLine.length >= 3) {
+                    String metric = nextLine[0];
+                    String host = nextLine[1];
+                    int tcpPort = Integer.parseInt(nextLine[2]);
+                    TimelyBalancedHost tbh = balancedMetricResolver.findHost(host, tcpPort);
+                    if (tbh == null) {
+                        tbh = balancedMetricResolver.getRoundRobinHost(null);
+                    } else {
+                        LOG.trace("Found assigment: {} to {}:{}", metric, host, tcpPort);
+                    }
+                    assignedMetricToHostMap.put(metric, tbh);
+                }
+            }
+        } catch (IOException e) {
+            LOG.error(e.getMessage(), e);
+            throw e;
+        }
+        return assignedMetricToHostMap;
+    }
+
+    private void writeToStream(OutputStream outputStream) throws IOException {
+
+        CsvWriter writer = null;
+        try {
+            writer = new CsvWriter(outputStream, ',', Charset.forName("UTF-8"));
+            writer.setUseTextQualifier(false);
+            writer.write("metric");
+            writer.write("host");
+            writer.write("tcpPort");
+            writer.endRecord();
+            for (Map.Entry<String, TimelyBalancedHost> e : metricToHostMap.entrySet()) {
+                if (!nonCachedMetrics.contains(e.getKey())) {
+                    writer.write(e.getKey());
+                    writer.write(e.getValue().getHost());
+                    writer.write(Integer.toString(e.getValue().getTcpPort()));
+                    writer.endRecord();
+                    LOG.trace("Saving assigment: {} to {}:{}", e.getKey(), e.getValue().getHost(),
+                            e.getValue().getTcpPort());
+                }
+            }
+        } finally {
+            if (writer != null) {
+                writer.close();
+            }
+        }
+    }
+
+    public void createAssignmentFile() {
+        LOG.info("Creating assignment file: " + balancerConfig.getAssignmentFile());
+        PeristerType peristerType = PeristerType.valueOf(balancerConfig.getMetricAssignmentPersisterType());
+        switch (peristerType) {
+            case HDFS:
+                try {
+                    Configuration configuration = new Configuration();
+                    FileSystem fs = FileSystem.get(configuration);
+                    Path assignmentFile = new Path(balancerConfig.getAssignmentFile());
+                    fs.create(assignmentFile);
+                } catch (Exception e) {
+                    LOG.error(e.getMessage(), e);
+                }
+                break;
+            case FILE:
+                try {
+                    File assignmentFile = new File(balancerConfig.getAssignmentFile());
+                    String parentPath = assignmentFile.getParent();
+                    File directory = new File(parentPath);
+                    if (!directory.exists()) {
+
+                        directory.mkdirs();
+                    }
+                    if (!assignmentFile.exists()) {
+                        assignmentFile.createNewFile();
+                    }
+                } catch (Exception e) {
+                    LOG.error(e.getMessage(), e);
+                }
+                break;
+        }
+    }
+
+    public InputStream getInputStream(BalancerConfiguration balancerConfiguration) throws Exception {
+        PeristerType peristerType = PeristerType.valueOf(balancerConfiguration.getMetricAssignmentPersisterType());
+        switch (peristerType) {
+            case HDFS:
+                Configuration configuration = new Configuration();
+                FileSystem fs = FileSystem.get(configuration);
+                Path assignmentFile = new Path(balancerConfiguration.getAssignmentFile());
+                return fs.open(assignmentFile);
+            case FILE:
+                return new FileInputStream(new File(balancerConfiguration.getAssignmentFile()));
+            default:
+                throw new IllegalArgumentException(
+                        "Unknown metricPersisterType " + balancerConfiguration.getMetricAssignmentPersisterType());
+        }
+    }
+
+    public OutputStream getOutputStream(BalancerConfiguration balancerConfiguration) throws IOException {
+        PeristerType peristerType = PeristerType.valueOf(balancerConfiguration.getMetricAssignmentPersisterType());
+        switch (peristerType) {
+            case HDFS:
+                Configuration configuration = new Configuration();
+                FileSystem fs = FileSystem.get(configuration);
+                Path assignmentFile = new Path(balancerConfiguration.getAssignmentFile());
+                if (!fs.exists(assignmentFile.getParent())) {
+                    fs.mkdirs(assignmentFile.getParent());
+                }
+                return fs.create(assignmentFile, true);
+            case FILE:
+                return new FileOutputStream(new File(balancerConfiguration.getAssignmentFile()));
+            default:
+                throw new IllegalArgumentException(
+                        "Unknown metricPersisterType " + balancerConfiguration.getMetricAssignmentPersisterType());
+        }
+    }
+}

balancer/src/main/java/timely/balancer/resolver/MetricResolver.java

@@ -7,4 +7,9 @@ public interface MetricResolver {
     public TimelyBalancedHost getHostPortKeyIngest(String metric);
 
     public TimelyBalancedHost getHostPortKey(String metric);
+
+    public void start();
+
+    public void stop();
+
 }

balancer/src/main/java/timely/balancer/resolver/eventing/MetricAssignedCallback.java

@@ -0,0 +1,7 @@
+package timely.balancer.resolver.eventing;
+
+public abstract class MetricAssignedCallback {
+
+    public abstract void onEvent(MetricAssignedEvent event);
+
+}

balancer/src/main/java/timely/balancer/resolver/eventing/MetricAssignedEvent.java

@@ -0,0 +1,39 @@
+package timely.balancer.resolver.eventing;
+
+import timely.balancer.connection.TimelyBalancedHost;
+
+public class MetricAssignedEvent {
+
+    public enum Reason {
+        ASSIGN_ROUND_ROBIN, HOST_DOWN_ROUND_ROBIN, ASSIGN_FALLBACK_SEQUENTIAL, HOST_REMOVED, REBALANCE, ASSIGN_FILE
+    }
+
+    private final String metric;
+    private final TimelyBalancedHost losingHost;
+    private final TimelyBalancedHost gainingHost;
+    private final Reason reason;
+
+    public MetricAssignedEvent(String metric, TimelyBalancedHost losingHost, TimelyBalancedHost gainingHost,
+            Reason reason) {
+        this.metric = metric;
+        this.losingHost = losingHost;
+        this.gainingHost = gainingHost;
+        this.reason = reason;
+    }
+
+    public String getMetric() {
+        return metric;
+    }
+
+    public TimelyBalancedHost getLosingHost() {
+        return losingHost;
+    }
+
+    public TimelyBalancedHost getGainingHost() {
+        return gainingHost;
+    }
+
+    public Reason getReason() {
+        return reason;
+    }
+}

balancer/src/main/java/timely/balancer/resolver/eventing/MetricBalanceCallback.java

@@ -0,0 +1,7 @@
+package timely.balancer.resolver.eventing;
+
+public abstract class MetricBalanceCallback {
+
+    public abstract void onEvent(MetricBalanceEvent event);
+
+}

balancer/src/main/java/timely/balancer/resolver/eventing/MetricBalanceEvent.java

@@ -0,0 +1,33 @@
+package timely.balancer.resolver.eventing;
+
+import timely.balancer.resolver.BalancedMetricResolver;
+
+public class MetricBalanceEvent {
+
+    public enum ProgressType {
+        BEGIN, BALANCER_LOCK_ACQUIRED, END
+    }
+
+    private final ProgressType progressType;
+    private final BalancedMetricResolver.BalanceType balanceType;
+    private final Long numRessigned;
+
+    public MetricBalanceEvent(ProgressType progressType, BalancedMetricResolver.BalanceType balanceType,
+            long numReassigned) {
+        this.progressType = progressType;
+        this.balanceType = balanceType;
+        this.numRessigned = numReassigned;
+    }
+
+    public BalancedMetricResolver.BalanceType getBalanceType() {
+        return balanceType;
+    }
+
+    public ProgressType getProgressType() {
+        return progressType;
+    }
+
+    public Long getNumRessigned() {
+        return numRessigned;
+    }
+}

balancer/src/main/java/timely/balancer/resolver/eventing/MetricHostCallback.java

@@ -0,0 +1,7 @@
+package timely.balancer.resolver.eventing;
+
+public abstract class MetricHostCallback {
+
+    public abstract void onEvent(MetricHostEvent event);
+
+}

balancer/src/main/java/timely/balancer/resolver/eventing/MetricHostEvent.java

@@ -0,0 +1,26 @@
+package timely.balancer.resolver.eventing;
+
+import timely.balancer.connection.TimelyBalancedHost;
+
+public class MetricHostEvent {
+
+    public enum ActionType {
+        ADDED, REMOVED, SUCCESS, FAILURE
+    }
+
+    private final TimelyBalancedHost timelyBalancedHost;
+    private final ActionType actionType;
+
+    public MetricHostEvent(TimelyBalancedHost timelyBalancedHost, ActionType actionType) {
+        this.timelyBalancedHost = timelyBalancedHost;
+        this.actionType = actionType;
+    }
+
+    public TimelyBalancedHost getTimelyBalancedHost() {
+        return timelyBalancedHost;
+    }
+
+    public ActionType getActionType() {
+        return actionType;
+    }
+}

balancer/src/main/spotbugs/excludes.xml

@@ -10,4 +10,9 @@
     <Method name="getBuf" />
     <Bug pattern="EI_EXPOSE_REP" />
   </Match>
+  <Match>
+    <Class name="timely.balancer.resolver.MetricAssignmentPerister" />
+    <Method name="createAssignmentFile" />
+    <Bug pattern="RV_RETURN_VALUE_IGNORED_BAD_PRACTICE" />
+  </Match>
 </FindBugsFilter>

balancer/src/test/java/timely/balancer/resolver/ServiceRegistry.java

@@ -0,0 +1,76 @@
+package timely.balancer.resolver;
+
+import static timely.Server.SERVICE_DISCOVERY_PATH;
+
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.x.discovery.ServiceDiscovery;
+import org.apache.curator.x.discovery.ServiceDiscoveryBuilder;
+import org.apache.curator.x.discovery.ServiceInstance;
+import org.apache.curator.x.discovery.ServiceInstanceBuilder;
+import org.apache.zookeeper.data.Stat;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import timely.ServerDetails;
+import timely.balancer.connection.TimelyBalancedHost;
+
+public class ServiceRegistry {
+
+    private static final Logger LOG = LoggerFactory.getLogger(ServiceRegistry.class);
+
+    public static void registerService(CuratorFramework curatorFramework, TimelyBalancedHost tbh) {
+        try {
+            try {
+                Stat stat = curatorFramework.checkExists().forPath(SERVICE_DISCOVERY_PATH);
+                if (stat == null) {
+                    curatorFramework.create().creatingParentContainersIfNeeded().forPath(SERVICE_DISCOVERY_PATH);
+                }
+            } catch (Exception e) {
+                LOG.error(e.getMessage());
+            }
+
+            ServiceInstance<ServerDetails> serviceInstance = serviceInstance(tbh);
+            ServiceDiscovery<ServerDetails> discovery = ServiceDiscoveryBuilder.builder(ServerDetails.class)
+                    .client(curatorFramework).basePath(SERVICE_DISCOVERY_PATH).build();
+            discovery.start();
+            discovery.registerService(serviceInstance);
+        } catch (Exception e) {
+            LOG.error(e.getMessage());
+        }
+    }
+
+    public static void unregisterService(CuratorFramework curatorFramework, TimelyBalancedHost tbh) {
+        try {
+            try {
+                Stat stat = curatorFramework.checkExists().forPath(SERVICE_DISCOVERY_PATH);
+                if (stat == null) {
+                    curatorFramework.create().creatingParentContainersIfNeeded().forPath(SERVICE_DISCOVERY_PATH);
+                }
+            } catch (Exception e) {
+                LOG.error(e.getMessage());
+            }
+
+            ServiceInstance<ServerDetails> serviceInstance = serviceInstance(tbh);
+            ServiceDiscovery<ServerDetails> discovery = ServiceDiscoveryBuilder.builder(ServerDetails.class)
+                    .client(curatorFramework).basePath(SERVICE_DISCOVERY_PATH).build();
+            discovery.start();
+            discovery.unregisterService(serviceInstance);
+        } catch (Exception e) {
+            LOG.error(e.getMessage());
+        }
+    }
+
+    private static ServiceInstance<ServerDetails> serviceInstance(TimelyBalancedHost config) throws Exception {
+        ServerDetails serverDetails = new ServerDetails();
+        serverDetails.setHost(config.getHost());
+        serverDetails.setTcpPort(config.getTcpPort());
+        serverDetails.setHttpPort(config.getHttpPort());
+        serverDetails.setWsPort(config.getWsPort());
+        serverDetails.setUdpPort(config.getUdpPort());
+
+        ServiceInstanceBuilder<ServerDetails> builder = ServiceInstance.builder();
+        String serviceName = config.getHost() + ":" + config.getTcpPort();
+        ServiceInstance<ServerDetails> serviceInstance = builder.id(serviceName).name("timely-server")
+                .address(config.getHost()).port(config.getTcpPort()).payload(serverDetails).build();
+        return serviceInstance;
+    }
+}

balancer/src/test/java/timely/balancer/resolver/TestBalancedMetricResolver.java

@@ -0,0 +1,270 @@
+package timely.balancer.resolver;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Random;
+import java.util.UUID;
+
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.CuratorFrameworkFactory;
+import org.apache.curator.retry.RetryNTimes;
+import org.apache.zookeeper.server.ServerConfig;
+import org.apache.zookeeper.server.ZooKeeperServerMain;
+import org.apache.zookeeper.server.quorum.QuorumPeerConfig;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.util.SocketUtils;
+import timely.balancer.configuration.BalancerConfiguration;
+import timely.balancer.connection.TimelyBalancedHost;
+import timely.balancer.resolver.eventing.MetricAssignedCallback;
+import timely.balancer.resolver.eventing.MetricAssignedEvent;
+import timely.balancer.resolver.eventing.MetricBalanceCallback;
+import timely.balancer.resolver.eventing.MetricBalanceEvent;
+import timely.balancer.resolver.eventing.MetricHostCallback;
+import timely.balancer.resolver.eventing.MetricHostEvent;
+
+public class TestBalancedMetricResolver {
+
+    private final Logger LOG = LoggerFactory.getLogger(getClass());
+    private CuratorFramework curatorFramework;
+    private SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd HHmmss.SSS");
+    private Random r = new Random(System.currentTimeMillis());
+    private List<TimelyBalancedHost> timelyHosts;
+    private Map<MetricAssignedEvent, Integer> metricBalanceEventCounter = new HashMap<>();
+
+    @Before
+    public void startup() {
+        QuorumPeerConfig currentConfig = createZookeeper();
+        InetSocketAddress address = currentConfig.getClientPortAddress();
+        String zkServer = address.getHostName() + ":" + address.getPort();
+        curatorFramework = CuratorFrameworkFactory.newClient(zkServer, 1000, 5000, new RetryNTimes(2, 50));
+        curatorFramework.start();
+        timelyHosts = new ArrayList<>();
+    }
+
+    @After
+    public void shutdown() {
+        curatorFramework.close();
+    }
+
+    private QuorumPeerConfig createZookeeper() {
+        Properties startupProperties = new Properties();
+        File file = new File(System.getProperty("java.io.tmpdir") + File.separator + UUID.randomUUID());
+        file.deleteOnExit();
+        startupProperties.setProperty("dataDir", file.getAbsolutePath());
+        startupProperties.setProperty("clientPort", String.valueOf(SocketUtils.findAvailableTcpPort(3000, 4000)));
+        QuorumPeerConfig quorumConfiguration = new QuorumPeerConfig();
+
+        try {
+            quorumConfiguration.parseProperties(startupProperties);
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+
+        ZooKeeperServerMain zooKeeperServer = new ZooKeeperServerMain();
+        final ServerConfig configuration = new ServerConfig();
+        configuration.readFrom(quorumConfiguration);
+
+        new Thread() {
+
+            public void run() {
+                try {
+                    zooKeeperServer.runFromConfig(configuration);
+                } catch (IOException e) {
+                    LOG.error("ZooKeeper Failed", e);
+                }
+            }
+        }.start();
+
+        return quorumConfiguration;
+    }
+
+    private BalancerConfiguration getBalancerConfiguration() {
+        BalancerConfiguration balancerConfiguration = new BalancerConfiguration();
+        balancerConfiguration.setMetricAssignmentPersisterType("FILE");
+        File file = new File(System.getProperty("java.io.tmpdir") + File.separator + UUID.randomUUID());
+        file.deleteOnExit();
+        balancerConfiguration.setAssignmentFile(file.getAbsolutePath());
+        balancerConfiguration.setBalanceDelay(0);
+        balancerConfiguration.setBalancePeriod(1000);
+        balancerConfiguration.setPersistDelay(0);
+        balancerConfiguration.setPersistDelay(1000);
+        balancerConfiguration.setCheckServerHealthInterval(10);
+        return balancerConfiguration;
+    }
+
+    private void createTimelyHosts(BalancerConfiguration balancerConfiguration, int numHosts) {
+        for (int port = 1000, x = 0; x < numHosts; x++, port += 10) {
+            TimelyBalancedHost tbh = TimelyBalancedHost.of("localhost", port);
+            tbh.setBalancerConfig(balancerConfiguration);
+            timelyHosts.add(tbh);
+            ServiceRegistry.registerService(curatorFramework, tbh);
+        }
+    }
+
+    private void registerCallbacks(BalancedMetricResolver resolver) {
+        resolver.registerCallback(new MetricAssignedCallback() {
+
+            @Override
+            public void onEvent(MetricAssignedEvent event) {
+                LOG.debug(String.format("%s metric:%s lose:%s:%d gain:%s:%d reason:%s",
+                        event.getClass().getSimpleName(), event.getMetric(),
+                        event.getLosingHost() == null ? null : event.getLosingHost().getHost(),
+                        event.getLosingHost() == null ? null : event.getLosingHost().getTcpPort(),
+                        event.getGainingHost().getHost(), event.getGainingHost().getTcpPort(), event.getReason()));
+            }
+        });
+
+        resolver.registerCallback(new MetricHostCallback() {
+
+            @Override
+            public void onEvent(MetricHostEvent event) {
+                LOG.debug(String.format("%s host:%s:%d type:%s", event.getClass().getSimpleName(),
+                        event.getTimelyBalancedHost().getHost(), event.getTimelyBalancedHost().getTcpPort(),
+                        event.getActionType()));
+            }
+        });
+
+        resolver.registerCallback(new MetricBalanceCallback() {
+
+            @Override
+            public void onEvent(MetricBalanceEvent event) {
+                LOG.debug(String.format("%s type:%s progress:%s reassigned:%d", event.getClass().getSimpleName(),
+                        event.getBalanceType(), event.getProgressType(), event.getNumRessigned()));
+            }
+        });
+    }
+
+    private TimelyBalancedHost randomServerDown(TestHealthChecker healthChecker, int numHosts) {
+        TimelyBalancedHost tbh = timelyHosts.get(r.nextInt(numHosts));
+        LOG.debug("Marking server down:" + TimelyBalancedHost.toStringShort(tbh));
+        healthChecker.serverDown(tbh);
+        return tbh;
+    }
+
+    private TimelyBalancedHost randomServerToggle(TestHealthChecker healthChecker, int numHosts) {
+        TimelyBalancedHost tbh = timelyHosts.get(r.nextInt(numHosts));
+        LOG.debug("Marking server " + (tbh.isUp() ? "down:" : "up:") + TimelyBalancedHost.toStringShort(tbh));
+        healthChecker.serverToggle(tbh);
+        return tbh;
+    }
+
+    private TimelyBalancedHost randomServerUnregister(int numHosts) {
+        TimelyBalancedHost tbh = timelyHosts.get(r.nextInt(numHosts));
+        LOG.debug("Unregistering:" + TimelyBalancedHost.toStringShort(tbh));
+        ServiceRegistry.unregisterService(curatorFramework, tbh);
+        return tbh;
+    }
+
+    @Test
+    public void test1() {
+        LOG.debug("Configure test");
+        int numHosts = 10;
+        BalancerConfiguration balancerConfiguration = getBalancerConfiguration();
+        createTimelyHosts(balancerConfiguration, numHosts);
+        TestHealthChecker healthChecker = new TestHealthChecker(balancerConfiguration, timelyHosts);
+        BalancedMetricResolver balancedMetricResolver = new BalancedMetricResolver(curatorFramework,
+                balancerConfiguration, healthChecker);
+        registerCallbacks(balancedMetricResolver);
+        LOG.debug("Start balancedMetricResolver");
+        balancedMetricResolver.start();
+        long now = System.currentTimeMillis();
+        int numMetrics = 100;
+        for (int x = 0; x < numMetrics; x++) {
+            balancedMetricResolver.getHostPortKeyIngest(String.format("metric%03d", x));
+        }
+
+        LOG.debug("Begin monitoring");
+        long cycle = 0;
+        List<TimelyBalancedHost> serversUnregistered = new ArrayList<>();
+        while (System.currentTimeMillis() < now + 15000) {
+            try {
+                Thread.sleep(100);
+                for (int x = 0; x < 10000; x++) {
+                    int mNum = r.nextInt(numMetrics);
+                    balancedMetricResolver.getHostPortKeyIngest(String.format("metric%03d", mNum));
+                }
+                // if (cycle % 100 == 0) {
+                // randomServerDown(healthChecker, numHosts);
+                // }
+                // if (cycle % 2 == 0) {
+                // randomServerToggle(healthChecker, numHosts);
+                // }
+                if (cycle % 100 == 0) {
+                    serversUnregistered.add(randomServerUnregister(numHosts));
+                }
+                if (cycle % 500 == 0) {
+                    serversUnregistered.stream().forEach(tbh -> ServiceRegistry.registerService(curatorFramework, tbh));
+                    serversUnregistered.clear();
+                }
+
+            } catch (InterruptedException e) {
+
+            }
+            cycle++;
+        }
+        LOG.debug("Stop balancedMetricResolver");
+        balancedMetricResolver.stop();
+    }
+
+    @Test
+    public void test2() {
+        LOG.debug("Configure test");
+        int numHosts = 10;
+        BalancerConfiguration balancerConfiguration = getBalancerConfiguration();
+        createTimelyHosts(balancerConfiguration, numHosts);
+        TestHealthChecker healthChecker = new TestHealthChecker(balancerConfiguration, timelyHosts);
+        BalancedMetricResolver balancedMetricResolver = new BalancedMetricResolver(curatorFramework,
+                balancerConfiguration, healthChecker);
+        registerCallbacks(balancedMetricResolver);
+        LOG.debug("Start balancedMetricResolver");
+        balancedMetricResolver.start();
+        long now = System.currentTimeMillis();
+        int numMetrics = 100;
+        for (int x = 0; x < numMetrics; x++) {
+            balancedMetricResolver.getHostPortKeyIngest(String.format("metric%03d", x));
+        }
+
+        LOG.debug("Begin monitoring");
+        long cycle = 0;
+        List<TimelyBalancedHost> serversUnregistered = new ArrayList<>();
+        while (System.currentTimeMillis() < now + 15000) {
+            try {
+                Thread.sleep(100);
+                for (int x = 0; x < 10000; x++) {
+                    int mNum = r.nextInt(numMetrics);
+                    balancedMetricResolver.getHostPortKeyIngest(String.format("metric%03d", mNum));
+                }
+                // if (cycle % 100 == 0) {
+                // randomServerDown(healthChecker, numHosts);
+                // }
+                // if (cycle % 2 == 0) {
+                // randomServerToggle(healthChecker, numHosts);
+                // }
+                if (cycle % 100 == 0) {
+                    serversUnregistered.add(randomServerUnregister(numHosts));
+                }
+                if (cycle % 500 == 0) {
+                    serversUnregistered.stream().forEach(tbh -> ServiceRegistry.registerService(curatorFramework, tbh));
+                    serversUnregistered.clear();
+                }
+
+            } catch (InterruptedException e) {
+
+            }
+            cycle++;
+        }
+        LOG.debug("Stop balancedMetricResolver");
+        balancedMetricResolver.stop();
+    }
+}

balancer/src/test/java/timely/balancer/resolver/TestHealthChecker.java

@@ -0,0 +1,89 @@
+package timely.balancer.resolver;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import timely.balancer.configuration.BalancerConfiguration;
+import timely.balancer.connection.TimelyBalancedHost;
+import timely.balancer.healthcheck.HealthChecker;
+import timely.balancer.healthcheck.TcpHealthChecker;
+
+public class TestHealthChecker implements HealthChecker {
+
+    private final Logger LOG = LoggerFactory.getLogger(TcpHealthChecker.class);
+    private List<TimelyBalancedHost> timelyHosts = new ArrayList<>();
+    private Map<TimelyBalancedHost, Boolean> serverUpMap = Collections.synchronizedMap(new HashMap<>());
+    private ScheduledExecutorService executorService = Executors.newScheduledThreadPool(1);
+
+    public TestHealthChecker(BalancerConfiguration config, List<TimelyBalancedHost> timelyHosts) {
+        setTimelyHosts(timelyHosts);
+        this.executorService.scheduleAtFixedRate(() -> {
+            synchronized (timelyHosts) {
+                for (TimelyBalancedHost h : timelyHosts) {
+                    try {
+                        if (TestHealthChecker.this.isServerHealthy(h)) {
+                            h.reportSuccess();
+                        } else {
+                            h.reportFailure();
+                        }
+                    } catch (Exception e) {
+                        LOG.error(e.getMessage(), e);
+                        h.reportFailure();
+                    }
+                }
+            }
+        }, 0, config.getCheckServerHealthInterval(), TimeUnit.MILLISECONDS);
+    }
+
+    @Override
+    public void setTimelyHosts(List<TimelyBalancedHost> timelyHosts) {
+        synchronized (timelyHosts) {
+            this.timelyHosts.clear();
+            this.timelyHosts.addAll(timelyHosts);
+            for (TimelyBalancedHost h : timelyHosts) {
+                serverUpMap.put(h, true);
+            }
+        }
+
+    }
+
+    public boolean isServerHealthy(TimelyBalancedHost h) {
+        return serverUpMap.get(h);
+    }
+
+    public void serverUp(TimelyBalancedHost h) {
+        serverUpMap.put(h, true);
+    }
+
+    public void serverDown(TimelyBalancedHost h) {
+        serverUpMap.put(h, false);
+    }
+
+    public void serverToggle(TimelyBalancedHost h) {
+        synchronized (serverUpMap) {
+            serverUpMap.put(h, !serverUpMap.get(h));
+        }
+    }
+
+    @Override
+    public void close() throws Exception {
+        this.executorService.shutdown();
+        try {
+            this.executorService.awaitTermination(5, TimeUnit.SECONDS);
+        } catch (InterruptedException e) {
+
+        } finally {
+            if (!this.executorService.isTerminated()) {
+                this.executorService.shutdownNow();
+            }
+        }
+    }
+}

balancer/src/test/resources/log4j2.xml

@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Configuration status="WARN">
+  <Appenders>
+    <Console name="CONSOLE" target="SYSTEM_OUT">
+      <PatternLayout pattern="%d{yyyyMMdd HHmmss.SSS} [%t] %-5level %logger{36} - %msg%n"/>
+    </Console>
+  </Appenders>
+  <Loggers>
+    <Logger name="org.apache.accumulo.core.client.impl.ThriftTransportPool" level="OFF" additivity="false" />
+    <Logger name="org.apache.accumulo.fate.zookeeper.ZooCache" level="OFF" additivity="false" />
+    <Logger name="org.apache.zookeeper" level="OFF" additivity="false" />
+    <Logger name="timely.balancer.resolver" level="TRACE" additivity="true" />
+    <Logger name="timely.balancer.resolver.TestBalancedMetricResolver" level="DEBUG" additivity="true" />
+    <Root level="ERROR">
+      <AppenderRef ref="CONSOLE"/>
+    </Root>
+  </Loggers>
+</Configuration>

client/pom.xml

@@ -4,12 +4,13 @@
   <parent>
     <groupId>timely</groupId>
     <artifactId>timely</artifactId>
-    <version>1.1.1-SNAPSHOT</version>
+    <version>1.2.8-SNAPSHOT</version>
   </parent>
   <artifactId>timely-client</artifactId>
   <properties>
     <!-- PEP 440 compliant version gets substituted into src/main/python/template/setup.py -->
-    <python.client.version>1.1.1.dev0</python.client.version>
+    <!-- use .dev0 instead of -SNAPSHOT -->
+    <python.client.version>1.2.8.dev0</python.client.version>
   </properties>
   <dependencies>
     <dependency>

client/src/main/java/timely/client/http/HttpClient.java

@@ -46,20 +46,26 @@ public class HttpClient {
 
     public static CloseableHttpClient get(SSLContext ssl, CookieStore cookieStore, boolean hostVerificationEnabled,
             boolean clientAuth) {
-        RequestConfig defaultRequestConfig = RequestConfig.custom().setCookieSpec(CookieSpecs.STANDARD).build();
         HostnameVerifier hostnameVerifier;
         if (hostVerificationEnabled) {
             hostnameVerifier = SSLConnectionSocketFactory.getDefaultHostnameVerifier();
         } else {
             hostnameVerifier = new NoopHostnameVerifier();
         }
-        HttpClientBuilder builder = HttpClients.custom().setSSLContext(ssl).setDefaultCookieStore(cookieStore)
-                .setDefaultRequestConfig(defaultRequestConfig).setSSLHostnameVerifier(hostnameVerifier);
+        RequestConfig defaultRequestConfig = RequestConfig.custom().setCookieSpec(CookieSpecs.STANDARD).build();
+        HttpClientBuilder builder;
+        if (cookieStore == null) {
+            builder = HttpClients.custom().setSSLContext(ssl).disableCookieManagement()
+                    .setDefaultRequestConfig(defaultRequestConfig).setSSLHostnameVerifier(hostnameVerifier);
+        } else {
+            builder = HttpClients.custom().setSSLContext(ssl).setDefaultCookieStore(cookieStore)
+                    .setDefaultRequestConfig(defaultRequestConfig).setSSLHostnameVerifier(hostnameVerifier);
+        }
 
         if (clientAuth) {
-            SSLConnectionSocketFactory sslCF = new SSLConnectionSocketFactory(ssl,
+            SSLConnectionSocketFactory sslCSF = new SSLConnectionSocketFactory(ssl,
                     new String[] { "TLSv1.1", "TLSv1.2" }, null, hostnameVerifier);
-            builder.setSSLSocketFactory(sslCF);
+            builder.setSSLSocketFactory(sslCSF);
         }
         return builder.build();
     }

client/src/main/java/timely/client/tcp/TcpClient.java

@@ -1,7 +1,8 @@
 package timely.client.tcp;
 
+import java.io.BufferedWriter;
 import java.io.IOException;
-import java.io.PrintWriter;
+import java.io.OutputStreamWriter;
 import java.net.Socket;
 
 import org.slf4j.Logger;
@@ -14,7 +15,7 @@ public class TcpClient implements AutoCloseable {
     private final String host;
     private final int port;
     private Socket sock = null;
-    private PrintWriter out = null;
+    private BufferedWriter out = null;
     private long connectTime = 0L;
     private long backoff = 2000;
     private int bufferSize = -1;
@@ -47,7 +48,6 @@ public class TcpClient implements AutoCloseable {
      * 
      * @param metric
      *            newline terminated string representation of Timely metric
-     * @throws IOException
      */
     public synchronized void write(String metric) throws IOException {
         if (connect() != 0) {
@@ -60,7 +60,7 @@ public class TcpClient implements AutoCloseable {
         }
     }
 
-    public synchronized void flush() {
+    public synchronized void flush() throws IOException {
         if (null != out) {
             out.flush();
             writesSinceFlush = 0;
@@ -94,12 +94,12 @@ public class TcpClient implements AutoCloseable {
     }
 
     private synchronized int connect() {
-        if (null == sock || !sock.isConnected() || out.checkError()) {
+        if (null == sock || !sock.isConnected()) {
             if (System.currentTimeMillis() > (connectTime + backoff)) {
                 try {
                     connectTime = System.currentTimeMillis();
                     sock = new Socket(host, port);
-                    out = new PrintWriter(sock.getOutputStream(), false);
+                    out = new BufferedWriter(new OutputStreamWriter(sock.getOutputStream()));
                     backoff = 2000;
                     LOG.trace("Connected to Timely at {}:{}", host, port);
                 } catch (IOException e) {

client/src/main/java/timely/client/websocket/WebSocketClient.java

@@ -39,8 +39,8 @@ public class WebSocketClient implements AutoCloseable {
     private final String timelyHostname;
     private final int timelyHttpsPort;
     private final int timelyWssPort;
-    private final boolean clientAuth;
     private final boolean doLogin;
+    private final boolean clientAuth;
     private final String timelyUsername;
     private final String timelyPassword;
     private final boolean hostVerificationEnabled;

client/src/main/java/timely/model/ObjectSizeOf.java

@@ -62,7 +62,6 @@ public interface ObjectSizeOf {
          * ObjectSizeOf interface and if implemented on the object will use that.
          * Otherwise it will do a simple navigation of the fields using reflection.
          *
-         * @param o
          * @return an approximation of the object size
          */
         public static long getObjectSize(Object o, boolean useSizeInBytesMethod, boolean debug) {

client/src/main/java/timely/model/Tag.java

@@ -77,8 +77,6 @@ public class Tag implements Comparable<Tag>, Serializable {
 
     /**
      * Joins key and value into key=value
-     * 
-     * @return
      */
     public String join() {
         return tagParser.combine(this);

client/src/main/java/timely/model/parse/TagListParser.java

@@ -47,7 +47,6 @@ public class TagListParser implements Parser<List<Tag>>, Combiner<Collection<Tag
      * An alternative combiner for a map of string to string instead of a list of
      * tags
      * 
-     * @param tags
      * @return string representation
      */
     public String combine(Map<String, String> tags) {

client/src/main/python/timely/DataOperations.py

@@ -23,7 +23,7 @@ def pivot(df, metric, groupByColumn=None):
     return dataFrame
 
 
-def resample(df, sample, how='mean'):
+def resample(df, sample, how='mean', interpolate=True, fill=None):
 
     dataFrame = pandas.DataFrame(df, copy=True)
     if dataFrame is not None:

client/src/main/python/timely/TimelyMetric.py

@@ -16,14 +16,15 @@ utc = UTC()
 class TimelyWebSocketClient(WebSocketClient):
 
     def __init__(self, hostport, metric, tags, startTime, endTime, connect_timeout=WebSocketClient.DEFAULT_CONNECT_TIMEOUT,
-                 request_timeout=WebSocketClient.DEFAULT_REQUEST_TIMEOUT):
+                 request_timeout=WebSocketClient.DEFAULT_REQUEST_TIMEOUT, client_cert=None, client_key=None, ca_certs=None, validate_cert=False):
 
         self.metric = metric
         self.tags = tags
         self.startTime = startTime
         self.endTime = endTime
         self.subscriptionId = random.randint(1000000000, 9999999999)
-        WebSocketClient.__init__(self, hostport, '/websocket', {'metric' : metric}, connect_timeout, request_timeout)
+        WebSocketClient.__init__(self, hostport, '/websocket', {'metric' : metric}, connect_timeout, request_timeout,
+                                 client_cert=client_cert, client_key=client_key, ca_certs=ca_certs, validate_cert=validate_cert)
 
     def _on_message(self, msg):
         # implement in subclass
@@ -72,7 +73,8 @@ class TimelyMetric(TimelyWebSocketClient):
     client = None
     endtime = None
 
-    def __init__(self, hostport, metric, tags, beginStr, endStr, period, sample):
+    def __init__(self, hostport, metric, tags, beginStr, endStr, period, sample,
+                 client_cert=None, client_key=None, ca_certs=None, validate_cert=False):
         self.series = pandas.Series()
         self.metric = metric
         self.tags = tags
@@ -82,7 +84,8 @@ class TimelyMetric(TimelyWebSocketClient):
         self.data = []
         self.timeDateRange = TimeDateRange(beginStr, endStr, period)
 
-        TimelyWebSocketClient.__init__(self, hostport, metric, tags, self.timeDateRange.getBeginMs(), self.timeDateRange.getEndMs())
+        TimelyWebSocketClient.__init__(self, hostport, metric, tags, self.timeDateRange.getBeginMs(), self.timeDateRange.getEndMs(),
+                                       client_cert=client_cert, client_key=client_key, ca_certs=ca_certs, validate_cert=validate_cert)
 
     def debugOn(self):
         self.debug = True

client/src/main/python/timely/WebSocketClient.py

@@ -15,10 +15,14 @@ class WebSocketClient():
     APPLICATION_JSON = 'application/json'
 
     def __init__(self, hostport, path, params, connect_timeout=DEFAULT_CONNECT_TIMEOUT,
-                 request_timeout=DEFAULT_REQUEST_TIMEOUT):
+                 request_timeout=DEFAULT_REQUEST_TIMEOUT, client_cert=None, client_key=None, ca_certs=None, validate_cert=False):
 
         self.connect_timeout = connect_timeout
         self.request_timeout = request_timeout
+        self.client_cert=client_cert
+        self.client_key=client_key
+        self.ca_certs=ca_certs
+        self.validate_cert=validate_cert
         self.url = 'wss://' + hostport
         if (path is not None and len(path) > 0):
             if (path.startswith('/')):
@@ -39,7 +43,8 @@ class WebSocketClient():
         """Connect to the server.
         :param str url: server URL.
         """
-        request = websocket.httpclient.HTTPRequest(self.url, connect_timeout=self.connect_timeout, request_timeout=self.request_timeout, validate_cert=False)
+        request = websocket.httpclient.HTTPRequest(self.url, connect_timeout=self.connect_timeout, request_timeout=self.request_timeout, validate_cert=self.validate_cert,
+                                                   client_cert=self.client_cert, client_key=self.client_key, ca_certs=self.ca_certs)
         websocket.websocket_connect(request, io_loop=ioloop, callback=self._connect_callback)
 
     def send(self, data):

client/src/main/python/timely/sampleAnalytic.py

@@ -5,7 +5,9 @@ import os
 from timely.TimelyAnalyticConfiguration import TimelyAnalyticConfiguration
 
 hostport = "127.0.0.1:5244"
-timelyMetric = timely.TimelyMetric.TimelyMetric(hostport, 'timely.keys.metric.inserted', None, None, None, '24 hours', None).fetch()
+client_cert = None
+client_key = None
+timelyMetric = timely.TimelyMetric.TimelyMetric(hostport, 'timely.keys.metric.inserted', None, None, None, '24 hours', None, client_cert=client_cert, client_key=client_key).fetch()
 
 print(timelyMetric.getDataFrame())
 exit()

client/src/main/spotbugs/excludes.xml

@@ -5,4 +5,9 @@
     <Method name="connect" />
     <Class name="timely.client.tcp.TcpClient"/>
   </Match>
+  <Match>
+    <!-- Must ignore these everywhere, because of a javac byte code generation bug -->
+    <!-- https://github.com/spotbugs/spotbugs/issues/756 -->
+    <Bug pattern="RCN_REDUNDANT_NULLCHECK_WOULD_HAVE_BEEN_A_NPE" />
+  </Match>
 </FindBugsFilter>

client/src/test/java/timely/api/SerializationTest.java

@@ -47,9 +47,9 @@ public class SerializationTest {
         testSerialization(remove);
     }
 
-    @SuppressWarnings("unchecked")
     private <T> void testSerialization(T o) throws Exception {
         String ser = mapper.writeValueAsString(o);
+        @SuppressWarnings("unchecked")
         T des = mapper.readValue(ser, (Class<T>) o.getClass());
         Assert.assertEquals(o, des);
     }

collectd-nsq-plugin/pom.xml

@@ -4,7 +4,7 @@
   <parent>
     <groupId>timely</groupId>
     <artifactId>timely</artifactId>
-    <version>1.1.1-SNAPSHOT</version>
+    <version>1.2.8-SNAPSHOT</version>
   </parent>
   <artifactId>collectd-nsq-plugin</artifactId>
   <dependencies>

collectd-nsq-plugin/src/main/java/timely/collectd/plugin/PooledCloseableHttpClientFactory.java

@@ -10,7 +10,7 @@ import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.impl.client.HttpClientBuilder;
 import org.collectd.api.OConfigItem;
 
-public class PooledCloseableHttpClientFactory implements PooledObjectFactory {
+public class PooledCloseableHttpClientFactory implements PooledObjectFactory<CloseableHttpClient> {
 
     private int connectionRequestTimeout = 10000; // 10 seconds in milliseconds
     private int connectTimeout = 30000; // 30 seconds in milliseconds
@@ -43,19 +43,19 @@ public class PooledCloseableHttpClientFactory implements PooledObjectFactory {
     }
 
     @Override
-    public PooledObject makeObject() throws Exception {
+    public PooledObject<CloseableHttpClient> makeObject() throws Exception {
         RequestConfig requestConfig = RequestConfig.custom().setConnectionRequestTimeout(connectionRequestTimeout)
                 .setConnectTimeout(connectTimeout).setSocketTimeout(socketTimeout).build();
         HttpClientBuilder builder = HttpClientBuilder.create();
         builder.setDefaultRequestConfig(requestConfig);
         builder.setConnectionTimeToLive(connectionTimeToLive, TimeUnit.MILLISECONDS);
-        return new DefaultPooledObject(builder.build());
+        return new DefaultPooledObject<>(builder.build());
     }
 
     @Override
-    public void destroyObject(PooledObject pooledObject) throws Exception {
+    public void destroyObject(PooledObject<CloseableHttpClient> pooledObject) throws Exception {
         try {
-            ((CloseableHttpClient) pooledObject.getObject()).close();
+            pooledObject.getObject().close();
         } catch (Exception e) {
             // do nothing
         } finally {
@@ -65,17 +65,17 @@ public class PooledCloseableHttpClientFactory implements PooledObjectFactory {
     }
 
     @Override
-    public boolean validateObject(PooledObject pooledObject) {
+    public boolean validateObject(PooledObject<CloseableHttpClient> pooledObject) {
         return pooledObject.getObject() != null;
     }
 
     @Override
-    public void activateObject(PooledObject pooledObject) throws Exception {
+    public void activateObject(PooledObject<CloseableHttpClient> pooledObject) throws Exception {
         // do nothing
     }
 
     @Override
-    public void passivateObject(PooledObject pooledObject) throws Exception {
+    public void passivateObject(PooledObject<CloseableHttpClient> pooledObject) throws Exception {
         // do nothing
     }
 

collectd-nsq-plugin/src/main/java/timely/collectd/plugin/WriteNSQPlugin.java

@@ -47,6 +47,7 @@ public class WriteNSQPlugin extends CollectDPluginParent
         Collectd.registerWrite(WriteNSQPlugin.class.getName(), this);
     }
 
+    @Override
     public int write(ValueList vl) {
         ByteArrayOutputStream baos = new ByteArrayOutputStream();
         DataOutputStream out = new DataOutputStream(baos);
@@ -110,15 +111,17 @@ public class WriteNSQPlugin extends CollectDPluginParent
         }
     }
 
+    @Override
     public int shutdown() {
         Collectd.logInfo("Shutting down connection to NSQ at " + host + ":" + port);
         clientPool.close();
         return 0;
     }
 
+    @Override
     public int config(OConfigItem config) {
         super.config(config);
-        GenericObjectPoolConfig poolConfig = new GenericObjectPoolConfig();
+        GenericObjectPoolConfig<CloseableHttpClient> poolConfig = new GenericObjectPoolConfig<>();
         // use max size for maxTotal and maxIdle
         // no need to activate or passivate
         poolConfig.setMaxTotal(POOL_MAX_SIZE);
@@ -156,7 +159,7 @@ public class WriteNSQPlugin extends CollectDPluginParent
             Collectd.logError("NSQ host, port, and topic must be configured");
             return -1;
         }
-        this.clientPool = new GenericObjectPool(pooledCloseableHttpClientFactory, poolConfig);
+        this.clientPool = new GenericObjectPool<>(pooledCloseableHttpClientFactory, poolConfig);
         this.endpoint = "http://" + host + ":" + port + "/mpub?topic=" + topic;
         return 0;
     }

collectd-plugin-parent/pom.xml

@@ -4,7 +4,7 @@
   <parent>
     <groupId>timely</groupId>
     <artifactId>timely</artifactId>
-    <version>1.1.1-SNAPSHOT</version>
+    <version>1.2.8-SNAPSHOT</version>
   </parent>
   <artifactId>collectd-plugin-parent</artifactId>
   <dependencies>

collectd-timely-plugin/pom.xml

@@ -4,7 +4,7 @@
   <parent>
     <groupId>timely</groupId>
     <artifactId>timely</artifactId>
-    <version>1.1.1-SNAPSHOT</version>
+    <version>1.2.8-SNAPSHOT</version>
   </parent>
   <artifactId>collectd-timely-plugin</artifactId>
   <dependencies>

collectd-timely-plugin/src/main/java/timely/collectd/plugin/PooledSocketFactory.java

@@ -8,7 +8,7 @@ import org.apache.commons.pool2.impl.DefaultPooledObject;
 import org.collectd.api.Collectd;
 import org.collectd.api.OConfigItem;
 
-public class PooledSocketFactory implements PooledObjectFactory {
+public class PooledSocketFactory implements PooledObjectFactory<Socket> {
 
     private String host = null;
     private int port = 0;
@@ -81,12 +81,12 @@ public class PooledSocketFactory implements PooledObjectFactory {
     }
 
     @Override
-    public PooledObject makeObject() throws Exception {
-        return new DefaultPooledObject(connect());
+    public PooledObject<Socket> makeObject() throws Exception {
+        return new DefaultPooledObject<>(connect());
     }
 
     @Override
-    public void destroyObject(PooledObject pooledObject) throws Exception {
+    public void destroyObject(PooledObject<Socket> pooledObject) throws Exception {
         try {
             Collectd.logDebug("Shutting down connection to Timely at " + host + ":" + port);
             Socket socket = (Socket) pooledObject.getObject();
@@ -103,7 +103,7 @@ public class PooledSocketFactory implements PooledObjectFactory {
     }
 
     @Override
-    public boolean validateObject(PooledObject pooledObject) {
+    public boolean validateObject(PooledObject<Socket> pooledObject) {
         if (connectionTimeToLive > -1) {
             if (System.currentTimeMillis() > pooledObject.getCreateTime() + connectionTimeToLive) {
                 return false;
@@ -114,12 +114,12 @@ public class PooledSocketFactory implements PooledObjectFactory {
     }
 
     @Override
-    public void activateObject(PooledObject pooledObject) throws Exception {
+    public void activateObject(PooledObject<Socket> pooledObject) throws Exception {
         // do nothing
     }
 
     @Override
-    public void passivateObject(PooledObject pooledObject) throws Exception {
+    public void passivateObject(PooledObject<Socket> pooledObject) throws Exception {
         // do nothing
     }
 }

collectd-timely-plugin/src/main/java/timely/collectd/plugin/WriteTimelyPlugin.java

@@ -86,13 +86,13 @@ public class WriteTimelyPlugin extends CollectDPluginParent
         PooledSocketFactory socketFactory = new PooledSocketFactory();
         retval = socketFactory.config(config);
         if (retval == 0) {
-            GenericObjectPoolConfig poolConfig = new GenericObjectPoolConfig();
+            GenericObjectPoolConfig<Socket> poolConfig = new GenericObjectPoolConfig<>();
             // use max size for maxTotal and maxIdle
             // no need to activate and passivate
             poolConfig.setMaxTotal(POOL_MAX_SIZE);
             poolConfig.setMaxIdle(POOL_MAX_SIZE);
             poolConfig.setTestOnReturn(true);
-            socketPool = new GenericObjectPool(socketFactory, poolConfig);
+            socketPool = new GenericObjectPool<>(socketFactory, poolConfig);
         }
         return retval;
     }

common/pom.xml

@@ -4,7 +4,7 @@
   <parent>
     <groupId>timely</groupId>
     <artifactId>timely</artifactId>
-    <version>1.1.1-SNAPSHOT</version>
+    <version>1.2.8-SNAPSHOT</version>
   </parent>
   <artifactId>timely-common</artifactId>
   <dependencies>

grafana-auth/bin/grafana-auth.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+
+if [[ `uname` == "Darwin" ]]; then
+        THIS_SCRIPT=`python -c 'import os,sys; print os.path.realpath(sys.argv[1])' $0`
+        TCNATIVE_SUFFIX="jnilib"
+else
+        THIS_SCRIPT=`readlink -f $0`
+        TCNATIVE_SUFFIX="so"
+fi
+
+THIS_DIR="${THIS_SCRIPT%/*}"
+NATIVE_DIR="${THIS_DIR}/META-INF/native"
+BASE_DIR=${THIS_DIR}/..
+TMP_DIR="${BASE_DIR}/tmp"
+CONF_DIR="${BASE_DIR}/conf"
+LIB_DIR="${BASE_DIR}/lib"
+NUM_SERVER_THREADS=4
+
+if [[ -e ${TMP_DIR} ]]; then
+  rm -rf ${TMP_DIR}
+fi
+mkdir ${TMP_DIR}
+
+if [[ -e ${NATIVE_DIR} ]]; then
+  rm -rf ${NATIVE_DIR}
+fi
+mkdir -p ${NATIVE_DIR}
+
+pushd ${BASE_DIR}/bin
+$JAVA_HOME/bin/jar xf ${LIB_DIR}/netty-tcnative-boringssl-static*.jar META-INF/native/libnetty_tcnative_linux_x86_64.${TCNATIVE_SUFFIX}
+$JAVA_HOME/bin/jar xf ${LIB_DIR}/netty-all*.jar META-INF/native/libnetty_transport_native_epoll_x86_64.${TCNATIVE_SUFFIX}
+popd
+
+export CLASSPATH="${CONF_DIR}:${LIB_DIR}/*:${HADOOP_CONF_DIR}"
+JVM_ARGS="-Xmx1G -Xms1G -Dio.netty.eventLoopThreads=${NUM_SERVER_THREADS} Dlog4j.configurationFile=${CONF_DIR}/log4j2-spring.xml"
+JVM_ARGS="${JVM_ARGS} -DLog4jContextSelector=org.apache.logging.log4j.core.async.AsyncLoggerContextSelector"
+JVM_ARGS="${JVM_ARGS} -Djava.library.path=${NATIVE_DIR}"
+
+#JVM_ARGS="${JVM_ARGS} -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5005"
+
+echo "$JAVA_HOME/bin/java ${JVM_ARGS} timely.grafana.auth.GrafanaAuth --spring.config.name=grafana-auth"
+$JAVA_HOME/bin/java ${JVM_ARGS} timely.grafana.auth.GrafanaAuth --spring.config.name=grafana-auth

grafana-auth/conf/grafana-auth.yml

@@ -0,0 +1,55 @@
+grafana-auth:
+  grafana:
+    host: localhost
+    port: 3001
+  security:
+    session-max-age: 86400
+    cache-expiration-minutes: 60
+    cache-refresh-minutes: 5
+    allow-anonymous-http-access: true
+    allow-anonymous-ws-access: true
+    serverSsl:
+      certificate-file:
+      key-file:
+      #key-password:
+      use-generated-keypair: true
+      trust-store-file:
+      use-openssl: true
+      #use-ciphers:
+    clientSsl:
+      keyFile:
+      keyType: pkcs12
+      #keyPassword:
+      trustStoreFile:
+      trustStoreType: jks
+      #trustStorePassword:
+  http:
+    host: localhost
+    ip: 127.0.0.1
+    port: 3000
+    redirect-path: /secure-me
+    strict-transport-max-age: 608400
+    cors:
+      allow-any-origin: false
+      allow-null-origin: false
+      allow-credentials: true
+      allowed-origins:
+      allowed-methods: DELETE,GET,HEAD,OPTIONS,PUT,POST
+      allowed-headers: content-type
+    httpClientPool:
+      minIdlePerKey: 0
+      maxIdlePerKey: 40
+      maxTotalPerKey: 40
+      maxTotal: -1
+      lifo: true
+      fairness: false
+      maxWaitMillis: -1
+      minEvictableIdleTimeMillis: 1800000
+      evictorShutdownTimeoutMillis: 10000
+      softMinEvictableIdleTimeMillis: -1
+      numTestsPerEvictionRun: 3
+      timeBetweenEvictionRunsMillis: -1
+      blockWhenExhausted: true
+      jmxEnabled: true
+      testOnBorrow: false
+      testOnReturn: false

grafana-auth/conf/log4j2-error-console.xml

@@ -0,0 +1,13 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Configuration status="warn" monitorInterval="60">
+  <Appenders>
+    <Console name="Console" target="SYSTEM_OUT">
+      <PatternLayout pattern="%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n"/>
+    </Console>
+  </Appenders>
+  <Loggers>
+    <Root level="error" includeLocation="false">
+      <AppenderRef ref="Console" level="error" />
+    </Root>
+  </Loggers>
+</Configuration>

grafana-auth/conf/log4j2-spring.xml

@@ -0,0 +1,29 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Configuration status="warn" monitorInterval="60">
+  <Properties>
+    <Property name="PID">????</Property>
+    <Property name="LOG_PATTERN">%clr{%d{yyyy-MM-dd HH:mm:ss.SSS}}{faint} %clr{%5p} %clr{${sys:PID}}{magenta} %clr{---}{faint} %clr{[%15.15t]}{faint} %clr{%-40.40c{1.}}{cyan} %clr{:}{faint} %m%n%wEx</Property>
+  </Properties>
+  <Appenders>
+    <RollingRandomAccessFile name="Rolling" fileName="/srv/logs/timely/grafana-auth.log"
+    						 filePattern="/srv/logs/timely/$${date:yyyy-MM}/grafana-auth-%d{MM-dd-yyyy}-%i.log.gz">
+      <PatternLayout pattern="${LOG_PATTERN}"/>
+      <Policies>
+      	<SizeBasedTriggeringPolicy size="1024 MB" />
+      </Policies>
+      <DefaultRolloverStrategy max="10" />
+    </RollingRandomAccessFile>
+    <Console name="Console" target="SYSTEM_OUT">
+      <PatternLayout pattern="${LOG_PATTERN}"/>
+    </Console>
+  </Appenders>
+  <Loggers>
+    <Root level="trace" includeLocation="false">
+      <AppenderRef ref="Console" level="warn" />
+      <AppenderRef ref="Rolling" level="trace" />
+    </Root>
+    <Logger name="timely" additivity="false">
+      <AppenderRef ref="Console" level="debug" />
+    </Logger>
+  </Loggers>
+</Configuration>

grafana-auth/conf/security.xml

@@ -0,0 +1,71 @@
+<beans:beans xmlns="http://www.springframework.org/schema/security"
+             xmlns:beans="http://www.springframework.org/schema/beans"
+             xmlns:util="http://www.springframework.org/schema/util"
+             xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+             xsi:schemaLocation="http://www.springframework.org/schema/beans
+								 http://www.springframework.org/schema/beans/spring-beans-3.0.xsd
+								 http://www.springframework.org/schema/security
+								 http://www.springframework.org/schema/security/spring-security.xsd
+								 http://www.springframework.org/schema/util
+								 http://www.springframework.org/schema/util/spring-util.xsd">
+		          
+	<beans:bean id="x509PrincipalExtractor" class="org.springframework.security.web.authentication.preauth.x509.SubjectDnX509PrincipalExtractor">
+		<beans:property name="subjectDnRegex" value="CN=(.*?)(?:,|$)"/>
+	</beans:bean>
+
+	<beans:bean id="timelyUserDetailsService" class="timely.auth.UserDetailsService">
+		<beans:property name="users">
+			<util:list>
+				<beans:bean class="timely.auth.TimelyUser">
+					<beans:constructor-arg name="subjectDn">
+						<beans:value>CN=example.com</beans:value>
+					</beans:constructor-arg>
+					<beans:constructor-arg name="auths">
+						<util:list>
+							<beans:value>D</beans:value>
+							<beans:value>E</beans:value>
+							<beans:value>F</beans:value>
+						</util:list>
+					</beans:constructor-arg>
+					<beans:constructor-arg name="roles">
+						<util:list>
+							<beans:value>G</beans:value>
+							<beans:value>H</beans:value>
+							<beans:value>I</beans:value>
+						</util:list>
+					</beans:constructor-arg>
+				</beans:bean>
+			</util:list>
+		</beans:property>
+	</beans:bean>
+    
+    <beans:bean id="x509AuthorizationProvider" class="org.springframework.security.web.authentication.preauth.PreAuthenticatedAuthenticationProvider">
+    	<beans:property name="preAuthenticatedUserDetailsService" ref="timelyUserDetailsService" />
+    </beans:bean>
+    
+    <user-service id="usernamePasswordService">
+    	<user name="test" password="test1" authorities="A,B,C" />
+    </user-service>
+
+	<beans:bean id="noOpPasswordEncoder" class="org.springframework.security.crypto.password.NoOpPasswordEncoder"/>
+
+	<beans:bean id="UserNamePasswordAuthenticationProvider" class="org.springframework.security.authentication.dao.DaoAuthenticationProvider">
+		<beans:property name="userDetailsService" ref="usernamePasswordService"/>
+		<beans:property name="passwordEncoder" ref="noOpPasswordEncoder"/>
+	</beans:bean>
+	    
+	<authentication-manager alias="authenticationManager">
+		<!-- X509 PreAuthenticated authorization provider -->
+		<authentication-provider ref="x509AuthorizationProvider"/>
+		<!-- Basic authorization provider, validates provided username and password -->
+		<authentication-provider ref="UserNamePasswordAuthenticationProvider" />
+	</authentication-manager>
+
+	<util:list id="requiredRoles" value-type="java.lang.String">
+		<beans:value></beans:value>
+	</util:list>
+
+	<util:list id="requiredAuths" value-type="java.lang.String">
+		<beans:value></beans:value>
+	</util:list>
+</beans:beans>

grafana-auth/pom.xml

@@ -0,0 +1,295 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>timely</groupId>
+    <artifactId>timely</artifactId>
+    <version>1.2.8-SNAPSHOT</version>
+  </parent>
+  <artifactId>grafana-auth</artifactId>
+  <dependencies>
+    <dependency>
+      <groupId>com.fasterxml.jackson.core</groupId>
+      <artifactId>jackson-annotations</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>com.fasterxml.jackson.core</groupId>
+      <artifactId>jackson-core</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>com.fasterxml.jackson.core</groupId>
+      <artifactId>jackson-databind</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>com.fasterxml.jackson.datatype</groupId>
+      <artifactId>jackson-datatype-jdk8</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>com.github.ben-manes.caffeine</groupId>
+      <artifactId>caffeine</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>com.google.flatbuffers</groupId>
+      <artifactId>flatbuffers-java</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>commons-beanutils</groupId>
+      <artifactId>commons-beanutils</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>io.github.lukehutch</groupId>
+      <artifactId>fast-classpath-scanner</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>io.netty</groupId>
+      <artifactId>netty-all</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>net.sourceforge.javacsv</groupId>
+      <artifactId>javacsv</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.accumulo</groupId>
+      <artifactId>accumulo-core</artifactId>
+      <exclusions>
+        <exclusion>
+          <groupId>org.slf4j</groupId>
+          <artifactId>slf4j-log4j12</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.accumulo</groupId>
+      <artifactId>accumulo-minicluster</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.commons</groupId>
+      <artifactId>commons-collections4</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.commons</groupId>
+      <artifactId>commons-lang3</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.commons</groupId>
+      <artifactId>commons-pool2</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.curator</groupId>
+      <artifactId>curator-client</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.curator</groupId>
+      <artifactId>curator-framework</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.curator</groupId>
+      <artifactId>curator-recipes</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.curator</groupId>
+      <artifactId>curator-x-discovery</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-client</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-common</artifactId>
+      <exclusions>
+        <exclusion>
+          <groupId>jdk.tools</groupId>
+          <artifactId>jdk.tools</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-minicluster</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-api</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.springframework</groupId>
+      <artifactId>spring-websocket</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.springframework.boot</groupId>
+      <artifactId>spring-boot-starter</artifactId>
+      <exclusions>
+        <exclusion>
+          <groupId>org.springframework.boot</groupId>
+          <artifactId>spring-boot-starter-logging</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+    <dependency>
+      <groupId>org.springframework.boot</groupId>
+      <artifactId>spring-boot-starter-log4j2</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.springframework.boot</groupId>
+      <artifactId>spring-boot-starter-validation</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.springframework.security</groupId>
+      <artifactId>spring-security-config</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.springframework.security</groupId>
+      <artifactId>spring-security-web</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>timely</groupId>
+      <artifactId>timely-balancer</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>timely</groupId>
+      <artifactId>timely-client</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>timely</groupId>
+      <artifactId>timely-common</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>timely</groupId>
+      <artifactId>timely-server</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>com.lmax</groupId>
+      <artifactId>disruptor</artifactId>
+      <scope>runtime</scope>
+    </dependency>
+    <dependency>
+      <groupId>io.netty</groupId>
+      <artifactId>netty-tcnative-boringssl-static</artifactId>
+      <scope>runtime</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.logging.log4j</groupId>
+      <artifactId>log4j-api</artifactId>
+      <scope>runtime</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.logging.log4j</groupId>
+      <artifactId>log4j-core</artifactId>
+      <scope>runtime</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.logging.log4j</groupId>
+      <artifactId>log4j-slf4j-impl</artifactId>
+      <scope>runtime</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.javassist</groupId>
+      <artifactId>javassist</artifactId>
+      <scope>runtime</scope>
+    </dependency>
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.jsoup</groupId>
+      <artifactId>jsoup</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.springframework.boot</groupId>
+      <artifactId>spring-boot-test-autoconfigure</artifactId>
+      <scope>test</scope>
+    </dependency>
+  </dependencies>
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-assembly-plugin</artifactId>
+        <executions>
+          <execution>
+            <id>assemble</id>
+            <goals>
+              <goal>single</goal>
+            </goals>
+            <phase>package</phase>
+            <configuration>
+              <descriptors>
+                <descriptor>src/main/dist/assembly.xml</descriptor>
+              </descriptors>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-dependency-plugin</artifactId>
+        <executions>
+          <execution>
+            <id>copy-dependencies</id>
+            <goals>
+              <goal>copy-dependencies</goal>
+            </goals>
+            <phase>prepare-package</phase>
+            <configuration>
+              <excludeArtifactIds>junit</excludeArtifactIds>
+              <outputDirectory>${project.build.directory}/lib</outputDirectory>
+            </configuration>
+          </execution>
+          <execution>
+            <id>unpack-native</id>
+            <goals>
+              <goal>unpack</goal>
+            </goals>
+            <phase>process-test-classes</phase>
+            <configuration>
+              <artifactItems>
+                <artifactItem>
+                  <groupId>io.netty</groupId>
+                  <artifactId>netty-tcnative-boringssl-static</artifactId>
+                  <version>${version.netty-tcnative}</version>
+                  <type>jar</type>
+                  <overWrite>false</overWrite>
+                  <outputDirectory>${project.build.directory}/native</outputDirectory>
+                  <includes>**/*.so</includes>
+                </artifactItem>
+              </artifactItems>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-failsafe-plugin</artifactId>
+      </plugin>
+      <plugin>
+        <groupId>org.codehaus.mojo</groupId>
+        <artifactId>build-helper-maven-plugin</artifactId>
+        <executions>
+          <execution>
+            <id>add-source</id>
+            <goals>
+              <goal>add-source</goal>
+            </goals>
+            <phase>generate-sources</phase>
+            <configuration>
+              <sources>
+                <source>src/main/generated-java</source>
+              </sources>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+    </plugins>
+    <extensions>
+      <!-- Use os-maven-plugin to initialize the "os.detected" properties -->
+      <extension>
+        <groupId>kr.motd.maven</groupId>
+        <artifactId>os-maven-plugin</artifactId>
+      </extension>
+    </extensions>
+  </build>
+</project>

grafana-auth/src/main/dist/assembly.xml

@@ -0,0 +1,55 @@
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3" 
+  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3 http://maven.apache.org/xsd/assembly-1.1.3.xsd">
+  <id>dist</id>
+  <formats>
+    <format>tar.gz</format>
+  </formats>
+  <fileSets>
+    <fileSet>
+      <directory>${project.basedir}</directory>
+      <outputDirectory></outputDirectory>
+      <includes>
+        <include>README*</include>
+        <include>LICENSE*</include>
+        <include>NOTICE*</include>
+      </includes>
+    </fileSet>
+    <fileSet>
+      <directory>${project.build.directory}</directory>
+      <outputDirectory>lib</outputDirectory>
+      <includes>
+        <include>timely*.jar</include>
+        <include>grafana*.jar</include>
+      </includes>
+    </fileSet>
+    <fileSet>
+      <directory>${project.build.directory}/lib</directory>
+      <outputDirectory>lib</outputDirectory>
+      <includes>
+        <include>*.jar</include>
+      </includes>
+     </fileSet>
+    <fileSet>
+      <directory>${project.basedir}/bin</directory>
+      <outputDirectory>bin</outputDirectory>
+      <includes>
+        <include>**/*</include>
+      </includes>
+     </fileSet>
+    <fileSet>
+      <directory>${project.basedir}/conf</directory>
+      <outputDirectory>conf</outputDirectory>
+      <includes>
+        <include>*</include>
+      </includes>
+     </fileSet>
+	<fileSet>
+	  <directory>${project.basedir}/src/main/resources/webapp</directory>
+	  <outputDirectory>bin/webapp</outputDirectory>
+	  <includes>
+		<include>*</include>
+	  </includes>
+	</fileSet>
+  </fileSets>
+</assembly>

grafana-auth/src/main/java/timely/grafana/auth/GrafanaAuth.java

@@ -0,0 +1,322 @@
+package timely.grafana.auth;
+
+import java.io.File;
+import java.net.InetSocketAddress;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+
+import javax.net.ssl.SSLContext;
+
+import io.netty.bootstrap.ServerBootstrap;
+import io.netty.buffer.PooledByteBufAllocator;
+import io.netty.channel.Channel;
+import io.netty.channel.ChannelFuture;
+import io.netty.channel.ChannelHandler;
+import io.netty.channel.ChannelInitializer;
+import io.netty.channel.ChannelOption;
+import io.netty.channel.EventLoopGroup;
+import io.netty.channel.epoll.EpollEventLoopGroup;
+import io.netty.channel.epoll.EpollServerSocketChannel;
+import io.netty.channel.nio.NioEventLoopGroup;
+import io.netty.channel.socket.ServerSocketChannel;
+import io.netty.channel.socket.SocketChannel;
+import io.netty.channel.socket.nio.NioServerSocketChannel;
+import io.netty.handler.codec.http.HttpContentCompressor;
+import io.netty.handler.codec.http.HttpContentDecompressor;
+import io.netty.handler.codec.http.HttpObjectAggregator;
+import io.netty.handler.codec.http.HttpRequestDecoder;
+import io.netty.handler.codec.http.HttpResponseEncoder;
+import io.netty.handler.logging.LoggingHandler;
+import io.netty.handler.ssl.ClientAuth;
+import io.netty.handler.ssl.OpenSslServerContext;
+import io.netty.handler.ssl.OpenSslServerSessionContext;
+import io.netty.handler.ssl.SslContext;
+import io.netty.handler.ssl.SslContextBuilder;
+import io.netty.handler.ssl.SslProvider;
+import io.netty.handler.ssl.util.SelfSignedCertificate;
+import io.netty.handler.stream.ChunkedWriteHandler;
+import io.netty.util.internal.SystemPropertyUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.boot.WebApplicationType;
+import org.springframework.boot.builder.SpringApplicationBuilder;
+import org.springframework.context.ConfigurableApplicationContext;
+import timely.auth.AuthCache;
+import timely.balancer.configuration.ClientSsl;
+import timely.balancer.connection.http.HttpClientPool;
+import timely.client.http.HttpClient;
+import timely.configuration.ServerSsl;
+import timely.grafana.auth.configuration.GrafanaAuthConfiguration;
+import timely.grafana.auth.configuration.SpringBootstrap;
+import timely.grafana.auth.netty.http.GrafanaRelayHandler;
+import timely.grafana.auth.netty.http.GrafanaRequestDecoder;
+import timely.netty.http.HttpStaticFileServerHandler;
+import timely.netty.http.NonSslRedirectHandler;
+import timely.netty.http.TimelyExceptionHandler;
+import timely.netty.http.auth.X509LoginRequestHandler;
+
+public class GrafanaAuth {
+
+    private static final Logger LOG = LoggerFactory.getLogger(GrafanaAuth.class);
+    private static final int EPOLL_MIN_MAJOR_VERSION = 2;
+    private static final int EPOLL_MIN_MINOR_VERSION = 6;
+    private static final int EPOLL_MIN_PATCH_VERSION = 32;
+    private static final String OS_NAME = "os.name";
+    private static final String OS_VERSION = "os.version";
+    protected static final CountDownLatch LATCH = new CountDownLatch(1);
+    static ConfigurableApplicationContext applicationContext;
+
+    private final GrafanaAuthConfiguration grafanaAuthConfig;
+    private EventLoopGroup httpWorkerGroup = null;
+    private EventLoopGroup httpBossGroup = null;
+    protected Channel httpChannelHandle = null;
+    protected volatile boolean shutdown = false;
+
+    public GrafanaAuth(GrafanaAuthConfiguration grafanaAuthConfig) throws Exception {
+        this.grafanaAuthConfig = grafanaAuthConfig;
+    }
+
+    protected static ConfigurableApplicationContext initializeContext(String[] args) {
+        SpringApplicationBuilder builder = new SpringApplicationBuilder(SpringBootstrap.class);
+        builder.web(WebApplicationType.NONE);
+        builder.registerShutdownHook(false);
+        return builder.run(args);
+    }
+
+    private void shutdownHook() {
+
+        final Runnable shutdownRunner = () -> {
+            if (!shutdown) {
+                shutdown();
+            }
+        };
+        final Thread hook = new Thread(shutdownRunner, "shutdown-hook-thread");
+        Runtime.getRuntime().addShutdownHook(hook);
+    }
+
+    public void shutdown() {
+        List<ChannelFuture> channelFutures = new ArrayList<>();
+
+        if (httpChannelHandle != null) {
+            LOG.info("Closing httpChannelHandle");
+            channelFutures.add(httpChannelHandle.close());
+        }
+
+        // wait for the channels to shutdown
+        channelFutures.forEach(f -> {
+            try {
+                f.get();
+            } catch (final Exception e) {
+                LOG.error("Channel:" + f.channel().config() + " -> " + e.getMessage(), e);
+            }
+        });
+
+        int quietPeriod = grafanaAuthConfig.getShutdownQuietPeriod();
+        List<Future<?>> groupFutures = new ArrayList<>();
+
+        if (httpBossGroup != null) {
+            LOG.info("Shutting down httpBossGroup");
+            groupFutures.add(httpBossGroup.shutdownGracefully(quietPeriod, 10, TimeUnit.SECONDS));
+        }
+
+        if (httpWorkerGroup != null) {
+            LOG.info("Shutting down httpWorkerGroup");
+            groupFutures.add(httpWorkerGroup.shutdownGracefully(quietPeriod, 10, TimeUnit.SECONDS));
+        }
+
+        groupFutures.parallelStream().forEach(f -> {
+            try {
+                f.get();
+            } catch (final Exception e) {
+                LOG.error("Group:" + f.toString() + " -> " + e.getMessage(), e);
+            }
+        });
+
+        if (applicationContext != null) {
+            LOG.info("Closing applicationContext");
+            applicationContext.close();
+        }
+
+        this.shutdown = true;
+        LOG.info("Server shut down.");
+    }
+
+    private static boolean useEpoll() {
+
+        // Should we just return true if this is Linux and if we get an error
+        // during Epoll
+        // setup handle it there?
+        final String os = SystemPropertyUtil.get(OS_NAME).toLowerCase().trim();
+        final String[] version = SystemPropertyUtil.get(OS_VERSION).toLowerCase().trim().split("\\.");
+        if (os.startsWith("linux") && version.length >= 3) {
+            final int major = Integer.parseInt(version[0]);
+            if (major > EPOLL_MIN_MAJOR_VERSION) {
+                return true;
+            } else if (major == EPOLL_MIN_MAJOR_VERSION) {
+                final int minor = Integer.parseInt(version[1]);
+                if (minor > EPOLL_MIN_MINOR_VERSION) {
+                    return true;
+                } else if (minor == EPOLL_MIN_MINOR_VERSION) {
+                    final int patch = Integer.parseInt(version[2].substring(0, 2));
+                    return patch >= EPOLL_MIN_PATCH_VERSION;
+                } else {
+                    return false;
+                }
+            } else {
+                return false;
+            }
+        } else {
+            return false;
+        }
+    }
+
+    protected SslContext createSSLContext(GrafanaAuthConfiguration config) throws Exception {
+
+        ServerSsl sslCfg = config.getSecurity().getServerSsl();
+        Boolean generate = sslCfg.isUseGeneratedKeypair();
+        SslContextBuilder ssl;
+        if (generate) {
+            LOG.warn("Using generated self signed server certificate");
+            Date begin = new Date();
+            Date end = new Date(begin.getTime() + 86400000);
+            SelfSignedCertificate ssc = new SelfSignedCertificate("localhost", begin, end);
+            ssl = SslContextBuilder.forServer(ssc.certificate(), ssc.privateKey());
+        } else {
+            String cert = sslCfg.getCertificateFile();
+            String key = sslCfg.getKeyFile();
+            String keyPass = sslCfg.getKeyPassword();
+            if (null == cert || null == key) {
+                throw new IllegalArgumentException("Check your SSL properties, something is wrong.");
+            }
+            ssl = SslContextBuilder.forServer(new File(cert), new File(key), keyPass);
+        }
+
+        ssl.ciphers(sslCfg.getUseCiphers());
+
+        // Can't set to REQUIRE because the CORS pre-flight requests will fail.
+        ssl.clientAuth(ClientAuth.OPTIONAL);
+
+        Boolean useOpenSSL = sslCfg.isUseOpenssl();
+        if (useOpenSSL) {
+            ssl.sslProvider(SslProvider.OPENSSL);
+        } else {
+            ssl.sslProvider(SslProvider.JDK);
+        }
+        String trustStore = sslCfg.getTrustStoreFile();
+        if (null != trustStore) {
+            if (!trustStore.isEmpty()) {
+                ssl.trustManager(new File(trustStore));
+            }
+        }
+        return ssl.build();
+    }
+
+    protected SSLContext createSSLClientContext(GrafanaAuthConfiguration config) throws Exception {
+
+        ClientSsl ssl = config.getSecurity().getClientSsl();
+        return HttpClient.getSSLContext(ssl.getTrustStoreFile(), ssl.getTrustStoreType(), ssl.getTrustStorePassword(),
+                ssl.getKeyFile(), ssl.getKeyType(), ssl.getKeyPassword());
+    }
+
+    public void run() throws Exception {
+
+        AuthCache.configure(grafanaAuthConfig.getSecurity());
+        final boolean useEpoll = useEpoll();
+        Class<? extends ServerSocketChannel> channelClass;
+        if (useEpoll) {
+            httpWorkerGroup = new EpollEventLoopGroup();
+            httpBossGroup = new EpollEventLoopGroup();
+            channelClass = EpollServerSocketChannel.class;
+        } else {
+            httpWorkerGroup = new NioEventLoopGroup();
+            httpBossGroup = new NioEventLoopGroup();
+            channelClass = NioServerSocketChannel.class;
+        }
+        LOG.info("Using channel class {}", channelClass.getSimpleName());
+
+        final int httpPort = grafanaAuthConfig.getHttp().getPort();
+        final String httpIp = grafanaAuthConfig.getHttp().getIp();
+        SslContext sslCtx = createSSLContext(grafanaAuthConfig);
+        if (sslCtx instanceof OpenSslServerContext) {
+            OpenSslServerContext openssl = (OpenSslServerContext) sslCtx;
+            String application = "GrafanaAuth_" + httpPort;
+            OpenSslServerSessionContext opensslCtx = openssl.sessionContext();
+            opensslCtx.setSessionCacheEnabled(true);
+            opensslCtx.setSessionCacheSize(128);
+            opensslCtx.setSessionIdContext(application.getBytes(StandardCharsets.UTF_8));
+            opensslCtx.setSessionTimeout(grafanaAuthConfig.getSecurity().getSessionMaxAge());
+        }
+        SSLContext clientSSLContext = createSSLClientContext(grafanaAuthConfig);
+        final ServerBootstrap httpServer = new ServerBootstrap();
+        httpServer.group(httpBossGroup, httpWorkerGroup);
+        httpServer.channel(channelClass);
+        httpServer.handler(new LoggingHandler());
+        HttpClientPool httpClientPool = new HttpClientPool(grafanaAuthConfig.getSecurity(), grafanaAuthConfig.getHttp(),
+                clientSSLContext);
+        httpServer.childHandler(setupHttpChannel(grafanaAuthConfig, sslCtx, httpClientPool));
+        httpServer.option(ChannelOption.ALLOCATOR, PooledByteBufAllocator.DEFAULT);
+        httpServer.option(ChannelOption.SO_BACKLOG, 128);
+        httpServer.option(ChannelOption.SO_KEEPALIVE, true);
+        httpChannelHandle = httpServer.bind(httpIp, httpPort).sync().channel();
+        final String httpAddress = ((InetSocketAddress) httpChannelHandle.localAddress()).getAddress().getHostAddress();
+
+        shutdownHook();
+        LOG.info("Server started. Listening on {}:{} for HTTP traffic", httpAddress, httpPort);
+    }
+
+    public static void main(String[] args) throws Exception {
+
+        GrafanaAuth.applicationContext = GrafanaAuth.initializeContext(args);
+        GrafanaAuthConfiguration grafanaAuthConf = GrafanaAuth.applicationContext
+                .getBean(GrafanaAuthConfiguration.class);
+
+        GrafanaAuth grafanaAuth = new GrafanaAuth(grafanaAuthConf);
+        try {
+            grafanaAuth.run();
+            LATCH.await();
+        } catch (final InterruptedException e) {
+            LOG.info("Server shutting down.");
+        } catch (Exception e) {
+            LOG.error("Error running server.", e);
+        } finally {
+            try {
+                grafanaAuth.shutdown();
+            } catch (Exception e) {
+                System.exit(1);
+            }
+        }
+    }
+
+    protected ChannelHandler setupHttpChannel(GrafanaAuthConfiguration config, SslContext sslCtx,
+            HttpClientPool httpClientPool) {
+
+        return new ChannelInitializer<SocketChannel>() {
+
+            @Override
+            protected void initChannel(SocketChannel ch) throws Exception {
+
+                ch.pipeline().addLast("ssl", new NonSslRedirectHandler(config.getHttp(), sslCtx));
+                ch.pipeline().addLast("encoder", new HttpResponseEncoder());
+                ch.pipeline().addLast("decoder", new HttpRequestDecoder());
+                ch.pipeline().addLast("compressor", new HttpContentCompressor());
+                ch.pipeline().addLast("decompressor", new HttpContentDecompressor());
+                // high maximum contentLength so that grafana snapshots can be delivered
+                // might not be necessary if inbound chunking (while proxying) is handled
+                ch.pipeline().addLast("aggregator", new HttpObjectAggregator(2097152));
+                ch.pipeline().addLast("chunker", new ChunkedWriteHandler());
+                ch.pipeline().addLast("grafanaDecoder",
+                        new GrafanaRequestDecoder(config.getSecurity(), config.getHttp()));
+                ch.pipeline().addLast("fileServer", new HttpStaticFileServerHandler());
+                ch.pipeline().addLast("login", new X509LoginRequestHandler(config.getSecurity(), config.getHttp()));
+                ch.pipeline().addLast("httpRelay", new GrafanaRelayHandler(config, httpClientPool));
+                ch.pipeline().addLast("error", new TimelyExceptionHandler()
+                        .setIgnoreSslHandshakeErrors(config.getSecurity().getServerSsl().isUseGeneratedKeypair()));
+            }
+        };
+    }
+}

grafana-auth/src/main/java/timely/grafana/auth/configuration/Grafana.java

@@ -0,0 +1,32 @@
+package timely.grafana.auth.configuration;
+
+public class Grafana {
+
+    private String host = "localhost";
+    private Integer port = 3000;
+    private String protocol = "http";
+
+    public void setHost(String host) {
+        this.host = host;
+    }
+
+    public String getHost() {
+        return host;
+    }
+
+    public void setPort(Integer port) {
+        this.port = port;
+    }
+
+    public Integer getPort() {
+        return port;
+    }
+
+    public void setProtocol(String protocol) {
+        this.protocol = protocol;
+    }
+
+    public String getProtocol() {
+        return protocol;
+    }
+}

grafana-auth/src/main/java/timely/grafana/auth/configuration/GrafanaAuthConfiguration.java

@@ -0,0 +1,54 @@
+package timely.grafana.auth.configuration;
+
+import javax.validation.Valid;
+
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.boot.context.properties.NestedConfigurationProperty;
+import org.springframework.stereotype.Component;
+import org.springframework.validation.annotation.Validated;
+import timely.balancer.configuration.BalancerHttp;
+import timely.balancer.configuration.BalancerSecurity;
+
+@Validated
+@Component
+@ConfigurationProperties(prefix = "grafana-auth")
+public class GrafanaAuthConfiguration {
+
+    @Valid
+    @NestedConfigurationProperty
+    private BalancerHttp http = new BalancerHttp();
+
+    @Valid
+    @NestedConfigurationProperty
+    private BalancerSecurity security = new BalancerSecurity();
+
+    @Valid
+    @NestedConfigurationProperty
+    private Grafana grafana = new Grafana();
+
+    private Integer shutdownQuietPeriod = 5;
+
+    public BalancerSecurity getSecurity() {
+        return security;
+    }
+
+    public BalancerHttp getHttp() {
+        return http;
+    }
+
+    public Grafana getGrafana() {
+        return grafana;
+    }
+
+    /**
+     * Time to wait (in seconds) for connections to finish and to make sure no new
+     * connections happen before shutting down Netty event loop groups.
+     */
+    public int getShutdownQuietPeriod() {
+        return this.shutdownQuietPeriod;
+    }
+
+    public void setShutdownQuietPeriod(Integer quietPeriod) {
+        this.shutdownQuietPeriod = quietPeriod;
+    }
+}

grafana-auth/src/main/java/timely/grafana/auth/configuration/SpringBootstrap.java

@@ -0,0 +1,7 @@
+package timely.grafana.auth.configuration;
+
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+
+@SpringBootApplication(scanBasePackageClasses = { SpringBootstrap.class })
+public class SpringBootstrap {
+}

grafana-auth/src/main/java/timely/grafana/auth/netty/http/GrafanaRelayHandler.java

@@ -0,0 +1,234 @@
+package timely.grafana.auth.netty.http;
+
+import static io.netty.handler.codec.http.HttpHeaderNames.CONTENT_LENGTH;
+
+import java.io.ByteArrayOutputStream;
+import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import com.google.common.collect.Multimap;
+import io.netty.buffer.Unpooled;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.channel.SimpleChannelInboundHandler;
+import io.netty.handler.codec.http.DefaultFullHttpResponse;
+import io.netty.handler.codec.http.FullHttpRequest;
+import io.netty.handler.codec.http.FullHttpResponse;
+import io.netty.handler.codec.http.HttpMethod;
+import io.netty.handler.codec.http.HttpResponseStatus;
+import io.netty.handler.codec.http.HttpVersion;
+import io.netty.util.ReferenceCountUtil;
+import org.apache.http.Header;
+import org.apache.http.HttpEntity;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpDelete;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpHead;
+import org.apache.http.client.methods.HttpOptions;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.methods.HttpPut;
+import org.apache.http.entity.StringEntity;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.message.BasicHeader;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.util.StreamUtils;
+import timely.api.request.timeseries.HttpRequest;
+import timely.api.response.TimelyException;
+import timely.auth.TimelyPrincipal;
+import timely.auth.TimelyUser;
+import timely.auth.util.DnUtils;
+import timely.auth.util.HttpHeaderUtils;
+import timely.balancer.connection.TimelyBalancedHost;
+import timely.balancer.connection.http.HttpClientPool;
+import timely.grafana.auth.configuration.GrafanaAuthConfiguration;
+import timely.grafana.request.GrafanaRequest;
+import timely.netty.http.TimelyHttpHandler;
+import timely.netty.http.auth.TimelyAuthenticationToken;
+
+public class GrafanaRelayHandler extends SimpleChannelInboundHandler<HttpRequest> implements TimelyHttpHandler {
+
+    private static final Logger LOG = LoggerFactory.getLogger(GrafanaRelayHandler.class);
+    private final HttpClientPool httpClientPool;
+    private GrafanaAuthConfiguration config;
+
+    public GrafanaRelayHandler(GrafanaAuthConfiguration config, HttpClientPool httpClientPool) {
+        this.config = config;
+        this.httpClientPool = httpClientPool;
+    }
+
+    @Override
+    public void channelRead0(ChannelHandlerContext ctx, HttpRequest msg) throws Exception {
+
+        CloseableHttpClient client = null;
+        CloseableHttpResponse relayedResponse = null;
+        FullHttpRequest request = null;
+        GrafanaRequest grafanaRequest;
+        TimelyBalancedHost k = null;
+        String relayURI = null;
+        try {
+            try {
+                request = msg.getHttpRequest();
+                String originalURI = request.uri();
+                originalURI = encodeURI(originalURI);
+
+                if (msg instanceof GrafanaRequest) {
+                    grafanaRequest = (GrafanaRequest) msg;
+                } else {
+                    throw new UnsupportedOperationException("Request not supported");
+                }
+                TimelyAuthenticationToken token = grafanaRequest.getToken();
+                if (token == null) {
+                    throw new IllegalStateException("No token on grafana request");
+                }
+
+                String protocol = config.getGrafana().getProtocol();
+                String host = config.getGrafana().getHost();
+                Integer port = config.getGrafana().getPort();
+                k = TimelyBalancedHost.of(host, -1, port, -1, -1, null);
+                client = httpClientPool.borrowObject(k);
+                Multimap<String, String> headers = HttpHeaderUtils.toMultimap(request.headers());
+
+                // If incoming user sets these headers, do not relay them
+                headers.removeAll("x-webauth-user");
+                headers.removeAll("x-webauth-name");
+                TimelyPrincipal principal = token.getTimelyPrincipal();
+                TimelyUser user = principal.getPrimaryUser();
+                String subjectDn = user.getDn().subjectDN();
+                String userName = DnUtils.getCommonName(subjectDn);
+                String loginName = DnUtils.getShortName(subjectDn);
+                headers.put("x-webauth-user", loginName);
+                headers.put("x-webauth-name", userName);
+                List<Header> relayedHeaderList = new ArrayList<>();
+                for (Map.Entry<String, String> h : headers.entries()) {
+                    if (!h.getKey().equals(CONTENT_LENGTH.toString())) {
+                        relayedHeaderList.add(new BasicHeader(h.getKey(), h.getValue()));
+                    }
+                }
+                Header[] headerArray = new Header[relayedHeaderList.size()];
+                relayedHeaderList.toArray(headerArray);
+                relayURI = protocol + "://" + host + ":" + port + originalURI;
+                if (request.method().equals(HttpMethod.HEAD)) {
+                    HttpHead relayedRequest = new HttpHead(relayURI);
+                    if (headerArray.length > 0) {
+                        relayedRequest.setHeaders(headerArray);
+                    }
+                    relayedResponse = client.execute(relayedRequest);
+                } else if (request.method().equals(HttpMethod.GET)) {
+                    HttpGet relayedRequest = new HttpGet(relayURI);
+                    if (headerArray.length > 0) {
+                        relayedRequest.setHeaders(headerArray);
+                    }
+                    relayedResponse = client.execute(relayedRequest);
+                } else if (request.method().equals(HttpMethod.POST)) {
+                    HttpPost relayedRequest = new HttpPost(relayURI);
+                    String content = request.content().toString(StandardCharsets.UTF_8);
+                    StringEntity entity = new StringEntity(content);
+                    relayedRequest.setEntity(entity);
+                    if (headerArray.length > 0) {
+                        relayedRequest.setHeaders(headerArray);
+                    }
+                    relayedResponse = client.execute(relayedRequest);
+                } else if (request.method().equals(HttpMethod.PUT)) {
+                    HttpPut relayedRequest = new HttpPut(relayURI);
+                    String content = request.content().toString(StandardCharsets.UTF_8);
+                    StringEntity entity = new StringEntity(content);
+                    relayedRequest.setEntity(entity);
+                    if (headerArray.length > 0) {
+                        relayedRequest.setHeaders(headerArray);
+                    }
+                    relayedResponse = client.execute(relayedRequest);
+                } else if (request.method().equals(HttpMethod.DELETE)) {
+                    HttpDelete relayedRequest = new HttpDelete(relayURI);
+                    if (headerArray.length > 0) {
+                        relayedRequest.setHeaders(headerArray);
+                    }
+                    relayedResponse = client.execute(relayedRequest);
+                } else if (request.method().equals(HttpMethod.OPTIONS)) {
+                    HttpOptions relayedRequest = new HttpOptions(relayURI);
+                    if (headerArray.length > 0) {
+                        relayedRequest.setHeaders(headerArray);
+                    }
+                    relayedResponse = client.execute(relayedRequest);
+                }
+            } catch (Exception e) {
+                String message = e.getMessage();
+                if (message == null) {
+                    LOG.error("", e);
+                } else {
+                    LOG.error(message, e);
+                }
+                this.sendHttpError(ctx, new TimelyException(HttpResponseStatus.INTERNAL_SERVER_ERROR.code(),
+                        e.getMessage(), e.getLocalizedMessage(), e));
+                return;
+            } finally {
+                if (request != null && request instanceof AutoCloseable) {
+                    ReferenceCountUtil.release(request);
+                }
+            }
+
+            FullHttpResponse response = null;
+            if (relayedResponse == null) {
+                this.sendHttpError(ctx, new TimelyException(HttpResponseStatus.METHOD_NOT_ALLOWED.code(),
+                        "Method not allowed", "Method not allowed"));
+            } else {
+                LOG.debug("RequestURL:{} ResponseCode:{}", relayURI, relayedResponse.getStatusLine().getStatusCode());
+                HttpEntity httpEntity = relayedResponse.getEntity();
+                if (httpEntity == null) {
+                    response = new DefaultFullHttpResponse(HttpVersion.HTTP_1_1,
+                            HttpResponseStatus.valueOf(relayedResponse.getStatusLine().getStatusCode()));
+                } else {
+                    ByteArrayOutputStream baos = new MyByteArrayOutputStream();
+                    StreamUtils.copy(httpEntity.getContent(), baos);
+                    response = new DefaultFullHttpResponse(HttpVersion.HTTP_1_1,
+                            HttpResponseStatus.valueOf(relayedResponse.getStatusLine().getStatusCode()),
+                            Unpooled.copiedBuffer(baos.toByteArray()));
+                    response.headers().set(CONTENT_LENGTH, response.content().readableBytes());
+                }
+                for (Header h : relayedResponse.getAllHeaders()) {
+                    response.headers().add(h.getName(), h.getValue());
+                }
+                sendResponse(ctx, response);
+            }
+        } finally {
+            if (relayedResponse != null) {
+                relayedResponse.close();
+            }
+            if (client != null && k != null) {
+                httpClientPool.returnObject(k, client);
+            } else {
+                LOG.error("NOT RETURNING CONNECTION! " + msg.getHttpRequest().uri());
+            }
+        }
+    }
+
+    static public class MyByteArrayOutputStream extends ByteArrayOutputStream {
+
+        public MyByteArrayOutputStream() {
+        }
+
+        public MyByteArrayOutputStream(int size) {
+            super(size);
+        }
+
+        public int getCount() {
+            return count;
+        }
+
+        public byte[] getBuf() {
+            return buf;
+        }
+    }
+
+    static public String encodeURI(String s) {
+        try {
+            s = s.replaceAll("\\{", URLEncoder.encode("{", "UTF-8"));
+            s = s.replaceAll("\\}", URLEncoder.encode("}", "UTF-8"));
+        } catch (Exception e) {
+
+        }
+        return s;
+    }
+}

grafana-auth/src/main/java/timely/grafana/auth/netty/http/GrafanaRequestDecoder.java

@@ -0,0 +1,133 @@
+package timely.grafana.auth.netty.http;
+
+import java.security.cert.X509Certificate;
+import java.util.Collection;
+import java.util.List;
+
+import com.google.common.collect.Multimap;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.handler.codec.MessageToMessageDecoder;
+import io.netty.handler.codec.http.FullHttpRequest;
+import io.netty.handler.codec.http.HttpHeaderNames;
+import io.netty.handler.codec.http.QueryStringDecoder;
+import io.netty.handler.codec.http.cookie.ServerCookieDecoder;
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import timely.api.request.AuthenticatedRequest;
+import timely.api.request.timeseries.HttpRequest;
+import timely.api.response.StrictTransportResponse;
+import timely.api.response.TimelyException;
+import timely.auth.AuthCache;
+import timely.auth.AuthenticationService;
+import timely.auth.SubjectIssuerDNPair;
+import timely.auth.TimelyPrincipal;
+import timely.auth.util.HttpHeaderUtils;
+import timely.configuration.Http;
+import timely.configuration.Security;
+import timely.grafana.request.GrafanaRequest;
+import timely.netty.Constants;
+import timely.netty.http.TimelyHttpHandler;
+import timely.netty.http.auth.TimelyAuthenticationToken;
+
+public class GrafanaRequestDecoder extends MessageToMessageDecoder<FullHttpRequest> implements TimelyHttpHandler {
+
+    private static final Logger LOG = LoggerFactory.getLogger(GrafanaRequestDecoder.class);
+    private static final String LOG_RECEIVED_REQUEST = "Received HTTP request {}";
+    private static final String LOG_PARSED_REQUEST = "Parsed request {}";
+
+    private final Security security;
+    private final String nonSecureRedirectAddress;
+
+    public GrafanaRequestDecoder(Security security, Http http) {
+
+        this.security = security;
+        this.nonSecureRedirectAddress = http.getRedirectPath();
+    }
+
+    public static String getSessionId(FullHttpRequest msg) {
+        Multimap<String, String> headers = HttpHeaderUtils.toMultimap(msg.headers());
+        Collection<String> cookies = headers.get(HttpHeaderNames.COOKIE.toString());
+        final StringBuilder buf = new StringBuilder();
+        cookies.forEach(h -> {
+            ServerCookieDecoder.STRICT.decode(h).forEach(c -> {
+                if (c.name().equals(Constants.COOKIE_NAME)) {
+                    if (buf.length() == 0) {
+                        buf.append(c.value());
+                    }
+                }
+            });
+        });
+        if (buf.length() == 0) {
+            return null;
+        } else {
+            return buf.toString();
+        }
+    }
+
+    @Override
+    protected void decode(ChannelHandlerContext ctx, FullHttpRequest msg, List<Object> out) throws Exception {
+
+        LOG.trace(LOG_RECEIVED_REQUEST, msg);
+
+        final String uri = msg.uri();
+        final QueryStringDecoder decoder = new QueryStringDecoder(uri);
+        if (decoder.path().equals(nonSecureRedirectAddress)) {
+            out.add(new StrictTransportResponse());
+            return;
+        }
+
+        final String sessionId = getSessionId(msg);
+        LOG.trace("SessionID: " + sessionId);
+
+        HttpRequest request;
+        try {
+            request = new GrafanaRequest();
+            Multimap<String, String> headers = HttpHeaderUtils.toMultimap(msg.headers());
+            ((AuthenticatedRequest) request).addHeaders(headers);
+            X509Certificate clientCert = AuthenticationService.getClientCertificate(ctx);
+            TimelyAuthenticationToken token;
+            if (StringUtils.isNotBlank(sessionId)) {
+                TimelyPrincipal principal;
+                ((AuthenticatedRequest) request).setSessionId(sessionId);
+                if (AuthCache.containsKey(sessionId)) {
+                    principal = AuthCache.get(sessionId);
+                } else {
+                    principal = TimelyPrincipal.anonymousPrincipal();
+                }
+                SubjectIssuerDNPair dn = principal.getPrimaryUser().getDn();
+                token = AuthenticationService.getAuthenticationToken(null, dn.subjectDN(), dn.issuerDN());
+            } else if (clientCert != null) {
+                token = AuthenticationService.getAuthenticationToken(clientCert, headers);
+            } else {
+                SubjectIssuerDNPair dn = TimelyPrincipal.anonymousPrincipal().getPrimaryUser().getDn();
+                token = AuthenticationService.getAuthenticationToken(null, dn.subjectDN(), dn.issuerDN());
+            }
+            ((AuthenticatedRequest) request).setToken(token);
+            request.setHttpRequest(msg.copy());
+            LOG.trace(LOG_PARSED_REQUEST, request);
+            request.validate();
+            out.add(request);
+        } catch (UnsupportedOperationException | NullPointerException e) {
+            // Return the original http request to route to the static file
+            // server
+            msg.retain();
+            out.add(msg);
+            return;
+        }
+        try {
+            try {
+                AuthenticationService.enforceAccess((AuthenticatedRequest) request);
+            } catch (TimelyException e) {
+                if (!security.isAllowAnonymousHttpAccess()) {
+                    throw e;
+                }
+            }
+        } catch (Exception e) {
+            out.clear();
+            throw e;
+        }
+
+    }
+
+}

grafana-auth/src/main/java/timely/grafana/request/GrafanaRequest.java

@@ -0,0 +1,32 @@
+package timely.grafana.request;
+
+import io.netty.handler.codec.http.FullHttpRequest;
+import io.netty.handler.codec.http.QueryStringDecoder;
+import timely.api.annotation.Http;
+import timely.api.request.AuthenticatedRequest;
+import timely.api.request.HttpGetRequest;
+import timely.api.request.HttpPostRequest;
+
+@Http(path = "/")
+public class GrafanaRequest extends AuthenticatedRequest implements HttpGetRequest, HttpPostRequest {
+
+    private FullHttpRequest httpRequest = null;
+
+    @Override
+    public HttpGetRequest parseQueryParameters(QueryStringDecoder decoder) throws Exception {
+        return new GrafanaRequest();
+    }
+
+    @Override
+    public HttpPostRequest parseBody(String content) throws Exception {
+        return new GrafanaRequest();
+    }
+
+    public void setHttpRequest(FullHttpRequest httpRequest) {
+        this.httpRequest = httpRequest;
+    }
+
+    public FullHttpRequest getHttpRequest() {
+        return httpRequest;
+    }
+}

grafana-auth/src/main/spotbugs/excludes.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<FindBugsFilter>
+  <Match>
+    <Class name="timely.grafana.auth.netty.http.GrafanaRelayHandler$MyByteArrayOutputStream" />
+    <Method name="getBuf" />
+    <Bug pattern="EI_EXPOSE_REP" />
+  </Match>
+</FindBugsFilter>

grafana/pom.xml

@@ -4,7 +4,7 @@
   <parent>
     <groupId>timely</groupId>
     <artifactId>timely</artifactId>
-    <version>1.1.1-SNAPSHOT</version>
+    <version>1.2.8-SNAPSHOT</version>
   </parent>
   <artifactId>timely-app</artifactId>
   <packaging>pom</packaging>

grafana/timely-app/dist/datasource/datasource.js

@@ -509,7 +509,7 @@ System.register(['lodash', 'angular', '../../../app/core/utils/datemath'], funct
                 } else {
                   return target.metric === metricData.metric && _.every(target.tags, function (tagV, tagK) {
                     interpolatedTagValue = this.templateSrv.replace(tagV, options.scopedVars, 'pipe');
-                    return metricData.tags[tagK] === interpolatedTagValue || interpolatedTagValue === "*";
+                    return metricData.tags[tagK] === interpolatedTagValue || interpolatedTagValue === ".*";
                   }.bind(this));
                 }
               }.bind(this));

grafana/timely-app/dist/datasource/datasource.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/datasource/datasource.js"],"names":["_","angular","dateMath","TimelyDatasource","instanceSettings","$q","backendSrv","templateSrv","contextSrv","type","name","jsonData","httpsPort","timelyHost","wsPort","basicAuths","url","_makeHttpsUrl","withCredentials","basicAuth","supportMetrics","tagKeys","aggregatorsPromise","filterTypesPromise","options","start","convertToTSDBTime","rangeRaw","from","end","to","qs","each","targets","target","metric","push","convertTargetToQuery","bind","queries","compact","isEmpty","d","defer","resolve","data","promise","groupByTags","query","filters","length","val","tagk","tags","key","performTimeSeriesQuery","then","response","metricToTargetMapping","mapMetricsToTargets","result","map","metricData","index","_saveTagKeys","transformMetricData","datasourceRequest","method","status","message","title","eventList","aggregator","annotation","results","annotationObject","annotations","isGlobal","globalAnnotations","event","description","time","Math","floor","startTime","text","notes","when","interpolated","replace","err","reject","responseTransform","value","metrics_regex","tag_names_regex","tag_values_regex","tag_names_suggest_regex","tag_values_suggest_regex","metrics_query","match","_performSuggestQuery","tag_names_query","_performMetricKeyLookup","tag_values_query","_performMetricKeyValueLookup","tag_names_suggest_query","tag_values_suggest_query","msResolution","responsePromises","reqBody","showQuery","_addCredentialOptions","headers","Promise","all","queryResponses","combinedResponse","x","concat","Object","keys","aggregateTags","tag","_get","q","max","keysArray","split","trim","keysQuery","splice","join","m","limit","tagvs","r","indexOf","tagks","tagv","relativeUrl","params","isArray","sort","md","metricLabel","createMetricLabel","dps","v","k","datapoints","alias","scopedVars","clone","label","tagData","toPairs","has","hide","shouldComputeRate","rate","rateOptions","counter","isCounter","counterMax","parseInt","counterResetValue","resetValue","disableDownsampling","interval","downsampleInterval","parseFloat","downsample","downsampleAggregator","downsampleFillPolicy","copy","filter_key","filter","tag_key","metrics","interpolatedTagValue","findIndex","every","tagV","tagK","date","roundUp","parse","valueOf"],"mappings":";;;;;;;;;;;;;;;AAAOA,O;;AACAC,a;;AACKC,c;;;;;;;;;;;;;;;;;;;;;kCAECC,gB;;AAEX;AACA,kCAAYC,gBAAZ,EAA8BC,EAA9B,EAAkCC,UAAlC,EAA8CC,WAA9C,EAA2DC,UAA3D,EAAuE;AAAA;;AACrE,eAAKC,IAAL,GAAYL,iBAAiBK,IAA7B;AACA,eAAKC,IAAL,GAAYN,iBAAiBM,IAA7B;;AAEA,eAAKC,QAAL,GAAgBP,iBAAiBO,QAAjB,IAA6B,EAA7C;AACA,eAAKC,SAAL,GAAiBR,iBAAiBO,QAAjB,CAA0BC,SAA3C;AACA,eAAKC,UAAL,GAAkBT,iBAAiBO,QAAjB,CAA0BE,UAA5C;AACA,eAAKC,MAAL,GAAcV,iBAAiBO,QAAjB,CAA0BG,MAAxC;AACA,cAAI,CAACV,iBAAiBO,QAAjB,CAA0BI,UAA/B,EAA0C;AACxCX,6BAAiBO,QAAjB,CAA0BI,UAA1B,GAAuC,KAAvC;AACD;;AAED,eAAKC,GAAL,GAAW,KAAKC,aAAL,EAAX;AACA,eAAKC,eAAL,GAAuB,IAAvB;AACA,eAAKC,SAAL,GAAiB,KAAjB;;AAEA,eAAKC,cAAL,GAAsB,IAAtB;AACA,eAAKC,OAAL,GAAe,EAAf;;AAEA,eAAKhB,EAAL,GAAUA,EAAV;AACA,eAAKC,UAAL,GAAkBA,UAAlB;AACA,eAAKC,WAAL,GAAmBA,WAAnB;AACA,eAAKC,UAAL,GAAkBA,UAAlB;;AAEA,eAAKc,kBAAL,GAA0B,IAA1B;AACA,eAAKC,kBAAL,GAA0B,IAA1B;AAED;;AAED;;;;;gCACMC,O,EAAQ;AACZ,gBAAIC,QAAQ,KAAKC,iBAAL,CAAuBF,QAAQG,QAAR,CAAiBC,IAAxC,EAA8C,KAA9C,CAAZ;AACA,gBAAIC,MAAM,KAAKH,iBAAL,CAAuBF,QAAQG,QAAR,CAAiBG,EAAxC,EAA4C,IAA5C,CAAV;AACA,gBAAIC,KAAK,EAAT;;AAEA/B,cAAEgC,IAAF,CAAOR,QAAQS,OAAf,EAAwB,UAASC,MAAT,EAAiB;AACvC,kBAAI,CAACA,OAAOC,MAAZ,EAAoB;AAAE;AAAS;AAC/BJ,iBAAGK,IAAH,CAAQ,KAAKC,oBAAL,CAA0BH,MAA1B,EAAkCV,OAAlC,CAAR;AACD,aAHuB,CAGtBc,IAHsB,CAGjB,IAHiB,CAAxB;;AAKA,gBAAIC,UAAUvC,EAAEwC,OAAF,CAAUT,EAAV,CAAd;;AAEA;AACA,gBAAI/B,EAAEyC,OAAF,CAAUF,OAAV,CAAJ,EAAwB;AACtB,kBAAIG,IAAI,KAAKrC,EAAL,CAAQsC,KAAR,EAAR;AACAD,gBAAEE,OAAF,CAAU,EAAEC,MAAM,EAAR,EAAV;AACA,qBAAOH,EAAEI,OAAT;AACD;;AAED,gBAAIC,cAAc,EAAlB;AACA/C,cAAEgC,IAAF,CAAOO,OAAP,EAAgB,UAASS,KAAT,EAAgB;AAC9B,kBAAIA,MAAMC,OAAN,IAAiBD,MAAMC,OAAN,CAAcC,MAAd,GAAuB,CAA5C,EAA+C;AAC7ClD,kBAAEgC,IAAF,CAAOgB,MAAMC,OAAb,EAAsB,UAASE,GAAT,EAAc;AAClCJ,8BAAYI,IAAIC,IAAhB,IAAwB,IAAxB;AACD,iBAFD;AAGD,eAJD,MAIO;AACLpD,kBAAEgC,IAAF,CAAOgB,MAAMK,IAAb,EAAmB,UAASF,GAAT,EAAcG,GAAd,EAAmB;AACpCP,8BAAYO,GAAZ,IAAmB,IAAnB;AACD,iBAFD;AAGD;AACF,aAVD;;AAYA,mBAAO,KAAKC,sBAAL,CAA4BhB,OAA5B,EAAqCd,KAArC,EAA4CI,GAA5C,EAAiD2B,IAAjD,CAAsD,UAASC,QAAT,EAAmB;AAC9E,kBAAIC,wBAAwB,KAAKC,mBAAL,CAAyBF,SAASZ,IAAlC,EAAwCrB,OAAxC,CAA5B;AACA,kBAAIoC,SAAS5D,EAAE6D,GAAF,CAAMJ,SAASZ,IAAf,EAAqB,UAASiB,UAAT,EAAqBC,KAArB,EAA4B;AAC5DA,wBAAQL,sBAAsBK,KAAtB,CAAR;AACA,oBAAIA,UAAU,CAAC,CAAf,EAAkB;AAChBA,0BAAQ,CAAR;AACD;AACD,qBAAKC,YAAL,CAAkBF,UAAlB;;AAEA,uBAAO,KAAKG,mBAAL,CAAyBH,UAAzB,EAAqCf,WAArC,EAAkDvB,QAAQS,OAAR,CAAgB8B,KAAhB,CAAlD,EAA0EvC,OAA1E,CAAP;AACD,eARiC,CAQhCc,IARgC,CAQ3B,IAR2B,CAArB,CAAb;AASA,qBAAO,EAAEO,MAAMe,MAAR,EAAP;AACD,aAZ4D,CAY3DtB,IAZ2D,CAYtD,IAZsD,CAAtD,CAAP;AAaD;;;2CAGe;AACd,mBAAO,KAAKhC,UAAL,CAAgB4D,iBAAhB,CAAkC;AACvClD,mBAAK,KAAKA,GAAL,GAAW,UADuB;AAEvCmD,sBAAQ;AAF+B,aAAlC,EAGJX,IAHI,CAGC,oBAAY;AAClB,kBAAIC,SAASW,MAAT,KAAoB,GLine truncated
+{"version":3,"sources":["../../src/datasource/datasource.js"],"names":["_","angular","dateMath","TimelyDatasource","instanceSettings","$q","backendSrv","templateSrv","contextSrv","type","name","jsonData","httpsPort","timelyHost","wsPort","basicAuths","url","_makeHttpsUrl","withCredentials","basicAuth","supportMetrics","tagKeys","aggregatorsPromise","filterTypesPromise","options","start","convertToTSDBTime","rangeRaw","from","end","to","qs","each","targets","target","metric","push","convertTargetToQuery","bind","queries","compact","isEmpty","d","defer","resolve","data","promise","groupByTags","query","filters","length","val","tagk","tags","key","performTimeSeriesQuery","then","response","metricToTargetMapping","mapMetricsToTargets","result","map","metricData","index","_saveTagKeys","transformMetricData","datasourceRequest","method","status","message","title","eventList","aggregator","annotation","results","annotationObject","annotations","isGlobal","globalAnnotations","event","description","time","Math","floor","startTime","text","notes","when","interpolated","replace","err","reject","responseTransform","value","metrics_regex","tag_names_regex","tag_values_regex","tag_names_suggest_regex","tag_values_suggest_regex","metrics_query","match","_performSuggestQuery","tag_names_query","_performMetricKeyLookup","tag_values_query","_performMetricKeyValueLookup","tag_names_suggest_query","tag_values_suggest_query","msResolution","responsePromises","reqBody","showQuery","_addCredentialOptions","headers","Promise","all","queryResponses","combinedResponse","x","concat","Object","keys","aggregateTags","tag","_get","q","max","keysArray","split","trim","keysQuery","splice","join","m","limit","tagvs","r","indexOf","tagks","tagv","relativeUrl","params","isArray","sort","md","metricLabel","createMetricLabel","dps","v","k","datapoints","alias","scopedVars","clone","label","tagData","toPairs","has","hide","shouldComputeRate","rate","rateOptions","counter","isCounter","counterMax","parseInt","counterResetValue","resetValue","disableDownsampling","interval","downsampleInterval","parseFloat","downsample","downsampleAggregator","downsampleFillPolicy","copy","filter_key","filter","tag_key","metrics","interpolatedTagValue","findIndex","every","tagV","tagK","date","roundUp","parse","valueOf"],"mappings":";;;;;;;;;;;;;;;AAAOA,O;;AACAC,a;;AACKC,c;;;;;;;;;;;;;;;;;;;;;kCAECC,gB;;AAEX;AACA,kCAAYC,gBAAZ,EAA8BC,EAA9B,EAAkCC,UAAlC,EAA8CC,WAA9C,EAA2DC,UAA3D,EAAuE;AAAA;;AACrE,eAAKC,IAAL,GAAYL,iBAAiBK,IAA7B;AACA,eAAKC,IAAL,GAAYN,iBAAiBM,IAA7B;;AAEA,eAAKC,QAAL,GAAgBP,iBAAiBO,QAAjB,IAA6B,EAA7C;AACA,eAAKC,SAAL,GAAiBR,iBAAiBO,QAAjB,CAA0BC,SAA3C;AACA,eAAKC,UAAL,GAAkBT,iBAAiBO,QAAjB,CAA0BE,UAA5C;AACA,eAAKC,MAAL,GAAcV,iBAAiBO,QAAjB,CAA0BG,MAAxC;AACA,cAAI,CAACV,iBAAiBO,QAAjB,CAA0BI,UAA/B,EAA0C;AACxCX,6BAAiBO,QAAjB,CAA0BI,UAA1B,GAAuC,KAAvC;AACD;;AAED,eAAKC,GAAL,GAAW,KAAKC,aAAL,EAAX;AACA,eAAKC,eAAL,GAAuB,IAAvB;AACA,eAAKC,SAAL,GAAiB,KAAjB;;AAEA,eAAKC,cAAL,GAAsB,IAAtB;AACA,eAAKC,OAAL,GAAe,EAAf;;AAEA,eAAKhB,EAAL,GAAUA,EAAV;AACA,eAAKC,UAAL,GAAkBA,UAAlB;AACA,eAAKC,WAAL,GAAmBA,WAAnB;AACA,eAAKC,UAAL,GAAkBA,UAAlB;;AAEA,eAAKc,kBAAL,GAA0B,IAA1B;AACA,eAAKC,kBAAL,GAA0B,IAA1B;AAED;;AAED;;;;;gCACMC,O,EAAQ;AACZ,gBAAIC,QAAQ,KAAKC,iBAAL,CAAuBF,QAAQG,QAAR,CAAiBC,IAAxC,EAA8C,KAA9C,CAAZ;AACA,gBAAIC,MAAM,KAAKH,iBAAL,CAAuBF,QAAQG,QAAR,CAAiBG,EAAxC,EAA4C,IAA5C,CAAV;AACA,gBAAIC,KAAK,EAAT;;AAEA/B,cAAEgC,IAAF,CAAOR,QAAQS,OAAf,EAAwB,UAASC,MAAT,EAAiB;AACvC,kBAAI,CAACA,OAAOC,MAAZ,EAAoB;AAAE;AAAS;AAC/BJ,iBAAGK,IAAH,CAAQ,KAAKC,oBAAL,CAA0BH,MAA1B,EAAkCV,OAAlC,CAAR;AACD,aAHuB,CAGtBc,IAHsB,CAGjB,IAHiB,CAAxB;;AAKA,gBAAIC,UAAUvC,EAAEwC,OAAF,CAAUT,EAAV,CAAd;;AAEA;AACA,gBAAI/B,EAAEyC,OAAF,CAAUF,OAAV,CAAJ,EAAwB;AACtB,kBAAIG,IAAI,KAAKrC,EAAL,CAAQsC,KAAR,EAAR;AACAD,gBAAEE,OAAF,CAAU,EAAEC,MAAM,EAAR,EAAV;AACA,qBAAOH,EAAEI,OAAT;AACD;;AAED,gBAAIC,cAAc,EAAlB;AACA/C,cAAEgC,IAAF,CAAOO,OAAP,EAAgB,UAASS,KAAT,EAAgB;AAC9B,kBAAIA,MAAMC,OAAN,IAAiBD,MAAMC,OAAN,CAAcC,MAAd,GAAuB,CAA5C,EAA+C;AAC7ClD,kBAAEgC,IAAF,CAAOgB,MAAMC,OAAb,EAAsB,UAASE,GAAT,EAAc;AAClCJ,8BAAYI,IAAIC,IAAhB,IAAwB,IAAxB;AACD,iBAFD;AAGD,eAJD,MAIO;AACLpD,kBAAEgC,IAAF,CAAOgB,MAAMK,IAAb,EAAmB,UAASF,GAAT,EAAcG,GAAd,EAAmB;AACpCP,8BAAYO,GAAZ,IAAmB,IAAnB;AACD,iBAFD;AAGD;AACF,aAVD;;AAYA,mBAAO,KAAKC,sBAAL,CAA4BhB,OAA5B,EAAqCd,KAArC,EAA4CI,GAA5C,EAAiD2B,IAAjD,CAAsD,UAASC,QAAT,EAAmB;AAC9E,kBAAIC,wBAAwB,KAAKC,mBAAL,CAAyBF,SAASZ,IAAlC,EAAwCrB,OAAxC,CAA5B;AACA,kBAAIoC,SAAS5D,EAAE6D,GAAF,CAAMJ,SAASZ,IAAf,EAAqB,UAASiB,UAAT,EAAqBC,KAArB,EAA4B;AAC5DA,wBAAQL,sBAAsBK,KAAtB,CAAR;AACA,oBAAIA,UAAU,CAAC,CAAf,EAAkB;AAChBA,0BAAQ,CAAR;AACD;AACD,qBAAKC,YAAL,CAAkBF,UAAlB;;AAEA,uBAAO,KAAKG,mBAAL,CAAyBH,UAAzB,EAAqCf,WAArC,EAAkDvB,QAAQS,OAAR,CAAgB8B,KAAhB,CAAlD,EAA0EvC,OAA1E,CAAP;AACD,eARiC,CAQhCc,IARgC,CAQ3B,IAR2B,CAArB,CAAb;AASA,qBAAO,EAAEO,MAAMe,MAAR,EAAP;AACD,aAZ4D,CAY3DtB,IAZ2D,CAYtD,IAZsD,CAAtD,CAAP;AAaD;;;2CAGe;AACd,mBAAO,KAAKhC,UAAL,CAAgB4D,iBAAhB,CAAkC;AACvClD,mBAAK,KAAKA,GAAL,GAAW,UADuB;AAEvCmD,sBAAQ;AAF+B,aAAlC,EAGJX,IAHI,CAGC,oBAAY;AAClB,kBAAIC,SAASW,MAAT,KAAoB,GLine truncated

grafana/timely-app/dist/plugin.json

@@ -23,8 +23,8 @@
       {"name": "Project site", "url": "https://github.com/NationalSecurityAgency/timely"},
       {"name": "License & Terms", "url": "https://raw.githubusercontent.com/NationalSecurityAgency/timely/master/LICENSE"}
     ],
-    "version": "1.1.0",
-    "updated": "2019-01-15"
+    "version": "1.2.8-SNAPSHOT",
+    "updated": "2020-02-21"
   },
 
   "includes": [

grafana/timely-app/package.json

@@ -1,7 +1,7 @@
 {
   "name": "timely-app",
   "private": true,
-  "version": "1.1.0",
+  "version": "1.2.8-SNAPSHOT",
   "description": "",
   "main": "index.js",
   "scripts": {

grafana/timely-app/plugin.json

@@ -23,8 +23,8 @@
       {"name": "Project site", "url": "https://github.com/NationalSecurityAgency/timely"},
       {"name": "License & Terms", "url": "https://raw.githubusercontent.com/NationalSecurityAgency/timely/master/LICENSE"}
     ],
-    "version": "1.1.0",
-    "updated": "2019-01-15"
+    "version": "1.2.8-SNAPSHOT",
+    "updated": "2020-02-21"
   },
 
   "includes": [

grafana/timely-app/src/datasource/datasource.js

@@ -446,7 +446,7 @@ export class TimelyDatasource {
         } else {
           return target.metric === metricData.metric && _.every(target.tags, function(tagV, tagK) {
                 interpolatedTagValue = this.templateSrv.replace(tagV, options.scopedVars, 'pipe');
-                return metricData.tags[tagK] === interpolatedTagValue || interpolatedTagValue === "*";
+                return metricData.tags[tagK] === interpolatedTagValue || interpolatedTagValue === ".*";
               }.bind(this));
         }
       }.bind(this));

pom.xml

@@ -3,7 +3,7 @@
   <modelVersion>4.0.0</modelVersion>
   <groupId>timely</groupId>
   <artifactId>timely</artifactId>
-  <version>1.1.1-SNAPSHOT</version>
+  <version>1.2.8-SNAPSHOT</version>
   <packaging>pom</packaging>
   <name>Timely</name>
   <description>A secure and scalable time series database based on Apache Accumulo</description>
@@ -20,6 +20,7 @@
     <module>grafana</module>
     <module>analytics</module>
     <module>balancer</module>
+    <module>grafana-auth</module>
   </modules>
   <scm>
     <connection>scm:git:https://github.com/NationalSecurityAgency/timely.git</connection>
@@ -40,50 +41,50 @@
     <os.detection.classifierWithLikes>fedora</os.detection.classifierWithLikes>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
     <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
-    <version.accumulo>1.9.2</version.accumulo>
+    <version.accumulo>1.9.3</version.accumulo>
     <version.buildnumber-maven>1.4</version.buildnumber-maven>
-    <version.caffeine>2.3.3</version.caffeine>
+    <version.caffeine>2.6.2</version.caffeine>
     <version.collectd>5.7.1</version.collectd>
-    <version.commons-collections4>4.1</version.commons-collections4>
-    <version.commons-configuration>1.10</version.commons-configuration>
-    <version.commons-io>2.6</version.commons-io>
-    <version.commons-lang3>3.5</version.commons-lang3>
-    <version.commons-pool2>2.6.0</version.commons-pool2>
-    <version.curator>2.11.1</version.curator>
-    <version.disruptor>3.3.6</version.disruptor>
+    <version.commons-collections4>4.3</version.commons-collections4>
+    <version.commons-lang3>3.8.1</version.commons-lang3>
+    <version.commons-pool2>2.6.1</version.commons-pool2>
+    <version.curator>4.1.0</version.curator>
+    <version.disruptor>3.4.2</version.disruptor>
+    <version.easymock>4.0.2</version.easymock>
     <version.eclipse-lifecycle>1.0.0</version.eclipse-lifecycle>
-    <version.fast-classpath-scanner>2.0.5</version.fast-classpath-scanner>
+    <version.fast-classpath-scanner>3.1.13</version.fast-classpath-scanner>
     <version.flink>1.1.4</version.flink>
     <version.google-flatbuffers>1.10.0</version.google-flatbuffers>
-    <version.guava>19.0</version.guava>
-    <version.hadoop>2.7.3</version.hadoop>
-    <version.httpcomponents>4.5.2</version.httpcomponents>
-    <version.impsort>1.2.0</version.impsort>
-    <version.jackson>2.9.8</version.jackson>
-    <version.jacoco>0.7.6.201602180812</version.jacoco>
+    <version.guava>27.1-jre</version.guava>
+    <version.hadoop>2.7.7</version.hadoop>
+    <version.httpcomponents>4.5.6</version.httpcomponents>
+    <version.impsort>1.3.0</version.impsort>
+    <version.jackson>2.10.0</version.jackson>
+    <version.jackson.databind>2.10.0</version.jackson.databind>
+    <version.jacoco>0.8.4</version.jacoco>
     <version.javacsv>2.0</version.javacsv>
-    <version.javassist>3.21.0-GA</version.javassist>
+    <version.javassist>3.24.1-GA</version.javassist>
     <version.jcip-annotations>1.0</version.jcip-annotations>
-    <version.jsoup>1.9.2</version.jsoup>
+    <version.jsoup>1.11.3</version.jsoup>
     <version.junit>4.12</version.junit>
-    <version.log4j>2.6.2</version.log4j>
-    <version.maven-assembly>2.6</version.maven-assembly>
-    <version.maven-build-helper>1.12</version.maven-build-helper>
-    <version.maven-compiler>3.3</version.maven-compiler>
-    <version.maven-dependency>2.10</version.maven-dependency>
-    <version.maven-jar>3.0.1</version.maven-jar>
-    <version.maven-pmd>3.6</version.maven-pmd>
-    <version.maven-rpm>2.1.4</version.maven-rpm>
-    <version.maven-shade>2.4.2</version.maven-shade>
-    <version.maven-surefire>2.19.1</version.maven-surefire>
-    <version.netty>4.0.56.Final</version.netty>
-    <version.netty-tcnative>2.0.7.Final</version.netty-tcnative>
+    <version.log4j>2.11.2</version.log4j>
+    <version.maven-assembly>3.1.1</version.maven-assembly>
+    <version.maven-build-helper>3.0.0</version.maven-build-helper>
+    <version.maven-compiler>3.8.1</version.maven-compiler>
+    <version.maven-dependency>3.1.1</version.maven-dependency>
+    <version.maven-jar>3.1.2</version.maven-jar>
+    <version.maven-pmd>3.12.0</version.maven-pmd>
+    <version.maven-rpm>2.2.0</version.maven-rpm>
+    <version.maven-shade>3.2.1</version.maven-shade>
+    <version.maven-surefire>3.0.0-M3</version.maven-surefire>
+    <version.netty>4.1.42.Final</version.netty>
+    <version.netty-tcnative>2.0.25.Final</version.netty-tcnative>
     <version.os-maven>1.5.0.Final</version.os-maven>
     <version.powermock>1.6.4</version.powermock>
-    <version.slf4j>1.7.21</version.slf4j>
-    <version.spotbugs-maven>3.1.7</version.spotbugs-maven>
-    <version.spring-boot>1.4.1.RELEASE</version.spring-boot>
-    <version.tyrus>1.13</version.tyrus>
+    <version.slf4j>1.7.26</version.slf4j>
+    <version.spotbugs-maven>3.1.12.1</version.spotbugs-maven>
+    <version.spring-boot>2.1.3.RELEASE</version.spring-boot>
+    <version.tyrus>1.15</version.tyrus>
   </properties>
   <dependencyManagement>
     <dependencies>
@@ -100,7 +101,7 @@
       <dependency>
         <groupId>com.fasterxml.jackson.core</groupId>
         <artifactId>jackson-databind</artifactId>
-        <version>${version.jackson}</version>
+        <version>${version.jackson.databind}</version>
       </dependency>
       <dependency>
         <groupId>com.fasterxml.jackson.datatype</groupId>
@@ -140,13 +141,18 @@
       <dependency>
         <groupId>io.github.lukehutch</groupId>
         <artifactId>fast-classpath-scanner</artifactId>
-        <version>2.0.5</version>
+        <version>${version.fast-classpath-scanner}</version>
       </dependency>
       <dependency>
         <groupId>io.netty</groupId>
         <artifactId>netty-all</artifactId>
         <version>${version.netty}</version>
       </dependency>
+      <dependency>
+        <groupId>io.netty</groupId>
+        <artifactId>netty-tcnative-boringssl-static</artifactId>
+        <version>${version.netty-tcnative}</version>
+      </dependency>
       <dependency>
         <groupId>net.sourceforge.javacsv</groupId>
         <artifactId>javacsv</artifactId>
@@ -156,6 +162,12 @@
         <groupId>org.apache.accumulo</groupId>
         <artifactId>accumulo-core</artifactId>
         <version>${version.accumulo}</version>
+        <exclusions>
+          <exclusion>
+            <groupId>io.netty</groupId>
+            <artifactId>netty</artifactId>
+          </exclusion>
+        </exclusions>
       </dependency>
       <dependency>
         <groupId>org.apache.accumulo</groupId>
@@ -309,6 +321,11 @@
         <artifactId>collectd-plugin-parent</artifactId>
         <version>${project.version}</version>
       </dependency>
+      <dependency>
+        <groupId>timely</groupId>
+        <artifactId>timely-balancer</artifactId>
+        <version>${project.version}</version>
+      </dependency>
       <dependency>
         <groupId>timely</groupId>
         <artifactId>timely-client</artifactId>
@@ -381,6 +398,12 @@
         <version>${version.jcip-annotations}</version>
         <scope>test</scope>
       </dependency>
+      <dependency>
+        <groupId>org.easymock</groupId>
+        <artifactId>easymock</artifactId>
+        <version>${version.easymock}</version>
+        <scope>test</scope>
+      </dependency>
       <dependency>
         <groupId>org.powermock</groupId>
         <artifactId>powermock-easymock-release-full</artifactId>
@@ -396,7 +419,7 @@
         <plugin>
           <groupId>net.revelc.code.formatter</groupId>
           <artifactId>formatter-maven-plugin</artifactId>
-          <version>2.8.0</version>
+          <version>2.10.0</version>
           <configuration>
             <compilerCompliance>${maven.compiler.source}</compilerCompliance>
             <compilerSource>${maven.compiler.source}</compilerSource>
@@ -414,7 +437,7 @@
         <plugin>
           <groupId>com.github.ekryd.sortpom</groupId>
           <artifactId>sortpom-maven-plugin</artifactId>
-          <version>2.8.0</version>
+          <version>2.10.0</version>
           <configuration>
             <predefinedSortOrder>recommended_2008_06</predefinedSortOrder>
             <createBackupFile>false</createBackupFile>
@@ -446,7 +469,7 @@
           <artifactId>maven-failsafe-plugin</artifactId>
           <version>${version.maven-surefire}</version>
           <configuration>
-            <argLine>${failsafeArgLine} -Djava.library.path=${project.build.directory}/native/META-INF/native/libnetty-tcnative.so -Dzookeeper.sasl.client=false</argLine>
+            <argLine>${failsafeArgLine} -Djava.library.path=${project.build.directory}/native/META-INF/native/libnetty_tcnative_linux_x86_64.so -Dzookeeper.sasl.client=false</argLine>
             <includes>
               <include>**/*.java</include>
             </includes>
@@ -489,7 +512,7 @@
           <artifactId>maven-surefire-plugin</artifactId>
           <version>${version.maven-surefire}</version>
           <configuration>
-            <argLine>${sureFireArgLine} -Djava.library.path=${project.build.directory}/native/META-INF/native/libnetty-tcnative.so -Dzookeeper.sasl.client=false</argLine>
+            <argLine>${sureFireArgLine} -Djava.library.path=${project.build.directory}/native/META-INF/native/libnetty-tcnative_linux_x86_64.so -Dzookeeper.sasl.client=false</argLine>
             <excludedGroups>timely.test.IntegrationTest</excludedGroups>
             <redirectTestOutputToFile>true</redirectTestOutputToFile>
           </configuration>
@@ -558,7 +581,7 @@
             <configuration>
               <rules>
                 <requireMavenVersion>
-                  <version>3.0.5</version>
+                  <version>3.5.0</version>
                 </requireMavenVersion>
                 <requireJavaVersion>
                   <version>[${maven.compiler.target},)</version>
@@ -729,6 +752,9 @@
           <name>m2e.version</name>
         </property>
       </activation>
+      <properties>
+        <maven.compiler.release>8</maven.compiler.release>
+      </properties>
       <build>
         <pluginManagement>
           <plugins>
@@ -789,5 +815,14 @@
         </pluginManagement>
       </build>
     </profile>
+    <profile>
+      <id>jdk-release-flag</id>
+      <activation>
+        <jdk>[9,)</jdk>
+      </activation>
+      <properties>
+        <maven.compiler.release>8</maven.compiler.release>
+      </properties>
+    </profile>
   </profiles>
 </project>

server/bin/get-tablet-summary.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+if [[ `uname` == "Darwin" ]]; then
+        THIS_SCRIPT=`python -c 'import os,sys; print os.path.realpath(sys.argv[1])' $0`
+else
+        THIS_SCRIPT=`readlink -f $0`
+fi
+
+THIS_DIR="${THIS_SCRIPT%/*}"
+BASE_DIR=${THIS_DIR}/..
+CONF_DIR="${BASE_DIR}/conf"
+LIB_DIR="${BASE_DIR}/lib"
+
+export CLASSPATH="${CONF_DIR}:${LIB_DIR}/*"
+JVM_ARGS="-Xmx1024m -Xms128m"
+JVM_ARGS="${JVM_ARGS} -Dlogging.config=${CONF_DIR}/log4j2-error-console.xml"
+JVM_ARGS="${JVM_ARGS} -Dlog4j.configurationFile=${CONF_DIR}/log4j2-error-console.xml"
+JVM_ARGS="${JVM_ARGS} -DLog4jContextSelector=org.apache.logging.log4j.core.async.AsyncLoggerContextSelector"
+
+echo "$JAVA_HOME/bin/java ${JVM_ARGS} timely.store.compaction.util.TabletMetadataConsole --spring.config.name=timely"
+$JAVA_HOME/bin/java ${JVM_ARGS} timely.store.compaction.util.TabletMetadataConsole --spring.config.name=timely

server/bin/timely-server.sh

@@ -27,15 +27,14 @@ fi
 mkdir -p ${NATIVE_DIR}
 
 pushd ${BASE_DIR}/bin
-$JAVA_HOME/bin/jar xf ${LIB_DIR}/netty-tcnative*.jar META-INF/native/libnetty_tcnative.${TCNATIVE_SUFFIX}
-$JAVA_HOME/bin/jar xf ${LIB_DIR}/netty-transport-native-epoll*.jar META-INF/native/libnetty_transport_native_epoll_x86_64.${TCNATIVE_SUFFIX}
+$JAVA_HOME/bin/jar xf ${LIB_DIR}/netty-tcnative-boringssl-static*.jar META-INF/native/libnetty_tcnative_linux_x86_64.${TCNATIVE_SUFFIX}
+$JAVA_HOME/bin/jar xf ${LIB_DIR}/netty-all*.jar META-INF/native/libnetty_transport_native_epoll_x86_64.${TCNATIVE_SUFFIX}
 popd
 
-export CLASSPATH="${CONF_DIR}:${LIB_DIR}/*"
-JVM_ARGS="-Xmx256m -Xms256m -Dio.netty.eventLoopThreads=${NUM_SERVER_THREADS}"
+export CLASSPATH="${CONF_DIR}:${LIB_DIR}/*:${HADOOP_CONF_DIR}"
+JVM_ARGS="-Xmx256m -Xms256m -Dio.netty.eventLoopThreads=${NUM_SERVER_THREADS} -Dlog4j.configurationFile=${CONF_DIR}/log4j2-spring.xml"
 JVM_ARGS="${JVM_ARGS} -DLog4jContextSelector=org.apache.logging.log4j.core.async.AsyncLoggerContextSelector"
 JVM_ARGS="${JVM_ARGS} -Djava.library.path=${NATIVE_DIR}"
-JVM_ARGS="${JVM_ARGS} -Djava.library.path=${NATIVE_DIR}/libnetty-${TCNATIVE_SUFFIX}"
 
 echo "$JAVA_HOME/bin/java ${JVM_ARGS} timely.Server --spring.config.name=timely"
 $JAVA_HOME/bin/java ${JVM_ARGS} timely.Server --spring.config.name=timely

server/bin/timely-standalone.sh

@@ -2,17 +2,12 @@
 
 if [[ `uname` == "Darwin" ]]; then
         THIS_SCRIPT=`python -c 'import os,sys; print os.path.realpath(sys.argv[1])' $0`
-        TCNATIVE_SUFFIX="tcnative.jnilib"
+        TCNATIVE_SUFFIX="jnilib"
 else
         THIS_SCRIPT=`readlink -f $0`
-        TCNATIVE_SUFFIX="tcnative.so"
+        TCNATIVE_SUFFIX="so"
 fi
 
-# netty tcnative file reference
-#netty-tcnative-1.1.33.Fork18-linux-x86_64-fedora.jar -> libnetty-tcnative.so
-#netty-tcnative-1.1.33.Fork18-linux-x86_64.jar -> libnetty-tcnative.so
-#netty-tcnative-1.1.33.Fork18-osx-x86_64.jar -> libnetty-tcnative.jnilib
-
 THIS_DIR="${THIS_SCRIPT%/*}"
 NATIVE_DIR="${THIS_DIR}/META-INF/native"
 BASE_DIR=${THIS_DIR}/..
@@ -32,11 +27,12 @@ fi
 mkdir -p ${NATIVE_DIR}
 
 pushd ${BASE_DIR}/bin
-$JAVA_HOME/bin/jar xf ${LIB_DIR}/netty-tcnative*.jar META-INF/native/libnetty-${TCNATIVE_SUFFIX}
+$JAVA_HOME/bin/jar xf ${LIB_DIR}/netty-tcnative-boringssl-static*.jar META-INF/native/libnetty_tcnative_linux_x86_64.${TCNATIVE_SUFFIX}
+$JAVA_HOME/bin/jar xf ${LIB_DIR}/netty-all*.jar META-INF/native/libnetty_transport_native_epoll_x86_64.${TCNATIVE_SUFFIX}
 popd
 
-export CLASSPATH="${CONF_DIR}:${LIB_DIR}/*"
-JVM_ARGS="-Xmx256m -Xms256m -Dio.netty.eventLoopThreads=${NUM_SERVER_THREADS}"
+export CLASSPATH="${CONF_DIR}:${LIB_DIR}/*:${HADOOP_CONF_DIR}"
+JVM_ARGS="-Xmx256m -Xms256m -Dio.netty.eventLoopThreads=${NUM_SERVER_THREADS} -Dlog4j.configurationFile=${CONF_DIR}/log4j2-spring.xml"
 JVM_ARGS="${JVM_ARGS} -DLog4jContextSelector=org.apache.logging.log4j.core.async.AsyncLoggerContextSelector"
 JVM_ARGS="${JVM_ARGS} -Djava.library.path=${NATIVE_DIR}"
 # for debugging

server/conf/security.xml

@@ -15,15 +15,27 @@
 
     <beans:bean id="timelyUserDetailsService" class="timely.auth.UserDetailsService">
     	<beans:property name="users">
-    		<util:map key-type="java.lang.String">
-    			<beans:entry key="CN=example.com">
-   					<util:list>
-   						<beans:value>D</beans:value>
-   						<beans:value>E</beans:value>
-   						<beans:value>F</beans:value>
-   					</util:list>
-    			</beans:entry>
-    		</util:map>
+    		<util:list>
+				<beans:bean class="timely.auth.TimelyUser">
+					<beans:constructor-arg name="subjectDn">
+						<beans:value>CN=example.com</beans:value>
+					</beans:constructor-arg>
+					<beans:constructor-arg name="auths">
+						<util:list>
+							<beans:value>D</beans:value>
+							<beans:value>E</beans:value>
+							<beans:value>F</beans:value>
+						</util:list>
+					</beans:constructor-arg>
+					<beans:constructor-arg name="roles">
+						<util:list>
+							<beans:value>G</beans:value>
+							<beans:value>H</beans:value>
+							<beans:value>I</beans:value>
+						</util:list>
+					</beans:constructor-arg>
+				</beans:bean>
+    		</util:list>
     	</beans:property>
     </beans:bean>
     
@@ -34,9 +46,12 @@
     <user-service id="usernamePasswordService">
     	<user name="test" password="test1" authorities="A,B,C" />
     </user-service>
-    
-    <beans:bean id="UserNamePasswordAuthenticationProvider" class="org.springframework.security.authentication.dao.DaoAuthenticationProvider">
+
+	<beans:bean id="noOpPasswordEncoder" class="org.springframework.security.crypto.password.NoOpPasswordEncoder"/>
+
+	<beans:bean id="UserNamePasswordAuthenticationProvider" class="org.springframework.security.authentication.dao.DaoAuthenticationProvider">
 		<beans:property name="userDetailsService" ref="usernamePasswordService"/>
+		<beans:property name="passwordEncoder" ref="noOpPasswordEncoder"/>
 	</beans:bean>
 	    
 	<authentication-manager alias="authenticationManager">
@@ -45,5 +60,12 @@
 		<!-- Basic authorization provider, validates provided username and password -->
 		<authentication-provider ref="UserNamePasswordAuthenticationProvider" />
 	</authentication-manager>
-	
+
+	<util:list id="requiredRoles" value-type="java.lang.String">
+		<beans:value></beans:value>
+	</util:list>
+
+	<util:list id="requiredAuths" value-type="java.lang.String">
+		<beans:value></beans:value>
+	</util:list>
 </beans:beans>

server/conf/timely-docker.yml

@@ -18,9 +18,11 @@ timely:
       max-downsample-memory: 10000000
   security:
     session-max-age: 86400
+    cache-expiration-minutes: 60
+    cache-refresh-minutes: 5
     allow-anonymous-http-access: true
     allow-anonymous-ws-access: true
-    ssl:
+    server-ssl:
       certificate-file: /timely-server-src/server/pki/timely.csr
       key-file: /timely-server-src/server/pki/timely-pkcs8.key
       key-password: password
@@ -50,9 +52,9 @@ timely:
     timeout: 60
     subscription-lag: 120
   meta-cache:
-    expiration-minutes: 60
-    initial-capacity: 2000
-    max-capacity: 10000
+    expiration-minutes: 1440
+    cache-refresh-minutes: 60
+    max-tag-values: 100
   visibility-cache:
     expiration-minutes: 60
     initial-capacity: 2000

server/conf/timely-standalone.yml

@@ -3,6 +3,8 @@ timely:
   meta-table: timely.meta
   metric-age-off-days:
     default: 7
+  cache:
+    enabled: false
   metrics-report-ignored-tags:
   accumulo:
     instance-name: TimelyStandalone
@@ -18,9 +20,11 @@ timely:
       max-downsample-memory: 10000000
   security:
     session-max-age: 86400
+    cache-expiration-minutes: 60
+    cache-refresh-minutes: 5
     allow-anonymous-http-access: true
     allow-anonymous-ws-access: true
-    ssl:
+    server-ssl:
       certificate-file:
       key-file:
       key-password: timely
@@ -54,9 +58,9 @@ timely:
     scanner-read-ahead: 1
     subscription-batch-size: 5000
   meta-cache:
-    expiration-minutes: 60
-    initial-capacity: 2000
-    max-capacity: 10000
+    expiration-minutes: 1440
+    cache-refresh-minutes: 60
+    max-tag-values: 100
   visibility-cache:
     expiration-minutes: 60
     initial-capacity: 2000

server/conf/timely.yml

@@ -8,6 +8,7 @@ timely:
     age-off-hours:
       default: 26
   metrics-report-ignored-tags:
+  defaultVisibility:
   accumulo:
     instance-name: accumulo
     zookeepers: localhost:2181
@@ -22,9 +23,11 @@ timely:
       max-downsample-memory: 10000000
   security:
     session-max-age: 86400
+    cache-expiration-minutes: 60
+    cache-refresh-minutes: 5
     allow-anonymous-http-access: true
     allow-anonymous-ws-access: true
-    serverSsl:
+    server-ssl:
       certificate-file:
       key-file:
       key-password:
@@ -59,10 +62,9 @@ timely:
     scanner-read-ahead: 1
     subscription-batch-size: 5000
   meta-cache:
-    expiration-minutes: 60
-    initial-capacity: 2000
-    max-capacity: 10000
+    expiration-minutes: 1440
+    cache-refresh-minutes: 60
+    max-tag-values: 100
   visibility-cache:
     expiration-minutes: 60
-    initial-capacity: 2000
-    max-capacity: 10000
+

server/pom.xml

@@ -4,7 +4,7 @@
   <parent>
     <groupId>timely</groupId>
     <artifactId>timely</artifactId>
-    <version>1.1.1-SNAPSHOT</version>
+    <version>1.2.8-SNAPSHOT</version>
   </parent>
   <artifactId>timely-server</artifactId>
   <dependencies>
@@ -108,6 +108,10 @@
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-minicluster</artifactId>
     </dependency>
+    <dependency>
+      <groupId>org.glassfish.tyrus</groupId>
+      <artifactId>tyrus-client</artifactId>
+    </dependency>
     <dependency>
       <groupId>org.slf4j</groupId>
       <artifactId>slf4j-api</artifactId>
@@ -149,9 +153,7 @@
     </dependency>
     <dependency>
       <groupId>io.netty</groupId>
-      <artifactId>netty-tcnative</artifactId>
-      <version>${version.netty-tcnative}</version>
-      <classifier>${os.detected.classifier}</classifier>
+      <artifactId>netty-tcnative-boringssl-static</artifactId>
       <scope>runtime</scope>
     </dependency>
     <dependency>
@@ -179,11 +181,22 @@
       <artifactId>junit</artifactId>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.easymock</groupId>
+      <artifactId>easymock</artifactId>
+      <scope>test</scope>
+    </dependency>
     <dependency>
       <groupId>org.jsoup</groupId>
       <artifactId>jsoup</artifactId>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.powermock</groupId>
+      <artifactId>powermock-easymock-release-full</artifactId>
+      <classifier>full</classifier>
+      <scope>test</scope>
+    </dependency>
     <dependency>
       <groupId>org.springframework.boot</groupId>
       <artifactId>spring-boot-test-autoconfigure</artifactId>
@@ -240,9 +253,8 @@
               <artifactItems>
                 <artifactItem>
                   <groupId>io.netty</groupId>
-                  <artifactId>netty-tcnative</artifactId>
+                  <artifactId>netty-tcnative-boringssl-static</artifactId>
                   <version>${version.netty-tcnative}</version>
-                  <classifier>${os.detected.classifier}</classifier>
                   <type>jar</type>
                   <overWrite>false</overWrite>
                   <outputDirectory>${project.build.directory}/native</outputDirectory>
@@ -284,37 +296,4 @@
       </extension>
     </extensions>
   </build>
-  <profiles>
-    <profile>
-      <id>native-epoll-linux</id>
-      <activation>
-        <os>
-          <family>!mac</family>
-        </os>
-      </activation>
-      <dependencies>
-        <dependency>
-          <groupId>io.netty</groupId>
-          <artifactId>netty-transport-native-epoll</artifactId>
-          <version>${version.netty}</version>
-          <classifier>linux-x86_64</classifier>
-        </dependency>
-      </dependencies>
-    </profile>
-    <profile>
-      <id>native-epoll</id>
-      <activation>
-        <os>
-          <family>mac</family>
-        </os>
-      </activation>
-      <dependencies>
-        <dependency>
-          <groupId>io.netty</groupId>
-          <artifactId>netty-transport-native-epoll</artifactId>
-          <version>${version.netty}</version>
-        </dependency>
-      </dependencies>
-    </profile>
-  </profiles>
 </project>

server/src/main/java/timely/Server.java

@@ -24,6 +24,7 @@ import io.netty.channel.ChannelHandler;
 import io.netty.channel.ChannelInitializer;
 import io.netty.channel.ChannelOption;
 import io.netty.channel.EventLoopGroup;
+import io.netty.channel.WriteBufferWaterMark;
 import io.netty.channel.epoll.EpollChannelOption;
 import io.netty.channel.epoll.EpollDatagramChannel;
 import io.netty.channel.epoll.EpollEventLoopGroup;
@@ -44,6 +45,7 @@ import io.netty.handler.codec.http.HttpRequestDecoder;
 import io.netty.handler.codec.http.HttpResponseEncoder;
 import io.netty.handler.codec.http.HttpServerCodec;
 import io.netty.handler.codec.http.cors.CorsConfig;
+import io.netty.handler.codec.http.cors.CorsConfigBuilder;
 import io.netty.handler.codec.http.cors.CorsHandler;
 import io.netty.handler.codec.http.websocketx.WebSocketServerProtocolHandler;
 import io.netty.handler.logging.LoggingHandler;
@@ -58,6 +60,7 @@ import io.netty.handler.stream.ChunkedWriteHandler;
 import io.netty.handler.timeout.IdleStateHandler;
 import io.netty.util.NettyRuntime;
 import io.netty.util.internal.SystemPropertyUtil;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
 import org.apache.curator.RetryPolicy;
 import org.apache.curator.framework.CuratorFramework;
 import org.apache.curator.framework.CuratorFrameworkFactory;
@@ -70,9 +73,9 @@ import org.apache.zookeeper.CreateMode;
 import org.apache.zookeeper.data.Stat;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.boot.WebApplicationType;
 import org.springframework.boot.builder.SpringApplicationBuilder;
 import org.springframework.context.ConfigurableApplicationContext;
-import timely.api.response.TimelyException;
 import timely.auth.AuthCache;
 import timely.auth.VisibilityCache;
 import timely.configuration.Configuration;
@@ -208,13 +211,13 @@ public class Server {
             payload.setWsPort(config.getWebsocket().getPort());
             payload.setUdpPort(config.getServer().getUdpPort());
 
-            ServiceInstanceBuilder builder = ServiceInstance.builder();
+            ServiceInstanceBuilder<ServerDetails> builder = ServiceInstance.builder();
             String serviceName = host + ":" + config.getServer().getTcpPort();
-            ServiceInstance serviceInstance = builder.id(serviceName).name("timely-server")
+            ServiceInstance<ServerDetails> serviceInstance = builder.id(serviceName).name("timely-server")
                     .address(config.getServer().getIp()).port(config.getServer().getTcpPort()).payload(payload).build();
 
-            ServiceDiscovery discovery = ServiceDiscoveryBuilder.builder(ServerDetails.class).client(curatorFramework)
-                    .basePath(SERVICE_DISCOVERY_PATH).build();
+            ServiceDiscovery<ServerDetails> discovery = ServiceDiscoveryBuilder.builder(ServerDetails.class)
+                    .client(curatorFramework).basePath(SERVICE_DISCOVERY_PATH).build();
             discovery.start();
             discovery.registerService(serviceInstance);
 
@@ -230,11 +233,12 @@ public class Server {
 
     public static void main(String[] args) throws Exception {
 
-        Configuration conf = initializeConfiguration(args);
+        Server.applicationContext = initializeContext(args);
+        Configuration conf = Server.applicationContext.getBean(Configuration.class);
 
-        Server s = new Server(conf);
+        Server server = new Server(conf);
         try {
-            s.run();
+            server.run();
             LATCH.await();
         } catch (final InterruptedException e) {
             LOG.info("Server shutting down.");
@@ -242,16 +246,18 @@ public class Server {
             LOG.error("Error running server.", e);
         } finally {
             try {
-                s.shutdown();
+                server.shutdown();
             } catch (Exception e) {
                 System.exit(1);
             }
         }
     }
 
-    protected static Configuration initializeConfiguration(String[] args) {
-        applicationContext = new SpringApplicationBuilder(SpringBootstrap.class).web(false).run(args);
-        return applicationContext.getBean(Configuration.class);
+    protected static ConfigurableApplicationContext initializeContext(String[] args) {
+        SpringApplicationBuilder builder = new SpringApplicationBuilder(SpringBootstrap.class);
+        builder.web(WebApplicationType.NONE);
+        builder.registerShutdownHook(false);
+        return builder.run(args);
     }
 
     private void shutdownHook() {
@@ -266,6 +272,8 @@ public class Server {
     }
 
     public void shutdown() {
+        LOG.warn("Server shutdown 1");
+
         List<ChannelFuture> channelFutures = new ArrayList<>();
 
         if (tcpChannelHandle != null) {
@@ -283,8 +291,9 @@ public class Server {
             channelFutures.add(wsChannelHandle.close());
         }
 
+        int udpChannel = 1;
         for (Channel c : udpChannelHandleList) {
-            LOG.info("Closing udpChannelHandle");
+            LOG.info("Closing udpChannelHandle #" + udpChannel++);
             channelFutures.add(c.close());
         }
 
@@ -293,14 +302,15 @@ public class Server {
             try {
                 f.get();
             } catch (final Exception e) {
-                LOG.error("Error while shutting down channel: {}", f.channel().config());
-                LOG.error("{}", e.getMessage());
-                e.printStackTrace();
+                LOG.error("Channel:" + f.channel().config() + " -> " + e.getMessage(), e);
             }
         });
 
-        Integer quietPeriod = config.getServer().getShutdownQuietPeriod();
+        LOG.warn("Server shutdown 2");
+
+        int quietPeriod = config.getServer().getShutdownQuietPeriod();
         List<Future<?>> groupFutures = new ArrayList<>();
+
         if (tcpBossGroup != null) {
             LOG.info("Shutting down tcpBossGroup");
             groupFutures.add(tcpBossGroup.shutdownGracefully(quietPeriod, 10, TimeUnit.SECONDS));
@@ -336,6 +346,8 @@ public class Server {
             groupFutures.add(udpBossGroup.shutdownGracefully(quietPeriod, 10, TimeUnit.SECONDS));
         }
 
+        LOG.warn("Server shutdown 3");
+
         if (udpWorkerGroup != null) {
             LOG.info("Shutting down udpWorkerGroup");
             groupFutures.add(udpWorkerGroup.shutdownGracefully(quietPeriod, 10, TimeUnit.SECONDS));
@@ -345,41 +357,70 @@ public class Server {
             try {
                 f.get();
             } catch (final Exception e) {
-                LOG.error("Error while shutting down group: {}", f.toString());
-                LOG.error("{}", e.getMessage());
-                e.printStackTrace();
+                LOG.error("Group:" + f.toString() + " -> " + e.getMessage(), e);
             }
         });
 
-        try {
-            LOG.info("Flushing datastore.");
-            dataStore.flush();
-        } catch (TimelyException e) {
-            LOG.error("Error flushing to server during shutdown", e);
-        }
+        LOG.warn("Server shutdown 4");
 
         try {
-            LOG.info("Closing MetaCacheFactory");
+            LOG.info("Closing dataStore");
+            dataStore.close();
+        } catch (Exception e) {
+            LOG.error(e.getMessage(), e);
+        }
+
+        LOG.warn("Server shutdown 5");
+
+        if (dataStoreCache != null) {
+            try {
+                LOG.info("Closing dataStoreCache");
+                dataStoreCache.close();
+            } catch (Exception e) {
+                LOG.error(e.getMessage(), e);
+            }
+        }
+
+        LOG.warn("Server shutdown 6");
+
+        try {
+            LOG.info("Closing metaCacheFactory");
             MetaCacheFactory.close();
         } catch (Exception e) {
-            LOG.error("Error closing MetaCacheFactory during shutdown", e);
+            LOG.error(e.getMessage(), e);
         }
 
+        LOG.warn("Server shutdown 7");
+
         try {
-            LOG.info("Closing WebSocketRequestDecoder");
+            LOG.info("Closing webSocketRequestDecoder subscriptions");
             WebSocketRequestDecoder.close();
         } catch (Exception e) {
-            LOG.error("Error closing WebSocketRequestDecoder during shutdown", e);
+            LOG.error(e.getMessage(), e);
         }
 
+        LOG.warn("Server shutdown 8");
+
         if (curatorFramework != null) {
-            curatorFramework.close();
+            try {
+                LOG.info("Closing curatorFramework");
+                curatorFramework.close();
+            } catch (Exception e) {
+                LOG.error(e.getMessage(), e);
+            }
         }
 
+        LOG.warn("Server shutdown 9");
+
         if (applicationContext != null) {
-            LOG.info("Closing applicationContext");
-            applicationContext.close();
+            try {
+                LOG.info("Closing applicationContext");
+                applicationContext.close();
+            } catch (Exception e) {
+                LOG.error(e.getMessage(), e);
+            }
         }
+        LOG.warn("Server shutdown 10");
         this.shutdown = true;
         LOG.info("Server shut down.");
     }
@@ -414,7 +455,9 @@ public class Server {
     public void run() throws Exception {
 
         RetryPolicy retryPolicy = new RetryForever(1000);
-        curatorFramework = CuratorFrameworkFactory.newClient(config.getAccumulo().getZookeepers(), 30000, 10000,
+        int timeout = Long.valueOf(AccumuloConfiguration.getTimeInMillis(config.getAccumulo().getZookeeperTimeout()))
+                .intValue();
+        curatorFramework = CuratorFrameworkFactory.newClient(config.getAccumulo().getZookeepers(), timeout, 10000,
                 retryPolicy);
         curatorFramework.start();
         ensureZkPaths(curatorFramework, zkPaths);
@@ -430,7 +473,7 @@ public class Server {
         // Initialize the MetaCache
         MetaCacheFactory.getCache(config);
         // initialize the auth cache
-        AuthCache.setSessionMaxAge(config.getSecurity());
+        AuthCache.configure(config.getSecurity());
         // Initialize the VisibilityCache
         VisibilityCache.init(config);
         final boolean useEpoll = useEpoll();
@@ -507,10 +550,9 @@ public class Server {
         wsServer.option(ChannelOption.ALLOCATOR, PooledByteBufAllocator.DEFAULT);
         wsServer.option(ChannelOption.SO_BACKLOG, 128);
         wsServer.option(ChannelOption.SO_KEEPALIVE, true);
-        /* Not sure if next three lines are necessary */
+        /* Not sure if next two lines are necessary */
         wsServer.option(ChannelOption.SO_SNDBUF, 1048576);
-        wsServer.option(ChannelOption.WRITE_BUFFER_HIGH_WATER_MARK, 838860);
-        wsServer.option(ChannelOption.WRITE_BUFFER_LOW_WATER_MARK, 620145);
+        wsServer.option(ChannelOption.WRITE_BUFFER_WATER_MARK, new WriteBufferWaterMark(620145, 838860));
         wsChannelHandle = wsServer.bind(wsIp, wsPort).sync().channel();
         final String wsAddress = ((InetSocketAddress) wsChannelHandle.localAddress()).getAddress().getHostAddress();
 
@@ -542,7 +584,7 @@ public class Server {
         if (generate) {
             LOG.warn("Using generated self signed server certificate");
             Date begin = new Date();
-            Date end = new Date(begin.getTime() + 86400000);
+            Date end = new Date(begin.getTime() + TimeUnit.DAYS.toMillis(7));
             SelfSignedCertificate ssc = new SelfSignedCertificate("localhost", begin, end);
             ssl = SslContextBuilder.forServer(ssc.certificate(), ssc.privateKey());
         } else {
@@ -587,14 +629,14 @@ public class Server {
                 ch.pipeline().addLast("decoder", new HttpRequestDecoder());
                 ch.pipeline().addLast("compressor", new HttpContentCompressor());
                 ch.pipeline().addLast("decompressor", new HttpContentDecompressor());
-                ch.pipeline().addLast("aggregator", new HttpObjectAggregator(8192));
+                ch.pipeline().addLast("aggregator", new HttpObjectAggregator(65536));
                 ch.pipeline().addLast("chunker", new ChunkedWriteHandler());
                 final Cors corsCfg = config.getHttp().getCors();
-                final CorsConfig.Builder ccb;
+                final CorsConfigBuilder ccb;
                 if (corsCfg.isAllowAnyOrigin()) {
-                    ccb = new CorsConfig.Builder();
+                    ccb = CorsConfigBuilder.forAnyOrigin();
                 } else {
-                    ccb = new CorsConfig.Builder(corsCfg.getAllowedOrigins().stream().toArray(String[]::new));
+                    ccb = CorsConfigBuilder.forOrigins(corsCfg.getAllowedOrigins().stream().toArray(String[]::new));
                 }
                 if (corsCfg.isAllowNullOrigin()) {
                     ccb.allowNullOrigin();
@@ -609,7 +651,8 @@ public class Server {
                 ch.pipeline().addLast("cors", new CorsHandler(cors));
                 ch.pipeline().addLast("queryDecoder",
                         new timely.netty.http.HttpRequestDecoder(config.getSecurity(), config.getHttp()));
-                ch.pipeline().addLast("fileServer", new HttpStaticFileServerHandler());
+                ch.pipeline().addLast("fileServer", new HttpStaticFileServerHandler()
+                        .setIgnoreSslHandshakeErrors(config.getSecurity().getServerSsl().isUseGeneratedKeypair()));
                 ch.pipeline().addLast("strict", new StrictTransportHandler(config));
                 ch.pipeline().addLast("login", new X509LoginRequestHandler(config.getSecurity(), config.getHttp()));
                 ch.pipeline().addLast("doLogin",
@@ -622,7 +665,9 @@ public class Server {
                 ch.pipeline().addLast("version", new HttpVersionRequestHandler());
                 ch.pipeline().addLast("cache", new HttpCacheRequestHandler(dataStoreCache));
                 ch.pipeline().addLast("put", new HttpMetricPutHandler(dataStore));
-                ch.pipeline().addLast("error", new TimelyExceptionHandler());
+                ch.pipeline().addLast("error", new TimelyExceptionHandler()
+                        .setIgnoreSslHandshakeErrors(config.getSecurity().getServerSsl().isUseGeneratedKeypair()));
+
             }
         };
     }
@@ -635,7 +680,7 @@ public class Server {
                 ch.pipeline().addLast("logger", new LoggingHandler());
                 ch.pipeline().addLast("packetDecoder", new UdpPacketToByteBuf());
                 ch.pipeline().addLast("buffer", new MetricsBufferDecoder());
-                ch.pipeline().addLast("frame", new DelimiterBasedFrameDecoder(8192, true, Delimiters.lineDelimiter()));
+                ch.pipeline().addLast("frame", new DelimiterBasedFrameDecoder(65536, true, Delimiters.lineDelimiter()));
                 ch.pipeline().addLast("putDecoder", new UdpDecoder());
                 ch.pipeline().addLast("putHandler", new TcpPutHandler(dataStore));
             }
@@ -648,7 +693,7 @@ public class Server {
             @Override
             protected void initChannel(SocketChannel ch) throws Exception {
                 ch.pipeline().addLast("buffer", new MetricsBufferDecoder());
-                ch.pipeline().addLast("frame", new DelimiterBasedFrameDecoder(8192, true, Delimiters.lineDelimiter()));
+                ch.pipeline().addLast("frame", new DelimiterBasedFrameDecoder(65536, true, Delimiters.lineDelimiter()));
                 ch.pipeline().addLast("putDecoder", new TcpDecoder());
                 ch.pipeline().addLast("putHandler", new TcpPutHandler(dataStore));
                 ch.pipeline().addLast("versionHandler", new TcpVersionHandler());
@@ -663,11 +708,11 @@ public class Server {
             protected void initChannel(SocketChannel ch) throws Exception {
                 ch.pipeline().addLast("ssl", sslCtx.newHandler(ch.alloc()));
                 ch.pipeline().addLast("httpServer", new HttpServerCodec());
-                ch.pipeline().addLast("aggregator", new HttpObjectAggregator(8192));
+                ch.pipeline().addLast("aggregator", new HttpObjectAggregator(65536));
                 ch.pipeline().addLast("sessionExtractor", new WebSocketFullRequestHandler());
                 ch.pipeline().addLast("idle-handler", new IdleStateHandler(conf.getWebsocket().getTimeout(), 0, 0));
                 ch.pipeline().addLast("ws-protocol",
-                        new WebSocketServerProtocolHandler(WS_PATH, null, true, 65536, true));
+                        new WebSocketServerProtocolHandler(WS_PATH, null, true, 65536, false, true));
                 ch.pipeline().addLast("wsDecoder", new WebSocketRequestDecoder(config.getSecurity()));
                 ch.pipeline().addLast("aggregators", new WSAggregatorsRequestHandler());
                 ch.pipeline().addLast("metrics", new WSMetricsRequestHandler(config));

server/src/main/java/timely/StandaloneServer.java

@@ -52,7 +52,8 @@ public class StandaloneServer extends Server {
             System.exit(1);
         }
 
-        Configuration conf = initializeConfiguration(args);
+        StandaloneServer.applicationContext = initializeContext(args);
+        Configuration conf = StandaloneServer.applicationContext.getBean(Configuration.class);
 
         File accumuloDir = new File(tmp, "accumulo");
         MiniAccumuloConfig macConfig = new MiniAccumuloConfig(accumuloDir, "secret");

server/src/main/java/timely/adapter/accumulo/MetaAdapter.java

@@ -0,0 +1,39 @@
+package timely.adapter.accumulo;
+
+import org.apache.accumulo.core.data.Key;
+import timely.api.model.Meta;
+import timely.model.Metric;
+
+/**
+ * Creates Accumulo data structures from {@link Metric}
+ */
+public class MetaAdapter {
+
+    public static Key createMetricKey(String metricName, Long timestamp) {
+        return new Key(Meta.METRIC_PREFIX + metricName, "", "", timestamp);
+    }
+
+    public static Key createTagKey(String metricName, String tagName, Long timestamp) {
+        return new Key(Meta.TAG_PREFIX + metricName, tagName, "", timestamp);
+    }
+
+    public static Key createValueKey(String metricName, String tagName, String tagValue, Long timestamp) {
+        return new Key(Meta.VALUE_PREFIX + metricName, tagName, tagValue, timestamp);
+    }
+
+    public static Meta parse(Key key) {
+        String row = key.getRow().toString();
+        if (row.startsWith(Meta.METRIC_PREFIX)) {
+            return Meta.parse(key, null, Meta.METRIC_PREFIX);
+        } else if (row.startsWith(Meta.TAG_PREFIX)) {
+            return Meta.parse(key, null, Meta.TAG_PREFIX);
+        } else if (row.startsWith(Meta.VALUE_PREFIX)) {
+            return Meta.parse(key, null, Meta.VALUE_PREFIX);
+        }
+        throw new IllegalStateException("Invalid key in meta " + key.toString());
+    }
+
+    public static String decodeRowKey(Key key) {
+        return parse(key).getMetric();
+    }
+}

server/src/main/java/timely/api/model/Meta.java

@@ -5,6 +5,10 @@ import java.util.List;
 
 import org.apache.accumulo.core.data.Key;
 import org.apache.accumulo.core.data.Value;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.builder.CompareToBuilder;
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
 
 public class Meta implements Comparable<Meta> {
 
@@ -49,60 +53,79 @@ public class Meta implements Comparable<Meta> {
 
     public List<Key> toKeys() {
         List<Key> keys = new ArrayList<>();
-        keys.add(new Key(METRIC_PREFIX + metric));
-        keys.add(new Key(TAG_PREFIX + metric, tagKey));
-        keys.add(new Key(VALUE_PREFIX + metric, tagKey, tagValue));
+        if (StringUtils.isNotBlank(metric)) {
+            keys.add(new Key(METRIC_PREFIX + metric));
+        }
+        if (StringUtils.isNotBlank(tagKey)) {
+            keys.add(new Key(TAG_PREFIX + metric, tagKey));
+        }
+        if (StringUtils.isNotBlank(tagKey) && StringUtils.isNotBlank(tagValue)) {
+            keys.add(new Key(VALUE_PREFIX + metric, tagKey, tagValue));
+        }
         return keys;
     }
 
-    public static Meta parse(Key k, Value v) {
+    public static String prefix(Key key) {
+        String row = key.getRow().toString();
+        if (row.startsWith(Meta.METRIC_PREFIX)) {
+            return Meta.METRIC_PREFIX;
+        } else if (row.startsWith(Meta.TAG_PREFIX)) {
+            return Meta.TAG_PREFIX;
+        } else if (row.startsWith(Meta.VALUE_PREFIX)) {
+            return Meta.VALUE_PREFIX;
+        }
+        throw new IllegalStateException("Invalid key in meta " + key.toString());
+    }
+
+    public static Meta parse(Key key) {
+        return parse(key, null);
+    }
+
+    public static Meta parse(Key key, Value value) {
+        String row = key.getRow().toString();
+        if (row.startsWith(Meta.METRIC_PREFIX)) {
+            return Meta.parse(key, value, Meta.METRIC_PREFIX);
+        } else if (row.startsWith(Meta.TAG_PREFIX)) {
+            return Meta.parse(key, value, Meta.TAG_PREFIX);
+        } else if (row.startsWith(Meta.VALUE_PREFIX)) {
+            return Meta.parse(key, value, Meta.VALUE_PREFIX);
+        }
+        throw new IllegalStateException("Invalid key in meta " + key.toString());
+    }
+
+    public static Meta parse(Key k, Value v, String prefix) {
         if (k.getColumnQualifier().getLength() > 0) {
-            return new Meta(k.getRow().toString().substring(METRIC_PREFIX.length()), k.getColumnFamily().toString(),
+            return new Meta(k.getRow().toString().substring(prefix.length()), k.getColumnFamily().toString(),
                     k.getColumnQualifier().toString());
         } else if (k.getColumnFamily().getLength() > 0) {
-            return new Meta(k.getRow().toString().substring(METRIC_PREFIX.length()), k.getColumnFamily().toString(),
-                    null);
+            return new Meta(k.getRow().toString().substring(prefix.length()), k.getColumnFamily().toString(), null);
 
         } else {
-            return new Meta(k.getRow().toString().substring(METRIC_PREFIX.length()), null, null);
+            return new Meta(k.getRow().toString().substring(prefix.length()), null, null);
         }
     }
 
     @Override
     public int hashCode() {
-        final int prime = 31;
-        int result = 1;
-        result = prime * result + ((metric == null) ? 0 : metric.hashCode());
-        result = prime * result + ((tagKey == null) ? 0 : tagKey.hashCode());
-        result = prime * result + ((tagValue == null) ? 0 : tagValue.hashCode());
-        return result;
+        HashCodeBuilder builder = new HashCodeBuilder();
+        builder.append(metric);
+        builder.append(tagKey);
+        builder.append(tagValue);
+        return builder.toHashCode();
     }
 
     @Override
-    public boolean equals(Object obj) {
-        if (this == obj)
-            return true;
-        if (obj == null)
+    public boolean equals(Object o) {
+        if (o instanceof Meta) {
+            Meta obj = (Meta) o;
+            EqualsBuilder builder = new EqualsBuilder();
+            builder.append(this.metric, obj.metric);
+            builder.append(this.tagKey, obj.tagKey);
+            builder.append(this.tagValue, obj.tagValue);
+            return builder.isEquals();
+        } else {
             return false;
-        if (getClass() != obj.getClass())
-            return false;
-        Meta other = (Meta) obj;
-        if (metric == null) {
-            if (other.metric != null)
-                return false;
-        } else if (!metric.equals(other.metric))
-            return false;
-        if (tagKey == null) {
-            if (other.tagKey != null)
-                return false;
-        } else if (!tagKey.equals(other.tagKey))
-            return false;
-        if (tagValue == null) {
-            if (other.tagValue != null)
-                return false;
-        } else if (!tagValue.equals(other.tagValue))
-            return false;
-        return true;
+        }
     }
 
     @Override
@@ -112,15 +135,10 @@ public class Meta implements Comparable<Meta> {
 
     @Override
     public int compareTo(Meta o) {
-        int result = metric.compareTo(o.metric);
-        if (result != 0) {
-            return result;
-        }
-        result = tagKey.compareTo(o.tagKey);
-        if (result != 0) {
-            return result;
-        }
-        return tagValue.compareTo(o.tagValue);
+        CompareToBuilder builder = new CompareToBuilder();
+        builder.append(this.metric, o.metric);
+        builder.append(this.tagKey, o.tagKey);
+        builder.append(this.tagValue, o.tagValue);
+        return builder.toComparison();
     }
-
 }

server/src/main/java/timely/api/request/AuthenticatedRequest.java

@@ -9,7 +9,7 @@ import timely.auth.util.HttpHeaderUtils;
 import timely.netty.http.auth.TimelyAuthenticationToken;
 
 /**
- * Base class for requests that require authentication
+ * Base class for requests that require auth
  */
 @JsonIgnoreProperties({ "userName", "requestHeader" })
 public class AuthenticatedRequest implements Request {
@@ -44,7 +44,7 @@ public class AuthenticatedRequest implements Request {
         StringBuilder buf = new StringBuilder();
         buf.append("{");
         buf.append("Session ID: ").append(sessionId);
-        buf.append("Token: ").append(token.getTimelyPrincipal().getName());
+        buf.append("Token: ").append(token == null ? null : token.getTimelyPrincipal().getName());
         buf.append(", Request Headers: ").append(requestHeaders.toString());
         buf.append("}");
         return buf.toString();

server/src/main/java/timely/api/request/TcpRequest.java

@@ -4,8 +4,6 @@ public interface TcpRequest extends Request {
 
     /**
      * Parse a line of text received and populate this object
-     * 
-     * @param line
      */
     public void parse(String line);
 

server/src/main/java/timely/api/request/UdpRequest.java

@@ -4,8 +4,6 @@ public interface UdpRequest {
 
     /**
      * Parse a line of text received and populate this object
-     * 
-     * @param line
      */
     public void parse(String line);
 

server/src/main/java/timely/api/response/timeseries/MetricsResponse.java

@@ -18,7 +18,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode;
 import io.netty.buffer.Unpooled;
 import io.netty.handler.codec.http.DefaultFullHttpResponse;
 import io.netty.handler.codec.http.FullHttpResponse;
-import io.netty.handler.codec.http.HttpHeaders.Names;
+import io.netty.handler.codec.http.HttpHeaderNames;
 import io.netty.handler.codec.http.HttpResponseStatus;
 import io.netty.handler.codec.http.HttpVersion;
 import io.netty.handler.codec.http.websocketx.TextWebSocketFrame;
@@ -129,8 +129,8 @@ public class MetricsResponse {
         }
         FullHttpResponse response = new DefaultFullHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.OK,
                 Unpooled.copiedBuffer(buf));
-        response.headers().set(Names.CONTENT_TYPE, responseType);
-        response.headers().set(Names.CONTENT_LENGTH, response.content().readableBytes());
+        response.headers().set(HttpHeaderNames.CONTENT_TYPE, responseType);
+        response.headers().set(HttpHeaderNames.CONTENT_LENGTH, response.content().readableBytes());
         return response;
     }
 
@@ -156,8 +156,10 @@ public class MetricsResponse {
         b.append(TR_END);
         String prevMetric = null;
         StringBuilder tags = new StringBuilder();
+        Map<String, Long> tagMap = new HashMap<>();
         for (Meta m : tree) {
             if (prevMetric != null && !m.getMetric().equals(prevMetric)) {
+                tagMap.clear();
                 b.append(TR_START);
                 b.append(TD_START).append(prevMetric).append(TD_END);
                 b.append(TD_START).append(tags.toString()).append(TD_END);
@@ -168,8 +170,12 @@ public class MetricsResponse {
             if (prevMetric == null) {
                 prevMetric = m.getMetric();
             }
+            long numValues = tagMap.getOrDefault(m.getTagKey(), 0l);
             if (!(this.ignoredTags.contains(m.getTagKey()))) {
-                tags.append(m.getTagKey()).append("=").append(m.getTagValue()).append(" ");
+                if (numValues <= conf.getMetaCache().getMaxTagValues()) {
+                    tags.append(m.getTagKey()).append("=").append(m.getTagValue()).append(" ");
+                    tagMap.put(m.getTagKey(), ++numValues);
+                }
             }
         }
         b.append(TR_START);

server/src/main/java/timely/auth/AuthCache.java

@@ -20,39 +20,96 @@ import timely.netty.http.auth.TimelyAuthenticationToken;
 
 public class AuthCache {
 
-    private static Logger LOG = LoggerFactory.getLogger(AuthCache.class);
+    private static final Logger LOG = LoggerFactory.getLogger(AuthCache.class);
     private static Cache<String, TimelyPrincipal> CACHE = null;
 
-    private static int sessionMaxAge = -1;
+    private static int cacheExpirationMinutes = -1;
+    private static int cacheRefreshMinutes = -1;
 
     /**
      * For tests only
      */
-    public static void resetSessionMaxAge() {
-        sessionMaxAge = -1;
+    public static void resetConfiguration() {
+        cacheExpirationMinutes = -1;
+        cacheRefreshMinutes = -1;
     }
 
-    public static void setSessionMaxAge(Security security) {
-        if (-1 != sessionMaxAge) {
-            throw new IllegalStateException("Cache session max age already configured.");
+    public static void configure(Security security) {
+        if (-1 != cacheExpirationMinutes || -1 != cacheRefreshMinutes) {
+            throw new IllegalStateException("Cache already configured.");
         }
-        sessionMaxAge = security.getSessionMaxAge();
+        cacheExpirationMinutes = security.getCacheExpirationMinutes();
+        cacheRefreshMinutes = security.getCacheRefreshMinutes();
     }
 
-    public static Cache<String, TimelyPrincipal> getCache() {
-        if (-1 == sessionMaxAge) {
+    private static Cache<String, TimelyPrincipal> getCache() {
+        if (-1 == cacheExpirationMinutes) {
             throw new IllegalStateException("Cache session max age not configured.");
         }
         if (null == CACHE) {
-            CACHE = Caffeine.newBuilder().expireAfterAccess(sessionMaxAge, TimeUnit.SECONDS).build();
+            Caffeine<Object, Object> caffeine = Caffeine.newBuilder();
+            caffeine.expireAfterWrite(cacheExpirationMinutes, TimeUnit.MINUTES);
+            if (cacheRefreshMinutes > 0) {
+                caffeine.refreshAfterWrite(cacheRefreshMinutes, TimeUnit.MINUTES);
+                CACHE = caffeine.build(key -> getTimelyPrincipal(key));
+            } else {
+                CACHE = caffeine.build();
+            }
         }
         return CACHE;
     }
 
+    private static TimelyPrincipal getTimelyPrincipal(String key) {
+        LOG.debug("Refreshing TimelyPrincipal {}", key);
+        TimelyPrincipal timelyPrincipal = null;
+        String[] dnArray = key.split(" -> ");
+        if (dnArray.length == 1) {
+            SubjectIssuerDNPair pair = SubjectIssuerDNPair.parse(dnArray[0]);
+            TimelyAuthenticationToken token = AuthenticationService.getAuthenticationToken(new Object(),
+                    pair.subjectDN(), pair.issuerDN());
+            timelyPrincipal = AuthenticationService.authenticate(token, pair, token.getTimelyPrincipal().getName(),
+                    false);
+        } else if (dnArray.length > 1) {
+            Collection<TimelyUser> timelyUsers = new ArrayList<>();
+            for (String s : dnArray) {
+                SubjectIssuerDNPair pair = SubjectIssuerDNPair.parse(s);
+                TimelyAuthenticationToken token = AuthenticationService.getAuthenticationToken(new Object(),
+                        pair.subjectDN(), pair.issuerDN());
+                TimelyPrincipal p = AuthenticationService.authenticate(token, pair,
+                        token.getTimelyPrincipal().getName(), false);
+                timelyUsers.addAll(p.getProxiedUsers());
+            }
+            timelyPrincipal = new TimelyPrincipal(timelyUsers);
+        }
+        return timelyPrincipal;
+    }
+
+    public static boolean containsKey(String key) {
+        return getCache().getIfPresent(key) != null;
+    }
+
+    public static TimelyPrincipal get(String key) {
+        return getCache().getIfPresent(key);
+    }
+
+    public static void put(String key, TimelyPrincipal principal) {
+        getCache().put(key, principal);
+    }
+
+    public static void remove(String key) {
+        getCache().invalidate(key);
+    }
+
+    public static void clear() {
+        if (CACHE != null) {
+            CACHE.invalidateAll();
+        }
+    }
+
     protected static Collection<Authorizations> getAuthorizations(String entityName) {
         if (!StringUtils.isEmpty(entityName)) {
             List<Authorizations> authorizationsList = new ArrayList<>();
-            TimelyPrincipal principal = CACHE.asMap().get(entityName);
+            TimelyPrincipal principal = AuthCache.get(entityName);
             if (principal != null) {
                 for (Collection<String> authCollection : principal.getAuthorizations()) {
                     authorizationsList.add(AuthorizationsUtil.toAuthorizations(authCollection));

server/src/main/java/timely/auth/AuthenticationService.java

@@ -2,8 +2,14 @@ package timely.auth;
 
 import java.security.cert.X509Certificate;
 import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
 import java.util.List;
 import java.util.ServiceConfigurationError;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
 
 import com.google.common.collect.Multimap;
 import io.netty.channel.ChannelHandlerContext;
@@ -17,24 +23,34 @@ import org.springframework.context.ApplicationContext;
 import org.springframework.context.support.ClassPathXmlApplicationContext;
 import org.springframework.security.authentication.AuthenticationManager;
 import org.springframework.security.core.Authentication;
-import org.springframework.security.web.authentication.preauth.x509.SubjectDnX509PrincipalExtractor;
 import timely.api.request.AuthenticatedRequest;
 import timely.api.response.TimelyException;
-import timely.auth.util.AuthenticationUtils;
+import timely.auth.util.DnUtils;
 import timely.netty.http.auth.TimelyAuthenticationToken;
+import timely.netty.http.auth.TimelyUserDetails;
 
 public class AuthenticationService {
 
-    private static Logger LOG = LoggerFactory.getLogger(AuthenticationService.class);
+    private static final Logger LOG = LoggerFactory.getLogger(AuthenticationService.class);
     private static ApplicationContext springContext = null;
     private static AuthenticationManager authManager = null;
-    private static SubjectDnX509PrincipalExtractor x509 = null;
+    private static ConcurrentHashMap<String, Object> fetchingEntity = new ConcurrentHashMap<>();
+    private static ConcurrentHashMap<String, Object> fetchingPrincipal = new ConcurrentHashMap<>();
+    private static Collection<String> requiredRoles;
+    private static Collection<String> requiredAuths;
 
     static {
         try {
             springContext = new ClassPathXmlApplicationContext("security.xml");
-            authManager = (AuthenticationManager) springContext.getBean("authenticationManager");
-            x509 = (SubjectDnX509PrincipalExtractor) springContext.getBean("x509PrincipalExtractor");
+            authManager = springContext.getBean("authenticationManager", AuthenticationManager.class);
+            @SuppressWarnings("unchecked")
+            List<String> uncheckedRoles = springContext.getBean("requiredRoles", List.class);
+            requiredRoles = uncheckedRoles;
+            requiredRoles.removeIf(String::isEmpty);
+            @SuppressWarnings("unchecked")
+            List<String> uncheckedAuths = springContext.getBean("requiredAuths", List.class);
+            requiredAuths = uncheckedAuths;
+            requiredAuths.removeIf(String::isEmpty);
         } catch (BeansException e) {
             throw new ServiceConfigurationError("Error setting up Authentication objects: " + e.getMessage(),
                     e.getRootCause());
@@ -45,23 +61,66 @@ public class AuthenticationService {
         return authManager;
     }
 
-    public static TimelyAuthenticationToken authenticate(Authentication authentication) {
-        return authenticate(authentication, null);
+    public static TimelyPrincipal authenticate(Authentication authentication, String cacheName) {
+        return authenticate(authentication, SubjectIssuerDNPair.of(cacheName), cacheName);
     }
 
-    public static TimelyAuthenticationToken authenticate(Authentication authentication, X509Certificate clientCert) {
-        Authentication token = getAuthenticationManager().authenticate(authentication);
-        return AuthenticationUtils.getTimelyAuthenticationToken(token, clientCert);
+    public static TimelyPrincipal authenticate(Authentication authentication, SubjectIssuerDNPair pair, String entity) {
+        return authenticate(authentication, pair, entity, true);
+    }
+
+    public static TimelyPrincipal authenticate(Authentication authentication, SubjectIssuerDNPair pair, String entity,
+            boolean useCache) {
+
+        // only one thread should call the authenticationManager for a given entity
+        TimelyPrincipal principal = null;
+        if (useCache) {
+            principal = AuthCache.get(entity);
+        }
+        // once an entity is in the AuthCache, we can bypass all of this
+        if (principal == null) {
+            Object newObj = new Object();
+            Object o = fetchingEntity.getOrDefault(entity, newObj);
+            if (o == null) {
+                o = newObj;
+            }
+            try {
+                synchronized (o) {
+                    // if multiple threads are waiting on this synchronized, then the 2nd and all
+                    // others will get the principal here
+                    if (useCache) {
+                        principal = AuthCache.get(entity);
+                    }
+                    if (principal == null) {
+                        Authentication token = getAuthenticationManager().authenticate(authentication);
+                        principal = getTimelyPrincipal(token, pair);
+                        LOG.trace("Got entity principal for {} from AuthenticationManager", entity);
+                        if (useCache) {
+                            AuthCache.put(entity, principal);
+                        }
+                    } else {
+                        LOG.trace("Got entity principal for {} from AuthCache on 2nd attempt", entity);
+                    }
+                }
+            } finally {
+                fetchingEntity.remove(entity);
+            }
+        } else {
+            LOG.trace("Got entity principal for {} from AuthCache on 1st attempt", entity);
+        }
+        return principal;
     }
 
     public static void enforceAccess(AuthenticatedRequest request) throws Exception {
         String sessionId = request.getSessionId();
         X509Certificate clientCert = request.getToken().getClientCert();
+        TimelyPrincipal verifiedPrincipal = null;
         if (StringUtils.isBlank(sessionId) && clientCert == null) {
             throw new TimelyException(HttpResponseStatus.UNAUTHORIZED.code(), "User must log in",
                     "Anonymous access is disabled.  User must either send a client certificate or log in");
         } else if (StringUtils.isNotBlank(sessionId)) {
-            if (!AuthCache.getCache().asMap().containsKey(sessionId)) {
+            verifiedPrincipal = AuthCache.get(sessionId);
+            if (verifiedPrincipal == null) {
                 throw new TimelyException(HttpResponseStatus.UNAUTHORIZED.code(), "User must log in",
                         "Unknown session id was submitted, log in again");
             }
@@ -69,31 +128,92 @@ public class AuthenticationService {
             try {
                 TimelyAuthenticationToken token = request.getToken();
                 TimelyPrincipal requestPrincipal = token.getTimelyPrincipal();
-                if (!AuthCache.getCache().asMap().containsKey(requestPrincipal.getName())) {
-                    List<TimelyUser> authenticatedTimelyUsers = new ArrayList<>();
-                    for (TimelyUser user : requestPrincipal.getProxiedUsers()) {
-                        SubjectIssuerDNPair p = user.getDn();
-                        TimelyAuthenticationToken entityToken = getAuthenticationToken(clientCert, p.subjectDN(),
-                                p.issuerDN());
-                        TimelyPrincipal entityPrincipal = AuthCache.getCache().asMap()
-                                .get(entityToken.getTimelyPrincipal().getName());
-                        if (entityPrincipal == null) {
-                            TimelyAuthenticationToken t = authenticate(entityToken, clientCert);
-                            entityPrincipal = t.getTimelyPrincipal();
-                            AuthCache.getCache().asMap().put(entityPrincipal.getName(), entityPrincipal);
-                        }
-                        authenticatedTimelyUsers.addAll(entityPrincipal.getProxiedUsers());
+                String entity = requestPrincipal.getName();
+                verifiedPrincipal = AuthCache.get(entity);
+                if (verifiedPrincipal == null) {
+                    Object newObj = new Object();
+                    Object o = fetchingPrincipal.getOrDefault(entity, newObj);
+                    if (o == null) {
+                        o = newObj;
                     }
-                    TimelyPrincipal principal = new TimelyPrincipal(authenticatedTimelyUsers);
-                    AuthCache.getCache().asMap().put(principal.getName(), principal);
-                    LOG.debug("Authenticated user {} for request {} with authorizations {}", principal.getName(),
-                            request.toString(), principal.getAuthorizationsString());
+                    try {
+                        synchronized (o) {
+                            // check again if in the AuthCache
+                            verifiedPrincipal = AuthCache.get(entity);
+                            if (verifiedPrincipal == null) {
+                                // first thread here, so authenticate all entities in requestPrincipal
+                                List<TimelyUser> authenticatedTimelyUsers = new ArrayList<>();
+                                for (TimelyUser user : requestPrincipal.getProxiedUsers()) {
+                                    SubjectIssuerDNPair p = user.getDn();
+                                    TimelyAuthenticationToken entityToken = getAuthenticationToken(clientCert,
+                                            p.subjectDN(), p.issuerDN());
+                                    TimelyPrincipal entityPrincipal = authenticate(entityToken, p,
+                                            entityToken.getTimelyPrincipal().getName());
+                                    authenticatedTimelyUsers.addAll(entityPrincipal.getProxiedUsers());
+                                }
+                                verifiedPrincipal = new TimelyPrincipal(authenticatedTimelyUsers);
+                                AuthCache.put(verifiedPrincipal.getName(), verifiedPrincipal);
+                                LOG.debug("Authenticated user {} for request {} with authorizations {}",
+                                        verifiedPrincipal.getName(), request.toString(),
+                                        verifiedPrincipal.getAuthorizationsString());
+                            } else {
+                                LOG.trace("Verified principal {} in AuthCache on 2nd attempt", entity);
+                            }
+                        }
+                    } finally {
+                        fetchingPrincipal.remove(entity);
+                    }
+                } else {
+                    LOG.trace("Verified principal {} in AuthCache on 1st attempt", entity);
                 }
-            } catch (Exception e) {
+            } catch (RuntimeException e) {
                 LOG.error(e.getMessage(), e);
                 throw new TimelyException(HttpResponseStatus.UNAUTHORIZED.code(), "Access denied", e.getMessage());
             }
         }
+        try {
+            checkAllowedAccess(verifiedPrincipal.getPrimaryUser());
+        } catch (TimelyException e) {
+            // validation based on roles and/or auths failed
+            // remove combined principal and primary user from cache
+            // i.e. don't cache a failure
+            AuthCache.remove(verifiedPrincipal.getName());
+            AuthCache.remove(verifiedPrincipal.getPrimaryUser().getName());
+            throw e;
+        }
+
+    }
+
+    private static void checkAllowedAccess(TimelyUser primaryUser) throws TimelyException {
+        Set<String> missingRoles = new TreeSet<>();
+        Set<String> missingAuths = new TreeSet<>();
+        if (requiredRoles != null) {
+            if (!primaryUser.getRoles().containsAll(requiredRoles)) {
+                missingRoles.addAll(requiredRoles);
+                missingRoles.removeAll(primaryUser.getRoles());
+            }
+        }
+        if (requiredAuths != null) {
+            if (!primaryUser.getAuths().containsAll(requiredAuths)) {
+                missingAuths.addAll(requiredRoles);
+                missingAuths.removeAll(primaryUser.getAuths());
+            }
+        }
+
+        if (!missingRoles.isEmpty() || !missingAuths.isEmpty()) {
+            String message = "";
+            if (!missingRoles.isEmpty() && !missingAuths.isEmpty()) {
+                message = "User:" + primaryUser.getName() + " is missing role(s):" + missingRoles + " and auth(s):"
+                        + missingAuths;
+
+            } else if (!missingRoles.isEmpty()) {
+                message = "User:" + primaryUser.getName() + " is missing role(s):" + missingRoles;
+            } else if (missingAuths.isEmpty()) {
+                message = "User:" + primaryUser.getName() + " is missing auths(s):" + missingAuths;
+            }
+            LOG.debug(message);
+            throw new TimelyException(HttpResponseStatus.UNAUTHORIZED.code(), "Access denied", message);
+        }
     }
 
     public static X509Certificate getClientCertificate(ChannelHandlerContext ctx) {
@@ -125,7 +245,7 @@ public class AuthenticationService {
         return authenticationToken;
     }
 
-    public static TimelyAuthenticationToken getAuthenticationToken(X509Certificate clientCert, String subjectDn,
+    public static TimelyAuthenticationToken getAuthenticationToken(Object clientCert, String subjectDn,
             String issuerDn) {
         TimelyAuthenticationToken authenticationToken = null;
         try {
@@ -135,4 +255,21 @@ public class AuthenticationService {
         }
         return authenticationToken;
     }
+
+    private static TimelyPrincipal getTimelyPrincipal(Authentication authentication,
+            SubjectIssuerDNPair subjectIssuerDNPair) {
+        String subjectDn = subjectIssuerDNPair.subjectDN();
+        TimelyUser.UserType userType = DnUtils.isServerDN(subjectDn) ? TimelyUser.UserType.SERVER
+                : TimelyUser.UserType.USER;
+        Collection<String> auths = authentication.getAuthorities().stream().map(a -> a.getAuthority())
+                .collect(Collectors.toList());
+        Object o = authentication.getPrincipal();
+        Collection<String> roles = null;
+        if (o instanceof TimelyUserDetails) {
+            roles = ((TimelyUserDetails) o).getRoles();
+        }
+        TimelyUser timelyUser = new TimelyUser(subjectIssuerDNPair, userType, auths, roles, null,
+                System.currentTimeMillis());
+        return new TimelyPrincipal(Arrays.asList(timelyUser));
+    }
 }

server/src/main/java/timely/auth/ScannerBaseDelegate.java

@@ -16,8 +16,6 @@ import org.apache.accumulo.core.data.Value;
 import org.apache.accumulo.core.data.thrift.IterInfo;
 import org.apache.accumulo.core.security.Authorizations;
 import org.apache.hadoop.io.Text;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 /**
  * A simple wrapper around a {@link ScannerBase} that overrides the methods that
@@ -27,7 +25,6 @@ import org.slf4j.LoggerFactory;
  */
 public class ScannerBaseDelegate implements ScannerBase {
 
-    private static final Logger log = LoggerFactory.getLogger(ScannerBaseDelegate.class);
     private static final String SYSTEM_ITERATOR_NAME_PREFIX = "sys_";
 
     protected final ScannerBase delegate;
@@ -139,10 +136,11 @@ public class ScannerBaseDelegate implements ScannerBase {
     @Override
     public void clearScanIterators() {
         if (delegate instanceof ScannerOptions) {
-            ScannerOptionsHelper opts = new ScannerOptionsHelper((ScannerOptions) delegate);
-            for (IteratorSetting iteratorSetting : opts.getIterators()) {
-                if (!iteratorSetting.getName().startsWith(SYSTEM_ITERATOR_NAME_PREFIX)) {
-                    delegate.removeScanIterator(iteratorSetting.getName());
+            try (ScannerOptionsHelper opts = new ScannerOptionsHelper((ScannerOptions) delegate)) {
+                for (IteratorSetting iteratorSetting : opts.getIterators()) {
+                    if (!iteratorSetting.getName().startsWith(SYSTEM_ITERATOR_NAME_PREFIX)) {
+                        delegate.removeScanIterator(iteratorSetting.getName());
+                    }
                 }
             }
         } else {

server/src/main/java/timely/auth/SubjectIssuerDNPair.java

@@ -13,6 +13,7 @@ import timely.auth.util.ProxiedEntityUtils;
  */
 public class SubjectIssuerDNPair implements Serializable {
 
+    private static final long serialVersionUID = 1L;
     private final String subjectDN;
     private final String issuerDN;