Refactor BalancedMetricResolver to allow HDFS and FILE backing, add callbacks for major events, begin working on tests

Refactor BalancedMetricResolver, allow FILE or HDFS persister, add callbacks for events, begin working on testing Fixes
closes #199 Fix issues when using a generated (self-signed) server certificate
2026-11-10 03:41:25 -10:00 · 2020-04-16 16:32:55 -04:00 · 2020-04-14 18:18:32 -04:00 · 2020-04-14 18:18:27 -04:00 · 2020-04-14 17:03:14 -04:00 · 2020-04-14 17:03:14 -04:00
231 changed files with 10406 additions and 1962 deletions
@@ -4,7 +4,7 @@
  <parent>
    <groupId>timely</groupId>
    <artifactId>timely</artifactId>
-    <version>1.0.1-SNAPSHOT</version>
+    <version>1.2.8-SNAPSHOT</version>
  </parent>
  <artifactId>timely-analytics</artifactId>
  <!--
@@ -68,6 +68,7 @@
    <pluginManagement>
      <plugins>
        <plugin>
+          <groupId>org.apache.maven.plugins</groupId>
          <artifactId>maven-compiler-plugin</artifactId>
          <configuration>
            <compilerId>jdt</compilerId>
@@ -76,7 +77,7 @@
            <dependency>
              <groupId>org.eclipse.tycho</groupId>
              <artifactId>tycho-compiler-jdt</artifactId>
-              <version>0.21.0</version>
+              <version>1.4.0</version>
            </dependency>
          </dependencies>
        </plugin>
@@ -66,7 +66,7 @@ public class SubscriptionSource extends RichSourceFunction<MetricResponse> imple
        LOG.info("Opening summarization job.");
        super.open(parameters);
        client = new WebSocketSubscriptionClient(jp.getTimelyHostname(), jp.getTimelyHttpsPort(), jp.getTimelyWssPort(),
-                jp.isDoLogin(), jp.getTimelyUsername(), jp.getTimelyPassword(), jp.getKeyStoreFile(),
+                jp.isClientAuth(), jp.isDoLogin(), jp.getTimelyUsername(), jp.getTimelyPassword(), jp.getKeyStoreFile(),
                jp.getKeyStoreType(), jp.getKeyStorePass(), jp.getTrustStoreFile(), jp.getTrustStoreType(),
                jp.getTrustStorePass(), jp.isHostVerificationEnabled(), jp.getBufferSize());
    }
@@ -96,7 +96,7 @@ public class SubscriptionSource extends RichSourceFunction<MetricResponse> imple

            @Override
            public void onOpen(Session session, EndpointConfig config) {
-                session.addMessageHandler(new MessageHandler.Whole<String>() {
+                session.addMessageHandler(String.class, new MessageHandler.Whole<String>() {

                    private final SimpleDateFormat formatter = new SimpleDateFormat("yyyyMMdd-HH");
                    private long lastWatermarkTime = 0L;
@@ -67,7 +67,7 @@ public class SummarizationJob {
        })
        .name("Metric Windows")
        .addSink(
-    		new SocketClientSink<MetricHistogram>(jp.getTimelyHostname(), jp.getTimelyTcpPort(),
+            new SocketClientSink<MetricHistogram>(jp.getTimelyHostname(), jp.getTimelyTcpPort(),
                    new MetricHistogram(), 2, true) {

                private final Logger LOG = LoggerFactory
@@ -18,6 +18,7 @@ public class SummarizationJobParameters extends GlobalJobParameters implements S
    private final int timelyTcpPort;
    private final int timelyHttpsPort;
    private final int timelyWssPort;
+    private final boolean clientAuth;
    private final boolean doLogin;
    private final String timelyUsername;
    private final String timelyPassword;
@@ -40,6 +41,7 @@ public class SummarizationJobParameters extends GlobalJobParameters implements S
        timelyTcpPort = params.getInt("timelyTcpPort", 4241);
        timelyHttpsPort = params.getInt("timelyHttpsPort", 4242);
        timelyWssPort = params.getInt("timelyWssPort", 4243);
+        clientAuth = params.getBoolean("clientAuth", false);
        doLogin = params.getBoolean("doLogin", false);
        timelyUsername = params.get("timelyUsername", null);
        timelyPassword = params.get("timelyPassword", null);
@@ -69,6 +71,7 @@ public class SummarizationJobParameters extends GlobalJobParameters implements S
        timelyTcpPort = conf.getInteger("timelyTcpPort", 4241);
        timelyHttpsPort = conf.getInteger("timelyHttpsPort", 4242);
        timelyWssPort = conf.getInteger("timelyWssPort", 4243);
+        clientAuth = conf.getBoolean("clientAuth", false);
        doLogin = conf.getBoolean("doLogin", false);
        timelyUsername = conf.getString("timelyUsername", null);
        timelyPassword = conf.getString("timelyPassword", null);
@@ -112,6 +115,10 @@ public class SummarizationJobParameters extends GlobalJobParameters implements S
        return timelyWssPort;
    }

+    public boolean isClientAuth() {
+        return clientAuth;
+    }
+
    public boolean isDoLogin() {
        return doLogin;
    }
@@ -1,4 +1,4 @@
-#!/bin/bash -vx
+#!/bin/bash

 if [[ `uname` == "Darwin" ]]; then
        THIS_SCRIPT=`python -c 'import os,sys; print os.path.realpath(sys.argv[1])' $0`
@@ -8,11 +8,6 @@ else
        TCNATIVE_SUFFIX="so"
 fi

-# netty tcnative file reference
-#netty-tcnative-1.1.33.Fork18-linux-x86_64-fedora.jar -> libnetty-tcnative.so
-#netty-tcnative-1.1.33.Fork18-linux-x86_64.jar -> libnetty-tcnative.so
-#netty-tcnative-1.1.33.Fork18-osx-x86_64.jar -> libnetty-tcnative.jnilib
-
 THIS_DIR="${THIS_SCRIPT%/*}"
 NATIVE_DIR="${THIS_DIR}/META-INF/native"
 BASE_DIR=${THIS_DIR}/..
@@ -32,14 +27,15 @@ fi
 mkdir -p ${NATIVE_DIR}

 pushd ${BASE_DIR}/bin
-$JAVA_HOME/bin/jar xf ${LIB_DIR}/netty-tcnative*.jar META-INF/native/libnetty_tcnative.${NATIVE_SUFFIX}
-$JAVA_HOME/bin/jar xf ${LIB_DIR}/netty-transport-native-epoll*.jar META-INF/native/libnetty_transport_native_epoll_x86_64.${NATIVE_SUFFIX}
+$JAVA_HOME/bin/jar xf ${LIB_DIR}/netty-tcnative-boringssl-static*.jar META-INF/native/libnetty_tcnative_linux_x86_64.${TCNATIVE_SUFFIX}
+$JAVA_HOME/bin/jar xf ${LIB_DIR}/netty-all*.jar META-INF/native/libnetty_transport_native_epoll_x86_64.${TCNATIVE_SUFFIX}
 popd

-export CLASSPATH="${CONF_DIR}:${LIB_DIR}/*"
-JVM_ARGS="-Xmx1G -Xms1G -Dio.netty.eventLoopThreads=${NUM_SERVER_THREADS}"
+export CLASSPATH="${CONF_DIR}:${LIB_DIR}/*:${HADOOP_CONF_DIR}"
+JVM_ARGS="-Xmx1G -Xms1G -Dio.netty.eventLoopThreads=${NUM_SERVER_THREADS} -Dlog4j.configurationFile=${CONF_DIR}/log4j2-spring.xml"
 JVM_ARGS="${JVM_ARGS} -DLog4jContextSelector=org.apache.logging.log4j.core.async.AsyncLoggerContextSelector"
 JVM_ARGS="${JVM_ARGS} -Djava.library.path=${NATIVE_DIR}"
+
 #JVM_ARGS="${JVM_ARGS} -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5005"

 echo "$JAVA_HOME/bin/java ${JVM_ARGS} timely.balancer.Balancer --spring.config.name=timely --spring.profiles.active=balancer"
@@ -22,5 +22,8 @@
      <AppenderRef ref="Console" level="warn" />
      <AppenderRef ref="Rolling" level="trace" />
    </Root>
+    <Logger name="timely" additivity="false">
+      <AppenderRef ref="Console" level="debug" />
+    </Logger>
  </Loggers>
 </Configuration>
@@ -11,21 +11,33 @@
 		          
 	<beans:bean id="x509PrincipalExtractor" class="org.springframework.security.web.authentication.preauth.x509.SubjectDnX509PrincipalExtractor">
 		<beans:property name="subjectDnRegex" value="CN=(.*?)(?:,|$)"/>
-	</beans:bean>	
+	</beans:bean>

-    <beans:bean id="timelyUserDetailsService" class="timely.auth.UserDetailsService">
-    	<beans:property name="users">
-    		<util:map key-type="java.lang.String">
-    			<beans:entry key="example.com">
-   					<util:list>
-   						<beans:value>D</beans:value>
-   						<beans:value>E</beans:value>
-   						<beans:value>F</beans:value>
-   					</util:list>
-    			</beans:entry>
-    		</util:map>
-    	</beans:property>
-    </beans:bean>
+	<beans:bean id="timelyUserDetailsService" class="timely.auth.UserDetailsService">
+		<beans:property name="users">
+			<util:list>
+				<beans:bean class="timely.auth.TimelyUser">
+					<beans:constructor-arg name="subjectDn">
+						<beans:value>CN=example.com</beans:value>
+					</beans:constructor-arg>
+					<beans:constructor-arg name="auths">
+						<util:list>
+							<beans:value>D</beans:value>
+							<beans:value>E</beans:value>
+							<beans:value>F</beans:value>
+						</util:list>
+					</beans:constructor-arg>
+					<beans:constructor-arg name="roles">
+						<util:list>
+							<beans:value>G</beans:value>
+							<beans:value>H</beans:value>
+							<beans:value>I</beans:value>
+						</util:list>
+					</beans:constructor-arg>
+				</beans:bean>
+			</util:list>
+		</beans:property>
+	</beans:bean>
    
    <beans:bean id="x509AuthorizationProvider" class="org.springframework.security.web.authentication.preauth.PreAuthenticatedAuthenticationProvider">
    	<beans:property name="preAuthenticatedUserDetailsService" ref="timelyUserDetailsService" />
@@ -34,9 +46,12 @@
    <user-service id="usernamePasswordService">
    	<user name="test" password="test1" authorities="A,B,C" />
    </user-service>
-    
-    <beans:bean id="UserNamePasswordAuthenticationProvider" class="org.springframework.security.authentication.dao.DaoAuthenticationProvider">
+
+	<beans:bean id="noOpPasswordEncoder" class="org.springframework.security.crypto.password.NoOpPasswordEncoder"/>
+
+	<beans:bean id="UserNamePasswordAuthenticationProvider" class="org.springframework.security.authentication.dao.DaoAuthenticationProvider">
 		<beans:property name="userDetailsService" ref="usernamePasswordService"/>
+		<beans:property name="passwordEncoder" ref="noOpPasswordEncoder"/>
 	</beans:bean>
 	    
 	<authentication-manager alias="authenticationManager">
@@ -45,5 +60,12 @@
 		<!-- Basic authorization provider, validates provided username and password -->
 		<authentication-provider ref="UserNamePasswordAuthenticationProvider" />
 	</authentication-manager>
-	
+
+	<util:list id="requiredRoles" value-type="java.lang.String">
+		<beans:value></beans:value>
+	</util:list>
+
+	<util:list id="requiredAuths" value-type="java.lang.String">
+		<beans:value></beans:value>
+	</util:list>
 </beans:beans>
@@ -8,12 +8,15 @@ timely-balancer:
  assignmentFile: /timely/balancer/assignments.csv
  security:
    session-max-age: 86400
-    allow-anonymous-access: true
+    cache-expiration-minutes: 60
+    cache-refresh-minutes: 5
+    allow-anonymous-http-access: true
+    allow-anonymous-ws-access: true
    serverSsl:
      certificate-file:
      key-file:
      #key-password:
-      use-generated-keypair: false
+      use-generated-keypair: true
      trust-store-file:
      use-openssl: true
      #use-ciphers:
@@ -31,7 +34,7 @@ timely-balancer:
    nonCachedMetrics:
  server:
    ip: 127.0.0.1
-    tcp-port: 4242
+    tcpPort: 4242
    numTcpPools: 1
    tcpBufferSize: 100
    tcpClientPool:
@@ -51,7 +54,7 @@ timely-balancer:
      jmxEnabled: true
      testOnBorrow: false
      testOnReturn: false
-    udp-port: 4245
+    udpPort: 4245
    udpClientPool:
      minIdlePerKey: 0
      maxIdlePerKey: 40
@@ -108,6 +111,7 @@ timely-balancer:
    flush-interval-seconds: 30
    scanner-read-ahead: 1
    subscription-batch-size: 5000
+    incomingBufferSize: 50000000
    wsClientPool:
      minIdlePerKey: 0
      maxIdlePerKey: 40
@@ -4,7 +4,7 @@
  <parent>
    <groupId>timely</groupId>
    <artifactId>timely</artifactId>
-    <version>1.0.1-SNAPSHOT</version>
+    <version>1.2.8-SNAPSHOT</version>
  </parent>
  <artifactId>timely-balancer</artifactId>
  <dependencies>
@@ -108,6 +108,10 @@
      <groupId>org.apache.hadoop</groupId>
      <artifactId>hadoop-minicluster</artifactId>
    </dependency>
+    <dependency>
+      <groupId>org.glassfish.tyrus</groupId>
+      <artifactId>tyrus-client</artifactId>
+    </dependency>
    <dependency>
      <groupId>org.slf4j</groupId>
      <artifactId>slf4j-api</artifactId>
@@ -161,9 +165,7 @@
    </dependency>
    <dependency>
      <groupId>io.netty</groupId>
-      <artifactId>netty-tcnative</artifactId>
-      <version>${version.netty-tcnative}</version>
-      <classifier>${os.detected.classifier}</classifier>
+      <artifactId>netty-tcnative-boringssl-static</artifactId>
      <scope>runtime</scope>
    </dependency>
    <dependency>
@@ -201,11 +203,6 @@
      <artifactId>spring-boot-test-autoconfigure</artifactId>
      <scope>test</scope>
    </dependency>
-    <dependency>
-      <groupId>timely</groupId>
-      <artifactId>timely-common</artifactId>
-      <scope>test</scope>
-    </dependency>
  </dependencies>
  <build>
    <plugins>
@@ -252,9 +249,8 @@
              <artifactItems>
                <artifactItem>
                  <groupId>io.netty</groupId>
-                  <artifactId>netty-tcnative</artifactId>
+                  <artifactId>netty-tcnative-boringssl-static</artifactId>
                  <version>${version.netty-tcnative}</version>
-                  <classifier>${os.detected.classifier}</classifier>
                  <type>jar</type>
                  <overWrite>false</overWrite>
                  <outputDirectory>${project.build.directory}/native</outputDirectory>
@@ -296,37 +292,4 @@
      </extension>
    </extensions>
  </build>
-  <profiles>
-    <profile>
-      <id>native-epoll-linux</id>
-      <activation>
-        <os>
-          <family>!mac</family>
-        </os>
-      </activation>
-      <dependencies>
-        <dependency>
-          <groupId>io.netty</groupId>
-          <artifactId>netty-transport-native-epoll</artifactId>
-          <version>${version.netty}</version>
-          <classifier>linux-x86_64</classifier>
-        </dependency>
-      </dependencies>
-    </profile>
-    <profile>
-      <id>native-epoll</id>
-      <activation>
-        <os>
-          <family>mac</family>
-        </os>
-      </activation>
-      <dependencies>
-        <dependency>
-          <groupId>io.netty</groupId>
-          <artifactId>netty-transport-native-epoll</artifactId>
-          <version>${version.netty}</version>
-        </dependency>
-      </dependencies>
-    </profile>
-  </profiles>
 </project>
@@ -33,7 +33,7 @@
      <directory>${project.basedir}/bin</directory>
      <outputDirectory>bin</outputDirectory>
      <includes>
-        <include>*</include>
+        <include>**/*</include>
      </includes>
     </fileSet>
    <fileSet>
@@ -1,7 +1,7 @@
 package timely.balancer;

-import java.util.Timer;
-import java.util.TimerTask;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;

 import com.google.common.util.concurrent.AtomicDouble;
@@ -20,38 +20,26 @@ public class ArrivalRate {
    private AtomicDouble longTermRate = new AtomicDouble(0);
    private AtomicLong longTermLastReset = new AtomicLong(0);

-    public ArrivalRate(Timer timer) {
+    public ArrivalRate(ScheduledExecutorService arrivalRateExecutor) {
        long now = System.currentTimeMillis();
        shortTermLastReset.set(now);
        mediumTermLastReset.set(now);
        longTermLastReset.set(now);

        // 2 minute
-        timer.schedule(new TimerTask() {
-
-            @Override
-            public void run() {
-                resetShort();
-            }
-        }, 120000, 120000);
+        arrivalRateExecutor.scheduleAtFixedRate(() -> {
+            resetShort();
+        }, 2, 2, TimeUnit.MINUTES);

        // 5 minute
-        timer.schedule(new TimerTask() {
-
-            @Override
-            public void run() {
-                resetMedium();
-            }
-        }, 300000, 300000);
+        arrivalRateExecutor.scheduleAtFixedRate(() -> {
+            resetMedium();
+        }, 5, 5, TimeUnit.MINUTES);

        // 15 minute
-        timer.schedule(new TimerTask() {
-
-            @Override
-            public void run() {
-                resetLong();
-            }
-        }, 900000, 900000);
+        arrivalRateExecutor.scheduleAtFixedRate(() -> {
+            resetLong();
+        }, 15, 15, TimeUnit.MINUTES);
    }

    private void resetShort() {
@@ -1,5 +1,9 @@
 package timely.balancer;

+import static timely.Server.SERVICE_DISCOVERY_PATH;
+import static timely.store.cache.DataStoreCache.NON_CACHED_METRICS;
+import static timely.store.cache.DataStoreCache.NON_CACHED_METRICS_LOCK_PATH;
+
 import java.io.File;
 import java.net.InetSocketAddress;
 import java.nio.charset.StandardCharsets;
@@ -21,6 +25,7 @@ import io.netty.channel.ChannelHandler;
 import io.netty.channel.ChannelInitializer;
 import io.netty.channel.ChannelOption;
 import io.netty.channel.EventLoopGroup;
+import io.netty.channel.WriteBufferWaterMark;
 import io.netty.channel.epoll.EpollChannelOption;
 import io.netty.channel.epoll.EpollDatagramChannel;
 import io.netty.channel.epoll.EpollEventLoopGroup;
@@ -52,10 +57,19 @@ import io.netty.handler.stream.ChunkedWriteHandler;
 import io.netty.handler.timeout.IdleStateHandler;
 import io.netty.util.NettyRuntime;
 import io.netty.util.internal.SystemPropertyUtil;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.curator.RetryPolicy;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.CuratorFrameworkFactory;
+import org.apache.curator.retry.RetryForever;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.data.Stat;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.boot.WebApplicationType;
 import org.springframework.boot.builder.SpringApplicationBuilder;
 import org.springframework.context.ConfigurableApplicationContext;
+import timely.auth.AuthCache;
 import timely.balancer.configuration.BalancerConfiguration;
 import timely.balancer.configuration.ClientSsl;
 import timely.balancer.configuration.SpringBootstrap;
@@ -63,7 +77,7 @@ import timely.balancer.connection.http.HttpClientPool;
 import timely.balancer.connection.tcp.TcpClientPool;
 import timely.balancer.connection.udp.UdpClientPool;
 import timely.balancer.connection.ws.WsClientPool;
-import timely.balancer.healthcheck.HealthChecker;
+import timely.balancer.healthcheck.TcpHealthChecker;
 import timely.balancer.netty.http.HttpRelayHandler;
 import timely.balancer.netty.tcp.TcpRelayHandler;
 import timely.balancer.netty.udp.UdpRelayHandler;
@@ -81,7 +95,7 @@ import timely.netty.tcp.TcpDecoder;
 import timely.netty.tcp.TcpVersionHandler;
 import timely.netty.udp.UdpDecoder;
 import timely.netty.udp.UdpPacketToByteBuf;
-import timely.netty.websocket.WebSocketHttpCookieHandler;
+import timely.netty.websocket.WebSocketFullRequestHandler;
 import timely.netty.websocket.WebSocketRequestDecoder;
 import timely.netty.websocket.subscription.WSTimelyExceptionHandler;

@@ -115,9 +129,13 @@ public class Balancer {
    protected Channel httpChannelHandle = null;
    protected Channel wsChannelHandle = null;
    protected List<Channel> udpChannelHandleList = new ArrayList<>();
+    private CuratorFramework curatorFramework;
    protected volatile boolean shutdown = false;
    private static final int DEFAULT_EVENT_LOOP_THREADS;

+    private String[] zkPaths = new String[] { LEADER_LATCH_PATH, ASSIGNMENTS_LAST_UPDATED_PATH, ASSIGNMENTS_LOCK_PATH,
+            SERVICE_DISCOVERY_PATH, NON_CACHED_METRICS, NON_CACHED_METRICS_LOCK_PATH };
+
    static {
        DEFAULT_EVENT_LOOP_THREADS = Math.max(1,
                SystemPropertyUtil.getInt("io.netty.eventLoopThreads", NettyRuntime.availableProcessors() * 2));
@@ -127,8 +145,11 @@ public class Balancer {
        this.balancerConfig = balancerConf;
    }

-    protected static ConfigurableApplicationContext initializeConfiguration(String[] args) {
-        return new SpringApplicationBuilder(SpringBootstrap.class).web(false).run(args);
+    protected static ConfigurableApplicationContext initializeContext(String[] args) {
+        SpringApplicationBuilder builder = new SpringApplicationBuilder(SpringBootstrap.class);
+        builder.web(WebApplicationType.NONE);
+        builder.registerShutdownHook(false);
+        return builder.run(args);
    }

    private void shutdownHook() {
@@ -160,8 +181,9 @@ public class Balancer {
            channelFutures.add(wsChannelHandle.close());
        }

+        int udpChannel = 1;
        for (Channel c : udpChannelHandleList) {
-            LOG.info("Closing udpChannelHandle");
+            LOG.info("Closing udpChannelHandle #" + udpChannel++);
            channelFutures.add(c.close());
        }

@@ -170,14 +192,13 @@ public class Balancer {
            try {
                f.get();
            } catch (final Exception e) {
-                LOG.error("Error while shutting down channel: {}", f.channel().config());
-                LOG.error("{}", e.getMessage());
-                e.printStackTrace();
+                LOG.error("Channel:" + f.channel().config() + " -> " + e.getMessage(), e);
            }
        });

-        Integer quietPeriod = balancerConfig.getServer().getShutdownQuietPeriod();
+        int quietPeriod = balancerConfig.getServer().getShutdownQuietPeriod();
        List<Future<?>> groupFutures = new ArrayList<>();
+
        if (tcpBossGroup != null) {
            LOG.info("Shutting down tcpBossGroup");
            groupFutures.add(tcpBossGroup.shutdownGracefully(quietPeriod, 10, TimeUnit.SECONDS));
@@ -222,9 +243,7 @@ public class Balancer {
            try {
                f.get();
            } catch (final Exception e) {
-                LOG.error("Error while shutting down group: {}", f.toString());
-                LOG.error("{}", e.getMessage());
-                e.printStackTrace();
+                LOG.error("Group:" + f.toString() + " -> " + e.getMessage(), e);
            }
        });

@@ -235,10 +254,29 @@ public class Balancer {
            LOG.error("Error closing WebSocketRequestDecoder during shutdown", e);
        }

+        if (this.metricResolver != null) {
+            try {
+                LOG.info("Closing metricResolver");
+                this.metricResolver.stop();
+            } catch (Exception e) {
+                LOG.error(e.getMessage(), e);
+            }
+        }
+
+        if (curatorFramework != null) {
+            try {
+                LOG.info("Closing curatorFramework");
+                curatorFramework.close();
+            } catch (Exception e) {
+                LOG.error(e.getMessage(), e);
+            }
+        }
+
        if (applicationContext != null) {
            LOG.info("Closing applicationContext");
            applicationContext.close();
        }
+
        this.shutdown = true;
        LOG.info("Server shut down.");
    }
@@ -320,8 +358,33 @@ public class Balancer {
                ssl.getKeyFile(), ssl.getKeyType(), ssl.getKeyPassword());
    }

+    private void ensureZkPaths(CuratorFramework curatorFramework, String[] paths) {
+        for (String s : paths) {
+            try {
+                Stat stat = curatorFramework.checkExists().forPath(s);
+                if (stat == null) {
+                    curatorFramework.create().creatingParentContainersIfNeeded().withMode(CreateMode.PERSISTENT)
+                            .forPath(s);
+                }
+            } catch (Exception e) {
+                LOG.info(e.getMessage());
+            }
+
+        }
+    }
+
    public void run() throws Exception {

+        // start curator framework
+        RetryPolicy retryPolicy = new RetryForever(1000);
+        int timeout = Long.valueOf(AccumuloConfiguration.getTimeInMillis(balancerConfig.getZooKeeper().getTimeout()))
+                .intValue();
+        curatorFramework = CuratorFrameworkFactory.newClient(balancerConfig.getZooKeeper().getServers(), timeout, 10000,
+                retryPolicy);
+        curatorFramework.start();
+        ensureZkPaths(curatorFramework, zkPaths);
+
+        AuthCache.configure(balancerConfig.getSecurity());
        final boolean useEpoll = useEpoll();
        Class<? extends ServerSocketChannel> channelClass;
        Class<? extends Channel> datagramChannelClass;
@@ -354,8 +417,9 @@ public class Balancer {
        for (int x = 0; x < balancerConfig.getServer().getNumTcpPools(); x++) {
            tcpClientPools.add(new TcpClientPool(this.balancerConfig));
        }
-        HealthChecker healthChecker = new HealthChecker(this.balancerConfig, tcpClientPools.get(0));
-        this.metricResolver = new BalancedMetricResolver(this.balancerConfig, healthChecker);
+        TcpHealthChecker healthChecker = new TcpHealthChecker(this.balancerConfig, tcpClientPools.get(0));
+        this.metricResolver = new BalancedMetricResolver(curatorFramework, this.balancerConfig, healthChecker);
+        this.metricResolver.start();

        final ServerBootstrap tcpServer = new ServerBootstrap();
        tcpServer.group(tcpBossGroup, tcpWorkerGroup);
@@ -387,7 +451,8 @@ public class Balancer {
        httpServer.group(httpBossGroup, httpWorkerGroup);
        httpServer.channel(channelClass);
        httpServer.handler(new LoggingHandler());
-        HttpClientPool httpClientPool = new HttpClientPool(balancerConfig, clientSSLContext);
+        HttpClientPool httpClientPool = new HttpClientPool(balancerConfig.getSecurity(), balancerConfig.getHttp(),
+                clientSSLContext);
        httpServer.childHandler(setupHttpChannel(balancerConfig, sslCtx, metricResolver, httpClientPool));
        httpServer.option(ChannelOption.ALLOCATOR, PooledByteBufAllocator.DEFAULT);
        httpServer.option(ChannelOption.SO_BACKLOG, 128);
@@ -406,10 +471,9 @@ public class Balancer {
        wsServer.option(ChannelOption.ALLOCATOR, PooledByteBufAllocator.DEFAULT);
        wsServer.option(ChannelOption.SO_BACKLOG, 128);
        wsServer.option(ChannelOption.SO_KEEPALIVE, true);
-        /* Not sure if next three lines are necessary */
+        /* Not sure if next two lines are necessary */
        wsServer.option(ChannelOption.SO_SNDBUF, 1048576);
-        wsServer.option(ChannelOption.WRITE_BUFFER_HIGH_WATER_MARK, 838860);
-        wsServer.option(ChannelOption.WRITE_BUFFER_LOW_WATER_MARK, 620145);
+        wsServer.option(ChannelOption.WRITE_BUFFER_WATER_MARK, new WriteBufferWaterMark(620145, 838860));
        wsChannelHandle = wsServer.bind(wsIp, wsPort).sync().channel();
        final String wsAddress = ((InetSocketAddress) wsChannelHandle.localAddress()).getAddress().getHostAddress();

@@ -434,12 +498,12 @@ public class Balancer {

    public static void main(String[] args) throws Exception {

-        Balancer.applicationContext = Balancer.initializeConfiguration(args);
-        BalancerConfiguration balancerConf = applicationContext.getBean(BalancerConfiguration.class);
+        Balancer.applicationContext = Balancer.initializeContext(args);
+        BalancerConfiguration balancerConf = Balancer.applicationContext.getBean(BalancerConfiguration.class);

-        Balancer b = new Balancer(balancerConf);
+        Balancer balancer = new Balancer(balancerConf);
        try {
-            b.run();
+            balancer.run();
            LATCH.await();
        } catch (final InterruptedException e) {
            LOG.info("Server shutting down.");
@@ -447,7 +511,7 @@ public class Balancer {
            LOG.error("Error running server.", e);
        } finally {
            try {
-                b.shutdown();
+                balancer.shutdown();
            } catch (Exception e) {
                System.exit(1);
            }
@@ -467,16 +531,17 @@ public class Balancer {
                ch.pipeline().addLast("decoder", new HttpRequestDecoder());
                ch.pipeline().addLast("compressor", new HttpContentCompressor());
                ch.pipeline().addLast("decompressor", new HttpContentDecompressor());
-                ch.pipeline().addLast("aggregator", new HttpObjectAggregator(8192));
+                ch.pipeline().addLast("aggregator", new HttpObjectAggregator(65536));
                ch.pipeline().addLast("chunker", new ChunkedWriteHandler());
                ch.pipeline().addLast("queryDecoder", new timely.netty.http.HttpRequestDecoder(
                        balancerConfig.getSecurity(), balancerConfig.getHttp()));
-                ch.pipeline().addLast("fileServer", new HttpStaticFileServerHandler());
+                ch.pipeline().addLast("fileServer", new HttpStaticFileServerHandler().setIgnoreSslHandshakeErrors(
+                        balancerConfig.getSecurity().getServerSsl().isUseGeneratedKeypair()));
                ch.pipeline().addLast("login",
                        new X509LoginRequestHandler(balancerConfig.getSecurity(), balancerConfig.getHttp()));
-                ch.pipeline().addLast("httpRelay",
-                        new HttpRelayHandler(balancerConfig, metricResolver, httpClientPool));
-                ch.pipeline().addLast("error", new TimelyExceptionHandler());
+                ch.pipeline().addLast("httpRelay", new HttpRelayHandler(metricResolver, httpClientPool));
+                ch.pipeline().addLast("error", new TimelyExceptionHandler().setIgnoreSslHandshakeErrors(
+                        balancerConfig.getSecurity().getServerSsl().isUseGeneratedKeypair()));
            }
        };
    }
@@ -489,7 +554,7 @@ public class Balancer {
                ch.pipeline().addLast("logger", new LoggingHandler());
                ch.pipeline().addLast("packetDecoder", new UdpPacketToByteBuf());
                ch.pipeline().addLast("buffer", new MetricsBufferDecoder());
-                ch.pipeline().addLast("frame", new DelimiterBasedFrameDecoder(8192, true, Delimiters.lineDelimiter()));
+                ch.pipeline().addLast("frame", new DelimiterBasedFrameDecoder(65536, true, Delimiters.lineDelimiter()));
                ch.pipeline().addLast("putDecoder", new UdpDecoder());
                ch.pipeline().addLast("udpRelayHandler", new UdpRelayHandler(metricResolver, udpClientPool));
            }
@@ -502,7 +567,7 @@ public class Balancer {
            @Override
            protected void initChannel(SocketChannel ch) throws Exception {
                ch.pipeline().addLast("buffer", new MetricsBufferDecoder());
-                ch.pipeline().addLast("frame", new DelimiterBasedFrameDecoder(8192, true, Delimiters.lineDelimiter()));
+                ch.pipeline().addLast("frame", new DelimiterBasedFrameDecoder(65536, true, Delimiters.lineDelimiter()));
                ch.pipeline().addLast("putDecoder", new TcpDecoder());
                ch.pipeline().addLast("tcpRelayHandler", new TcpRelayHandler(metricResolver, tcpClientPools));
                ch.pipeline().addLast("versionHandler", new TcpVersionHandler());
@@ -518,13 +583,13 @@ public class Balancer {
            protected void initChannel(SocketChannel ch) throws Exception {
                ch.pipeline().addLast("ssl", sslCtx.newHandler(ch.alloc()));
                ch.pipeline().addLast("httpServer", new HttpServerCodec());
-                ch.pipeline().addLast("aggregator", new HttpObjectAggregator(8192));
-                ch.pipeline().addLast("sessionExtractor", new WebSocketHttpCookieHandler(balancerConfig.getSecurity()));
+                ch.pipeline().addLast("aggregator", new HttpObjectAggregator(65536));
+                ch.pipeline().addLast("sessionExtractor", new WebSocketFullRequestHandler());

                ch.pipeline().addLast("idle-handler",
                        new IdleStateHandler(balancerConfig.getWebsocket().getTimeout(), 0, 0));
                ch.pipeline().addLast("ws-protocol",
-                        new WebSocketServerProtocolHandler(WS_PATH, null, true, 65536, true));
+                        new WebSocketServerProtocolHandler(WS_PATH, null, true, 65536, false, true));
                ch.pipeline().addLast("wsDecoder", new WebSocketRequestDecoder(balancerConfig.getSecurity()));
                ch.pipeline().addLast("httpRelay", new WsRelayHandler(balancerConfig, metricResolver, wsClientPool));
                ch.pipeline().addLast("error", new WSTimelyExceptionHandler());
@@ -5,8 +5,10 @@ import javax.validation.Valid;
 import org.springframework.boot.context.properties.ConfigurationProperties;
 import org.springframework.boot.context.properties.NestedConfigurationProperty;
 import org.springframework.stereotype.Component;
+import org.springframework.validation.annotation.Validated;
 import timely.configuration.Cache;

+@Validated
@Component
@ConfigurationProperties(prefix = "timely-balancer")
 public class BalancerConfiguration {
@@ -47,6 +49,16 @@ public class BalancerConfiguration {

    private int serverSuccessesBeforeUp = 3;

+    private String metricAssignmentPersisterType = "HDFS";
+
+    private long balanceDelay = 900000;
+
+    private long balancePeriod = 900000;
+
+    private long persistDelay = 10000;
+
+    private long persistPeriod = 60000;
+
    public ZooKeeper getZooKeeper() {
        return zooKeeper;
    }
@@ -118,4 +130,44 @@ public class BalancerConfiguration {
    public void setServerSuccessesBeforeUp(int serverSuccessesBeforeUp) {
        this.serverSuccessesBeforeUp = serverSuccessesBeforeUp;
    }
+
+    public String getMetricAssignmentPersisterType() {
+        return metricAssignmentPersisterType;
+    }
+
+    public void setMetricAssignmentPersisterType(String metricAssignmentPersisterType) {
+        this.metricAssignmentPersisterType = metricAssignmentPersisterType;
+    }
+
+    public void setBalanceDelay(long balanceDelay) {
+        this.balanceDelay = balanceDelay;
+    }
+
+    public long getBalanceDelay() {
+        return balanceDelay;
+    }
+
+    public void setBalancePeriod(long balancePeriod) {
+        this.balancePeriod = balancePeriod;
+    }
+
+    public long getBalancePeriod() {
+        return balancePeriod;
+    }
+
+    public void setPersistDelay(long persistDelay) {
+        this.persistDelay = persistDelay;
+    }
+
+    public long getPersistDelay() {
+        return persistDelay;
+    }
+
+    public void setPersistPeriod(long persistPeriod) {
+        this.persistPeriod = persistPeriod;
+    }
+
+    public long getPersistPeriod() {
+        return persistPeriod;
+    }
 }
@@ -3,6 +3,7 @@ package timely.balancer.configuration;
 import javax.validation.Valid;

 import org.apache.commons.pool2.impl.GenericKeyedObjectPoolConfig;
+import org.apache.http.impl.client.CloseableHttpClient;
 import org.springframework.boot.context.properties.NestedConfigurationProperty;
 import timely.configuration.Http;

@@ -10,9 +11,9 @@ public class BalancerHttp extends Http {

    @Valid
    @NestedConfigurationProperty
-    private GenericKeyedObjectPoolConfig httpClientPool = new GenericKeyedObjectPoolConfig();
+    private GenericKeyedObjectPoolConfig<CloseableHttpClient> httpClientPool = new GenericKeyedObjectPoolConfig<>();

-    public GenericKeyedObjectPoolConfig getHttpClientPool() {
+    public GenericKeyedObjectPoolConfig<CloseableHttpClient> getHttpClientPool() {
        return httpClientPool;
    }
 }
@@ -4,27 +4,29 @@ import javax.validation.Valid;

 import org.apache.commons.pool2.impl.GenericKeyedObjectPoolConfig;
 import org.springframework.boot.context.properties.NestedConfigurationProperty;
+import timely.client.tcp.TcpClient;
+import timely.client.udp.UdpClient;
 import timely.configuration.Server;

 public class BalancerServer extends Server {

    @Valid
    @NestedConfigurationProperty
-    private GenericKeyedObjectPoolConfig udpClientPool = new GenericKeyedObjectPoolConfig();
+    private GenericKeyedObjectPoolConfig<UdpClient> udpClientPool = new GenericKeyedObjectPoolConfig<>();

    @Valid
    @NestedConfigurationProperty
-    private GenericKeyedObjectPoolConfig tcpClientPool = new GenericKeyedObjectPoolConfig();
+    private GenericKeyedObjectPoolConfig<TcpClient> tcpClientPool = new GenericKeyedObjectPoolConfig<>();

    private int numTcpPools = 1;

    private int tcpBufferSize = -1;

-    public GenericKeyedObjectPoolConfig getUdpClientPool() {
+    public GenericKeyedObjectPoolConfig<UdpClient> getUdpClientPool() {
        return udpClientPool;
    }

-    public GenericKeyedObjectPoolConfig getTcpClientPool() {
+    public GenericKeyedObjectPoolConfig<TcpClient> getTcpClientPool() {
        return tcpClientPool;
    }

@@ -4,15 +4,26 @@ import javax.validation.Valid;

 import org.apache.commons.pool2.impl.GenericKeyedObjectPoolConfig;
 import org.springframework.boot.context.properties.NestedConfigurationProperty;
+import timely.client.websocket.subscription.WebSocketSubscriptionClient;
 import timely.configuration.Websocket;

 public class BalancerWebsocket extends Websocket {

    @Valid
    @NestedConfigurationProperty
-    private GenericKeyedObjectPoolConfig wsClientPool = new GenericKeyedObjectPoolConfig();
+    private GenericKeyedObjectPoolConfig<WebSocketSubscriptionClient> wsClientPool = new GenericKeyedObjectPoolConfig<>();

-    public GenericKeyedObjectPoolConfig getWsClientPool() {
+    public GenericKeyedObjectPoolConfig<WebSocketSubscriptionClient> getWsClientPool() {
        return wsClientPool;
    }
+
+    public int incomingBufferSize = 50000000;
+
+    public int getIncomingBufferSize() {
+        return incomingBufferSize;
+    }
+
+    public void setIncomingBufferSize(int incomingBufferSize) {
+        this.incomingBufferSize = incomingBufferSize;
+    }
 }
@@ -8,6 +8,8 @@ public class ClientSsl {
    private String trustStoreFile;
    private String trustStoreType;
    private String trustStorePassword;
+    private boolean useClientCert = true;
+    private boolean hostVerificationEnabled = true;

    public String getKeyFile() {
        return keyFile;
@@ -55,6 +57,21 @@ public class ClientSsl {

    public void setTrustStorePassword(String trustStorePassword) {
        this.trustStorePassword = trustStorePassword;
+    }

+    public void setUseClientCert(boolean useClientCert) {
+        this.useClientCert = useClientCert;
+    }
+
+    public boolean isUseClientCert() {
+        return useClientCert;
+    }
+
+    public void setHostVerificationEnabled(boolean hostVerificationEnabled) {
+        this.hostVerificationEnabled = hostVerificationEnabled;
+    }
+
+    public boolean isHostVerificationEnabled() {
+        return hostVerificationEnabled;
    }
 }
@@ -1,9 +1,14 @@
 package timely.balancer.configuration;

+import javax.validation.constraints.NotBlank;
+
 public class ZooKeeper {

    private String servers;

+    @NotBlank
+    private String timeout = "120s";
+
    public String getServers() {
        return servers;
    }
@@ -11,4 +16,12 @@ public class ZooKeeper {
    public void setServers(String servers) {
        this.servers = servers;
    }
+
+    public String getTimeout() {
+        return timeout;
+    }
+
+    public void setTimeout(String timeout) {
+        this.timeout = timeout;
+    }
 }
@@ -8,12 +8,9 @@ import org.slf4j.LoggerFactory;
 import timely.balancer.ArrivalRate;
 import timely.balancer.configuration.BalancerConfiguration;

-import java.util.Timer;
-
 public class TimelyBalancedHost {

    private static final Logger LOG = LoggerFactory.getLogger(TimelyBalancedHost.class);
-    static private Timer arrivalRateTimer = new Timer("ArrivalRateTimerTBH", true);

    private String host;
    private int tcpPort;
@@ -24,34 +21,29 @@ public class TimelyBalancedHost {
    private BalancerConfiguration balancerConfig;
    private int failures = 0;
    private int successes = 0;
-    private ArrivalRate arrivalRate = new ArrivalRate(arrivalRateTimer);
+    private ArrivalRate arrivalRate = null;

-    public TimelyBalancedHost() {
+    private TimelyBalancedHost() {

    }

-    public TimelyBalancedHost(String host, int tcpPort, int httpPort, int wsPort, int udpPort) {
+    private TimelyBalancedHost(String host, int tcpPort, int httpPort, int wsPort, int udpPort,
+            ArrivalRate arrivalRate) {
        this.host = host;
        this.tcpPort = tcpPort;
        this.httpPort = httpPort;
        this.wsPort = wsPort;
        this.udpPort = udpPort;
+        this.arrivalRate = arrivalRate;
    }

-    public TimelyBalancedHost(String host, int basePort) {
-        this.host = host;
-        this.tcpPort = basePort;
-        this.httpPort = basePort + 1;
-        this.wsPort = basePort + 2;
-        this.udpPort = basePort + 3;
-    }
-
-    static public TimelyBalancedHost of(String host, int tcpPort, int httpPort, int wsPort, int udpPort) {
-        return new TimelyBalancedHost(host, tcpPort, httpPort, wsPort, udpPort);
+    static public TimelyBalancedHost of(String host, int tcpPort, int httpPort, int wsPort, int udpPort,
+            ArrivalRate arrivalRate) {
+        return new TimelyBalancedHost(host, tcpPort, httpPort, wsPort, udpPort, arrivalRate);
    }

    static public TimelyBalancedHost of(String host, int basePort) {
-        return new TimelyBalancedHost(host, basePort);
+        return new TimelyBalancedHost(host, basePort, basePort + 1, basePort + 2, basePort + 3, null);
    }

    public void setBalancerConfig(BalancerConfiguration balancerConfig) {
@@ -141,11 +133,17 @@ public class TimelyBalancedHost {
    }

    public void arrived() {
-        arrivalRate.arrived();
+        if (this.arrivalRate != null) {
+            this.arrivalRate.arrived();
+        }
    }

    public double getArrivalRate() {
-        return arrivalRate.getRate();
+        if (this.arrivalRate == null) {
+            return 0;
+        } else {
+            return this.arrivalRate.getRate();
+        }
    }

    @Override
@@ -186,4 +184,11 @@ public class TimelyBalancedHost {
        builder.append(isUp);
        return builder.toString();
    }
+
+    public static String toStringShort(TimelyBalancedHost tbh) {
+        if (tbh == null) {
+            return null;
+        }
+        return tbh.host + ":" + tbh.tcpPort;
+    }
 }
@@ -5,22 +5,25 @@ import javax.net.ssl.SSLContext;
 import org.apache.commons.pool2.KeyedPooledObjectFactory;
 import org.apache.commons.pool2.PooledObject;
 import org.apache.commons.pool2.impl.DefaultPooledObject;
-import org.apache.http.impl.client.BasicCookieStore;
 import org.apache.http.impl.client.CloseableHttpClient;
+import timely.balancer.configuration.BalancerSecurity;
 import timely.balancer.connection.TimelyBalancedHost;

 public class HttpClientFactory implements KeyedPooledObjectFactory<TimelyBalancedHost, CloseableHttpClient> {

    private final SSLContext sslContext;
+    private final BalancerSecurity security;

-    public HttpClientFactory(SSLContext sslContext) {
+    public HttpClientFactory(BalancerSecurity security, SSLContext sslContext) {
+        this.security = security;
        this.sslContext = sslContext;
    }

    @Override
    public PooledObject<CloseableHttpClient> makeObject(TimelyBalancedHost k) throws Exception {
-        BasicCookieStore cookieJar = new BasicCookieStore();
-        return new DefaultPooledObject<>(timely.client.http.HttpClient.get(this.sslContext, cookieJar, false));
+        // disable cookie management because we are sharing connections
+        return new DefaultPooledObject<>(timely.client.http.HttpClient.get(this.sslContext, null,
+                security.getClientSsl().isHostVerificationEnabled(), security.getClientSsl().isUseClientCert()));
    }

    @Override
@@ -4,16 +4,13 @@ import javax.net.ssl.SSLContext;

 import org.apache.commons.pool2.impl.GenericKeyedObjectPool;
 import org.apache.http.impl.client.CloseableHttpClient;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import timely.balancer.configuration.BalancerConfiguration;
+import timely.balancer.configuration.BalancerHttp;
+import timely.balancer.configuration.BalancerSecurity;
 import timely.balancer.connection.TimelyBalancedHost;

 public class HttpClientPool extends GenericKeyedObjectPool<TimelyBalancedHost, CloseableHttpClient> {

-    private static final Logger LOG = LoggerFactory.getLogger(HttpClientPool.class);
-
-    public HttpClientPool(BalancerConfiguration config, SSLContext sslContext) {
-        super(new HttpClientFactory(sslContext), config.getHttp().getHttpClientPool());
+    public HttpClientPool(BalancerSecurity balancerSecurity, BalancerHttp http, SSLContext sslContext) {
+        super(new HttpClientFactory(balancerSecurity, sslContext), http.getHttpClientPool());
    }
 }
@@ -12,21 +12,21 @@ import timely.client.websocket.subscription.WebSocketSubscriptionClient;
 public class WsClientFactory implements KeyedPooledObjectFactory<TimelyBalancedHost, WebSocketSubscriptionClient> {

    private final SSLContext sslContext;
-    private final boolean doLogin;
    private BalancerConfiguration balancerConfig;

    public WsClientFactory(BalancerConfiguration balancerConfig, SSLContext sslContext) {
-        this.sslContext = sslContext;
-        this.doLogin = balancerConfig.isLoginRequired();
        this.balancerConfig = balancerConfig;
+        this.sslContext = sslContext;
    }

    @Override
    public PooledObject<WebSocketSubscriptionClient> makeObject(TimelyBalancedHost k) throws Exception {

-        int bufferSize = balancerConfig.getWebsocket().getSubscriptionBatchSize() * 500;
+        int bufferSize = balancerConfig.getWebsocket().getIncomingBufferSize();
        WebSocketSubscriptionClient client = new WebSocketSubscriptionClient(sslContext, k.getHost(), k.getHttpPort(),
-                k.getWsPort(), doLogin, "", "", false, bufferSize);
+                k.getWsPort(), balancerConfig.getSecurity().getClientSsl().isUseClientCert(),
+                balancerConfig.isLoginRequired(), "", "",
+                balancerConfig.getSecurity().getClientSsl().isHostVerificationEnabled(), bufferSize);
        return new DefaultPooledObject<>(client);
    }

@@ -1,53 +1,12 @@
 package timely.balancer.healthcheck;

-import java.util.ArrayList;
 import java.util.List;
-import java.util.Timer;
-import java.util.TimerTask;

-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import timely.balancer.configuration.BalancerConfiguration;
 import timely.balancer.connection.TimelyBalancedHost;
-import timely.balancer.connection.tcp.TcpClientPool;

-public class HealthChecker {
+public interface HealthChecker {

-    private static final Logger LOG = LoggerFactory.getLogger(HealthChecker.class);
-    private List<TimelyBalancedHost> timelyHosts = new ArrayList<>();
-    private Timer timer;
-    final private TCPServerHealthCheck tcpServerHealthCheck;
-    private TimerTask check = new TimerTask() {
+    public void setTimelyHosts(List<TimelyBalancedHost> timelyHosts);

-        @Override
-        public void run() {
-            synchronized (timelyHosts) {
-                for (TimelyBalancedHost h : timelyHosts) {
-                    try {
-                        if (tcpServerHealthCheck.isServerHealthy(h)) {
-                            h.reportSuccess();
-                        } else {
-                            h.reportFailure();
-                        }
-                    } catch (Exception e) {
-                        LOG.error(e.getMessage(), e);
-                        h.reportFailure();
-                    }
-                }
-            }
-        }
-    };
-
-    public HealthChecker(BalancerConfiguration config, TcpClientPool tcpClientPool) {
-        this.tcpServerHealthCheck = new TCPServerHealthCheck(tcpClientPool);
-        this.timer = new Timer("HealthCheckerThread", true);
-        this.timer.scheduleAtFixedRate(check, 0, config.getCheckServerHealthInterval());
-    }
-
-    public void setTimelyHosts(List<TimelyBalancedHost> timelyHosts) {
-        synchronized (timelyHosts) {
-            this.timelyHosts.clear();
-            this.timelyHosts.addAll(timelyHosts);
-        }
-    }
+    public void close() throws Exception;
 }
@@ -0,0 +1,62 @@
+package timely.balancer.healthcheck;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import timely.balancer.configuration.BalancerConfiguration;
+import timely.balancer.connection.TimelyBalancedHost;
+import timely.balancer.connection.tcp.TcpClientPool;
+
+public class TcpHealthChecker implements HealthChecker {
+
+    private static final Logger LOG = LoggerFactory.getLogger(TcpHealthChecker.class);
+    private List<TimelyBalancedHost> timelyHosts = new ArrayList<>();
+    private ScheduledExecutorService executorService = Executors.newScheduledThreadPool(1);
+    final private TCPServerHealthCheck tcpServerHealthCheck;
+
+    public TcpHealthChecker(BalancerConfiguration config, TcpClientPool tcpClientPool) {
+        this.tcpServerHealthCheck = new TCPServerHealthCheck(tcpClientPool);
+        this.executorService.scheduleAtFixedRate(() -> {
+            synchronized (timelyHosts) {
+                for (TimelyBalancedHost h : timelyHosts) {
+                    try {
+                        if (tcpServerHealthCheck.isServerHealthy(h)) {
+                            h.reportSuccess();
+                        } else {
+                            h.reportFailure();
+                        }
+                    } catch (Exception e) {
+                        LOG.error(e.getMessage(), e);
+                        h.reportFailure();
+                    }
+                }
+            }
+        }, 0, config.getCheckServerHealthInterval(), TimeUnit.MILLISECONDS);
+    }
+
+    @Override
+    public void setTimelyHosts(List<TimelyBalancedHost> timelyHosts) {
+        synchronized (timelyHosts) {
+            this.timelyHosts.clear();
+            this.timelyHosts.addAll(timelyHosts);
+        }
+    }
+
+    public void close() throws Exception {
+        this.executorService.shutdown();
+        try {
+            this.executorService.awaitTermination(5, TimeUnit.SECONDS);
+        } catch (InterruptedException e) {
+
+        } finally {
+            if (!this.executorService.isTerminated()) {
+                this.executorService.shutdownNow();
+            }
+        }
+    }
+}
@@ -1,9 +1,8 @@
 package timely.balancer.netty.http;

-import static io.netty.handler.codec.http.HttpHeaders.Names.CONTENT_LENGTH;
+import static io.netty.handler.codec.http.HttpHeaderNames.CONTENT_LENGTH;

 import java.io.ByteArrayOutputStream;
-import java.net.URI;
 import java.net.URLEncoder;
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
@@ -12,6 +11,7 @@ import java.util.Iterator;
 import java.util.List;
 import java.util.Map;

+import com.google.common.collect.Multimap;
 import io.netty.buffer.Unpooled;
 import io.netty.channel.ChannelHandlerContext;
 import io.netty.channel.SimpleChannelInboundHandler;
@@ -33,26 +33,26 @@ import org.apache.http.message.BasicHeader;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.util.StreamUtils;
+import timely.api.request.AuthenticatedRequest;
 import timely.api.request.MetricRequest;
 import timely.api.request.timeseries.HttpRequest;
 import timely.api.request.timeseries.QueryRequest;
 import timely.api.response.TimelyException;
-import timely.balancer.configuration.BalancerConfiguration;
+import timely.auth.util.HttpHeaderUtils;
+import timely.auth.util.ProxiedEntityUtils;
 import timely.balancer.connection.TimelyBalancedHost;
 import timely.balancer.connection.http.HttpClientPool;
 import timely.balancer.resolver.MetricResolver;
 import timely.netty.http.TimelyHttpHandler;
+import timely.netty.http.auth.TimelyAuthenticationToken;

 public class HttpRelayHandler extends SimpleChannelInboundHandler<HttpRequest> implements TimelyHttpHandler {

    private static final Logger LOG = LoggerFactory.getLogger(HttpRelayHandler.class);
-    private final BalancerConfiguration balancerConf;
    private final HttpClientPool httpClientPool;
    private MetricResolver metricResolver;

-    public HttpRelayHandler(BalancerConfiguration balancerConf, MetricResolver metricResolver,
-            HttpClientPool httpClientPool) {
-        this.balancerConf = balancerConf;
+    public HttpRelayHandler(MetricResolver metricResolver, HttpClientPool httpClientPool) {
        this.metricResolver = metricResolver;
        this.httpClientPool = httpClientPool;
    }
@@ -67,7 +67,7 @@ public class HttpRelayHandler extends SimpleChannelInboundHandler<HttpRequest> i
        try {
            try {
                request = msg.getHttpRequest();
-                String originalURI = request.getUri();
+                String originalURI = request.uri();
                originalURI = encodeURI(originalURI);

                String metric = null;
@@ -86,14 +86,22 @@ public class HttpRelayHandler extends SimpleChannelInboundHandler<HttpRequest> i
                } else {
                    k = metricResolver.getHostPortKey(null);
                }
-                k = metricResolver.getHostPortKey(metric);
                client = httpClientPool.borrowObject(k);

-                List<Map.Entry<String, String>> headerList = request.headers().entries();
-                List<Header> relayedHeaderList = new ArrayList<>();
+                Multimap<String, String> headers = HttpHeaderUtils.toMultimap(request.headers());
+                if (msg instanceof AuthenticatedRequest) {
+                    AuthenticatedRequest authenticatedRequest = (AuthenticatedRequest) msg;
+                    TimelyAuthenticationToken token = authenticatedRequest.getToken();
+                    if (token != null) {
+                        if (token.getClientCert() != null) {
+                            ProxiedEntityUtils.addProxyHeaders(headers, token.getClientCert());
+                        }
+                    }
+                }

-                for (Map.Entry<String, String> h : headerList) {
-                    if (!h.getKey().equals(CONTENT_LENGTH)) {
+                List<Header> relayedHeaderList = new ArrayList<>();
+                for (Map.Entry<String, String> h : headers.entries()) {
+                    if (!h.getKey().equals(CONTENT_LENGTH.toString())) {
                        relayedHeaderList.add(new BasicHeader(h.getKey(), h.getValue()));
                    }
                }
@@ -101,14 +109,14 @@ public class HttpRelayHandler extends SimpleChannelInboundHandler<HttpRequest> i
                relayedHeaderList.toArray(headerArray);

                String relayURI = "https://" + k.getHost() + ":" + k.getHttpPort() + originalURI;
-                if (request.getMethod().equals(HttpMethod.GET)) {
+                if (request.method().equals(HttpMethod.GET)) {
                    HttpUriRequest relayedRequest = new HttpGet(relayURI);
                    if (headerArray.length > 0) {
                        relayedRequest.setHeaders(headerArray);
                    }
                    relayedResponse = client.execute(relayedRequest);

-                } else if (request.getMethod().equals(HttpMethod.POST)) {
+                } else if (request.method().equals(HttpMethod.POST)) {
                    HttpPost relayedRequest = new HttpPost(relayURI);

                    String content = request.content().toString(StandardCharsets.UTF_8);
@@ -143,7 +151,6 @@ public class HttpRelayHandler extends SimpleChannelInboundHandler<HttpRequest> i
                this.sendHttpError(ctx, new TimelyException(HttpResponseStatus.METHOD_NOT_ALLOWED.code(),
                        "Method not allowed", "Method not allowed"));
            } else {
-                // buf = JsonUtil.getObjectMapper().writeValueAsBytes(msg);
                ByteArrayOutputStream baos = new MyByteArrayOutputStream();
                StreamUtils.copy(relayedResponse.getEntity().getContent(), baos);

@@ -153,8 +160,6 @@ public class HttpRelayHandler extends SimpleChannelInboundHandler<HttpRequest> i
                for (Header h : relayedResponse.getAllHeaders()) {
                    response.headers().add(h.getName(), h.getValue());
                }
-                // response.headers().set(HttpHeaders.Names.CONTENT_TYPE,
-                // Constants.JSON_TYPE);
                response.headers().set(CONTENT_LENGTH, response.content().readableBytes());
                sendResponse(ctx, response);
            }
@@ -165,7 +170,7 @@ public class HttpRelayHandler extends SimpleChannelInboundHandler<HttpRequest> i
            if (client != null && k != null) {
                httpClientPool.returnObject(k, client);
            } else {
-                LOG.error("NOT RETURNING CONNECTION! " + msg.getHttpRequest().getUri());
+                LOG.error("NOT RETURNING CONNECTION! " + msg.getHttpRequest().uri());
            }
        }
    }
@@ -197,13 +202,4 @@ public class HttpRelayHandler extends SimpleChannelInboundHandler<HttpRequest> i
        }
        return s;
    }
-
-    public static void main(String[] args) {
-
-        String s = "https://127.0.0.1:4243/api/query?start=1356998400000&end=1528398304000&m=sum:rate{false,100,0}:sys.thermal.gauge{class=aws}";
-        URI uri = URI.create(encodeURI(s));
-
-        System.out.println(uri.toString());
-    }
-
 }
@@ -1,12 +1,16 @@
 package timely.balancer.netty.ws;

+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
 import java.util.concurrent.TimeUnit;

 import javax.websocket.CloseReason;
 import javax.websocket.EndpointConfig;
-import javax.websocket.MessageHandler;
 import javax.websocket.Session;

+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.Multimap;
 import io.netty.channel.ChannelHandlerContext;
 import io.netty.handler.codec.http.HttpResponseStatus;
 import io.netty.handler.codec.http.websocketx.PingWebSocketFrame;
@@ -15,25 +19,48 @@ import io.netty.util.concurrent.ScheduledFuture;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import timely.api.response.TimelyException;
+import timely.auth.util.ProxiedEntityUtils;
 import timely.client.websocket.ClientHandler;
+import timely.netty.http.auth.TimelyAuthenticationToken;

 public class WsClientHandler extends ClientHandler {

    private static final Logger LOG = LoggerFactory.getLogger(WsClientHandler.class);
    private final ChannelHandlerContext ctx;
+    private final TimelyAuthenticationToken token;
    private ScheduledFuture<?> ping;

-    public WsClientHandler(ChannelHandlerContext ctx, int pingRate) {
+    public WsClientHandler(ChannelHandlerContext ctx, TimelyAuthenticationToken token, int pingRate) {
        this.ctx = ctx;
+        this.token = token;
        this.ping = this.ctx.executor().scheduleAtFixedRate(() -> {
            LOG.trace("Sending ping on channel {}", ctx.channel());
            ctx.writeAndFlush(new PingWebSocketFrame());
        }, pingRate, pingRate, TimeUnit.SECONDS);
    }

+    @Override
+    public void beforeRequest(Map<String, List<String>> headers) {
+        if (token.getClientCert() != null) {
+            Multimap<String, String> proxyRequestHeaders = HashMultimap.create();
+            ProxiedEntityUtils.addProxyHeaders(proxyRequestHeaders, token.getClientCert());
+            for (String s : proxyRequestHeaders.keySet()) {
+                headers.put(s, new ArrayList<>(proxyRequestHeaders.get(s)));
+            }
+        }
+
+        Multimap<String, String> originalRequestHeaders = token.getHttpHeaders();
+        for (String s : originalRequestHeaders.keySet()) {
+            if (!headers.containsKey(s)) {
+                // add pre-existing values if key does not exist in proxyRequest
+                headers.put(s, new ArrayList<>(originalRequestHeaders.get(s)));
+            }
+        }
+    }
+
    @Override
    public void onOpen(Session session, EndpointConfig config) {
-        session.addMessageHandler((MessageHandler.Whole<String>) message -> {
+        session.addMessageHandler(String.class, message -> {
            ctx.writeAndFlush(new TextWebSocketFrame(message));
            LOG.debug("Message received on Websocket session {}: {}", session.getId(), message);
        });
@@ -9,6 +9,7 @@ import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.SerializationFeature;
 import com.fasterxml.jackson.datatype.jdk8.Jdk8Module;
+import com.google.common.collect.Multimap;
 import io.netty.channel.ChannelFuture;
 import io.netty.channel.ChannelFutureListener;
 import io.netty.channel.ChannelHandlerContext;
@@ -24,6 +25,7 @@ import timely.api.request.subscription.CreateSubscription;
 import timely.api.request.subscription.RemoveSubscription;
 import timely.api.request.subscription.SubscriptionRequest;
 import timely.api.response.TimelyException;
+import timely.auth.util.ProxiedEntityUtils;
 import timely.balancer.configuration.BalancerConfiguration;
 import timely.balancer.connection.TimelyBalancedHost;
 import timely.balancer.connection.ws.WsClientPool;
@@ -64,6 +66,10 @@ public class WsRelayHandler extends SimpleChannelInboundHandler<SubscriptionRequ
        String subscriptionId;
        try {
            String metric = null;
+
+            Multimap<String, String> headers = msg.getRequestHeaders();
+            ProxiedEntityUtils.addProxyHeaders(headers, msg.getToken().getClientCert());
+
            if (msg instanceof CreateSubscription) {
                CreateSubscription create = (CreateSubscription) msg;
                final String currentSubscriptionId = create.getSubscriptionId();
@@ -118,7 +124,8 @@ public class WsRelayHandler extends SimpleChannelInboundHandler<SubscriptionRequ
                    if (hostClientHolder == null) {
                        k = metricResolver.getHostPortKey(metric);
                        client = wsClientPool.borrowObject(k);
-                        client.open(new WsClientHandler(origContext, (balancerConfig.getWebsocket().getTimeout() / 2)));
+                        client.open(new WsClientHandler(origContext, add.getToken(),
+                                (balancerConfig.getWebsocket().getTimeout() / 2)));
                        metricToClientMap.put(metric, new WsClientHolder(k, client));
                    } else {
                        client = hostClientHolder.getClient();
@@ -4,8 +4,14 @@ import static timely.Server.SERVICE_DISCOVERY_PATH;
 import static timely.balancer.Balancer.ASSIGNMENTS_LAST_UPDATED_PATH;
 import static timely.balancer.Balancer.ASSIGNMENTS_LOCK_PATH;
 import static timely.balancer.Balancer.LEADER_LATCH_PATH;
+import static timely.balancer.resolver.eventing.MetricAssignedEvent.Reason.ASSIGN_FALLBACK_SEQUENTIAL;
+import static timely.balancer.resolver.eventing.MetricAssignedEvent.Reason.ASSIGN_ROUND_ROBIN;
+import static timely.balancer.resolver.eventing.MetricAssignedEvent.Reason.HOST_DOWN_ROUND_ROBIN;
+import static timely.balancer.resolver.eventing.MetricAssignedEvent.Reason.HOST_REMOVED;
+import static timely.balancer.resolver.eventing.MetricAssignedEvent.Reason.REBALANCE;
+import static timely.store.cache.DataStoreCache.NON_CACHED_METRICS;
+import static timely.store.cache.DataStoreCache.NON_CACHED_METRICS_LOCK_PATH;

-import java.nio.charset.Charset;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -17,20 +23,19 @@ import java.util.List;
 import java.util.Map;
 import java.util.Random;
 import java.util.Set;
-import java.util.Timer;
-import java.util.TimerTask;
 import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.locks.ReentrantReadWriteLock;

-import com.csvreader.CsvReader;
-import com.csvreader.CsvWriter;
 import org.apache.commons.lang.StringUtils;
-import org.apache.curator.RetryPolicy;
+import org.apache.commons.lang3.SerializationUtils;
 import org.apache.curator.framework.CuratorFramework;
-import org.apache.curator.framework.CuratorFrameworkFactory;
-import org.apache.curator.framework.recipes.atomic.DistributedAtomicLong;
+import org.apache.curator.framework.recipes.atomic.DistributedAtomicValue;
 import org.apache.curator.framework.recipes.cache.TreeCache;
 import org.apache.curator.framework.recipes.cache.TreeCacheEvent;
 import org.apache.curator.framework.recipes.cache.TreeCacheListener;
@@ -38,18 +43,13 @@ import org.apache.curator.framework.recipes.leader.LeaderLatch;
 import org.apache.curator.framework.recipes.leader.LeaderLatchListener;
 import org.apache.curator.framework.recipes.locks.InterProcessReadWriteLock;
 import org.apache.curator.framework.state.ConnectionState;
-import org.apache.curator.retry.RetryUntilElapsed;
+import org.apache.curator.retry.RetryForever;
 import org.apache.curator.x.discovery.ServiceCache;
 import org.apache.curator.x.discovery.ServiceDiscovery;
 import org.apache.curator.x.discovery.ServiceDiscoveryBuilder;
 import org.apache.curator.x.discovery.ServiceInstance;
 import org.apache.curator.x.discovery.details.ServiceCacheListener;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.zookeeper.data.Stat;
+import org.apache.zookeeper.CreateMode;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import timely.ServerDetails;
@@ -57,136 +57,184 @@ import timely.balancer.ArrivalRate;
 import timely.balancer.configuration.BalancerConfiguration;
 import timely.balancer.connection.TimelyBalancedHost;
 import timely.balancer.healthcheck.HealthChecker;
+import timely.balancer.resolver.eventing.MetricAssignedCallback;
+import timely.balancer.resolver.eventing.MetricAssignedEvent;
+import timely.balancer.resolver.eventing.MetricBalanceCallback;
+import timely.balancer.resolver.eventing.MetricBalanceEvent;
+import timely.balancer.resolver.eventing.MetricHostCallback;
+import timely.balancer.resolver.eventing.MetricHostEvent;

 public class BalancedMetricResolver implements MetricResolver {

    private static final Logger LOG = LoggerFactory.getLogger(BalancedMetricResolver.class);
    private Map<String, TimelyBalancedHost> metricToHostMap = new TreeMap<>();
+    private Set<String> unassignedMetrics = new HashSet<>();
    private Map<String, ArrivalRate> metricMap = new HashMap<>();
    private ReentrantReadWriteLock balancerLock = new ReentrantReadWriteLock();
    private List<TimelyBalancedHost> serverList = new ArrayList<>();
    private Random r = new Random();
-    final private HealthChecker healthChecker;
-    private Timer timer = new Timer("RebalanceTimer", true);
-    private Timer arrivalRateTimer = new Timer("AriivalRateTimerResolver", true);
+    private final HealthChecker healthChecker;
+    private ScheduledExecutorService assignmentExecutor = Executors.newScheduledThreadPool(2);
+    private ScheduledExecutorService arrivalRateExecutor = Executors.newScheduledThreadPool(2);
    private int roundRobinCounter = 0;
    private Set<String> nonCachedMetrics = new HashSet<>();
-    private ReentrantReadWriteLock nonCachedMetricsLock = new ReentrantReadWriteLock();
-    private BalancerConfiguration balancerConfig;
-    private CuratorFramework curatorFramework;
-    private RetryPolicy retryPolicy = new RetryUntilElapsed(60000, 1000);
+    private DistributedAtomicValue nonCachedMetricsIP;
+    private ReentrantReadWriteLock nonCachedMetricsLocalLock = new ReentrantReadWriteLock();
+    private InterProcessReadWriteLock nonCachedMetricsIPRWLock;
+    private AtomicLong assignmentsLastUpdatedLocal = new AtomicLong(0);
    private LeaderLatch leaderLatch;
    private AtomicBoolean isLeader = new AtomicBoolean(false);
-    private InterProcessReadWriteLock assignmentsLock;
-    private DistributedAtomicLong assignmentsLastUpdatedInHdfs;
-    private AtomicLong assignmentsLastUpdatedLocal = new AtomicLong(0);
+    private InterProcessReadWriteLock assignmentsIPRWLock;
+    private MetricAssignmentPerister assignmentPerister;

-    private String[] zkPaths = new String[] { LEADER_LATCH_PATH, ASSIGNMENTS_LAST_UPDATED_PATH, ASSIGNMENTS_LOCK_PATH,
-            SERVICE_DISCOVERY_PATH };
+    final private BalancerConfiguration balancerConfig;
+    final private CuratorFramework curatorFramework;

-    private enum BalanceType {
+    private List<MetricAssignedCallback> metricAssignedCallbacks = new ArrayList<>();
+    private List<MetricBalanceCallback> metricBalanceCallbacks = new ArrayList<>();
+    private List<MetricHostCallback> metricHostCallbacks = new ArrayList<>();
+
+    public enum BalanceType {
        HIGH_LOW, HIGH_AVG, AVG_LOW;
    }

-    public BalancedMetricResolver(BalancerConfiguration balancerConfig, HealthChecker healthChecker) {
+    public BalancedMetricResolver(CuratorFramework curatorFramework, BalancerConfiguration balancerConfig,
+            HealthChecker healthChecker) {
+        this.curatorFramework = curatorFramework;
        this.balancerConfig = balancerConfig;
        this.healthChecker = healthChecker;
+    }
+
+    public void start() {
+        assignmentsIPRWLock = new InterProcessReadWriteLock(curatorFramework, ASSIGNMENTS_LOCK_PATH);
+        testIPRWLock(curatorFramework, assignmentsIPRWLock, ASSIGNMENTS_LOCK_PATH);
+
+        this.assignmentPerister = new MetricAssignmentPerister(this, balancerConfig, metricToHostMap, nonCachedMetrics,
+                curatorFramework, assignmentsIPRWLock, nonCachedMetricsLocalLock, assignmentsLastUpdatedLocal,
+                balancerLock);

-        // start curator framework
-        curatorFramework = CuratorFrameworkFactory.newClient(balancerConfig.getZooKeeper().getServers(), 30000, 1000,
-                retryPolicy);
-        curatorFramework.start();
-        ensureZkPaths(curatorFramework, zkPaths);
-        assignmentsLock = new InterProcessReadWriteLock(curatorFramework, ASSIGNMENTS_LOCK_PATH);
        startLeaderLatch(curatorFramework);
        startServiceListener(curatorFramework);
-        assignmentsLastUpdatedInHdfs = new DistributedAtomicLong(curatorFramework, ASSIGNMENTS_LAST_UPDATED_PATH,
-                retryPolicy);

-        TreeCacheListener listener = new TreeCacheListener() {
-
-            @Override
-            public void childEvent(CuratorFramework curatorFramework, TreeCacheEvent event) throws Exception {
-                LOG.info("Handling event {}. assignmentsLastUpdatedInHdfs:{}", event.getType().toString(),
-                        new Date(assignmentsLastUpdatedInHdfs.get().postValue()));
-                if (event.getType().equals(TreeCacheEvent.Type.NODE_UPDATED)) {
-                    long lastLocalUpdate = assignmentsLastUpdatedLocal.get();
-                    long lastHdfsUpdate = assignmentsLastUpdatedInHdfs.get().postValue();
-                    if (lastHdfsUpdate > lastLocalUpdate) {
-                        LOG.info("Reading assignments from hdfs lastHdfsUpdate ({}) > lastLocalUpdate ({})",
-                                new Date(lastHdfsUpdate), new Date(lastLocalUpdate));
-                        readAssignmentsFromHdfs();
-                    } else {
-                        LOG.info("Not reading assignments from hdfs lastHdfsUpdate ({}) <= lastLocalUpdate ({})",
-                                new Date(lastHdfsUpdate), new Date(lastLocalUpdate));
-                    }
-                }
+        TreeCacheListener assignmentListener = (curatorFramework1, event) -> {
+            LOG.debug("Handling assignments event {}. assignmentsLastUpdated:{}", event.getType().toString(),
+                    new Date(assignmentPerister.getAssignmentsLastUpdatedByLeader()));
+            if (event.getType().equals(TreeCacheEvent.Type.NODE_UPDATED)) {
+                assignmentPerister.readAssignments(true);
            }
        };

-        try {
-            TreeCache treeCache = new TreeCache(curatorFramework, ASSIGNMENTS_LAST_UPDATED_PATH);
-            treeCache.getListenable().addListener(listener);
-            treeCache.start();
+        try (TreeCache assignmentTreeCache = new TreeCache(curatorFramework, ASSIGNMENTS_LAST_UPDATED_PATH)) {
+            assignmentTreeCache.getListenable().addListener(assignmentListener);
+            assignmentTreeCache.start();
        } catch (Exception e) {
            LOG.error(e.getMessage(), e);
        }

-        nonCachedMetricsLock.writeLock().lock();
-        try {
-            nonCachedMetrics.addAll(balancerConfig.getCache().getNonCachedMetrics());
-        } finally {
-            nonCachedMetricsLock.writeLock().unlock();
-        }
-
-        readAssignmentsFromHdfs();
-
-        timer.schedule(new TimerTask() {
+        nonCachedMetricsIPRWLock = new InterProcessReadWriteLock(curatorFramework, NON_CACHED_METRICS_LOCK_PATH);
+        testIPRWLock(curatorFramework, nonCachedMetricsIPRWLock, NON_CACHED_METRICS_LOCK_PATH);
+        nonCachedMetricsIP = new DistributedAtomicValue(curatorFramework, NON_CACHED_METRICS, new RetryForever(1000));
+        TreeCacheListener nonCachedMetricsListener = new TreeCacheListener() {

            @Override
-            public void run() {
-                if (isLeader.get()) {
-                    try {
-                        balance();
-                    } catch (Exception e) {
-                        LOG.error(e.getMessage(), e);
-                    }
+            public void childEvent(CuratorFramework curatorFramework, TreeCacheEvent event) throws Exception {
+                if (event.getType().equals(TreeCacheEvent.Type.NODE_UPDATED)) {
+                    LOG.debug("Handling nonCachedMetricsIP event {}", event.getType().toString());
+                    readNonCachedMetricsIP();
                }
            }
-        }, 900000, 900000);
+        };

-        timer.schedule(new TimerTask() {
+        try (TreeCache nonCachedMetricsTreeCache = new TreeCache(curatorFramework, NON_CACHED_METRICS)) {
+            nonCachedMetricsTreeCache.getListenable().addListener(nonCachedMetricsListener);
+            nonCachedMetricsTreeCache.start();
+        } catch (Exception e) {
+            LOG.error(e.getMessage(), e);
+        }

-            @Override
-            public void run() {
+        nonCachedMetricsLocalLock.writeLock().lock();
+        try {
+            addNonCachedMetrics(balancerConfig.getCache().getNonCachedMetrics());
+        } finally {
+            nonCachedMetricsLocalLock.writeLock().unlock();
+        }
+        readNonCachedMetricsIP();
+
+        assignmentPerister.readAssignments(false);
+
+        assignmentExecutor.scheduleAtFixedRate(() -> {
+            if (isLeader.get()) {
                try {
-                    if (isLeader.get()) {
-                        long lastLocalUpdate = assignmentsLastUpdatedLocal.get();
-                        long lastHdfsUpdate = assignmentsLastUpdatedInHdfs.get().postValue();
-                        if (lastLocalUpdate > lastHdfsUpdate) {
-                            LOG.info("Writing assignments to hdfs lastLocalUpdate ({}) > lastHdfsUpdate ({})",
-                                    new Date(lastLocalUpdate), new Date(lastHdfsUpdate));
-                            writeAssigmentsToHdfs();
-                        }
-                    }
+                    balance();
                } catch (Exception e) {
                    LOG.error(e.getMessage(), e);
                }
            }
-        }, 10000, 60000);
-    }
+        }, balancerConfig.getBalanceDelay(), balancerConfig.getBalancePeriod(), TimeUnit.MILLISECONDS);

-    private void ensureZkPaths(CuratorFramework curatorFramework, String[] paths) {
-        for (String s : paths) {
+        assignmentExecutor.scheduleAtFixedRate(() -> {
            try {
-                Stat stat = curatorFramework.checkExists().forPath(s);
-                if (stat == null) {
-                    curatorFramework.create().creatingParentContainersIfNeeded().forPath(s);
+                if (isLeader.get()) {
+                    long lastLocalUpdate = assignmentsLastUpdatedLocal.get();
+                    long lastUpdateByLeader = assignmentPerister.getAssignmentsLastUpdatedByLeader();
+                    if (lastLocalUpdate > lastUpdateByLeader) {
+                        LOG.debug("Leader writing assignments lastLocalUpdate ({}) > lastUpdateByLeader ({})",
+                                new Date(lastLocalUpdate), new Date(lastUpdateByLeader));
+                        assignmentPerister.writeAssignments();
+                    } else {
+                        LOG.trace("Leader not writing assignments lastLocalUpdate ({}) <= lastUpdateByLeader ({})",
+                                new Date(lastLocalUpdate), new Date(lastUpdateByLeader));
+                    }
+                } else {
+                    assignmentPerister.readAssignments(true);
                }
            } catch (Exception e) {
-                LOG.info(e.getMessage());
+                LOG.error(e.getMessage(), e);
            }
+        }, balancerConfig.getPersistDelay(), balancerConfig.getPersistPeriod(), TimeUnit.MILLISECONDS);
+    }

+    private void readNonCachedMetricsIP() {
+
+        Set<String> currentNonCachedMetricsDistributed = new TreeSet<>();
+        try {
+            boolean acquired = false;
+            while (!acquired) {
+                acquired = nonCachedMetricsIPRWLock.readLock().acquire(60, TimeUnit.SECONDS);
+            }
+            byte[] currentNonCachedMetricsDistributedBytes = nonCachedMetricsIP.get().postValue();
+            if (currentNonCachedMetricsDistributedBytes != null) {
+                try {
+                    currentNonCachedMetricsDistributed = SerializationUtils
+                            .deserialize(currentNonCachedMetricsDistributedBytes);
+                } catch (Exception e) {
+                    LOG.error(e.getMessage(), e);
+                    currentNonCachedMetricsDistributed = new TreeSet<>();
+                }
+            }
+        } catch (Exception e) {
+            LOG.error(e.getMessage(), e);
+        } finally {
+            try {
+                nonCachedMetricsIPRWLock.readLock().release();
+            } catch (Exception e) {
+                LOG.error(e.getMessage(), e);
+            }
+        }
+
+        nonCachedMetricsLocalLock.writeLock().lock();
+        try {
+            if (nonCachedMetrics.containsAll(currentNonCachedMetricsDistributed)) {
+                LOG.debug("local nonCachedMetrics already contains {}", currentNonCachedMetricsDistributed);
+            } else {
+                currentNonCachedMetricsDistributed.removeAll(nonCachedMetrics);
+                LOG.debug("Adding {} to local nonCachedMetrics", currentNonCachedMetricsDistributed);
+                nonCachedMetrics.addAll(currentNonCachedMetricsDistributed);
+            }
+        } catch (Exception e) {
+            LOG.error(e.getMessage(), e);
+        } finally {
+            nonCachedMetricsLocalLock.writeLock().unlock();
        }
    }

@@ -201,7 +249,7 @@ public class BalancedMetricResolver implements MetricResolver {
                public void isLeader() {
                    LOG.info("this balancer is the leader");
                    isLeader.set(true);
-                    writeAssigmentsToHdfs();
+                    assignmentPerister.writeAssignments();
                }

                @Override
@@ -217,28 +265,29 @@ public class BalancedMetricResolver implements MetricResolver {

    private void startServiceListener(CuratorFramework curatorFramework) {
        try {
-            ServiceDiscovery discovery = ServiceDiscoveryBuilder.builder(ServerDetails.class).client(curatorFramework)
-                    .basePath(SERVICE_DISCOVERY_PATH).build();
+            ServiceDiscovery<ServerDetails> discovery = ServiceDiscoveryBuilder.builder(ServerDetails.class)
+                    .client(curatorFramework).basePath(SERVICE_DISCOVERY_PATH).build();
            discovery.start();
-            Collection<ServiceInstance> instances = discovery.queryForInstances("timely-server");
+            Collection<ServiceInstance<ServerDetails>> instances = discovery.queryForInstances("timely-server");

            balancerLock.writeLock().lock();
            try {
-                for (ServiceInstance si : instances) {
-                    ServerDetails pl = (ServerDetails) si.getPayload();
+                for (ServiceInstance<ServerDetails> si : instances) {
+                    ServerDetails pl = si.getPayload();
                    TimelyBalancedHost tbh = TimelyBalancedHost.of(pl.getHost(), pl.getTcpPort(), pl.getHttpPort(),
-                            pl.getWsPort(), pl.getUdpPort());
-                    LOG.info("adding service {} host:{} tcpPort:{} httpPort:{} wsPort:{} udpPort:{}", si.getId(),
-                            pl.getHost(), pl.getTcpPort(), pl.getHttpPort(), pl.getWsPort(), pl.getUdpPort());
+                            pl.getWsPort(), pl.getUdpPort(), new ArrivalRate(arrivalRateExecutor));
+                    LOG.info("adding service {} host:{}", si.getId(), TimelyBalancedHost.toStringShort(tbh));
                    tbh.setBalancerConfig(balancerConfig);
                    serverList.add(tbh);
+                    metricHostEvent(tbh, MetricHostEvent.ActionType.ADDED);
                }
                healthChecker.setTimelyHosts(serverList);
            } finally {
                balancerLock.writeLock().unlock();
            }

-            final ServiceCache serviceCache = discovery.serviceCacheBuilder().name("timely-server").build();
+            final ServiceCache<ServerDetails> serviceCache = discovery.serviceCacheBuilder().name("timely-server")
+                    .build();
            ServiceCacheListener listener = new ServiceCacheListener() {

                @Override
@@ -246,12 +295,13 @@ public class BalancedMetricResolver implements MetricResolver {
                    boolean rebalanceNeeded = false;
                    balancerLock.writeLock().lock();
                    try {
-                        List<ServiceInstance> instances = serviceCache.getInstances();
+                        List<ServiceInstance<ServerDetails>> instances = serviceCache.getInstances();
                        Set<TimelyBalancedHost> availableHosts = new HashSet<>();
-                        for (ServiceInstance si : instances) {
+                        for (ServiceInstance<ServerDetails> si : instances) {
                            ServerDetails pl = (ServerDetails) si.getPayload();
                            TimelyBalancedHost tbh = TimelyBalancedHost.of(pl.getHost(), pl.getTcpPort(),
-                                    pl.getHttpPort(), pl.getWsPort(), pl.getUdpPort());
+                                    pl.getHttpPort(), pl.getWsPort(), pl.getUdpPort(),
+                                    new ArrivalRate(arrivalRateExecutor));
                            tbh.setBalancerConfig(balancerConfig);
                            availableHosts.add(tbh);
                        }
@@ -265,9 +315,8 @@ public class BalancedMetricResolver implements MetricResolver {
                                availableHosts.remove(h);
                            } else {
                                itr.remove();
-                                LOG.info("removing service {}:{} host:{} tcpPort:{} httpPort:{} wsPort:{} udpPort:{}",
-                                        h.getHost(), h.getTcpPort(), h.getHost(), h.getTcpPort(), h.getHttpPort(),
-                                        h.getWsPort(), h.getUdpPort());
+                                metricHostEvent(h, MetricHostEvent.ActionType.REMOVED);
+                                LOG.info("removing service {}", TimelyBalancedHost.toStringShort(h));
                                for (Map.Entry<String, TimelyBalancedHost> e : metricToHostMap.entrySet()) {
                                    if (e.getValue().equals(h)) {
                                        reassignMetrics.add(e.getKey());
@@ -279,28 +328,33 @@ public class BalancedMetricResolver implements MetricResolver {

                        // add new hosts that were not previously known
                        for (TimelyBalancedHost h : availableHosts) {
-                            LOG.info("adding service {}:{} host:{} tcpPort:{} httpPort:{} wsPort:{} udpPort:{}",
-                                    h.getHost(), h.getTcpPort(), h.getHost(), h.getTcpPort(), h.getHttpPort(),
-                                    h.getWsPort(), h.getUdpPort());
+                            LOG.info("adding service {}", TimelyBalancedHost.toStringShort(h));
                            serverList.add(h);
+                            metricHostEvent(h, MetricHostEvent.ActionType.ADDED);
                            rebalanceNeeded = true;
                        }
                        healthChecker.setTimelyHosts(serverList);
-
+                        if (!availableHosts.isEmpty()) {
+                            reassignMetrics.addAll(unassignedMetrics);
+                            unassignedMetrics.clear();
+                        }
                        if (isLeader.get()) {
                            for (String s : reassignMetrics) {
                                TimelyBalancedHost h = getRoundRobinHost(null);
-                                assignMetric(s, h);
-                                LOG.debug("Assigned server removed.  Assigning {} to server {}:{}", s, h.getTcpPort());
+                                assignMetric(s, h, HOST_REMOVED);
+                                LOG.debug("Assigned server removed.  Assigning {} to server {}", s,
+                                        TimelyBalancedHost.toStringShort(h));
                            }
-                            writeAssigmentsToHdfs();
                        }

                    } finally {
                        balancerLock.writeLock().unlock();
                    }
-                    if (isLeader.get() && rebalanceNeeded) {
-                        balance();
+                    if (isLeader.get()) {
+                        if (rebalanceNeeded) {
+                            balance();
+                        }
+                        assignmentPerister.writeAssignments();
                    }
                }

@@ -360,7 +414,7 @@ public class BalancedMetricResolver implements MetricResolver {
        return tbh;
    }

-    private TimelyBalancedHost getRoundRobinHost(TimelyBalancedHost notThisOne) {
+    protected TimelyBalancedHost getRoundRobinHost(TimelyBalancedHost notThisOne) {
        TimelyBalancedHost tbh = null;
        balancerLock.readLock().lock();
        try {
@@ -457,21 +511,21 @@ public class BalancedMetricResolver implements MetricResolver {
            }

            if (highHosts.isEmpty() && lowHosts.isEmpty()) {
-                LOG.info("Host's arrival rates are within {} of the average", controlBandPercentage);
+                LOG.debug("Host's arrival rates are within {} of the average", controlBandPercentage);
            } else if (!highHosts.isEmpty() && !lowHosts.isEmpty()) {
-                LOG.info("begin rebalancing {}", BalanceType.HIGH_LOW);
+                LOG.debug("begin rebalancing {}", BalanceType.HIGH_LOW);
                numReassigned = rebalance(highHosts, lowHosts, calculatedRates, averageArrivalRate,
                        BalanceType.HIGH_LOW);
-                LOG.info("end rebalancing {} - reassigned {}", BalanceType.HIGH_LOW, numReassigned);
+                LOG.debug("end rebalancing {} - reassigned {}", BalanceType.HIGH_LOW, numReassigned);
            } else if (lowHosts.isEmpty()) {
-                LOG.info("begin rebalancing {}", BalanceType.HIGH_AVG);
+                LOG.debug("begin rebalancing {}", BalanceType.HIGH_AVG);
                numReassigned = rebalance(highHosts, avgHosts, calculatedRates, averageArrivalRate,
                        BalanceType.HIGH_AVG);
-                LOG.info("end rebalancing {} - reassigned {}", BalanceType.HIGH_AVG, numReassigned);
+                LOG.debug("end rebalancing {} - reassigned {}", BalanceType.HIGH_AVG, numReassigned);
            } else {
-                LOG.info("begin rebalancing {}", BalanceType.AVG_LOW);
+                LOG.debug("begin rebalancing {}", BalanceType.AVG_LOW);
                numReassigned = rebalance(avgHosts, lowHosts, calculatedRates, averageArrivalRate, BalanceType.AVG_LOW);
-                LOG.info("end rebalancing {} - reassigned {}", BalanceType.AVG_LOW, numReassigned);
+                LOG.debug("end rebalancing {} - reassigned {}", BalanceType.AVG_LOW, numReassigned);
            }
        }
        return numReassigned;
@@ -482,8 +536,10 @@ public class BalancedMetricResolver implements MetricResolver {

        int numReassigned = 0;
        if (isLeader.get()) {
+            metricBalanceEvent(MetricBalanceEvent.ProgressType.BEGIN, balanceType, 0);
            balancerLock.writeLock().lock();
            try {
+                metricBalanceEvent(MetricBalanceEvent.ProgressType.BALANCER_LOCK_ACQUIRED, balanceType, 0);
                Map<String, ArrivalRate> tempMetricMap = new HashMap<>();
                tempMetricMap.putAll(metricMap);

@@ -513,8 +569,8 @@ public class BalancedMetricResolver implements MetricResolver {
                        }
                    }

-                    LOG.trace("focusHost {}:{} desiredChange:{} rateSortedMetrics.size():{}", h.getHost(),
-                            h.getTcpPort(), desiredChange, rateSortedMetrics.size());
+                    LOG.trace("focusHost {} desiredChange:{} rateSortedMetrics.size():{}",
+                            TimelyBalancedHost.toStringShort(h), desiredChange, rateSortedMetrics.size());

                    boolean doneWithHost = false;
                    while (!doneWithHost) {
@@ -543,7 +599,7 @@ public class BalancedMetricResolver implements MetricResolver {
                                } else {
                                    String metric = e.getValue();
                                    Double metricRate = e.getKey();
-                                    assignMetric(metric, candidateHost);
+                                    assignMetric(metric, candidateHost, REBALANCE);
                                    numReassigned++;
                                    calculatedRates.put(candidateHost, calculatedRates.get(candidateHost) + metricRate);
                                    calculatedRates.put(h, calculatedRates.get(h) - metricRate);
@@ -552,9 +608,9 @@ public class BalancedMetricResolver implements MetricResolver {
                                    itr.remove();
                                    tempMetricMap.remove(metric);
                                    LOG.info(
-                                            "rebalancing: reassigning metric:{} rate:{} from server {}:{} to {}:{} remaining delta {}",
-                                            metric, metricRate, h.getHost(), h.getTcpPort(), candidateHost.getHost(),
-                                            candidateHost.getTcpPort(), desiredChange);
+                                            "rebalancing: reassigning metric:{} rate:{} from server {} to {} remaining delta {}",
+                                            metric, metricRate, TimelyBalancedHost.toStringShort(h),
+                                            TimelyBalancedHost.toStringShort(candidateHost), desiredChange);
                                }
                            }
                        }
@@ -577,18 +633,94 @@ public class BalancedMetricResolver implements MetricResolver {
                }
            } finally {
                balancerLock.writeLock().unlock();
+                metricBalanceEvent(MetricBalanceEvent.ProgressType.END, balanceType, numReassigned);
            }
        }
        return numReassigned;
    }

-    private boolean shouldCache(String metricName) {
+    private void addNonCachedMetrics(Collection<String> nonCachedMetricsUpdate) {
+        if (!nonCachedMetricsUpdate.isEmpty()) {
+            try {
+                LOG.debug("Adding {} to local nonCachedMetrics", nonCachedMetricsUpdate);
+                nonCachedMetricsLocalLock.writeLock().lock();
+                try {
+                    nonCachedMetrics.addAll(nonCachedMetricsUpdate);
+                } finally {
+                    nonCachedMetricsLocalLock.writeLock().unlock();
+                }
+
+                try {
+                    boolean acquired = false;
+                    while (!acquired) {
+                        acquired = nonCachedMetricsIPRWLock.writeLock().acquire(60, TimeUnit.SECONDS);
+                    }
+                    byte[] currentNonCachedMetricsDistributedBytes = nonCachedMetricsIP.get().postValue();
+                    Set<String> currentNonCachedMetricsIP;
+                    if (currentNonCachedMetricsDistributedBytes == null) {
+                        currentNonCachedMetricsIP = new TreeSet<>();
+                    } else {
+                        currentNonCachedMetricsIP = SerializationUtils
+                                .deserialize(currentNonCachedMetricsDistributedBytes);
+                    }
+                    if (currentNonCachedMetricsIP.containsAll(nonCachedMetricsUpdate)) {
+                        LOG.debug("nonCachedMetricsIP already contains {}", nonCachedMetricsUpdate);
+                    } else {
+                        nonCachedMetricsUpdate.removeAll(currentNonCachedMetricsIP);
+                        LOG.debug("Adding {} to nonCachedMetricsIP", nonCachedMetricsUpdate);
+                        TreeSet<String> updateSet = new TreeSet<>();
+                        updateSet.addAll(currentNonCachedMetricsIP);
+                        updateSet.addAll(nonCachedMetricsUpdate);
+                        byte[] updateValue = SerializationUtils.serialize(updateSet);
+                        nonCachedMetricsIP.trySet(updateValue);
+                        if (!nonCachedMetricsIP.get().succeeded()) {
+                            nonCachedMetricsIP.forceSet(updateValue);
+                        }
+                    }
+                } finally {
+                    nonCachedMetricsIPRWLock.writeLock().release();
+                }
+            } catch (Exception e) {
+                LOG.error(e.getMessage(), e);
+            }
+        }
+    }
+
+    private void testIPRWLock(CuratorFramework curatorFramework, InterProcessReadWriteLock lock, String path) {
+        try {
+            lock.writeLock().acquire(10, TimeUnit.SECONDS);
+        } catch (Exception e1) {
+            try {
+                curatorFramework.delete().deletingChildrenIfNeeded().forPath(path);
+                curatorFramework.create().creatingParentContainersIfNeeded().withMode(CreateMode.PERSISTENT)
+                        .forPath(path);
+            } catch (Exception e2) {
+                LOG.info(e2.getMessage(), e2);
+            }
+        } finally {
+            try {
+                lock.writeLock().release();
+            } catch (Exception e3) {
+                LOG.error(e3.getMessage(), e3);
+            }
+        }
+    }
+
+    protected boolean shouldCache(String metricName) {

        if (StringUtils.isBlank(metricName)) {
            return false;
        } else {
            balancerLock.readLock().lock();
            try {
+                nonCachedMetricsLocalLock.readLock().lock();
+                try {
+                    if (nonCachedMetrics.contains(metricName)) {
+                        return false;
+                    }
+                } finally {
+                    nonCachedMetricsLocalLock.readLock().unlock();
+                }
                if (metricToHostMap.containsKey(metricName)) {
                    return true;
                }
@@ -596,27 +728,17 @@ public class BalancedMetricResolver implements MetricResolver {
                balancerLock.readLock().unlock();
            }

-            nonCachedMetricsLock.readLock().lock();
+            nonCachedMetricsLocalLock.writeLock().lock();
            try {
-                if (nonCachedMetrics.contains(metricName)) {
-                    return false;
-                }
-
                for (String r : nonCachedMetrics) {
                    if (metricName.matches(r)) {
-                        LOG.info("Adding {} to list of non-cached metrics", metricName);
-                        nonCachedMetricsLock.readLock().unlock();
-                        nonCachedMetricsLock.writeLock().lock();
-                        nonCachedMetrics.add(metricName);
+                        LOG.debug("Adding {} to list of non-cached metrics", metricName);
+                        addNonCachedMetrics(Collections.singleton(metricName));
                        return false;
                    }
                }
            } finally {
-                if (nonCachedMetricsLock.isWriteLockedByCurrentThread()) {
-                    nonCachedMetricsLock.writeLock().unlock();
-                } else {
-                    nonCachedMetricsLock.readLock().unlock();
-                }
+                nonCachedMetricsLocalLock.writeLock().unlock();
            }
            return true;
        }
@@ -633,7 +755,7 @@ public class BalancedMetricResolver implements MetricResolver {
            if (chooseMetricSpecificHost) {
                ArrivalRate rate = metricMap.get(metric);
                if (rate == null) {
-                    rate = new ArrivalRate(arrivalRateTimer);
+                    rate = new ArrivalRate(arrivalRateExecutor);
                    if (!balancerLock.isWriteLockedByCurrentThread()) {
                        balancerLock.readLock().unlock();
                        balancerLock.writeLock().lock();
@@ -646,7 +768,7 @@ public class BalancedMetricResolver implements MetricResolver {
            }

            if (StringUtils.isBlank(metric)) {
-                tbh = getRandomHost(null);
+                tbh = getRoundRobinHost(null);
            } else {
                tbh = metricToHostMap.get(metric);
                if (tbh == null) {
@@ -655,17 +777,17 @@ public class BalancedMetricResolver implements MetricResolver {
                        balancerLock.readLock().unlock();
                        balancerLock.writeLock().lock();
                    }
-                    assignMetric(metric, tbh);
+                    assignMetric(metric, tbh, ASSIGN_ROUND_ROBIN);
                } else if (!tbh.isUp()) {
                    TimelyBalancedHost oldTbh = tbh;
-                    tbh = getLeastUsedHost();
-                    LOG.info("rebalancing from host that is down: reassigning metric {} from server {}:{} to {}:{}",
-                            metric, oldTbh.getHost(), oldTbh.getTcpPort(), tbh.getHost(), tbh.getTcpPort());
+                    tbh = getRoundRobinHost(oldTbh);
+                    LOG.debug("rebalancing from host that is down: reassigning metric {} from server {} to {}", metric,
+                            TimelyBalancedHost.toStringShort(oldTbh), TimelyBalancedHost.toStringShort(tbh));
                    if (!balancerLock.isWriteLockedByCurrentThread()) {
                        balancerLock.readLock().unlock();
                        balancerLock.writeLock().lock();
                    }
-                    assignMetric(metric, tbh);
+                    assignMetric(metric, tbh, HOST_DOWN_ROUND_ROBIN);
                }
            }

@@ -682,7 +804,7 @@ public class BalancedMetricResolver implements MetricResolver {
                        balancerLock.readLock().unlock();
                        balancerLock.writeLock().lock();
                    }
-                    assignMetric(metric, tbh);
+                    assignMetric(metric, tbh, ASSIGN_FALLBACK_SEQUENTIAL);
                }
            }
        } finally {
@@ -711,20 +833,23 @@ public class BalancedMetricResolver implements MetricResolver {
                metric = null;
            }

-            if (tbh == null || !tbh.isUp()) {
-                for (int x = 0; tbh == null && x < serverList.size(); x++) {
-                    tbh = serverList.get(Math.abs(r.nextInt() & Integer.MAX_VALUE) % serverList.size());
-                    if (!tbh.isUp()) {
-                        tbh = null;
-                    }
+            if (tbh == null) {
+                tbh = getRoundRobinHost(null);
+                if (!balancerLock.isWriteLockedByCurrentThread()) {
+                    balancerLock.readLock().unlock();
+                    balancerLock.writeLock().lock();
                }
-                if (tbh != null && StringUtils.isNotBlank(metric)) {
-                    if (!balancerLock.isWriteLockedByCurrentThread()) {
-                        balancerLock.readLock().unlock();
-                        balancerLock.writeLock().lock();
-                    }
-                    assignMetric(metric, tbh);
+                assignMetric(metric, tbh, ASSIGN_ROUND_ROBIN);
+            } else if (!tbh.isUp()) {
+                TimelyBalancedHost oldTbh = tbh;
+                tbh = getRoundRobinHost(oldTbh);
+                LOG.debug("rebalancing from host that is down: reassigning metric {} from server {} to {}", metric,
+                        TimelyBalancedHost.toStringShort(oldTbh), TimelyBalancedHost.toStringShort(tbh));
+                if (!balancerLock.isWriteLockedByCurrentThread()) {
+                    balancerLock.readLock().unlock();
+                    balancerLock.writeLock().lock();
                }
+                assignMetric(metric, tbh, HOST_DOWN_ROUND_ROBIN);
            }

            // if all else fails
@@ -740,7 +865,7 @@ public class BalancedMetricResolver implements MetricResolver {
                        balancerLock.readLock().unlock();
                        balancerLock.writeLock().lock();
                    }
-                    assignMetric(metric, tbh);
+                    assignMetric(metric, tbh, ASSIGN_FALLBACK_SEQUENTIAL);
                }
            }
        } finally {
@@ -753,7 +878,7 @@ public class BalancedMetricResolver implements MetricResolver {
        return tbh;
    }

-    private TimelyBalancedHost findHost(String host, int tcpPort) {
+    protected TimelyBalancedHost findHost(String host, int tcpPort) {
        TimelyBalancedHost tbh = null;
        for (TimelyBalancedHost h : serverList) {
            if (h.getHost().equals(host) && h.getTcpPort() == tcpPort) {
@@ -764,130 +889,100 @@ public class BalancedMetricResolver implements MetricResolver {
        return tbh;
    }

-    private void readAssignmentsFromHdfs() {
-
-        Map<String, TimelyBalancedHost> assignedMetricToHostMap = new TreeMap<>();
-        try {
-            assignmentsLock.readLock().acquire();
-            Configuration configuration = new Configuration();
-            FileSystem fs = FileSystem.get(configuration);
-            Path assignmentFile = new Path(balancerConfig.getAssignmentFile());
-            FSDataInputStream iStream = fs.open(assignmentFile);
-
-            CsvReader reader = new CsvReader(iStream, ',', Charset.forName("UTF-8"));
-            reader.setUseTextQualifier(false);
-
-            // skip the headers
-            boolean success = true;
-            success = reader.readHeaders();
-
-            while (success) {
-                success = reader.readRecord();
-                String[] nextLine = reader.getValues();
-                if (nextLine.length >= 3) {
-                    String metric = nextLine[0];
-                    String host = nextLine[1];
-                    int tcpPort = Integer.parseInt(nextLine[2]);
-                    TimelyBalancedHost tbh = findHost(host, tcpPort);
-                    if (tbh == null) {
-                        tbh = getRoundRobinHost(null);
-                    } else {
-                        LOG.trace("Found assigment: {} to {}:{}", metric, host, tcpPort);
-                    }
-                    assignedMetricToHostMap.put(metric, tbh);
-                }
-            }
-
-            balancerLock.writeLock().lock();
-            try {
-                metricToHostMap.clear();
-                for (Map.Entry<String, TimelyBalancedHost> e : assignedMetricToHostMap.entrySet()) {
-                    if (shouldCache(e.getKey())) {
-                        metricToHostMap.put(e.getKey(), e.getValue());
-                    }
-                }
-            } finally {
-                balancerLock.writeLock().unlock();
-            }
-            assignmentsLastUpdatedLocal.set(assignmentsLastUpdatedInHdfs.get().postValue());
-            LOG.info("Read {} assignments from hdfs lastHdfsUpdate = lastLocalUpdate ({})", metricToHostMap.size(),
-                    new Date(assignmentsLastUpdatedLocal.get()));
-        } catch (Exception e) {
-            LOG.error(e.getMessage(), e);
-        } finally {
-            try {
-                assignmentsLock.readLock().release();
-            } catch (Exception e) {
-                LOG.error(e.getMessage(), e);
-            }
-        }
-    }
-
-    private void writeAssigmentsToHdfs() {
-
-        CsvWriter writer = null;
-        try {
-            assignmentsLock.writeLock().acquire();
-            balancerLock.readLock().lock();
-            try {
-                if (!metricToHostMap.isEmpty()) {
-                    Configuration configuration = new Configuration();
-                    FileSystem fs = FileSystem.get(configuration);
-                    Path assignmentFile = new Path(balancerConfig.getAssignmentFile());
-                    if (!fs.exists(assignmentFile.getParent())) {
-                        fs.mkdirs(assignmentFile.getParent());
-                    }
-                    FSDataOutputStream oStream = fs.create(assignmentFile, true);
-                    writer = new CsvWriter(oStream, ',', Charset.forName("UTF-8"));
-                    writer.setUseTextQualifier(false);
-                    writer.write("metric");
-                    writer.write("host");
-                    writer.write("tcpPort");
-                    writer.endRecord();
-                    for (Map.Entry<String, TimelyBalancedHost> e : metricToHostMap.entrySet()) {
-                        writer.write(e.getKey());
-                        writer.write(e.getValue().getHost());
-                        writer.write(Integer.toString(e.getValue().getTcpPort()));
-                        writer.endRecord();
-                        LOG.trace("Saving assigment: {} to {}:{}", e.getKey(), e.getValue().getHost(),
-                                e.getValue().getTcpPort());
-                    }
-
-                    long now = System.currentTimeMillis();
-                    assignmentsLastUpdatedLocal.set(now);
-                    assignmentsLastUpdatedInHdfs.trySet(now);
-                    if (!assignmentsLastUpdatedInHdfs.get().succeeded()) {
-                        assignmentsLastUpdatedInHdfs.forceSet(now);
-                    }
-                    LOG.info("Wrote {} assignments to hdfs lastHdfsUpdate = lastLocalUpdate ({})",
-                            metricToHostMap.size(), new Date(assignmentsLastUpdatedLocal.get()));
-                }
-            } finally {
-                balancerLock.readLock().unlock();
-            }
-        } catch (Exception e) {
-            LOG.error(e.getMessage(), e);
-        } finally {
-            if (writer != null) {
-                writer.close();
-            }
-            try {
-                assignmentsLock.writeLock().release();
-            } catch (Exception e) {
-                LOG.error(e.getMessage(), e);
-            }
-        }
-    }
-
-    private void assignMetric(String metric, TimelyBalancedHost tbh) {
+    private void assignMetric(String metric, TimelyBalancedHost tbh, MetricAssignedEvent.Reason reason) {
        if (isLeader.get()) {
-            balancerLock.writeLock().lock();
-            try {
-                metricToHostMap.put(metric, tbh);
-                assignmentsLastUpdatedLocal.set(System.currentTimeMillis());
-            } finally {
-                balancerLock.writeLock().unlock();
+            if (StringUtils.isNotBlank(metric) && tbh != null) {
+                balancerLock.writeLock().lock();
+                try {
+                    metricAssignedEvent(metric, metricToHostMap.get(metric), tbh, reason);
+                    metricToHostMap.put(metric, tbh);
+                    assignmentsLastUpdatedLocal.set(System.currentTimeMillis());
+                } finally {
+                    balancerLock.writeLock().unlock();
+                }
+            } else {
+                unassignedMetrics.add(metric);
+                // Exception e = new IllegalStateException("Bad assignment metric:" + metric + "
+                // host:" + tbh);
+                LOG.warn("Bad assignment metric:" + metric + " host:" + tbh);
            }
        }
    }
-}
+
+    public void registerCallback(MetricAssignedCallback callback) {
+        this.metricAssignedCallbacks.add(callback);
+    }
+
+    public void registerCallback(MetricBalanceCallback callback) {
+        this.metricBalanceCallbacks.add(callback);
+    }
+
+    public void registerCallback(MetricHostCallback callback) {
+        this.metricHostCallbacks.add(callback);
+    }
+
+    protected void metricAssignedEvent(String metric, TimelyBalancedHost losing, TimelyBalancedHost gaining,
+            MetricAssignedEvent.Reason reason) {
+        if (!metricAssignedCallbacks.isEmpty()) {
+            MetricAssignedEvent event = new MetricAssignedEvent(metric, losing, gaining, reason);
+            for (MetricAssignedCallback callback : metricAssignedCallbacks) {
+                callback.onEvent(event);
+            }
+        }
+    }
+
+    protected void metricBalanceEvent(MetricBalanceEvent.ProgressType progressType,
+            BalancedMetricResolver.BalanceType balanceType, long numReassigned) {
+        if (!metricBalanceCallbacks.isEmpty()) {
+            MetricBalanceEvent event = new MetricBalanceEvent(progressType, balanceType, numReassigned);
+            for (MetricBalanceCallback callback : metricBalanceCallbacks) {
+                callback.onEvent(event);
+            }
+        }
+    }
+
+    protected void metricHostEvent(TimelyBalancedHost host, MetricHostEvent.ActionType actionType) {
+        if (!metricHostCallbacks.isEmpty()) {
+            MetricHostEvent event = new MetricHostEvent(host, actionType);
+            for (MetricHostCallback callback : metricHostCallbacks) {
+                callback.onEvent(event);
+            }
+        }
+    }
+
+    protected void writeAssignments(Map<String, TimelyBalancedHost> metricToHostMap) {
+        assignmentPerister.writeAssignments(metricToHostMap);
+    }
+
+    @Override
+    public void stop() {
+
+        this.assignmentExecutor.shutdown();
+        try {
+            this.assignmentExecutor.awaitTermination(5, TimeUnit.SECONDS);
+        } catch (InterruptedException e) {
+
+        } finally {
+            if (!this.assignmentExecutor.isTerminated()) {
+                this.assignmentExecutor.shutdownNow();
+            }
+        }
+
+        this.arrivalRateExecutor.shutdown();
+        try {
+            this.arrivalRateExecutor.awaitTermination(5, TimeUnit.SECONDS);
+        } catch (InterruptedException e) {
+
+        } finally {
+            if (!this.arrivalRateExecutor.isTerminated()) {
+                this.arrivalRateExecutor.shutdownNow();
+            }
+        }
+
+        try {
+            this.healthChecker.close();
+        } catch (Exception e) {
+            LOG.error(e.getMessage(), e);
+        }
+    }
+}
@@ -0,0 +1,320 @@
+package timely.balancer.resolver;
+
+import static timely.balancer.Balancer.ASSIGNMENTS_LAST_UPDATED_PATH;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.charset.Charset;
+import java.util.Date;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.apache.commons.lang.StringUtils;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.recipes.atomic.DistributedAtomicLong;
+import org.apache.curator.framework.recipes.locks.InterProcessReadWriteLock;
+import org.apache.curator.retry.RetryForever;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import timely.balancer.configuration.BalancerConfiguration;
+import timely.balancer.connection.TimelyBalancedHost;
+import timely.balancer.resolver.eventing.MetricAssignedEvent;
+
+public class MetricAssignmentPerister {
+
+    private final Logger LOG = LoggerFactory.getLogger(getClass());
+
+    private enum PeristerType {
+        HDFS, FILE
+    }
+
+    // passed in through constructor
+    BalancedMetricResolver balancedMetricResolver;
+    private BalancerConfiguration balancerConfig;
+    private Map<String, TimelyBalancedHost> metricToHostMap;
+    private Set<String> nonCachedMetrics;
+    private InterProcessReadWriteLock assignmentsIPRWLock;
+    private ReentrantReadWriteLock nonCachedMetricsLocalLock;
+    private ReentrantReadWriteLock balancerLock;
+    private AtomicLong assignmentsLastUpdatedLocal;
+
+    // local
+    private DistributedAtomicLong assignmentsLastUpdatedInHdfs;
+
+    public MetricAssignmentPerister(BalancedMetricResolver balancedMetricResolver, BalancerConfiguration balancerConfig,
+            Map<String, TimelyBalancedHost> metricToHostMap, Set<String> nonCachedMetrics,
+            CuratorFramework curatorFramework, InterProcessReadWriteLock assignmentsIPRWLock,
+            ReentrantReadWriteLock nonCachedMetricsLocalLock, AtomicLong assignmentsLastUpdatedLocal,
+            ReentrantReadWriteLock balancerLock) {
+
+        this.balancedMetricResolver = balancedMetricResolver;
+        this.balancerConfig = balancerConfig;
+        this.metricToHostMap = metricToHostMap;
+        this.nonCachedMetrics = nonCachedMetrics;
+        this.assignmentsIPRWLock = assignmentsIPRWLock;
+        this.nonCachedMetricsLocalLock = nonCachedMetricsLocalLock;
+        this.assignmentsLastUpdatedLocal = assignmentsLastUpdatedLocal;
+        this.balancerLock = balancerLock;
+        assignmentsLastUpdatedInHdfs = new DistributedAtomicLong(curatorFramework, ASSIGNMENTS_LAST_UPDATED_PATH,
+                new RetryForever(1000));
+    }
+
+    public long getAssignmentsLastUpdatedByLeader() throws Exception {
+        return assignmentsLastUpdatedInHdfs.get().postValue();
+    }
+
+    public void readAssignments(boolean checkIfNecessary) {
+
+        try {
+            long lastLocalUpdate = assignmentsLastUpdatedLocal.get();
+            long lastHdfsUpdate = assignmentsLastUpdatedInHdfs.get().postValue();
+            if (checkIfNecessary) {
+                if (lastHdfsUpdate <= lastLocalUpdate) {
+                    LOG.debug("Not reading assignments from storage lastStorageUpdate ({}) <= lastLocalUpdate ({})",
+                            new Date(lastHdfsUpdate), new Date(lastLocalUpdate));
+                    return;
+                }
+            }
+            LOG.debug("Reading assignments from storage lastStorageUpdate ({}) > lastLocalUpdate ({})",
+                    new Date(lastHdfsUpdate), new Date(lastLocalUpdate));
+        } catch (Exception e) {
+            LOG.error(e.getMessage(), e);
+        }
+
+        // proceed with reading from storage
+
+        try (InputStream inputStream = getInputStream(balancerConfig)) {
+            boolean acquired = false;
+            while (!acquired) {
+                acquired = assignmentsIPRWLock.readLock().acquire(60, TimeUnit.SECONDS);
+            }
+            balancerLock.writeLock().lock();
+            // get potential new list from implementation
+            Map<String, TimelyBalancedHost> assignedMetricToHostMap = readFromStream(inputStream);
+            metricToHostMap.clear();
+            for (Map.Entry<String, TimelyBalancedHost> e : assignedMetricToHostMap.entrySet()) {
+                if (StringUtils.isNotBlank(e.getKey()) && e.getValue() != null) {
+                    if (balancedMetricResolver.shouldCache(e.getKey())) {
+                        balancedMetricResolver.metricAssignedEvent(e.getKey(), null, e.getValue(),
+                                MetricAssignedEvent.Reason.ASSIGN_FILE);
+                        metricToHostMap.put(e.getKey(), e.getValue());
+                    }
+                } else {
+                    Exception e1 = new IllegalStateException(
+                            "Bad assignment metric:" + e.getKey() + " host:" + e.getValue());
+                    LOG.warn(e1.getMessage(), e1);
+                }
+            }
+            nonCachedMetricsLocalLock.readLock().lock();
+            try {
+                Iterator<Map.Entry<String, TimelyBalancedHost>> itr = metricToHostMap.entrySet().iterator();
+                while (itr.hasNext()) {
+                    Map.Entry<String, TimelyBalancedHost> e = itr.next();
+                    if (nonCachedMetrics.contains(e.getKey())) {
+                        itr.remove();
+                    }
+                }
+            } finally {
+                nonCachedMetricsLocalLock.readLock().unlock();
+            }
+            assignmentsLastUpdatedLocal.set(assignmentsLastUpdatedInHdfs.get().postValue());
+            LOG.info("Read {} assignments from storage lastStorageUpdate = lastLocalUpdate ({})",
+                    metricToHostMap.size(), new Date(assignmentsLastUpdatedLocal.get()));
+        } catch (FileNotFoundException e) {
+            createAssignmentFile();
+            metricToHostMap.clear();
+        } catch (Exception e) {
+            LOG.error(e.getMessage(), e);
+        } finally {
+            balancerLock.writeLock().unlock();
+            try {
+                assignmentsIPRWLock.readLock().release();
+            } catch (Exception e) {
+                LOG.error(e.getMessage(), e);
+            }
+        }
+    }
+
+    public void writeAssignments() {
+        writeAssignments(metricToHostMap);
+    }
+
+    protected void writeAssignments(Map<String, TimelyBalancedHost> metricToHostMap) {
+
+        try {
+            boolean acquired = false;
+            while (!acquired) {
+                acquired = assignmentsIPRWLock.writeLock().acquire(60, TimeUnit.SECONDS);
+            }
+            balancerLock.readLock().lock();
+            nonCachedMetricsLocalLock.readLock().lock();
+            try (OutputStream outputStream = getOutputStream(balancerConfig)) {
+                if (!metricToHostMap.isEmpty()) {
+                    writeToStream(outputStream);
+                    long now = System.currentTimeMillis();
+                    assignmentsLastUpdatedLocal.set(now);
+                    assignmentsLastUpdatedInHdfs.trySet(now);
+                    if (!assignmentsLastUpdatedInHdfs.get().succeeded()) {
+                        assignmentsLastUpdatedInHdfs.forceSet(now);
+                    }
+                    LOG.debug("Wrote {} assignments to storage lastStorageUpdate = lastLocalUpdate ({})",
+                            metricToHostMap.size(), new Date(assignmentsLastUpdatedLocal.get()));
+                }
+            } finally {
+                nonCachedMetricsLocalLock.readLock().unlock();
+                balancerLock.readLock().unlock();
+            }
+        } catch (Exception e) {
+            LOG.error(e.getMessage(), e);
+        } finally {
+            try {
+                assignmentsIPRWLock.writeLock().release();
+            } catch (Exception e) {
+                LOG.error(e.getMessage(), e);
+            }
+        }
+    }
+
+    public Map<String, TimelyBalancedHost> readFromStream(InputStream inputStream) throws IOException {
+        Map<String, TimelyBalancedHost> assignedMetricToHostMap = new TreeMap<>();
+        try {
+            CsvReader reader = new CsvReader(inputStream, ',', Charset.forName("UTF-8"));
+            reader.setUseTextQualifier(false);
+
+            // skip the headers
+            boolean success = true;
+            success = reader.readHeaders();
+
+            while (success) {
+                success = reader.readRecord();
+                String[] nextLine = reader.getValues();
+                if (nextLine.length >= 3) {
+                    String metric = nextLine[0];
+                    String host = nextLine[1];
+                    int tcpPort = Integer.parseInt(nextLine[2]);
+                    TimelyBalancedHost tbh = balancedMetricResolver.findHost(host, tcpPort);
+                    if (tbh == null) {
+                        tbh = balancedMetricResolver.getRoundRobinHost(null);
+                    } else {
+                        LOG.trace("Found assigment: {} to {}:{}", metric, host, tcpPort);
+                    }
+                    assignedMetricToHostMap.put(metric, tbh);
+                }
+            }
+        } catch (IOException e) {
+            LOG.error(e.getMessage(), e);
+            throw e;
+        }
+        return assignedMetricToHostMap;
+    }
+
+    private void writeToStream(OutputStream outputStream) throws IOException {
+
+        CsvWriter writer = null;
+        try {
+            writer = new CsvWriter(outputStream, ',', Charset.forName("UTF-8"));
+            writer.setUseTextQualifier(false);
+            writer.write("metric");
+            writer.write("host");
+            writer.write("tcpPort");
+            writer.endRecord();
+            for (Map.Entry<String, TimelyBalancedHost> e : metricToHostMap.entrySet()) {
+                if (!nonCachedMetrics.contains(e.getKey())) {
+                    writer.write(e.getKey());
+                    writer.write(e.getValue().getHost());
+                    writer.write(Integer.toString(e.getValue().getTcpPort()));
+                    writer.endRecord();
+                    LOG.trace("Saving assigment: {} to {}:{}", e.getKey(), e.getValue().getHost(),
+                            e.getValue().getTcpPort());
+                }
+            }
+        } finally {
+            if (writer != null) {
+                writer.close();
+            }
+        }
+    }
+
+    public void createAssignmentFile() {
+        LOG.info("Creating assignment file: " + balancerConfig.getAssignmentFile());
+        PeristerType peristerType = PeristerType.valueOf(balancerConfig.getMetricAssignmentPersisterType());
+        switch (peristerType) {
+            case HDFS:
+                try {
+                    Configuration configuration = new Configuration();
+                    FileSystem fs = FileSystem.get(configuration);
+                    Path assignmentFile = new Path(balancerConfig.getAssignmentFile());
+                    fs.create(assignmentFile);
+                } catch (Exception e) {
+                    LOG.error(e.getMessage(), e);
+                }
+                break;
+            case FILE:
+                try {
+                    File assignmentFile = new File(balancerConfig.getAssignmentFile());
+                    String parentPath = assignmentFile.getParent();
+                    File directory = new File(parentPath);
+                    if (!directory.exists()) {
+
+                        directory.mkdirs();
+                    }
+                    if (!assignmentFile.exists()) {
+                        assignmentFile.createNewFile();
+                    }
+                } catch (Exception e) {
+                    LOG.error(e.getMessage(), e);
+                }
+                break;
+        }
+    }
+
+    public InputStream getInputStream(BalancerConfiguration balancerConfiguration) throws Exception {
+        PeristerType peristerType = PeristerType.valueOf(balancerConfiguration.getMetricAssignmentPersisterType());
+        switch (peristerType) {
+            case HDFS:
+                Configuration configuration = new Configuration();
+                FileSystem fs = FileSystem.get(configuration);
+                Path assignmentFile = new Path(balancerConfiguration.getAssignmentFile());
+                return fs.open(assignmentFile);
+            case FILE:
+                return new FileInputStream(new File(balancerConfiguration.getAssignmentFile()));
+            default:
+                throw new IllegalArgumentException(
+                        "Unknown metricPersisterType " + balancerConfiguration.getMetricAssignmentPersisterType());
+        }
+    }
+
+    public OutputStream getOutputStream(BalancerConfiguration balancerConfiguration) throws IOException {
+        PeristerType peristerType = PeristerType.valueOf(balancerConfiguration.getMetricAssignmentPersisterType());
+        switch (peristerType) {
+            case HDFS:
+                Configuration configuration = new Configuration();
+                FileSystem fs = FileSystem.get(configuration);
+                Path assignmentFile = new Path(balancerConfiguration.getAssignmentFile());
+                if (!fs.exists(assignmentFile.getParent())) {
+                    fs.mkdirs(assignmentFile.getParent());
+                }
+                return fs.create(assignmentFile, true);
+            case FILE:
+                return new FileOutputStream(new File(balancerConfiguration.getAssignmentFile()));
+            default:
+                throw new IllegalArgumentException(
+                        "Unknown metricPersisterType " + balancerConfiguration.getMetricAssignmentPersisterType());
+        }
+    }
+}
@@ -7,4 +7,9 @@ public interface MetricResolver {
    public TimelyBalancedHost getHostPortKeyIngest(String metric);

    public TimelyBalancedHost getHostPortKey(String metric);
+
+    public void start();
+
+    public void stop();
+
 }
@@ -0,0 +1,7 @@
+package timely.balancer.resolver.eventing;
+
+public abstract class MetricAssignedCallback {
+
+    public abstract void onEvent(MetricAssignedEvent event);
+
+}
@@ -0,0 +1,39 @@
+package timely.balancer.resolver.eventing;
+
+import timely.balancer.connection.TimelyBalancedHost;
+
+public class MetricAssignedEvent {
+
+    public enum Reason {
+        ASSIGN_ROUND_ROBIN, HOST_DOWN_ROUND_ROBIN, ASSIGN_FALLBACK_SEQUENTIAL, HOST_REMOVED, REBALANCE, ASSIGN_FILE
+    }
+
+    private final String metric;
+    private final TimelyBalancedHost losingHost;
+    private final TimelyBalancedHost gainingHost;
+    private final Reason reason;
+
+    public MetricAssignedEvent(String metric, TimelyBalancedHost losingHost, TimelyBalancedHost gainingHost,
+            Reason reason) {
+        this.metric = metric;
+        this.losingHost = losingHost;
+        this.gainingHost = gainingHost;
+        this.reason = reason;
+    }
+
+    public String getMetric() {
+        return metric;
+    }
+
+    public TimelyBalancedHost getLosingHost() {
+        return losingHost;
+    }
+
+    public TimelyBalancedHost getGainingHost() {
+        return gainingHost;
+    }
+
+    public Reason getReason() {
+        return reason;
+    }
+}
@@ -0,0 +1,7 @@
+package timely.balancer.resolver.eventing;
+
+public abstract class MetricBalanceCallback {
+
+    public abstract void onEvent(MetricBalanceEvent event);
+
+}
@@ -0,0 +1,33 @@
+package timely.balancer.resolver.eventing;
+
+import timely.balancer.resolver.BalancedMetricResolver;
+
+public class MetricBalanceEvent {
+
+    public enum ProgressType {
+        BEGIN, BALANCER_LOCK_ACQUIRED, END
+    }
+
+    private final ProgressType progressType;
+    private final BalancedMetricResolver.BalanceType balanceType;
+    private final Long numRessigned;
+
+    public MetricBalanceEvent(ProgressType progressType, BalancedMetricResolver.BalanceType balanceType,
+            long numReassigned) {
+        this.progressType = progressType;
+        this.balanceType = balanceType;
+        this.numRessigned = numReassigned;
+    }
+
+    public BalancedMetricResolver.BalanceType getBalanceType() {
+        return balanceType;
+    }
+
+    public ProgressType getProgressType() {
+        return progressType;
+    }
+
+    public Long getNumRessigned() {
+        return numRessigned;
+    }
+}
@@ -0,0 +1,7 @@
+package timely.balancer.resolver.eventing;
+
+public abstract class MetricHostCallback {
+
+    public abstract void onEvent(MetricHostEvent event);
+
+}
@@ -0,0 +1,26 @@
+package timely.balancer.resolver.eventing;
+
+import timely.balancer.connection.TimelyBalancedHost;
+
+public class MetricHostEvent {
+
+    public enum ActionType {
+        ADDED, REMOVED, SUCCESS, FAILURE
+    }
+
+    private final TimelyBalancedHost timelyBalancedHost;
+    private final ActionType actionType;
+
+    public MetricHostEvent(TimelyBalancedHost timelyBalancedHost, ActionType actionType) {
+        this.timelyBalancedHost = timelyBalancedHost;
+        this.actionType = actionType;
+    }
+
+    public TimelyBalancedHost getTimelyBalancedHost() {
+        return timelyBalancedHost;
+    }
+
+    public ActionType getActionType() {
+        return actionType;
+    }
+}
@@ -10,4 +10,9 @@
    <Method name="getBuf" />
    <Bug pattern="EI_EXPOSE_REP" />
  </Match>
+  <Match>
+    <Class name="timely.balancer.resolver.MetricAssignmentPerister" />
+    <Method name="createAssignmentFile" />
+    <Bug pattern="RV_RETURN_VALUE_IGNORED_BAD_PRACTICE" />
+  </Match>
 </FindBugsFilter>
@@ -0,0 +1,76 @@
+package timely.balancer.resolver;
+
+import static timely.Server.SERVICE_DISCOVERY_PATH;
+
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.x.discovery.ServiceDiscovery;
+import org.apache.curator.x.discovery.ServiceDiscoveryBuilder;
+import org.apache.curator.x.discovery.ServiceInstance;
+import org.apache.curator.x.discovery.ServiceInstanceBuilder;
+import org.apache.zookeeper.data.Stat;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import timely.ServerDetails;
+import timely.balancer.connection.TimelyBalancedHost;
+
+public class ServiceRegistry {
+
+    private static final Logger LOG = LoggerFactory.getLogger(ServiceRegistry.class);
+
+    public static void registerService(CuratorFramework curatorFramework, TimelyBalancedHost tbh) {
+        try {
+            try {
+                Stat stat = curatorFramework.checkExists().forPath(SERVICE_DISCOVERY_PATH);
+                if (stat == null) {
+                    curatorFramework.create().creatingParentContainersIfNeeded().forPath(SERVICE_DISCOVERY_PATH);
+                }
+            } catch (Exception e) {
+                LOG.error(e.getMessage());
+            }
+
+            ServiceInstance<ServerDetails> serviceInstance = serviceInstance(tbh);
+            ServiceDiscovery<ServerDetails> discovery = ServiceDiscoveryBuilder.builder(ServerDetails.class)
+                    .client(curatorFramework).basePath(SERVICE_DISCOVERY_PATH).build();
+            discovery.start();
+            discovery.registerService(serviceInstance);
+        } catch (Exception e) {
+            LOG.error(e.getMessage());
+        }
+    }
+
+    public static void unregisterService(CuratorFramework curatorFramework, TimelyBalancedHost tbh) {
+        try {
+            try {
+                Stat stat = curatorFramework.checkExists().forPath(SERVICE_DISCOVERY_PATH);
+                if (stat == null) {
+                    curatorFramework.create().creatingParentContainersIfNeeded().forPath(SERVICE_DISCOVERY_PATH);
+                }
+            } catch (Exception e) {
+                LOG.error(e.getMessage());
+            }
+
+            ServiceInstance<ServerDetails> serviceInstance = serviceInstance(tbh);
+            ServiceDiscovery<ServerDetails> discovery = ServiceDiscoveryBuilder.builder(ServerDetails.class)
+                    .client(curatorFramework).basePath(SERVICE_DISCOVERY_PATH).build();
+            discovery.start();
+            discovery.unregisterService(serviceInstance);
+        } catch (Exception e) {
+            LOG.error(e.getMessage());
+        }
+    }
+
+    private static ServiceInstance<ServerDetails> serviceInstance(TimelyBalancedHost config) throws Exception {
+        ServerDetails serverDetails = new ServerDetails();
+        serverDetails.setHost(config.getHost());
+        serverDetails.setTcpPort(config.getTcpPort());
+        serverDetails.setHttpPort(config.getHttpPort());
+        serverDetails.setWsPort(config.getWsPort());
+        serverDetails.setUdpPort(config.getUdpPort());
+
+        ServiceInstanceBuilder<ServerDetails> builder = ServiceInstance.builder();
+        String serviceName = config.getHost() + ":" + config.getTcpPort();
+        ServiceInstance<ServerDetails> serviceInstance = builder.id(serviceName).name("timely-server")
+                .address(config.getHost()).port(config.getTcpPort()).payload(serverDetails).build();
+        return serviceInstance;
+    }
+}
@@ -0,0 +1,270 @@
+package timely.balancer.resolver;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Random;
+import java.util.UUID;
+
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.CuratorFrameworkFactory;
+import org.apache.curator.retry.RetryNTimes;
+import org.apache.zookeeper.server.ServerConfig;
+import org.apache.zookeeper.server.ZooKeeperServerMain;
+import org.apache.zookeeper.server.quorum.QuorumPeerConfig;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.util.SocketUtils;
+import timely.balancer.configuration.BalancerConfiguration;
+import timely.balancer.connection.TimelyBalancedHost;
+import timely.balancer.resolver.eventing.MetricAssignedCallback;
+import timely.balancer.resolver.eventing.MetricAssignedEvent;
+import timely.balancer.resolver.eventing.MetricBalanceCallback;
+import timely.balancer.resolver.eventing.MetricBalanceEvent;
+import timely.balancer.resolver.eventing.MetricHostCallback;
+import timely.balancer.resolver.eventing.MetricHostEvent;
+
+public class TestBalancedMetricResolver {
+
+    private final Logger LOG = LoggerFactory.getLogger(getClass());
+    private CuratorFramework curatorFramework;
+    private SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd HHmmss.SSS");
+    private Random r = new Random(System.currentTimeMillis());
+    private List<TimelyBalancedHost> timelyHosts;
+    private Map<MetricAssignedEvent, Integer> metricBalanceEventCounter = new HashMap<>();
+
+    @Before
+    public void startup() {
+        QuorumPeerConfig currentConfig = createZookeeper();
+        InetSocketAddress address = currentConfig.getClientPortAddress();
+        String zkServer = address.getHostName() + ":" + address.getPort();
+        curatorFramework = CuratorFrameworkFactory.newClient(zkServer, 1000, 5000, new RetryNTimes(2, 50));
+        curatorFramework.start();
+        timelyHosts = new ArrayList<>();
+    }
+
+    @After
+    public void shutdown() {
+        curatorFramework.close();
+    }
+
+    private QuorumPeerConfig createZookeeper() {
+        Properties startupProperties = new Properties();
+        File file = new File(System.getProperty("java.io.tmpdir") + File.separator + UUID.randomUUID());
+        file.deleteOnExit();
+        startupProperties.setProperty("dataDir", file.getAbsolutePath());
+        startupProperties.setProperty("clientPort", String.valueOf(SocketUtils.findAvailableTcpPort(3000, 4000)));
+        QuorumPeerConfig quorumConfiguration = new QuorumPeerConfig();
+
+        try {
+            quorumConfiguration.parseProperties(startupProperties);
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+
+        ZooKeeperServerMain zooKeeperServer = new ZooKeeperServerMain();
+        final ServerConfig configuration = new ServerConfig();
+        configuration.readFrom(quorumConfiguration);
+
+        new Thread() {
+
+            public void run() {
+                try {
+                    zooKeeperServer.runFromConfig(configuration);
+                } catch (IOException e) {
+                    LOG.error("ZooKeeper Failed", e);
+                }
+            }
+        }.start();
+
+        return quorumConfiguration;
+    }
+
+    private BalancerConfiguration getBalancerConfiguration() {
+        BalancerConfiguration balancerConfiguration = new BalancerConfiguration();
+        balancerConfiguration.setMetricAssignmentPersisterType("FILE");
+        File file = new File(System.getProperty("java.io.tmpdir") + File.separator + UUID.randomUUID());
+        file.deleteOnExit();
+        balancerConfiguration.setAssignmentFile(file.getAbsolutePath());
+        balancerConfiguration.setBalanceDelay(0);
+        balancerConfiguration.setBalancePeriod(1000);
+        balancerConfiguration.setPersistDelay(0);
+        balancerConfiguration.setPersistDelay(1000);
+        balancerConfiguration.setCheckServerHealthInterval(10);
+        return balancerConfiguration;
+    }
+
+    private void createTimelyHosts(BalancerConfiguration balancerConfiguration, int numHosts) {
+        for (int port = 1000, x = 0; x < numHosts; x++, port += 10) {
+            TimelyBalancedHost tbh = TimelyBalancedHost.of("localhost", port);
+            tbh.setBalancerConfig(balancerConfiguration);
+            timelyHosts.add(tbh);
+            ServiceRegistry.registerService(curatorFramework, tbh);
+        }
+    }
+
+    private void registerCallbacks(BalancedMetricResolver resolver) {
+        resolver.registerCallback(new MetricAssignedCallback() {
+
+            @Override
+            public void onEvent(MetricAssignedEvent event) {
+                LOG.debug(String.format("%s metric:%s lose:%s:%d gain:%s:%d reason:%s",
+                        event.getClass().getSimpleName(), event.getMetric(),
+                        event.getLosingHost() == null ? null : event.getLosingHost().getHost(),
+                        event.getLosingHost() == null ? null : event.getLosingHost().getTcpPort(),
+                        event.getGainingHost().getHost(), event.getGainingHost().getTcpPort(), event.getReason()));
+            }
+        });
+
+        resolver.registerCallback(new MetricHostCallback() {
+
+            @Override
+            public void onEvent(MetricHostEvent event) {
+                LOG.debug(String.format("%s host:%s:%d type:%s", event.getClass().getSimpleName(),
+                        event.getTimelyBalancedHost().getHost(), event.getTimelyBalancedHost().getTcpPort(),
+                        event.getActionType()));
+            }
+        });
+
+        resolver.registerCallback(new MetricBalanceCallback() {
+
+            @Override
+            public void onEvent(MetricBalanceEvent event) {
+                LOG.debug(String.format("%s type:%s progress:%s reassigned:%d", event.getClass().getSimpleName(),
+                        event.getBalanceType(), event.getProgressType(), event.getNumRessigned()));
+            }
+        });
+    }
+
+    private TimelyBalancedHost randomServerDown(TestHealthChecker healthChecker, int numHosts) {
+        TimelyBalancedHost tbh = timelyHosts.get(r.nextInt(numHosts));
+        LOG.debug("Marking server down:" + TimelyBalancedHost.toStringShort(tbh));
+        healthChecker.serverDown(tbh);
+        return tbh;
+    }
+
+    private TimelyBalancedHost randomServerToggle(TestHealthChecker healthChecker, int numHosts) {
+        TimelyBalancedHost tbh = timelyHosts.get(r.nextInt(numHosts));
+        LOG.debug("Marking server " + (tbh.isUp() ? "down:" : "up:") + TimelyBalancedHost.toStringShort(tbh));
+        healthChecker.serverToggle(tbh);
+        return tbh;
+    }
+
+    private TimelyBalancedHost randomServerUnregister(int numHosts) {
+        TimelyBalancedHost tbh = timelyHosts.get(r.nextInt(numHosts));
+        LOG.debug("Unregistering:" + TimelyBalancedHost.toStringShort(tbh));
+        ServiceRegistry.unregisterService(curatorFramework, tbh);
+        return tbh;
+    }
+
+    @Test
+    public void test1() {
+        LOG.debug("Configure test");
+        int numHosts = 10;
+        BalancerConfiguration balancerConfiguration = getBalancerConfiguration();
+        createTimelyHosts(balancerConfiguration, numHosts);
+        TestHealthChecker healthChecker = new TestHealthChecker(balancerConfiguration, timelyHosts);
+        BalancedMetricResolver balancedMetricResolver = new BalancedMetricResolver(curatorFramework,
+                balancerConfiguration, healthChecker);
+        registerCallbacks(balancedMetricResolver);
+        LOG.debug("Start balancedMetricResolver");
+        balancedMetricResolver.start();
+        long now = System.currentTimeMillis();
+        int numMetrics = 100;
+        for (int x = 0; x < numMetrics; x++) {
+            balancedMetricResolver.getHostPortKeyIngest(String.format("metric%03d", x));
+        }
+
+        LOG.debug("Begin monitoring");
+        long cycle = 0;
+        List<TimelyBalancedHost> serversUnregistered = new ArrayList<>();
+        while (System.currentTimeMillis() < now + 15000) {
+            try {
+                Thread.sleep(100);
+                for (int x = 0; x < 10000; x++) {
+                    int mNum = r.nextInt(numMetrics);
+                    balancedMetricResolver.getHostPortKeyIngest(String.format("metric%03d", mNum));
+                }
+                // if (cycle % 100 == 0) {
+                // randomServerDown(healthChecker, numHosts);
+                // }
+                // if (cycle % 2 == 0) {
+                // randomServerToggle(healthChecker, numHosts);
+                // }
+                if (cycle % 100 == 0) {
+                    serversUnregistered.add(randomServerUnregister(numHosts));
+                }
+                if (cycle % 500 == 0) {
+                    serversUnregistered.stream().forEach(tbh -> ServiceRegistry.registerService(curatorFramework, tbh));
+                    serversUnregistered.clear();
+                }
+
+            } catch (InterruptedException e) {
+
+            }
+            cycle++;
+        }
+        LOG.debug("Stop balancedMetricResolver");
+        balancedMetricResolver.stop();
+    }
+
+    @Test
+    public void test2() {
+        LOG.debug("Configure test");
+        int numHosts = 10;
+        BalancerConfiguration balancerConfiguration = getBalancerConfiguration();
+        createTimelyHosts(balancerConfiguration, numHosts);
+        TestHealthChecker healthChecker = new TestHealthChecker(balancerConfiguration, timelyHosts);
+        BalancedMetricResolver balancedMetricResolver = new BalancedMetricResolver(curatorFramework,
+                balancerConfiguration, healthChecker);
+        registerCallbacks(balancedMetricResolver);
+        LOG.debug("Start balancedMetricResolver");
+        balancedMetricResolver.start();
+        long now = System.currentTimeMillis();
+        int numMetrics = 100;
+        for (int x = 0; x < numMetrics; x++) {
+            balancedMetricResolver.getHostPortKeyIngest(String.format("metric%03d", x));
+        }
+
+        LOG.debug("Begin monitoring");
+        long cycle = 0;
+        List<TimelyBalancedHost> serversUnregistered = new ArrayList<>();
+        while (System.currentTimeMillis() < now + 15000) {
+            try {
+                Thread.sleep(100);
+                for (int x = 0; x < 10000; x++) {
+                    int mNum = r.nextInt(numMetrics);
+                    balancedMetricResolver.getHostPortKeyIngest(String.format("metric%03d", mNum));
+                }
+                // if (cycle % 100 == 0) {
+                // randomServerDown(healthChecker, numHosts);
+                // }
+                // if (cycle % 2 == 0) {
+                // randomServerToggle(healthChecker, numHosts);
+                // }
+                if (cycle % 100 == 0) {
+                    serversUnregistered.add(randomServerUnregister(numHosts));
+                }
+                if (cycle % 500 == 0) {
+                    serversUnregistered.stream().forEach(tbh -> ServiceRegistry.registerService(curatorFramework, tbh));
+                    serversUnregistered.clear();
+                }
+
+            } catch (InterruptedException e) {
+
+            }
+            cycle++;
+        }
+        LOG.debug("Stop balancedMetricResolver");
+        balancedMetricResolver.stop();
+    }
+}
@@ -0,0 +1,89 @@
+package timely.balancer.resolver;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import timely.balancer.configuration.BalancerConfiguration;
+import timely.balancer.connection.TimelyBalancedHost;
+import timely.balancer.healthcheck.HealthChecker;
+import timely.balancer.healthcheck.TcpHealthChecker;
+
+public class TestHealthChecker implements HealthChecker {
+
+    private final Logger LOG = LoggerFactory.getLogger(TcpHealthChecker.class);
+    private List<TimelyBalancedHost> timelyHosts = new ArrayList<>();
+    private Map<TimelyBalancedHost, Boolean> serverUpMap = Collections.synchronizedMap(new HashMap<>());
+    private ScheduledExecutorService executorService = Executors.newScheduledThreadPool(1);
+
+    public TestHealthChecker(BalancerConfiguration config, List<TimelyBalancedHost> timelyHosts) {
+        setTimelyHosts(timelyHosts);
+        this.executorService.scheduleAtFixedRate(() -> {
+            synchronized (timelyHosts) {
+                for (TimelyBalancedHost h : timelyHosts) {
+                    try {
+                        if (TestHealthChecker.this.isServerHealthy(h)) {
+                            h.reportSuccess();
+                        } else {
+                            h.reportFailure();
+                        }
+                    } catch (Exception e) {
+                        LOG.error(e.getMessage(), e);
+                        h.reportFailure();
+                    }
+                }
+            }
+        }, 0, config.getCheckServerHealthInterval(), TimeUnit.MILLISECONDS);
+    }
+
+    @Override
+    public void setTimelyHosts(List<TimelyBalancedHost> timelyHosts) {
+        synchronized (timelyHosts) {
+            this.timelyHosts.clear();
+            this.timelyHosts.addAll(timelyHosts);
+            for (TimelyBalancedHost h : timelyHosts) {
+                serverUpMap.put(h, true);
+            }
+        }
+
+    }
+
+    public boolean isServerHealthy(TimelyBalancedHost h) {
+        return serverUpMap.get(h);
+    }
+
+    public void serverUp(TimelyBalancedHost h) {
+        serverUpMap.put(h, true);
+    }
+
+    public void serverDown(TimelyBalancedHost h) {
+        serverUpMap.put(h, false);
+    }
+
+    public void serverToggle(TimelyBalancedHost h) {
+        synchronized (serverUpMap) {
+            serverUpMap.put(h, !serverUpMap.get(h));
+        }
+    }
+
+    @Override
+    public void close() throws Exception {
+        this.executorService.shutdown();
+        try {
+            this.executorService.awaitTermination(5, TimeUnit.SECONDS);
+        } catch (InterruptedException e) {
+
+        } finally {
+            if (!this.executorService.isTerminated()) {
+                this.executorService.shutdownNow();
+            }
+        }
+    }
+}
@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Configuration status="WARN">
+  <Appenders>
+    <Console name="CONSOLE" target="SYSTEM_OUT">
+      <PatternLayout pattern="%d{yyyyMMdd HHmmss.SSS} [%t] %-5level %logger{36} - %msg%n"/>
+    </Console>
+  </Appenders>
+  <Loggers>
+    <Logger name="org.apache.accumulo.core.client.impl.ThriftTransportPool" level="OFF" additivity="false" />
+    <Logger name="org.apache.accumulo.fate.zookeeper.ZooCache" level="OFF" additivity="false" />
+    <Logger name="org.apache.zookeeper" level="OFF" additivity="false" />
+    <Logger name="timely.balancer.resolver" level="TRACE" additivity="true" />
+    <Logger name="timely.balancer.resolver.TestBalancedMetricResolver" level="DEBUG" additivity="true" />
+    <Root level="ERROR">
+      <AppenderRef ref="CONSOLE"/>
+    </Root>
+  </Loggers>
+</Configuration>
@@ -4,12 +4,13 @@
  <parent>
    <groupId>timely</groupId>
    <artifactId>timely</artifactId>
-    <version>1.0.1-SNAPSHOT</version>
+    <version>1.2.8-SNAPSHOT</version>
  </parent>
  <artifactId>timely-client</artifactId>
  <properties>
    <!-- PEP 440 compliant version gets substituted into src/main/python/template/setup.py -->
-    <python.client.version>0.0.7.dev0</python.client.version>
+    <!-- use .dev0 instead of -SNAPSHOT -->
+    <python.client.version>1.2.8.dev0</python.client.version>
  </properties>
  <dependencies>
    <dependency>
@@ -107,4 +108,43 @@
      </plugin>
    </plugins>
  </build>
+  <profiles>
+    <profile>
+      <id>python</id>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>org.codehaus.mojo</groupId>
+            <artifactId>exec-maven-plugin</artifactId>
+            <executions>
+              <execution>
+                <id>python-dist</id>
+                <goals>
+                  <goal>exec</goal>
+                </goals>
+                <phase>package</phase>
+                <configuration>
+                  <executable>python</executable>
+                  <workingDirectory>${project.basedir}/src/main/python</workingDirectory>
+                  <commandlineArgs>setup.py sdist --dist-dir ${project.build.directory}</commandlineArgs>
+                </configuration>
+              </execution>
+              <execution>
+                <id>python-clean</id>
+                <goals>
+                  <goal>exec</goal>
+                </goals>
+                <phase>package</phase>
+                <configuration>
+                  <executable>rm</executable>
+                  <workingDirectory>${project.basedir}/src/main/python</workingDirectory>
+                  <commandlineArgs>-rf timely.egg-info setup.py</commandlineArgs>
+                </configuration>
+              </execution>
+            </executions>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
+  </profiles>
 </project>
@@ -1,12 +1,13 @@
 package timely.client.http;

+import javax.net.ssl.HostnameVerifier;
 import javax.net.ssl.SSLContext;

 import org.apache.http.client.CookieStore;
 import org.apache.http.client.config.CookieSpecs;
 import org.apache.http.client.config.RequestConfig;
-import org.apache.http.conn.ssl.DefaultHostnameVerifier;
 import org.apache.http.conn.ssl.NoopHostnameVerifier;
+import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
 import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.impl.client.HttpClientBuilder;
 import org.apache.http.impl.client.HttpClients;
@@ -40,14 +41,31 @@ public class HttpClient {
    }

    public static CloseableHttpClient get(SSLContext ssl, CookieStore cookieStore, boolean hostVerificationEnabled) {
-        RequestConfig defaultRequestConfig = RequestConfig.custom().setCookieSpec(CookieSpecs.STANDARD).build();
+        return get(ssl, cookieStore, hostVerificationEnabled, false);
+    }

-        HttpClientBuilder builder = HttpClients.custom().setSSLContext(ssl).setDefaultCookieStore(cookieStore)
-                .setDefaultRequestConfig(defaultRequestConfig);
+    public static CloseableHttpClient get(SSLContext ssl, CookieStore cookieStore, boolean hostVerificationEnabled,
+            boolean clientAuth) {
+        HostnameVerifier hostnameVerifier;
        if (hostVerificationEnabled) {
-            builder.setSSLHostnameVerifier(new DefaultHostnameVerifier());
+            hostnameVerifier = SSLConnectionSocketFactory.getDefaultHostnameVerifier();
        } else {
-            builder.setSSLHostnameVerifier(new NoopHostnameVerifier());
+            hostnameVerifier = new NoopHostnameVerifier();
+        }
+        RequestConfig defaultRequestConfig = RequestConfig.custom().setCookieSpec(CookieSpecs.STANDARD).build();
+        HttpClientBuilder builder;
+        if (cookieStore == null) {
+            builder = HttpClients.custom().setSSLContext(ssl).disableCookieManagement()
+                    .setDefaultRequestConfig(defaultRequestConfig).setSSLHostnameVerifier(hostnameVerifier);
+        } else {
+            builder = HttpClients.custom().setSSLContext(ssl).setDefaultCookieStore(cookieStore)
+                    .setDefaultRequestConfig(defaultRequestConfig).setSSLHostnameVerifier(hostnameVerifier);
+        }
+
+        if (clientAuth) {
+            SSLConnectionSocketFactory sslCSF = new SSLConnectionSocketFactory(ssl,
+                    new String[] { "TLSv1.1", "TLSv1.2" }, null, hostnameVerifier);
+            builder.setSSLSocketFactory(sslCSF);
        }
        return builder.build();
    }
@@ -1,7 +1,8 @@
 package timely.client.tcp;

+import java.io.BufferedWriter;
 import java.io.IOException;
-import java.io.PrintWriter;
+import java.io.OutputStreamWriter;
 import java.net.Socket;

 import org.slf4j.Logger;
@@ -14,7 +15,7 @@ public class TcpClient implements AutoCloseable {
    private final String host;
    private final int port;
    private Socket sock = null;
-    private PrintWriter out = null;
+    private BufferedWriter out = null;
    private long connectTime = 0L;
    private long backoff = 2000;
    private int bufferSize = -1;
@@ -47,7 +48,6 @@ public class TcpClient implements AutoCloseable {
     * 
     * @param metric
     *            newline terminated string representation of Timely metric
-     * @throws IOException
     */
    public synchronized void write(String metric) throws IOException {
        if (connect() != 0) {
@@ -60,7 +60,7 @@ public class TcpClient implements AutoCloseable {
        }
    }

-    public synchronized void flush() {
+    public synchronized void flush() throws IOException {
        if (null != out) {
            out.flush();
            writesSinceFlush = 0;
@@ -94,12 +94,12 @@ public class TcpClient implements AutoCloseable {
    }

    private synchronized int connect() {
-        if (null == sock || !sock.isConnected() || out.checkError()) {
+        if (null == sock || !sock.isConnected()) {
            if (System.currentTimeMillis() > (connectTime + backoff)) {
                try {
                    connectTime = System.currentTimeMillis();
                    sock = new Socket(host, port);
-                    out = new PrintWriter(sock.getOutputStream(), false);
+                    out = new BufferedWriter(new OutputStreamWriter(sock.getOutputStream()));
                    backoff = 2000;
                    LOG.trace("Connected to Timely at {}:{}", host, port);
                } catch (IOException e) {
@@ -1,5 +1,8 @@
 package timely.client.websocket;

+import java.util.List;
+import java.util.Map;
+
 import javax.websocket.CloseReason;
 import javax.websocket.Endpoint;
 import javax.websocket.EndpointConfig;
@@ -35,4 +38,6 @@ public abstract class ClientHandler extends Endpoint {
        LOG.error("Error occurred on Websocket session" + session.getId(), error);
    }

+    public void beforeRequest(Map<String, List<String>> headers) {
+    }
 }
@@ -14,13 +14,15 @@ import org.apache.http.cookie.Cookie;
 public class TimelyEndpointConfig implements ClientEndpointConfig {

    private String sessionCookie = null;
+    private final ClientHandler clientEndpoint;

    private static final String FORMAT = "%s=%s;";

-    public TimelyEndpointConfig(Cookie sessionCookie) {
+    public TimelyEndpointConfig(ClientHandler clientEndpoint, Cookie sessionCookie) {
        if (null != sessionCookie) {
            this.sessionCookie = String.format(FORMAT, sessionCookie.getName(), sessionCookie.getValue());
        }
+        this.clientEndpoint = clientEndpoint;
    }

    @Override
@@ -58,6 +60,7 @@ public class TimelyEndpointConfig implements ClientEndpointConfig {
                if (null != sessionCookie) {
                    headers.put("Cookie", Collections.singletonList(sessionCookie));
                }
+                TimelyEndpointConfig.this.clientEndpoint.beforeRequest(headers);
            }
        };
    }
@@ -40,6 +40,7 @@ public class WebSocketClient implements AutoCloseable {
    private final int timelyHttpsPort;
    private final int timelyWssPort;
    private final boolean doLogin;
+    private final boolean clientAuth;
    private final String timelyUsername;
    private final String timelyPassword;
    private final boolean hostVerificationEnabled;
@@ -51,13 +52,14 @@ public class WebSocketClient implements AutoCloseable {
    protected volatile boolean closed = true;

    protected WebSocketClient(SSLContext ssl, String timelyHostname, int timelyHttpsPort, int timelyWssPort,
-            boolean doLogin, String timelyUsername, String timelyPassword, boolean hostVerificationEnabled,
-            int bufferSize) {
+            boolean clientAuth, boolean doLogin, String timelyUsername, String timelyPassword,
+            boolean hostVerificationEnabled, int bufferSize) {
        this.ssl = ssl;
        this.timelyHostname = timelyHostname;
        this.timelyHttpsPort = timelyHttpsPort;
        this.timelyWssPort = timelyWssPort;
        this.doLogin = doLogin;
+        this.clientAuth = clientAuth;
        this.timelyUsername = timelyUsername;
        this.timelyPassword = timelyPassword;
        this.hostVerificationEnabled = hostVerificationEnabled;
@@ -67,28 +69,28 @@ public class WebSocketClient implements AutoCloseable {
        Preconditions.checkNotNull(timelyHttpsPort, "%s must be supplied", "Timely HTTPS port");
        Preconditions.checkNotNull(timelyWssPort, "%s must be supplied", "Timely WSS port");

-        if (doLogin && ((StringUtils.isEmpty(timelyUsername) && !StringUtils.isEmpty(timelyPassword)
-                || (!StringUtils.isEmpty(timelyUsername) && StringUtils.isEmpty(timelyPassword))))) {
-            throw new IllegalArgumentException("Both Timely username and password must be empty or non-empty");
+        if (doLogin && !clientAuth) {
+            if ((StringUtils.isEmpty(timelyUsername) && !StringUtils.isEmpty(timelyPassword)
+                    || (!StringUtils.isEmpty(timelyUsername) && StringUtils.isEmpty(timelyPassword)))) {
+                throw new IllegalArgumentException("Both Timely username and password must be empty or non-empty");
+            }
        }
-
    }

-    protected WebSocketClient(String timelyHostname, int timelyHttpsPort, int timelyWssPort, boolean doLogin,
-            String timelyUsername, String timelyPassword, String keyStoreFile, String keyStoreType, String keyStorePass,
-            String trustStoreFile, String trustStoreType, String trustStorePass, boolean hostVerificationEnabled,
-            int bufferSize) {
+    protected WebSocketClient(String timelyHostname, int timelyHttpsPort, int timelyWssPort, boolean clientAuth,
+            boolean doLogin, String timelyUsername, String timelyPassword, String keyStoreFile, String keyStoreType,
+            String keyStorePass, String trustStoreFile, String trustStoreType, String trustStorePass,
+            boolean hostVerificationEnabled, int bufferSize) {
        this(HttpClient.getSSLContext(trustStoreFile, trustStoreType, trustStorePass, keyStoreFile, keyStoreType,
-                keyStorePass), timelyHostname, timelyHttpsPort, timelyWssPort, doLogin, timelyUsername, timelyPassword,
-                hostVerificationEnabled, bufferSize);
+                keyStorePass), timelyHostname, timelyHttpsPort, timelyWssPort, clientAuth, doLogin, timelyUsername,
+                timelyPassword, hostVerificationEnabled, bufferSize);
    }

    public void open(ClientHandler clientEndpoint) throws IOException, DeploymentException, URISyntaxException {
-
        Cookie sessionCookie = null;
        if (doLogin) {
            BasicCookieStore cookieJar = new BasicCookieStore();
-            try (CloseableHttpClient client = HttpClient.get(ssl, cookieJar, hostVerificationEnabled)) {
+            try (CloseableHttpClient client = HttpClient.get(ssl, cookieJar, hostVerificationEnabled, clientAuth)) {

                String target = "https://" + timelyHostname + ":" + timelyHttpsPort + "/login";

@@ -140,8 +142,8 @@ public class WebSocketClient implements AutoCloseable {
        webSocketClient.getProperties().put(ClientProperties.SSL_ENGINE_CONFIGURATOR, sslEngine);
        webSocketClient.getProperties().put(ClientProperties.INCOMING_BUFFER_SIZE, bufferSize);
        String wssPath = "wss://" + timelyHostname + ":" + timelyWssPort + "/websocket";
-        session = webSocketClient.connectToServer(clientEndpoint, new TimelyEndpointConfig(sessionCookie),
-                new URI(wssPath));
+        session = webSocketClient.connectToServer(clientEndpoint,
+                new TimelyEndpointConfig(clientEndpoint, sessionCookie), new URI(wssPath));

        final ByteBuffer pingData = ByteBuffer.allocate(0);
        webSocketClient.getScheduledExecutorService().scheduleAtFixedRate(() -> {
@@ -27,21 +27,21 @@ public class WebSocketSubscriptionClient extends WebSocketClient {
    private final String subscriptionId;

    public WebSocketSubscriptionClient(SSLContext ssl, String timelyHostname, int timelyHttpsPort, int timelyWssPort,
-            boolean doLogin, String timelyUsername, String timelyPassword, boolean hostVerificationEnabled,
-            int bufferSize) {
-        super(ssl, timelyHostname, timelyHttpsPort, timelyWssPort, doLogin, timelyUsername, timelyPassword,
+            boolean clientAuth, boolean doLogin, String timelyUsername, String timelyPassword,
+            boolean hostVerificationEnabled, int bufferSize) {
+        super(ssl, timelyHostname, timelyHttpsPort, timelyWssPort, clientAuth, doLogin, timelyUsername, timelyPassword,
                hostVerificationEnabled, bufferSize);
        subscriptionId = UUID.randomUUID().toString();
        LOG.trace("Created WebSocketClient with subscriptionId {}", this.subscriptionId);
    }

-    public WebSocketSubscriptionClient(String timelyHostname, int timelyHttpsPort, int timelyWssPort, boolean doLogin,
-            String timelyUsername, String timelyPassword, String keyStoreFile, String keyStoreType, String keyStorePass,
-            String trustStoreFile, String trustStoreType, String trustStorePass, boolean hostVerificationEnabled,
-            int bufferSize) {
-        super(timelyHostname, timelyHttpsPort, timelyWssPort, doLogin, timelyUsername, timelyPassword, keyStoreFile,
-                keyStoreType, keyStorePass, trustStoreFile, trustStoreType, trustStorePass, hostVerificationEnabled,
-                bufferSize);
+    public WebSocketSubscriptionClient(String timelyHostname, int timelyHttpsPort, int timelyWssPort,
+            boolean clientAuth, boolean doLogin, String timelyUsername, String timelyPassword, String keyStoreFile,
+            String keyStoreType, String keyStorePass, String trustStoreFile, String trustStoreType,
+            String trustStorePass, boolean hostVerificationEnabled, int bufferSize) {
+        super(timelyHostname, timelyHttpsPort, timelyWssPort, clientAuth, doLogin, timelyUsername, timelyPassword,
+                keyStoreFile, keyStoreType, keyStorePass, trustStoreFile, trustStoreType, trustStorePass,
+                hostVerificationEnabled, bufferSize);
        subscriptionId = UUID.randomUUID().toString();
        LOG.trace("Created WebSocketClient with subscriptionId {}", this.subscriptionId);
    }
@@ -62,7 +62,6 @@ public interface ObjectSizeOf {
         * ObjectSizeOf interface and if implemented on the object will use that.
         * Otherwise it will do a simple navigation of the fields using reflection.
         *
-         * @param o
         * @return an approximation of the object size
         */
        public static long getObjectSize(Object o, boolean useSizeInBytesMethod, boolean debug) {
@@ -77,8 +77,6 @@ public class Tag implements Comparable<Tag>, Serializable {

    /**
     * Joins key and value into key=value
-     * 
-     * @return
     */
    public String join() {
        return tagParser.combine(this);
@@ -47,7 +47,6 @@ public class TagListParser implements Parser<List<Tag>>, Combiner<Collection<Tag
     * An alternative combiner for a map of string to string instead of a list of
     * tags
     * 
-     * @param tags
     * @return string representation
     */
    public String combine(Map<String, String> tags) {
@@ -23,7 +23,7 @@ def pivot(df, metric, groupByColumn=None):
    return dataFrame


-def resample(df, sample, how='mean'):
+def resample(df, sample, how='mean', interpolate=True, fill=None):

    dataFrame = pandas.DataFrame(df, copy=True)
    if dataFrame is not None:
@@ -2,7 +2,6 @@ import time
 from datetime import datetime
 from datetime import timedelta
 from datetime import tzinfo
-from pandas.tslib import Timedelta
 import pandas

 class TimeDateError(ValueError):
@@ -16,14 +16,15 @@ utc = UTC()
 class TimelyWebSocketClient(WebSocketClient):

    def __init__(self, hostport, metric, tags, startTime, endTime, connect_timeout=WebSocketClient.DEFAULT_CONNECT_TIMEOUT,
-                 request_timeout=WebSocketClient.DEFAULT_REQUEST_TIMEOUT):
+                 request_timeout=WebSocketClient.DEFAULT_REQUEST_TIMEOUT, client_cert=None, client_key=None, ca_certs=None, validate_cert=False):

        self.metric = metric
        self.tags = tags
        self.startTime = startTime
        self.endTime = endTime
        self.subscriptionId = random.randint(1000000000, 9999999999)
-        WebSocketClient.__init__(self, hostport, '/websocket', {'metric' : metric}, connect_timeout, request_timeout)
+        WebSocketClient.__init__(self, hostport, '/websocket', {'metric' : metric}, connect_timeout, request_timeout,
+                                 client_cert=client_cert, client_key=client_key, ca_certs=ca_certs, validate_cert=validate_cert)

    def _on_message(self, msg):
        # implement in subclass
@@ -72,7 +73,8 @@ class TimelyMetric(TimelyWebSocketClient):
    client = None
    endtime = None

-    def __init__(self, hostport, metric, tags, beginStr, endStr, period, sample):
+    def __init__(self, hostport, metric, tags, beginStr, endStr, period, sample,
+                 client_cert=None, client_key=None, ca_certs=None, validate_cert=False):
        self.series = pandas.Series()
        self.metric = metric
        self.tags = tags
@@ -82,7 +84,8 @@ class TimelyMetric(TimelyWebSocketClient):
        self.data = []
        self.timeDateRange = TimeDateRange(beginStr, endStr, period)

-        TimelyWebSocketClient.__init__(self, hostport, metric, tags, self.timeDateRange.getBeginMs(), self.timeDateRange.getEndMs())
+        TimelyWebSocketClient.__init__(self, hostport, metric, tags, self.timeDateRange.getBeginMs(), self.timeDateRange.getEndMs(),
+                                       client_cert=client_cert, client_key=client_key, ca_certs=ca_certs, validate_cert=validate_cert)

    def debugOn(self):
        self.debug = True
@@ -15,10 +15,14 @@ class WebSocketClient():
    APPLICATION_JSON = 'application/json'

    def __init__(self, hostport, path, params, connect_timeout=DEFAULT_CONNECT_TIMEOUT,
-                 request_timeout=DEFAULT_REQUEST_TIMEOUT):
+                 request_timeout=DEFAULT_REQUEST_TIMEOUT, client_cert=None, client_key=None, ca_certs=None, validate_cert=False):

        self.connect_timeout = connect_timeout
        self.request_timeout = request_timeout
+        self.client_cert=client_cert
+        self.client_key=client_key
+        self.ca_certs=ca_certs
+        self.validate_cert=validate_cert
        self.url = 'wss://' + hostport
        if (path is not None and len(path) > 0):
            if (path.startswith('/')):
@@ -39,7 +43,8 @@ class WebSocketClient():
        """Connect to the server.
        :param str url: server URL.
        """
-        request = websocket.httpclient.HTTPRequest(self.url, connect_timeout=self.connect_timeout, request_timeout=self.request_timeout, validate_cert=False)
+        request = websocket.httpclient.HTTPRequest(self.url, connect_timeout=self.connect_timeout, request_timeout=self.request_timeout, validate_cert=self.validate_cert,
+                                                   client_cert=self.client_cert, client_key=self.client_key, ca_certs=self.ca_certs)
        websocket.websocket_connect(request, io_loop=ioloop, callback=self._connect_callback)

    def send(self, data):
@@ -5,7 +5,9 @@ import os
 from timely.TimelyAnalyticConfiguration import TimelyAnalyticConfiguration

 hostport = "127.0.0.1:5244"
-timelyMetric = timely.TimelyMetric.TimelyMetric(hostport, 'timely.keys.metric.inserted', None, None, None, '24 hours', None).fetch()
+client_cert = None
+client_key = None
+timelyMetric = timely.TimelyMetric.TimelyMetric(hostport, 'timely.keys.metric.inserted', None, None, None, '24 hours', None, client_cert=client_cert, client_key=client_key).fetch()

 print(timelyMetric.getDataFrame())
 exit()
@@ -5,4 +5,9 @@
    <Method name="connect" />
    <Class name="timely.client.tcp.TcpClient"/>
  </Match>
+  <Match>
+    <!-- Must ignore these everywhere, because of a javac byte code generation bug -->
+    <!-- https://github.com/spotbugs/spotbugs/issues/756 -->
+    <Bug pattern="RCN_REDUNDANT_NULLCHECK_WOULD_HAVE_BEEN_A_NPE" />
+  </Match>
 </FindBugsFilter>
@@ -47,9 +47,9 @@ public class SerializationTest {
        testSerialization(remove);
    }

-    @SuppressWarnings("unchecked")
    private <T> void testSerialization(T o) throws Exception {
        String ser = mapper.writeValueAsString(o);
+        @SuppressWarnings("unchecked")
        T des = mapper.readValue(ser, (Class<T>) o.getClass());
        Assert.assertEquals(o, des);
    }
@@ -4,7 +4,7 @@
  <parent>
    <groupId>timely</groupId>
    <artifactId>timely</artifactId>
-    <version>1.0.1-SNAPSHOT</version>
+    <version>1.2.8-SNAPSHOT</version>
  </parent>
  <artifactId>collectd-nsq-plugin</artifactId>
  <dependencies>
@@ -10,7 +10,7 @@ import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.impl.client.HttpClientBuilder;
 import org.collectd.api.OConfigItem;

-public class PooledCloseableHttpClientFactory implements PooledObjectFactory {
+public class PooledCloseableHttpClientFactory implements PooledObjectFactory<CloseableHttpClient> {

    private int connectionRequestTimeout = 10000; // 10 seconds in milliseconds
    private int connectTimeout = 30000; // 30 seconds in milliseconds
@@ -43,19 +43,19 @@ public class PooledCloseableHttpClientFactory implements PooledObjectFactory {
    }

    @Override
-    public PooledObject makeObject() throws Exception {
+    public PooledObject<CloseableHttpClient> makeObject() throws Exception {
        RequestConfig requestConfig = RequestConfig.custom().setConnectionRequestTimeout(connectionRequestTimeout)
                .setConnectTimeout(connectTimeout).setSocketTimeout(socketTimeout).build();
        HttpClientBuilder builder = HttpClientBuilder.create();
        builder.setDefaultRequestConfig(requestConfig);
        builder.setConnectionTimeToLive(connectionTimeToLive, TimeUnit.MILLISECONDS);
-        return new DefaultPooledObject(builder.build());
+        return new DefaultPooledObject<>(builder.build());
    }

    @Override
-    public void destroyObject(PooledObject pooledObject) throws Exception {
+    public void destroyObject(PooledObject<CloseableHttpClient> pooledObject) throws Exception {
        try {
-            ((CloseableHttpClient) pooledObject.getObject()).close();
+            pooledObject.getObject().close();
        } catch (Exception e) {
            // do nothing
        } finally {
@@ -65,17 +65,17 @@ public class PooledCloseableHttpClientFactory implements PooledObjectFactory {
    }

    @Override
-    public boolean validateObject(PooledObject pooledObject) {
+    public boolean validateObject(PooledObject<CloseableHttpClient> pooledObject) {
        return pooledObject.getObject() != null;
    }

    @Override
-    public void activateObject(PooledObject pooledObject) throws Exception {
+    public void activateObject(PooledObject<CloseableHttpClient> pooledObject) throws Exception {
        // do nothing
    }

    @Override
-    public void passivateObject(PooledObject pooledObject) throws Exception {
+    public void passivateObject(PooledObject<CloseableHttpClient> pooledObject) throws Exception {
        // do nothing
    }

@@ -47,6 +47,7 @@ public class WriteNSQPlugin extends CollectDPluginParent
        Collectd.registerWrite(WriteNSQPlugin.class.getName(), this);
    }

+    @Override
    public int write(ValueList vl) {
        ByteArrayOutputStream baos = new ByteArrayOutputStream();
        DataOutputStream out = new DataOutputStream(baos);
@@ -110,15 +111,17 @@ public class WriteNSQPlugin extends CollectDPluginParent
        }
    }

+    @Override
    public int shutdown() {
        Collectd.logInfo("Shutting down connection to NSQ at " + host + ":" + port);
        clientPool.close();
        return 0;
    }

+    @Override
    public int config(OConfigItem config) {
        super.config(config);
-        GenericObjectPoolConfig poolConfig = new GenericObjectPoolConfig();
+        GenericObjectPoolConfig<CloseableHttpClient> poolConfig = new GenericObjectPoolConfig<>();
        // use max size for maxTotal and maxIdle
        // no need to activate or passivate
        poolConfig.setMaxTotal(POOL_MAX_SIZE);
@@ -156,7 +159,7 @@ public class WriteNSQPlugin extends CollectDPluginParent
            Collectd.logError("NSQ host, port, and topic must be configured");
            return -1;
        }
-        this.clientPool = new GenericObjectPool(pooledCloseableHttpClientFactory, poolConfig);
+        this.clientPool = new GenericObjectPool<>(pooledCloseableHttpClientFactory, poolConfig);
        this.endpoint = "http://" + host + ":" + port + "/mpub?topic=" + topic;
        return 0;
    }
@@ -4,7 +4,7 @@
  <parent>
    <groupId>timely</groupId>
    <artifactId>timely</artifactId>
-    <version>1.0.1-SNAPSHOT</version>
+    <version>1.2.8-SNAPSHOT</version>
  </parent>
  <artifactId>collectd-plugin-parent</artifactId>
  <dependencies>
@@ -4,7 +4,7 @@
  <parent>
    <groupId>timely</groupId>
    <artifactId>timely</artifactId>
-    <version>1.0.1-SNAPSHOT</version>
+    <version>1.2.8-SNAPSHOT</version>
  </parent>
  <artifactId>collectd-timely-plugin</artifactId>
  <dependencies>
@@ -8,7 +8,7 @@ import org.apache.commons.pool2.impl.DefaultPooledObject;
 import org.collectd.api.Collectd;
 import org.collectd.api.OConfigItem;

-public class PooledSocketFactory implements PooledObjectFactory {
+public class PooledSocketFactory implements PooledObjectFactory<Socket> {

    private String host = null;
    private int port = 0;
@@ -81,12 +81,12 @@ public class PooledSocketFactory implements PooledObjectFactory {
    }

    @Override
-    public PooledObject makeObject() throws Exception {
-        return new DefaultPooledObject(connect());
+    public PooledObject<Socket> makeObject() throws Exception {
+        return new DefaultPooledObject<>(connect());
    }

    @Override
-    public void destroyObject(PooledObject pooledObject) throws Exception {
+    public void destroyObject(PooledObject<Socket> pooledObject) throws Exception {
        try {
            Collectd.logDebug("Shutting down connection to Timely at " + host + ":" + port);
            Socket socket = (Socket) pooledObject.getObject();
@@ -103,7 +103,7 @@ public class PooledSocketFactory implements PooledObjectFactory {
    }

    @Override
-    public boolean validateObject(PooledObject pooledObject) {
+    public boolean validateObject(PooledObject<Socket> pooledObject) {
        if (connectionTimeToLive > -1) {
            if (System.currentTimeMillis() > pooledObject.getCreateTime() + connectionTimeToLive) {
                return false;
@@ -114,12 +114,12 @@ public class PooledSocketFactory implements PooledObjectFactory {
    }

    @Override
-    public void activateObject(PooledObject pooledObject) throws Exception {
+    public void activateObject(PooledObject<Socket> pooledObject) throws Exception {
        // do nothing
    }

    @Override
-    public void passivateObject(PooledObject pooledObject) throws Exception {
+    public void passivateObject(PooledObject<Socket> pooledObject) throws Exception {
        // do nothing
    }
 }
@@ -86,13 +86,13 @@ public class WriteTimelyPlugin extends CollectDPluginParent
        PooledSocketFactory socketFactory = new PooledSocketFactory();
        retval = socketFactory.config(config);
        if (retval == 0) {
-            GenericObjectPoolConfig poolConfig = new GenericObjectPoolConfig();
+            GenericObjectPoolConfig<Socket> poolConfig = new GenericObjectPoolConfig<>();
            // use max size for maxTotal and maxIdle
            // no need to activate and passivate
            poolConfig.setMaxTotal(POOL_MAX_SIZE);
            poolConfig.setMaxIdle(POOL_MAX_SIZE);
            poolConfig.setTestOnReturn(true);
-            socketPool = new GenericObjectPool(socketFactory, poolConfig);
+            socketPool = new GenericObjectPool<>(socketFactory, poolConfig);
        }
        return retval;
    }
@@ -4,7 +4,7 @@
  <parent>
    <groupId>timely</groupId>
    <artifactId>timely</artifactId>
-    <version>1.0.1-SNAPSHOT</version>
+    <version>1.2.8-SNAPSHOT</version>
  </parent>
  <artifactId>timely-common</artifactId>
  <dependencies>
@@ -0,0 +1,42 @@
+#!/bin/bash
+
+if [[ `uname` == "Darwin" ]]; then
+        THIS_SCRIPT=`python -c 'import os,sys; print os.path.realpath(sys.argv[1])' $0`
+        TCNATIVE_SUFFIX="jnilib"
+else
+        THIS_SCRIPT=`readlink -f $0`
+        TCNATIVE_SUFFIX="so"
+fi
+
+THIS_DIR="${THIS_SCRIPT%/*}"
+NATIVE_DIR="${THIS_DIR}/META-INF/native"
+BASE_DIR=${THIS_DIR}/..
+TMP_DIR="${BASE_DIR}/tmp"
+CONF_DIR="${BASE_DIR}/conf"
+LIB_DIR="${BASE_DIR}/lib"
+NUM_SERVER_THREADS=4
+
+if [[ -e ${TMP_DIR} ]]; then
+  rm -rf ${TMP_DIR}
+fi
+mkdir ${TMP_DIR}
+
+if [[ -e ${NATIVE_DIR} ]]; then
+  rm -rf ${NATIVE_DIR}
+fi
+mkdir -p ${NATIVE_DIR}
+
+pushd ${BASE_DIR}/bin
+$JAVA_HOME/bin/jar xf ${LIB_DIR}/netty-tcnative-boringssl-static*.jar META-INF/native/libnetty_tcnative_linux_x86_64.${TCNATIVE_SUFFIX}
+$JAVA_HOME/bin/jar xf ${LIB_DIR}/netty-all*.jar META-INF/native/libnetty_transport_native_epoll_x86_64.${TCNATIVE_SUFFIX}
+popd
+
+export CLASSPATH="${CONF_DIR}:${LIB_DIR}/*:${HADOOP_CONF_DIR}"
+JVM_ARGS="-Xmx1G -Xms1G -Dio.netty.eventLoopThreads=${NUM_SERVER_THREADS} Dlog4j.configurationFile=${CONF_DIR}/log4j2-spring.xml"
+JVM_ARGS="${JVM_ARGS} -DLog4jContextSelector=org.apache.logging.log4j.core.async.AsyncLoggerContextSelector"
+JVM_ARGS="${JVM_ARGS} -Djava.library.path=${NATIVE_DIR}"
+
+#JVM_ARGS="${JVM_ARGS} -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5005"
+
+echo "$JAVA_HOME/bin/java ${JVM_ARGS} timely.grafana.auth.GrafanaAuth --spring.config.name=grafana-auth"
+$JAVA_HOME/bin/java ${JVM_ARGS} timely.grafana.auth.GrafanaAuth --spring.config.name=grafana-auth
@@ -0,0 +1,55 @@
+grafana-auth:
+  grafana:
+    host: localhost
+    port: 3001
+  security:
+    session-max-age: 86400
+    cache-expiration-minutes: 60
+    cache-refresh-minutes: 5
+    allow-anonymous-http-access: true
+    allow-anonymous-ws-access: true
+    serverSsl:
+      certificate-file:
+      key-file:
+      #key-password:
+      use-generated-keypair: true
+      trust-store-file:
+      use-openssl: true
+      #use-ciphers:
+    clientSsl:
+      keyFile:
+      keyType: pkcs12
+      #keyPassword:
+      trustStoreFile:
+      trustStoreType: jks
+      #trustStorePassword:
+  http:
+    host: localhost
+    ip: 127.0.0.1
+    port: 3000
+    redirect-path: /secure-me
+    strict-transport-max-age: 608400
+    cors:
+      allow-any-origin: false
+      allow-null-origin: false
+      allow-credentials: true
+      allowed-origins:
+      allowed-methods: DELETE,GET,HEAD,OPTIONS,PUT,POST
+      allowed-headers: content-type
+    httpClientPool:
+      minIdlePerKey: 0
+      maxIdlePerKey: 40
+      maxTotalPerKey: 40
+      maxTotal: -1
+      lifo: true
+      fairness: false
+      maxWaitMillis: -1
+      minEvictableIdleTimeMillis: 1800000
+      evictorShutdownTimeoutMillis: 10000
+      softMinEvictableIdleTimeMillis: -1
+      numTestsPerEvictionRun: 3
+      timeBetweenEvictionRunsMillis: -1
+      blockWhenExhausted: true
+      jmxEnabled: true
+      testOnBorrow: false
+      testOnReturn: false
@@ -0,0 +1,13 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Configuration status="warn" monitorInterval="60">
+  <Appenders>
+    <Console name="Console" target="SYSTEM_OUT">
+      <PatternLayout pattern="%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n"/>
+    </Console>
+  </Appenders>
+  <Loggers>
+    <Root level="error" includeLocation="false">
+      <AppenderRef ref="Console" level="error" />
+    </Root>
+  </Loggers>
+</Configuration>
@@ -0,0 +1,29 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Configuration status="warn" monitorInterval="60">
+  <Properties>
+    <Property name="PID">????</Property>
+    <Property name="LOG_PATTERN">%clr{%d{yyyy-MM-dd HH:mm:ss.SSS}}{faint} %clr{%5p} %clr{${sys:PID}}{magenta} %clr{---}{faint} %clr{[%15.15t]}{faint} %clr{%-40.40c{1.}}{cyan} %clr{:}{faint} %m%n%wEx</Property>
+  </Properties>
+  <Appenders>
+    <RollingRandomAccessFile name="Rolling" fileName="/srv/logs/timely/grafana-auth.log"
+    						 filePattern="/srv/logs/timely/$${date:yyyy-MM}/grafana-auth-%d{MM-dd-yyyy}-%i.log.gz">
+      <PatternLayout pattern="${LOG_PATTERN}"/>
+      <Policies>
+      	<SizeBasedTriggeringPolicy size="1024 MB" />
+      </Policies>
+      <DefaultRolloverStrategy max="10" />
+    </RollingRandomAccessFile>
+    <Console name="Console" target="SYSTEM_OUT">
+      <PatternLayout pattern="${LOG_PATTERN}"/>
+    </Console>
+  </Appenders>
+  <Loggers>
+    <Root level="trace" includeLocation="false">
+      <AppenderRef ref="Console" level="warn" />
+      <AppenderRef ref="Rolling" level="trace" />
+    </Root>
+    <Logger name="timely" additivity="false">
+      <AppenderRef ref="Console" level="debug" />
+    </Logger>
+  </Loggers>
+</Configuration>
@@ -0,0 +1,71 @@
+<beans:beans xmlns="http://www.springframework.org/schema/security"
+             xmlns:beans="http://www.springframework.org/schema/beans"
+             xmlns:util="http://www.springframework.org/schema/util"
+             xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+             xsi:schemaLocation="http://www.springframework.org/schema/beans
+								 http://www.springframework.org/schema/beans/spring-beans-3.0.xsd
+								 http://www.springframework.org/schema/security
+								 http://www.springframework.org/schema/security/spring-security.xsd
+								 http://www.springframework.org/schema/util
+								 http://www.springframework.org/schema/util/spring-util.xsd">
+		          
+	<beans:bean id="x509PrincipalExtractor" class="org.springframework.security.web.authentication.preauth.x509.SubjectDnX509PrincipalExtractor">
+		<beans:property name="subjectDnRegex" value="CN=(.*?)(?:,|$)"/>
+	</beans:bean>
+
+	<beans:bean id="timelyUserDetailsService" class="timely.auth.UserDetailsService">
+		<beans:property name="users">
+			<util:list>
+				<beans:bean class="timely.auth.TimelyUser">
+					<beans:constructor-arg name="subjectDn">
+						<beans:value>CN=example.com</beans:value>
+					</beans:constructor-arg>
+					<beans:constructor-arg name="auths">
+						<util:list>
+							<beans:value>D</beans:value>
+							<beans:value>E</beans:value>
+							<beans:value>F</beans:value>
+						</util:list>
+					</beans:constructor-arg>
+					<beans:constructor-arg name="roles">
+						<util:list>
+							<beans:value>G</beans:value>
+							<beans:value>H</beans:value>
+							<beans:value>I</beans:value>
+						</util:list>
+					</beans:constructor-arg>
+				</beans:bean>
+			</util:list>
+		</beans:property>
+	</beans:bean>
+    
+    <beans:bean id="x509AuthorizationProvider" class="org.springframework.security.web.authentication.preauth.PreAuthenticatedAuthenticationProvider">
+    	<beans:property name="preAuthenticatedUserDetailsService" ref="timelyUserDetailsService" />
+    </beans:bean>
+    
+    <user-service id="usernamePasswordService">
+    	<user name="test" password="test1" authorities="A,B,C" />
+    </user-service>
+
+	<beans:bean id="noOpPasswordEncoder" class="org.springframework.security.crypto.password.NoOpPasswordEncoder"/>
+
+	<beans:bean id="UserNamePasswordAuthenticationProvider" class="org.springframework.security.authentication.dao.DaoAuthenticationProvider">
+		<beans:property name="userDetailsService" ref="usernamePasswordService"/>
+		<beans:property name="passwordEncoder" ref="noOpPasswordEncoder"/>
+	</beans:bean>
+	    
+	<authentication-manager alias="authenticationManager">
+		<!-- X509 PreAuthenticated authorization provider -->
+		<authentication-provider ref="x509AuthorizationProvider"/>
+		<!-- Basic authorization provider, validates provided username and password -->
+		<authentication-provider ref="UserNamePasswordAuthenticationProvider" />
+	</authentication-manager>
+
+	<util:list id="requiredRoles" value-type="java.lang.String">
+		<beans:value></beans:value>
+	</util:list>
+
+	<util:list id="requiredAuths" value-type="java.lang.String">
+		<beans:value></beans:value>
+	</util:list>
+</beans:beans>
@@ -0,0 +1,295 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>timely</groupId>
+    <artifactId>timely</artifactId>
+    <version>1.2.8-SNAPSHOT</version>
+  </parent>
+  <artifactId>grafana-auth</artifactId>
+  <dependencies>
+    <dependency>
+      <groupId>com.fasterxml.jackson.core</groupId>
+      <artifactId>jackson-annotations</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>com.fasterxml.jackson.core</groupId>
+      <artifactId>jackson-core</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>com.fasterxml.jackson.core</groupId>
+      <artifactId>jackson-databind</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>com.fasterxml.jackson.datatype</groupId>
+      <artifactId>jackson-datatype-jdk8</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>com.github.ben-manes.caffeine</groupId>
+      <artifactId>caffeine</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>com.google.flatbuffers</groupId>
+      <artifactId>flatbuffers-java</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>commons-beanutils</groupId>
+      <artifactId>commons-beanutils</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>io.github.lukehutch</groupId>
+      <artifactId>fast-classpath-scanner</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>io.netty</groupId>
+      <artifactId>netty-all</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>net.sourceforge.javacsv</groupId>
+      <artifactId>javacsv</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.accumulo</groupId>
+      <artifactId>accumulo-core</artifactId>
+      <exclusions>
+        <exclusion>
+          <groupId>org.slf4j</groupId>
+          <artifactId>slf4j-log4j12</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.accumulo</groupId>
+      <artifactId>accumulo-minicluster</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.commons</groupId>
+      <artifactId>commons-collections4</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.commons</groupId>
+      <artifactId>commons-lang3</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.commons</groupId>
+      <artifactId>commons-pool2</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.curator</groupId>
+      <artifactId>curator-client</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.curator</groupId>
+      <artifactId>curator-framework</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.curator</groupId>
+      <artifactId>curator-recipes</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.curator</groupId>
+      <artifactId>curator-x-discovery</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-client</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-common</artifactId>
+      <exclusions>
+        <exclusion>
+          <groupId>jdk.tools</groupId>
+          <artifactId>jdk.tools</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-minicluster</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-api</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.springframework</groupId>
+      <artifactId>spring-websocket</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.springframework.boot</groupId>
+      <artifactId>spring-boot-starter</artifactId>
+      <exclusions>
+        <exclusion>
+          <groupId>org.springframework.boot</groupId>
+          <artifactId>spring-boot-starter-logging</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+    <dependency>
+      <groupId>org.springframework.boot</groupId>
+      <artifactId>spring-boot-starter-log4j2</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.springframework.boot</groupId>
+      <artifactId>spring-boot-starter-validation</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.springframework.security</groupId>
+      <artifactId>spring-security-config</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.springframework.security</groupId>
+      <artifactId>spring-security-web</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>timely</groupId>
+      <artifactId>timely-balancer</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>timely</groupId>
+      <artifactId>timely-client</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>timely</groupId>
+      <artifactId>timely-common</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>timely</groupId>
+      <artifactId>timely-server</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>com.lmax</groupId>
+      <artifactId>disruptor</artifactId>
+      <scope>runtime</scope>
+    </dependency>
+    <dependency>
+      <groupId>io.netty</groupId>
+      <artifactId>netty-tcnative-boringssl-static</artifactId>
+      <scope>runtime</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.logging.log4j</groupId>
+      <artifactId>log4j-api</artifactId>
+      <scope>runtime</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.logging.log4j</groupId>
+      <artifactId>log4j-core</artifactId>
+      <scope>runtime</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.logging.log4j</groupId>
+      <artifactId>log4j-slf4j-impl</artifactId>
+      <scope>runtime</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.javassist</groupId>
+      <artifactId>javassist</artifactId>
+      <scope>runtime</scope>
+    </dependency>
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.jsoup</groupId>
+      <artifactId>jsoup</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.springframework.boot</groupId>
+      <artifactId>spring-boot-test-autoconfigure</artifactId>
+      <scope>test</scope>
+    </dependency>
+  </dependencies>
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-assembly-plugin</artifactId>
+        <executions>
+          <execution>
+            <id>assemble</id>
+            <goals>
+              <goal>single</goal>
+            </goals>
+            <phase>package</phase>
+            <configuration>
+              <descriptors>
+                <descriptor>src/main/dist/assembly.xml</descriptor>
+              </descriptors>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-dependency-plugin</artifactId>
+        <executions>
+          <execution>
+            <id>copy-dependencies</id>
+            <goals>
+              <goal>copy-dependencies</goal>
+            </goals>
+            <phase>prepare-package</phase>
+            <configuration>
+              <excludeArtifactIds>junit</excludeArtifactIds>
+              <outputDirectory>${project.build.directory}/lib</outputDirectory>
+            </configuration>
+          </execution>
+          <execution>
+            <id>unpack-native</id>
+            <goals>
+              <goal>unpack</goal>
+            </goals>
+            <phase>process-test-classes</phase>
+            <configuration>
+              <artifactItems>
+                <artifactItem>
+                  <groupId>io.netty</groupId>
+                  <artifactId>netty-tcnative-boringssl-static</artifactId>
+                  <version>${version.netty-tcnative}</version>
+                  <type>jar</type>
+                  <overWrite>false</overWrite>
+                  <outputDirectory>${project.build.directory}/native</outputDirectory>
+                  <includes>**/*.so</includes>
+                </artifactItem>
+              </artifactItems>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-failsafe-plugin</artifactId>
+      </plugin>
+      <plugin>
+        <groupId>org.codehaus.mojo</groupId>
+        <artifactId>build-helper-maven-plugin</artifactId>
+        <executions>
+          <execution>
+            <id>add-source</id>
+            <goals>
+              <goal>add-source</goal>
+            </goals>
+            <phase>generate-sources</phase>
+            <configuration>
+              <sources>
+                <source>src/main/generated-java</source>
+              </sources>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+    </plugins>
+    <extensions>
+      <!-- Use os-maven-plugin to initialize the "os.detected" properties -->
+      <extension>
+        <groupId>kr.motd.maven</groupId>
+        <artifactId>os-maven-plugin</artifactId>
+      </extension>
+    </extensions>
+  </build>
+</project>
@@ -0,0 +1,55 @@
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3" 
+  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3 http://maven.apache.org/xsd/assembly-1.1.3.xsd">
+  <id>dist</id>
+  <formats>
+    <format>tar.gz</format>
+  </formats>
+  <fileSets>
+    <fileSet>
+      <directory>${project.basedir}</directory>
+      <outputDirectory></outputDirectory>
+      <includes>
+        <include>README*</include>
+        <include>LICENSE*</include>
+        <include>NOTICE*</include>
+      </includes>
+    </fileSet>
+    <fileSet>
+      <directory>${project.build.directory}</directory>
+      <outputDirectory>lib</outputDirectory>
+      <includes>
+        <include>timely*.jar</include>
+        <include>grafana*.jar</include>
+      </includes>
+    </fileSet>
+    <fileSet>
+      <directory>${project.build.directory}/lib</directory>
+      <outputDirectory>lib</outputDirectory>
+      <includes>
+        <include>*.jar</include>
+      </includes>
+     </fileSet>
+    <fileSet>
+      <directory>${project.basedir}/bin</directory>
+      <outputDirectory>bin</outputDirectory>
+      <includes>
+        <include>**/*</include>
+      </includes>
+     </fileSet>
+    <fileSet>
+      <directory>${project.basedir}/conf</directory>
+      <outputDirectory>conf</outputDirectory>
+      <includes>
+        <include>*</include>
+      </includes>
+     </fileSet>
+	<fileSet>
+	  <directory>${project.basedir}/src/main/resources/webapp</directory>
+	  <outputDirectory>bin/webapp</outputDirectory>
+	  <includes>
+		<include>*</include>
+	  </includes>
+	</fileSet>
+  </fileSets>
+</assembly>
@@ -0,0 +1,322 @@
+package timely.grafana.auth;
+
+import java.io.File;
+import java.net.InetSocketAddress;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+
+import javax.net.ssl.SSLContext;
+
+import io.netty.bootstrap.ServerBootstrap;
+import io.netty.buffer.PooledByteBufAllocator;
+import io.netty.channel.Channel;
+import io.netty.channel.ChannelFuture;
+import io.netty.channel.ChannelHandler;
+import io.netty.channel.ChannelInitializer;
+import io.netty.channel.ChannelOption;
+import io.netty.channel.EventLoopGroup;
+import io.netty.channel.epoll.EpollEventLoopGroup;
+import io.netty.channel.epoll.EpollServerSocketChannel;
+import io.netty.channel.nio.NioEventLoopGroup;
+import io.netty.channel.socket.ServerSocketChannel;
+import io.netty.channel.socket.SocketChannel;
+import io.netty.channel.socket.nio.NioServerSocketChannel;
+import io.netty.handler.codec.http.HttpContentCompressor;
+import io.netty.handler.codec.http.HttpContentDecompressor;
+import io.netty.handler.codec.http.HttpObjectAggregator;
+import io.netty.handler.codec.http.HttpRequestDecoder;
+import io.netty.handler.codec.http.HttpResponseEncoder;
+import io.netty.handler.logging.LoggingHandler;
+import io.netty.handler.ssl.ClientAuth;
+import io.netty.handler.ssl.OpenSslServerContext;
+import io.netty.handler.ssl.OpenSslServerSessionContext;
+import io.netty.handler.ssl.SslContext;
+import io.netty.handler.ssl.SslContextBuilder;
+import io.netty.handler.ssl.SslProvider;
+import io.netty.handler.ssl.util.SelfSignedCertificate;
+import io.netty.handler.stream.ChunkedWriteHandler;
+import io.netty.util.internal.SystemPropertyUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.boot.WebApplicationType;
+import org.springframework.boot.builder.SpringApplicationBuilder;
+import org.springframework.context.ConfigurableApplicationContext;
+import timely.auth.AuthCache;
+import timely.balancer.configuration.ClientSsl;
+import timely.balancer.connection.http.HttpClientPool;
+import timely.client.http.HttpClient;
+import timely.configuration.ServerSsl;
+import timely.grafana.auth.configuration.GrafanaAuthConfiguration;
+import timely.grafana.auth.configuration.SpringBootstrap;
+import timely.grafana.auth.netty.http.GrafanaRelayHandler;
+import timely.grafana.auth.netty.http.GrafanaRequestDecoder;
+import timely.netty.http.HttpStaticFileServerHandler;
+import timely.netty.http.NonSslRedirectHandler;
+import timely.netty.http.TimelyExceptionHandler;
+import timely.netty.http.auth.X509LoginRequestHandler;
+
+public class GrafanaAuth {
+
+    private static final Logger LOG = LoggerFactory.getLogger(GrafanaAuth.class);
+    private static final int EPOLL_MIN_MAJOR_VERSION = 2;
+    private static final int EPOLL_MIN_MINOR_VERSION = 6;
+    private static final int EPOLL_MIN_PATCH_VERSION = 32;
+    private static final String OS_NAME = "os.name";
+    private static final String OS_VERSION = "os.version";
+    protected static final CountDownLatch LATCH = new CountDownLatch(1);
+    static ConfigurableApplicationContext applicationContext;
+
+    private final GrafanaAuthConfiguration grafanaAuthConfig;
+    private EventLoopGroup httpWorkerGroup = null;
+    private EventLoopGroup httpBossGroup = null;
+    protected Channel httpChannelHandle = null;
+    protected volatile boolean shutdown = false;
+
+    public GrafanaAuth(GrafanaAuthConfiguration grafanaAuthConfig) throws Exception {
+        this.grafanaAuthConfig = grafanaAuthConfig;
+    }
+
+    protected static ConfigurableApplicationContext initializeContext(String[] args) {
+        SpringApplicationBuilder builder = new SpringApplicationBuilder(SpringBootstrap.class);
+        builder.web(WebApplicationType.NONE);
+        builder.registerShutdownHook(false);
+        return builder.run(args);
+    }
+
+    private void shutdownHook() {
+
+        final Runnable shutdownRunner = () -> {
+            if (!shutdown) {
+                shutdown();
+            }
+        };
+        final Thread hook = new Thread(shutdownRunner, "shutdown-hook-thread");
+        Runtime.getRuntime().addShutdownHook(hook);
+    }
+
+    public void shutdown() {
+        List<ChannelFuture> channelFutures = new ArrayList<>();
+
+        if (httpChannelHandle != null) {
+            LOG.info("Closing httpChannelHandle");
+            channelFutures.add(httpChannelHandle.close());
+        }
+
+        // wait for the channels to shutdown
+        channelFutures.forEach(f -> {
+            try {
+                f.get();
+            } catch (final Exception e) {
+                LOG.error("Channel:" + f.channel().config() + " -> " + e.getMessage(), e);
+            }
+        });
+
+        int quietPeriod = grafanaAuthConfig.getShutdownQuietPeriod();
+        List<Future<?>> groupFutures = new ArrayList<>();
+
+        if (httpBossGroup != null) {
+            LOG.info("Shutting down httpBossGroup");
+            groupFutures.add(httpBossGroup.shutdownGracefully(quietPeriod, 10, TimeUnit.SECONDS));
+        }
+
+        if (httpWorkerGroup != null) {
+            LOG.info("Shutting down httpWorkerGroup");
+            groupFutures.add(httpWorkerGroup.shutdownGracefully(quietPeriod, 10, TimeUnit.SECONDS));
+        }
+
+        groupFutures.parallelStream().forEach(f -> {
+            try {
+                f.get();
+            } catch (final Exception e) {
+                LOG.error("Group:" + f.toString() + " -> " + e.getMessage(), e);
+            }
+        });
+
+        if (applicationContext != null) {
+            LOG.info("Closing applicationContext");
+            applicationContext.close();
+        }
+
+        this.shutdown = true;
+        LOG.info("Server shut down.");
+    }
+
+    private static boolean useEpoll() {
+
+        // Should we just return true if this is Linux and if we get an error
+        // during Epoll
+        // setup handle it there?
+        final String os = SystemPropertyUtil.get(OS_NAME).toLowerCase().trim();
+        final String[] version = SystemPropertyUtil.get(OS_VERSION).toLowerCase().trim().split("\\.");
+        if (os.startsWith("linux") && version.length >= 3) {
+            final int major = Integer.parseInt(version[0]);
+            if (major > EPOLL_MIN_MAJOR_VERSION) {
+                return true;
+            } else if (major == EPOLL_MIN_MAJOR_VERSION) {
+                final int minor = Integer.parseInt(version[1]);
+                if (minor > EPOLL_MIN_MINOR_VERSION) {
+                    return true;
+                } else if (minor == EPOLL_MIN_MINOR_VERSION) {
+                    final int patch = Integer.parseInt(version[2].substring(0, 2));
+                    return patch >= EPOLL_MIN_PATCH_VERSION;
+                } else {
+                    return false;
+                }
+            } else {
+                return false;
+            }
+        } else {
+            return false;
+        }
+    }
+
+    protected SslContext createSSLContext(GrafanaAuthConfiguration config) throws Exception {
+
+        ServerSsl sslCfg = config.getSecurity().getServerSsl();
+        Boolean generate = sslCfg.isUseGeneratedKeypair();
+        SslContextBuilder ssl;
+        if (generate) {
+            LOG.warn("Using generated self signed server certificate");
+            Date begin = new Date();
+            Date end = new Date(begin.getTime() + 86400000);
+            SelfSignedCertificate ssc = new SelfSignedCertificate("localhost", begin, end);
+            ssl = SslContextBuilder.forServer(ssc.certificate(), ssc.privateKey());
+        } else {
+            String cert = sslCfg.getCertificateFile();
+            String key = sslCfg.getKeyFile();
+            String keyPass = sslCfg.getKeyPassword();
+            if (null == cert || null == key) {
+                throw new IllegalArgumentException("Check your SSL properties, something is wrong.");
+            }
+            ssl = SslContextBuilder.forServer(new File(cert), new File(key), keyPass);
+        }
+
+        ssl.ciphers(sslCfg.getUseCiphers());
+
+        // Can't set to REQUIRE because the CORS pre-flight requests will fail.
+        ssl.clientAuth(ClientAuth.OPTIONAL);
+
+        Boolean useOpenSSL = sslCfg.isUseOpenssl();
+        if (useOpenSSL) {
+            ssl.sslProvider(SslProvider.OPENSSL);
+        } else {
+            ssl.sslProvider(SslProvider.JDK);
+        }
+        String trustStore = sslCfg.getTrustStoreFile();
+        if (null != trustStore) {
+            if (!trustStore.isEmpty()) {
+                ssl.trustManager(new File(trustStore));
+            }
+        }
+        return ssl.build();
+    }
+
+    protected SSLContext createSSLClientContext(GrafanaAuthConfiguration config) throws Exception {
+
+        ClientSsl ssl = config.getSecurity().getClientSsl();
+        return HttpClient.getSSLContext(ssl.getTrustStoreFile(), ssl.getTrustStoreType(), ssl.getTrustStorePassword(),
+                ssl.getKeyFile(), ssl.getKeyType(), ssl.getKeyPassword());
+    }
+
+    public void run() throws Exception {
+
+        AuthCache.configure(grafanaAuthConfig.getSecurity());
+        final boolean useEpoll = useEpoll();
+        Class<? extends ServerSocketChannel> channelClass;
+        if (useEpoll) {
+            httpWorkerGroup = new EpollEventLoopGroup();
+            httpBossGroup = new EpollEventLoopGroup();
+            channelClass = EpollServerSocketChannel.class;
+        } else {
+            httpWorkerGroup = new NioEventLoopGroup();
+            httpBossGroup = new NioEventLoopGroup();
+            channelClass = NioServerSocketChannel.class;
+        }
+        LOG.info("Using channel class {}", channelClass.getSimpleName());
+
+        final int httpPort = grafanaAuthConfig.getHttp().getPort();
+        final String httpIp = grafanaAuthConfig.getHttp().getIp();
+        SslContext sslCtx = createSSLContext(grafanaAuthConfig);
+        if (sslCtx instanceof OpenSslServerContext) {
+            OpenSslServerContext openssl = (OpenSslServerContext) sslCtx;
+            String application = "GrafanaAuth_" + httpPort;
+            OpenSslServerSessionContext opensslCtx = openssl.sessionContext();
+            opensslCtx.setSessionCacheEnabled(true);
+            opensslCtx.setSessionCacheSize(128);
+            opensslCtx.setSessionIdContext(application.getBytes(StandardCharsets.UTF_8));
+            opensslCtx.setSessionTimeout(grafanaAuthConfig.getSecurity().getSessionMaxAge());
+        }
+        SSLContext clientSSLContext = createSSLClientContext(grafanaAuthConfig);
+        final ServerBootstrap httpServer = new ServerBootstrap();
+        httpServer.group(httpBossGroup, httpWorkerGroup);
+        httpServer.channel(channelClass);
+        httpServer.handler(new LoggingHandler());
+        HttpClientPool httpClientPool = new HttpClientPool(grafanaAuthConfig.getSecurity(), grafanaAuthConfig.getHttp(),
+                clientSSLContext);
+        httpServer.childHandler(setupHttpChannel(grafanaAuthConfig, sslCtx, httpClientPool));
+        httpServer.option(ChannelOption.ALLOCATOR, PooledByteBufAllocator.DEFAULT);
+        httpServer.option(ChannelOption.SO_BACKLOG, 128);
+        httpServer.option(ChannelOption.SO_KEEPALIVE, true);
+        httpChannelHandle = httpServer.bind(httpIp, httpPort).sync().channel();
+        final String httpAddress = ((InetSocketAddress) httpChannelHandle.localAddress()).getAddress().getHostAddress();
+
+        shutdownHook();
+        LOG.info("Server started. Listening on {}:{} for HTTP traffic", httpAddress, httpPort);
+    }
+
+    public static void main(String[] args) throws Exception {
+
+        GrafanaAuth.applicationContext = GrafanaAuth.initializeContext(args);
+        GrafanaAuthConfiguration grafanaAuthConf = GrafanaAuth.applicationContext
+                .getBean(GrafanaAuthConfiguration.class);
+
+        GrafanaAuth grafanaAuth = new GrafanaAuth(grafanaAuthConf);
+        try {
+            grafanaAuth.run();
+            LATCH.await();
+        } catch (final InterruptedException e) {
+            LOG.info("Server shutting down.");
+        } catch (Exception e) {
+            LOG.error("Error running server.", e);
+        } finally {
+            try {
+                grafanaAuth.shutdown();
+            } catch (Exception e) {
+                System.exit(1);
+            }
+        }
+    }
+
+    protected ChannelHandler setupHttpChannel(GrafanaAuthConfiguration config, SslContext sslCtx,
+            HttpClientPool httpClientPool) {
+
+        return new ChannelInitializer<SocketChannel>() {
+
+            @Override
+            protected void initChannel(SocketChannel ch) throws Exception {
+
+                ch.pipeline().addLast("ssl", new NonSslRedirectHandler(config.getHttp(), sslCtx));
+                ch.pipeline().addLast("encoder", new HttpResponseEncoder());
+                ch.pipeline().addLast("decoder", new HttpRequestDecoder());
+                ch.pipeline().addLast("compressor", new HttpContentCompressor());
+                ch.pipeline().addLast("decompressor", new HttpContentDecompressor());
+                // high maximum contentLength so that grafana snapshots can be delivered
+                // might not be necessary if inbound chunking (while proxying) is handled
+                ch.pipeline().addLast("aggregator", new HttpObjectAggregator(2097152));
+                ch.pipeline().addLast("chunker", new ChunkedWriteHandler());
+                ch.pipeline().addLast("grafanaDecoder",
+                        new GrafanaRequestDecoder(config.getSecurity(), config.getHttp()));
+                ch.pipeline().addLast("fileServer", new HttpStaticFileServerHandler());
+                ch.pipeline().addLast("login", new X509LoginRequestHandler(config.getSecurity(), config.getHttp()));
+                ch.pipeline().addLast("httpRelay", new GrafanaRelayHandler(config, httpClientPool));
+                ch.pipeline().addLast("error", new TimelyExceptionHandler()
+                        .setIgnoreSslHandshakeErrors(config.getSecurity().getServerSsl().isUseGeneratedKeypair()));
+            }
+        };
+    }
+}
@@ -0,0 +1,32 @@
+package timely.grafana.auth.configuration;
+
+public class Grafana {
+
+    private String host = "localhost";
+    private Integer port = 3000;
+    private String protocol = "http";
+
+    public void setHost(String host) {
+        this.host = host;
+    }
+
+    public String getHost() {
+        return host;
+    }
+
+    public void setPort(Integer port) {
+        this.port = port;
+    }
+
+    public Integer getPort() {
+        return port;
+    }
+
+    public void setProtocol(String protocol) {
+        this.protocol = protocol;
+    }
+
+    public String getProtocol() {
+        return protocol;
+    }
+}
@@ -0,0 +1,54 @@
+package timely.grafana.auth.configuration;
+
+import javax.validation.Valid;
+
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.boot.context.properties.NestedConfigurationProperty;
+import org.springframework.stereotype.Component;
+import org.springframework.validation.annotation.Validated;
+import timely.balancer.configuration.BalancerHttp;
+import timely.balancer.configuration.BalancerSecurity;
+
+@Validated
+@Component
+@ConfigurationProperties(prefix = "grafana-auth")
+public class GrafanaAuthConfiguration {
+
+    @Valid
+    @NestedConfigurationProperty
+    private BalancerHttp http = new BalancerHttp();
+
+    @Valid
+    @NestedConfigurationProperty
+    private BalancerSecurity security = new BalancerSecurity();
+
+    @Valid
+    @NestedConfigurationProperty
+    private Grafana grafana = new Grafana();
+
+    private Integer shutdownQuietPeriod = 5;
+
+    public BalancerSecurity getSecurity() {
+        return security;
+    }
+
+    public BalancerHttp getHttp() {
+        return http;
+    }
+
+    public Grafana getGrafana() {
+        return grafana;
+    }
+
+    /**
+     * Time to wait (in seconds) for connections to finish and to make sure no new
+     * connections happen before shutting down Netty event loop groups.
+     */
+    public int getShutdownQuietPeriod() {
+        return this.shutdownQuietPeriod;
+    }
+
+    public void setShutdownQuietPeriod(Integer quietPeriod) {
+        this.shutdownQuietPeriod = quietPeriod;
+    }
+}
@@ -0,0 +1,7 @@
+package timely.grafana.auth.configuration;
+
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+
+@SpringBootApplication(scanBasePackageClasses = { SpringBootstrap.class })
+public class SpringBootstrap {
+}
@@ -0,0 +1,234 @@
+package timely.grafana.auth.netty.http;
+
+import static io.netty.handler.codec.http.HttpHeaderNames.CONTENT_LENGTH;
+
+import java.io.ByteArrayOutputStream;
+import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import com.google.common.collect.Multimap;
+import io.netty.buffer.Unpooled;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.channel.SimpleChannelInboundHandler;
+import io.netty.handler.codec.http.DefaultFullHttpResponse;
+import io.netty.handler.codec.http.FullHttpRequest;
+import io.netty.handler.codec.http.FullHttpResponse;
+import io.netty.handler.codec.http.HttpMethod;
+import io.netty.handler.codec.http.HttpResponseStatus;
+import io.netty.handler.codec.http.HttpVersion;
+import io.netty.util.ReferenceCountUtil;
+import org.apache.http.Header;
+import org.apache.http.HttpEntity;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpDelete;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpHead;
+import org.apache.http.client.methods.HttpOptions;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.methods.HttpPut;
+import org.apache.http.entity.StringEntity;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.message.BasicHeader;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.util.StreamUtils;
+import timely.api.request.timeseries.HttpRequest;
+import timely.api.response.TimelyException;
+import timely.auth.TimelyPrincipal;
+import timely.auth.TimelyUser;
+import timely.auth.util.DnUtils;
+import timely.auth.util.HttpHeaderUtils;
+import timely.balancer.connection.TimelyBalancedHost;
+import timely.balancer.connection.http.HttpClientPool;
+import timely.grafana.auth.configuration.GrafanaAuthConfiguration;
+import timely.grafana.request.GrafanaRequest;
+import timely.netty.http.TimelyHttpHandler;
+import timely.netty.http.auth.TimelyAuthenticationToken;
+
+public class GrafanaRelayHandler extends SimpleChannelInboundHandler<HttpRequest> implements TimelyHttpHandler {
+
+    private static final Logger LOG = LoggerFactory.getLogger(GrafanaRelayHandler.class);
+    private final HttpClientPool httpClientPool;
+    private GrafanaAuthConfiguration config;
+
+    public GrafanaRelayHandler(GrafanaAuthConfiguration config, HttpClientPool httpClientPool) {
+        this.config = config;
+        this.httpClientPool = httpClientPool;
+    }
+
+    @Override
+    public void channelRead0(ChannelHandlerContext ctx, HttpRequest msg) throws Exception {
+
+        CloseableHttpClient client = null;
+        CloseableHttpResponse relayedResponse = null;
+        FullHttpRequest request = null;
+        GrafanaRequest grafanaRequest;
+        TimelyBalancedHost k = null;
+        String relayURI = null;
+        try {
+            try {
+                request = msg.getHttpRequest();
+                String originalURI = request.uri();
+                originalURI = encodeURI(originalURI);
+
+                if (msg instanceof GrafanaRequest) {
+                    grafanaRequest = (GrafanaRequest) msg;
+                } else {
+                    throw new UnsupportedOperationException("Request not supported");
+                }
+                TimelyAuthenticationToken token = grafanaRequest.getToken();
+                if (token == null) {
+                    throw new IllegalStateException("No token on grafana request");
+                }
+
+                String protocol = config.getGrafana().getProtocol();
+                String host = config.getGrafana().getHost();
+                Integer port = config.getGrafana().getPort();
+                k = TimelyBalancedHost.of(host, -1, port, -1, -1, null);
+                client = httpClientPool.borrowObject(k);
+                Multimap<String, String> headers = HttpHeaderUtils.toMultimap(request.headers());
+
+                // If incoming user sets these headers, do not relay them
+                headers.removeAll("x-webauth-user");
+                headers.removeAll("x-webauth-name");
+                TimelyPrincipal principal = token.getTimelyPrincipal();
+                TimelyUser user = principal.getPrimaryUser();
+                String subjectDn = user.getDn().subjectDN();
+                String userName = DnUtils.getCommonName(subjectDn);
+                String loginName = DnUtils.getShortName(subjectDn);
+                headers.put("x-webauth-user", loginName);
+                headers.put("x-webauth-name", userName);
+                List<Header> relayedHeaderList = new ArrayList<>();
+                for (Map.Entry<String, String> h : headers.entries()) {
+                    if (!h.getKey().equals(CONTENT_LENGTH.toString())) {
+                        relayedHeaderList.add(new BasicHeader(h.getKey(), h.getValue()));
+                    }
+                }
+                Header[] headerArray = new Header[relayedHeaderList.size()];
+                relayedHeaderList.toArray(headerArray);
+                relayURI = protocol + "://" + host + ":" + port + originalURI;
+                if (request.method().equals(HttpMethod.HEAD)) {
+                    HttpHead relayedRequest = new HttpHead(relayURI);
+                    if (headerArray.length > 0) {
+                        relayedRequest.setHeaders(headerArray);
+                    }
+                    relayedResponse = client.execute(relayedRequest);
+                } else if (request.method().equals(HttpMethod.GET)) {
+                    HttpGet relayedRequest = new HttpGet(relayURI);
+                    if (headerArray.length > 0) {
+                        relayedRequest.setHeaders(headerArray);
+                    }
+                    relayedResponse = client.execute(relayedRequest);
+                } else if (request.method().equals(HttpMethod.POST)) {
+                    HttpPost relayedRequest = new HttpPost(relayURI);
+                    String content = request.content().toString(StandardCharsets.UTF_8);
+                    StringEntity entity = new StringEntity(content);
+                    relayedRequest.setEntity(entity);
+                    if (headerArray.length > 0) {
+                        relayedRequest.setHeaders(headerArray);
+                    }
+                    relayedResponse = client.execute(relayedRequest);
+                } else if (request.method().equals(HttpMethod.PUT)) {
+                    HttpPut relayedRequest = new HttpPut(relayURI);
+                    String content = request.content().toString(StandardCharsets.UTF_8);
+                    StringEntity entity = new StringEntity(content);
+                    relayedRequest.setEntity(entity);
+                    if (headerArray.length > 0) {
+                        relayedRequest.setHeaders(headerArray);
+                    }
+                    relayedResponse = client.execute(relayedRequest);
+                } else if (request.method().equals(HttpMethod.DELETE)) {
+                    HttpDelete relayedRequest = new HttpDelete(relayURI);
+                    if (headerArray.length > 0) {
+                        relayedRequest.setHeaders(headerArray);
+                    }
+                    relayedResponse = client.execute(relayedRequest);
+                } else if (request.method().equals(HttpMethod.OPTIONS)) {
+                    HttpOptions relayedRequest = new HttpOptions(relayURI);
+                    if (headerArray.length > 0) {
+                        relayedRequest.setHeaders(headerArray);
+                    }
+                    relayedResponse = client.execute(relayedRequest);
+                }
+            } catch (Exception e) {
+                String message = e.getMessage();
+                if (message == null) {
+                    LOG.error("", e);
+                } else {
+                    LOG.error(message, e);
+                }
+                this.sendHttpError(ctx, new TimelyException(HttpResponseStatus.INTERNAL_SERVER_ERROR.code(),
+                        e.getMessage(), e.getLocalizedMessage(), e));
+                return;
+            } finally {
+                if (request != null && request instanceof AutoCloseable) {
+                    ReferenceCountUtil.release(request);
+                }
+            }
+
+            FullHttpResponse response = null;
+            if (relayedResponse == null) {
+                this.sendHttpError(ctx, new TimelyException(HttpResponseStatus.METHOD_NOT_ALLOWED.code(),
+                        "Method not allowed", "Method not allowed"));
+            } else {
+                LOG.debug("RequestURL:{} ResponseCode:{}", relayURI, relayedResponse.getStatusLine().getStatusCode());
+                HttpEntity httpEntity = relayedResponse.getEntity();
+                if (httpEntity == null) {
+                    response = new DefaultFullHttpResponse(HttpVersion.HTTP_1_1,
+                            HttpResponseStatus.valueOf(relayedResponse.getStatusLine().getStatusCode()));
+                } else {
+                    ByteArrayOutputStream baos = new MyByteArrayOutputStream();
+                    StreamUtils.copy(httpEntity.getContent(), baos);
+                    response = new DefaultFullHttpResponse(HttpVersion.HTTP_1_1,
+                            HttpResponseStatus.valueOf(relayedResponse.getStatusLine().getStatusCode()),
+                            Unpooled.copiedBuffer(baos.toByteArray()));
+                    response.headers().set(CONTENT_LENGTH, response.content().readableBytes());
+                }
+                for (Header h : relayedResponse.getAllHeaders()) {
+                    response.headers().add(h.getName(), h.getValue());
+                }
+                sendResponse(ctx, response);
+            }
+        } finally {
+            if (relayedResponse != null) {
+                relayedResponse.close();
+            }
+            if (client != null && k != null) {
+                httpClientPool.returnObject(k, client);
+            } else {
+                LOG.error("NOT RETURNING CONNECTION! " + msg.getHttpRequest().uri());
+            }
+        }
+    }
+
+    static public class MyByteArrayOutputStream extends ByteArrayOutputStream {
+
+        public MyByteArrayOutputStream() {
+        }
+
+        public MyByteArrayOutputStream(int size) {
+            super(size);
+        }
+
+        public int getCount() {
+            return count;
+        }
+
+        public byte[] getBuf() {
+            return buf;
+        }
+    }
+
+    static public String encodeURI(String s) {
+        try {
+            s = s.replaceAll("\\{", URLEncoder.encode("{", "UTF-8"));
+            s = s.replaceAll("\\}", URLEncoder.encode("}", "UTF-8"));
+        } catch (Exception e) {
+
+        }
+        return s;
+    }
+}
@@ -0,0 +1,133 @@
+package timely.grafana.auth.netty.http;
+
+import java.security.cert.X509Certificate;
+import java.util.Collection;
+import java.util.List;
+
+import com.google.common.collect.Multimap;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.handler.codec.MessageToMessageDecoder;
+import io.netty.handler.codec.http.FullHttpRequest;
+import io.netty.handler.codec.http.HttpHeaderNames;
+import io.netty.handler.codec.http.QueryStringDecoder;
+import io.netty.handler.codec.http.cookie.ServerCookieDecoder;
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import timely.api.request.AuthenticatedRequest;
+import timely.api.request.timeseries.HttpRequest;
+import timely.api.response.StrictTransportResponse;
+import timely.api.response.TimelyException;
+import timely.auth.AuthCache;
+import timely.auth.AuthenticationService;
+import timely.auth.SubjectIssuerDNPair;
+import timely.auth.TimelyPrincipal;
+import timely.auth.util.HttpHeaderUtils;
+import timely.configuration.Http;
+import timely.configuration.Security;
+import timely.grafana.request.GrafanaRequest;
+import timely.netty.Constants;
+import timely.netty.http.TimelyHttpHandler;
+import timely.netty.http.auth.TimelyAuthenticationToken;
+
+public class GrafanaRequestDecoder extends MessageToMessageDecoder<FullHttpRequest> implements TimelyHttpHandler {
+
+    private static final Logger LOG = LoggerFactory.getLogger(GrafanaRequestDecoder.class);
+    private static final String LOG_RECEIVED_REQUEST = "Received HTTP request {}";
+    private static final String LOG_PARSED_REQUEST = "Parsed request {}";
+
+    private final Security security;
+    private final String nonSecureRedirectAddress;
+
+    public GrafanaRequestDecoder(Security security, Http http) {
+
+        this.security = security;
+        this.nonSecureRedirectAddress = http.getRedirectPath();
+    }
+
+    public static String getSessionId(FullHttpRequest msg) {
+        Multimap<String, String> headers = HttpHeaderUtils.toMultimap(msg.headers());
+        Collection<String> cookies = headers.get(HttpHeaderNames.COOKIE.toString());
+        final StringBuilder buf = new StringBuilder();
+        cookies.forEach(h -> {
+            ServerCookieDecoder.STRICT.decode(h).forEach(c -> {
+                if (c.name().equals(Constants.COOKIE_NAME)) {
+                    if (buf.length() == 0) {
+                        buf.append(c.value());
+                    }
+                }
+            });
+        });
+        if (buf.length() == 0) {
+            return null;
+        } else {
+            return buf.toString();
+        }
+    }
+
+    @Override
+    protected void decode(ChannelHandlerContext ctx, FullHttpRequest msg, List<Object> out) throws Exception {
+
+        LOG.trace(LOG_RECEIVED_REQUEST, msg);
+
+        final String uri = msg.uri();
+        final QueryStringDecoder decoder = new QueryStringDecoder(uri);
+        if (decoder.path().equals(nonSecureRedirectAddress)) {
+            out.add(new StrictTransportResponse());
+            return;
+        }
+
+        final String sessionId = getSessionId(msg);
+        LOG.trace("SessionID: " + sessionId);
+
+        HttpRequest request;
+        try {
+            request = new GrafanaRequest();
+            Multimap<String, String> headers = HttpHeaderUtils.toMultimap(msg.headers());
+            ((AuthenticatedRequest) request).addHeaders(headers);
+            X509Certificate clientCert = AuthenticationService.getClientCertificate(ctx);
+            TimelyAuthenticationToken token;
+            if (StringUtils.isNotBlank(sessionId)) {
+                TimelyPrincipal principal;
+                ((AuthenticatedRequest) request).setSessionId(sessionId);
+                if (AuthCache.containsKey(sessionId)) {
+                    principal = AuthCache.get(sessionId);
+                } else {
+                    principal = TimelyPrincipal.anonymousPrincipal();
+                }
+                SubjectIssuerDNPair dn = principal.getPrimaryUser().getDn();
+                token = AuthenticationService.getAuthenticationToken(null, dn.subjectDN(), dn.issuerDN());
+            } else if (clientCert != null) {
+                token = AuthenticationService.getAuthenticationToken(clientCert, headers);
+            } else {
+                SubjectIssuerDNPair dn = TimelyPrincipal.anonymousPrincipal().getPrimaryUser().getDn();
+                token = AuthenticationService.getAuthenticationToken(null, dn.subjectDN(), dn.issuerDN());
+            }
+            ((AuthenticatedRequest) request).setToken(token);
+            request.setHttpRequest(msg.copy());
+            LOG.trace(LOG_PARSED_REQUEST, request);
+            request.validate();
+            out.add(request);
+        } catch (UnsupportedOperationException | NullPointerException e) {
+            // Return the original http request to route to the static file
+            // server
+            msg.retain();
+            out.add(msg);
+            return;
+        }
+        try {
+            try {
+                AuthenticationService.enforceAccess((AuthenticatedRequest) request);
+            } catch (TimelyException e) {
+                if (!security.isAllowAnonymousHttpAccess()) {
+                    throw e;
+                }
+            }
+        } catch (Exception e) {
+            out.clear();
+            throw e;
+        }
+
+    }
+
+}
@@ -0,0 +1,32 @@
+package timely.grafana.request;
+
+import io.netty.handler.codec.http.FullHttpRequest;
+import io.netty.handler.codec.http.QueryStringDecoder;
+import timely.api.annotation.Http;
+import timely.api.request.AuthenticatedRequest;
+import timely.api.request.HttpGetRequest;
+import timely.api.request.HttpPostRequest;
+
+@Http(path = "/")
+public class GrafanaRequest extends AuthenticatedRequest implements HttpGetRequest, HttpPostRequest {
+
+    private FullHttpRequest httpRequest = null;
+
+    @Override
+    public HttpGetRequest parseQueryParameters(QueryStringDecoder decoder) throws Exception {
+        return new GrafanaRequest();
+    }
+
+    @Override
+    public HttpPostRequest parseBody(String content) throws Exception {
+        return new GrafanaRequest();
+    }
+
+    public void setHttpRequest(FullHttpRequest httpRequest) {
+        this.httpRequest = httpRequest;
+    }
+
+    public FullHttpRequest getHttpRequest() {
+        return httpRequest;
+    }
+}
@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<FindBugsFilter>
+  <Match>
+    <Class name="timely.grafana.auth.netty.http.GrafanaRelayHandler$MyByteArrayOutputStream" />
+    <Method name="getBuf" />
+    <Bug pattern="EI_EXPOSE_REP" />
+  </Match>
+</FindBugsFilter>
@@ -4,7 +4,7 @@
  <parent>
    <groupId>timely</groupId>
    <artifactId>timely</artifactId>
-    <version>1.0.1-SNAPSHOT</version>
+    <version>1.2.8-SNAPSHOT</version>
  </parent>
  <artifactId>timely-app</artifactId>
  <packaging>pom</packaging>
@@ -509,7 +509,7 @@ System.register(['lodash', 'angular', '../../../app/core/utils/datemath'], funct
                } else {
                  return target.metric === metricData.metric && _.every(target.tags, function (tagV, tagK) {
                    interpolatedTagValue = this.templateSrv.replace(tagV, options.scopedVars, 'pipe');
-                    return metricData.tags[tagK] === interpolatedTagValue || interpolatedTagValue === "*";
+                    return metricData.tags[tagK] === interpolatedTagValue || interpolatedTagValue === ".*";
                  }.bind(this));
                }
              }.bind(this));
@@ -1 +1 @@
-{"version":3,"sources":["../../src/datasource/datasource.js"],"names":["_","angular","dateMath","TimelyDatasource","instanceSettings","$q","backendSrv","templateSrv","contextSrv","type","name","jsonData","httpsPort","timelyHost","wsPort","basicAuths","url","_makeHttpsUrl","withCredentials","basicAuth","supportMetrics","tagKeys","aggregatorsPromise","filterTypesPromise","options","start","convertToTSDBTime","rangeRaw","from","end","to","qs","each","targets","target","metric","push","convertTargetToQuery","bind","queries","compact","isEmpty","d","defer","resolve","data","promise","groupByTags","query","filters","length","val","tagk","tags","key","performTimeSeriesQuery","then","response","metricToTargetMapping","mapMetricsToTargets","result","map","metricData","index","_saveTagKeys","transformMetricData","datasourceRequest","method","status","message","title","eventList","aggregator","annotation","results","annotationObject","annotations","isGlobal","globalAnnotations","event","description","time","Math","floor","startTime","text","notes","when","interpolated","replace","err","reject","responseTransform","value","metrics_regex","tag_names_regex","tag_values_regex","tag_names_suggest_regex","tag_values_suggest_regex","metrics_query","match","_performSuggestQuery","tag_names_query","_performMetricKeyLookup","tag_values_query","_performMetricKeyValueLookup","tag_names_suggest_query","tag_values_suggest_query","msResolution","responsePromises","reqBody","showQuery","_addCredentialOptions","headers","Promise","all","queryResponses","combinedResponse","x","concat","Object","keys","aggregateTags","tag","_get","q","max","keysArray","split","trim","keysQuery","splice","join","m","limit","tagvs","r","indexOf","tagks","tagv","relativeUrl","params","isArray","sort","md","metricLabel","createMetricLabel","dps","v","k","datapoints","alias","scopedVars","clone","label","tagData","toPairs","has","hide","shouldComputeRate","rate","rateOptions","counter","isCounter","counterMax","parseInt","counterResetValue","resetValue","disableDownsampling","interval","downsampleInterval","parseFloat","downsample","downsampleAggregator","downsampleFillPolicy","copy","filter_key","filter","tag_key","metrics","interpolatedTagValue","findIndex","every","tagV","tagK","date","roundUp","parse","valueOf"],"mappings":";;;;;;;;;;;;;;;AAAOA,O;;AACAC,a;;AACKC,c;;;;;;;;;;;;;;;;;;;;;kCAECC,gB;;AAEX;AACA,kCAAYC,gBAAZ,EAA8BC,EAA9B,EAAkCC,UAAlC,EAA8CC,WAA9C,EAA2DC,UAA3D,EAAuE;AAAA;;AACrE,eAAKC,IAAL,GAAYL,iBAAiBK,IAA7B;AACA,eAAKC,IAAL,GAAYN,iBAAiBM,IAA7B;;AAEA,eAAKC,QAAL,GAAgBP,iBAAiBO,QAAjB,IAA6B,EAA7C;AACA,eAAKC,SAAL,GAAiBR,iBAAiBO,QAAjB,CAA0BC,SAA3C;AACA,eAAKC,UAAL,GAAkBT,iBAAiBO,QAAjB,CAA0BE,UAA5C;AACA,eAAKC,MAAL,GAAcV,iBAAiBO,QAAjB,CAA0BG,MAAxC;AACA,cAAI,CAACV,iBAAiBO,QAAjB,CAA0BI,UAA/B,EAA0C;AACxCX,6BAAiBO,QAAjB,CAA0BI,UAA1B,GAAuC,KAAvC;AACD;;AAED,eAAKC,GAAL,GAAW,KAAKC,aAAL,EAAX;AACA,eAAKC,eAAL,GAAuB,IAAvB;AACA,eAAKC,SAAL,GAAiB,KAAjB;;AAEA,eAAKC,cAAL,GAAsB,IAAtB;AACA,eAAKC,OAAL,GAAe,EAAf;;AAEA,eAAKhB,EAAL,GAAUA,EAAV;AACA,eAAKC,UAAL,GAAkBA,UAAlB;AACA,eAAKC,WAAL,GAAmBA,WAAnB;AACA,eAAKC,UAAL,GAAkBA,UAAlB;;AAEA,eAAKc,kBAAL,GAA0B,IAA1B;AACA,eAAKC,kBAAL,GAA0B,IAA1B;AAED;;AAED;;;;;gCACMC,O,EAAQ;AACZ,gBAAIC,QAAQ,KAAKC,iBAAL,CAAuBF,QAAQG,QAAR,CAAiBC,IAAxC,EAA8C,KAA9C,CAAZ;AACA,gBAAIC,MAAM,KAAKH,iBAAL,CAAuBF,QAAQG,QAAR,CAAiBG,EAAxC,EAA4C,IAA5C,CAAV;AACA,gBAAIC,KAAK,EAAT;;AAEA/B,cAAEgC,IAAF,CAAOR,QAAQS,OAAf,EAAwB,UAASC,MAAT,EAAiB;AACvC,kBAAI,CAACA,OAAOC,MAAZ,EAAoB;AAAE;AAAS;AAC/BJ,iBAAGK,IAAH,CAAQ,KAAKC,oBAAL,CAA0BH,MAA1B,EAAkCV,OAAlC,CAAR;AACD,aAHuB,CAGtBc,IAHsB,CAGjB,IAHiB,CAAxB;;AAKA,gBAAIC,UAAUvC,EAAEwC,OAAF,CAAUT,EAAV,CAAd;;AAEA;AACA,gBAAI/B,EAAEyC,OAAF,CAAUF,OAAV,CAAJ,EAAwB;AACtB,kBAAIG,IAAI,KAAKrC,EAAL,CAAQsC,KAAR,EAAR;AACAD,gBAAEE,OAAF,CAAU,EAAEC,MAAM,EAAR,EAAV;AACA,qBAAOH,EAAEI,OAAT;AACD;;AAED,gBAAIC,cAAc,EAAlB;AACA/C,cAAEgC,IAAF,CAAOO,OAAP,EAAgB,UAASS,KAAT,EAAgB;AAC9B,kBAAIA,MAAMC,OAAN,IAAiBD,MAAMC,OAAN,CAAcC,MAAd,GAAuB,CAA5C,EAA+C;AAC7ClD,kBAAEgC,IAAF,CAAOgB,MAAMC,OAAb,EAAsB,UAASE,GAAT,EAAc;AAClCJ,8BAAYI,IAAIC,IAAhB,IAAwB,IAAxB;AACD,iBAFD;AAGD,eAJD,MAIO;AACLpD,kBAAEgC,IAAF,CAAOgB,MAAMK,IAAb,EAAmB,UAASF,GAAT,EAAcG,GAAd,EAAmB;AACpCP,8BAAYO,GAAZ,IAAmB,IAAnB;AACD,iBAFD;AAGD;AACF,aAVD;;AAYA,mBAAO,KAAKC,sBAAL,CAA4BhB,OAA5B,EAAqCd,KAArC,EAA4CI,GAA5C,EAAiD2B,IAAjD,CAAsD,UAASC,QAAT,EAAmB;AAC9E,kBAAIC,wBAAwB,KAAKC,mBAAL,CAAyBF,SAASZ,IAAlC,EAAwCrB,OAAxC,CAA5B;AACA,kBAAIoC,SAAS5D,EAAE6D,GAAF,CAAMJ,SAASZ,IAAf,EAAqB,UAASiB,UAAT,EAAqBC,KAArB,EAA4B;AAC5DA,wBAAQL,sBAAsBK,KAAtB,CAAR;AACA,oBAAIA,UAAU,CAAC,CAAf,EAAkB;AAChBA,0BAAQ,CAAR;AACD;AACD,qBAAKC,YAAL,CAAkBF,UAAlB;;AAEA,uBAAO,KAAKG,mBAAL,CAAyBH,UAAzB,EAAqCf,WAArC,EAAkDvB,QAAQS,OAAR,CAAgB8B,KAAhB,CAAlD,EAA0EvC,OAA1E,CAAP;AACD,eARiC,CAQhCc,IARgC,CAQ3B,IAR2B,CAArB,CAAb;AASA,qBAAO,EAAEO,MAAMe,MAAR,EAAP;AACD,aAZ4D,CAY3DtB,IAZ2D,CAYtD,IAZsD,CAAtD,CAAP;AAaD;;;2CAGe;AACd,mBAAO,KAAKhC,UAAL,CAAgB4D,iBAAhB,CAAkC;AACvClD,mBAAK,KAAKA,GAAL,GAAW,UADuB;AAEvCmD,sBAAQ;AAF+B,aAAlC,EAGJX,IAHI,CAGC,oBAAY;AAClB,kBAAIC,SAASW,MAAT,KAAoB,GLine truncated
+{"version":3,"sources":["../../src/datasource/datasource.js"],"names":["_","angular","dateMath","TimelyDatasource","instanceSettings","$q","backendSrv","templateSrv","contextSrv","type","name","jsonData","httpsPort","timelyHost","wsPort","basicAuths","url","_makeHttpsUrl","withCredentials","basicAuth","supportMetrics","tagKeys","aggregatorsPromise","filterTypesPromise","options","start","convertToTSDBTime","rangeRaw","from","end","to","qs","each","targets","target","metric","push","convertTargetToQuery","bind","queries","compact","isEmpty","d","defer","resolve","data","promise","groupByTags","query","filters","length","val","tagk","tags","key","performTimeSeriesQuery","then","response","metricToTargetMapping","mapMetricsToTargets","result","map","metricData","index","_saveTagKeys","transformMetricData","datasourceRequest","method","status","message","title","eventList","aggregator","annotation","results","annotationObject","annotations","isGlobal","globalAnnotations","event","description","time","Math","floor","startTime","text","notes","when","interpolated","replace","err","reject","responseTransform","value","metrics_regex","tag_names_regex","tag_values_regex","tag_names_suggest_regex","tag_values_suggest_regex","metrics_query","match","_performSuggestQuery","tag_names_query","_performMetricKeyLookup","tag_values_query","_performMetricKeyValueLookup","tag_names_suggest_query","tag_values_suggest_query","msResolution","responsePromises","reqBody","showQuery","_addCredentialOptions","headers","Promise","all","queryResponses","combinedResponse","x","concat","Object","keys","aggregateTags","tag","_get","q","max","keysArray","split","trim","keysQuery","splice","join","m","limit","tagvs","r","indexOf","tagks","tagv","relativeUrl","params","isArray","sort","md","metricLabel","createMetricLabel","dps","v","k","datapoints","alias","scopedVars","clone","label","tagData","toPairs","has","hide","shouldComputeRate","rate","rateOptions","counter","isCounter","counterMax","parseInt","counterResetValue","resetValue","disableDownsampling","interval","downsampleInterval","parseFloat","downsample","downsampleAggregator","downsampleFillPolicy","copy","filter_key","filter","tag_key","metrics","interpolatedTagValue","findIndex","every","tagV","tagK","date","roundUp","parse","valueOf"],"mappings":";;;;;;;;;;;;;;;AAAOA,O;;AACAC,a;;AACKC,c;;;;;;;;;;;;;;;;;;;;;kCAECC,gB;;AAEX;AACA,kCAAYC,gBAAZ,EAA8BC,EAA9B,EAAkCC,UAAlC,EAA8CC,WAA9C,EAA2DC,UAA3D,EAAuE;AAAA;;AACrE,eAAKC,IAAL,GAAYL,iBAAiBK,IAA7B;AACA,eAAKC,IAAL,GAAYN,iBAAiBM,IAA7B;;AAEA,eAAKC,QAAL,GAAgBP,iBAAiBO,QAAjB,IAA6B,EAA7C;AACA,eAAKC,SAAL,GAAiBR,iBAAiBO,QAAjB,CAA0BC,SAA3C;AACA,eAAKC,UAAL,GAAkBT,iBAAiBO,QAAjB,CAA0BE,UAA5C;AACA,eAAKC,MAAL,GAAcV,iBAAiBO,QAAjB,CAA0BG,MAAxC;AACA,cAAI,CAACV,iBAAiBO,QAAjB,CAA0BI,UAA/B,EAA0C;AACxCX,6BAAiBO,QAAjB,CAA0BI,UAA1B,GAAuC,KAAvC;AACD;;AAED,eAAKC,GAAL,GAAW,KAAKC,aAAL,EAAX;AACA,eAAKC,eAAL,GAAuB,IAAvB;AACA,eAAKC,SAAL,GAAiB,KAAjB;;AAEA,eAAKC,cAAL,GAAsB,IAAtB;AACA,eAAKC,OAAL,GAAe,EAAf;;AAEA,eAAKhB,EAAL,GAAUA,EAAV;AACA,eAAKC,UAAL,GAAkBA,UAAlB;AACA,eAAKC,WAAL,GAAmBA,WAAnB;AACA,eAAKC,UAAL,GAAkBA,UAAlB;;AAEA,eAAKc,kBAAL,GAA0B,IAA1B;AACA,eAAKC,kBAAL,GAA0B,IAA1B;AAED;;AAED;;;;;gCACMC,O,EAAQ;AACZ,gBAAIC,QAAQ,KAAKC,iBAAL,CAAuBF,QAAQG,QAAR,CAAiBC,IAAxC,EAA8C,KAA9C,CAAZ;AACA,gBAAIC,MAAM,KAAKH,iBAAL,CAAuBF,QAAQG,QAAR,CAAiBG,EAAxC,EAA4C,IAA5C,CAAV;AACA,gBAAIC,KAAK,EAAT;;AAEA/B,cAAEgC,IAAF,CAAOR,QAAQS,OAAf,EAAwB,UAASC,MAAT,EAAiB;AACvC,kBAAI,CAACA,OAAOC,MAAZ,EAAoB;AAAE;AAAS;AAC/BJ,iBAAGK,IAAH,CAAQ,KAAKC,oBAAL,CAA0BH,MAA1B,EAAkCV,OAAlC,CAAR;AACD,aAHuB,CAGtBc,IAHsB,CAGjB,IAHiB,CAAxB;;AAKA,gBAAIC,UAAUvC,EAAEwC,OAAF,CAAUT,EAAV,CAAd;;AAEA;AACA,gBAAI/B,EAAEyC,OAAF,CAAUF,OAAV,CAAJ,EAAwB;AACtB,kBAAIG,IAAI,KAAKrC,EAAL,CAAQsC,KAAR,EAAR;AACAD,gBAAEE,OAAF,CAAU,EAAEC,MAAM,EAAR,EAAV;AACA,qBAAOH,EAAEI,OAAT;AACD;;AAED,gBAAIC,cAAc,EAAlB;AACA/C,cAAEgC,IAAF,CAAOO,OAAP,EAAgB,UAASS,KAAT,EAAgB;AAC9B,kBAAIA,MAAMC,OAAN,IAAiBD,MAAMC,OAAN,CAAcC,MAAd,GAAuB,CAA5C,EAA+C;AAC7ClD,kBAAEgC,IAAF,CAAOgB,MAAMC,OAAb,EAAsB,UAASE,GAAT,EAAc;AAClCJ,8BAAYI,IAAIC,IAAhB,IAAwB,IAAxB;AACD,iBAFD;AAGD,eAJD,MAIO;AACLpD,kBAAEgC,IAAF,CAAOgB,MAAMK,IAAb,EAAmB,UAASF,GAAT,EAAcG,GAAd,EAAmB;AACpCP,8BAAYO,GAAZ,IAAmB,IAAnB;AACD,iBAFD;AAGD;AACF,aAVD;;AAYA,mBAAO,KAAKC,sBAAL,CAA4BhB,OAA5B,EAAqCd,KAArC,EAA4CI,GAA5C,EAAiD2B,IAAjD,CAAsD,UAASC,QAAT,EAAmB;AAC9E,kBAAIC,wBAAwB,KAAKC,mBAAL,CAAyBF,SAASZ,IAAlC,EAAwCrB,OAAxC,CAA5B;AACA,kBAAIoC,SAAS5D,EAAE6D,GAAF,CAAMJ,SAASZ,IAAf,EAAqB,UAASiB,UAAT,EAAqBC,KAArB,EAA4B;AAC5DA,wBAAQL,sBAAsBK,KAAtB,CAAR;AACA,oBAAIA,UAAU,CAAC,CAAf,EAAkB;AAChBA,0BAAQ,CAAR;AACD;AACD,qBAAKC,YAAL,CAAkBF,UAAlB;;AAEA,uBAAO,KAAKG,mBAAL,CAAyBH,UAAzB,EAAqCf,WAArC,EAAkDvB,QAAQS,OAAR,CAAgB8B,KAAhB,CAAlD,EAA0EvC,OAA1E,CAAP;AACD,eARiC,CAQhCc,IARgC,CAQ3B,IAR2B,CAArB,CAAb;AASA,qBAAO,EAAEO,MAAMe,MAAR,EAAP;AACD,aAZ4D,CAY3DtB,IAZ2D,CAYtD,IAZsD,CAAtD,CAAP;AAaD;;;2CAGe;AACd,mBAAO,KAAKhC,UAAL,CAAgB4D,iBAAhB,CAAkC;AACvClD,mBAAK,KAAKA,GAAL,GAAW,UADuB;AAEvCmD,sBAAQ;AAF+B,aAAlC,EAGJX,IAHI,CAGC,oBAAY;AAClB,kBAAIC,SAASW,MAAT,KAAoB,GLine truncated
@@ -23,8 +23,8 @@
      {"name": "Project site", "url": "https://github.com/NationalSecurityAgency/timely"},
      {"name": "License & Terms", "url": "https://raw.githubusercontent.com/NationalSecurityAgency/timely/master/LICENSE"}
    ],
-    "version": "0.0.6-SNAPSHOT",
-    "updated": "2017-08-11"
+    "version": "1.2.8-SNAPSHOT",
+    "updated": "2020-02-21"
  },

  "includes": [
@@ -1,7 +1,7 @@
 {
  "name": "timely-app",
  "private": true,
-  "version": "0.0.2",
+  "version": "1.2.8-SNAPSHOT",
  "description": "",
  "main": "index.js",
  "scripts": {
@@ -23,8 +23,8 @@
      {"name": "Project site", "url": "https://github.com/NationalSecurityAgency/timely"},
      {"name": "License & Terms", "url": "https://raw.githubusercontent.com/NationalSecurityAgency/timely/master/LICENSE"}
    ],
-    "version": "0.0.6-SNAPSHOT",
-    "updated": "2017-08-11"
+    "version": "1.2.8-SNAPSHOT",
+    "updated": "2020-02-21"
  },

  "includes": [
@@ -446,7 +446,7 @@ export class TimelyDatasource {
        } else {
          return target.metric === metricData.metric && _.every(target.tags, function(tagV, tagK) {
                interpolatedTagValue = this.templateSrv.replace(tagV, options.scopedVars, 'pipe');
-                return metricData.tags[tagK] === interpolatedTagValue || interpolatedTagValue === "*";
+                return metricData.tags[tagK] === interpolatedTagValue || interpolatedTagValue === ".*";
              }.bind(this));
        }
      }.bind(this));
@@ -3,7 +3,7 @@
  <modelVersion>4.0.0</modelVersion>
  <groupId>timely</groupId>
  <artifactId>timely</artifactId>
-  <version>1.0.1-SNAPSHOT</version>
+  <version>1.2.8-SNAPSHOT</version>
  <packaging>pom</packaging>
  <name>Timely</name>
  <description>A secure and scalable time series database based on Apache Accumulo</description>
@@ -20,6 +20,7 @@
    <module>grafana</module>
    <module>analytics</module>
    <module>balancer</module>
+    <module>grafana-auth</module>
  </modules>
  <scm>
    <connection>scm:git:https://github.com/NationalSecurityAgency/timely.git</connection>
@@ -40,50 +41,50 @@
    <os.detection.classifierWithLikes>fedora</os.detection.classifierWithLikes>
    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
    <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
-    <version.accumulo>1.7.4</version.accumulo>
+    <version.accumulo>1.9.3</version.accumulo>
    <version.buildnumber-maven>1.4</version.buildnumber-maven>
-    <version.caffeine>2.3.3</version.caffeine>
-    <version.collectd>5.5.1</version.collectd>
-    <version.commons-collections4>4.1</version.commons-collections4>
-    <version.commons-configuration>1.10</version.commons-configuration>
-    <version.commons-io>2.6</version.commons-io>
-    <version.commons-lang3>3.5</version.commons-lang3>
-    <version.commons-pool2>2.6.0</version.commons-pool2>
-    <version.curator>2.11.1</version.curator>
-    <version.disruptor>3.3.6</version.disruptor>
+    <version.caffeine>2.6.2</version.caffeine>
+    <version.collectd>5.7.1</version.collectd>
+    <version.commons-collections4>4.3</version.commons-collections4>
+    <version.commons-lang3>3.8.1</version.commons-lang3>
+    <version.commons-pool2>2.6.1</version.commons-pool2>
+    <version.curator>4.1.0</version.curator>
+    <version.disruptor>3.4.2</version.disruptor>
+    <version.easymock>4.0.2</version.easymock>
    <version.eclipse-lifecycle>1.0.0</version.eclipse-lifecycle>
-    <version.fast-classpath-scanner>2.0.5</version.fast-classpath-scanner>
+    <version.fast-classpath-scanner>3.1.13</version.fast-classpath-scanner>
    <version.flink>1.1.4</version.flink>
    <version.google-flatbuffers>1.10.0</version.google-flatbuffers>
-    <version.guava>19.0</version.guava>
-    <version.hadoop>2.7.3</version.hadoop>
-    <version.httpcomponents>4.5.2</version.httpcomponents>
-    <version.impsort>1.2.0</version.impsort>
-    <version.jackson>2.9.7</version.jackson>
-    <version.jacoco>0.7.6.201602180812</version.jacoco>
+    <version.guava>27.1-jre</version.guava>
+    <version.hadoop>2.7.7</version.hadoop>
+    <version.httpcomponents>4.5.6</version.httpcomponents>
+    <version.impsort>1.3.0</version.impsort>
+    <version.jackson>2.10.0</version.jackson>
+    <version.jackson.databind>2.10.0</version.jackson.databind>
+    <version.jacoco>0.8.4</version.jacoco>
    <version.javacsv>2.0</version.javacsv>
-    <version.javassist>3.21.0-GA</version.javassist>
+    <version.javassist>3.24.1-GA</version.javassist>
    <version.jcip-annotations>1.0</version.jcip-annotations>
-    <version.jsoup>1.9.2</version.jsoup>
+    <version.jsoup>1.11.3</version.jsoup>
    <version.junit>4.12</version.junit>
-    <version.log4j>2.6.2</version.log4j>
-    <version.maven-assembly>2.6</version.maven-assembly>
-    <version.maven-build-helper>1.12</version.maven-build-helper>
-    <version.maven-compiler>3.3</version.maven-compiler>
-    <version.maven-dependency>2.10</version.maven-dependency>
-    <version.maven-jar>3.0.1</version.maven-jar>
-    <version.maven-pmd>3.6</version.maven-pmd>
-    <version.maven-rpm>2.1.4</version.maven-rpm>
-    <version.maven-shade>2.4.2</version.maven-shade>
-    <version.maven-surefire>2.19.1</version.maven-surefire>
-    <version.netty>4.0.56.Final</version.netty>
-    <version.netty-tcnative>2.0.7.Final</version.netty-tcnative>
+    <version.log4j>2.11.2</version.log4j>
+    <version.maven-assembly>3.1.1</version.maven-assembly>
+    <version.maven-build-helper>3.0.0</version.maven-build-helper>
+    <version.maven-compiler>3.8.1</version.maven-compiler>
+    <version.maven-dependency>3.1.1</version.maven-dependency>
+    <version.maven-jar>3.1.2</version.maven-jar>
+    <version.maven-pmd>3.12.0</version.maven-pmd>
+    <version.maven-rpm>2.2.0</version.maven-rpm>
+    <version.maven-shade>3.2.1</version.maven-shade>
+    <version.maven-surefire>3.0.0-M3</version.maven-surefire>
+    <version.netty>4.1.42.Final</version.netty>
+    <version.netty-tcnative>2.0.25.Final</version.netty-tcnative>
    <version.os-maven>1.5.0.Final</version.os-maven>
    <version.powermock>1.6.4</version.powermock>
-    <version.slf4j>1.7.21</version.slf4j>
-    <version.spotbugs-maven>3.1.7</version.spotbugs-maven>
-    <version.spring-boot>1.4.1.RELEASE</version.spring-boot>
-    <version.tyrus>1.13</version.tyrus>
+    <version.slf4j>1.7.26</version.slf4j>
+    <version.spotbugs-maven>3.1.12.1</version.spotbugs-maven>
+    <version.spring-boot>2.1.3.RELEASE</version.spring-boot>
+    <version.tyrus>1.15</version.tyrus>
  </properties>
  <dependencyManagement>
    <dependencies>
@@ -100,6 +101,11 @@
      <dependency>
        <groupId>com.fasterxml.jackson.core</groupId>
        <artifactId>jackson-databind</artifactId>
+        <version>${version.jackson.databind}</version>
+      </dependency>
+      <dependency>
+        <groupId>com.fasterxml.jackson.datatype</groupId>
+        <artifactId>jackson-datatype-guava</artifactId>
        <version>${version.jackson}</version>
      </dependency>
      <dependency>
@@ -135,13 +141,18 @@
      <dependency>
        <groupId>io.github.lukehutch</groupId>
        <artifactId>fast-classpath-scanner</artifactId>
-        <version>2.0.5</version>
+        <version>${version.fast-classpath-scanner}</version>
      </dependency>
      <dependency>
        <groupId>io.netty</groupId>
        <artifactId>netty-all</artifactId>
        <version>${version.netty}</version>
      </dependency>
+      <dependency>
+        <groupId>io.netty</groupId>
+        <artifactId>netty-tcnative-boringssl-static</artifactId>
+        <version>${version.netty-tcnative}</version>
+      </dependency>
      <dependency>
        <groupId>net.sourceforge.javacsv</groupId>
        <artifactId>javacsv</artifactId>
@@ -151,6 +162,12 @@
        <groupId>org.apache.accumulo</groupId>
        <artifactId>accumulo-core</artifactId>
        <version>${version.accumulo}</version>
+        <exclusions>
+          <exclusion>
+            <groupId>io.netty</groupId>
+            <artifactId>netty</artifactId>
+          </exclusion>
+        </exclusions>
      </dependency>
      <dependency>
        <groupId>org.apache.accumulo</groupId>
@@ -304,6 +321,11 @@
        <artifactId>collectd-plugin-parent</artifactId>
        <version>${project.version}</version>
      </dependency>
+      <dependency>
+        <groupId>timely</groupId>
+        <artifactId>timely-balancer</artifactId>
+        <version>${project.version}</version>
+      </dependency>
      <dependency>
        <groupId>timely</groupId>
        <artifactId>timely-client</artifactId>
@@ -376,6 +398,12 @@
        <version>${version.jcip-annotations}</version>
        <scope>test</scope>
      </dependency>
+      <dependency>
+        <groupId>org.easymock</groupId>
+        <artifactId>easymock</artifactId>
+        <version>${version.easymock}</version>
+        <scope>test</scope>
+      </dependency>
      <dependency>
        <groupId>org.powermock</groupId>
        <artifactId>powermock-easymock-release-full</artifactId>
@@ -391,7 +419,7 @@
        <plugin>
          <groupId>net.revelc.code.formatter</groupId>
          <artifactId>formatter-maven-plugin</artifactId>
-          <version>2.8.0</version>
+          <version>2.10.0</version>
          <configuration>
            <compilerCompliance>${maven.compiler.source}</compilerCompliance>
            <compilerSource>${maven.compiler.source}</compilerSource>
@@ -409,7 +437,7 @@
        <plugin>
          <groupId>com.github.ekryd.sortpom</groupId>
          <artifactId>sortpom-maven-plugin</artifactId>
-          <version>2.8.0</version>
+          <version>2.10.0</version>
          <configuration>
            <predefinedSortOrder>recommended_2008_06</predefinedSortOrder>
            <createBackupFile>false</createBackupFile>
@@ -441,7 +469,7 @@
          <artifactId>maven-failsafe-plugin</artifactId>
          <version>${version.maven-surefire}</version>
          <configuration>
-            <argLine>${failsafeArgLine} -Djava.library.path=${project.build.directory}/native/META-INF/native/libnetty-tcnative.so -Dzookeeper.sasl.client=false</argLine>
+            <argLine>${failsafeArgLine} -Djava.library.path=${project.build.directory}/native/META-INF/native/libnetty_tcnative_linux_x86_64.so -Dzookeeper.sasl.client=false</argLine>
            <includes>
              <include>**/*.java</include>
            </includes>
@@ -484,7 +512,7 @@
          <artifactId>maven-surefire-plugin</artifactId>
          <version>${version.maven-surefire}</version>
          <configuration>
-            <argLine>${sureFireArgLine} -Djava.library.path=${project.build.directory}/native/META-INF/native/libnetty-tcnative.so -Dzookeeper.sasl.client=false</argLine>
+            <argLine>${sureFireArgLine} -Djava.library.path=${project.build.directory}/native/META-INF/native/libnetty-tcnative_linux_x86_64.so -Dzookeeper.sasl.client=false</argLine>
            <excludedGroups>timely.test.IntegrationTest</excludedGroups>
            <redirectTestOutputToFile>true</redirectTestOutputToFile>
          </configuration>
@@ -553,7 +581,7 @@
            <configuration>
              <rules>
                <requireMavenVersion>
-                  <version>3.0.5</version>
+                  <version>3.5.0</version>
                </requireMavenVersion>
                <requireJavaVersion>
                  <version>[${maven.compiler.target},)</version>
@@ -724,6 +752,9 @@
          <name>m2e.version</name>
        </property>
      </activation>
+      <properties>
+        <maven.compiler.release>8</maven.compiler.release>
+      </properties>
      <build>
        <pluginManagement>
          <plugins>
@@ -784,5 +815,14 @@
        </pluginManagement>
      </build>
    </profile>
+    <profile>
+      <id>jdk-release-flag</id>
+      <activation>
+        <jdk>[9,)</jdk>
+      </activation>
+      <properties>
+        <maven.compiler.release>8</maven.compiler.release>
+      </properties>
+    </profile>
  </profiles>
 </project>
@@ -0,0 +1,21 @@
+#!/bin/bash
+
+if [[ `uname` == "Darwin" ]]; then
+        THIS_SCRIPT=`python -c 'import os,sys; print os.path.realpath(sys.argv[1])' $0`
+else
+        THIS_SCRIPT=`readlink -f $0`
+fi
+
+THIS_DIR="${THIS_SCRIPT%/*}"
+BASE_DIR=${THIS_DIR}/..
+CONF_DIR="${BASE_DIR}/conf"
+LIB_DIR="${BASE_DIR}/lib"
+
+export CLASSPATH="${CONF_DIR}:${LIB_DIR}/*"
+JVM_ARGS="-Xmx1024m -Xms128m"
+JVM_ARGS="${JVM_ARGS} -Dlogging.config=${CONF_DIR}/log4j2-error-console.xml"
+JVM_ARGS="${JVM_ARGS} -Dlog4j.configurationFile=${CONF_DIR}/log4j2-error-console.xml"
+JVM_ARGS="${JVM_ARGS} -DLog4jContextSelector=org.apache.logging.log4j.core.async.AsyncLoggerContextSelector"
+
+echo "$JAVA_HOME/bin/java ${JVM_ARGS} timely.store.compaction.util.TabletMetadataConsole --spring.config.name=timely"
+$JAVA_HOME/bin/java ${JVM_ARGS} timely.store.compaction.util.TabletMetadataConsole --spring.config.name=timely
@@ -8,11 +8,6 @@ else
        TCNATIVE_SUFFIX="so"
 fi

-# netty tcnative file reference
-#netty-tcnative-1.1.33.Fork18-linux-x86_64-fedora.jar -> libnetty-tcnative.so
-#netty-tcnative-1.1.33.Fork18-linux-x86_64.jar -> libnetty-tcnative.so
-#netty-tcnative-1.1.33.Fork18-osx-x86_64.jar -> libnetty-tcnative.jnilib
-
 THIS_DIR="${THIS_SCRIPT%/*}"
 NATIVE_DIR="${THIS_DIR}/META-INF/native"
 BASE_DIR=${THIS_DIR}/..
@@ -32,12 +27,12 @@ fi
 mkdir -p ${NATIVE_DIR}

 pushd ${BASE_DIR}/bin
-$JAVA_HOME/bin/jar xf ${LIB_DIR}/netty-tcnative*.jar META-INF/native/libnetty_tcnative.${NATIVE_SUFFIX}
-$JAVA_HOME/bin/jar xf ${LIB_DIR}/netty-transport-native-epoll*.jar META-INF/native/libnetty_transport_native_epoll_x86_64.${NATIVE_SUFFIX}
+$JAVA_HOME/bin/jar xf ${LIB_DIR}/netty-tcnative-boringssl-static*.jar META-INF/native/libnetty_tcnative_linux_x86_64.${TCNATIVE_SUFFIX}
+$JAVA_HOME/bin/jar xf ${LIB_DIR}/netty-all*.jar META-INF/native/libnetty_transport_native_epoll_x86_64.${TCNATIVE_SUFFIX}
 popd

-export CLASSPATH="${CONF_DIR}:${LIB_DIR}/*"
-JVM_ARGS="-Xmx256m -Xms256m -Dio.netty.eventLoopThreads=${NUM_SERVER_THREADS}"
+export CLASSPATH="${CONF_DIR}:${LIB_DIR}/*:${HADOOP_CONF_DIR}"
+JVM_ARGS="-Xmx256m -Xms256m -Dio.netty.eventLoopThreads=${NUM_SERVER_THREADS} -Dlog4j.configurationFile=${CONF_DIR}/log4j2-spring.xml"
 JVM_ARGS="${JVM_ARGS} -DLog4jContextSelector=org.apache.logging.log4j.core.async.AsyncLoggerContextSelector"
 JVM_ARGS="${JVM_ARGS} -Djava.library.path=${NATIVE_DIR}"

@@ -2,17 +2,12 @@

 if [[ `uname` == "Darwin" ]]; then
        THIS_SCRIPT=`python -c 'import os,sys; print os.path.realpath(sys.argv[1])' $0`
-        TCNATIVE_SUFFIX="tcnative.jnilib"
+        TCNATIVE_SUFFIX="jnilib"
 else
        THIS_SCRIPT=`readlink -f $0`
-        TCNATIVE_SUFFIX="tcnative.so"
+        TCNATIVE_SUFFIX="so"
 fi

-# netty tcnative file reference
-#netty-tcnative-1.1.33.Fork18-linux-x86_64-fedora.jar -> libnetty-tcnative.so
-#netty-tcnative-1.1.33.Fork18-linux-x86_64.jar -> libnetty-tcnative.so
-#netty-tcnative-1.1.33.Fork18-osx-x86_64.jar -> libnetty-tcnative.jnilib
-
 THIS_DIR="${THIS_SCRIPT%/*}"
 NATIVE_DIR="${THIS_DIR}/META-INF/native"
 BASE_DIR=${THIS_DIR}/..
@@ -32,11 +27,12 @@ fi
 mkdir -p ${NATIVE_DIR}

 pushd ${BASE_DIR}/bin
-$JAVA_HOME/bin/jar xf ${LIB_DIR}/netty-tcnative*.jar META-INF/native/libnetty-${TCNATIVE_SUFFIX}
+$JAVA_HOME/bin/jar xf ${LIB_DIR}/netty-tcnative-boringssl-static*.jar META-INF/native/libnetty_tcnative_linux_x86_64.${TCNATIVE_SUFFIX}
+$JAVA_HOME/bin/jar xf ${LIB_DIR}/netty-all*.jar META-INF/native/libnetty_transport_native_epoll_x86_64.${TCNATIVE_SUFFIX}
 popd

-export CLASSPATH="${CONF_DIR}:${LIB_DIR}/*"
-JVM_ARGS="-Xmx256m -Xms256m -Dio.netty.eventLoopThreads=${NUM_SERVER_THREADS}"
+export CLASSPATH="${CONF_DIR}:${LIB_DIR}/*:${HADOOP_CONF_DIR}"
+JVM_ARGS="-Xmx256m -Xms256m -Dio.netty.eventLoopThreads=${NUM_SERVER_THREADS} -Dlog4j.configurationFile=${CONF_DIR}/log4j2-spring.xml"
 JVM_ARGS="${JVM_ARGS} -DLog4jContextSelector=org.apache.logging.log4j.core.async.AsyncLoggerContextSelector"
 JVM_ARGS="${JVM_ARGS} -Djava.library.path=${NATIVE_DIR}"
 # for debugging
@@ -26,12 +26,12 @@
    </Console>
  </Appenders>
  <Loggers>
-    <Logger name="timely.subscription" level="trace" additivity="false">
-      <AppenderRef ref="Subscription" />
-    </Logger>
-    <Root level="trace" includeLocation="false">
+    <Root level="debug" includeLocation="false">
      <AppenderRef ref="Console" level="warn" />
-      <AppenderRef ref="Rolling" level="trace" />
+      <AppenderRef ref="Rolling" level="debug" />
    </Root>
+    <Logger name="timely" additivity="false">
+      <AppenderRef ref="Console" level="debug" />
+    </Logger>
  </Loggers>
 </Configuration>
@@ -15,15 +15,27 @@

    <beans:bean id="timelyUserDetailsService" class="timely.auth.UserDetailsService">
    	<beans:property name="users">
-    		<util:map key-type="java.lang.String">
-    			<beans:entry key="example.com">
-   					<util:list>
-   						<beans:value>D</beans:value>
-   						<beans:value>E</beans:value>
-   						<beans:value>F</beans:value>
-   					</util:list>
-    			</beans:entry>
-    		</util:map>
+    		<util:list>
+				<beans:bean class="timely.auth.TimelyUser">
+					<beans:constructor-arg name="subjectDn">
+						<beans:value>CN=example.com</beans:value>
+					</beans:constructor-arg>
+					<beans:constructor-arg name="auths">
+						<util:list>
+							<beans:value>D</beans:value>
+							<beans:value>E</beans:value>
+							<beans:value>F</beans:value>
+						</util:list>
+					</beans:constructor-arg>
+					<beans:constructor-arg name="roles">
+						<util:list>
+							<beans:value>G</beans:value>
+							<beans:value>H</beans:value>
+							<beans:value>I</beans:value>
+						</util:list>
+					</beans:constructor-arg>
+				</beans:bean>
+    		</util:list>
    	</beans:property>
    </beans:bean>
    
@@ -34,9 +46,12 @@
    <user-service id="usernamePasswordService">
    	<user name="test" password="test1" authorities="A,B,C" />
    </user-service>
-    
-    <beans:bean id="UserNamePasswordAuthenticationProvider" class="org.springframework.security.authentication.dao.DaoAuthenticationProvider">
+
+	<beans:bean id="noOpPasswordEncoder" class="org.springframework.security.crypto.password.NoOpPasswordEncoder"/>
+
+	<beans:bean id="UserNamePasswordAuthenticationProvider" class="org.springframework.security.authentication.dao.DaoAuthenticationProvider">
 		<beans:property name="userDetailsService" ref="usernamePasswordService"/>
+		<beans:property name="passwordEncoder" ref="noOpPasswordEncoder"/>
 	</beans:bean>
 	    
 	<authentication-manager alias="authenticationManager">
@@ -45,5 +60,12 @@
 		<!-- Basic authorization provider, validates provided username and password -->
 		<authentication-provider ref="UserNamePasswordAuthenticationProvider" />
 	</authentication-manager>
-	
+
+	<util:list id="requiredRoles" value-type="java.lang.String">
+		<beans:value></beans:value>
+	</util:list>
+
+	<util:list id="requiredAuths" value-type="java.lang.String">
+		<beans:value></beans:value>
+	</util:list>
 </beans:beans>
@@ -18,8 +18,11 @@ timely:
      max-downsample-memory: 10000000
  security:
    session-max-age: 86400
-    allow-anonymous-access: true
-    ssl:
+    cache-expiration-minutes: 60
+    cache-refresh-minutes: 5
+    allow-anonymous-http-access: true
+    allow-anonymous-ws-access: true
+    server-ssl:
      certificate-file: /timely-server-src/server/pki/timely.csr
      key-file: /timely-server-src/server/pki/timely-pkcs8.key
      key-password: password
@@ -49,9 +52,9 @@ timely:
    timeout: 60
    subscription-lag: 120
  meta-cache:
-    expiration-minutes: 60
-    initial-capacity: 2000
-    max-capacity: 10000
+    expiration-minutes: 1440
+    cache-refresh-minutes: 60
+    max-tag-values: 100
  visibility-cache:
    expiration-minutes: 60
    initial-capacity: 2000
@@ -3,6 +3,8 @@ timely:
  meta-table: timely.meta
  metric-age-off-days:
    default: 7
+  cache:
+    enabled: false
  metrics-report-ignored-tags:
  accumulo:
    instance-name: TimelyStandalone
@@ -18,8 +20,11 @@ timely:
      max-downsample-memory: 10000000
  security:
    session-max-age: 86400
-    allow-anonymous-access: true
-    ssl:
+    cache-expiration-minutes: 60
+    cache-refresh-minutes: 5
+    allow-anonymous-http-access: true
+    allow-anonymous-ws-access: true
+    server-ssl:
      certificate-file:
      key-file:
      key-password: timely
@@ -53,9 +58,9 @@ timely:
    scanner-read-ahead: 1
    subscription-batch-size: 5000
  meta-cache:
-    expiration-minutes: 60
-    initial-capacity: 2000
-    max-capacity: 10000
+    expiration-minutes: 1440
+    cache-refresh-minutes: 60
+    max-tag-values: 100
  visibility-cache:
    expiration-minutes: 60
    initial-capacity: 2000
@@ -4,14 +4,15 @@ timely:
  metric-age-off-days:
    default: 7
  cache:
-    enabled:
+    enabled: true
    age-off-hours:
-      default: 30
+      default: 26
  metrics-report-ignored-tags:
+  defaultVisibility:
  accumulo:
-    instance-name:
-    zookeepers:
-    username:
+    instance-name: accumulo
+    zookeepers: localhost:2181
+    username: root
    password:
    write:
      latency: 5s
@@ -22,23 +23,26 @@ timely:
      max-downsample-memory: 10000000
  security:
    session-max-age: 86400
-    allow-anonymous-access: false
-    ssl:
+    cache-expiration-minutes: 60
+    cache-refresh-minutes: 5
+    allow-anonymous-http-access: true
+    allow-anonymous-ws-access: true
+    server-ssl:
      certificate-file:
      key-file:
      key-password:
-      use-generated-keypair: false
+      use-generated-keypair: true
      trust-store-file:
      use-openssl: true
      #use-ciphers:
  server:
-    ip:
-    tcp-port:
-    udp-port:
+    ip: 0.0.0.0
+    tcp-port: 4242
+    udp-port: 4542
  http:
-    host:
-    ip:
-    port:
+    host: localhost
+    ip: 0.0.0.0
+    port: 4243
    redirect-path: /secure-me
    strict-transport-max-age: 608400
    cors:
@@ -49,8 +53,8 @@ timely:
      allowed-methods: DELETE,GET,HEAD,OPTIONS,PUT,POST
      allowed-headers: content-type
  websocket:
-    ip:
-    port:
+    ip: 0.0.0.0
+    port: 4244
    timeout: 60
    subscription-lag: 120
    scanner-batch-size: 10000
@@ -58,10 +62,9 @@ timely:
    scanner-read-ahead: 1
    subscription-batch-size: 5000
  meta-cache:
-    expiration-minutes: 60
-    initial-capacity: 2000
-    max-capacity: 10000
+    expiration-minutes: 1440
+    cache-refresh-minutes: 60
+    max-tag-values: 100
  visibility-cache:
    expiration-minutes: 60
-    initial-capacity: 2000
-    max-capacity: 10000
+
@@ -4,7 +4,7 @@
  <parent>
    <groupId>timely</groupId>
    <artifactId>timely</artifactId>
-    <version>1.0.1-SNAPSHOT</version>
+    <version>1.2.8-SNAPSHOT</version>
  </parent>
  <artifactId>timely-server</artifactId>
  <dependencies>
@@ -20,6 +20,10 @@
      <groupId>com.fasterxml.jackson.core</groupId>
      <artifactId>jackson-databind</artifactId>
    </dependency>
+    <dependency>
+      <groupId>com.fasterxml.jackson.datatype</groupId>
+      <artifactId>jackson-datatype-guava</artifactId>
+    </dependency>
    <dependency>
      <groupId>com.fasterxml.jackson.datatype</groupId>
      <artifactId>jackson-datatype-jdk8</artifactId>
@@ -104,6 +108,10 @@
      <groupId>org.apache.hadoop</groupId>
      <artifactId>hadoop-minicluster</artifactId>
    </dependency>
+    <dependency>
+      <groupId>org.glassfish.tyrus</groupId>
+      <artifactId>tyrus-client</artifactId>
+    </dependency>
    <dependency>
      <groupId>org.slf4j</groupId>
      <artifactId>slf4j-api</artifactId>
@@ -145,9 +153,7 @@
    </dependency>
    <dependency>
      <groupId>io.netty</groupId>
-      <artifactId>netty-tcnative</artifactId>
-      <version>${version.netty-tcnative}</version>
-      <classifier>${os.detected.classifier}</classifier>
+      <artifactId>netty-tcnative-boringssl-static</artifactId>
      <scope>runtime</scope>
    </dependency>
    <dependency>
@@ -175,11 +181,22 @@
      <artifactId>junit</artifactId>
      <scope>test</scope>
    </dependency>
+    <dependency>
+      <groupId>org.easymock</groupId>
+      <artifactId>easymock</artifactId>
+      <scope>test</scope>
+    </dependency>
    <dependency>
      <groupId>org.jsoup</groupId>
      <artifactId>jsoup</artifactId>
      <scope>test</scope>
    </dependency>
+    <dependency>
+      <groupId>org.powermock</groupId>
+      <artifactId>powermock-easymock-release-full</artifactId>
+      <classifier>full</classifier>
+      <scope>test</scope>
+    </dependency>
    <dependency>
      <groupId>org.springframework.boot</groupId>
      <artifactId>spring-boot-test-autoconfigure</artifactId>
@@ -236,9 +253,8 @@
              <artifactItems>
                <artifactItem>
                  <groupId>io.netty</groupId>
-                  <artifactId>netty-tcnative</artifactId>
+                  <artifactId>netty-tcnative-boringssl-static</artifactId>
                  <version>${version.netty-tcnative}</version>
-                  <classifier>${os.detected.classifier}</classifier>
                  <type>jar</type>
                  <overWrite>false</overWrite>
                  <outputDirectory>${project.build.directory}/native</outputDirectory>
@@ -280,37 +296,4 @@
      </extension>
    </extensions>
  </build>
-  <profiles>
-    <profile>
-      <id>native-epoll-linux</id>
-      <activation>
-        <os>
-          <family>!mac</family>
-        </os>
-      </activation>
-      <dependencies>
-        <dependency>
-          <groupId>io.netty</groupId>
-          <artifactId>netty-transport-native-epoll</artifactId>
-          <version>${version.netty}</version>
-          <classifier>linux-x86_64</classifier>
-        </dependency>
-      </dependencies>
-    </profile>
-    <profile>
-      <id>native-epoll</id>
-      <activation>
-        <os>
-          <family>mac</family>
-        </os>
-      </activation>
-      <dependencies>
-        <dependency>
-          <groupId>io.netty</groupId>
-          <artifactId>netty-transport-native-epoll</artifactId>
-          <version>${version.netty}</version>
-        </dependency>
-      </dependencies>
-    </profile>
-  </profiles>
 </project>
@@ -33,7 +33,7 @@
      <directory>${project.basedir}/bin</directory>
      <outputDirectory>bin</outputDirectory>
      <includes>
-        <include>*</include>
+        <include>**/*</include>
      </includes>
     </fileSet>
    <fileSet>
@@ -1,5 +1,8 @@
 package timely;

+import static timely.store.cache.DataStoreCache.NON_CACHED_METRICS;
+import static timely.store.cache.DataStoreCache.NON_CACHED_METRICS_LOCK_PATH;
+
 import java.io.File;
 import java.net.InetAddress;
 import java.net.InetSocketAddress;
@@ -21,6 +24,7 @@ import io.netty.channel.ChannelHandler;
 import io.netty.channel.ChannelInitializer;
 import io.netty.channel.ChannelOption;
 import io.netty.channel.EventLoopGroup;
+import io.netty.channel.WriteBufferWaterMark;
 import io.netty.channel.epoll.EpollChannelOption;
 import io.netty.channel.epoll.EpollDatagramChannel;
 import io.netty.channel.epoll.EpollEventLoopGroup;
@@ -41,6 +45,7 @@ import io.netty.handler.codec.http.HttpRequestDecoder;
 import io.netty.handler.codec.http.HttpResponseEncoder;
 import io.netty.handler.codec.http.HttpServerCodec;
 import io.netty.handler.codec.http.cors.CorsConfig;
+import io.netty.handler.codec.http.cors.CorsConfigBuilder;
 import io.netty.handler.codec.http.cors.CorsHandler;
 import io.netty.handler.codec.http.websocketx.WebSocketServerProtocolHandler;
 import io.netty.handler.logging.LoggingHandler;
@@ -55,6 +60,7 @@ import io.netty.handler.stream.ChunkedWriteHandler;
 import io.netty.handler.timeout.IdleStateHandler;
 import io.netty.util.NettyRuntime;
 import io.netty.util.internal.SystemPropertyUtil;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
 import org.apache.curator.RetryPolicy;
 import org.apache.curator.framework.CuratorFramework;
 import org.apache.curator.framework.CuratorFrameworkFactory;
@@ -63,12 +69,13 @@ import org.apache.curator.x.discovery.ServiceDiscovery;
 import org.apache.curator.x.discovery.ServiceDiscoveryBuilder;
 import org.apache.curator.x.discovery.ServiceInstance;
 import org.apache.curator.x.discovery.ServiceInstanceBuilder;
+import org.apache.zookeeper.CreateMode;
 import org.apache.zookeeper.data.Stat;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.boot.WebApplicationType;
 import org.springframework.boot.builder.SpringApplicationBuilder;
 import org.springframework.context.ConfigurableApplicationContext;
-import timely.api.response.TimelyException;
 import timely.auth.AuthCache;
 import timely.auth.VisibilityCache;
 import timely.configuration.Configuration;
@@ -97,7 +104,7 @@ import timely.netty.udp.UdpDecoder;
 import timely.netty.udp.UdpPacketToByteBuf;
 import timely.netty.websocket.WSMetricPutHandler;
 import timely.netty.websocket.WSVersionRequestHandler;
-import timely.netty.websocket.WebSocketHttpCookieHandler;
+import timely.netty.websocket.WebSocketFullRequestHandler;
 import timely.netty.websocket.WebSocketRequestDecoder;
 import timely.netty.websocket.subscription.WSAddSubscriptionRequestHandler;
 import timely.netty.websocket.subscription.WSCloseSubscriptionRequestHandler;
@@ -125,6 +132,7 @@ public class Server {
    private static final String OS_VERSION = "os.version";
    private static final String WS_PATH = "/websocket";

+    private CuratorFramework curatorFramework;
    protected static final CountDownLatch LATCH = new CountDownLatch(1);
    static ConfigurableApplicationContext applicationContext;

@@ -146,6 +154,9 @@ public class Server {
    protected volatile boolean shutdown = false;
    private final int DEFAULT_EVENT_LOOP_THREADS;

+    private String[] zkPaths = new String[] { SERVICE_DISCOVERY_PATH, NON_CACHED_METRICS,
+            NON_CACHED_METRICS_LOCK_PATH };
+
    private static boolean useEpoll() {

        // Should we just return true if this is Linux and if we get an error
@@ -175,13 +186,8 @@ public class Server {
        }
    }

-    public void registerService() {
+    public void registerService(CuratorFramework curatorFramework) {
        try {
-            RetryPolicy retryPolicy = new RetryForever(1000);
-            CuratorFramework curatorFramework = CuratorFrameworkFactory.newClient(config.getAccumulo().getZookeepers(),
-                    30000, 10000, retryPolicy);
-            curatorFramework.start();
-
            try {
                Stat stat = curatorFramework.checkExists().forPath(SERVICE_DISCOVERY_PATH);
                if (stat == null) {
@@ -205,13 +211,13 @@ public class Server {
            payload.setWsPort(config.getWebsocket().getPort());
            payload.setUdpPort(config.getServer().getUdpPort());

-            ServiceInstanceBuilder builder = ServiceInstance.builder();
+            ServiceInstanceBuilder<ServerDetails> builder = ServiceInstance.builder();
            String serviceName = host + ":" + config.getServer().getTcpPort();
-            ServiceInstance serviceInstance = builder.id(serviceName).name("timely-server")
+            ServiceInstance<ServerDetails> serviceInstance = builder.id(serviceName).name("timely-server")
                    .address(config.getServer().getIp()).port(config.getServer().getTcpPort()).payload(payload).build();

-            ServiceDiscovery discovery = ServiceDiscoveryBuilder.builder(ServerDetails.class).client(curatorFramework)
-                    .basePath(SERVICE_DISCOVERY_PATH).build();
+            ServiceDiscovery<ServerDetails> discovery = ServiceDiscoveryBuilder.builder(ServerDetails.class)
+                    .client(curatorFramework).basePath(SERVICE_DISCOVERY_PATH).build();
            discovery.start();
            discovery.registerService(serviceInstance);

@@ -227,11 +233,12 @@ public class Server {

    public static void main(String[] args) throws Exception {

-        Configuration conf = initializeConfiguration(args);
+        Server.applicationContext = initializeContext(args);
+        Configuration conf = Server.applicationContext.getBean(Configuration.class);

-        Server s = new Server(conf);
+        Server server = new Server(conf);
        try {
-            s.run();
+            server.run();
            LATCH.await();
        } catch (final InterruptedException e) {
            LOG.info("Server shutting down.");
@@ -239,16 +246,18 @@ public class Server {
            LOG.error("Error running server.", e);
        } finally {
            try {
-                s.shutdown();
+                server.shutdown();
            } catch (Exception e) {
                System.exit(1);
            }
        }
    }

-    protected static Configuration initializeConfiguration(String[] args) {
-        applicationContext = new SpringApplicationBuilder(SpringBootstrap.class).web(false).run(args);
-        return applicationContext.getBean(Configuration.class);
+    protected static ConfigurableApplicationContext initializeContext(String[] args) {
+        SpringApplicationBuilder builder = new SpringApplicationBuilder(SpringBootstrap.class);
+        builder.web(WebApplicationType.NONE);
+        builder.registerShutdownHook(false);
+        return builder.run(args);
    }

    private void shutdownHook() {
@@ -263,6 +272,8 @@ public class Server {
    }

    public void shutdown() {
+        LOG.warn("Server shutdown 1");
+
        List<ChannelFuture> channelFutures = new ArrayList<>();

        if (tcpChannelHandle != null) {
@@ -280,8 +291,9 @@ public class Server {
            channelFutures.add(wsChannelHandle.close());
        }

+        int udpChannel = 1;
        for (Channel c : udpChannelHandleList) {
-            LOG.info("Closing udpChannelHandle");
+            LOG.info("Closing udpChannelHandle #" + udpChannel++);
            channelFutures.add(c.close());
        }

@@ -290,14 +302,15 @@ public class Server {
            try {
                f.get();
            } catch (final Exception e) {
-                LOG.error("Error while shutting down channel: {}", f.channel().config());
-                LOG.error("{}", e.getMessage());
-                e.printStackTrace();
+                LOG.error("Channel:" + f.channel().config() + " -> " + e.getMessage(), e);
            }
        });

-        Integer quietPeriod = config.getServer().getShutdownQuietPeriod();
+        LOG.warn("Server shutdown 2");
+
+        int quietPeriod = config.getServer().getShutdownQuietPeriod();
        List<Future<?>> groupFutures = new ArrayList<>();
+
        if (tcpBossGroup != null) {
            LOG.info("Shutting down tcpBossGroup");
            groupFutures.add(tcpBossGroup.shutdownGracefully(quietPeriod, 10, TimeUnit.SECONDS));
@@ -333,6 +346,8 @@ public class Server {
            groupFutures.add(udpBossGroup.shutdownGracefully(quietPeriod, 10, TimeUnit.SECONDS));
        }

+        LOG.warn("Server shutdown 3");
+
        if (udpWorkerGroup != null) {
            LOG.info("Shutting down udpWorkerGroup");
            groupFutures.add(udpWorkerGroup.shutdownGracefully(quietPeriod, 10, TimeUnit.SECONDS));
@@ -342,37 +357,70 @@ public class Server {
            try {
                f.get();
            } catch (final Exception e) {
-                LOG.error("Error while shutting down group: {}", f.toString());
-                LOG.error("{}", e.getMessage());
-                e.printStackTrace();
+                LOG.error("Group:" + f.toString() + " -> " + e.getMessage(), e);
            }
        });

-        try {
-            LOG.info("Flushing datastore.");
-            dataStore.flush();
-        } catch (TimelyException e) {
-            LOG.error("Error flushing to server during shutdown", e);
-        }
+        LOG.warn("Server shutdown 4");

        try {
-            LOG.info("Closing MetaCacheFactory");
+            LOG.info("Closing dataStore");
+            dataStore.close();
+        } catch (Exception e) {
+            LOG.error(e.getMessage(), e);
+        }
+
+        LOG.warn("Server shutdown 5");
+
+        if (dataStoreCache != null) {
+            try {
+                LOG.info("Closing dataStoreCache");
+                dataStoreCache.close();
+            } catch (Exception e) {
+                LOG.error(e.getMessage(), e);
+            }
+        }
+
+        LOG.warn("Server shutdown 6");
+
+        try {
+            LOG.info("Closing metaCacheFactory");
            MetaCacheFactory.close();
        } catch (Exception e) {
-            LOG.error("Error closing MetaCacheFactory during shutdown", e);
+            LOG.error(e.getMessage(), e);
        }

+        LOG.warn("Server shutdown 7");
+
        try {
-            LOG.info("Closing WebSocketRequestDecoder");
+            LOG.info("Closing webSocketRequestDecoder subscriptions");
            WebSocketRequestDecoder.close();
        } catch (Exception e) {
-            LOG.error("Error closing WebSocketRequestDecoder during shutdown", e);
+            LOG.error(e.getMessage(), e);
        }

-        if (applicationContext != null) {
-            LOG.info("Closing applicationContext");
-            applicationContext.close();
+        LOG.warn("Server shutdown 8");
+
+        if (curatorFramework != null) {
+            try {
+                LOG.info("Closing curatorFramework");
+                curatorFramework.close();
+            } catch (Exception e) {
+                LOG.error(e.getMessage(), e);
+            }
        }
+
+        LOG.warn("Server shutdown 9");
+
+        if (applicationContext != null) {
+            try {
+                LOG.info("Closing applicationContext");
+                applicationContext.close();
+            } catch (Exception e) {
+                LOG.error(e.getMessage(), e);
+            }
+        }
+        LOG.warn("Server shutdown 10");
        this.shutdown = true;
        LOG.info("Server shut down.");
    }
@@ -390,20 +438,42 @@ public class Server {
        this.config = conf;
    }

+    private void ensureZkPaths(CuratorFramework curatorFramework, String[] paths) {
+        for (String s : paths) {
+            try {
+                Stat stat = curatorFramework.checkExists().forPath(s);
+                if (stat == null) {
+                    curatorFramework.create().creatingParentContainersIfNeeded().withMode(CreateMode.PERSISTENT)
+                            .forPath(s);
+                }
+            } catch (Exception e) {
+                LOG.info(e.getMessage());
+            }
+        }
+    }
+
    public void run() throws Exception {

+        RetryPolicy retryPolicy = new RetryForever(1000);
+        int timeout = Long.valueOf(AccumuloConfiguration.getTimeInMillis(config.getAccumulo().getZookeeperTimeout()))
+                .intValue();
+        curatorFramework = CuratorFrameworkFactory.newClient(config.getAccumulo().getZookeepers(), timeout, 10000,
+                retryPolicy);
+        curatorFramework.start();
+        ensureZkPaths(curatorFramework, zkPaths);
+
        int nettyThreads = Math.max(1,
                SystemPropertyUtil.getInt("io.netty.eventLoopThreads", Runtime.getRuntime().availableProcessors() * 2));
        dataStore = DataStoreFactory.create(config, nettyThreads);
        if (config.getCache().isEnabled()) {
-            dataStoreCache = new DataStoreCache(config);
+            dataStoreCache = new DataStoreCache(curatorFramework, config);
            dataStoreCache.setInternalMetrics(dataStore.getInternalMetrics());
            dataStore.setCache(dataStoreCache);
        }
        // Initialize the MetaCache
        MetaCacheFactory.getCache(config);
        // initialize the auth cache
-        AuthCache.setSessionMaxAge(config);
+        AuthCache.configure(config.getSecurity());
        // Initialize the VisibilityCache
        VisibilityCache.init(config);
        final boolean useEpoll = useEpoll();
@@ -480,10 +550,9 @@ public class Server {
        wsServer.option(ChannelOption.ALLOCATOR, PooledByteBufAllocator.DEFAULT);
        wsServer.option(ChannelOption.SO_BACKLOG, 128);
        wsServer.option(ChannelOption.SO_KEEPALIVE, true);
-        /* Not sure if next three lines are necessary */
+        /* Not sure if next two lines are necessary */
        wsServer.option(ChannelOption.SO_SNDBUF, 1048576);
-        wsServer.option(ChannelOption.WRITE_BUFFER_HIGH_WATER_MARK, 838860);
-        wsServer.option(ChannelOption.WRITE_BUFFER_LOW_WATER_MARK, 620145);
+        wsServer.option(ChannelOption.WRITE_BUFFER_WATER_MARK, new WriteBufferWaterMark(620145, 838860));
        wsChannelHandle = wsServer.bind(wsIp, wsPort).sync().channel();
        final String wsAddress = ((InetSocketAddress) wsChannelHandle.localAddress()).getAddress().getHostAddress();

@@ -500,7 +569,7 @@ public class Server {
            udpServer.option(EpollChannelOption.SO_REUSEPORT, true);
            udpChannelHandleList.add(udpServer.bind(udpIp, udpPort).sync().channel());
        }
-        registerService();
+        registerService(curatorFramework);
        shutdownHook();
        LOG.info(
                "Server started. Listening on {}:{} for TCP traffic, {}:{} for HTTP traffic, {}:{} for WebSocket traffic, and {}:{} for UDP traffic",
@@ -515,7 +584,7 @@ public class Server {
        if (generate) {
            LOG.warn("Using generated self signed server certificate");
            Date begin = new Date();
-            Date end = new Date(begin.getTime() + 86400000);
+            Date end = new Date(begin.getTime() + TimeUnit.DAYS.toMillis(7));
            SelfSignedCertificate ssc = new SelfSignedCertificate("localhost", begin, end);
            ssl = SslContextBuilder.forServer(ssc.certificate(), ssc.privateKey());
        } else {
@@ -560,14 +629,14 @@ public class Server {
                ch.pipeline().addLast("decoder", new HttpRequestDecoder());
                ch.pipeline().addLast("compressor", new HttpContentCompressor());
                ch.pipeline().addLast("decompressor", new HttpContentDecompressor());
-                ch.pipeline().addLast("aggregator", new HttpObjectAggregator(8192));
+                ch.pipeline().addLast("aggregator", new HttpObjectAggregator(65536));
                ch.pipeline().addLast("chunker", new ChunkedWriteHandler());
                final Cors corsCfg = config.getHttp().getCors();
-                final CorsConfig.Builder ccb;
+                final CorsConfigBuilder ccb;
                if (corsCfg.isAllowAnyOrigin()) {
-                    ccb = new CorsConfig.Builder();
+                    ccb = CorsConfigBuilder.forAnyOrigin();
                } else {
-                    ccb = new CorsConfig.Builder(corsCfg.getAllowedOrigins().stream().toArray(String[]::new));
+                    ccb = CorsConfigBuilder.forOrigins(corsCfg.getAllowedOrigins().stream().toArray(String[]::new));
                }
                if (corsCfg.isAllowNullOrigin()) {
                    ccb.allowNullOrigin();
@@ -582,7 +651,8 @@ public class Server {
                ch.pipeline().addLast("cors", new CorsHandler(cors));
                ch.pipeline().addLast("queryDecoder",
                        new timely.netty.http.HttpRequestDecoder(config.getSecurity(), config.getHttp()));
-                ch.pipeline().addLast("fileServer", new HttpStaticFileServerHandler());
+                ch.pipeline().addLast("fileServer", new HttpStaticFileServerHandler()
+                        .setIgnoreSslHandshakeErrors(config.getSecurity().getServerSsl().isUseGeneratedKeypair()));
                ch.pipeline().addLast("strict", new StrictTransportHandler(config));
                ch.pipeline().addLast("login", new X509LoginRequestHandler(config.getSecurity(), config.getHttp()));
                ch.pipeline().addLast("doLogin",
@@ -595,7 +665,9 @@ public class Server {
                ch.pipeline().addLast("version", new HttpVersionRequestHandler());
                ch.pipeline().addLast("cache", new HttpCacheRequestHandler(dataStoreCache));
                ch.pipeline().addLast("put", new HttpMetricPutHandler(dataStore));
-                ch.pipeline().addLast("error", new TimelyExceptionHandler());
+                ch.pipeline().addLast("error", new TimelyExceptionHandler()
+                        .setIgnoreSslHandshakeErrors(config.getSecurity().getServerSsl().isUseGeneratedKeypair()));
+
            }
        };
    }
@@ -608,7 +680,7 @@ public class Server {
                ch.pipeline().addLast("logger", new LoggingHandler());
                ch.pipeline().addLast("packetDecoder", new UdpPacketToByteBuf());
                ch.pipeline().addLast("buffer", new MetricsBufferDecoder());
-                ch.pipeline().addLast("frame", new DelimiterBasedFrameDecoder(8192, true, Delimiters.lineDelimiter()));
+                ch.pipeline().addLast("frame", new DelimiterBasedFrameDecoder(65536, true, Delimiters.lineDelimiter()));
                ch.pipeline().addLast("putDecoder", new UdpDecoder());
                ch.pipeline().addLast("putHandler", new TcpPutHandler(dataStore));
            }
@@ -621,7 +693,7 @@ public class Server {
            @Override
            protected void initChannel(SocketChannel ch) throws Exception {
                ch.pipeline().addLast("buffer", new MetricsBufferDecoder());
-                ch.pipeline().addLast("frame", new DelimiterBasedFrameDecoder(8192, true, Delimiters.lineDelimiter()));
+                ch.pipeline().addLast("frame", new DelimiterBasedFrameDecoder(65536, true, Delimiters.lineDelimiter()));
                ch.pipeline().addLast("putDecoder", new TcpDecoder());
                ch.pipeline().addLast("putHandler", new TcpPutHandler(dataStore));
                ch.pipeline().addLast("versionHandler", new TcpVersionHandler());
@@ -636,11 +708,11 @@ public class Server {
            protected void initChannel(SocketChannel ch) throws Exception {
                ch.pipeline().addLast("ssl", sslCtx.newHandler(ch.alloc()));
                ch.pipeline().addLast("httpServer", new HttpServerCodec());
-                ch.pipeline().addLast("aggregator", new HttpObjectAggregator(8192));
-                ch.pipeline().addLast("sessionExtractor", new WebSocketHttpCookieHandler(config.getSecurity()));
+                ch.pipeline().addLast("aggregator", new HttpObjectAggregator(65536));
+                ch.pipeline().addLast("sessionExtractor", new WebSocketFullRequestHandler());
                ch.pipeline().addLast("idle-handler", new IdleStateHandler(conf.getWebsocket().getTimeout(), 0, 0));
                ch.pipeline().addLast("ws-protocol",
-                        new WebSocketServerProtocolHandler(WS_PATH, null, true, 65536, true));
+                        new WebSocketServerProtocolHandler(WS_PATH, null, true, 65536, false, true));
                ch.pipeline().addLast("wsDecoder", new WebSocketRequestDecoder(config.getSecurity()));
                ch.pipeline().addLast("aggregators", new WSAggregatorsRequestHandler());
                ch.pipeline().addLast("metrics", new WSMetricsRequestHandler(config));
@@ -649,7 +721,8 @@ public class Server {
                ch.pipeline().addLast("suggest", new WSSuggestRequestHandler(dataStore));
                ch.pipeline().addLast("version", new WSVersionRequestHandler());
                ch.pipeline().addLast("put", new WSMetricPutHandler(dataStore));
-                ch.pipeline().addLast("create", new WSCreateSubscriptionRequestHandler(dataStore, config));
+                ch.pipeline().addLast("create",
+                        new WSCreateSubscriptionRequestHandler(dataStore, dataStoreCache, config));
                ch.pipeline().addLast("add", new WSAddSubscriptionRequestHandler());
                ch.pipeline().addLast("remove", new WSRemoveSubscriptionRequestHandler());
                ch.pipeline().addLast("close", new WSCloseSubscriptionRequestHandler());
Author	SHA1	Message	Date
Bill Oley	3f4ce0623d	Refactor BalancedMetricResolver to allow HDFS and FILE backing, add callbacks for major events, begin working on tests Refactor BalancedMetricResolver, allow FILE or HDFS persister, add callbacks for events, begin working on testing Fixes	2020-04-16 16:32:55 -04:00
Bill Oley	2aedf9c2e5	closes #199 Fix issues when using a generated (self-signed) server certificate	2020-04-14 18:18:32 -04:00
Bill Oley	9ef8758dfb	closes #198 Cleaner shutdown of resources for Timely server, balancer, grafana-auth	2020-04-14 18:18:27 -04:00
Bill Oley	489bfcc541	closes #197 Fix scan range in MetaCacheImpl	2020-04-14 17:03:14 -04:00
Bill Oley	cb2679d40e	closes #200 Initialize non-cached metrics stored in zookeeper	2020-04-14 17:03:14 -04:00
Bill Oley	cb02950919	Refactor BalancedMetricResolver, allow FILE or HDFS persister, add callbacks for events, begin working on testing	2020-03-28 14:21:55 -04:00
Bill Oley	71cdaec2e4	Refactor BalancedMetricResolver to allow HDFS and FILE backing, add callbacks for major events, begin working on tests	2020-03-28 14:19:53 -04:00
Bill Oley	c19e1d8b2f	Set version to 1.2.8-SNAPSHOT	2020-02-21 10:20:48 -05:00
Bill Oley	cb73e8e75c	Tag version 1.2.7	2020-02-21 10:02:31 -05:00
Bill Oley	4c508c1a7d	Merge pull request #195 from dtspence/TIMELY-194 Fixed #194, corrected datasource.js to look at wildcard tag value	2020-02-21 08:55:37 -05:00
dtspence	c6c2c0d41e	Fixed #194 , corrected datasource.js to look at wildcard tag value The alignment process to match tag query results with the metric definition looks at the tag k/v pairs and makes a special consideration for a wildcard tag value. The change adjusts datasource.js so that the '.*' notation will match a query result and metric definition.	2020-02-05 19:56:03 +00:00
Christopher Tubbs	a24cef78b3	Update jackson (multiple CVEs) (#193 )	2019-11-07 23:05:38 -05:00
Bill Oley	08f99f6540	Set version to 1.2.7-SNAPSHOT	2019-10-30 12:33:49 -04:00
Bill Oley	be11da7d15	Tag version 1.2.6	2019-10-30 12:30:33 -04:00
Bill Oley	4bc5753328	fixes #192 Thread deadlock in BalancedMetricResolver	2019-10-29 12:24:10 -04:00
Bill Oley	2ccb89a06d	re #191 Correct typos	2019-10-18 09:16:03 -04:00
Bill Oley	dfc3840ad4	re #191 Delete cached metrics when they are no longer being updated	2019-10-17 17:49:24 -04:00
Bill Oley	6235034d8b	BalancedMetricResolver in the non-leader balancer should read assignments at intervals	2019-10-17 17:49:24 -04:00
Bill Oley	d8de6ac06f	Make zookeeper timeout configurable	2019-10-17 17:49:24 -04:00
Christopher Tubbs	f381946abe	Merge pull request #190 from NationalSecurityAgency/dependabot/maven/io.netty-netty-all-4.1.42.Final Bump netty-all from 4.1.35.Final to 4.1.42.Final	2019-10-17 09:35:13 -07:00
dependabot[bot]	48bbe6a561	Bump netty-all from 4.1.35.Final to 4.1.42.Final Bumps [netty-all](https://github.com/netty/netty) from 4.1.35.Final to 4.1.42.Final. - [Release notes](https://github.com/netty/netty/releases) - [Commits](https://github.com/netty/netty/compare/netty-4.1.35.Final...netty-4.1.42.Final) Signed-off-by: dependabot[bot] <[email protected]>	2019-10-11 18:58:57 +00:00
Christopher Tubbs	0f6110ea59	Update jackson to 2.9.10 * Fix CVE-2019-16335 * Fix CVE-2019-14540	2019-09-26 23:40:08 -04:00
Bill Oley	461b5e7e78	Pom version to 1.2.6-SNAPSHOT	2019-08-22 12:17:45 -04:00
Bill Oley	4ec7d94c28	Tag version 1.2.5	2019-08-22 12:12:49 -04:00
Bill Oley	77e910ae06	Use BufferedOutputWriter instead of PrintWriter, pass Exceptions instead of having to check error condition	2019-08-22 10:09:27 -04:00
Bill Oley	a4998af887	Move jackson-databind version to the properties section	2019-08-08 13:50:08 -04:00
Bill Oley	2858549ede	Merge pull request #188 from ctubbsii/jdk11-build-support Bump versions / permit build with JDK11	2019-08-08 13:35:54 -04:00
Christopher Tubbs	57f0e1408f	Bump versions / permit build with JDK11 Support builds on newer JDKs * Update tycho-compiler-jdt in analytics/pom.xml * Ignore spotbugs RCN_REDUNDANT_NULLCHECK_WOULD_HAVE_BEEN_A_NPE for JDK11 * Bump several plugins to latest (maven-compiler-plugin, etc.) * Add profile for using -release flag in Eclipse and with JDK9 and later Fix recently added warnings * Add try-with-resources for TreeCache objects to remove recent warnings * Suppress generics warning in MetricCompactionStrategy * Update TabletMetadataConsole to avoid deprecated Accumulo ClientConfiguration API Other * Bump jackson-databind for CVE-2019-14439, CVE-2019-14379 * Bump hadoop 2.7 to latest in series (2.7.3->2.7.7) for build	2019-08-06 16:14:13 -04:00
Bill Oley	151adc025a	Merge pull request #185 from dtspence/TIMELY-184 Fixes #184 Metric compaction strategy and summary implementation	2019-08-06 15:15:45 -04:00
Bill Oley	5abd9d7200	closes #187 Prevent deadlock in balancer distributed read/write lock	2019-08-06 12:00:57 -04:00
Bill Oley	fa2e211c72	closes #186 Make balancer websocket incomingBufferSize its own setting	2019-08-06 11:47:36 -04:00
dtspence	a21438976e	TIMELY-184: Metric compaction strategy and summary implementation.	2019-08-01 14:05:35 +00:00
Christopher Tubbs	ba37047f3a	Update jackson-databind (CVE-2019-12814)	2019-07-23 20:38:46 -04:00
Bill Oley	9305470c69	re #183 Check tagKey not null or blank in Meta.toKeys	2019-06-19 09:08:20 -04:00
Bill Oley	200233542e	re #183 Add timestamps to meta tags to allow proper age off	2019-06-17 15:36:51 -04:00
Bill Oley	3cd78dc624	Merge branch 'drewfarris-config-fixup'	2019-05-30 12:55:35 -04:00
Bill Oley	9027cac6e3	Merge branch 'config-fixup' of https://github.com/drewfarris/timely into drewfarris-config-fixup Conflicts: server/conf/timely-docker.yml server/conf/timely-standalone.yml server/conf/timely.yml	2019-05-30 12:50:31 -04:00
Bill Oley	3663b4e6d5	Pom version to 1.2.5-SNAPSHOT	2019-05-30 12:38:42 -04:00
Bill Oley	64973d0674	Merge pull request #182 from ctubbsii/fix-jackson Fix most compiler warnings and many other issues	2019-05-30 12:25:00 -04:00
Christopher Tubbs	5512306f1e	Fix most compiler warnings and other issues Build: * Fix netty-tcnative-boringssl-static version specs in pom.xml * Bump the version of accumulo to latest release (1.9.3) * Update jackson to 2.9.9 (CVE-2019-12086) Warnings: * Remove all use of deprecated netty stuffs and use their replacements in the netty APIs * Add missing generics parameters everywhere * Remove unused code (unused private fields, such as loggers) * Remove standard javadoc tags that were missing descriptions (these do not add value to the javadocs, since default `@param`, `@throws`, and `@return` tags are provided by javadoc) * Add missing serialVersionUid fields to Serializable classes * Fix a resource leak warning about not closing ScannerOptionsHelper (NOTE: 3 other resource leak warnings exist, but did not attempt to fix those, since they were non-trivial) * Remove deprecated annotations from hibernate and replace with their non-deprecated counterparts from javax.validation.constraints * Remove use of deprecated Accumulo ClientConfiguration constructor Other: * Use lowercase header names, for HTTP/2 compatibility (and to stop incorrectly assuming case-sensitivity for HTTP/1) * Narrow the scope of necessary `@SuppressWarnings("unchecked")` occurrences to a variable instead of suppressing for an entire method or an entire class * IDE added some missing overrides annotations * (style consistency) Replace tabs with spaces * Set log4j2 root logger level for tests to WARN instead of TRACE	2019-05-24 12:12:00 -04:00
Bill Oley	d35ae08188	Tag version 1.2.4	2019-05-22 17:14:42 -04:00
Bill Oley	e448657afb	Correction in websocket pipeline from Netty API change	2019-05-22 16:49:11 -04:00
Bill Oley	1dcd893711	Add age off iterator for meta table Add age off iterator for meta table	2019-05-22 16:49:08 -04:00
Bill Oley	409f5375d1	Change pom version to 1.2.4-SNAPSHOT	2019-05-22 16:48:09 -04:00
Bill Oley	c5df8139ca	Correct typos	2019-05-22 16:46:55 -04:00
Bill Oley	dcd23d4fc8	fixes #181 Use netty-tcnative-boringssl-static	2019-04-18 11:56:54 -04:00
Bill Oley	cc7326fc19	Tag version 1.2.2	2019-04-12 11:24:53 -04:00
Bill Oley	0d2b6e61c8	Correct typos	2019-04-12 10:55:04 -04:00
Bill Oley	b7cd8765c5	Correct typos and minor changed to MetaCache	2019-04-10 15:31:53 -04:00
Bill Oley	a9d27d5b1b	re #179 Allow AuthCache to periodically refresh credentials, cache meta table up to a configurable number of tags	2019-04-08 10:50:14 -04:00
Bill Oley	7d67eb218d	Update netty-tcnative dependency, remove x509 principal extractor from AuthenticationService	2019-04-03 15:52:19 -04:00
Bill Oley	5fb7e3dd2b	Bump pom versions to 1.2.2-SNAPSHOT	2019-04-02 12:57:22 -04:00
Bill Oley	5a89861cec	Tag version 1.2.1	2019-04-02 12:53:04 -04:00
Bill Oley	a3bc9a01ee	Update dependency versions Update dependencies Update dependency versions	2019-04-02 12:43:36 -04:00
Bill Oley	dba494b06d	Bump pom versions to 1.2.1-SNAPSHOT	2019-02-25 15:41:31 -05:00
Bill Oley	5826884c3c	Tag version 1.2.0	2019-02-25 15:11:57 -05:00
Bill Oley	f94c731bbc	Remove unused modeule directory introduced in error	2019-02-25 15:11:12 -05:00
Bill Oley	4e042bcb94	Allow all http methods in grafana auth proxy, disable cookies in HttpPools	2019-02-22 16:17:36 -05:00
Bill Oley	ab7f499134	closes #180 Allow a configurable default visibility for metrics	2019-02-21 15:33:25 -05:00
Bill Oley	c098f9f3e4	re #179 Create an authentication app for grafana that uses Timely's authentication framework and grafana's auth-proxy re #179 Create authentication proxy for grafana	2019-02-21 13:28:12 -05:00
Bill Oley	94cfcbc0c4	Allow authentication provider to return roles, check users against requiredRoles and requiredAuths for access index on master: `674e1b0` Improve synchronization in AuthenticationService Check validated users against requiredRoles and requiredAuths Conflicts: server/conf/security.xml server/src/main/java/timely/auth/AuthenticationService.java More work	2019-02-15 11:24:07 -05:00
Bill Oley	674e1b0ab5	Improve synchronization in AuthenticationService	2019-02-08 16:53:35 -05:00
Bill Oley	4eedb84547	Add synchronization to AuthenticationService to prevent simultaneous authentications of the same user by multiple threads	2019-02-07 17:17:08 -05:00
Bill Oley	bef86eac37	Fix websocket proxying with a client certificate, add ability to configure client cert in python websocket client	2019-02-07 17:16:00 -05:00
Bill Oley	27bb16e403	re #178 Allow proxying of http and ws requests More work Correct errors	2019-02-05 11:41:35 -05:00
Drew Farris	e2bbe4e212	Minor fix-ups to default configuration files. . Replace ssl with server-ssl in timely-standalone.yml and timely-docker.yml . Add a value for cache.enabled in timely.yml	2019-01-19 22:07:07 -05:00
Bill Oley	a95d2d7ea1	Bump pom versions to 1.1.1-SNAPSHOT	2019-01-16 13:55:47 -05:00
Bill Oley	f751175190	Update version of collectd	2019-01-16 13:53:43 -05:00
Bill Oley	648199f330	Revert "Bump pom versions to 1.1.1-SNAPSHOT" This reverts commit `fcfc3f7bc0`.	2019-01-16 13:53:26 -05:00
Bill Oley	fcfc3f7bc0	Bump pom versions to 1.1.1-SNAPSHOT	2019-01-15 15:48:43 -05:00
Bill Oley	200bc39902	Tag version 1.1.0 Grafana app changes	2019-01-15 15:47:14 -05:00
Bill Oley	f854c9e394	Add build profile for python client package	2019-01-15 15:08:30 -05:00
Bill Oley	73ddae1f8d	Correct typo	2019-01-15 14:15:20 -05:00
Bill Oley	cb7184503e	Allow Websocket requests to take advantage of cached metrics	2019-01-11 16:12:26 -05:00
Christopher Tubbs	3a357b50ef	Update jackson-databind dependency (Also, remove redundant test dependency on timely-common.)	2019-01-10 16:07:25 -05:00
Bill Oley	689aee0678	Fix deadlock in tracking nonCachedMetrics across processes	2019-01-08 12:06:41 -05:00
Bill Oley	32f6b9d4c9	Include the bin/testing directory in assembly	2019-01-04 11:14:11 -05:00
Bill Oley	512f318387	Add distributed tracking of nonCachedMetrics between the timely servers and the balancers work in progress Add distributed tracking of noncachedMetrics	2019-01-04 11:04:24 -05:00