Refactor BalancedMetricResolver to allow HDFS and FILE backing, add callbacks for major events, begin working on tests

Refactor BalancedMetricResolver, allow FILE or HDFS persister, add callbacks for events, begin working on testing

Fixes

balancer/pom.xml

@@ -108,6 +108,10 @@
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-minicluster</artifactId>
     </dependency>
+    <dependency>
+      <groupId>org.glassfish.tyrus</groupId>
+      <artifactId>tyrus-client</artifactId>
+    </dependency>
     <dependency>
       <groupId>org.slf4j</groupId>
       <artifactId>slf4j-api</artifactId>

balancer/src/main/java/timely/balancer/Balancer.java

@@ -77,7 +77,7 @@ import timely.balancer.connection.http.HttpClientPool;
 import timely.balancer.connection.tcp.TcpClientPool;
 import timely.balancer.connection.udp.UdpClientPool;
 import timely.balancer.connection.ws.WsClientPool;
-import timely.balancer.healthcheck.HealthChecker;
+import timely.balancer.healthcheck.TcpHealthChecker;
 import timely.balancer.netty.http.HttpRelayHandler;
 import timely.balancer.netty.tcp.TcpRelayHandler;
 import timely.balancer.netty.udp.UdpRelayHandler;
@@ -257,7 +257,7 @@ public class Balancer {
         if (this.metricResolver != null) {
             try {
                 LOG.info("Closing metricResolver");
-                this.metricResolver.close();
+                this.metricResolver.stop();
             } catch (Exception e) {
                 LOG.error(e.getMessage(), e);
             }
@@ -417,8 +417,9 @@ public class Balancer {
         for (int x = 0; x < balancerConfig.getServer().getNumTcpPools(); x++) {
             tcpClientPools.add(new TcpClientPool(this.balancerConfig));
         }
-        HealthChecker healthChecker = new HealthChecker(this.balancerConfig, tcpClientPools.get(0));
+        TcpHealthChecker healthChecker = new TcpHealthChecker(this.balancerConfig, tcpClientPools.get(0));
         this.metricResolver = new BalancedMetricResolver(curatorFramework, this.balancerConfig, healthChecker);
+        this.metricResolver.start();
 
         final ServerBootstrap tcpServer = new ServerBootstrap();
         tcpServer.group(tcpBossGroup, tcpWorkerGroup);

balancer/src/main/java/timely/balancer/configuration/BalancerConfiguration.java

@@ -49,6 +49,16 @@ public class BalancerConfiguration {
 
     private int serverSuccessesBeforeUp = 3;
 
+    private String metricAssignmentPersisterType = "HDFS";
+
+    private long balanceDelay = 900000;
+
+    private long balancePeriod = 900000;
+
+    private long persistDelay = 10000;
+
+    private long persistPeriod = 60000;
+
     public ZooKeeper getZooKeeper() {
         return zooKeeper;
     }
@@ -120,4 +130,44 @@ public class BalancerConfiguration {
     public void setServerSuccessesBeforeUp(int serverSuccessesBeforeUp) {
         this.serverSuccessesBeforeUp = serverSuccessesBeforeUp;
     }
+
+    public String getMetricAssignmentPersisterType() {
+        return metricAssignmentPersisterType;
+    }
+
+    public void setMetricAssignmentPersisterType(String metricAssignmentPersisterType) {
+        this.metricAssignmentPersisterType = metricAssignmentPersisterType;
+    }
+
+    public void setBalanceDelay(long balanceDelay) {
+        this.balanceDelay = balanceDelay;
+    }
+
+    public long getBalanceDelay() {
+        return balanceDelay;
+    }
+
+    public void setBalancePeriod(long balancePeriod) {
+        this.balancePeriod = balancePeriod;
+    }
+
+    public long getBalancePeriod() {
+        return balancePeriod;
+    }
+
+    public void setPersistDelay(long persistDelay) {
+        this.persistDelay = persistDelay;
+    }
+
+    public long getPersistDelay() {
+        return persistDelay;
+    }
+
+    public void setPersistPeriod(long persistPeriod) {
+        this.persistPeriod = persistPeriod;
+    }
+
+    public long getPersistPeriod() {
+        return persistPeriod;
+    }
 }

balancer/src/main/java/timely/balancer/connection/TimelyBalancedHost.java

@@ -42,6 +42,10 @@ public class TimelyBalancedHost {
         return new TimelyBalancedHost(host, tcpPort, httpPort, wsPort, udpPort, arrivalRate);
     }
 
+    static public TimelyBalancedHost of(String host, int basePort) {
+        return new TimelyBalancedHost(host, basePort, basePort + 1, basePort + 2, basePort + 3, null);
+    }
+
     public void setBalancerConfig(BalancerConfiguration balancerConfig) {
         this.balancerConfig = balancerConfig;
     }
@@ -180,4 +184,11 @@ public class TimelyBalancedHost {
         builder.append(isUp);
         return builder.toString();
     }
+
+    public static String toStringShort(TimelyBalancedHost tbh) {
+        if (tbh == null) {
+            return null;
+        }
+        return tbh.host + ":" + tbh.tcpPort;
+    }
 }

balancer/src/main/java/timely/balancer/healthcheck/HealthChecker.java

@@ -1,61 +1,12 @@
 package timely.balancer.healthcheck;
 
-import java.util.ArrayList;
 import java.util.List;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
 
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import timely.balancer.configuration.BalancerConfiguration;
 import timely.balancer.connection.TimelyBalancedHost;
-import timely.balancer.connection.tcp.TcpClientPool;
 
-public class HealthChecker {
+public interface HealthChecker {
 
-    private static final Logger LOG = LoggerFactory.getLogger(HealthChecker.class);
-    private List<TimelyBalancedHost> timelyHosts = new ArrayList<>();
-    private ScheduledExecutorService executorService = Executors.newScheduledThreadPool(1);
-    final private TCPServerHealthCheck tcpServerHealthCheck;
+    public void setTimelyHosts(List<TimelyBalancedHost> timelyHosts);
 
-    public HealthChecker(BalancerConfiguration config, TcpClientPool tcpClientPool) {
-        this.tcpServerHealthCheck = new TCPServerHealthCheck(tcpClientPool);
-        this.executorService.scheduleAtFixedRate(() -> {
-            synchronized (timelyHosts) {
-                for (TimelyBalancedHost h : timelyHosts) {
-                    try {
-                        if (tcpServerHealthCheck.isServerHealthy(h)) {
-                            h.reportSuccess();
-                        } else {
-                            h.reportFailure();
-                        }
-                    } catch (Exception e) {
-                        LOG.error(e.getMessage(), e);
-                        h.reportFailure();
-                    }
-                }
-            }
-        }, 0, config.getCheckServerHealthInterval(), TimeUnit.MILLISECONDS);
-    }
-
-    public void setTimelyHosts(List<TimelyBalancedHost> timelyHosts) {
-        synchronized (timelyHosts) {
-            this.timelyHosts.clear();
-            this.timelyHosts.addAll(timelyHosts);
-        }
-    }
-
-    public void close() throws Exception {
-        this.executorService.shutdown();
-        try {
-            this.executorService.awaitTermination(5, TimeUnit.SECONDS);
-        } catch (InterruptedException e) {
-
-        } finally {
-            if (!this.executorService.isTerminated()) {
-                this.executorService.shutdownNow();
-            }
-        }
-    }
+    public void close() throws Exception;
 }

balancer/src/main/java/timely/balancer/healthcheck/TcpHealthChecker.java

@@ -0,0 +1,62 @@
+package timely.balancer.healthcheck;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import timely.balancer.configuration.BalancerConfiguration;
+import timely.balancer.connection.TimelyBalancedHost;
+import timely.balancer.connection.tcp.TcpClientPool;
+
+public class TcpHealthChecker implements HealthChecker {
+
+    private static final Logger LOG = LoggerFactory.getLogger(TcpHealthChecker.class);
+    private List<TimelyBalancedHost> timelyHosts = new ArrayList<>();
+    private ScheduledExecutorService executorService = Executors.newScheduledThreadPool(1);
+    final private TCPServerHealthCheck tcpServerHealthCheck;
+
+    public TcpHealthChecker(BalancerConfiguration config, TcpClientPool tcpClientPool) {
+        this.tcpServerHealthCheck = new TCPServerHealthCheck(tcpClientPool);
+        this.executorService.scheduleAtFixedRate(() -> {
+            synchronized (timelyHosts) {
+                for (TimelyBalancedHost h : timelyHosts) {
+                    try {
+                        if (tcpServerHealthCheck.isServerHealthy(h)) {
+                            h.reportSuccess();
+                        } else {
+                            h.reportFailure();
+                        }
+                    } catch (Exception e) {
+                        LOG.error(e.getMessage(), e);
+                        h.reportFailure();
+                    }
+                }
+            }
+        }, 0, config.getCheckServerHealthInterval(), TimeUnit.MILLISECONDS);
+    }
+
+    @Override
+    public void setTimelyHosts(List<TimelyBalancedHost> timelyHosts) {
+        synchronized (timelyHosts) {
+            this.timelyHosts.clear();
+            this.timelyHosts.addAll(timelyHosts);
+        }
+    }
+
+    public void close() throws Exception {
+        this.executorService.shutdown();
+        try {
+            this.executorService.awaitTermination(5, TimeUnit.SECONDS);
+        } catch (InterruptedException e) {
+
+        } finally {
+            if (!this.executorService.isTerminated()) {
+                this.executorService.shutdownNow();
+            }
+        }
+    }
+}

balancer/src/main/java/timely/balancer/resolver/BalancedMetricResolver.java

@@ -4,11 +4,14 @@ import static timely.Server.SERVICE_DISCOVERY_PATH;
 import static timely.balancer.Balancer.ASSIGNMENTS_LAST_UPDATED_PATH;
 import static timely.balancer.Balancer.ASSIGNMENTS_LOCK_PATH;
 import static timely.balancer.Balancer.LEADER_LATCH_PATH;
+import static timely.balancer.resolver.eventing.MetricAssignedEvent.Reason.ASSIGN_FALLBACK_SEQUENTIAL;
+import static timely.balancer.resolver.eventing.MetricAssignedEvent.Reason.ASSIGN_ROUND_ROBIN;
+import static timely.balancer.resolver.eventing.MetricAssignedEvent.Reason.HOST_DOWN_ROUND_ROBIN;
+import static timely.balancer.resolver.eventing.MetricAssignedEvent.Reason.HOST_REMOVED;
+import static timely.balancer.resolver.eventing.MetricAssignedEvent.Reason.REBALANCE;
 import static timely.store.cache.DataStoreCache.NON_CACHED_METRICS;
 import static timely.store.cache.DataStoreCache.NON_CACHED_METRICS_LOCK_PATH;
 
-import java.io.FileNotFoundException;
-import java.nio.charset.Charset;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -29,12 +32,9 @@ import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 
-import com.csvreader.CsvReader;
-import com.csvreader.CsvWriter;
 import org.apache.commons.lang.StringUtils;
 import org.apache.commons.lang3.SerializationUtils;
 import org.apache.curator.framework.CuratorFramework;
-import org.apache.curator.framework.recipes.atomic.DistributedAtomicLong;
 import org.apache.curator.framework.recipes.atomic.DistributedAtomicValue;
 import org.apache.curator.framework.recipes.cache.TreeCache;
 import org.apache.curator.framework.recipes.cache.TreeCacheEvent;
@@ -49,11 +49,6 @@ import org.apache.curator.x.discovery.ServiceDiscovery;
 import org.apache.curator.x.discovery.ServiceDiscoveryBuilder;
 import org.apache.curator.x.discovery.ServiceInstance;
 import org.apache.curator.x.discovery.details.ServiceCacheListener;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
 import org.apache.zookeeper.CreateMode;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -62,16 +57,23 @@ import timely.balancer.ArrivalRate;
 import timely.balancer.configuration.BalancerConfiguration;
 import timely.balancer.connection.TimelyBalancedHost;
 import timely.balancer.healthcheck.HealthChecker;
+import timely.balancer.resolver.eventing.MetricAssignedCallback;
+import timely.balancer.resolver.eventing.MetricAssignedEvent;
+import timely.balancer.resolver.eventing.MetricBalanceCallback;
+import timely.balancer.resolver.eventing.MetricBalanceEvent;
+import timely.balancer.resolver.eventing.MetricHostCallback;
+import timely.balancer.resolver.eventing.MetricHostEvent;
 
 public class BalancedMetricResolver implements MetricResolver {
 
     private static final Logger LOG = LoggerFactory.getLogger(BalancedMetricResolver.class);
     private Map<String, TimelyBalancedHost> metricToHostMap = new TreeMap<>();
+    private Set<String> unassignedMetrics = new HashSet<>();
     private Map<String, ArrivalRate> metricMap = new HashMap<>();
     private ReentrantReadWriteLock balancerLock = new ReentrantReadWriteLock();
     private List<TimelyBalancedHost> serverList = new ArrayList<>();
     private Random r = new Random();
-    final private HealthChecker healthChecker;
+    private final HealthChecker healthChecker;
     private ScheduledExecutorService assignmentExecutor = Executors.newScheduledThreadPool(2);
     private ScheduledExecutorService arrivalRateExecutor = Executors.newScheduledThreadPool(2);
     private int roundRobinCounter = 0;
@@ -79,38 +81,46 @@ public class BalancedMetricResolver implements MetricResolver {
     private DistributedAtomicValue nonCachedMetricsIP;
     private ReentrantReadWriteLock nonCachedMetricsLocalLock = new ReentrantReadWriteLock();
     private InterProcessReadWriteLock nonCachedMetricsIPRWLock;
-    private BalancerConfiguration balancerConfig;
+    private AtomicLong assignmentsLastUpdatedLocal = new AtomicLong(0);
     private LeaderLatch leaderLatch;
     private AtomicBoolean isLeader = new AtomicBoolean(false);
     private InterProcessReadWriteLock assignmentsIPRWLock;
-    private DistributedAtomicLong assignmentsLastUpdatedInHdfs;
-    private AtomicLong assignmentsLastUpdatedLocal = new AtomicLong(0);
+    private MetricAssignmentPerister assignmentPerister;
 
-    private enum BalanceType {
-        HIGH_LOW, HIGH_AVG, AVG_LOW
+    final private BalancerConfiguration balancerConfig;
+    final private CuratorFramework curatorFramework;
+
+    private List<MetricAssignedCallback> metricAssignedCallbacks = new ArrayList<>();
+    private List<MetricBalanceCallback> metricBalanceCallbacks = new ArrayList<>();
+    private List<MetricHostCallback> metricHostCallbacks = new ArrayList<>();
+
+    public enum BalanceType {
+        HIGH_LOW, HIGH_AVG, AVG_LOW;
     }
 
     public BalancedMetricResolver(CuratorFramework curatorFramework, BalancerConfiguration balancerConfig,
             HealthChecker healthChecker) {
+        this.curatorFramework = curatorFramework;
         this.balancerConfig = balancerConfig;
         this.healthChecker = healthChecker;
+    }
 
+    public void start() {
         assignmentsIPRWLock = new InterProcessReadWriteLock(curatorFramework, ASSIGNMENTS_LOCK_PATH);
         testIPRWLock(curatorFramework, assignmentsIPRWLock, ASSIGNMENTS_LOCK_PATH);
+
+        this.assignmentPerister = new MetricAssignmentPerister(this, balancerConfig, metricToHostMap, nonCachedMetrics,
+                curatorFramework, assignmentsIPRWLock, nonCachedMetricsLocalLock, assignmentsLastUpdatedLocal,
+                balancerLock);
+
         startLeaderLatch(curatorFramework);
         startServiceListener(curatorFramework);
-        assignmentsLastUpdatedInHdfs = new DistributedAtomicLong(curatorFramework, ASSIGNMENTS_LAST_UPDATED_PATH,
-                new RetryForever(1000));
 
-        TreeCacheListener assignmentListener = new TreeCacheListener() {
-
-            @Override
-            public void childEvent(CuratorFramework curatorFramework, TreeCacheEvent event) throws Exception {
-                LOG.debug("Handling assignments event {}. assignmentsLastUpdatedInHdfs:{}", event.getType().toString(),
-                        new Date(assignmentsLastUpdatedInHdfs.get().postValue()));
-                if (event.getType().equals(TreeCacheEvent.Type.NODE_UPDATED)) {
-                    readAssignmentsFromHdfs(true);
-                }
+        TreeCacheListener assignmentListener = (curatorFramework1, event) -> {
+            LOG.debug("Handling assignments event {}. assignmentsLastUpdated:{}", event.getType().toString(),
+                    new Date(assignmentPerister.getAssignmentsLastUpdatedByLeader()));
+            if (event.getType().equals(TreeCacheEvent.Type.NODE_UPDATED)) {
+                assignmentPerister.readAssignments(true);
             }
         };
 
@@ -150,7 +160,7 @@ public class BalancedMetricResolver implements MetricResolver {
         }
         readNonCachedMetricsIP();
 
-        readAssignmentsFromHdfs(false);
+        assignmentPerister.readAssignments(false);
 
         assignmentExecutor.scheduleAtFixedRate(() -> {
             if (isLeader.get()) {
@@ -160,28 +170,28 @@ public class BalancedMetricResolver implements MetricResolver {
                     LOG.error(e.getMessage(), e);
                 }
             }
-        }, 15, 15, TimeUnit.MINUTES);
+        }, balancerConfig.getBalanceDelay(), balancerConfig.getBalancePeriod(), TimeUnit.MILLISECONDS);
 
         assignmentExecutor.scheduleAtFixedRate(() -> {
             try {
                 if (isLeader.get()) {
                     long lastLocalUpdate = assignmentsLastUpdatedLocal.get();
-                    long lastHdfsUpdate = assignmentsLastUpdatedInHdfs.get().postValue();
-                    if (lastLocalUpdate > lastHdfsUpdate) {
-                        LOG.debug("Leader writing assignments to hdfs lastLocalUpdate ({}) > lastHdfsUpdate ({})",
-                                new Date(lastLocalUpdate), new Date(lastHdfsUpdate));
-                        writeAssignmentsToHdfs();
+                    long lastUpdateByLeader = assignmentPerister.getAssignmentsLastUpdatedByLeader();
+                    if (lastLocalUpdate > lastUpdateByLeader) {
+                        LOG.debug("Leader writing assignments lastLocalUpdate ({}) > lastUpdateByLeader ({})",
+                                new Date(lastLocalUpdate), new Date(lastUpdateByLeader));
+                        assignmentPerister.writeAssignments();
                     } else {
-                        LOG.trace("Leader not writing assignments to hdfs lastLocalUpdate ({}) <= lastHdfsUpdate ({})",
-                                new Date(lastLocalUpdate), new Date(lastHdfsUpdate));
+                        LOG.trace("Leader not writing assignments lastLocalUpdate ({}) <= lastUpdateByLeader ({})",
+                                new Date(lastLocalUpdate), new Date(lastUpdateByLeader));
                     }
                 } else {
-                    readAssignmentsFromHdfs(true);
+                    assignmentPerister.readAssignments(true);
                 }
             } catch (Exception e) {
                 LOG.error(e.getMessage(), e);
             }
-        }, 10, 60, TimeUnit.SECONDS);
+        }, balancerConfig.getPersistDelay(), balancerConfig.getPersistPeriod(), TimeUnit.MILLISECONDS);
     }
 
     private void readNonCachedMetricsIP() {
@@ -239,7 +249,7 @@ public class BalancedMetricResolver implements MetricResolver {
                 public void isLeader() {
                     LOG.info("this balancer is the leader");
                     isLeader.set(true);
-                    writeAssignmentsToHdfs();
+                    assignmentPerister.writeAssignments();
                 }
 
                 @Override
@@ -266,10 +276,10 @@ public class BalancedMetricResolver implements MetricResolver {
                     ServerDetails pl = si.getPayload();
                     TimelyBalancedHost tbh = TimelyBalancedHost.of(pl.getHost(), pl.getTcpPort(), pl.getHttpPort(),
                             pl.getWsPort(), pl.getUdpPort(), new ArrivalRate(arrivalRateExecutor));
-                    LOG.info("adding service {} host:{} tcpPort:{} httpPort:{} wsPort:{} udpPort:{}", si.getId(),
-                            pl.getHost(), pl.getTcpPort(), pl.getHttpPort(), pl.getWsPort(), pl.getUdpPort());
+                    LOG.info("adding service {} host:{}", si.getId(), TimelyBalancedHost.toStringShort(tbh));
                     tbh.setBalancerConfig(balancerConfig);
                     serverList.add(tbh);
+                    metricHostEvent(tbh, MetricHostEvent.ActionType.ADDED);
                 }
                 healthChecker.setTimelyHosts(serverList);
             } finally {
@@ -305,9 +315,8 @@ public class BalancedMetricResolver implements MetricResolver {
                                 availableHosts.remove(h);
                             } else {
                                 itr.remove();
-                                LOG.info("removing service {}:{} host:{} tcpPort:{} httpPort:{} wsPort:{} udpPort:{}",
-                                        h.getHost(), h.getTcpPort(), h.getHost(), h.getTcpPort(), h.getHttpPort(),
-                                        h.getWsPort(), h.getUdpPort());
+                                metricHostEvent(h, MetricHostEvent.ActionType.REMOVED);
+                                LOG.info("removing service {}", TimelyBalancedHost.toStringShort(h));
                                 for (Map.Entry<String, TimelyBalancedHost> e : metricToHostMap.entrySet()) {
                                     if (e.getValue().equals(h)) {
                                         reassignMetrics.add(e.getKey());
@@ -319,19 +328,22 @@ public class BalancedMetricResolver implements MetricResolver {
 
                         // add new hosts that were not previously known
                         for (TimelyBalancedHost h : availableHosts) {
-                            LOG.info("adding service {}:{} host:{} tcpPort:{} httpPort:{} wsPort:{} udpPort:{}",
-                                    h.getHost(), h.getTcpPort(), h.getHost(), h.getTcpPort(), h.getHttpPort(),
-                                    h.getWsPort(), h.getUdpPort());
+                            LOG.info("adding service {}", TimelyBalancedHost.toStringShort(h));
                             serverList.add(h);
+                            metricHostEvent(h, MetricHostEvent.ActionType.ADDED);
                             rebalanceNeeded = true;
                         }
                         healthChecker.setTimelyHosts(serverList);
-
+                        if (!availableHosts.isEmpty()) {
+                            reassignMetrics.addAll(unassignedMetrics);
+                            unassignedMetrics.clear();
+                        }
                         if (isLeader.get()) {
                             for (String s : reassignMetrics) {
                                 TimelyBalancedHost h = getRoundRobinHost(null);
-                                assignMetric(s, h);
-                                LOG.debug("Assigned server removed.  Assigning {} to server {}:{}", s, h.getTcpPort());
+                                assignMetric(s, h, HOST_REMOVED);
+                                LOG.debug("Assigned server removed.  Assigning {} to server {}", s,
+                                        TimelyBalancedHost.toStringShort(h));
                             }
                         }
 
@@ -342,7 +354,7 @@ public class BalancedMetricResolver implements MetricResolver {
                         if (rebalanceNeeded) {
                             balance();
                         }
-                        writeAssignmentsToHdfs();
+                        assignmentPerister.writeAssignments();
                     }
                 }
 
@@ -402,7 +414,7 @@ public class BalancedMetricResolver implements MetricResolver {
         return tbh;
     }
 
-    private TimelyBalancedHost getRoundRobinHost(TimelyBalancedHost notThisOne) {
+    protected TimelyBalancedHost getRoundRobinHost(TimelyBalancedHost notThisOne) {
         TimelyBalancedHost tbh = null;
         balancerLock.readLock().lock();
         try {
@@ -524,8 +536,10 @@ public class BalancedMetricResolver implements MetricResolver {
 
         int numReassigned = 0;
         if (isLeader.get()) {
+            metricBalanceEvent(MetricBalanceEvent.ProgressType.BEGIN, balanceType, 0);
             balancerLock.writeLock().lock();
             try {
+                metricBalanceEvent(MetricBalanceEvent.ProgressType.BALANCER_LOCK_ACQUIRED, balanceType, 0);
                 Map<String, ArrivalRate> tempMetricMap = new HashMap<>();
                 tempMetricMap.putAll(metricMap);
 
@@ -555,8 +569,8 @@ public class BalancedMetricResolver implements MetricResolver {
                         }
                     }
 
-                    LOG.trace("focusHost {}:{} desiredChange:{} rateSortedMetrics.size():{}", h.getHost(),
-                            h.getTcpPort(), desiredChange, rateSortedMetrics.size());
+                    LOG.trace("focusHost {} desiredChange:{} rateSortedMetrics.size():{}",
+                            TimelyBalancedHost.toStringShort(h), desiredChange, rateSortedMetrics.size());
 
                     boolean doneWithHost = false;
                     while (!doneWithHost) {
@@ -585,7 +599,7 @@ public class BalancedMetricResolver implements MetricResolver {
                                 } else {
                                     String metric = e.getValue();
                                     Double metricRate = e.getKey();
-                                    assignMetric(metric, candidateHost);
+                                    assignMetric(metric, candidateHost, REBALANCE);
                                     numReassigned++;
                                     calculatedRates.put(candidateHost, calculatedRates.get(candidateHost) + metricRate);
                                     calculatedRates.put(h, calculatedRates.get(h) - metricRate);
@@ -594,9 +608,9 @@ public class BalancedMetricResolver implements MetricResolver {
                                     itr.remove();
                                     tempMetricMap.remove(metric);
                                     LOG.info(
-                                            "rebalancing: reassigning metric:{} rate:{} from server {}:{} to {}:{} remaining delta {}",
-                                            metric, metricRate, h.getHost(), h.getTcpPort(), candidateHost.getHost(),
-                                            candidateHost.getTcpPort(), desiredChange);
+                                            "rebalancing: reassigning metric:{} rate:{} from server {} to {} remaining delta {}",
+                                            metric, metricRate, TimelyBalancedHost.toStringShort(h),
+                                            TimelyBalancedHost.toStringShort(candidateHost), desiredChange);
                                 }
                             }
                         }
@@ -619,6 +633,7 @@ public class BalancedMetricResolver implements MetricResolver {
                 }
             } finally {
                 balancerLock.writeLock().unlock();
+                metricBalanceEvent(MetricBalanceEvent.ProgressType.END, balanceType, numReassigned);
             }
         }
         return numReassigned;
@@ -691,7 +706,7 @@ public class BalancedMetricResolver implements MetricResolver {
         }
     }
 
-    private boolean shouldCache(String metricName) {
+    protected boolean shouldCache(String metricName) {
 
         if (StringUtils.isBlank(metricName)) {
             return false;
@@ -753,7 +768,7 @@ public class BalancedMetricResolver implements MetricResolver {
             }
 
             if (StringUtils.isBlank(metric)) {
-                tbh = getRandomHost(null);
+                tbh = getRoundRobinHost(null);
             } else {
                 tbh = metricToHostMap.get(metric);
                 if (tbh == null) {
@@ -762,17 +777,17 @@ public class BalancedMetricResolver implements MetricResolver {
                         balancerLock.readLock().unlock();
                         balancerLock.writeLock().lock();
                     }
-                    assignMetric(metric, tbh);
+                    assignMetric(metric, tbh, ASSIGN_ROUND_ROBIN);
                 } else if (!tbh.isUp()) {
                     TimelyBalancedHost oldTbh = tbh;
-                    tbh = getLeastUsedHost();
-                    LOG.debug("rebalancing from host that is down: reassigning metric {} from server {}:{} to {}:{}",
-                            metric, oldTbh.getHost(), oldTbh.getTcpPort(), tbh.getHost(), tbh.getTcpPort());
+                    tbh = getRoundRobinHost(oldTbh);
+                    LOG.debug("rebalancing from host that is down: reassigning metric {} from server {} to {}", metric,
+                            TimelyBalancedHost.toStringShort(oldTbh), TimelyBalancedHost.toStringShort(tbh));
                     if (!balancerLock.isWriteLockedByCurrentThread()) {
                         balancerLock.readLock().unlock();
                         balancerLock.writeLock().lock();
                     }
-                    assignMetric(metric, tbh);
+                    assignMetric(metric, tbh, HOST_DOWN_ROUND_ROBIN);
                 }
             }
 
@@ -789,7 +804,7 @@ public class BalancedMetricResolver implements MetricResolver {
                         balancerLock.readLock().unlock();
                         balancerLock.writeLock().lock();
                     }
-                    assignMetric(metric, tbh);
+                    assignMetric(metric, tbh, ASSIGN_FALLBACK_SEQUENTIAL);
                 }
             }
         } finally {
@@ -818,20 +833,23 @@ public class BalancedMetricResolver implements MetricResolver {
                 metric = null;
             }
 
-            if (tbh == null || !tbh.isUp()) {
-                for (int x = 0; tbh == null && x < serverList.size(); x++) {
-                    tbh = serverList.get(Math.abs(r.nextInt() & Integer.MAX_VALUE) % serverList.size());
-                    if (!tbh.isUp()) {
-                        tbh = null;
-                    }
+            if (tbh == null) {
+                tbh = getRoundRobinHost(null);
+                if (!balancerLock.isWriteLockedByCurrentThread()) {
+                    balancerLock.readLock().unlock();
+                    balancerLock.writeLock().lock();
                 }
-                if (tbh != null && StringUtils.isNotBlank(metric)) {
-                    if (!balancerLock.isWriteLockedByCurrentThread()) {
-                        balancerLock.readLock().unlock();
-                        balancerLock.writeLock().lock();
-                    }
-                    assignMetric(metric, tbh);
+                assignMetric(metric, tbh, ASSIGN_ROUND_ROBIN);
+            } else if (!tbh.isUp()) {
+                TimelyBalancedHost oldTbh = tbh;
+                tbh = getRoundRobinHost(oldTbh);
+                LOG.debug("rebalancing from host that is down: reassigning metric {} from server {} to {}", metric,
+                        TimelyBalancedHost.toStringShort(oldTbh), TimelyBalancedHost.toStringShort(tbh));
+                if (!balancerLock.isWriteLockedByCurrentThread()) {
+                    balancerLock.readLock().unlock();
+                    balancerLock.writeLock().lock();
                 }
+                assignMetric(metric, tbh, HOST_DOWN_ROUND_ROBIN);
             }
 
             // if all else fails
@@ -847,7 +865,7 @@ public class BalancedMetricResolver implements MetricResolver {
                         balancerLock.readLock().unlock();
                         balancerLock.writeLock().lock();
                     }
-                    assignMetric(metric, tbh);
+                    assignMetric(metric, tbh, ASSIGN_FALLBACK_SEQUENTIAL);
                 }
             }
         } finally {
@@ -860,7 +878,7 @@ public class BalancedMetricResolver implements MetricResolver {
         return tbh;
     }
 
-    private TimelyBalancedHost findHost(String host, int tcpPort) {
+    protected TimelyBalancedHost findHost(String host, int tcpPort) {
         TimelyBalancedHost tbh = null;
         for (TimelyBalancedHost h : serverList) {
             if (h.getHost().equals(host) && h.getTcpPort() == tcpPort) {
@@ -871,199 +889,73 @@ public class BalancedMetricResolver implements MetricResolver {
         return tbh;
     }
 
-    private void createAssignmentFile() {
-        try {
-            LOG.info("Creating assignment file: " + balancerConfig.getAssignmentFile());
-            Configuration configuration = new Configuration();
-            FileSystem fs = FileSystem.get(configuration);
-            Path assignmentFile = new Path(balancerConfig.getAssignmentFile());
-            fs.create(assignmentFile);
-        } catch (Exception e) {
-            LOG.error(e.getMessage(), e);
-        }
-    }
-
-    private void readAssignmentsFromHdfs(boolean checkIfNecessary) {
-
-        try {
-            long lastLocalUpdate = assignmentsLastUpdatedLocal.get();
-            long lastHdfsUpdate = assignmentsLastUpdatedInHdfs.get().postValue();
-            if (checkIfNecessary) {
-                if (lastHdfsUpdate <= lastLocalUpdate) {
-                    LOG.debug("Not reading assignments from hdfs lastHdfsUpdate ({}) <= lastLocalUpdate ({})",
-                            new Date(lastHdfsUpdate), new Date(lastLocalUpdate));
-                    return;
-                }
-            }
-            LOG.debug("Reading assignments from hdfs lastHdfsUpdate ({}) > lastLocalUpdate ({})",
-                    new Date(lastHdfsUpdate), new Date(lastLocalUpdate));
-        } catch (Exception e) {
-            LOG.error(e.getMessage(), e);
-        }
-
-        // proceed with reading from HDFS
-
-        Map<String, TimelyBalancedHost> assignedMetricToHostMap = new TreeMap<>();
-        try {
-            boolean acquired = false;
-            while (!acquired) {
-                acquired = assignmentsIPRWLock.readLock().acquire(60, TimeUnit.SECONDS);
-            }
-            balancerLock.writeLock().lock();
-            Configuration configuration = new Configuration();
-            FileSystem fs = FileSystem.get(configuration);
-            Path assignmentFile = new Path(balancerConfig.getAssignmentFile());
-            FSDataInputStream iStream = fs.open(assignmentFile);
-
-            CsvReader reader = new CsvReader(iStream, ',', Charset.forName("UTF-8"));
-            reader.setUseTextQualifier(false);
-
-            // skip the headers
-            boolean success = true;
-            success = reader.readHeaders();
-
-            while (success) {
-                success = reader.readRecord();
-                String[] nextLine = reader.getValues();
-                if (nextLine.length >= 3) {
-                    String metric = nextLine[0];
-                    String host = nextLine[1];
-                    int tcpPort = Integer.parseInt(nextLine[2]);
-                    TimelyBalancedHost tbh = findHost(host, tcpPort);
-                    if (tbh == null) {
-                        tbh = getRoundRobinHost(null);
-                    } else {
-                        LOG.trace("Found assigment: {} to {}:{}", metric, host, tcpPort);
-                    }
-                    assignedMetricToHostMap.put(metric, tbh);
-                }
-            }
-
-            metricToHostMap.clear();
-            for (Map.Entry<String, TimelyBalancedHost> e : assignedMetricToHostMap.entrySet()) {
-                if (StringUtils.isNotBlank(e.getKey()) && e.getValue() != null) {
-                    if (shouldCache(e.getKey())) {
-                        metricToHostMap.put(e.getKey(), e.getValue());
-                    }
-                } else {
-                    Exception e1 = new IllegalStateException(
-                            "Bad assignment metric:" + e.getKey() + " host:" + e.getValue());
-                    LOG.warn(e1.getMessage(), e1);
-                }
-            }
-
-            nonCachedMetricsLocalLock.readLock().lock();
-            try {
-                Iterator<Map.Entry<String, TimelyBalancedHost>> itr = metricToHostMap.entrySet().iterator();
-                while (itr.hasNext()) {
-                    Map.Entry<String, TimelyBalancedHost> e = itr.next();
-                    if (nonCachedMetrics.contains(e.getKey())) {
-                        itr.remove();
-                    }
-                }
-            } finally {
-                nonCachedMetricsLocalLock.readLock().unlock();
-            }
-            assignmentsLastUpdatedLocal.set(assignmentsLastUpdatedInHdfs.get().postValue());
-            LOG.info("Read {} assignments from hdfs lastHdfsUpdate = lastLocalUpdate ({})", metricToHostMap.size(),
-                    new Date(assignmentsLastUpdatedLocal.get()));
-        } catch (FileNotFoundException e) {
-            createAssignmentFile();
-            metricToHostMap.clear();
-        } catch (Exception e) {
-            LOG.error(e.getMessage(), e);
-        } finally {
-            balancerLock.writeLock().unlock();
-            try {
-                assignmentsIPRWLock.readLock().release();
-            } catch (Exception e) {
-                LOG.error(e.getMessage(), e);
-            }
-        }
-    }
-
-    private void writeAssignmentsToHdfs() {
-
-        CsvWriter writer = null;
-        try {
-            boolean acquired = false;
-            while (!acquired) {
-                acquired = assignmentsIPRWLock.writeLock().acquire(60, TimeUnit.SECONDS);
-            }
-            balancerLock.readLock().lock();
-            nonCachedMetricsLocalLock.readLock().lock();
-            try {
-                if (!metricToHostMap.isEmpty()) {
-                    Configuration configuration = new Configuration();
-                    FileSystem fs = FileSystem.get(configuration);
-                    Path assignmentFile = new Path(balancerConfig.getAssignmentFile());
-                    if (!fs.exists(assignmentFile.getParent())) {
-                        fs.mkdirs(assignmentFile.getParent());
-                    }
-                    FSDataOutputStream oStream = fs.create(assignmentFile, true);
-                    writer = new CsvWriter(oStream, ',', Charset.forName("UTF-8"));
-                    writer.setUseTextQualifier(false);
-                    writer.write("metric");
-                    writer.write("host");
-                    writer.write("tcpPort");
-                    writer.endRecord();
-                    for (Map.Entry<String, TimelyBalancedHost> e : metricToHostMap.entrySet()) {
-                        if (!nonCachedMetrics.contains(e.getKey())) {
-                            writer.write(e.getKey());
-                            writer.write(e.getValue().getHost());
-                            writer.write(Integer.toString(e.getValue().getTcpPort()));
-                            writer.endRecord();
-                            LOG.trace("Saving assigment: {} to {}:{}", e.getKey(), e.getValue().getHost(),
-                                    e.getValue().getTcpPort());
-                        }
-                    }
-
-                    long now = System.currentTimeMillis();
-                    assignmentsLastUpdatedLocal.set(now);
-                    assignmentsLastUpdatedInHdfs.trySet(now);
-                    if (!assignmentsLastUpdatedInHdfs.get().succeeded()) {
-                        assignmentsLastUpdatedInHdfs.forceSet(now);
-                    }
-                    LOG.debug("Wrote {} assignments to hdfs lastHdfsUpdate = lastLocalUpdate ({})",
-                            metricToHostMap.size(), new Date(assignmentsLastUpdatedLocal.get()));
-                }
-            } finally {
-                nonCachedMetricsLocalLock.readLock().unlock();
-                balancerLock.readLock().unlock();
-            }
-        } catch (Exception e) {
-            LOG.error(e.getMessage(), e);
-        } finally {
-            if (writer != null) {
-                writer.close();
-            }
-            try {
-                assignmentsIPRWLock.writeLock().release();
-            } catch (Exception e) {
-                LOG.error(e.getMessage(), e);
-            }
-        }
-    }
-
-    private void assignMetric(String metric, TimelyBalancedHost tbh) {
+    private void assignMetric(String metric, TimelyBalancedHost tbh, MetricAssignedEvent.Reason reason) {
         if (isLeader.get()) {
             if (StringUtils.isNotBlank(metric) && tbh != null) {
                 balancerLock.writeLock().lock();
                 try {
+                    metricAssignedEvent(metric, metricToHostMap.get(metric), tbh, reason);
                     metricToHostMap.put(metric, tbh);
                     assignmentsLastUpdatedLocal.set(System.currentTimeMillis());
                 } finally {
                     balancerLock.writeLock().unlock();
                 }
             } else {
-                Exception e = new IllegalStateException("Bad assignment metric:" + metric + " host:" + tbh);
-                LOG.warn(e.getMessage(), e);
+                unassignedMetrics.add(metric);
+                // Exception e = new IllegalStateException("Bad assignment metric:" + metric + "
+                // host:" + tbh);
+                LOG.warn("Bad assignment metric:" + metric + " host:" + tbh);
             }
         }
     }
 
+    public void registerCallback(MetricAssignedCallback callback) {
+        this.metricAssignedCallbacks.add(callback);
+    }
+
+    public void registerCallback(MetricBalanceCallback callback) {
+        this.metricBalanceCallbacks.add(callback);
+    }
+
+    public void registerCallback(MetricHostCallback callback) {
+        this.metricHostCallbacks.add(callback);
+    }
+
+    protected void metricAssignedEvent(String metric, TimelyBalancedHost losing, TimelyBalancedHost gaining,
+            MetricAssignedEvent.Reason reason) {
+        if (!metricAssignedCallbacks.isEmpty()) {
+            MetricAssignedEvent event = new MetricAssignedEvent(metric, losing, gaining, reason);
+            for (MetricAssignedCallback callback : metricAssignedCallbacks) {
+                callback.onEvent(event);
+            }
+        }
+    }
+
+    protected void metricBalanceEvent(MetricBalanceEvent.ProgressType progressType,
+            BalancedMetricResolver.BalanceType balanceType, long numReassigned) {
+        if (!metricBalanceCallbacks.isEmpty()) {
+            MetricBalanceEvent event = new MetricBalanceEvent(progressType, balanceType, numReassigned);
+            for (MetricBalanceCallback callback : metricBalanceCallbacks) {
+                callback.onEvent(event);
+            }
+        }
+    }
+
+    protected void metricHostEvent(TimelyBalancedHost host, MetricHostEvent.ActionType actionType) {
+        if (!metricHostCallbacks.isEmpty()) {
+            MetricHostEvent event = new MetricHostEvent(host, actionType);
+            for (MetricHostCallback callback : metricHostCallbacks) {
+                callback.onEvent(event);
+            }
+        }
+    }
+
+    protected void writeAssignments(Map<String, TimelyBalancedHost> metricToHostMap) {
+        assignmentPerister.writeAssignments(metricToHostMap);
+    }
+
     @Override
-    public void close() throws Exception {
+    public void stop() {
 
         this.assignmentExecutor.shutdown();
         try {

balancer/src/main/java/timely/balancer/resolver/MetricAssignmentPerister.java

@@ -0,0 +1,320 @@
+package timely.balancer.resolver;
+
+import static timely.balancer.Balancer.ASSIGNMENTS_LAST_UPDATED_PATH;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.charset.Charset;
+import java.util.Date;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.apache.commons.lang.StringUtils;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.recipes.atomic.DistributedAtomicLong;
+import org.apache.curator.framework.recipes.locks.InterProcessReadWriteLock;
+import org.apache.curator.retry.RetryForever;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import timely.balancer.configuration.BalancerConfiguration;
+import timely.balancer.connection.TimelyBalancedHost;
+import timely.balancer.resolver.eventing.MetricAssignedEvent;
+
+public class MetricAssignmentPerister {
+
+    private final Logger LOG = LoggerFactory.getLogger(getClass());
+
+    private enum PeristerType {
+        HDFS, FILE
+    }
+
+    // passed in through constructor
+    BalancedMetricResolver balancedMetricResolver;
+    private BalancerConfiguration balancerConfig;
+    private Map<String, TimelyBalancedHost> metricToHostMap;
+    private Set<String> nonCachedMetrics;
+    private InterProcessReadWriteLock assignmentsIPRWLock;
+    private ReentrantReadWriteLock nonCachedMetricsLocalLock;
+    private ReentrantReadWriteLock balancerLock;
+    private AtomicLong assignmentsLastUpdatedLocal;
+
+    // local
+    private DistributedAtomicLong assignmentsLastUpdatedInHdfs;
+
+    public MetricAssignmentPerister(BalancedMetricResolver balancedMetricResolver, BalancerConfiguration balancerConfig,
+            Map<String, TimelyBalancedHost> metricToHostMap, Set<String> nonCachedMetrics,
+            CuratorFramework curatorFramework, InterProcessReadWriteLock assignmentsIPRWLock,
+            ReentrantReadWriteLock nonCachedMetricsLocalLock, AtomicLong assignmentsLastUpdatedLocal,
+            ReentrantReadWriteLock balancerLock) {
+
+        this.balancedMetricResolver = balancedMetricResolver;
+        this.balancerConfig = balancerConfig;
+        this.metricToHostMap = metricToHostMap;
+        this.nonCachedMetrics = nonCachedMetrics;
+        this.assignmentsIPRWLock = assignmentsIPRWLock;
+        this.nonCachedMetricsLocalLock = nonCachedMetricsLocalLock;
+        this.assignmentsLastUpdatedLocal = assignmentsLastUpdatedLocal;
+        this.balancerLock = balancerLock;
+        assignmentsLastUpdatedInHdfs = new DistributedAtomicLong(curatorFramework, ASSIGNMENTS_LAST_UPDATED_PATH,
+                new RetryForever(1000));
+    }
+
+    public long getAssignmentsLastUpdatedByLeader() throws Exception {
+        return assignmentsLastUpdatedInHdfs.get().postValue();
+    }
+
+    public void readAssignments(boolean checkIfNecessary) {
+
+        try {
+            long lastLocalUpdate = assignmentsLastUpdatedLocal.get();
+            long lastHdfsUpdate = assignmentsLastUpdatedInHdfs.get().postValue();
+            if (checkIfNecessary) {
+                if (lastHdfsUpdate <= lastLocalUpdate) {
+                    LOG.debug("Not reading assignments from storage lastStorageUpdate ({}) <= lastLocalUpdate ({})",
+                            new Date(lastHdfsUpdate), new Date(lastLocalUpdate));
+                    return;
+                }
+            }
+            LOG.debug("Reading assignments from storage lastStorageUpdate ({}) > lastLocalUpdate ({})",
+                    new Date(lastHdfsUpdate), new Date(lastLocalUpdate));
+        } catch (Exception e) {
+            LOG.error(e.getMessage(), e);
+        }
+
+        // proceed with reading from storage
+
+        try (InputStream inputStream = getInputStream(balancerConfig)) {
+            boolean acquired = false;
+            while (!acquired) {
+                acquired = assignmentsIPRWLock.readLock().acquire(60, TimeUnit.SECONDS);
+            }
+            balancerLock.writeLock().lock();
+            // get potential new list from implementation
+            Map<String, TimelyBalancedHost> assignedMetricToHostMap = readFromStream(inputStream);
+            metricToHostMap.clear();
+            for (Map.Entry<String, TimelyBalancedHost> e : assignedMetricToHostMap.entrySet()) {
+                if (StringUtils.isNotBlank(e.getKey()) && e.getValue() != null) {
+                    if (balancedMetricResolver.shouldCache(e.getKey())) {
+                        balancedMetricResolver.metricAssignedEvent(e.getKey(), null, e.getValue(),
+                                MetricAssignedEvent.Reason.ASSIGN_FILE);
+                        metricToHostMap.put(e.getKey(), e.getValue());
+                    }
+                } else {
+                    Exception e1 = new IllegalStateException(
+                            "Bad assignment metric:" + e.getKey() + " host:" + e.getValue());
+                    LOG.warn(e1.getMessage(), e1);
+                }
+            }
+            nonCachedMetricsLocalLock.readLock().lock();
+            try {
+                Iterator<Map.Entry<String, TimelyBalancedHost>> itr = metricToHostMap.entrySet().iterator();
+                while (itr.hasNext()) {
+                    Map.Entry<String, TimelyBalancedHost> e = itr.next();
+                    if (nonCachedMetrics.contains(e.getKey())) {
+                        itr.remove();
+                    }
+                }
+            } finally {
+                nonCachedMetricsLocalLock.readLock().unlock();
+            }
+            assignmentsLastUpdatedLocal.set(assignmentsLastUpdatedInHdfs.get().postValue());
+            LOG.info("Read {} assignments from storage lastStorageUpdate = lastLocalUpdate ({})",
+                    metricToHostMap.size(), new Date(assignmentsLastUpdatedLocal.get()));
+        } catch (FileNotFoundException e) {
+            createAssignmentFile();
+            metricToHostMap.clear();
+        } catch (Exception e) {
+            LOG.error(e.getMessage(), e);
+        } finally {
+            balancerLock.writeLock().unlock();
+            try {
+                assignmentsIPRWLock.readLock().release();
+            } catch (Exception e) {
+                LOG.error(e.getMessage(), e);
+            }
+        }
+    }
+
+    public void writeAssignments() {
+        writeAssignments(metricToHostMap);
+    }
+
+    protected void writeAssignments(Map<String, TimelyBalancedHost> metricToHostMap) {
+
+        try {
+            boolean acquired = false;
+            while (!acquired) {
+                acquired = assignmentsIPRWLock.writeLock().acquire(60, TimeUnit.SECONDS);
+            }
+            balancerLock.readLock().lock();
+            nonCachedMetricsLocalLock.readLock().lock();
+            try (OutputStream outputStream = getOutputStream(balancerConfig)) {
+                if (!metricToHostMap.isEmpty()) {
+                    writeToStream(outputStream);
+                    long now = System.currentTimeMillis();
+                    assignmentsLastUpdatedLocal.set(now);
+                    assignmentsLastUpdatedInHdfs.trySet(now);
+                    if (!assignmentsLastUpdatedInHdfs.get().succeeded()) {
+                        assignmentsLastUpdatedInHdfs.forceSet(now);
+                    }
+                    LOG.debug("Wrote {} assignments to storage lastStorageUpdate = lastLocalUpdate ({})",
+                            metricToHostMap.size(), new Date(assignmentsLastUpdatedLocal.get()));
+                }
+            } finally {
+                nonCachedMetricsLocalLock.readLock().unlock();
+                balancerLock.readLock().unlock();
+            }
+        } catch (Exception e) {
+            LOG.error(e.getMessage(), e);
+        } finally {
+            try {
+                assignmentsIPRWLock.writeLock().release();
+            } catch (Exception e) {
+                LOG.error(e.getMessage(), e);
+            }
+        }
+    }
+
+    public Map<String, TimelyBalancedHost> readFromStream(InputStream inputStream) throws IOException {
+        Map<String, TimelyBalancedHost> assignedMetricToHostMap = new TreeMap<>();
+        try {
+            CsvReader reader = new CsvReader(inputStream, ',', Charset.forName("UTF-8"));
+            reader.setUseTextQualifier(false);
+
+            // skip the headers
+            boolean success = true;
+            success = reader.readHeaders();
+
+            while (success) {
+                success = reader.readRecord();
+                String[] nextLine = reader.getValues();
+                if (nextLine.length >= 3) {
+                    String metric = nextLine[0];
+                    String host = nextLine[1];
+                    int tcpPort = Integer.parseInt(nextLine[2]);
+                    TimelyBalancedHost tbh = balancedMetricResolver.findHost(host, tcpPort);
+                    if (tbh == null) {
+                        tbh = balancedMetricResolver.getRoundRobinHost(null);
+                    } else {
+                        LOG.trace("Found assigment: {} to {}:{}", metric, host, tcpPort);
+                    }
+                    assignedMetricToHostMap.put(metric, tbh);
+                }
+            }
+        } catch (IOException e) {
+            LOG.error(e.getMessage(), e);
+            throw e;
+        }
+        return assignedMetricToHostMap;
+    }
+
+    private void writeToStream(OutputStream outputStream) throws IOException {
+
+        CsvWriter writer = null;
+        try {
+            writer = new CsvWriter(outputStream, ',', Charset.forName("UTF-8"));
+            writer.setUseTextQualifier(false);
+            writer.write("metric");
+            writer.write("host");
+            writer.write("tcpPort");
+            writer.endRecord();
+            for (Map.Entry<String, TimelyBalancedHost> e : metricToHostMap.entrySet()) {
+                if (!nonCachedMetrics.contains(e.getKey())) {
+                    writer.write(e.getKey());
+                    writer.write(e.getValue().getHost());
+                    writer.write(Integer.toString(e.getValue().getTcpPort()));
+                    writer.endRecord();
+                    LOG.trace("Saving assigment: {} to {}:{}", e.getKey(), e.getValue().getHost(),
+                            e.getValue().getTcpPort());
+                }
+            }
+        } finally {
+            if (writer != null) {
+                writer.close();
+            }
+        }
+    }
+
+    public void createAssignmentFile() {
+        LOG.info("Creating assignment file: " + balancerConfig.getAssignmentFile());
+        PeristerType peristerType = PeristerType.valueOf(balancerConfig.getMetricAssignmentPersisterType());
+        switch (peristerType) {
+            case HDFS:
+                try {
+                    Configuration configuration = new Configuration();
+                    FileSystem fs = FileSystem.get(configuration);
+                    Path assignmentFile = new Path(balancerConfig.getAssignmentFile());
+                    fs.create(assignmentFile);
+                } catch (Exception e) {
+                    LOG.error(e.getMessage(), e);
+                }
+                break;
+            case FILE:
+                try {
+                    File assignmentFile = new File(balancerConfig.getAssignmentFile());
+                    String parentPath = assignmentFile.getParent();
+                    File directory = new File(parentPath);
+                    if (!directory.exists()) {
+
+                        directory.mkdirs();
+                    }
+                    if (!assignmentFile.exists()) {
+                        assignmentFile.createNewFile();
+                    }
+                } catch (Exception e) {
+                    LOG.error(e.getMessage(), e);
+                }
+                break;
+        }
+    }
+
+    public InputStream getInputStream(BalancerConfiguration balancerConfiguration) throws Exception {
+        PeristerType peristerType = PeristerType.valueOf(balancerConfiguration.getMetricAssignmentPersisterType());
+        switch (peristerType) {
+            case HDFS:
+                Configuration configuration = new Configuration();
+                FileSystem fs = FileSystem.get(configuration);
+                Path assignmentFile = new Path(balancerConfiguration.getAssignmentFile());
+                return fs.open(assignmentFile);
+            case FILE:
+                return new FileInputStream(new File(balancerConfiguration.getAssignmentFile()));
+            default:
+                throw new IllegalArgumentException(
+                        "Unknown metricPersisterType " + balancerConfiguration.getMetricAssignmentPersisterType());
+        }
+    }
+
+    public OutputStream getOutputStream(BalancerConfiguration balancerConfiguration) throws IOException {
+        PeristerType peristerType = PeristerType.valueOf(balancerConfiguration.getMetricAssignmentPersisterType());
+        switch (peristerType) {
+            case HDFS:
+                Configuration configuration = new Configuration();
+                FileSystem fs = FileSystem.get(configuration);
+                Path assignmentFile = new Path(balancerConfiguration.getAssignmentFile());
+                if (!fs.exists(assignmentFile.getParent())) {
+                    fs.mkdirs(assignmentFile.getParent());
+                }
+                return fs.create(assignmentFile, true);
+            case FILE:
+                return new FileOutputStream(new File(balancerConfiguration.getAssignmentFile()));
+            default:
+                throw new IllegalArgumentException(
+                        "Unknown metricPersisterType " + balancerConfiguration.getMetricAssignmentPersisterType());
+        }
+    }
+}

balancer/src/main/java/timely/balancer/resolver/MetricResolver.java

@@ -8,5 +8,8 @@ public interface MetricResolver {
 
     public TimelyBalancedHost getHostPortKey(String metric);
 
-    public void close() throws Exception;
+    public void start();
+
+    public void stop();
+
 }

balancer/src/main/java/timely/balancer/resolver/eventing/MetricAssignedCallback.java

@@ -0,0 +1,7 @@
+package timely.balancer.resolver.eventing;
+
+public abstract class MetricAssignedCallback {
+
+    public abstract void onEvent(MetricAssignedEvent event);
+
+}

balancer/src/main/java/timely/balancer/resolver/eventing/MetricAssignedEvent.java

@@ -0,0 +1,39 @@
+package timely.balancer.resolver.eventing;
+
+import timely.balancer.connection.TimelyBalancedHost;
+
+public class MetricAssignedEvent {
+
+    public enum Reason {
+        ASSIGN_ROUND_ROBIN, HOST_DOWN_ROUND_ROBIN, ASSIGN_FALLBACK_SEQUENTIAL, HOST_REMOVED, REBALANCE, ASSIGN_FILE
+    }
+
+    private final String metric;
+    private final TimelyBalancedHost losingHost;
+    private final TimelyBalancedHost gainingHost;
+    private final Reason reason;
+
+    public MetricAssignedEvent(String metric, TimelyBalancedHost losingHost, TimelyBalancedHost gainingHost,
+            Reason reason) {
+        this.metric = metric;
+        this.losingHost = losingHost;
+        this.gainingHost = gainingHost;
+        this.reason = reason;
+    }
+
+    public String getMetric() {
+        return metric;
+    }
+
+    public TimelyBalancedHost getLosingHost() {
+        return losingHost;
+    }
+
+    public TimelyBalancedHost getGainingHost() {
+        return gainingHost;
+    }
+
+    public Reason getReason() {
+        return reason;
+    }
+}

balancer/src/main/java/timely/balancer/resolver/eventing/MetricBalanceCallback.java

@@ -0,0 +1,7 @@
+package timely.balancer.resolver.eventing;
+
+public abstract class MetricBalanceCallback {
+
+    public abstract void onEvent(MetricBalanceEvent event);
+
+}

balancer/src/main/java/timely/balancer/resolver/eventing/MetricBalanceEvent.java

@@ -0,0 +1,33 @@
+package timely.balancer.resolver.eventing;
+
+import timely.balancer.resolver.BalancedMetricResolver;
+
+public class MetricBalanceEvent {
+
+    public enum ProgressType {
+        BEGIN, BALANCER_LOCK_ACQUIRED, END
+    }
+
+    private final ProgressType progressType;
+    private final BalancedMetricResolver.BalanceType balanceType;
+    private final Long numRessigned;
+
+    public MetricBalanceEvent(ProgressType progressType, BalancedMetricResolver.BalanceType balanceType,
+            long numReassigned) {
+        this.progressType = progressType;
+        this.balanceType = balanceType;
+        this.numRessigned = numReassigned;
+    }
+
+    public BalancedMetricResolver.BalanceType getBalanceType() {
+        return balanceType;
+    }
+
+    public ProgressType getProgressType() {
+        return progressType;
+    }
+
+    public Long getNumRessigned() {
+        return numRessigned;
+    }
+}

balancer/src/main/java/timely/balancer/resolver/eventing/MetricHostCallback.java

@@ -0,0 +1,7 @@
+package timely.balancer.resolver.eventing;
+
+public abstract class MetricHostCallback {
+
+    public abstract void onEvent(MetricHostEvent event);
+
+}

balancer/src/main/java/timely/balancer/resolver/eventing/MetricHostEvent.java

@@ -0,0 +1,26 @@
+package timely.balancer.resolver.eventing;
+
+import timely.balancer.connection.TimelyBalancedHost;
+
+public class MetricHostEvent {
+
+    public enum ActionType {
+        ADDED, REMOVED, SUCCESS, FAILURE
+    }
+
+    private final TimelyBalancedHost timelyBalancedHost;
+    private final ActionType actionType;
+
+    public MetricHostEvent(TimelyBalancedHost timelyBalancedHost, ActionType actionType) {
+        this.timelyBalancedHost = timelyBalancedHost;
+        this.actionType = actionType;
+    }
+
+    public TimelyBalancedHost getTimelyBalancedHost() {
+        return timelyBalancedHost;
+    }
+
+    public ActionType getActionType() {
+        return actionType;
+    }
+}

balancer/src/main/spotbugs/excludes.xml

@@ -10,4 +10,9 @@
     <Method name="getBuf" />
     <Bug pattern="EI_EXPOSE_REP" />
   </Match>
+  <Match>
+    <Class name="timely.balancer.resolver.MetricAssignmentPerister" />
+    <Method name="createAssignmentFile" />
+    <Bug pattern="RV_RETURN_VALUE_IGNORED_BAD_PRACTICE" />
+  </Match>
 </FindBugsFilter>

balancer/src/test/java/timely/balancer/resolver/ServiceRegistry.java

@@ -0,0 +1,76 @@
+package timely.balancer.resolver;
+
+import static timely.Server.SERVICE_DISCOVERY_PATH;
+
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.x.discovery.ServiceDiscovery;
+import org.apache.curator.x.discovery.ServiceDiscoveryBuilder;
+import org.apache.curator.x.discovery.ServiceInstance;
+import org.apache.curator.x.discovery.ServiceInstanceBuilder;
+import org.apache.zookeeper.data.Stat;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import timely.ServerDetails;
+import timely.balancer.connection.TimelyBalancedHost;
+
+public class ServiceRegistry {
+
+    private static final Logger LOG = LoggerFactory.getLogger(ServiceRegistry.class);
+
+    public static void registerService(CuratorFramework curatorFramework, TimelyBalancedHost tbh) {
+        try {
+            try {
+                Stat stat = curatorFramework.checkExists().forPath(SERVICE_DISCOVERY_PATH);
+                if (stat == null) {
+                    curatorFramework.create().creatingParentContainersIfNeeded().forPath(SERVICE_DISCOVERY_PATH);
+                }
+            } catch (Exception e) {
+                LOG.error(e.getMessage());
+            }
+
+            ServiceInstance<ServerDetails> serviceInstance = serviceInstance(tbh);
+            ServiceDiscovery<ServerDetails> discovery = ServiceDiscoveryBuilder.builder(ServerDetails.class)
+                    .client(curatorFramework).basePath(SERVICE_DISCOVERY_PATH).build();
+            discovery.start();
+            discovery.registerService(serviceInstance);
+        } catch (Exception e) {
+            LOG.error(e.getMessage());
+        }
+    }
+
+    public static void unregisterService(CuratorFramework curatorFramework, TimelyBalancedHost tbh) {
+        try {
+            try {
+                Stat stat = curatorFramework.checkExists().forPath(SERVICE_DISCOVERY_PATH);
+                if (stat == null) {
+                    curatorFramework.create().creatingParentContainersIfNeeded().forPath(SERVICE_DISCOVERY_PATH);
+                }
+            } catch (Exception e) {
+                LOG.error(e.getMessage());
+            }
+
+            ServiceInstance<ServerDetails> serviceInstance = serviceInstance(tbh);
+            ServiceDiscovery<ServerDetails> discovery = ServiceDiscoveryBuilder.builder(ServerDetails.class)
+                    .client(curatorFramework).basePath(SERVICE_DISCOVERY_PATH).build();
+            discovery.start();
+            discovery.unregisterService(serviceInstance);
+        } catch (Exception e) {
+            LOG.error(e.getMessage());
+        }
+    }
+
+    private static ServiceInstance<ServerDetails> serviceInstance(TimelyBalancedHost config) throws Exception {
+        ServerDetails serverDetails = new ServerDetails();
+        serverDetails.setHost(config.getHost());
+        serverDetails.setTcpPort(config.getTcpPort());
+        serverDetails.setHttpPort(config.getHttpPort());
+        serverDetails.setWsPort(config.getWsPort());
+        serverDetails.setUdpPort(config.getUdpPort());
+
+        ServiceInstanceBuilder<ServerDetails> builder = ServiceInstance.builder();
+        String serviceName = config.getHost() + ":" + config.getTcpPort();
+        ServiceInstance<ServerDetails> serviceInstance = builder.id(serviceName).name("timely-server")
+                .address(config.getHost()).port(config.getTcpPort()).payload(serverDetails).build();
+        return serviceInstance;
+    }
+}

balancer/src/test/java/timely/balancer/resolver/TestBalancedMetricResolver.java

@@ -0,0 +1,270 @@
+package timely.balancer.resolver;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Random;
+import java.util.UUID;
+
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.CuratorFrameworkFactory;
+import org.apache.curator.retry.RetryNTimes;
+import org.apache.zookeeper.server.ServerConfig;
+import org.apache.zookeeper.server.ZooKeeperServerMain;
+import org.apache.zookeeper.server.quorum.QuorumPeerConfig;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.util.SocketUtils;
+import timely.balancer.configuration.BalancerConfiguration;
+import timely.balancer.connection.TimelyBalancedHost;
+import timely.balancer.resolver.eventing.MetricAssignedCallback;
+import timely.balancer.resolver.eventing.MetricAssignedEvent;
+import timely.balancer.resolver.eventing.MetricBalanceCallback;
+import timely.balancer.resolver.eventing.MetricBalanceEvent;
+import timely.balancer.resolver.eventing.MetricHostCallback;
+import timely.balancer.resolver.eventing.MetricHostEvent;
+
+public class TestBalancedMetricResolver {
+
+    private final Logger LOG = LoggerFactory.getLogger(getClass());
+    private CuratorFramework curatorFramework;
+    private SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd HHmmss.SSS");
+    private Random r = new Random(System.currentTimeMillis());
+    private List<TimelyBalancedHost> timelyHosts;
+    private Map<MetricAssignedEvent, Integer> metricBalanceEventCounter = new HashMap<>();
+
+    @Before
+    public void startup() {
+        QuorumPeerConfig currentConfig = createZookeeper();
+        InetSocketAddress address = currentConfig.getClientPortAddress();
+        String zkServer = address.getHostName() + ":" + address.getPort();
+        curatorFramework = CuratorFrameworkFactory.newClient(zkServer, 1000, 5000, new RetryNTimes(2, 50));
+        curatorFramework.start();
+        timelyHosts = new ArrayList<>();
+    }
+
+    @After
+    public void shutdown() {
+        curatorFramework.close();
+    }
+
+    private QuorumPeerConfig createZookeeper() {
+        Properties startupProperties = new Properties();
+        File file = new File(System.getProperty("java.io.tmpdir") + File.separator + UUID.randomUUID());
+        file.deleteOnExit();
+        startupProperties.setProperty("dataDir", file.getAbsolutePath());
+        startupProperties.setProperty("clientPort", String.valueOf(SocketUtils.findAvailableTcpPort(3000, 4000)));
+        QuorumPeerConfig quorumConfiguration = new QuorumPeerConfig();
+
+        try {
+            quorumConfiguration.parseProperties(startupProperties);
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+
+        ZooKeeperServerMain zooKeeperServer = new ZooKeeperServerMain();
+        final ServerConfig configuration = new ServerConfig();
+        configuration.readFrom(quorumConfiguration);
+
+        new Thread() {
+
+            public void run() {
+                try {
+                    zooKeeperServer.runFromConfig(configuration);
+                } catch (IOException e) {
+                    LOG.error("ZooKeeper Failed", e);
+                }
+            }
+        }.start();
+
+        return quorumConfiguration;
+    }
+
+    private BalancerConfiguration getBalancerConfiguration() {
+        BalancerConfiguration balancerConfiguration = new BalancerConfiguration();
+        balancerConfiguration.setMetricAssignmentPersisterType("FILE");
+        File file = new File(System.getProperty("java.io.tmpdir") + File.separator + UUID.randomUUID());
+        file.deleteOnExit();
+        balancerConfiguration.setAssignmentFile(file.getAbsolutePath());
+        balancerConfiguration.setBalanceDelay(0);
+        balancerConfiguration.setBalancePeriod(1000);
+        balancerConfiguration.setPersistDelay(0);
+        balancerConfiguration.setPersistDelay(1000);
+        balancerConfiguration.setCheckServerHealthInterval(10);
+        return balancerConfiguration;
+    }
+
+    private void createTimelyHosts(BalancerConfiguration balancerConfiguration, int numHosts) {
+        for (int port = 1000, x = 0; x < numHosts; x++, port += 10) {
+            TimelyBalancedHost tbh = TimelyBalancedHost.of("localhost", port);
+            tbh.setBalancerConfig(balancerConfiguration);
+            timelyHosts.add(tbh);
+            ServiceRegistry.registerService(curatorFramework, tbh);
+        }
+    }
+
+    private void registerCallbacks(BalancedMetricResolver resolver) {
+        resolver.registerCallback(new MetricAssignedCallback() {
+
+            @Override
+            public void onEvent(MetricAssignedEvent event) {
+                LOG.debug(String.format("%s metric:%s lose:%s:%d gain:%s:%d reason:%s",
+                        event.getClass().getSimpleName(), event.getMetric(),
+                        event.getLosingHost() == null ? null : event.getLosingHost().getHost(),
+                        event.getLosingHost() == null ? null : event.getLosingHost().getTcpPort(),
+                        event.getGainingHost().getHost(), event.getGainingHost().getTcpPort(), event.getReason()));
+            }
+        });
+
+        resolver.registerCallback(new MetricHostCallback() {
+
+            @Override
+            public void onEvent(MetricHostEvent event) {
+                LOG.debug(String.format("%s host:%s:%d type:%s", event.getClass().getSimpleName(),
+                        event.getTimelyBalancedHost().getHost(), event.getTimelyBalancedHost().getTcpPort(),
+                        event.getActionType()));
+            }
+        });
+
+        resolver.registerCallback(new MetricBalanceCallback() {
+
+            @Override
+            public void onEvent(MetricBalanceEvent event) {
+                LOG.debug(String.format("%s type:%s progress:%s reassigned:%d", event.getClass().getSimpleName(),
+                        event.getBalanceType(), event.getProgressType(), event.getNumRessigned()));
+            }
+        });
+    }
+
+    private TimelyBalancedHost randomServerDown(TestHealthChecker healthChecker, int numHosts) {
+        TimelyBalancedHost tbh = timelyHosts.get(r.nextInt(numHosts));
+        LOG.debug("Marking server down:" + TimelyBalancedHost.toStringShort(tbh));
+        healthChecker.serverDown(tbh);
+        return tbh;
+    }
+
+    private TimelyBalancedHost randomServerToggle(TestHealthChecker healthChecker, int numHosts) {
+        TimelyBalancedHost tbh = timelyHosts.get(r.nextInt(numHosts));
+        LOG.debug("Marking server " + (tbh.isUp() ? "down:" : "up:") + TimelyBalancedHost.toStringShort(tbh));
+        healthChecker.serverToggle(tbh);
+        return tbh;
+    }
+
+    private TimelyBalancedHost randomServerUnregister(int numHosts) {
+        TimelyBalancedHost tbh = timelyHosts.get(r.nextInt(numHosts));
+        LOG.debug("Unregistering:" + TimelyBalancedHost.toStringShort(tbh));
+        ServiceRegistry.unregisterService(curatorFramework, tbh);
+        return tbh;
+    }
+
+    @Test
+    public void test1() {
+        LOG.debug("Configure test");
+        int numHosts = 10;
+        BalancerConfiguration balancerConfiguration = getBalancerConfiguration();
+        createTimelyHosts(balancerConfiguration, numHosts);
+        TestHealthChecker healthChecker = new TestHealthChecker(balancerConfiguration, timelyHosts);
+        BalancedMetricResolver balancedMetricResolver = new BalancedMetricResolver(curatorFramework,
+                balancerConfiguration, healthChecker);
+        registerCallbacks(balancedMetricResolver);
+        LOG.debug("Start balancedMetricResolver");
+        balancedMetricResolver.start();
+        long now = System.currentTimeMillis();
+        int numMetrics = 100;
+        for (int x = 0; x < numMetrics; x++) {
+            balancedMetricResolver.getHostPortKeyIngest(String.format("metric%03d", x));
+        }
+
+        LOG.debug("Begin monitoring");
+        long cycle = 0;
+        List<TimelyBalancedHost> serversUnregistered = new ArrayList<>();
+        while (System.currentTimeMillis() < now + 15000) {
+            try {
+                Thread.sleep(100);
+                for (int x = 0; x < 10000; x++) {
+                    int mNum = r.nextInt(numMetrics);
+                    balancedMetricResolver.getHostPortKeyIngest(String.format("metric%03d", mNum));
+                }
+                // if (cycle % 100 == 0) {
+                // randomServerDown(healthChecker, numHosts);
+                // }
+                // if (cycle % 2 == 0) {
+                // randomServerToggle(healthChecker, numHosts);
+                // }
+                if (cycle % 100 == 0) {
+                    serversUnregistered.add(randomServerUnregister(numHosts));
+                }
+                if (cycle % 500 == 0) {
+                    serversUnregistered.stream().forEach(tbh -> ServiceRegistry.registerService(curatorFramework, tbh));
+                    serversUnregistered.clear();
+                }
+
+            } catch (InterruptedException e) {
+
+            }
+            cycle++;
+        }
+        LOG.debug("Stop balancedMetricResolver");
+        balancedMetricResolver.stop();
+    }
+
+    @Test
+    public void test2() {
+        LOG.debug("Configure test");
+        int numHosts = 10;
+        BalancerConfiguration balancerConfiguration = getBalancerConfiguration();
+        createTimelyHosts(balancerConfiguration, numHosts);
+        TestHealthChecker healthChecker = new TestHealthChecker(balancerConfiguration, timelyHosts);
+        BalancedMetricResolver balancedMetricResolver = new BalancedMetricResolver(curatorFramework,
+                balancerConfiguration, healthChecker);
+        registerCallbacks(balancedMetricResolver);
+        LOG.debug("Start balancedMetricResolver");
+        balancedMetricResolver.start();
+        long now = System.currentTimeMillis();
+        int numMetrics = 100;
+        for (int x = 0; x < numMetrics; x++) {
+            balancedMetricResolver.getHostPortKeyIngest(String.format("metric%03d", x));
+        }
+
+        LOG.debug("Begin monitoring");
+        long cycle = 0;
+        List<TimelyBalancedHost> serversUnregistered = new ArrayList<>();
+        while (System.currentTimeMillis() < now + 15000) {
+            try {
+                Thread.sleep(100);
+                for (int x = 0; x < 10000; x++) {
+                    int mNum = r.nextInt(numMetrics);
+                    balancedMetricResolver.getHostPortKeyIngest(String.format("metric%03d", mNum));
+                }
+                // if (cycle % 100 == 0) {
+                // randomServerDown(healthChecker, numHosts);
+                // }
+                // if (cycle % 2 == 0) {
+                // randomServerToggle(healthChecker, numHosts);
+                // }
+                if (cycle % 100 == 0) {
+                    serversUnregistered.add(randomServerUnregister(numHosts));
+                }
+                if (cycle % 500 == 0) {
+                    serversUnregistered.stream().forEach(tbh -> ServiceRegistry.registerService(curatorFramework, tbh));
+                    serversUnregistered.clear();
+                }
+
+            } catch (InterruptedException e) {
+
+            }
+            cycle++;
+        }
+        LOG.debug("Stop balancedMetricResolver");
+        balancedMetricResolver.stop();
+    }
+}

balancer/src/test/java/timely/balancer/resolver/TestHealthChecker.java

@@ -0,0 +1,89 @@
+package timely.balancer.resolver;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import timely.balancer.configuration.BalancerConfiguration;
+import timely.balancer.connection.TimelyBalancedHost;
+import timely.balancer.healthcheck.HealthChecker;
+import timely.balancer.healthcheck.TcpHealthChecker;
+
+public class TestHealthChecker implements HealthChecker {
+
+    private final Logger LOG = LoggerFactory.getLogger(TcpHealthChecker.class);
+    private List<TimelyBalancedHost> timelyHosts = new ArrayList<>();
+    private Map<TimelyBalancedHost, Boolean> serverUpMap = Collections.synchronizedMap(new HashMap<>());
+    private ScheduledExecutorService executorService = Executors.newScheduledThreadPool(1);
+
+    public TestHealthChecker(BalancerConfiguration config, List<TimelyBalancedHost> timelyHosts) {
+        setTimelyHosts(timelyHosts);
+        this.executorService.scheduleAtFixedRate(() -> {
+            synchronized (timelyHosts) {
+                for (TimelyBalancedHost h : timelyHosts) {
+                    try {
+                        if (TestHealthChecker.this.isServerHealthy(h)) {
+                            h.reportSuccess();
+                        } else {
+                            h.reportFailure();
+                        }
+                    } catch (Exception e) {
+                        LOG.error(e.getMessage(), e);
+                        h.reportFailure();
+                    }
+                }
+            }
+        }, 0, config.getCheckServerHealthInterval(), TimeUnit.MILLISECONDS);
+    }
+
+    @Override
+    public void setTimelyHosts(List<TimelyBalancedHost> timelyHosts) {
+        synchronized (timelyHosts) {
+            this.timelyHosts.clear();
+            this.timelyHosts.addAll(timelyHosts);
+            for (TimelyBalancedHost h : timelyHosts) {
+                serverUpMap.put(h, true);
+            }
+        }
+
+    }
+
+    public boolean isServerHealthy(TimelyBalancedHost h) {
+        return serverUpMap.get(h);
+    }
+
+    public void serverUp(TimelyBalancedHost h) {
+        serverUpMap.put(h, true);
+    }
+
+    public void serverDown(TimelyBalancedHost h) {
+        serverUpMap.put(h, false);
+    }
+
+    public void serverToggle(TimelyBalancedHost h) {
+        synchronized (serverUpMap) {
+            serverUpMap.put(h, !serverUpMap.get(h));
+        }
+    }
+
+    @Override
+    public void close() throws Exception {
+        this.executorService.shutdown();
+        try {
+            this.executorService.awaitTermination(5, TimeUnit.SECONDS);
+        } catch (InterruptedException e) {
+
+        } finally {
+            if (!this.executorService.isTerminated()) {
+                this.executorService.shutdownNow();
+            }
+        }
+    }
+}

balancer/src/test/resources/log4j2.xml

@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Configuration status="WARN">
+  <Appenders>
+    <Console name="CONSOLE" target="SYSTEM_OUT">
+      <PatternLayout pattern="%d{yyyyMMdd HHmmss.SSS} [%t] %-5level %logger{36} - %msg%n"/>
+    </Console>
+  </Appenders>
+  <Loggers>
+    <Logger name="org.apache.accumulo.core.client.impl.ThriftTransportPool" level="OFF" additivity="false" />
+    <Logger name="org.apache.accumulo.fate.zookeeper.ZooCache" level="OFF" additivity="false" />
+    <Logger name="org.apache.zookeeper" level="OFF" additivity="false" />
+    <Logger name="timely.balancer.resolver" level="TRACE" additivity="true" />
+    <Logger name="timely.balancer.resolver.TestBalancedMetricResolver" level="DEBUG" additivity="true" />
+    <Root level="ERROR">
+      <AppenderRef ref="CONSOLE"/>
+    </Root>
+  </Loggers>
+</Configuration>

client/src/main/java/timely/model/Metric.java

@@ -22,7 +22,7 @@ import timely.model.parse.TagParser;
 @JsonRootName("metric")
 @JsonInclude(JsonInclude.Include.NON_NULL)
 @JsonPropertyOrder({ "name" })
-public class Metric implements Serializable, ObjectSizeOf {
+public class Metric implements Serializable {
 
     private static final long serialVersionUID = 1L;
 
@@ -186,9 +186,4 @@ public class Metric implements Serializable, ObjectSizeOf {
             return this.metric;
         }
     }
-
-    @Override
-    public long sizeInBytes() {
-        return 0;
-    }
 }

client/src/main/java/timely/model/ObjectSizeOf.java

@@ -2,16 +2,13 @@ package timely.model;
 
 import java.lang.reflect.Array;
 import java.lang.reflect.Field;
+import java.lang.reflect.Method;
 import java.lang.reflect.Modifier;
-import java.util.ArrayList;
-import java.util.Collections;
 import java.util.HashSet;
-import java.util.List;
 import java.util.Set;
+import java.util.Stack;
 import java.util.logging.Logger;
 
-import org.apache.commons.lang3.StringUtils;
-
 /**
  * A simple interface that objects can implement to return the object size.
  */
@@ -25,29 +22,15 @@ public interface ObjectSizeOf {
     public static class ObjectInstance {
 
         private Object o;
-        private String name = "";
 
         public ObjectInstance(Object _o) {
             this.o = _o;
         }
 
-        public ObjectInstance(Object _o, String name) {
-            this.o = _o;
-            this.name = name;
-        }
-
         public Object getObject() {
             return o;
         }
 
-        public void setName(String name) {
-            this.name = name;
-        }
-
-        public String getName() {
-            return name;
-        }
-
         @Override
         public int hashCode() {
             return System.identityHashCode(o);
@@ -81,134 +64,110 @@ public interface ObjectSizeOf {
          *
          * @return an approximation of the object size
          */
+        public static long getObjectSize(Object o, boolean useSizeInBytesMethod, boolean debug) {
+            long estSize = getObjectSize(o, new HashSet<ObjectInstance>(), new Stack<ObjectInstance>(),
+                    useSizeInBytesMethod, debug);
+            return estSize;
+        }
 
         public static long getObjectSize(Object o) {
-            return getObjectSize(o, true, false);
+            long estSize = getObjectSize(o, new HashSet<ObjectInstance>(), new Stack<ObjectInstance>(), true, false);
+            return estSize;
         }
 
-        public static long getObjectSize(Object o, boolean useSizeInBytesMethod, boolean debug) {
-            List<String> output = null;
-            if (debug) {
-                output = new ArrayList<>();
-            }
-            long size = getObjectSize(new ObjectInstance(o, o.getClass().getSimpleName()), new HashSet<>(), 0,
-                    useSizeInBytesMethod, output);
-            if (debug) {
-                Collections.reverse(output);
-                for (String s : output)
-                    System.out.println(s);
-            }
-            return size;
-        }
-
-        public static long getObjectSize(ObjectInstance oi, Set<ObjectInstance> visited, int indent,
-                boolean useSizeInBytesMethod, List<String> output) {
-            long size = 0;
-            Object o = oi.getObject();
-            int arrayOverhead = 0;
-            int objectOverhead = 0;
-            int reference = 0;
-            boolean roundUp = false;
-            if (o != null && !visited.contains(oi)) {
-                visited.add(oi);
-                try {
-                    if (useSizeInBytesMethod) {
-                        if (o instanceof ObjectSizeOf) {
-                            size = ((ObjectSizeOf) o).sizeInBytes();
-                            if (output != null && size > 0) {
-                                output.add(StringUtils.repeat(' ', indent) + oi.getName() + " [" + size + "] ("
-                                        + o.getClass().getCanonicalName() + ") [ObjectSizeOf]");
-                            }
-                        }
-                    }
-                    if (size == 0) {
-                        // the hard way...
-                        // do not include Class related objects or
-                        // reflection objects
-                        if ((o instanceof Class) || (o instanceof ClassLoader) || (o.getClass().getPackage() != null
-                                && o.getClass().getPackage().getName().startsWith("java.lang.reflect"))) {
-                            size = 0;
-                        } else if (o instanceof Number || o instanceof Boolean || o instanceof Character) {
-                            size = NUMBER_SIZE;
-                        } else {
-                            // lets do a simple sizing
-                            Class<?> c = o.getClass();
-                            if (c.isArray()) {
-                                arrayOverhead++;
-                                int length = Array.getLength(o);
-                                if (c.getComponentType().isPrimitive()) {
-                                    size += length * getPrimitiveObjectSize(c.getComponentType());
+        public static long getObjectSize(Object o, Set<ObjectInstance> visited, Stack<ObjectInstance> stack,
+                boolean useSizeInBytesMethod, boolean debug) {
+            long totalSize = 0;
+            stack.add(new ObjectInstance(o));
+            while (!stack.isEmpty()) {
+                long size = 0;
+                ObjectInstance oi = stack.pop();
+                o = oi.getObject();
+                if (o != null && !visited.contains(oi)) {
+                    visited.add(oi);
+                    try {
+                        if (useSizeInBytesMethod) {
+                            try {
+                                if (o instanceof ObjectSizeOf) {
+                                    size = ((ObjectSizeOf) o).sizeInBytes();
                                 } else {
-                                    reference += length;
-                                    for (int i = 0, index = length - 1; i < length; i++, index--) {
-                                        Object element = Array.get(o, index);
-                                        if (element != null) {
-                                            size += getObjectSize(
-                                                    new ObjectInstance(element, oi.getName() + "[" + index + "]"),
-                                                    visited, indent + 2, useSizeInBytesMethod, output);
-                                        }
-                                    }
-                                }
-                            } else {
-                                objectOverhead++;
-                                while (c != null) {
-                                    for (Field field : c.getDeclaredFields()) {
-                                        if (Modifier.isStatic(field.getModifiers())) {
-                                            continue;
-                                        }
-                                        if (field.getType().isPrimitive()) {
-                                            long primSize = getPrimitiveObjectSize(field.getType());
-                                            if (output != null) {
-                                                output.add(StringUtils.repeat(' ', indent + 2) + field.getName() + " ["
-                                                        + primSize + "] (" + field.getType().getCanonicalName() + ")");
-                                            }
-                                            size += primSize;
-                                        } else {
-                                            reference++;
-                                            boolean accessible = field.isAccessible();
-                                            field.setAccessible(true);
-                                            try {
-                                                Object fieldObject = field.get(o);
-                                                if (fieldObject != null) {
-                                                    size += getObjectSize(
-                                                            new ObjectInstance(fieldObject, field.getName()), visited,
-                                                            indent + 2, useSizeInBytesMethod, output);
-                                                }
-                                            } catch (Exception e) {
-                                                // cannot get to field, so
-                                                // ignore it in this size
-                                                // calculation
-                                                e.printStackTrace();
-                                            }
-                                            field.setAccessible(accessible);
-                                        }
-                                    }
-                                    c = c.getSuperclass();
+                                    Method sizeInBytes = o.getClass().getMethod("sizeInBytes", (Class<?>[]) null);
+                                    size = (Long) sizeInBytes.invoke(o);
                                 }
+                            } catch (Throwable t) {
+                                // ok, lets do this the hard way...
                             }
-                            roundUp = true;
                         }
+                        if (size == 0) {
+                            // the hard way...
+                            // do not include Class related objects or
+                            // reflection objects
+                            if ((o instanceof Class) || (o instanceof ClassLoader) || (o.getClass().getPackage() != null
+                                    && o.getClass().getPackage().getName().startsWith("java.lang.reflect"))) {
+                                size = 0;
+                            } else if (o instanceof Number || o instanceof Boolean || o instanceof Character) {
+                                size = NUMBER_SIZE;
+                            } else {
+                                // lets do a simple sizing
+                                Class<?> c = o.getClass();
+                                if (c.isArray()) {
+                                    size = ARRAY_OVERHEAD;
+                                    int length = Array.getLength(o);
+                                    if (c.getComponentType().isPrimitive()) {
+                                        size += length * getPrimitiveObjectSize(c.getComponentType());
+                                    } else {
+                                        size += length * REFERENCE;
+                                        for (int i = 0; i < length; i++) {
+                                            Object element = Array.get(o, i);
+                                            if (element != null) {
+                                                stack.add(new ObjectInstance(element));
+                                            }
+                                        }
+                                    }
+                                } else {
+                                    size += OBJECT_OVERHEAD;
+                                    while (c != null) {
+                                        for (Field field : c.getDeclaredFields()) {
+                                            if (Modifier.isStatic(field.getModifiers())) {
+                                                continue;
+                                            }
+                                            if (field.getType().isPrimitive()) {
+                                                size += getPrimitiveObjectSize(field.getType());
+                                            } else {
+                                                size += REFERENCE;
+                                                boolean accessible = field.isAccessible();
+                                                field.setAccessible(true);
+                                                try {
+                                                    Object fieldObject = field.get(o);
+                                                    if (fieldObject != null) {
+                                                        stack.push(new ObjectInstance(fieldObject));
+                                                    }
+                                                } catch (Exception e) {
+                                                    // cannot get to field, so
+                                                    // ignore it in this size
+                                                    // calculation
+                                                    e.printStackTrace();
+                                                }
+                                                field.setAccessible(accessible);
+                                            }
+                                        }
+                                        c = c.getSuperclass();
+                                    }
+                                }
+                                size = roundUp(size);
+                            }
+                        }
+                    } catch (Throwable t) {
+                        log.warning("Unable to determine object size for " + o);
                     }
-                    long underlyingObjects = size;
-                    long overheads = REFERENCE * reference + OBJECT_OVERHEAD * objectOverhead
-                            + ARRAY_OVERHEAD * arrayOverhead;
-                    long addedOverheads = size + overheads;
-                    if (roundUp) {
-                        size = roundUp(addedOverheads);
-                    } else {
-                        size = addedOverheads;
+                    if (debug) {
+                        System.out.println(o.getClass().getCanonicalName() + "=" + size);
                     }
-                    if (output != null) {
-                        output.add(StringUtils.repeat(' ', indent) + oi.getName() + " [" + size + "] ("
-                                + o.getClass().getCanonicalName() + ") = " + underlyingObjects + "+" + overheads + "+"
-                                + (size - addedOverheads) + " REF=" + reference + " OBJ=" + objectOverhead + " ARRAY="
-                                + arrayOverhead + " ROUND=" + (size - addedOverheads));
-                    }
-                } catch (Throwable t) {
-                    log.warning("Unable to determine object size for " + o);
                 }
+                totalSize += size;
             }
-            return size;
+
+            return totalSize;
         }
 
         public static long roundUp(long size) {
@@ -232,42 +191,6 @@ public interface ObjectSizeOf {
                 return 0;
             }
         }
-
-        public static long getStringSize(String s) {
-            long size = 0;
-            size += ObjectSizeOf.Sizer.OBJECT_OVERHEAD;
-            size += ObjectSizeOf.Sizer.REFERENCE;
-            size += ObjectSizeOf.Sizer.getPrimitiveObjectSize(int.class);
-            try {
-                // change with Java 9
-                if (String.class.getDeclaredField("coder") != null) {
-                    size += ObjectSizeOf.Sizer.getPrimitiveObjectSize(byte.class);
-                }
-            } catch (Exception e) {
-
-            }
-            size += roundUp(ARRAY_OVERHEAD + (s.length() * ObjectSizeOf.Sizer.getPrimitiveObjectSize(byte.class)));
-            return roundUp(size);
-        }
-    }
-
-    public static void main(String[] args) {
-
-        Metric m = new Metric("mymetric", 100000l, 32);
-        m.addTag(new Tag("key1", "value1"));
-        m.addTag(new Tag("key2", "value2"));
-        m.addTag(new Tag("key3", "value3"));
-        m.addTag(new Tag("key4", "value4"));
-        m.addTag(new Tag("key5", "value5"));
-
-        long newSize1 = ObjectSizeOf.Sizer.getObjectSize(m, false, true);
-        System.out.println("new size #1: " + newSize1);
-
-        System.out.println();
-        System.out.println();
-        System.out.println();
-        long newSize2 = ObjectSizeOf.Sizer.getObjectSize(m, true, true);
-        System.out.println("new size #2: " + newSize2);
     }
 
 }

client/src/main/java/timely/model/Tag.java

@@ -18,7 +18,7 @@ import timely.model.parse.TagParser;
  * Tag consists of key value pair
  */
 @JsonRootName("tag")
-public class Tag implements Comparable<Tag>, ObjectSizeOf, Serializable {
+public class Tag implements Comparable<Tag>, Serializable {
 
     private static final long serialVersionUID = 1L;
 
@@ -27,12 +27,6 @@ public class Tag implements Comparable<Tag>, ObjectSizeOf, Serializable {
     private String key;
     private String value;
 
-    @JsonIgnore
-    private boolean mustRecalculateLength = true;
-
-    @JsonIgnore
-    private long length;
-
     public Tag() {
     }
 
@@ -61,7 +55,6 @@ public class Tag implements Comparable<Tag>, ObjectSizeOf, Serializable {
     public void set(String key, String value) {
         this.key = key;
         this.value = value;
-        this.mustRecalculateLength = true;
     }
 
     @JsonIgnore
@@ -71,7 +64,6 @@ public class Tag implements Comparable<Tag>, ObjectSizeOf, Serializable {
 
     public void setKey(String key) {
         this.key = key;
-        this.mustRecalculateLength = true;
     }
 
     @JsonIgnore
@@ -81,7 +73,6 @@ public class Tag implements Comparable<Tag>, ObjectSizeOf, Serializable {
 
     public void setValue(String value) {
         this.value = value;
-        this.mustRecalculateLength = true;
     }
 
     /**
@@ -115,26 +106,4 @@ public class Tag implements Comparable<Tag>, ObjectSizeOf, Serializable {
     public int compareTo(Tag other) {
         return new CompareToBuilder().append(this.key, other.key).append(this.value, other.value).toComparison();
     }
-
-    @JsonIgnore
-    @Override
-    public long sizeInBytes() {
-        if (this.mustRecalculateLength) {
-            calculateLength();
-            this.mustRecalculateLength = false;
-        }
-        return this.length;
-    }
-
-    private void calculateLength() {
-        long l = 0;
-        l += 2 * Sizer.REFERENCE;
-        l += Sizer.OBJECT_OVERHEAD;
-        l += Sizer.getPrimitiveObjectSize(long.class);
-        l += Sizer.getPrimitiveObjectSize(boolean.class);
-        l += Sizer.getStringSize(key);
-        l += Sizer.getStringSize(value);
-        this.length = Sizer.roundUp(l);
-    }
-
 }

client/src/main/java/timely/model/Value.java

@@ -5,14 +5,13 @@ import java.util.Comparator;
 import java.util.concurrent.TimeUnit;
 
 import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.google.common.base.Objects;
 
 /**
  * Value consists of a long timestamp and a double measurement
  */
-public class Value implements Comparator<Value>, ObjectSizeOf, Serializable {
+public class Value implements Comparator<Value>, Serializable {
 
     private static final long serialVersionUID = 1L;
 
@@ -91,15 +90,4 @@ public class Value implements Comparator<Value>, ObjectSizeOf, Serializable {
     public String toString() {
         return "Value{" + "timestamp=" + timestamp + ", measure=" + measure + "}";
     }
-
-    @JsonIgnore
-    @Override
-    public long sizeInBytes() {
-        long l = 0;
-        l += Sizer.OBJECT_OVERHEAD;
-        l += 2 * Sizer.REFERENCE;
-        l += Sizer.NUMBER_SIZE;
-        l += Sizer.NUMBER_SIZE;
-        return Sizer.roundUp(l);
-    }
 }

server/pom.xml

@@ -108,6 +108,10 @@
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-minicluster</artifactId>
     </dependency>
+    <dependency>
+      <groupId>org.glassfish.tyrus</groupId>
+      <artifactId>tyrus-client</artifactId>
+    </dependency>
     <dependency>
       <groupId>org.slf4j</groupId>
       <artifactId>slf4j-api</artifactId>

server/src/main/java/timely/Server.java

@@ -132,7 +132,6 @@ public class Server {
     private static final String OS_VERSION = "os.version";
     private static final String WS_PATH = "/websocket";
 
-    private SslContext sslCtx = null;
     private CuratorFramework curatorFramework;
     protected static final CountDownLatch LATCH = new CountDownLatch(1);
     static ConfigurableApplicationContext applicationContext;
@@ -239,7 +238,7 @@ public class Server {
 
         Server server = new Server(conf);
         try {
-            server.run(createSSLContext(conf));
+            server.run();
             LATCH.await();
         } catch (final InterruptedException e) {
             LOG.info("Server shutting down.");
@@ -273,6 +272,8 @@ public class Server {
     }
 
     public void shutdown() {
+        LOG.warn("Server shutdown 1");
+
         List<ChannelFuture> channelFutures = new ArrayList<>();
 
         if (tcpChannelHandle != null) {
@@ -305,6 +306,8 @@ public class Server {
             }
         });
 
+        LOG.warn("Server shutdown 2");
+
         int quietPeriod = config.getServer().getShutdownQuietPeriod();
         List<Future<?>> groupFutures = new ArrayList<>();
 
@@ -343,6 +346,8 @@ public class Server {
             groupFutures.add(udpBossGroup.shutdownGracefully(quietPeriod, 10, TimeUnit.SECONDS));
         }
 
+        LOG.warn("Server shutdown 3");
+
         if (udpWorkerGroup != null) {
             LOG.info("Shutting down udpWorkerGroup");
             groupFutures.add(udpWorkerGroup.shutdownGracefully(quietPeriod, 10, TimeUnit.SECONDS));
@@ -356,15 +361,17 @@ public class Server {
             }
         });
 
-        if (dataStore != null) {
-            try {
-                LOG.info("Closing dataStore");
-                dataStore.close();
-            } catch (Exception e) {
-                LOG.error(e.getMessage(), e);
-            }
+        LOG.warn("Server shutdown 4");
+
+        try {
+            LOG.info("Closing dataStore");
+            dataStore.close();
+        } catch (Exception e) {
+            LOG.error(e.getMessage(), e);
         }
 
+        LOG.warn("Server shutdown 5");
+
         if (dataStoreCache != null) {
             try {
                 LOG.info("Closing dataStoreCache");
@@ -374,6 +381,8 @@ public class Server {
             }
         }
 
+        LOG.warn("Server shutdown 6");
+
         try {
             LOG.info("Closing metaCacheFactory");
             MetaCacheFactory.close();
@@ -381,6 +390,8 @@ public class Server {
             LOG.error(e.getMessage(), e);
         }
 
+        LOG.warn("Server shutdown 7");
+
         try {
             LOG.info("Closing webSocketRequestDecoder subscriptions");
             WebSocketRequestDecoder.close();
@@ -388,6 +399,8 @@ public class Server {
             LOG.error(e.getMessage(), e);
         }
 
+        LOG.warn("Server shutdown 8");
+
         if (curatorFramework != null) {
             try {
                 LOG.info("Closing curatorFramework");
@@ -397,6 +410,8 @@ public class Server {
             }
         }
 
+        LOG.warn("Server shutdown 9");
+
         if (applicationContext != null) {
             try {
                 LOG.info("Closing applicationContext");
@@ -405,6 +420,7 @@ public class Server {
                 LOG.error(e.getMessage(), e);
             }
         }
+        LOG.warn("Server shutdown 10");
         this.shutdown = true;
         LOG.info("Server shut down.");
     }
@@ -436,7 +452,7 @@ public class Server {
         }
     }
 
-    public void run(SslContext sslCtx) throws Exception {
+    public void run() throws Exception {
 
         RetryPolicy retryPolicy = new RetryForever(1000);
         int timeout = Long.valueOf(AccumuloConfiguration.getTimeInMillis(config.getAccumulo().getZookeeperTimeout()))
@@ -503,6 +519,7 @@ public class Server {
 
         final int httpPort = config.getHttp().getPort();
         final String httpIp = config.getHttp().getIp();
+        SslContext sslCtx = createSSLContext(config);
         if (sslCtx instanceof OpenSslServerContext) {
             OpenSslServerContext openssl = (OpenSslServerContext) sslCtx;
             String application = "Timely_" + httpPort;
@@ -559,7 +576,7 @@ public class Server {
                 tcpAddress, tcpPort, httpAddress, httpPort, wsAddress, wsPort, wsAddress, udpPort);
     }
 
-    public static SslContext createSSLContext(Configuration config) throws Exception {
+    protected SslContext createSSLContext(Configuration config) throws Exception {
 
         ServerSsl sslCfg = config.getSecurity().getServerSsl();
         Boolean generate = sslCfg.isUseGeneratedKeypair();

server/src/main/java/timely/StandaloneServer.java

@@ -85,7 +85,7 @@ public class StandaloneServer extends Server {
         try {
             LOG.info("Starting StandaloneServer");
             StandaloneServer s = new StandaloneServer(conf);
-            s.run(createSSLContext(conf));
+            s.run();
         } catch (Exception e) {
             System.err.println("Error starting server");
             e.printStackTrace();

server/src/main/java/timely/adapter/accumulo/MetricAdapter.java

@@ -8,7 +8,6 @@ import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
-import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
 import com.google.common.base.Joiner;
@@ -40,8 +39,6 @@ public class MetricAdapter {
 
     private static final TagParser tagParser = new TagParser();
     private static final TagListParser tagListParser = new TagListParser();
-    private static Pattern equalsPattern = Pattern.compile("=");
-    private static Pattern commaPattern = Pattern.compile(",");
 
     public static final ColumnVisibility EMPTY_VISIBILITY = new ColumnVisibility();
     public static final String VISIBILITY_TAG = "viz";
@@ -62,10 +59,10 @@ public class MetricAdapter {
             // are used as a delimiter
             String k = t.getKey();
             String v = t.getValue();
-            k = equalsPattern.matcher(k).replaceAll("\\=");
-            k = commaPattern.matcher(k).replaceAll("\\,");
-            v = equalsPattern.matcher(v).replaceAll("\\=");
-            v = commaPattern.matcher(v).replaceAll("\\,");
+            k = k.replaceAll("=", "\\=");
+            k = k.replaceAll(",", "\\,");
+            v = v.replaceAll("=", "\\=");
+            v = v.replaceAll(",", "\\,");
             newTags.add(new Tag(k, v));
         }
         return newTags;
@@ -78,10 +75,10 @@ public class MetricAdapter {
             // are used as a delimiter
             String k = t.getKey();
             String v = t.getValue();
-            k = equalsPattern.matcher(k).replaceAll("\\=");
-            k = commaPattern.matcher(k).replaceAll("\\,");
-            v = equalsPattern.matcher(v).replaceAll("\\=");
-            v = commaPattern.matcher(v).replaceAll("\\,");
+            k = k.replaceAll("=", "\\=");
+            k = k.replaceAll(",", "\\,");
+            v = v.replaceAll("=", "\\=");
+            v = v.replaceAll(",", "\\,");
             newTags.put(k, v);
         }
         return newTags;

server/src/main/java/timely/configuration/ServerSsl.java

@@ -1,7 +1,6 @@
 package timely.configuration;
 
 import java.util.List;
-import java.util.Objects;
 
 import com.google.common.collect.Lists;
 import timely.validator.NotEmptyIfFieldSet;
@@ -77,24 +76,4 @@ public class ServerSsl {
     public void setUseCiphers(List<String> useCiphers) {
         this.useCiphers = useCiphers;
     }
-
-    @Override
-    public boolean equals(Object o) {
-        if (this == o)
-            return true;
-        if (o == null || getClass() != o.getClass())
-            return false;
-        ServerSsl serverSsl = (ServerSsl) o;
-        return useGeneratedKeypair == serverSsl.useGeneratedKeypair && useOpenssl == serverSsl.useOpenssl
-                && Objects.equals(certificateFile, serverSsl.certificateFile)
-                && Objects.equals(keyFile, serverSsl.keyFile) && Objects.equals(keyPassword, serverSsl.keyPassword)
-                && Objects.equals(trustStoreFile, serverSsl.trustStoreFile)
-                && Objects.equals(useCiphers, serverSsl.useCiphers);
-    }
-
-    @Override
-    public int hashCode() {
-        return Objects.hash(certificateFile, keyFile, keyPassword, trustStoreFile, useGeneratedKeypair, useOpenssl,
-                useCiphers);
-    }
 }

server/src/test/java/timely/sample/iterators/DownsampleIteratorTest.java

@@ -4,15 +4,8 @@ import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
 
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
 import java.util.Map.Entry;
-import java.util.Set;
-import java.util.SortedMap;
-import java.util.TreeMap;
 
 import org.apache.accumulo.core.client.IteratorSetting;
 import org.apache.accumulo.core.data.ColumnUpdate;
@@ -258,52 +251,6 @@ public class DownsampleIteratorTest {
         assertEquals((elapsedTime / sampleInterval) * numTagVariations, totalBuckets);
     }
 
-    @Test
-    public void test() {
-        String hello = new String("this is a longer string");
-        long maybe = ObjectSizeOf.Sizer.OBJECT_OVERHEAD + // for String
-                ObjectSizeOf.Sizer.REFERENCE + // for array of char
-                ObjectSizeOf.Sizer.getPrimitiveObjectSize(int.class) +
-
-                ObjectSizeOf.Sizer.ARRAY_OVERHEAD + // for array of char
-                hello.length() * ObjectSizeOf.Sizer.getPrimitiveObjectSize(char.class);
-
-        maybe = ObjectSizeOf.Sizer.roundUp(maybe);
-
-        long size = ObjectSizeOf.Sizer.getObjectSize(hello, true, true);
-        System.out.println("size = " + size);
-        System.out.println("maybe = " + maybe);
-    }
-
-    @Test
-    public void test2() {
-        System.out.println(ObjectSizeOf.Sizer.getObjectSize(new Long(100)));
-    }
-
-    @Test
-    public void testDownsampleMemoryCalculation() throws Exception {
-
-        int numTagVariations = 1;
-        int sampleInterval = 50;
-        int elapsedTime = 100;
-        int skipInterval = 10;
-        SortedMap<Key, Value> testData3 = createTestData3(elapsedTime, skipInterval, numTagVariations);
-        DownsampleIterator iter = new DownsampleIterator();
-        Map<Set<Tag>, Downsample> samples = runQuery(iter, testData3, sampleInterval, 1000);
-        assertEquals(numTagVariations, samples.size());
-        long totalBuckets = 0;
-        for (Entry<Set<Tag>, Downsample> entry : samples.entrySet()) {
-            Set<Tag> tagSet = entry.getKey();
-            Downsample downsample = entry.getValue();
-            for (Tag tag : tagSet) {
-                assertEquals(ObjectSizeOf.Sizer.getObjectSize(tag, true, true),
-                        ObjectSizeOf.Sizer.getObjectSize(tag, false, true));
-            }
-
-        }
-
-    }
-
     private Map<Set<Tag>, Downsample> runQuery(SortedKeyValueIterator<Key, Value> iter, SortedMap<Key, Value> testData,
             long period, long maxDownsampleMemory) throws Exception {
         IteratorSetting is = new IteratorSetting(100, DownsampleIterator.class);

server/src/test/java/timely/test/integration/DataStoreIT.java

@@ -59,7 +59,7 @@ public class DataStoreIT extends OneWaySSLBase {
         conf.setMetricAgeOffDays(ageOffSettings);
 
         final Server s = new Server(conf);
-        s.run(getSslContext());
+        s.run();
         try {
             put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2",
                     "sys.cpu.user " + (TEST_TIME + ONE_HOUR) + " 3.0 tag1=value1 tag2=value2",
@@ -106,7 +106,7 @@ public class DataStoreIT extends OneWaySSLBase {
         conf.setMetricAgeOffDays(ageOffSettings);
 
         final Server s = new Server(conf);
-        s.run(getSslContext());
+        s.run();
         // @formatter:off
         /*
          * TEST_TIME = System.currentTimeMillis() - ONE_DAY

server/src/test/java/timely/test/integration/MacITBase.java

@@ -5,8 +5,6 @@ import java.io.IOException;
 import java.nio.file.Files;
 import java.util.HashMap;
 
-import io.netty.handler.ssl.SslContext;
-import io.netty.handler.ssl.util.SelfSignedCertificate;
 import org.apache.accumulo.core.client.Connector;
 import org.apache.accumulo.minicluster.MiniAccumuloCluster;
 import org.apache.accumulo.minicluster.MiniAccumuloConfig;
@@ -14,10 +12,8 @@ import org.junit.Before;
 import org.junit.BeforeClass;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import timely.Server;
 import timely.auth.AuthCache;
 import timely.configuration.Configuration;
-import timely.configuration.ServerSsl;
 import timely.store.MetricAgeOffIterator;
 import timely.test.TestConfiguration;
 
@@ -28,11 +24,6 @@ public class MacITBase {
 
     private static final Logger LOG = LoggerFactory.getLogger(MacITBase.class);
 
-    protected static ServerSsl serverSsl = null;
-    protected static File clientTrustStoreFile = null;
-    protected static SslContext sslCtx = null;
-    protected static SelfSignedCertificate serverCert = null;
-
     private static final File tempDir;
 
     protected static final String MAC_ROOT_USER = "root";
@@ -59,6 +50,8 @@ public class MacITBase {
             conf = TestConfiguration.createMinimalConfigurationForTest();
             conf.getAccumulo().setInstanceName(mac.getInstanceName());
             conf.getAccumulo().setZookeepers(mac.getZooKeepers());
+            conf.getSecurity().getServerSsl().setUseOpenssl(false);
+            conf.getSecurity().getServerSsl().setUseGeneratedKeypair(true);
             conf.getWebsocket().setFlushIntervalSeconds(TestConfiguration.WAIT_SECONDS);
             HashMap<String, Integer> ageOffSettings = new HashMap<>();
             ageOffSettings.put(MetricAgeOffIterator.DEFAULT_AGEOFF_KEY, 7);
@@ -95,52 +88,4 @@ public class MacITBase {
     public void initializeAuthCache() {
         AuthCache.clear();
     }
-
-    protected static void setupSSL() throws Exception {
-        setupSSL(TestConfiguration.createMinimalConfigurationForTest(), false);
-    }
-
-    protected static void setupSSL(Configuration config, boolean twoWaySsl) throws Exception {
-        setupSSL(config, twoWaySsl, null);
-    }
-
-    protected static void setupSSL(Configuration config, boolean twoWaySsl, SelfSignedCertificate serverCert)
-            throws Exception {
-        ServerSsl serverSsl = config.getSecurity().getServerSsl();
-        boolean generateNewSsl = MacITBase.serverSsl == null || serverCert != null
-                || !MacITBase.serverSsl.equals(serverSsl);
-
-        if (generateNewSsl) {
-            // new serverCert and trustStoreFile
-            if (serverCert == null) {
-                MacITBase.serverCert = new SelfSignedCertificate();
-            } else {
-                MacITBase.serverCert = serverCert;
-            }
-            MacITBase.clientTrustStoreFile = MacITBase.serverCert.certificate().getAbsoluteFile();
-            MacITBase.serverSsl = serverSsl;
-            MacITBase.serverSsl.setCertificateFile(MacITBase.serverCert.certificate().getAbsolutePath());
-            MacITBase.serverSsl.setKeyFile(MacITBase.serverCert.privateKey().getAbsolutePath());
-        }
-
-        config.getSecurity().getServerSsl().setCertificateFile(MacITBase.serverSsl.getCertificateFile());
-        config.getSecurity().getServerSsl().setKeyFile(MacITBase.serverSsl.getKeyFile());
-        config.getSecurity().getServerSsl().setUseOpenssl(false);
-        config.getSecurity().getServerSsl().setUseGeneratedKeypair(false);
-        if (twoWaySsl) {
-            // Needed for 2way SSL
-            config.getSecurity().getServerSsl().setTrustStoreFile(MacITBase.serverSsl.getCertificateFile());
-            config.getSecurity().setAllowAnonymousHttpAccess(false);
-        } else {
-            config.getSecurity().setAllowAnonymousHttpAccess(true);
-        }
-        if (generateNewSsl) {
-            // new SslContext
-            OneWaySSLBase.sslCtx = Server.createSSLContext(config);
-        }
-    }
-
-    protected static SslContext getSslContext() {
-        return MacITBase.sslCtx;
-    }
 }

server/src/test/java/timely/test/integration/OneWaySSLAnonAccessIT.java

@@ -30,7 +30,7 @@ public class OneWaySSLAnonAccessIT extends OneWaySSLBase {
     @Test
     public void testQueryWithVisibility() throws Exception {
         final Server s = new Server(conf);
-        s.run(getSslContext());
+        s.run();
         try {
             put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2",
                     "sys.cpu.user " + (TEST_TIME + 1000) + " 3.0 tag1=value1 tag2=value2",

server/src/test/java/timely/test/integration/OneWaySSLBase.java

@@ -1,5 +1,6 @@
 package timely.test.integration;
 
+import java.io.File;
 import java.net.URL;
 
 import javax.net.ssl.HttpsURLConnection;
@@ -11,21 +12,25 @@ import io.netty.handler.ssl.JdkSslContext;
 import io.netty.handler.ssl.SslContext;
 import io.netty.handler.ssl.SslContextBuilder;
 import io.netty.handler.ssl.SslProvider;
+import io.netty.handler.ssl.util.SelfSignedCertificate;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
 import timely.auth.AuthCache;
+import timely.configuration.Configuration;
 
 /**
  * Base test class for SSL with anonymous access
  */
 public class OneWaySSLBase extends QueryBase {
 
+    protected static File clientTrustStoreFile = null;
+
     protected SSLSocketFactory getSSLSocketFactory() throws Exception {
         SslContextBuilder builder = SslContextBuilder.forClient();
         builder.applicationProtocolConfig(ApplicationProtocolConfig.DISABLED);
         builder.sslProvider(SslProvider.JDK);
-        builder.trustManager(MacITBase.clientTrustStoreFile); // Trust the server cert
+        builder.trustManager(clientTrustStoreFile); // Trust the server cert
         SslContext ctx = builder.build();
         Assert.assertTrue(ctx.isClient());
         Assert.assertTrue(ctx instanceof JdkSslContext);
@@ -34,9 +39,19 @@ public class OneWaySSLBase extends QueryBase {
         return jdkSslContext.getSocketFactory();
     }
 
+    protected static void setupSSL(Configuration config) throws Exception {
+        SelfSignedCertificate serverCert = new SelfSignedCertificate();
+        config.getSecurity().getServerSsl().setCertificateFile(serverCert.certificate().getAbsolutePath());
+        clientTrustStoreFile = serverCert.certificate().getAbsoluteFile();
+        config.getSecurity().getServerSsl().setKeyFile(serverCert.privateKey().getAbsolutePath());
+        config.getSecurity().getServerSsl().setUseOpenssl(false);
+        config.getSecurity().getServerSsl().setUseGeneratedKeypair(false);
+        config.getSecurity().setAllowAnonymousHttpAccess(true);
+    }
+
     @Before
     public void configureSSL() throws Exception {
-        setupSSL(conf, false);
+        setupSSL(conf);
     }
 
     @Override

server/src/test/java/timely/test/integration/OneWaySSLBasicAuthAccessIT.java

@@ -93,7 +93,7 @@ public class OneWaySSLBasicAuthAccessIT extends OneWaySSLBase {
     @Test
     public void testBasicAuthLogin() throws Exception {
         final Server s = new Server(conf);
-        s.run(getSslContext());
+        s.run();
         try {
             String metrics = "https://localhost:54322/api/metrics";
             query(metrics);
@@ -105,7 +105,7 @@ public class OneWaySSLBasicAuthAccessIT extends OneWaySSLBase {
     @Test(expected = UnauthorizedUserException.class)
     public void testBasicAuthLoginFailure() throws Exception {
         final Server s = new Server(conf);
-        s.run(getSslContext());
+        s.run();
         try {
             String metrics = "https://localhost:54322/api/metrics";
             query("test", "test2", metrics);
@@ -117,7 +117,7 @@ public class OneWaySSLBasicAuthAccessIT extends OneWaySSLBase {
     @Test
     public void testQueryWithVisibility() throws Exception {
         final Server s = new Server(conf);
-        s.run(getSslContext());
+        s.run();
         try {
             put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2",
                     "sys.cpu.user " + (TEST_TIME + 1000) + " 3.0 tag1=value1 tag2=value2",

server/src/test/java/timely/test/integration/StaticFileServerIT.java

@@ -13,7 +13,7 @@ public class StaticFileServerIT extends OneWaySSLBase {
     @Before
     public void before() throws Exception {
         s = new Server(conf);
-        s.run(getSslContext());
+        s.run();
     }
 
     @After

server/src/test/java/timely/test/integration/TwoWaySSLBase.java

@@ -1,92 +0,0 @@
-package timely.test.integration;
-
-import java.net.URL;
-import java.util.List;
-
-import javax.net.ssl.HostnameVerifier;
-import javax.net.ssl.HttpsURLConnection;
-import javax.net.ssl.SSLContext;
-import javax.net.ssl.SSLSession;
-import javax.net.ssl.SSLSocketFactory;
-
-import io.netty.handler.codec.http.HttpHeaderNames;
-import io.netty.handler.codec.http.cookie.ClientCookieDecoder;
-import io.netty.handler.codec.http.cookie.Cookie;
-import io.netty.handler.ssl.ApplicationProtocolConfig;
-import io.netty.handler.ssl.JdkSslContext;
-import io.netty.handler.ssl.SslContext;
-import io.netty.handler.ssl.SslContextBuilder;
-import io.netty.handler.ssl.SslProvider;
-import org.junit.After;
-import org.junit.Assert;
-import timely.auth.AuthCache;
-import timely.netty.Constants;
-
-/**
- * Base test class for SSL with anonymous access
- */
-public class TwoWaySSLBase extends QueryBase {
-
-    @After
-    public void tearDown() throws Exception {
-        AuthCache.resetConfiguration();
-    }
-
-    protected SSLSocketFactory getSSLSocketFactory() throws Exception {
-        SslContextBuilder builder = SslContextBuilder.forClient();
-        builder.applicationProtocolConfig(ApplicationProtocolConfig.DISABLED);
-        // Use server cert / key on client side
-        builder.keyManager(MacITBase.serverCert.key(), (String) null, MacITBase.serverCert.cert());
-        builder.sslProvider(SslProvider.JDK);
-        builder.trustManager(MacITBase.clientTrustStoreFile); // Trust the server cert
-        SslContext ctx = builder.build();
-        Assert.assertTrue(ctx.isClient());
-        Assert.assertTrue(ctx instanceof JdkSslContext);
-        JdkSslContext jdk = (JdkSslContext) ctx;
-        SSLContext jdkSslContext = jdk.context();
-        return jdkSslContext.getSocketFactory();
-    }
-
-    @Override
-    protected HttpsURLConnection getUrlConnection(URL url) throws Exception {
-        // Username and password not used in 2way SSL case
-        return getUrlConnection(null, null, url);
-    }
-
-    @Override
-    protected HttpsURLConnection getUrlConnection(String username, String password, URL url) throws Exception {
-        HttpsURLConnection.setDefaultSSLSocketFactory(getSSLSocketFactory());
-        URL loginURL = new URL(url.getProtocol() + "://" + url.getHost() + ":" + url.getPort() + "/login");
-        HttpsURLConnection con = (HttpsURLConnection) loginURL.openConnection();
-        con.setHostnameVerifier(new HostnameVerifier() {
-
-            @Override
-            public boolean verify(String arg0, SSLSession arg1) {
-                return true;
-            }
-        });
-        con.setRequestMethod("GET");
-        con.setDoOutput(true);
-        con.setRequestProperty("Content-Type", "application/json");
-        con.connect();
-        int responseCode = con.getResponseCode();
-        if (401 == responseCode) {
-            throw new UnauthorizedUserException();
-        }
-        Assert.assertEquals(200, responseCode);
-        List<String> cookies = con.getHeaderFields().get(HttpHeaderNames.SET_COOKIE.toString());
-        Assert.assertEquals(1, cookies.size());
-        Cookie sessionCookie = ClientCookieDecoder.STRICT.decode(cookies.get(0));
-        Assert.assertEquals(Constants.COOKIE_NAME, sessionCookie.name());
-        con = (HttpsURLConnection) url.openConnection();
-        con.setRequestProperty(HttpHeaderNames.COOKIE.toString(), sessionCookie.name() + "=" + sessionCookie.value());
-        con.setHostnameVerifier(new HostnameVerifier() {
-
-            @Override
-            public boolean verify(String arg0, SSLSession arg1) {
-                return true;
-            }
-        });
-        return con;
-    }
-}

server/src/test/java/timely/test/integration/TwoWaySSLFailureIT.java

@@ -1,11 +1,30 @@
 package timely.test.integration;
 
+import java.io.File;
+import java.net.URL;
+import java.util.List;
+
+import javax.net.ssl.HostnameVerifier;
+import javax.net.ssl.HttpsURLConnection;
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.SSLSession;
+import javax.net.ssl.SSLSocketFactory;
+
+import io.netty.handler.codec.http.HttpHeaderNames;
+import io.netty.handler.codec.http.cookie.ClientCookieDecoder;
+import io.netty.handler.codec.http.cookie.Cookie;
+import io.netty.handler.ssl.ApplicationProtocolConfig;
+import io.netty.handler.ssl.JdkSslContext;
+import io.netty.handler.ssl.SslContext;
+import io.netty.handler.ssl.SslContextBuilder;
+import io.netty.handler.ssl.SslProvider;
 import io.netty.handler.ssl.util.SelfSignedCertificate;
 import org.apache.accumulo.core.client.Connector;
 import org.apache.accumulo.minicluster.MiniAccumuloCluster;
 import org.apache.accumulo.minicluster.MiniAccumuloConfig;
 import org.junit.After;
 import org.junit.AfterClass;
+import org.junit.Assert;
 import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.ClassRule;
@@ -14,9 +33,10 @@ import org.junit.rules.TemporaryFolder;
 import timely.Server;
 import timely.auth.AuthCache;
 import timely.configuration.Configuration;
+import timely.netty.Constants;
 import timely.test.TestConfiguration;
 
-public class TwoWaySSLFailureIT extends TwoWaySSLBase {
+public class TwoWaySSLFailureIT extends QueryBase {
 
     @ClassRule
     public static final TemporaryFolder temp = new TemporaryFolder();
@@ -24,6 +44,87 @@ public class TwoWaySSLFailureIT extends TwoWaySSLBase {
     private static MiniAccumuloCluster mac = null;
     private static Configuration conf = null;
 
+    private static SelfSignedCertificate serverCert = null;
+    private static File clientTrustStoreFile = null;
+
+    static {
+        try {
+            // This fqdn does not match what is in security.xml
+            serverCert = new SelfSignedCertificate("CN=bad.example.com");
+            clientTrustStoreFile = serverCert.certificate().getAbsoluteFile();
+        } catch (Exception e) {
+            throw new RuntimeException("Error creating self signed certificate", e);
+        }
+    }
+
+    protected SSLSocketFactory getSSLSocketFactory() throws Exception {
+        SslContextBuilder builder = SslContextBuilder.forClient();
+        builder.applicationProtocolConfig(ApplicationProtocolConfig.DISABLED);
+        // Use server cert / key on client side
+        builder.keyManager(serverCert.key(), (String) null, serverCert.cert());
+        builder.sslProvider(SslProvider.JDK);
+        builder.trustManager(clientTrustStoreFile); // Trust the server cert
+        SslContext ctx = builder.build();
+        Assert.assertTrue(ctx.isClient());
+        Assert.assertTrue(ctx instanceof JdkSslContext);
+        JdkSslContext jdk = (JdkSslContext) ctx;
+        SSLContext jdkSslContext = jdk.context();
+        return jdkSslContext.getSocketFactory();
+    }
+
+    protected static void setupSSL(Configuration config) throws Exception {
+        config.getSecurity().getServerSsl().setCertificateFile(serverCert.certificate().getAbsolutePath());
+        config.getSecurity().getServerSsl().setKeyFile(serverCert.privateKey().getAbsolutePath());
+        // Needed for 2way SSL
+        config.getSecurity().getServerSsl().setTrustStoreFile(serverCert.certificate().getAbsolutePath());
+        config.getSecurity().getServerSsl().setUseOpenssl(false);
+        config.getSecurity().getServerSsl().setUseGeneratedKeypair(false);
+        config.getSecurity().setAllowAnonymousHttpAccess(false);
+    }
+
+    @Override
+    protected HttpsURLConnection getUrlConnection(URL url) throws Exception {
+        // Username and password not used in 2way SSL case
+        return getUrlConnection(null, null, url);
+    }
+
+    @Override
+    protected HttpsURLConnection getUrlConnection(String username, String password, URL url) throws Exception {
+        HttpsURLConnection.setDefaultSSLSocketFactory(getSSLSocketFactory());
+        URL loginURL = new URL(url.getProtocol() + "://" + url.getHost() + ":" + url.getPort() + "/login");
+        HttpsURLConnection con = (HttpsURLConnection) loginURL.openConnection();
+        con.setHostnameVerifier(new HostnameVerifier() {
+
+            @Override
+            public boolean verify(String arg0, SSLSession arg1) {
+                return true;
+            }
+        });
+        con.setRequestMethod("GET");
+        con.setDoOutput(true);
+        con.setRequestProperty("Content-Type", "application/json");
+        con.connect();
+        int responseCode = con.getResponseCode();
+        if (401 == responseCode) {
+            throw new UnauthorizedUserException();
+        }
+        Assert.assertEquals(200, responseCode);
+        List<String> cookies = con.getHeaderFields().get(HttpHeaderNames.SET_COOKIE.toString());
+        Assert.assertEquals(1, cookies.size());
+        Cookie sessionCookie = ClientCookieDecoder.STRICT.decode(cookies.get(0));
+        Assert.assertEquals(Constants.COOKIE_NAME, sessionCookie.name());
+        con = (HttpsURLConnection) url.openConnection();
+        con.setRequestProperty(HttpHeaderNames.COOKIE.toString(), sessionCookie.name() + "=" + sessionCookie.value());
+        con.setHostnameVerifier(new HostnameVerifier() {
+
+            @Override
+            public boolean verify(String arg0, SSLSession arg1) {
+                return true;
+            }
+        });
+        return con;
+    }
+
     @BeforeClass
     public static void beforeClass() throws Exception {
         temp.create();
@@ -33,8 +134,7 @@ public class TwoWaySSLFailureIT extends TwoWaySSLBase {
         conf = TestConfiguration.createMinimalConfigurationForTest();
         conf.getAccumulo().setInstanceName(mac.getInstanceName());
         conf.getAccumulo().setZookeepers(mac.getZooKeepers());
-        // This fqdn does not match what is in security.xml
-        setupSSL(conf, true, new SelfSignedCertificate("CN=bad.example.com"));
+        setupSSL(conf);
     }
 
     @AfterClass
@@ -63,7 +163,7 @@ public class TwoWaySSLFailureIT extends TwoWaySSLBase {
     @Test(expected = UnauthorizedUserException.class)
     public void testBasicAuthLoginFailure() throws Exception {
         final Server s = new Server(conf);
-        s.run(getSslContext());
+        s.run();
         try {
             String metrics = "https://localhost:54322/api/metrics";
             query(metrics);

server/src/test/java/timely/test/integration/TwoWaySSLIT.java

@@ -3,17 +3,33 @@ package timely.test.integration;
 import static com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly;
 import static org.junit.Assert.assertEquals;
 
+import java.io.File;
+import java.net.URL;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.concurrent.TimeUnit;
 
+import javax.net.ssl.HttpsURLConnection;
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.SSLSocketFactory;
+
+import io.netty.handler.codec.http.HttpHeaderNames;
+import io.netty.handler.codec.http.cookie.ClientCookieDecoder;
+import io.netty.handler.codec.http.cookie.Cookie;
+import io.netty.handler.ssl.ApplicationProtocolConfig;
+import io.netty.handler.ssl.JdkSslContext;
+import io.netty.handler.ssl.SslContext;
+import io.netty.handler.ssl.SslContextBuilder;
+import io.netty.handler.ssl.SslProvider;
+import io.netty.handler.ssl.util.SelfSignedCertificate;
 import org.apache.accumulo.core.client.Connector;
 import org.apache.accumulo.core.security.Authorizations;
 import org.apache.accumulo.minicluster.MiniAccumuloCluster;
 import org.apache.accumulo.minicluster.MiniAccumuloConfig;
 import org.junit.After;
 import org.junit.AfterClass;
+import org.junit.Assert;
 import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.ClassRule;
@@ -25,6 +41,7 @@ import timely.api.request.timeseries.QueryRequest;
 import timely.api.response.timeseries.QueryResponse;
 import timely.auth.AuthCache;
 import timely.configuration.Configuration;
+import timely.netty.Constants;
 import timely.test.IntegrationTest;
 import timely.test.TestConfiguration;
 
@@ -34,7 +51,7 @@ import timely.test.TestConfiguration;
  *
  */
 @Category(IntegrationTest.class)
-public class TwoWaySSLIT extends TwoWaySSLBase {
+public class TwoWaySSLIT extends QueryBase {
 
     private static final Long TEST_TIME = System.currentTimeMillis();
 
@@ -44,6 +61,74 @@ public class TwoWaySSLIT extends TwoWaySSLBase {
     private static MiniAccumuloCluster mac = null;
     private static Configuration conf = null;
 
+    protected static SelfSignedCertificate serverCert = null;
+    protected static File clientTrustStoreFile = null;
+
+    static {
+        try {
+            serverCert = new SelfSignedCertificate();
+            clientTrustStoreFile = serverCert.certificate().getAbsoluteFile();
+        } catch (Exception e) {
+            throw new RuntimeException("Error creating self signed certificate", e);
+        }
+    }
+
+    protected SSLSocketFactory getSSLSocketFactory() throws Exception {
+        SslContextBuilder builder = SslContextBuilder.forClient();
+        builder.applicationProtocolConfig(ApplicationProtocolConfig.DISABLED);
+        // Use server cert / key on client side.
+        builder.keyManager(serverCert.key(), (String) null, serverCert.cert());
+        builder.sslProvider(SslProvider.JDK);
+        builder.trustManager(clientTrustStoreFile); // Trust the server cert
+        SslContext ctx = builder.build();
+        Assert.assertTrue(ctx.isClient());
+        Assert.assertTrue(ctx instanceof JdkSslContext);
+        JdkSslContext jdk = (JdkSslContext) ctx;
+        SSLContext jdkSslContext = jdk.context();
+        return jdkSslContext.getSocketFactory();
+    }
+
+    protected static void setupSSL(Configuration config) throws Exception {
+        config.getSecurity().getServerSsl().setCertificateFile(serverCert.certificate().getAbsolutePath());
+        config.getSecurity().getServerSsl().setKeyFile(serverCert.privateKey().getAbsolutePath());
+        // Needed for 2way SSL
+        config.getSecurity().getServerSsl().setTrustStoreFile(serverCert.certificate().getAbsolutePath());
+        config.getSecurity().getServerSsl().setUseOpenssl(false);
+        config.getSecurity().getServerSsl().setUseGeneratedKeypair(false);
+        config.getSecurity().setAllowAnonymousHttpAccess(false);
+    }
+
+    @Override
+    protected HttpsURLConnection getUrlConnection(URL url) throws Exception {
+        // Username and password not used in 2way SSL case
+        return getUrlConnection(null, null, url);
+    }
+
+    @Override
+    protected HttpsURLConnection getUrlConnection(String username, String password, URL url) throws Exception {
+        HttpsURLConnection.setDefaultSSLSocketFactory(getSSLSocketFactory());
+        URL loginURL = new URL(url.getProtocol() + "://" + url.getHost() + ":" + url.getPort() + "/login");
+        HttpsURLConnection con = (HttpsURLConnection) loginURL.openConnection();
+        con.setHostnameVerifier((host, session) -> true);
+        con.setRequestMethod("GET");
+        con.setDoOutput(true);
+        con.setRequestProperty("Content-Type", "application/json");
+        con.connect();
+        int responseCode = con.getResponseCode();
+        if (401 == responseCode) {
+            throw new UnauthorizedUserException();
+        }
+        Assert.assertEquals(200, responseCode);
+        List<String> cookies = con.getHeaderFields().get(HttpHeaderNames.SET_COOKIE.toString());
+        Assert.assertEquals(1, cookies.size());
+        Cookie sessionCookie = ClientCookieDecoder.STRICT.decode(cookies.get(0));
+        Assert.assertEquals(Constants.COOKIE_NAME, sessionCookie.name());
+        con = (HttpsURLConnection) url.openConnection();
+        con.setRequestProperty(HttpHeaderNames.COOKIE.toString(), sessionCookie.name() + "=" + sessionCookie.value());
+        con.setHostnameVerifier((host, session) -> true);
+        return con;
+    }
+
     @BeforeClass
     public static void beforeClass() throws Exception {
         final MiniAccumuloConfig macConfig = new MiniAccumuloConfig(temp.newFolder("mac"), "secret");
@@ -52,6 +137,7 @@ public class TwoWaySSLIT extends TwoWaySSLBase {
         conf = TestConfiguration.createMinimalConfigurationForTest();
         conf.getAccumulo().setInstanceName(mac.getInstanceName());
         conf.getAccumulo().setZookeepers(mac.getZooKeepers());
+        setupSSL(conf);
     }
 
     @AfterClass
@@ -71,7 +157,6 @@ public class TwoWaySSLIT extends TwoWaySSLBase {
                 }
             }
         });
-        setupSSL(conf, true);
     }
 
     @After
@@ -82,7 +167,7 @@ public class TwoWaySSLIT extends TwoWaySSLBase {
     @Test
     public void testBasicAuthLogin() throws Exception {
         final Server s = new Server(conf);
-        s.run(getSslContext());
+        s.run();
         try {
             String metrics = "https://localhost:54322/api/metrics";
             query(metrics);
@@ -105,7 +190,7 @@ public class TwoWaySSLIT extends TwoWaySSLBase {
 
     public void testQueryWithVisibility(Configuration conf) throws Exception {
         final Server s = new Server(conf);
-        s.run(getSslContext());
+        s.run();
         try {
       // @formatter:off
             put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2",

server/src/test/java/timely/test/integration/TwoWaySSLOpenSSLIT.java

@@ -3,14 +3,32 @@ package timely.test.integration;
 import static com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly;
 import static org.junit.Assert.assertEquals;
 
+import java.io.File;
+import java.net.URL;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.concurrent.TimeUnit;
 
+import javax.net.ssl.HostnameVerifier;
+import javax.net.ssl.HttpsURLConnection;
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.SSLSession;
+import javax.net.ssl.SSLSocketFactory;
+
+import io.netty.handler.codec.http.HttpHeaderNames;
+import io.netty.handler.codec.http.cookie.ClientCookieDecoder;
+import io.netty.handler.codec.http.cookie.Cookie;
+import io.netty.handler.ssl.ApplicationProtocolConfig;
+import io.netty.handler.ssl.JdkSslContext;
+import io.netty.handler.ssl.SslContext;
+import io.netty.handler.ssl.SslContextBuilder;
+import io.netty.handler.ssl.SslProvider;
+import io.netty.handler.ssl.util.SelfSignedCertificate;
 import org.apache.accumulo.core.client.Connector;
 import org.apache.accumulo.core.security.Authorizations;
 import org.junit.After;
+import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
@@ -18,6 +36,8 @@ import timely.Server;
 import timely.api.request.timeseries.QueryRequest;
 import timely.api.response.timeseries.QueryResponse;
 import timely.auth.AuthCache;
+import timely.configuration.Configuration;
+import timely.netty.Constants;
 import timely.test.IntegrationTest;
 import timely.test.TestConfiguration;
 
@@ -28,13 +48,97 @@ import timely.test.TestConfiguration;
  *
  */
 @Category(IntegrationTest.class)
-public class TwoWaySSLOpenSSLIT extends TwoWaySSLBase {
+public class TwoWaySSLOpenSSLIT extends QueryBase {
 
     private static final Long TEST_TIME = System.currentTimeMillis();
 
+    protected static SelfSignedCertificate serverCert = null;
+    protected static File clientTrustStoreFile = null;
+
+    static {
+        try {
+            serverCert = new SelfSignedCertificate();
+            clientTrustStoreFile = serverCert.certificate().getAbsoluteFile();
+        } catch (Exception e) {
+            throw new RuntimeException("Error creating self signed certificate", e);
+        }
+    }
+
+    protected SSLSocketFactory getSSLSocketFactory() throws Exception {
+        SslContextBuilder builder = SslContextBuilder.forClient();
+        builder.applicationProtocolConfig(ApplicationProtocolConfig.DISABLED);
+        // Use server cert / key on client side.
+        builder.keyManager(serverCert.key(), (String) null, serverCert.cert());
+        builder.sslProvider(SslProvider.JDK);
+        builder.trustManager(clientTrustStoreFile); // Trust the server cert
+        SslContext ctx = builder.build();
+        Assert.assertTrue(ctx.isClient());
+        Assert.assertTrue(ctx instanceof JdkSslContext);
+        JdkSslContext jdk = (JdkSslContext) ctx;
+        SSLContext jdkSslContext = jdk.context();
+        return jdkSslContext.getSocketFactory();
+    }
+
+    protected static void setupSSL(Configuration config) throws Exception {
+        config.getSecurity().getServerSsl().setCertificateFile(serverCert.certificate().getAbsolutePath());
+        config.getSecurity().getServerSsl().setKeyFile(serverCert.privateKey().getAbsolutePath());
+        // Needed for 2way SSL
+        config.getSecurity().getServerSsl().setTrustStoreFile(serverCert.certificate().getAbsolutePath());
+        config.getSecurity().getServerSsl().setUseOpenssl(false);
+        config.getSecurity().getServerSsl().setUseGeneratedKeypair(false);
+        config.getSecurity().setAllowAnonymousHttpAccess(false);
+    }
+
+    @Before
+    public void configureSSL() throws Exception {
+        setupSSL(conf);
+    }
+
+    @Override
+    protected HttpsURLConnection getUrlConnection(URL url) throws Exception {
+        // Username and password not used in 2way SSL case
+        return getUrlConnection(null, null, url);
+    }
+
+    @Override
+    protected HttpsURLConnection getUrlConnection(String username, String password, URL url) throws Exception {
+        HttpsURLConnection.setDefaultSSLSocketFactory(getSSLSocketFactory());
+        URL loginURL = new URL(url.getProtocol() + "://" + url.getHost() + ":" + url.getPort() + "/login");
+        HttpsURLConnection con = (HttpsURLConnection) loginURL.openConnection();
+        con.setHostnameVerifier(new HostnameVerifier() {
+
+            @Override
+            public boolean verify(String arg0, SSLSession arg1) {
+                return true;
+            }
+        });
+        con.setRequestMethod("GET");
+        con.setDoOutput(true);
+        con.setRequestProperty("Content-Type", "application/json");
+        con.connect();
+        int responseCode = con.getResponseCode();
+        if (401 == responseCode) {
+            throw new UnauthorizedUserException();
+        }
+        Assert.assertEquals(200, responseCode);
+        List<String> cookies = con.getHeaderFields().get(HttpHeaderNames.SET_COOKIE.toString());
+        Assert.assertEquals(1, cookies.size());
+        Cookie sessionCookie = ClientCookieDecoder.STRICT.decode(cookies.get(0));
+        Assert.assertEquals(Constants.COOKIE_NAME, sessionCookie.name());
+        con = (HttpsURLConnection) url.openConnection();
+        con.setRequestProperty(HttpHeaderNames.COOKIE.toString(), sessionCookie.name() + "=" + sessionCookie.value());
+        con.setHostnameVerifier(new HostnameVerifier() {
+
+            @Override
+            public boolean verify(String arg0, SSLSession arg1) {
+                return true;
+            }
+        });
+        return con;
+    }
+
     @Before
     public void setup() throws Exception {
-        setupSSL(conf, true);
         Connector con = mac.getConnector("root", "secret");
         con.securityOperations().changeUserAuthorizations("root", new Authorizations("A", "B", "C", "D", "E", "F"));
     }
@@ -47,7 +151,7 @@ public class TwoWaySSLOpenSSLIT extends TwoWaySSLBase {
     @Test
     public void testBasicAuthLogin() throws Exception {
         final Server s = new Server(conf);
-        s.run(getSslContext());
+        s.run();
         try {
             String metrics = "https://localhost:54322/api/metrics";
             query(metrics);
@@ -59,7 +163,7 @@ public class TwoWaySSLOpenSSLIT extends TwoWaySSLBase {
     @Test
     public void testQueryWithVisibility() throws Exception {
         final Server s = new Server(conf);
-        s.run(getSslContext());
+        s.run();
         try {
             put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2",
                     "sys.cpu.user " + (TEST_TIME + 1000) + " 3.0 tag1=value1 tag2=value2",

server/src/test/java/timely/test/integration/client/TcpClientIT.java

@@ -49,7 +49,7 @@ public class TcpClientIT extends OneWaySSLBase {
     @Test
     public void testPut() throws Exception {
         final TestServer m = new TestServer(conf);
-        m.run(getSslContext());
+        m.run();
         try (TcpClient client = new TcpClient("127.0.0.1", 54321)) {
             client.open();
             client.write("put sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2\n");
@@ -80,7 +80,7 @@ public class TcpClientIT extends OneWaySSLBase {
     public void testPutMultiple() throws Exception {
 
         final TestServer m = new TestServer(conf);
-        m.run(getSslContext());
+        m.run();
         try (TcpClient client = new TcpClient("127.0.0.1", 54321)) {
             client.open();
             // @formatter:off
@@ -124,7 +124,7 @@ public class TcpClientIT extends OneWaySSLBase {
     @Test
     public void testPutInvalidTimestamp() throws Exception {
         final TestServer m = new TestServer(conf);
-        m.run(getSslContext());
+        m.run();
         try (TcpClient client = new TcpClient("127.0.0.1", 54321)) {
             client.open();
             client.write("put sys.cpu.user " + TEST_TIME + "Z" + " 1.0 tag1=value1 tag2=value2\n");
@@ -139,7 +139,7 @@ public class TcpClientIT extends OneWaySSLBase {
     @Test
     public void testPersistence() throws Exception {
         final Server s = new Server(conf);
-        s.run(getSslContext());
+        s.run();
         try {
             put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2",
                     "sys.cpu.idle " + (TEST_TIME + 1) + " 1.0 tag3=value3 tag4=value4",
@@ -183,7 +183,7 @@ public class TcpClientIT extends OneWaySSLBase {
     @Test
     public void testPersistenceWithVisibility() throws Exception {
         final Server s = new Server(conf);
-        s.run(getSslContext());
+        s.run();
         try {
             put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2",
                     "sys.cpu.idle " + (TEST_TIME + 1) + " 1.0 tag3=value3 tag4=value4 viz=(a|b)",

server/src/test/java/timely/test/integration/client/UdpClientIT.java

@@ -53,7 +53,6 @@ public class UdpClientIT extends MacITBase {
                 }
             }
         });
-        setupSSL();
     }
 
     @After
@@ -64,7 +63,7 @@ public class UdpClientIT extends MacITBase {
     @Test
     public void testPut() throws Exception {
         final TestServer m = new TestServer(conf);
-        m.run(getSslContext());
+        m.run();
         try (UdpClient client = new UdpClient("127.0.0.1", 54325)) {
             client.open();
             client.write(("put sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2\n"));
@@ -94,7 +93,7 @@ public class UdpClientIT extends MacITBase {
     public void testPutMultiple() throws Exception {
 
         final TestServer m = new TestServer(conf);
-        m.run(getSslContext());
+        m.run();
         try (UdpClient client = new UdpClient("127.0.0.1", 54325)) {
             client.open();
             client.write(("put sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2\n" + "put sys.cpu.idle "
@@ -136,7 +135,7 @@ public class UdpClientIT extends MacITBase {
     @Test
     public void testPutInvalidTimestamp() throws Exception {
         final TestServer m = new TestServer(conf);
-        m.run(getSslContext());
+        m.run();
         try (UdpClient client = new UdpClient("127.0.0.1", 54325)) {
             client.open();
             client.write(("put sys.cpu.user " + TEST_TIME + "Z" + " 1.0 tag1=value1 tag2=value2\n"));
@@ -150,7 +149,7 @@ public class UdpClientIT extends MacITBase {
     @Test
     public void testPersistence() throws Exception {
         final Server s = new Server(conf);
-        s.run(getSslContext());
+        s.run();
         try {
             put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2",
                     "sys.cpu.idle " + (TEST_TIME + 1) + " 1.0 tag3=value3 tag4=value4",
@@ -194,7 +193,7 @@ public class UdpClientIT extends MacITBase {
     @Test
     public void testPersistenceWithVisibility() throws Exception {
         final Server s = new Server(conf);
-        s.run(getSslContext());
+        s.run();
         try {
             put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2",
                     "sys.cpu.idle " + (TEST_TIME + 1) + " 1.0 tag3=value3 tag4=value4 viz=(a|b)",

server/src/test/java/timely/test/integration/client/WebSocketClientIT.java

@@ -65,7 +65,7 @@ public class WebSocketClientIT extends OneWaySSLBase {
         Connector con = mac.getConnector("root", "secret");
         con.securityOperations().changeUserAuthorizations("root", new Authorizations("A", "B", "C", "D", "E", "F"));
         s = new Server(conf);
-        s.run(getSslContext());
+        s.run();
         setupSslCtx();
     }
 

server/src/test/java/timely/test/integration/http/HTTPStrictTransportSecurityIT.java

@@ -23,7 +23,7 @@ public class HTTPStrictTransportSecurityIT extends OneWaySSLBase {
     @Before
     public void before() throws Exception {
         s = new Server(conf);
-        s.run(getSslContext());
+        s.run();
     }
 
     @After

server/src/test/java/timely/test/integration/http/HttpApiIT.java

@@ -58,7 +58,7 @@ public class HttpApiIT extends OneWaySSLBase {
     @Test
     public void testSuggest() throws Exception {
         final Server s = new Server(conf);
-        s.run(getSslContext());
+        s.run();
         try {
             put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2",
                     "sys.cpu.idle " + (TEST_TIME + 1) + " 1.0 tag3=value3 tag4=value4",
@@ -86,7 +86,7 @@ public class HttpApiIT extends OneWaySSLBase {
     @Test
     public void testMetrics() throws Exception {
         final Server s = new Server(conf);
-        s.run(getSslContext());
+        s.run();
         try {
       // @formatter:off
             put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2",
@@ -120,7 +120,7 @@ public class HttpApiIT extends OneWaySSLBase {
     // @formatter:off
 
         final Server s = new Server(conf);
-        s.run(getSslContext());
+        s.run();
         try {
             put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2",
                     "sys.cpu.idle " + (TEST_TIME + 1) + " 1.0 tag3=value3 tag4=value4",
@@ -175,7 +175,7 @@ public class HttpApiIT extends OneWaySSLBase {
     @Test
     public void testLookup() throws Exception {
         final Server s = new Server(conf);
-        s.run(getSslContext());
+        s.run();
         try {
             put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2",
                     "sys.cpu.user " + (TEST_TIME + 1) + " 1.0 tag3=value3",
@@ -211,7 +211,7 @@ public class HttpApiIT extends OneWaySSLBase {
     @Test
     public void testQueryWithMsResolution() throws Exception {
         final Server s = new Server(conf);
-        s.run(getSslContext());
+        s.run();
         try {
             put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2",
                     "sys.cpu.user " + (TEST_TIME + 1) + " 1.0 tag3=value3",
@@ -259,7 +259,7 @@ public class HttpApiIT extends OneWaySSLBase {
     @Test
     public void testQueryWithoutMsResolution() throws Exception {
         final Server s = new Server(conf);
-        s.run(getSslContext());
+        s.run();
         try {
             put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2",
                     "sys.cpu.user " + (TEST_TIME + 1) + " 1.0 tag3=value3",
@@ -298,7 +298,7 @@ public class HttpApiIT extends OneWaySSLBase {
     @Test
     public void testQueryWithNoTags() throws Exception {
         final Server s = new Server(conf);
-        s.run(getSslContext());
+        s.run();
         try {
             put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2",
                     "sys.cpu.user " + (TEST_TIME + 1) + " 1.0 tag3=value3",
@@ -334,7 +334,7 @@ public class HttpApiIT extends OneWaySSLBase {
     @Test
     public void testQueryWithNoTagsMultipleSeries() throws Exception {
         final Server s = new Server(conf);
-        s.run(getSslContext());
+        s.run();
         try {
             put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2 host=h1",
                     "sys.cpu.user " + TEST_TIME + " 2.0 tag1=value1 tag2=value2 host=h2",
@@ -375,7 +375,7 @@ public class HttpApiIT extends OneWaySSLBase {
     @Test(expected = NotSuccessfulException.class)
     public void testQueryWithNoMatchingTags() throws Exception {
         final Server s = new Server(conf);
-        s.run(getSslContext());
+        s.run();
         try {
             put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2 rack=r1",
                     "sys.cpu.user " + (TEST_TIME + 1) + " 1.0 tag3=value3 rack=r2",
@@ -399,7 +399,7 @@ public class HttpApiIT extends OneWaySSLBase {
     @Test
     public void testQueryWithTagWildcard() throws Exception {
         final Server s = new Server(conf);
-        s.run(getSslContext());
+        s.run();
     // @formatter:off
         try {
             put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2 rack=r1",
@@ -464,7 +464,7 @@ public class HttpApiIT extends OneWaySSLBase {
     @Test
     public void testQueryWithTagWildcard2() throws Exception {
         final Server s = new Server(conf);
-        s.run(getSslContext());
+        s.run();
     // @formatter:off
         try {
             put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2 rack=r1",
@@ -524,7 +524,7 @@ public class HttpApiIT extends OneWaySSLBase {
     @Test
     public void testQueryWithTagOr() throws Exception {
         final Server s = new Server(conf);
-        s.run(getSslContext());
+        s.run();
         try {
       // @formatter:off
             put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2 rack=r1",
@@ -608,7 +608,7 @@ public class HttpApiIT extends OneWaySSLBase {
     @Test
     public void testQueryWithTagRegex() throws Exception {
         final Server s = new Server(conf);
-        s.run(getSslContext());
+        s.run();
         try {
             System.out.println("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2 rack=r1");
             System.out.println("sys.cpu.user " + (TEST_TIME + 1) + " 1.0 tag3=value3 rack=r2");
@@ -674,7 +674,7 @@ public class HttpApiIT extends OneWaySSLBase {
     @Test
     public void testQueryWithTagRegex2() throws Exception {
         final Server s = new Server(conf);
-        s.run(getSslContext());
+        s.run();
         try {
             put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2 rack=r1",
                     "sys.cpu.user " + (TEST_TIME + 1) + " 1.0 tag3=value3 rack=r2",
@@ -730,7 +730,7 @@ public class HttpApiIT extends OneWaySSLBase {
     @Test
     public void testRateQuery() throws Exception {
         final Server s = new Server(conf);
-        s.run(getSslContext());
+        s.run();
         try {
             // @formatter:off
             put("sys.cpu.user " + (TEST_TIME) + " 1.0 tag1=value1 tag2=value2 rack=r1",
@@ -782,7 +782,7 @@ public class HttpApiIT extends OneWaySSLBase {
     @Test
     public void testRateCounterQuery() throws Exception {
         final Server s = new Server(conf);
-        s.run(getSslContext());
+        s.run();
         try {
       // @formatter:off
             put("sys.cpu.user " + (TEST_TIME) + " 1.0 tag1=value1 tag2=value2 rack=r1",
@@ -836,7 +836,7 @@ public class HttpApiIT extends OneWaySSLBase {
     @Test
     public void testRateCounterWithOptions() throws Exception {
         final Server s = new Server(conf);
-        s.run(getSslContext());
+        s.run();
         try {
       // @formatter:off
             put("sys.cpu.user " + (TEST_TIME) + " 1.0 tag1=value1 tag2=value2 rack=r1",
@@ -892,7 +892,7 @@ public class HttpApiIT extends OneWaySSLBase {
     @Test
     public void testGetVersion() throws Exception {
         final Server s = new Server(conf);
-        s.run(getSslContext());
+        s.run();
         try {
             String response = query("https://127.0.0.1:54322/version", "application/json");
             assertNotNull(response);
@@ -905,7 +905,7 @@ public class HttpApiIT extends OneWaySSLBase {
     @Test
     public void testPutMetric() throws Exception {
         final Server s = new Server(conf);
-        s.run(getSslContext());
+        s.run();
         try {
             Metric m = Metric.newBuilder().name("sys.cpu.user").value(TEST_TIME, 1.0D).tag(new Tag("tag1", "value1"))
                     .build();

server/src/test/java/timely/test/integration/tcp/TimelyTcpIT.java

@@ -56,7 +56,7 @@ public class TimelyTcpIT extends MacITBase {
     @Test
     public void testVersion() throws Exception {
         final TestServer m = new TestServer(conf);
-        m.run(getSslContext());
+        m.run();
         try (Socket sock = new Socket("127.0.0.1", 54321);
                 PrintWriter writer = new PrintWriter(sock.getOutputStream(), true);) {
             writer.write("version\n");
@@ -76,7 +76,7 @@ public class TimelyTcpIT extends MacITBase {
     @Test
     public void testPut() throws Exception {
         final TestServer m = new TestServer(conf);
-        m.run(getSslContext());
+        m.run();
         try (Socket sock = new Socket("127.0.0.1", 54321);
                 PrintWriter writer = new PrintWriter(sock.getOutputStream(), true);) {
             writer.write("put sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2\n");
@@ -107,7 +107,7 @@ public class TimelyTcpIT extends MacITBase {
     public void testPutMultiple() throws Exception {
 
         final TestServer m = new TestServer(conf);
-        m.run(getSslContext());
+        m.run();
         try (Socket sock = new Socket("127.0.0.1", 54321);
                 PrintWriter writer = new PrintWriter(sock.getOutputStream(), true)) {
             // @formatter:off
@@ -190,7 +190,7 @@ public class TimelyTcpIT extends MacITBase {
         LOG.debug("Sending {} bytes", data.length);
 
         final TestServer m = new TestServer(conf);
-        m.run(getSslContext());
+        m.run();
         try (Socket sock = new Socket("127.0.0.1", 54321);) {
             sock.getOutputStream().write(data);
             sock.getOutputStream().flush();
@@ -233,7 +233,7 @@ public class TimelyTcpIT extends MacITBase {
     @Test
     public void testPutInvalidTimestamp() throws Exception {
         final TestServer m = new TestServer(conf);
-        m.run(getSslContext());
+        m.run();
         try (Socket sock = new Socket("127.0.0.1", 54321);
                 PrintWriter writer = new PrintWriter(sock.getOutputStream(), true);
                 BufferedReader reader = new BufferedReader(new InputStreamReader(sock.getInputStream()));) {
@@ -249,7 +249,7 @@ public class TimelyTcpIT extends MacITBase {
     @Test
     public void testPersistence() throws Exception {
         final Server s = new Server(conf);
-        s.run(getSslContext());
+        s.run();
         try {
             put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2",
                     "sys.cpu.idle " + (TEST_TIME + 1) + " 1.0 tag3=value3 tag4=value4",
@@ -293,7 +293,7 @@ public class TimelyTcpIT extends MacITBase {
     @Test
     public void testPersistenceWithVisibility() throws Exception {
         final Server s = new Server(conf);
-        s.run(getSslContext());
+        s.run();
         try {
             put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2",
                     "sys.cpu.idle " + (TEST_TIME + 1) + " 1.0 tag3=value3 tag4=value4 viz=(a|b)",

server/src/test/java/timely/test/integration/udp/TimelyUdpIT.java

@@ -60,7 +60,6 @@ public class TimelyUdpIT extends MacITBase {
                 }
             }
         });
-        setupSSL();
     }
 
     @After
@@ -71,7 +70,7 @@ public class TimelyUdpIT extends MacITBase {
     @Test
     public void testPut() throws Exception {
         final TestServer m = new TestServer(conf);
-        m.run(getSslContext());
+        m.run();
         InetSocketAddress address = new InetSocketAddress("127.0.0.1", 54325);
         DatagramPacket packet = new DatagramPacket("".getBytes(UTF_8), 0, 0, address.getAddress(), 54325);
         try (DatagramSocket sock = new DatagramSocket()) {
@@ -103,7 +102,7 @@ public class TimelyUdpIT extends MacITBase {
     public void testPutMultiple() throws Exception {
 
         final TestServer m = new TestServer(conf);
-        m.run(getSslContext());
+        m.run();
         InetSocketAddress address = new InetSocketAddress("127.0.0.1", 54325);
         DatagramPacket packet = new DatagramPacket("".getBytes(UTF_8), 0, 0, address.getAddress(), 54325);
         // @formatter:off
@@ -186,7 +185,7 @@ public class TimelyUdpIT extends MacITBase {
         LOG.debug("Sending {} bytes", data.length);
 
         final TestServer m = new TestServer(conf);
-        m.run(getSslContext());
+        m.run();
         InetSocketAddress address = new InetSocketAddress("127.0.0.1", 54325);
         DatagramPacket packet = new DatagramPacket("".getBytes(UTF_8), 0, 0, address.getAddress(), 54325);
         try (DatagramSocket sock = new DatagramSocket()) {
@@ -230,7 +229,7 @@ public class TimelyUdpIT extends MacITBase {
     @Test
     public void testPutInvalidTimestamp() throws Exception {
         final TestServer m = new TestServer(conf);
-        m.run(getSslContext());
+        m.run();
         InetSocketAddress address = new InetSocketAddress("127.0.0.1", 54325);
         DatagramPacket packet = new DatagramPacket("".getBytes(UTF_8), 0, 0, address.getAddress(), 54325);
         try (DatagramSocket sock = new DatagramSocket();) {
@@ -246,7 +245,7 @@ public class TimelyUdpIT extends MacITBase {
     @Test
     public void testPersistence() throws Exception {
         final Server s = new Server(conf);
-        s.run(getSslContext());
+        s.run();
         try {
             put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2",
                     "sys.cpu.idle " + (TEST_TIME + 1) + " 1.0 tag3=value3 tag4=value4",
@@ -290,7 +289,7 @@ public class TimelyUdpIT extends MacITBase {
     @Test
     public void testPersistenceWithVisibility() throws Exception {
         final Server s = new Server(conf);
-        s.run(getSslContext());
+        s.run();
         try {
             put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2",
                     "sys.cpu.idle " + (TEST_TIME + 1) + " 1.0 tag3=value3 tag4=value4 viz=(a|b)",

server/src/test/java/timely/test/integration/websocket/WebSocketIT.java

@@ -189,7 +189,7 @@ public class WebSocketIT extends OneWaySSLBase {
     @Before
     public void setup() throws Exception {
         s = new Server(conf);
-        s.run(getSslContext());
+        s.run();
 
         Connector con = mac.getConnector("root", "secret");
         con.securityOperations().changeUserAuthorizations("root", new Authorizations("A", "B", "C", "D", "E", "F"));