Refactor BalancedMetricResolver to allow HDFS and FILE backing, add callbacks for major events, begin working on tests

Refactor BalancedMetricResolver, allow FILE or HDFS persister, add callbacks for events, begin working on testing

Fixes

analytics/pom.xml

@@ -4,7 +4,7 @@
   <parent>
     <groupId>timely</groupId>
     <artifactId>timely</artifactId>
-    <version>1.2.5-SNAPSHOT</version>
+    <version>1.2.8-SNAPSHOT</version>
   </parent>
   <artifactId>timely-analytics</artifactId>
   <!--
@@ -68,6 +68,7 @@
     <pluginManagement>
       <plugins>
         <plugin>
+          <groupId>org.apache.maven.plugins</groupId>
           <artifactId>maven-compiler-plugin</artifactId>
           <configuration>
             <compilerId>jdt</compilerId>
@@ -76,7 +77,7 @@
             <dependency>
               <groupId>org.eclipse.tycho</groupId>
               <artifactId>tycho-compiler-jdt</artifactId>
-              <version>0.21.0</version>
+              <version>1.4.0</version>
             </dependency>
           </dependencies>
         </plugin>

balancer/conf/timely-balancer.yml

@@ -111,6 +111,7 @@ timely-balancer:
     flush-interval-seconds: 30
     scanner-read-ahead: 1
     subscription-batch-size: 5000
+    incomingBufferSize: 50000000
     wsClientPool:
       minIdlePerKey: 0
       maxIdlePerKey: 40

balancer/pom.xml

@@ -4,7 +4,7 @@
   <parent>
     <groupId>timely</groupId>
     <artifactId>timely</artifactId>
-    <version>1.2.5-SNAPSHOT</version>
+    <version>1.2.8-SNAPSHOT</version>
   </parent>
   <artifactId>timely-balancer</artifactId>
   <dependencies>
@@ -108,6 +108,10 @@
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-minicluster</artifactId>
     </dependency>
+    <dependency>
+      <groupId>org.glassfish.tyrus</groupId>
+      <artifactId>tyrus-client</artifactId>
+    </dependency>
     <dependency>
       <groupId>org.slf4j</groupId>
       <artifactId>slf4j-api</artifactId>

balancer/src/main/java/timely/balancer/ArrivalRate.java

@@ -1,7 +1,7 @@
 package timely.balancer;
 
-import java.util.Timer;
-import java.util.TimerTask;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
 
 import com.google.common.util.concurrent.AtomicDouble;
@@ -20,38 +20,26 @@ public class ArrivalRate {
     private AtomicDouble longTermRate = new AtomicDouble(0);
     private AtomicLong longTermLastReset = new AtomicLong(0);
 
-    public ArrivalRate(Timer timer) {
+    public ArrivalRate(ScheduledExecutorService arrivalRateExecutor) {
         long now = System.currentTimeMillis();
         shortTermLastReset.set(now);
         mediumTermLastReset.set(now);
         longTermLastReset.set(now);
 
         // 2 minute
-        timer.schedule(new TimerTask() {
-
-            @Override
-            public void run() {
-                resetShort();
-            }
-        }, 120000, 120000);
+        arrivalRateExecutor.scheduleAtFixedRate(() -> {
+            resetShort();
+        }, 2, 2, TimeUnit.MINUTES);
 
         // 5 minute
-        timer.schedule(new TimerTask() {
-
-            @Override
-            public void run() {
-                resetMedium();
-            }
-        }, 300000, 300000);
+        arrivalRateExecutor.scheduleAtFixedRate(() -> {
+            resetMedium();
+        }, 5, 5, TimeUnit.MINUTES);
 
         // 15 minute
-        timer.schedule(new TimerTask() {
-
-            @Override
-            public void run() {
-                resetLong();
-            }
-        }, 900000, 900000);
+        arrivalRateExecutor.scheduleAtFixedRate(() -> {
+            resetLong();
+        }, 15, 15, TimeUnit.MINUTES);
     }
 
     private void resetShort() {

balancer/src/main/java/timely/balancer/Balancer.java

@@ -57,6 +57,7 @@ import io.netty.handler.stream.ChunkedWriteHandler;
 import io.netty.handler.timeout.IdleStateHandler;
 import io.netty.util.NettyRuntime;
 import io.netty.util.internal.SystemPropertyUtil;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
 import org.apache.curator.RetryPolicy;
 import org.apache.curator.framework.CuratorFramework;
 import org.apache.curator.framework.CuratorFrameworkFactory;
@@ -76,7 +77,7 @@ import timely.balancer.connection.http.HttpClientPool;
 import timely.balancer.connection.tcp.TcpClientPool;
 import timely.balancer.connection.udp.UdpClientPool;
 import timely.balancer.connection.ws.WsClientPool;
-import timely.balancer.healthcheck.HealthChecker;
+import timely.balancer.healthcheck.TcpHealthChecker;
 import timely.balancer.netty.http.HttpRelayHandler;
 import timely.balancer.netty.tcp.TcpRelayHandler;
 import timely.balancer.netty.udp.UdpRelayHandler;
@@ -144,8 +145,11 @@ public class Balancer {
         this.balancerConfig = balancerConf;
     }
 
-    protected static ConfigurableApplicationContext initializeConfiguration(String[] args) {
-        return new SpringApplicationBuilder(SpringBootstrap.class).web(WebApplicationType.NONE).run(args);
+    protected static ConfigurableApplicationContext initializeContext(String[] args) {
+        SpringApplicationBuilder builder = new SpringApplicationBuilder(SpringBootstrap.class);
+        builder.web(WebApplicationType.NONE);
+        builder.registerShutdownHook(false);
+        return builder.run(args);
     }
 
     private void shutdownHook() {
@@ -177,8 +181,9 @@ public class Balancer {
             channelFutures.add(wsChannelHandle.close());
         }
 
+        int udpChannel = 1;
         for (Channel c : udpChannelHandleList) {
-            LOG.info("Closing udpChannelHandle");
+            LOG.info("Closing udpChannelHandle #" + udpChannel++);
             channelFutures.add(c.close());
         }
 
@@ -187,14 +192,13 @@ public class Balancer {
             try {
                 f.get();
             } catch (final Exception e) {
-                LOG.error("Error while shutting down channel: {}", f.channel().config());
-                LOG.error("{}", e.getMessage());
-                e.printStackTrace();
+                LOG.error("Channel:" + f.channel().config() + " -> " + e.getMessage(), e);
             }
         });
 
-        Integer quietPeriod = balancerConfig.getServer().getShutdownQuietPeriod();
+        int quietPeriod = balancerConfig.getServer().getShutdownQuietPeriod();
         List<Future<?>> groupFutures = new ArrayList<>();
+
         if (tcpBossGroup != null) {
             LOG.info("Shutting down tcpBossGroup");
             groupFutures.add(tcpBossGroup.shutdownGracefully(quietPeriod, 10, TimeUnit.SECONDS));
@@ -239,9 +243,7 @@ public class Balancer {
             try {
                 f.get();
             } catch (final Exception e) {
-                LOG.error("Error while shutting down group: {}", f.toString());
-                LOG.error("{}", e.getMessage());
-                e.printStackTrace();
+                LOG.error("Group:" + f.toString() + " -> " + e.getMessage(), e);
             }
         });
 
@@ -252,14 +254,29 @@ public class Balancer {
             LOG.error("Error closing WebSocketRequestDecoder during shutdown", e);
         }
 
+        if (this.metricResolver != null) {
+            try {
+                LOG.info("Closing metricResolver");
+                this.metricResolver.stop();
+            } catch (Exception e) {
+                LOG.error(e.getMessage(), e);
+            }
+        }
+
+        if (curatorFramework != null) {
+            try {
+                LOG.info("Closing curatorFramework");
+                curatorFramework.close();
+            } catch (Exception e) {
+                LOG.error(e.getMessage(), e);
+            }
+        }
+
         if (applicationContext != null) {
             LOG.info("Closing applicationContext");
             applicationContext.close();
         }
 
-        if (curatorFramework != null) {
-            curatorFramework.close();
-        }
         this.shutdown = true;
         LOG.info("Server shut down.");
     }
@@ -360,7 +377,9 @@ public class Balancer {
 
         // start curator framework
         RetryPolicy retryPolicy = new RetryForever(1000);
-        curatorFramework = CuratorFrameworkFactory.newClient(balancerConfig.getZooKeeper().getServers(), 30000, 1000,
+        int timeout = Long.valueOf(AccumuloConfiguration.getTimeInMillis(balancerConfig.getZooKeeper().getTimeout()))
+                .intValue();
+        curatorFramework = CuratorFrameworkFactory.newClient(balancerConfig.getZooKeeper().getServers(), timeout, 10000,
                 retryPolicy);
         curatorFramework.start();
         ensureZkPaths(curatorFramework, zkPaths);
@@ -398,8 +417,9 @@ public class Balancer {
         for (int x = 0; x < balancerConfig.getServer().getNumTcpPools(); x++) {
             tcpClientPools.add(new TcpClientPool(this.balancerConfig));
         }
-        HealthChecker healthChecker = new HealthChecker(this.balancerConfig, tcpClientPools.get(0));
+        TcpHealthChecker healthChecker = new TcpHealthChecker(this.balancerConfig, tcpClientPools.get(0));
         this.metricResolver = new BalancedMetricResolver(curatorFramework, this.balancerConfig, healthChecker);
+        this.metricResolver.start();
 
         final ServerBootstrap tcpServer = new ServerBootstrap();
         tcpServer.group(tcpBossGroup, tcpWorkerGroup);
@@ -478,12 +498,12 @@ public class Balancer {
 
     public static void main(String[] args) throws Exception {
 
-        Balancer.applicationContext = Balancer.initializeConfiguration(args);
-        BalancerConfiguration balancerConf = applicationContext.getBean(BalancerConfiguration.class);
+        Balancer.applicationContext = Balancer.initializeContext(args);
+        BalancerConfiguration balancerConf = Balancer.applicationContext.getBean(BalancerConfiguration.class);
 
-        Balancer b = new Balancer(balancerConf);
+        Balancer balancer = new Balancer(balancerConf);
         try {
-            b.run();
+            balancer.run();
             LATCH.await();
         } catch (final InterruptedException e) {
             LOG.info("Server shutting down.");
@@ -491,7 +511,7 @@ public class Balancer {
             LOG.error("Error running server.", e);
         } finally {
             try {
-                b.shutdown();
+                balancer.shutdown();
             } catch (Exception e) {
                 System.exit(1);
             }
@@ -515,11 +535,13 @@ public class Balancer {
                 ch.pipeline().addLast("chunker", new ChunkedWriteHandler());
                 ch.pipeline().addLast("queryDecoder", new timely.netty.http.HttpRequestDecoder(
                         balancerConfig.getSecurity(), balancerConfig.getHttp()));
-                ch.pipeline().addLast("fileServer", new HttpStaticFileServerHandler());
+                ch.pipeline().addLast("fileServer", new HttpStaticFileServerHandler().setIgnoreSslHandshakeErrors(
+                        balancerConfig.getSecurity().getServerSsl().isUseGeneratedKeypair()));
                 ch.pipeline().addLast("login",
                         new X509LoginRequestHandler(balancerConfig.getSecurity(), balancerConfig.getHttp()));
                 ch.pipeline().addLast("httpRelay", new HttpRelayHandler(metricResolver, httpClientPool));
-                ch.pipeline().addLast("error", new TimelyExceptionHandler());
+                ch.pipeline().addLast("error", new TimelyExceptionHandler().setIgnoreSslHandshakeErrors(
+                        balancerConfig.getSecurity().getServerSsl().isUseGeneratedKeypair()));
             }
         };
     }

balancer/src/main/java/timely/balancer/configuration/BalancerConfiguration.java

@@ -49,6 +49,16 @@ public class BalancerConfiguration {
 
     private int serverSuccessesBeforeUp = 3;
 
+    private String metricAssignmentPersisterType = "HDFS";
+
+    private long balanceDelay = 900000;
+
+    private long balancePeriod = 900000;
+
+    private long persistDelay = 10000;
+
+    private long persistPeriod = 60000;
+
     public ZooKeeper getZooKeeper() {
         return zooKeeper;
     }
@@ -120,4 +130,44 @@ public class BalancerConfiguration {
     public void setServerSuccessesBeforeUp(int serverSuccessesBeforeUp) {
         this.serverSuccessesBeforeUp = serverSuccessesBeforeUp;
     }
+
+    public String getMetricAssignmentPersisterType() {
+        return metricAssignmentPersisterType;
+    }
+
+    public void setMetricAssignmentPersisterType(String metricAssignmentPersisterType) {
+        this.metricAssignmentPersisterType = metricAssignmentPersisterType;
+    }
+
+    public void setBalanceDelay(long balanceDelay) {
+        this.balanceDelay = balanceDelay;
+    }
+
+    public long getBalanceDelay() {
+        return balanceDelay;
+    }
+
+    public void setBalancePeriod(long balancePeriod) {
+        this.balancePeriod = balancePeriod;
+    }
+
+    public long getBalancePeriod() {
+        return balancePeriod;
+    }
+
+    public void setPersistDelay(long persistDelay) {
+        this.persistDelay = persistDelay;
+    }
+
+    public long getPersistDelay() {
+        return persistDelay;
+    }
+
+    public void setPersistPeriod(long persistPeriod) {
+        this.persistPeriod = persistPeriod;
+    }
+
+    public long getPersistPeriod() {
+        return persistPeriod;
+    }
 }

balancer/src/main/java/timely/balancer/configuration/BalancerWebsocket.java

@@ -16,4 +16,14 @@ public class BalancerWebsocket extends Websocket {
     public GenericKeyedObjectPoolConfig<WebSocketSubscriptionClient> getWsClientPool() {
         return wsClientPool;
     }
+
+    public int incomingBufferSize = 50000000;
+
+    public int getIncomingBufferSize() {
+        return incomingBufferSize;
+    }
+
+    public void setIncomingBufferSize(int incomingBufferSize) {
+        this.incomingBufferSize = incomingBufferSize;
+    }
 }

balancer/src/main/java/timely/balancer/configuration/ZooKeeper.java

@@ -1,9 +1,14 @@
 package timely.balancer.configuration;
 
+import javax.validation.constraints.NotBlank;
+
 public class ZooKeeper {
 
     private String servers;
 
+    @NotBlank
+    private String timeout = "120s";
+
     public String getServers() {
         return servers;
     }
@@ -11,4 +16,12 @@ public class ZooKeeper {
     public void setServers(String servers) {
         this.servers = servers;
     }
+
+    public String getTimeout() {
+        return timeout;
+    }
+
+    public void setTimeout(String timeout) {
+        this.timeout = timeout;
+    }
 }

balancer/src/main/java/timely/balancer/connection/TimelyBalancedHost.java

@@ -1,7 +1,5 @@
 package timely.balancer.connection;
 
-import java.util.Timer;
-
 import org.apache.commons.lang3.builder.EqualsBuilder;
 import org.apache.commons.lang3.builder.HashCodeBuilder;
 import org.apache.commons.lang3.builder.ToStringBuilder;
@@ -13,7 +11,6 @@ import timely.balancer.configuration.BalancerConfiguration;
 public class TimelyBalancedHost {
 
     private static final Logger LOG = LoggerFactory.getLogger(TimelyBalancedHost.class);
-    static private Timer arrivalRateTimer = new Timer("ArrivalRateTimerTBH", true);
 
     private String host;
     private int tcpPort;
@@ -24,34 +21,29 @@ public class TimelyBalancedHost {
     private BalancerConfiguration balancerConfig;
     private int failures = 0;
     private int successes = 0;
-    private ArrivalRate arrivalRate = new ArrivalRate(arrivalRateTimer);
+    private ArrivalRate arrivalRate = null;
 
-    public TimelyBalancedHost() {
+    private TimelyBalancedHost() {
 
     }
 
-    public TimelyBalancedHost(String host, int tcpPort, int httpPort, int wsPort, int udpPort) {
+    private TimelyBalancedHost(String host, int tcpPort, int httpPort, int wsPort, int udpPort,
+            ArrivalRate arrivalRate) {
         this.host = host;
         this.tcpPort = tcpPort;
         this.httpPort = httpPort;
         this.wsPort = wsPort;
         this.udpPort = udpPort;
+        this.arrivalRate = arrivalRate;
     }
 
-    public TimelyBalancedHost(String host, int basePort) {
-        this.host = host;
-        this.tcpPort = basePort;
-        this.httpPort = basePort + 1;
-        this.wsPort = basePort + 2;
-        this.udpPort = basePort + 3;
-    }
-
-    static public TimelyBalancedHost of(String host, int tcpPort, int httpPort, int wsPort, int udpPort) {
-        return new TimelyBalancedHost(host, tcpPort, httpPort, wsPort, udpPort);
+    static public TimelyBalancedHost of(String host, int tcpPort, int httpPort, int wsPort, int udpPort,
+            ArrivalRate arrivalRate) {
+        return new TimelyBalancedHost(host, tcpPort, httpPort, wsPort, udpPort, arrivalRate);
     }
 
     static public TimelyBalancedHost of(String host, int basePort) {
-        return new TimelyBalancedHost(host, basePort);
+        return new TimelyBalancedHost(host, basePort, basePort + 1, basePort + 2, basePort + 3, null);
     }
 
     public void setBalancerConfig(BalancerConfiguration balancerConfig) {
@@ -141,11 +133,17 @@ public class TimelyBalancedHost {
     }
 
     public void arrived() {
-        arrivalRate.arrived();
+        if (this.arrivalRate != null) {
+            this.arrivalRate.arrived();
+        }
     }
 
     public double getArrivalRate() {
-        return arrivalRate.getRate();
+        if (this.arrivalRate == null) {
+            return 0;
+        } else {
+            return this.arrivalRate.getRate();
+        }
     }
 
     @Override
@@ -186,4 +184,11 @@ public class TimelyBalancedHost {
         builder.append(isUp);
         return builder.toString();
     }
+
+    public static String toStringShort(TimelyBalancedHost tbh) {
+        if (tbh == null) {
+            return null;
+        }
+        return tbh.host + ":" + tbh.tcpPort;
+    }
 }

balancer/src/main/java/timely/balancer/connection/ws/WsClientFactory.java

@@ -22,7 +22,7 @@ public class WsClientFactory implements KeyedPooledObjectFactory<TimelyBalancedH
     @Override
     public PooledObject<WebSocketSubscriptionClient> makeObject(TimelyBalancedHost k) throws Exception {
 
-        int bufferSize = balancerConfig.getWebsocket().getSubscriptionBatchSize() * 500;
+        int bufferSize = balancerConfig.getWebsocket().getIncomingBufferSize();
         WebSocketSubscriptionClient client = new WebSocketSubscriptionClient(sslContext, k.getHost(), k.getHttpPort(),
                 k.getWsPort(), balancerConfig.getSecurity().getClientSsl().isUseClientCert(),
                 balancerConfig.isLoginRequired(), "", "",

balancer/src/main/java/timely/balancer/healthcheck/HealthChecker.java

@@ -1,53 +1,12 @@
 package timely.balancer.healthcheck;
 
-import java.util.ArrayList;
 import java.util.List;
-import java.util.Timer;
-import java.util.TimerTask;
 
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import timely.balancer.configuration.BalancerConfiguration;
 import timely.balancer.connection.TimelyBalancedHost;
-import timely.balancer.connection.tcp.TcpClientPool;
 
-public class HealthChecker {
+public interface HealthChecker {
 
-    private static final Logger LOG = LoggerFactory.getLogger(HealthChecker.class);
-    private List<TimelyBalancedHost> timelyHosts = new ArrayList<>();
-    private Timer timer;
-    final private TCPServerHealthCheck tcpServerHealthCheck;
-    private TimerTask check = new TimerTask() {
+    public void setTimelyHosts(List<TimelyBalancedHost> timelyHosts);
 
-        @Override
-        public void run() {
-            synchronized (timelyHosts) {
-                for (TimelyBalancedHost h : timelyHosts) {
-                    try {
-                        if (tcpServerHealthCheck.isServerHealthy(h)) {
-                            h.reportSuccess();
-                        } else {
-                            h.reportFailure();
-                        }
-                    } catch (Exception e) {
-                        LOG.error(e.getMessage(), e);
-                        h.reportFailure();
-                    }
-                }
-            }
-        }
-    };
-
-    public HealthChecker(BalancerConfiguration config, TcpClientPool tcpClientPool) {
-        this.tcpServerHealthCheck = new TCPServerHealthCheck(tcpClientPool);
-        this.timer = new Timer("HealthCheckerThread", true);
-        this.timer.scheduleAtFixedRate(check, 0, config.getCheckServerHealthInterval());
-    }
-
-    public void setTimelyHosts(List<TimelyBalancedHost> timelyHosts) {
-        synchronized (timelyHosts) {
-            this.timelyHosts.clear();
-            this.timelyHosts.addAll(timelyHosts);
-        }
-    }
+    public void close() throws Exception;
 }

balancer/src/main/java/timely/balancer/healthcheck/TcpHealthChecker.java

@@ -0,0 +1,62 @@
+package timely.balancer.healthcheck;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import timely.balancer.configuration.BalancerConfiguration;
+import timely.balancer.connection.TimelyBalancedHost;
+import timely.balancer.connection.tcp.TcpClientPool;
+
+public class TcpHealthChecker implements HealthChecker {
+
+    private static final Logger LOG = LoggerFactory.getLogger(TcpHealthChecker.class);
+    private List<TimelyBalancedHost> timelyHosts = new ArrayList<>();
+    private ScheduledExecutorService executorService = Executors.newScheduledThreadPool(1);
+    final private TCPServerHealthCheck tcpServerHealthCheck;
+
+    public TcpHealthChecker(BalancerConfiguration config, TcpClientPool tcpClientPool) {
+        this.tcpServerHealthCheck = new TCPServerHealthCheck(tcpClientPool);
+        this.executorService.scheduleAtFixedRate(() -> {
+            synchronized (timelyHosts) {
+                for (TimelyBalancedHost h : timelyHosts) {
+                    try {
+                        if (tcpServerHealthCheck.isServerHealthy(h)) {
+                            h.reportSuccess();
+                        } else {
+                            h.reportFailure();
+                        }
+                    } catch (Exception e) {
+                        LOG.error(e.getMessage(), e);
+                        h.reportFailure();
+                    }
+                }
+            }
+        }, 0, config.getCheckServerHealthInterval(), TimeUnit.MILLISECONDS);
+    }
+
+    @Override
+    public void setTimelyHosts(List<TimelyBalancedHost> timelyHosts) {
+        synchronized (timelyHosts) {
+            this.timelyHosts.clear();
+            this.timelyHosts.addAll(timelyHosts);
+        }
+    }
+
+    public void close() throws Exception {
+        this.executorService.shutdown();
+        try {
+            this.executorService.awaitTermination(5, TimeUnit.SECONDS);
+        } catch (InterruptedException e) {
+
+        } finally {
+            if (!this.executorService.isTerminated()) {
+                this.executorService.shutdownNow();
+            }
+        }
+    }
+}

balancer/src/main/java/timely/balancer/resolver/BalancedMetricResolver.java

@@ -4,10 +4,14 @@ import static timely.Server.SERVICE_DISCOVERY_PATH;
 import static timely.balancer.Balancer.ASSIGNMENTS_LAST_UPDATED_PATH;
 import static timely.balancer.Balancer.ASSIGNMENTS_LOCK_PATH;
 import static timely.balancer.Balancer.LEADER_LATCH_PATH;
+import static timely.balancer.resolver.eventing.MetricAssignedEvent.Reason.ASSIGN_FALLBACK_SEQUENTIAL;
+import static timely.balancer.resolver.eventing.MetricAssignedEvent.Reason.ASSIGN_ROUND_ROBIN;
+import static timely.balancer.resolver.eventing.MetricAssignedEvent.Reason.HOST_DOWN_ROUND_ROBIN;
+import static timely.balancer.resolver.eventing.MetricAssignedEvent.Reason.HOST_REMOVED;
+import static timely.balancer.resolver.eventing.MetricAssignedEvent.Reason.REBALANCE;
 import static timely.store.cache.DataStoreCache.NON_CACHED_METRICS;
 import static timely.store.cache.DataStoreCache.NON_CACHED_METRICS_LOCK_PATH;
 
-import java.nio.charset.Charset;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -19,21 +23,18 @@ import java.util.List;
 import java.util.Map;
 import java.util.Random;
 import java.util.Set;
-import java.util.Timer;
-import java.util.TimerTask;
 import java.util.TreeMap;
 import java.util.TreeSet;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 
-import com.csvreader.CsvReader;
-import com.csvreader.CsvWriter;
 import org.apache.commons.lang.StringUtils;
 import org.apache.commons.lang3.SerializationUtils;
 import org.apache.curator.framework.CuratorFramework;
-import org.apache.curator.framework.recipes.atomic.DistributedAtomicLong;
 import org.apache.curator.framework.recipes.atomic.DistributedAtomicValue;
 import org.apache.curator.framework.recipes.cache.TreeCache;
 import org.apache.curator.framework.recipes.cache.TreeCacheEvent;
@@ -48,11 +49,6 @@ import org.apache.curator.x.discovery.ServiceDiscovery;
 import org.apache.curator.x.discovery.ServiceDiscoveryBuilder;
 import org.apache.curator.x.discovery.ServiceInstance;
 import org.apache.curator.x.discovery.details.ServiceCacheListener;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
 import org.apache.zookeeper.CreateMode;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -61,69 +57,74 @@ import timely.balancer.ArrivalRate;
 import timely.balancer.configuration.BalancerConfiguration;
 import timely.balancer.connection.TimelyBalancedHost;
 import timely.balancer.healthcheck.HealthChecker;
+import timely.balancer.resolver.eventing.MetricAssignedCallback;
+import timely.balancer.resolver.eventing.MetricAssignedEvent;
+import timely.balancer.resolver.eventing.MetricBalanceCallback;
+import timely.balancer.resolver.eventing.MetricBalanceEvent;
+import timely.balancer.resolver.eventing.MetricHostCallback;
+import timely.balancer.resolver.eventing.MetricHostEvent;
 
 public class BalancedMetricResolver implements MetricResolver {
 
     private static final Logger LOG = LoggerFactory.getLogger(BalancedMetricResolver.class);
     private Map<String, TimelyBalancedHost> metricToHostMap = new TreeMap<>();
+    private Set<String> unassignedMetrics = new HashSet<>();
     private Map<String, ArrivalRate> metricMap = new HashMap<>();
     private ReentrantReadWriteLock balancerLock = new ReentrantReadWriteLock();
     private List<TimelyBalancedHost> serverList = new ArrayList<>();
     private Random r = new Random();
-    final private HealthChecker healthChecker;
-    private Timer timer = new Timer("RebalanceTimer", true);
-    private Timer arrivalRateTimer = new Timer("ArrivalRateTimerResolver", true);
+    private final HealthChecker healthChecker;
+    private ScheduledExecutorService assignmentExecutor = Executors.newScheduledThreadPool(2);
+    private ScheduledExecutorService arrivalRateExecutor = Executors.newScheduledThreadPool(2);
     private int roundRobinCounter = 0;
     private Set<String> nonCachedMetrics = new HashSet<>();
     private DistributedAtomicValue nonCachedMetricsIP;
     private ReentrantReadWriteLock nonCachedMetricsLocalLock = new ReentrantReadWriteLock();
     private InterProcessReadWriteLock nonCachedMetricsIPRWLock;
-    private BalancerConfiguration balancerConfig;
+    private AtomicLong assignmentsLastUpdatedLocal = new AtomicLong(0);
     private LeaderLatch leaderLatch;
     private AtomicBoolean isLeader = new AtomicBoolean(false);
     private InterProcessReadWriteLock assignmentsIPRWLock;
-    private DistributedAtomicLong assignmentsLastUpdatedInHdfs;
-    private AtomicLong assignmentsLastUpdatedLocal = new AtomicLong(0);
+    private MetricAssignmentPerister assignmentPerister;
 
-    private enum BalanceType {
-        HIGH_LOW, HIGH_AVG, AVG_LOW
+    final private BalancerConfiguration balancerConfig;
+    final private CuratorFramework curatorFramework;
+
+    private List<MetricAssignedCallback> metricAssignedCallbacks = new ArrayList<>();
+    private List<MetricBalanceCallback> metricBalanceCallbacks = new ArrayList<>();
+    private List<MetricHostCallback> metricHostCallbacks = new ArrayList<>();
+
+    public enum BalanceType {
+        HIGH_LOW, HIGH_AVG, AVG_LOW;
     }
 
     public BalancedMetricResolver(CuratorFramework curatorFramework, BalancerConfiguration balancerConfig,
             HealthChecker healthChecker) {
+        this.curatorFramework = curatorFramework;
         this.balancerConfig = balancerConfig;
         this.healthChecker = healthChecker;
+    }
 
+    public void start() {
         assignmentsIPRWLock = new InterProcessReadWriteLock(curatorFramework, ASSIGNMENTS_LOCK_PATH);
         testIPRWLock(curatorFramework, assignmentsIPRWLock, ASSIGNMENTS_LOCK_PATH);
+
+        this.assignmentPerister = new MetricAssignmentPerister(this, balancerConfig, metricToHostMap, nonCachedMetrics,
+                curatorFramework, assignmentsIPRWLock, nonCachedMetricsLocalLock, assignmentsLastUpdatedLocal,
+                balancerLock);
+
         startLeaderLatch(curatorFramework);
         startServiceListener(curatorFramework);
-        assignmentsLastUpdatedInHdfs = new DistributedAtomicLong(curatorFramework, ASSIGNMENTS_LAST_UPDATED_PATH,
-                new RetryForever(1000));
 
-        TreeCacheListener assignmentListener = new TreeCacheListener() {
-
-            @Override
-            public void childEvent(CuratorFramework curatorFramework, TreeCacheEvent event) throws Exception {
-                LOG.debug("Handling assignments event {}. assignmentsLastUpdatedInHdfs:{}", event.getType().toString(),
-                        new Date(assignmentsLastUpdatedInHdfs.get().postValue()));
-                if (event.getType().equals(TreeCacheEvent.Type.NODE_UPDATED)) {
-                    long lastLocalUpdate = assignmentsLastUpdatedLocal.get();
-                    long lastHdfsUpdate = assignmentsLastUpdatedInHdfs.get().postValue();
-                    if (lastHdfsUpdate > lastLocalUpdate) {
-                        LOG.debug("Reading assignments from hdfs lastHdfsUpdate ({}) > lastLocalUpdate ({})",
-                                new Date(lastHdfsUpdate), new Date(lastLocalUpdate));
-                        readAssignmentsFromHdfs();
-                    } else {
-                        LOG.debug("Not reading assignments from hdfs lastHdfsUpdate ({}) <= lastLocalUpdate ({})",
-                                new Date(lastHdfsUpdate), new Date(lastLocalUpdate));
-                    }
-                }
+        TreeCacheListener assignmentListener = (curatorFramework1, event) -> {
+            LOG.debug("Handling assignments event {}. assignmentsLastUpdated:{}", event.getType().toString(),
+                    new Date(assignmentPerister.getAssignmentsLastUpdatedByLeader()));
+            if (event.getType().equals(TreeCacheEvent.Type.NODE_UPDATED)) {
+                assignmentPerister.readAssignments(true);
             }
         };
 
-        try {
-            TreeCache assignmentTreeCache = new TreeCache(curatorFramework, ASSIGNMENTS_LAST_UPDATED_PATH);
+        try (TreeCache assignmentTreeCache = new TreeCache(curatorFramework, ASSIGNMENTS_LAST_UPDATED_PATH)) {
             assignmentTreeCache.getListenable().addListener(assignmentListener);
             assignmentTreeCache.start();
         } catch (Exception e) {
@@ -144,8 +145,7 @@ public class BalancedMetricResolver implements MetricResolver {
             }
         };
 
-        try {
-            TreeCache nonCachedMetricsTreeCache = new TreeCache(curatorFramework, NON_CACHED_METRICS);
+        try (TreeCache nonCachedMetricsTreeCache = new TreeCache(curatorFramework, NON_CACHED_METRICS)) {
             nonCachedMetricsTreeCache.getListenable().addListener(nonCachedMetricsListener);
             nonCachedMetricsTreeCache.start();
         } catch (Exception e) {
@@ -160,55 +160,55 @@ public class BalancedMetricResolver implements MetricResolver {
         }
         readNonCachedMetricsIP();
 
-        readAssignmentsFromHdfs();
+        assignmentPerister.readAssignments(false);
 
-        timer.schedule(new TimerTask() {
-
-            @Override
-            public void run() {
-                if (isLeader.get()) {
-                    try {
-                        balance();
-                    } catch (Exception e) {
-                        LOG.error(e.getMessage(), e);
-                    }
-                }
-            }
-        }, 900000, 900000);
-
-        timer.schedule(new TimerTask() {
-
-            @Override
-            public void run() {
+        assignmentExecutor.scheduleAtFixedRate(() -> {
+            if (isLeader.get()) {
                 try {
-                    if (isLeader.get()) {
-                        long lastLocalUpdate = assignmentsLastUpdatedLocal.get();
-                        long lastHdfsUpdate = assignmentsLastUpdatedInHdfs.get().postValue();
-                        if (lastLocalUpdate > lastHdfsUpdate) {
-                            LOG.debug("Writing assignments to hdfs lastLocalUpdate ({}) > lastHdfsUpdate ({})",
-                                    new Date(lastLocalUpdate), new Date(lastHdfsUpdate));
-                            writeAssigmentsToHdfs();
-                        }
-                    }
+                    balance();
                 } catch (Exception e) {
                     LOG.error(e.getMessage(), e);
                 }
             }
-        }, 10000, 60000);
+        }, balancerConfig.getBalanceDelay(), balancerConfig.getBalancePeriod(), TimeUnit.MILLISECONDS);
+
+        assignmentExecutor.scheduleAtFixedRate(() -> {
+            try {
+                if (isLeader.get()) {
+                    long lastLocalUpdate = assignmentsLastUpdatedLocal.get();
+                    long lastUpdateByLeader = assignmentPerister.getAssignmentsLastUpdatedByLeader();
+                    if (lastLocalUpdate > lastUpdateByLeader) {
+                        LOG.debug("Leader writing assignments lastLocalUpdate ({}) > lastUpdateByLeader ({})",
+                                new Date(lastLocalUpdate), new Date(lastUpdateByLeader));
+                        assignmentPerister.writeAssignments();
+                    } else {
+                        LOG.trace("Leader not writing assignments lastLocalUpdate ({}) <= lastUpdateByLeader ({})",
+                                new Date(lastLocalUpdate), new Date(lastUpdateByLeader));
+                    }
+                } else {
+                    assignmentPerister.readAssignments(true);
+                }
+            } catch (Exception e) {
+                LOG.error(e.getMessage(), e);
+            }
+        }, balancerConfig.getPersistDelay(), balancerConfig.getPersistPeriod(), TimeUnit.MILLISECONDS);
     }
 
     private void readNonCachedMetricsIP() {
 
         Set<String> currentNonCachedMetricsDistributed = new TreeSet<>();
         try {
-            nonCachedMetricsIPRWLock.readLock().acquire();
+            boolean acquired = false;
+            while (!acquired) {
+                acquired = nonCachedMetricsIPRWLock.readLock().acquire(60, TimeUnit.SECONDS);
+            }
             byte[] currentNonCachedMetricsDistributedBytes = nonCachedMetricsIP.get().postValue();
             if (currentNonCachedMetricsDistributedBytes != null) {
                 try {
                     currentNonCachedMetricsDistributed = SerializationUtils
                             .deserialize(currentNonCachedMetricsDistributedBytes);
                 } catch (Exception e) {
-                    LOG.error(e.getMessage());
+                    LOG.error(e.getMessage(), e);
                     currentNonCachedMetricsDistributed = new TreeSet<>();
                 }
             }
@@ -249,7 +249,7 @@ public class BalancedMetricResolver implements MetricResolver {
                 public void isLeader() {
                     LOG.info("this balancer is the leader");
                     isLeader.set(true);
-                    writeAssigmentsToHdfs();
+                    assignmentPerister.writeAssignments();
                 }
 
                 @Override
@@ -275,11 +275,11 @@ public class BalancedMetricResolver implements MetricResolver {
                 for (ServiceInstance<ServerDetails> si : instances) {
                     ServerDetails pl = si.getPayload();
                     TimelyBalancedHost tbh = TimelyBalancedHost.of(pl.getHost(), pl.getTcpPort(), pl.getHttpPort(),
-                            pl.getWsPort(), pl.getUdpPort());
-                    LOG.info("adding service {} host:{} tcpPort:{} httpPort:{} wsPort:{} udpPort:{}", si.getId(),
-                            pl.getHost(), pl.getTcpPort(), pl.getHttpPort(), pl.getWsPort(), pl.getUdpPort());
+                            pl.getWsPort(), pl.getUdpPort(), new ArrivalRate(arrivalRateExecutor));
+                    LOG.info("adding service {} host:{}", si.getId(), TimelyBalancedHost.toStringShort(tbh));
                     tbh.setBalancerConfig(balancerConfig);
                     serverList.add(tbh);
+                    metricHostEvent(tbh, MetricHostEvent.ActionType.ADDED);
                 }
                 healthChecker.setTimelyHosts(serverList);
             } finally {
@@ -300,7 +300,8 @@ public class BalancedMetricResolver implements MetricResolver {
                         for (ServiceInstance<ServerDetails> si : instances) {
                             ServerDetails pl = (ServerDetails) si.getPayload();
                             TimelyBalancedHost tbh = TimelyBalancedHost.of(pl.getHost(), pl.getTcpPort(),
-                                    pl.getHttpPort(), pl.getWsPort(), pl.getUdpPort());
+                                    pl.getHttpPort(), pl.getWsPort(), pl.getUdpPort(),
+                                    new ArrivalRate(arrivalRateExecutor));
                             tbh.setBalancerConfig(balancerConfig);
                             availableHosts.add(tbh);
                         }
@@ -314,9 +315,8 @@ public class BalancedMetricResolver implements MetricResolver {
                                 availableHosts.remove(h);
                             } else {
                                 itr.remove();
-                                LOG.info("removing service {}:{} host:{} tcpPort:{} httpPort:{} wsPort:{} udpPort:{}",
-                                        h.getHost(), h.getTcpPort(), h.getHost(), h.getTcpPort(), h.getHttpPort(),
-                                        h.getWsPort(), h.getUdpPort());
+                                metricHostEvent(h, MetricHostEvent.ActionType.REMOVED);
+                                LOG.info("removing service {}", TimelyBalancedHost.toStringShort(h));
                                 for (Map.Entry<String, TimelyBalancedHost> e : metricToHostMap.entrySet()) {
                                     if (e.getValue().equals(h)) {
                                         reassignMetrics.add(e.getKey());
@@ -328,28 +328,33 @@ public class BalancedMetricResolver implements MetricResolver {
 
                         // add new hosts that were not previously known
                         for (TimelyBalancedHost h : availableHosts) {
-                            LOG.info("adding service {}:{} host:{} tcpPort:{} httpPort:{} wsPort:{} udpPort:{}",
-                                    h.getHost(), h.getTcpPort(), h.getHost(), h.getTcpPort(), h.getHttpPort(),
-                                    h.getWsPort(), h.getUdpPort());
+                            LOG.info("adding service {}", TimelyBalancedHost.toStringShort(h));
                             serverList.add(h);
+                            metricHostEvent(h, MetricHostEvent.ActionType.ADDED);
                             rebalanceNeeded = true;
                         }
                         healthChecker.setTimelyHosts(serverList);
-
+                        if (!availableHosts.isEmpty()) {
+                            reassignMetrics.addAll(unassignedMetrics);
+                            unassignedMetrics.clear();
+                        }
                         if (isLeader.get()) {
                             for (String s : reassignMetrics) {
                                 TimelyBalancedHost h = getRoundRobinHost(null);
-                                assignMetric(s, h);
-                                LOG.debug("Assigned server removed.  Assigning {} to server {}:{}", s, h.getTcpPort());
+                                assignMetric(s, h, HOST_REMOVED);
+                                LOG.debug("Assigned server removed.  Assigning {} to server {}", s,
+                                        TimelyBalancedHost.toStringShort(h));
                             }
-                            writeAssigmentsToHdfs();
                         }
 
                     } finally {
                         balancerLock.writeLock().unlock();
                     }
-                    if (isLeader.get() && rebalanceNeeded) {
-                        balance();
+                    if (isLeader.get()) {
+                        if (rebalanceNeeded) {
+                            balance();
+                        }
+                        assignmentPerister.writeAssignments();
                     }
                 }
 
@@ -409,7 +414,7 @@ public class BalancedMetricResolver implements MetricResolver {
         return tbh;
     }
 
-    private TimelyBalancedHost getRoundRobinHost(TimelyBalancedHost notThisOne) {
+    protected TimelyBalancedHost getRoundRobinHost(TimelyBalancedHost notThisOne) {
         TimelyBalancedHost tbh = null;
         balancerLock.readLock().lock();
         try {
@@ -531,8 +536,10 @@ public class BalancedMetricResolver implements MetricResolver {
 
         int numReassigned = 0;
         if (isLeader.get()) {
+            metricBalanceEvent(MetricBalanceEvent.ProgressType.BEGIN, balanceType, 0);
             balancerLock.writeLock().lock();
             try {
+                metricBalanceEvent(MetricBalanceEvent.ProgressType.BALANCER_LOCK_ACQUIRED, balanceType, 0);
                 Map<String, ArrivalRate> tempMetricMap = new HashMap<>();
                 tempMetricMap.putAll(metricMap);
 
@@ -562,8 +569,8 @@ public class BalancedMetricResolver implements MetricResolver {
                         }
                     }
 
-                    LOG.trace("focusHost {}:{} desiredChange:{} rateSortedMetrics.size():{}", h.getHost(),
-                            h.getTcpPort(), desiredChange, rateSortedMetrics.size());
+                    LOG.trace("focusHost {} desiredChange:{} rateSortedMetrics.size():{}",
+                            TimelyBalancedHost.toStringShort(h), desiredChange, rateSortedMetrics.size());
 
                     boolean doneWithHost = false;
                     while (!doneWithHost) {
@@ -592,7 +599,7 @@ public class BalancedMetricResolver implements MetricResolver {
                                 } else {
                                     String metric = e.getValue();
                                     Double metricRate = e.getKey();
-                                    assignMetric(metric, candidateHost);
+                                    assignMetric(metric, candidateHost, REBALANCE);
                                     numReassigned++;
                                     calculatedRates.put(candidateHost, calculatedRates.get(candidateHost) + metricRate);
                                     calculatedRates.put(h, calculatedRates.get(h) - metricRate);
@@ -601,9 +608,9 @@ public class BalancedMetricResolver implements MetricResolver {
                                     itr.remove();
                                     tempMetricMap.remove(metric);
                                     LOG.info(
-                                            "rebalancing: reassigning metric:{} rate:{} from server {}:{} to {}:{} remaining delta {}",
-                                            metric, metricRate, h.getHost(), h.getTcpPort(), candidateHost.getHost(),
-                                            candidateHost.getTcpPort(), desiredChange);
+                                            "rebalancing: reassigning metric:{} rate:{} from server {} to {} remaining delta {}",
+                                            metric, metricRate, TimelyBalancedHost.toStringShort(h),
+                                            TimelyBalancedHost.toStringShort(candidateHost), desiredChange);
                                 }
                             }
                         }
@@ -626,6 +633,7 @@ public class BalancedMetricResolver implements MetricResolver {
                 }
             } finally {
                 balancerLock.writeLock().unlock();
+                metricBalanceEvent(MetricBalanceEvent.ProgressType.END, balanceType, numReassigned);
             }
         }
         return numReassigned;
@@ -643,7 +651,10 @@ public class BalancedMetricResolver implements MetricResolver {
                 }
 
                 try {
-                    nonCachedMetricsIPRWLock.writeLock().acquire();
+                    boolean acquired = false;
+                    while (!acquired) {
+                        acquired = nonCachedMetricsIPRWLock.writeLock().acquire(60, TimeUnit.SECONDS);
+                    }
                     byte[] currentNonCachedMetricsDistributedBytes = nonCachedMetricsIP.get().postValue();
                     Set<String> currentNonCachedMetricsIP;
                     if (currentNonCachedMetricsDistributedBytes == null) {
@@ -690,12 +701,12 @@ public class BalancedMetricResolver implements MetricResolver {
             try {
                 lock.writeLock().release();
             } catch (Exception e3) {
-                LOG.error(e3.getMessage());
+                LOG.error(e3.getMessage(), e3);
             }
         }
     }
 
-    private boolean shouldCache(String metricName) {
+    protected boolean shouldCache(String metricName) {
 
         if (StringUtils.isBlank(metricName)) {
             return false;
@@ -744,7 +755,7 @@ public class BalancedMetricResolver implements MetricResolver {
             if (chooseMetricSpecificHost) {
                 ArrivalRate rate = metricMap.get(metric);
                 if (rate == null) {
-                    rate = new ArrivalRate(arrivalRateTimer);
+                    rate = new ArrivalRate(arrivalRateExecutor);
                     if (!balancerLock.isWriteLockedByCurrentThread()) {
                         balancerLock.readLock().unlock();
                         balancerLock.writeLock().lock();
@@ -757,7 +768,7 @@ public class BalancedMetricResolver implements MetricResolver {
             }
 
             if (StringUtils.isBlank(metric)) {
-                tbh = getRandomHost(null);
+                tbh = getRoundRobinHost(null);
             } else {
                 tbh = metricToHostMap.get(metric);
                 if (tbh == null) {
@@ -766,17 +777,17 @@ public class BalancedMetricResolver implements MetricResolver {
                         balancerLock.readLock().unlock();
                         balancerLock.writeLock().lock();
                     }
-                    assignMetric(metric, tbh);
+                    assignMetric(metric, tbh, ASSIGN_ROUND_ROBIN);
                 } else if (!tbh.isUp()) {
                     TimelyBalancedHost oldTbh = tbh;
-                    tbh = getLeastUsedHost();
-                    LOG.debug("rebalancing from host that is down: reassigning metric {} from server {}:{} to {}:{}",
-                            metric, oldTbh.getHost(), oldTbh.getTcpPort(), tbh.getHost(), tbh.getTcpPort());
+                    tbh = getRoundRobinHost(oldTbh);
+                    LOG.debug("rebalancing from host that is down: reassigning metric {} from server {} to {}", metric,
+                            TimelyBalancedHost.toStringShort(oldTbh), TimelyBalancedHost.toStringShort(tbh));
                     if (!balancerLock.isWriteLockedByCurrentThread()) {
                         balancerLock.readLock().unlock();
                         balancerLock.writeLock().lock();
                     }
-                    assignMetric(metric, tbh);
+                    assignMetric(metric, tbh, HOST_DOWN_ROUND_ROBIN);
                 }
             }
 
@@ -793,7 +804,7 @@ public class BalancedMetricResolver implements MetricResolver {
                         balancerLock.readLock().unlock();
                         balancerLock.writeLock().lock();
                     }
-                    assignMetric(metric, tbh);
+                    assignMetric(metric, tbh, ASSIGN_FALLBACK_SEQUENTIAL);
                 }
             }
         } finally {
@@ -822,20 +833,23 @@ public class BalancedMetricResolver implements MetricResolver {
                 metric = null;
             }
 
-            if (tbh == null || !tbh.isUp()) {
-                for (int x = 0; tbh == null && x < serverList.size(); x++) {
-                    tbh = serverList.get(Math.abs(r.nextInt() & Integer.MAX_VALUE) % serverList.size());
-                    if (!tbh.isUp()) {
-                        tbh = null;
-                    }
+            if (tbh == null) {
+                tbh = getRoundRobinHost(null);
+                if (!balancerLock.isWriteLockedByCurrentThread()) {
+                    balancerLock.readLock().unlock();
+                    balancerLock.writeLock().lock();
                 }
-                if (tbh != null && StringUtils.isNotBlank(metric)) {
-                    if (!balancerLock.isWriteLockedByCurrentThread()) {
-                        balancerLock.readLock().unlock();
-                        balancerLock.writeLock().lock();
-                    }
-                    assignMetric(metric, tbh);
+                assignMetric(metric, tbh, ASSIGN_ROUND_ROBIN);
+            } else if (!tbh.isUp()) {
+                TimelyBalancedHost oldTbh = tbh;
+                tbh = getRoundRobinHost(oldTbh);
+                LOG.debug("rebalancing from host that is down: reassigning metric {} from server {} to {}", metric,
+                        TimelyBalancedHost.toStringShort(oldTbh), TimelyBalancedHost.toStringShort(tbh));
+                if (!balancerLock.isWriteLockedByCurrentThread()) {
+                    balancerLock.readLock().unlock();
+                    balancerLock.writeLock().lock();
                 }
+                assignMetric(metric, tbh, HOST_DOWN_ROUND_ROBIN);
             }
 
             // if all else fails
@@ -851,7 +865,7 @@ public class BalancedMetricResolver implements MetricResolver {
                         balancerLock.readLock().unlock();
                         balancerLock.writeLock().lock();
                     }
-                    assignMetric(metric, tbh);
+                    assignMetric(metric, tbh, ASSIGN_FALLBACK_SEQUENTIAL);
                 }
             }
         } finally {
@@ -864,7 +878,7 @@ public class BalancedMetricResolver implements MetricResolver {
         return tbh;
     }
 
-    private TimelyBalancedHost findHost(String host, int tcpPort) {
+    protected TimelyBalancedHost findHost(String host, int tcpPort) {
         TimelyBalancedHost tbh = null;
         for (TimelyBalancedHost h : serverList) {
             if (h.getHost().equals(host) && h.getTcpPort() == tcpPort) {
@@ -875,144 +889,100 @@ public class BalancedMetricResolver implements MetricResolver {
         return tbh;
     }
 
-    private void readAssignmentsFromHdfs() {
-
-        Map<String, TimelyBalancedHost> assignedMetricToHostMap = new TreeMap<>();
-        try {
-            assignmentsIPRWLock.readLock().acquire();
-            balancerLock.writeLock().lock();
-            Configuration configuration = new Configuration();
-            FileSystem fs = FileSystem.get(configuration);
-            Path assignmentFile = new Path(balancerConfig.getAssignmentFile());
-            FSDataInputStream iStream = fs.open(assignmentFile);
-
-            CsvReader reader = new CsvReader(iStream, ',', Charset.forName("UTF-8"));
-            reader.setUseTextQualifier(false);
-
-            // skip the headers
-            boolean success = true;
-            success = reader.readHeaders();
-
-            while (success) {
-                success = reader.readRecord();
-                String[] nextLine = reader.getValues();
-                if (nextLine.length >= 3) {
-                    String metric = nextLine[0];
-                    String host = nextLine[1];
-                    int tcpPort = Integer.parseInt(nextLine[2]);
-                    TimelyBalancedHost tbh = findHost(host, tcpPort);
-                    if (tbh == null) {
-                        tbh = getRoundRobinHost(null);
-                    } else {
-                        LOG.trace("Found assigment: {} to {}:{}", metric, host, tcpPort);
-                    }
-                    assignedMetricToHostMap.put(metric, tbh);
-                }
-            }
-
-            metricToHostMap.clear();
-            for (Map.Entry<String, TimelyBalancedHost> e : assignedMetricToHostMap.entrySet()) {
-                if (shouldCache(e.getKey())) {
-                    metricToHostMap.put(e.getKey(), e.getValue());
-                }
-            }
-
-            nonCachedMetricsLocalLock.readLock().lock();
-            try {
-                Iterator<Map.Entry<String, TimelyBalancedHost>> itr = metricToHostMap.entrySet().iterator();
-                while (itr.hasNext()) {
-                    Map.Entry<String, TimelyBalancedHost> e = itr.next();
-                    if (nonCachedMetrics.contains(e.getKey())) {
-                        itr.remove();
-                    }
-                }
-            } finally {
-                nonCachedMetricsLocalLock.readLock().unlock();
-            }
-            assignmentsLastUpdatedLocal.set(assignmentsLastUpdatedInHdfs.get().postValue());
-            LOG.info("Read {} assignments from hdfs lastHdfsUpdate = lastLocalUpdate ({})", metricToHostMap.size(),
-                    new Date(assignmentsLastUpdatedLocal.get()));
-        } catch (Exception e) {
-            LOG.error(e.getMessage(), e);
-        } finally {
-            balancerLock.writeLock().unlock();
-            try {
-                assignmentsIPRWLock.readLock().release();
-            } catch (Exception e) {
-                LOG.error(e.getMessage(), e);
-            }
-        }
-    }
-
-    private void writeAssigmentsToHdfs() {
-
-        CsvWriter writer = null;
-        try {
-            assignmentsIPRWLock.writeLock().acquire();
-            balancerLock.readLock().lock();
-            nonCachedMetricsLocalLock.readLock().lock();
-            try {
-                if (!metricToHostMap.isEmpty()) {
-                    Configuration configuration = new Configuration();
-                    FileSystem fs = FileSystem.get(configuration);
-                    Path assignmentFile = new Path(balancerConfig.getAssignmentFile());
-                    if (!fs.exists(assignmentFile.getParent())) {
-                        fs.mkdirs(assignmentFile.getParent());
-                    }
-                    FSDataOutputStream oStream = fs.create(assignmentFile, true);
-                    writer = new CsvWriter(oStream, ',', Charset.forName("UTF-8"));
-                    writer.setUseTextQualifier(false);
-                    writer.write("metric");
-                    writer.write("host");
-                    writer.write("tcpPort");
-                    writer.endRecord();
-                    for (Map.Entry<String, TimelyBalancedHost> e : metricToHostMap.entrySet()) {
-                        if (!nonCachedMetrics.contains(e.getKey())) {
-                            writer.write(e.getKey());
-                            writer.write(e.getValue().getHost());
-                            writer.write(Integer.toString(e.getValue().getTcpPort()));
-                            writer.endRecord();
-                            LOG.trace("Saving assigment: {} to {}:{}", e.getKey(), e.getValue().getHost(),
-                                    e.getValue().getTcpPort());
-                        }
-                    }
-
-                    long now = System.currentTimeMillis();
-                    assignmentsLastUpdatedLocal.set(now);
-                    assignmentsLastUpdatedInHdfs.trySet(now);
-                    if (!assignmentsLastUpdatedInHdfs.get().succeeded()) {
-                        assignmentsLastUpdatedInHdfs.forceSet(now);
-                    }
-                    LOG.debug("Wrote {} assignments to hdfs lastHdfsUpdate = lastLocalUpdate ({})",
-                            metricToHostMap.size(), new Date(assignmentsLastUpdatedLocal.get()));
-                }
-            } finally {
-                nonCachedMetricsLocalLock.readLock().unlock();
-                balancerLock.readLock().unlock();
-            }
-        } catch (Exception e) {
-            LOG.error(e.getMessage(), e);
-        } finally {
-            if (writer != null) {
-                writer.close();
-            }
-            try {
-                assignmentsIPRWLock.writeLock().release();
-            } catch (Exception e) {
-                LOG.error(e.getMessage(), e);
-            }
-        }
-    }
-
-    private void assignMetric(String metric, TimelyBalancedHost tbh) {
+    private void assignMetric(String metric, TimelyBalancedHost tbh, MetricAssignedEvent.Reason reason) {
         if (isLeader.get()) {
-            balancerLock.writeLock().lock();
-            try {
-                metricToHostMap.put(metric, tbh);
-                assignmentsLastUpdatedLocal.set(System.currentTimeMillis());
-            } finally {
-                balancerLock.writeLock().unlock();
+            if (StringUtils.isNotBlank(metric) && tbh != null) {
+                balancerLock.writeLock().lock();
+                try {
+                    metricAssignedEvent(metric, metricToHostMap.get(metric), tbh, reason);
+                    metricToHostMap.put(metric, tbh);
+                    assignmentsLastUpdatedLocal.set(System.currentTimeMillis());
+                } finally {
+                    balancerLock.writeLock().unlock();
+                }
+            } else {
+                unassignedMetrics.add(metric);
+                // Exception e = new IllegalStateException("Bad assignment metric:" + metric + "
+                // host:" + tbh);
+                LOG.warn("Bad assignment metric:" + metric + " host:" + tbh);
             }
         }
     }
+
+    public void registerCallback(MetricAssignedCallback callback) {
+        this.metricAssignedCallbacks.add(callback);
+    }
+
+    public void registerCallback(MetricBalanceCallback callback) {
+        this.metricBalanceCallbacks.add(callback);
+    }
+
+    public void registerCallback(MetricHostCallback callback) {
+        this.metricHostCallbacks.add(callback);
+    }
+
+    protected void metricAssignedEvent(String metric, TimelyBalancedHost losing, TimelyBalancedHost gaining,
+            MetricAssignedEvent.Reason reason) {
+        if (!metricAssignedCallbacks.isEmpty()) {
+            MetricAssignedEvent event = new MetricAssignedEvent(metric, losing, gaining, reason);
+            for (MetricAssignedCallback callback : metricAssignedCallbacks) {
+                callback.onEvent(event);
+            }
+        }
+    }
+
+    protected void metricBalanceEvent(MetricBalanceEvent.ProgressType progressType,
+            BalancedMetricResolver.BalanceType balanceType, long numReassigned) {
+        if (!metricBalanceCallbacks.isEmpty()) {
+            MetricBalanceEvent event = new MetricBalanceEvent(progressType, balanceType, numReassigned);
+            for (MetricBalanceCallback callback : metricBalanceCallbacks) {
+                callback.onEvent(event);
+            }
+        }
+    }
+
+    protected void metricHostEvent(TimelyBalancedHost host, MetricHostEvent.ActionType actionType) {
+        if (!metricHostCallbacks.isEmpty()) {
+            MetricHostEvent event = new MetricHostEvent(host, actionType);
+            for (MetricHostCallback callback : metricHostCallbacks) {
+                callback.onEvent(event);
+            }
+        }
+    }
+
+    protected void writeAssignments(Map<String, TimelyBalancedHost> metricToHostMap) {
+        assignmentPerister.writeAssignments(metricToHostMap);
+    }
+
+    @Override
+    public void stop() {
+
+        this.assignmentExecutor.shutdown();
+        try {
+            this.assignmentExecutor.awaitTermination(5, TimeUnit.SECONDS);
+        } catch (InterruptedException e) {
+
+        } finally {
+            if (!this.assignmentExecutor.isTerminated()) {
+                this.assignmentExecutor.shutdownNow();
+            }
+        }
+
+        this.arrivalRateExecutor.shutdown();
+        try {
+            this.arrivalRateExecutor.awaitTermination(5, TimeUnit.SECONDS);
+        } catch (InterruptedException e) {
+
+        } finally {
+            if (!this.arrivalRateExecutor.isTerminated()) {
+                this.arrivalRateExecutor.shutdownNow();
+            }
+        }
+
+        try {
+            this.healthChecker.close();
+        } catch (Exception e) {
+            LOG.error(e.getMessage(), e);
+        }
+    }
 }

balancer/src/main/java/timely/balancer/resolver/MetricAssignmentPerister.java

@@ -0,0 +1,320 @@
+package timely.balancer.resolver;
+
+import static timely.balancer.Balancer.ASSIGNMENTS_LAST_UPDATED_PATH;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.charset.Charset;
+import java.util.Date;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.apache.commons.lang.StringUtils;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.recipes.atomic.DistributedAtomicLong;
+import org.apache.curator.framework.recipes.locks.InterProcessReadWriteLock;
+import org.apache.curator.retry.RetryForever;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import timely.balancer.configuration.BalancerConfiguration;
+import timely.balancer.connection.TimelyBalancedHost;
+import timely.balancer.resolver.eventing.MetricAssignedEvent;
+
+public class MetricAssignmentPerister {
+
+    private final Logger LOG = LoggerFactory.getLogger(getClass());
+
+    private enum PeristerType {
+        HDFS, FILE
+    }
+
+    // passed in through constructor
+    BalancedMetricResolver balancedMetricResolver;
+    private BalancerConfiguration balancerConfig;
+    private Map<String, TimelyBalancedHost> metricToHostMap;
+    private Set<String> nonCachedMetrics;
+    private InterProcessReadWriteLock assignmentsIPRWLock;
+    private ReentrantReadWriteLock nonCachedMetricsLocalLock;
+    private ReentrantReadWriteLock balancerLock;
+    private AtomicLong assignmentsLastUpdatedLocal;
+
+    // local
+    private DistributedAtomicLong assignmentsLastUpdatedInHdfs;
+
+    public MetricAssignmentPerister(BalancedMetricResolver balancedMetricResolver, BalancerConfiguration balancerConfig,
+            Map<String, TimelyBalancedHost> metricToHostMap, Set<String> nonCachedMetrics,
+            CuratorFramework curatorFramework, InterProcessReadWriteLock assignmentsIPRWLock,
+            ReentrantReadWriteLock nonCachedMetricsLocalLock, AtomicLong assignmentsLastUpdatedLocal,
+            ReentrantReadWriteLock balancerLock) {
+
+        this.balancedMetricResolver = balancedMetricResolver;
+        this.balancerConfig = balancerConfig;
+        this.metricToHostMap = metricToHostMap;
+        this.nonCachedMetrics = nonCachedMetrics;
+        this.assignmentsIPRWLock = assignmentsIPRWLock;
+        this.nonCachedMetricsLocalLock = nonCachedMetricsLocalLock;
+        this.assignmentsLastUpdatedLocal = assignmentsLastUpdatedLocal;
+        this.balancerLock = balancerLock;
+        assignmentsLastUpdatedInHdfs = new DistributedAtomicLong(curatorFramework, ASSIGNMENTS_LAST_UPDATED_PATH,
+                new RetryForever(1000));
+    }
+
+    public long getAssignmentsLastUpdatedByLeader() throws Exception {
+        return assignmentsLastUpdatedInHdfs.get().postValue();
+    }
+
+    public void readAssignments(boolean checkIfNecessary) {
+
+        try {
+            long lastLocalUpdate = assignmentsLastUpdatedLocal.get();
+            long lastHdfsUpdate = assignmentsLastUpdatedInHdfs.get().postValue();
+            if (checkIfNecessary) {
+                if (lastHdfsUpdate <= lastLocalUpdate) {
+                    LOG.debug("Not reading assignments from storage lastStorageUpdate ({}) <= lastLocalUpdate ({})",
+                            new Date(lastHdfsUpdate), new Date(lastLocalUpdate));
+                    return;
+                }
+            }
+            LOG.debug("Reading assignments from storage lastStorageUpdate ({}) > lastLocalUpdate ({})",
+                    new Date(lastHdfsUpdate), new Date(lastLocalUpdate));
+        } catch (Exception e) {
+            LOG.error(e.getMessage(), e);
+        }
+
+        // proceed with reading from storage
+
+        try (InputStream inputStream = getInputStream(balancerConfig)) {
+            boolean acquired = false;
+            while (!acquired) {
+                acquired = assignmentsIPRWLock.readLock().acquire(60, TimeUnit.SECONDS);
+            }
+            balancerLock.writeLock().lock();
+            // get potential new list from implementation
+            Map<String, TimelyBalancedHost> assignedMetricToHostMap = readFromStream(inputStream);
+            metricToHostMap.clear();
+            for (Map.Entry<String, TimelyBalancedHost> e : assignedMetricToHostMap.entrySet()) {
+                if (StringUtils.isNotBlank(e.getKey()) && e.getValue() != null) {
+                    if (balancedMetricResolver.shouldCache(e.getKey())) {
+                        balancedMetricResolver.metricAssignedEvent(e.getKey(), null, e.getValue(),
+                                MetricAssignedEvent.Reason.ASSIGN_FILE);
+                        metricToHostMap.put(e.getKey(), e.getValue());
+                    }
+                } else {
+                    Exception e1 = new IllegalStateException(
+                            "Bad assignment metric:" + e.getKey() + " host:" + e.getValue());
+                    LOG.warn(e1.getMessage(), e1);
+                }
+            }
+            nonCachedMetricsLocalLock.readLock().lock();
+            try {
+                Iterator<Map.Entry<String, TimelyBalancedHost>> itr = metricToHostMap.entrySet().iterator();
+                while (itr.hasNext()) {
+                    Map.Entry<String, TimelyBalancedHost> e = itr.next();
+                    if (nonCachedMetrics.contains(e.getKey())) {
+                        itr.remove();
+                    }
+                }
+            } finally {
+                nonCachedMetricsLocalLock.readLock().unlock();
+            }
+            assignmentsLastUpdatedLocal.set(assignmentsLastUpdatedInHdfs.get().postValue());
+            LOG.info("Read {} assignments from storage lastStorageUpdate = lastLocalUpdate ({})",
+                    metricToHostMap.size(), new Date(assignmentsLastUpdatedLocal.get()));
+        } catch (FileNotFoundException e) {
+            createAssignmentFile();
+            metricToHostMap.clear();
+        } catch (Exception e) {
+            LOG.error(e.getMessage(), e);
+        } finally {
+            balancerLock.writeLock().unlock();
+            try {
+                assignmentsIPRWLock.readLock().release();
+            } catch (Exception e) {
+                LOG.error(e.getMessage(), e);
+            }
+        }
+    }
+
+    public void writeAssignments() {
+        writeAssignments(metricToHostMap);
+    }
+
+    protected void writeAssignments(Map<String, TimelyBalancedHost> metricToHostMap) {
+
+        try {
+            boolean acquired = false;
+            while (!acquired) {
+                acquired = assignmentsIPRWLock.writeLock().acquire(60, TimeUnit.SECONDS);
+            }
+            balancerLock.readLock().lock();
+            nonCachedMetricsLocalLock.readLock().lock();
+            try (OutputStream outputStream = getOutputStream(balancerConfig)) {
+                if (!metricToHostMap.isEmpty()) {
+                    writeToStream(outputStream);
+                    long now = System.currentTimeMillis();
+                    assignmentsLastUpdatedLocal.set(now);
+                    assignmentsLastUpdatedInHdfs.trySet(now);
+                    if (!assignmentsLastUpdatedInHdfs.get().succeeded()) {
+                        assignmentsLastUpdatedInHdfs.forceSet(now);
+                    }
+                    LOG.debug("Wrote {} assignments to storage lastStorageUpdate = lastLocalUpdate ({})",
+                            metricToHostMap.size(), new Date(assignmentsLastUpdatedLocal.get()));
+                }
+            } finally {
+                nonCachedMetricsLocalLock.readLock().unlock();
+                balancerLock.readLock().unlock();
+            }
+        } catch (Exception e) {
+            LOG.error(e.getMessage(), e);
+        } finally {
+            try {
+                assignmentsIPRWLock.writeLock().release();
+            } catch (Exception e) {
+                LOG.error(e.getMessage(), e);
+            }
+        }
+    }
+
+    public Map<String, TimelyBalancedHost> readFromStream(InputStream inputStream) throws IOException {
+        Map<String, TimelyBalancedHost> assignedMetricToHostMap = new TreeMap<>();
+        try {
+            CsvReader reader = new CsvReader(inputStream, ',', Charset.forName("UTF-8"));
+            reader.setUseTextQualifier(false);
+
+            // skip the headers
+            boolean success = true;
+            success = reader.readHeaders();
+
+            while (success) {
+                success = reader.readRecord();
+                String[] nextLine = reader.getValues();
+                if (nextLine.length >= 3) {
+                    String metric = nextLine[0];
+                    String host = nextLine[1];
+                    int tcpPort = Integer.parseInt(nextLine[2]);
+                    TimelyBalancedHost tbh = balancedMetricResolver.findHost(host, tcpPort);
+                    if (tbh == null) {
+                        tbh = balancedMetricResolver.getRoundRobinHost(null);
+                    } else {
+                        LOG.trace("Found assigment: {} to {}:{}", metric, host, tcpPort);
+                    }
+                    assignedMetricToHostMap.put(metric, tbh);
+                }
+            }
+        } catch (IOException e) {
+            LOG.error(e.getMessage(), e);
+            throw e;
+        }
+        return assignedMetricToHostMap;
+    }
+
+    private void writeToStream(OutputStream outputStream) throws IOException {
+
+        CsvWriter writer = null;
+        try {
+            writer = new CsvWriter(outputStream, ',', Charset.forName("UTF-8"));
+            writer.setUseTextQualifier(false);
+            writer.write("metric");
+            writer.write("host");
+            writer.write("tcpPort");
+            writer.endRecord();
+            for (Map.Entry<String, TimelyBalancedHost> e : metricToHostMap.entrySet()) {
+                if (!nonCachedMetrics.contains(e.getKey())) {
+                    writer.write(e.getKey());
+                    writer.write(e.getValue().getHost());
+                    writer.write(Integer.toString(e.getValue().getTcpPort()));
+                    writer.endRecord();
+                    LOG.trace("Saving assigment: {} to {}:{}", e.getKey(), e.getValue().getHost(),
+                            e.getValue().getTcpPort());
+                }
+            }
+        } finally {
+            if (writer != null) {
+                writer.close();
+            }
+        }
+    }
+
+    public void createAssignmentFile() {
+        LOG.info("Creating assignment file: " + balancerConfig.getAssignmentFile());
+        PeristerType peristerType = PeristerType.valueOf(balancerConfig.getMetricAssignmentPersisterType());
+        switch (peristerType) {
+            case HDFS:
+                try {
+                    Configuration configuration = new Configuration();
+                    FileSystem fs = FileSystem.get(configuration);
+                    Path assignmentFile = new Path(balancerConfig.getAssignmentFile());
+                    fs.create(assignmentFile);
+                } catch (Exception e) {
+                    LOG.error(e.getMessage(), e);
+                }
+                break;
+            case FILE:
+                try {
+                    File assignmentFile = new File(balancerConfig.getAssignmentFile());
+                    String parentPath = assignmentFile.getParent();
+                    File directory = new File(parentPath);
+                    if (!directory.exists()) {
+
+                        directory.mkdirs();
+                    }
+                    if (!assignmentFile.exists()) {
+                        assignmentFile.createNewFile();
+                    }
+                } catch (Exception e) {
+                    LOG.error(e.getMessage(), e);
+                }
+                break;
+        }
+    }
+
+    public InputStream getInputStream(BalancerConfiguration balancerConfiguration) throws Exception {
+        PeristerType peristerType = PeristerType.valueOf(balancerConfiguration.getMetricAssignmentPersisterType());
+        switch (peristerType) {
+            case HDFS:
+                Configuration configuration = new Configuration();
+                FileSystem fs = FileSystem.get(configuration);
+                Path assignmentFile = new Path(balancerConfiguration.getAssignmentFile());
+                return fs.open(assignmentFile);
+            case FILE:
+                return new FileInputStream(new File(balancerConfiguration.getAssignmentFile()));
+            default:
+                throw new IllegalArgumentException(
+                        "Unknown metricPersisterType " + balancerConfiguration.getMetricAssignmentPersisterType());
+        }
+    }
+
+    public OutputStream getOutputStream(BalancerConfiguration balancerConfiguration) throws IOException {
+        PeristerType peristerType = PeristerType.valueOf(balancerConfiguration.getMetricAssignmentPersisterType());
+        switch (peristerType) {
+            case HDFS:
+                Configuration configuration = new Configuration();
+                FileSystem fs = FileSystem.get(configuration);
+                Path assignmentFile = new Path(balancerConfiguration.getAssignmentFile());
+                if (!fs.exists(assignmentFile.getParent())) {
+                    fs.mkdirs(assignmentFile.getParent());
+                }
+                return fs.create(assignmentFile, true);
+            case FILE:
+                return new FileOutputStream(new File(balancerConfiguration.getAssignmentFile()));
+            default:
+                throw new IllegalArgumentException(
+                        "Unknown metricPersisterType " + balancerConfiguration.getMetricAssignmentPersisterType());
+        }
+    }
+}

balancer/src/main/java/timely/balancer/resolver/MetricResolver.java

@@ -7,4 +7,9 @@ public interface MetricResolver {
     public TimelyBalancedHost getHostPortKeyIngest(String metric);
 
     public TimelyBalancedHost getHostPortKey(String metric);
+
+    public void start();
+
+    public void stop();
+
 }

balancer/src/main/java/timely/balancer/resolver/eventing/MetricAssignedCallback.java

@@ -0,0 +1,7 @@
+package timely.balancer.resolver.eventing;
+
+public abstract class MetricAssignedCallback {
+
+    public abstract void onEvent(MetricAssignedEvent event);
+
+}

balancer/src/main/java/timely/balancer/resolver/eventing/MetricAssignedEvent.java

@@ -0,0 +1,39 @@
+package timely.balancer.resolver.eventing;
+
+import timely.balancer.connection.TimelyBalancedHost;
+
+public class MetricAssignedEvent {
+
+    public enum Reason {
+        ASSIGN_ROUND_ROBIN, HOST_DOWN_ROUND_ROBIN, ASSIGN_FALLBACK_SEQUENTIAL, HOST_REMOVED, REBALANCE, ASSIGN_FILE
+    }
+
+    private final String metric;
+    private final TimelyBalancedHost losingHost;
+    private final TimelyBalancedHost gainingHost;
+    private final Reason reason;
+
+    public MetricAssignedEvent(String metric, TimelyBalancedHost losingHost, TimelyBalancedHost gainingHost,
+            Reason reason) {
+        this.metric = metric;
+        this.losingHost = losingHost;
+        this.gainingHost = gainingHost;
+        this.reason = reason;
+    }
+
+    public String getMetric() {
+        return metric;
+    }
+
+    public TimelyBalancedHost getLosingHost() {
+        return losingHost;
+    }
+
+    public TimelyBalancedHost getGainingHost() {
+        return gainingHost;
+    }
+
+    public Reason getReason() {
+        return reason;
+    }
+}

balancer/src/main/java/timely/balancer/resolver/eventing/MetricBalanceCallback.java

@@ -0,0 +1,7 @@
+package timely.balancer.resolver.eventing;
+
+public abstract class MetricBalanceCallback {
+
+    public abstract void onEvent(MetricBalanceEvent event);
+
+}

balancer/src/main/java/timely/balancer/resolver/eventing/MetricBalanceEvent.java

@@ -0,0 +1,33 @@
+package timely.balancer.resolver.eventing;
+
+import timely.balancer.resolver.BalancedMetricResolver;
+
+public class MetricBalanceEvent {
+
+    public enum ProgressType {
+        BEGIN, BALANCER_LOCK_ACQUIRED, END
+    }
+
+    private final ProgressType progressType;
+    private final BalancedMetricResolver.BalanceType balanceType;
+    private final Long numRessigned;
+
+    public MetricBalanceEvent(ProgressType progressType, BalancedMetricResolver.BalanceType balanceType,
+            long numReassigned) {
+        this.progressType = progressType;
+        this.balanceType = balanceType;
+        this.numRessigned = numReassigned;
+    }
+
+    public BalancedMetricResolver.BalanceType getBalanceType() {
+        return balanceType;
+    }
+
+    public ProgressType getProgressType() {
+        return progressType;
+    }
+
+    public Long getNumRessigned() {
+        return numRessigned;
+    }
+}

balancer/src/main/java/timely/balancer/resolver/eventing/MetricHostCallback.java

@@ -0,0 +1,7 @@
+package timely.balancer.resolver.eventing;
+
+public abstract class MetricHostCallback {
+
+    public abstract void onEvent(MetricHostEvent event);
+
+}

balancer/src/main/java/timely/balancer/resolver/eventing/MetricHostEvent.java

@@ -0,0 +1,26 @@
+package timely.balancer.resolver.eventing;
+
+import timely.balancer.connection.TimelyBalancedHost;
+
+public class MetricHostEvent {
+
+    public enum ActionType {
+        ADDED, REMOVED, SUCCESS, FAILURE
+    }
+
+    private final TimelyBalancedHost timelyBalancedHost;
+    private final ActionType actionType;
+
+    public MetricHostEvent(TimelyBalancedHost timelyBalancedHost, ActionType actionType) {
+        this.timelyBalancedHost = timelyBalancedHost;
+        this.actionType = actionType;
+    }
+
+    public TimelyBalancedHost getTimelyBalancedHost() {
+        return timelyBalancedHost;
+    }
+
+    public ActionType getActionType() {
+        return actionType;
+    }
+}

balancer/src/main/spotbugs/excludes.xml

@@ -10,4 +10,9 @@
     <Method name="getBuf" />
     <Bug pattern="EI_EXPOSE_REP" />
   </Match>
+  <Match>
+    <Class name="timely.balancer.resolver.MetricAssignmentPerister" />
+    <Method name="createAssignmentFile" />
+    <Bug pattern="RV_RETURN_VALUE_IGNORED_BAD_PRACTICE" />
+  </Match>
 </FindBugsFilter>

balancer/src/test/java/timely/balancer/resolver/ServiceRegistry.java

@@ -0,0 +1,76 @@
+package timely.balancer.resolver;
+
+import static timely.Server.SERVICE_DISCOVERY_PATH;
+
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.x.discovery.ServiceDiscovery;
+import org.apache.curator.x.discovery.ServiceDiscoveryBuilder;
+import org.apache.curator.x.discovery.ServiceInstance;
+import org.apache.curator.x.discovery.ServiceInstanceBuilder;
+import org.apache.zookeeper.data.Stat;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import timely.ServerDetails;
+import timely.balancer.connection.TimelyBalancedHost;
+
+public class ServiceRegistry {
+
+    private static final Logger LOG = LoggerFactory.getLogger(ServiceRegistry.class);
+
+    public static void registerService(CuratorFramework curatorFramework, TimelyBalancedHost tbh) {
+        try {
+            try {
+                Stat stat = curatorFramework.checkExists().forPath(SERVICE_DISCOVERY_PATH);
+                if (stat == null) {
+                    curatorFramework.create().creatingParentContainersIfNeeded().forPath(SERVICE_DISCOVERY_PATH);
+                }
+            } catch (Exception e) {
+                LOG.error(e.getMessage());
+            }
+
+            ServiceInstance<ServerDetails> serviceInstance = serviceInstance(tbh);
+            ServiceDiscovery<ServerDetails> discovery = ServiceDiscoveryBuilder.builder(ServerDetails.class)
+                    .client(curatorFramework).basePath(SERVICE_DISCOVERY_PATH).build();
+            discovery.start();
+            discovery.registerService(serviceInstance);
+        } catch (Exception e) {
+            LOG.error(e.getMessage());
+        }
+    }
+
+    public static void unregisterService(CuratorFramework curatorFramework, TimelyBalancedHost tbh) {
+        try {
+            try {
+                Stat stat = curatorFramework.checkExists().forPath(SERVICE_DISCOVERY_PATH);
+                if (stat == null) {
+                    curatorFramework.create().creatingParentContainersIfNeeded().forPath(SERVICE_DISCOVERY_PATH);
+                }
+            } catch (Exception e) {
+                LOG.error(e.getMessage());
+            }
+
+            ServiceInstance<ServerDetails> serviceInstance = serviceInstance(tbh);
+            ServiceDiscovery<ServerDetails> discovery = ServiceDiscoveryBuilder.builder(ServerDetails.class)
+                    .client(curatorFramework).basePath(SERVICE_DISCOVERY_PATH).build();
+            discovery.start();
+            discovery.unregisterService(serviceInstance);
+        } catch (Exception e) {
+            LOG.error(e.getMessage());
+        }
+    }
+
+    private static ServiceInstance<ServerDetails> serviceInstance(TimelyBalancedHost config) throws Exception {
+        ServerDetails serverDetails = new ServerDetails();
+        serverDetails.setHost(config.getHost());
+        serverDetails.setTcpPort(config.getTcpPort());
+        serverDetails.setHttpPort(config.getHttpPort());
+        serverDetails.setWsPort(config.getWsPort());
+        serverDetails.setUdpPort(config.getUdpPort());
+
+        ServiceInstanceBuilder<ServerDetails> builder = ServiceInstance.builder();
+        String serviceName = config.getHost() + ":" + config.getTcpPort();
+        ServiceInstance<ServerDetails> serviceInstance = builder.id(serviceName).name("timely-server")
+                .address(config.getHost()).port(config.getTcpPort()).payload(serverDetails).build();
+        return serviceInstance;
+    }
+}

balancer/src/test/java/timely/balancer/resolver/TestBalancedMetricResolver.java

@@ -0,0 +1,270 @@
+package timely.balancer.resolver;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Random;
+import java.util.UUID;
+
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.CuratorFrameworkFactory;
+import org.apache.curator.retry.RetryNTimes;
+import org.apache.zookeeper.server.ServerConfig;
+import org.apache.zookeeper.server.ZooKeeperServerMain;
+import org.apache.zookeeper.server.quorum.QuorumPeerConfig;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.util.SocketUtils;
+import timely.balancer.configuration.BalancerConfiguration;
+import timely.balancer.connection.TimelyBalancedHost;
+import timely.balancer.resolver.eventing.MetricAssignedCallback;
+import timely.balancer.resolver.eventing.MetricAssignedEvent;
+import timely.balancer.resolver.eventing.MetricBalanceCallback;
+import timely.balancer.resolver.eventing.MetricBalanceEvent;
+import timely.balancer.resolver.eventing.MetricHostCallback;
+import timely.balancer.resolver.eventing.MetricHostEvent;
+
+public class TestBalancedMetricResolver {
+
+    private final Logger LOG = LoggerFactory.getLogger(getClass());
+    private CuratorFramework curatorFramework;
+    private SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd HHmmss.SSS");
+    private Random r = new Random(System.currentTimeMillis());
+    private List<TimelyBalancedHost> timelyHosts;
+    private Map<MetricAssignedEvent, Integer> metricBalanceEventCounter = new HashMap<>();
+
+    @Before
+    public void startup() {
+        QuorumPeerConfig currentConfig = createZookeeper();
+        InetSocketAddress address = currentConfig.getClientPortAddress();
+        String zkServer = address.getHostName() + ":" + address.getPort();
+        curatorFramework = CuratorFrameworkFactory.newClient(zkServer, 1000, 5000, new RetryNTimes(2, 50));
+        curatorFramework.start();
+        timelyHosts = new ArrayList<>();
+    }
+
+    @After
+    public void shutdown() {
+        curatorFramework.close();
+    }
+
+    private QuorumPeerConfig createZookeeper() {
+        Properties startupProperties = new Properties();
+        File file = new File(System.getProperty("java.io.tmpdir") + File.separator + UUID.randomUUID());
+        file.deleteOnExit();
+        startupProperties.setProperty("dataDir", file.getAbsolutePath());
+        startupProperties.setProperty("clientPort", String.valueOf(SocketUtils.findAvailableTcpPort(3000, 4000)));
+        QuorumPeerConfig quorumConfiguration = new QuorumPeerConfig();
+
+        try {
+            quorumConfiguration.parseProperties(startupProperties);
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+
+        ZooKeeperServerMain zooKeeperServer = new ZooKeeperServerMain();
+        final ServerConfig configuration = new ServerConfig();
+        configuration.readFrom(quorumConfiguration);
+
+        new Thread() {
+
+            public void run() {
+                try {
+                    zooKeeperServer.runFromConfig(configuration);
+                } catch (IOException e) {
+                    LOG.error("ZooKeeper Failed", e);
+                }
+            }
+        }.start();
+
+        return quorumConfiguration;
+    }
+
+    private BalancerConfiguration getBalancerConfiguration() {
+        BalancerConfiguration balancerConfiguration = new BalancerConfiguration();
+        balancerConfiguration.setMetricAssignmentPersisterType("FILE");
+        File file = new File(System.getProperty("java.io.tmpdir") + File.separator + UUID.randomUUID());
+        file.deleteOnExit();
+        balancerConfiguration.setAssignmentFile(file.getAbsolutePath());
+        balancerConfiguration.setBalanceDelay(0);
+        balancerConfiguration.setBalancePeriod(1000);
+        balancerConfiguration.setPersistDelay(0);
+        balancerConfiguration.setPersistDelay(1000);
+        balancerConfiguration.setCheckServerHealthInterval(10);
+        return balancerConfiguration;
+    }
+
+    private void createTimelyHosts(BalancerConfiguration balancerConfiguration, int numHosts) {
+        for (int port = 1000, x = 0; x < numHosts; x++, port += 10) {
+            TimelyBalancedHost tbh = TimelyBalancedHost.of("localhost", port);
+            tbh.setBalancerConfig(balancerConfiguration);
+            timelyHosts.add(tbh);
+            ServiceRegistry.registerService(curatorFramework, tbh);
+        }
+    }
+
+    private void registerCallbacks(BalancedMetricResolver resolver) {
+        resolver.registerCallback(new MetricAssignedCallback() {
+
+            @Override
+            public void onEvent(MetricAssignedEvent event) {
+                LOG.debug(String.format("%s metric:%s lose:%s:%d gain:%s:%d reason:%s",
+                        event.getClass().getSimpleName(), event.getMetric(),
+                        event.getLosingHost() == null ? null : event.getLosingHost().getHost(),
+                        event.getLosingHost() == null ? null : event.getLosingHost().getTcpPort(),
+                        event.getGainingHost().getHost(), event.getGainingHost().getTcpPort(), event.getReason()));
+            }
+        });
+
+        resolver.registerCallback(new MetricHostCallback() {
+
+            @Override
+            public void onEvent(MetricHostEvent event) {
+                LOG.debug(String.format("%s host:%s:%d type:%s", event.getClass().getSimpleName(),
+                        event.getTimelyBalancedHost().getHost(), event.getTimelyBalancedHost().getTcpPort(),
+                        event.getActionType()));
+            }
+        });
+
+        resolver.registerCallback(new MetricBalanceCallback() {
+
+            @Override
+            public void onEvent(MetricBalanceEvent event) {
+                LOG.debug(String.format("%s type:%s progress:%s reassigned:%d", event.getClass().getSimpleName(),
+                        event.getBalanceType(), event.getProgressType(), event.getNumRessigned()));
+            }
+        });
+    }
+
+    private TimelyBalancedHost randomServerDown(TestHealthChecker healthChecker, int numHosts) {
+        TimelyBalancedHost tbh = timelyHosts.get(r.nextInt(numHosts));
+        LOG.debug("Marking server down:" + TimelyBalancedHost.toStringShort(tbh));
+        healthChecker.serverDown(tbh);
+        return tbh;
+    }
+
+    private TimelyBalancedHost randomServerToggle(TestHealthChecker healthChecker, int numHosts) {
+        TimelyBalancedHost tbh = timelyHosts.get(r.nextInt(numHosts));
+        LOG.debug("Marking server " + (tbh.isUp() ? "down:" : "up:") + TimelyBalancedHost.toStringShort(tbh));
+        healthChecker.serverToggle(tbh);
+        return tbh;
+    }
+
+    private TimelyBalancedHost randomServerUnregister(int numHosts) {
+        TimelyBalancedHost tbh = timelyHosts.get(r.nextInt(numHosts));
+        LOG.debug("Unregistering:" + TimelyBalancedHost.toStringShort(tbh));
+        ServiceRegistry.unregisterService(curatorFramework, tbh);
+        return tbh;
+    }
+
+    @Test
+    public void test1() {
+        LOG.debug("Configure test");
+        int numHosts = 10;
+        BalancerConfiguration balancerConfiguration = getBalancerConfiguration();
+        createTimelyHosts(balancerConfiguration, numHosts);
+        TestHealthChecker healthChecker = new TestHealthChecker(balancerConfiguration, timelyHosts);
+        BalancedMetricResolver balancedMetricResolver = new BalancedMetricResolver(curatorFramework,
+                balancerConfiguration, healthChecker);
+        registerCallbacks(balancedMetricResolver);
+        LOG.debug("Start balancedMetricResolver");
+        balancedMetricResolver.start();
+        long now = System.currentTimeMillis();
+        int numMetrics = 100;
+        for (int x = 0; x < numMetrics; x++) {
+            balancedMetricResolver.getHostPortKeyIngest(String.format("metric%03d", x));
+        }
+
+        LOG.debug("Begin monitoring");
+        long cycle = 0;
+        List<TimelyBalancedHost> serversUnregistered = new ArrayList<>();
+        while (System.currentTimeMillis() < now + 15000) {
+            try {
+                Thread.sleep(100);
+                for (int x = 0; x < 10000; x++) {
+                    int mNum = r.nextInt(numMetrics);
+                    balancedMetricResolver.getHostPortKeyIngest(String.format("metric%03d", mNum));
+                }
+                // if (cycle % 100 == 0) {
+                // randomServerDown(healthChecker, numHosts);
+                // }
+                // if (cycle % 2 == 0) {
+                // randomServerToggle(healthChecker, numHosts);
+                // }
+                if (cycle % 100 == 0) {
+                    serversUnregistered.add(randomServerUnregister(numHosts));
+                }
+                if (cycle % 500 == 0) {
+                    serversUnregistered.stream().forEach(tbh -> ServiceRegistry.registerService(curatorFramework, tbh));
+                    serversUnregistered.clear();
+                }
+
+            } catch (InterruptedException e) {
+
+            }
+            cycle++;
+        }
+        LOG.debug("Stop balancedMetricResolver");
+        balancedMetricResolver.stop();
+    }
+
+    @Test
+    public void test2() {
+        LOG.debug("Configure test");
+        int numHosts = 10;
+        BalancerConfiguration balancerConfiguration = getBalancerConfiguration();
+        createTimelyHosts(balancerConfiguration, numHosts);
+        TestHealthChecker healthChecker = new TestHealthChecker(balancerConfiguration, timelyHosts);
+        BalancedMetricResolver balancedMetricResolver = new BalancedMetricResolver(curatorFramework,
+                balancerConfiguration, healthChecker);
+        registerCallbacks(balancedMetricResolver);
+        LOG.debug("Start balancedMetricResolver");
+        balancedMetricResolver.start();
+        long now = System.currentTimeMillis();
+        int numMetrics = 100;
+        for (int x = 0; x < numMetrics; x++) {
+            balancedMetricResolver.getHostPortKeyIngest(String.format("metric%03d", x));
+        }
+
+        LOG.debug("Begin monitoring");
+        long cycle = 0;
+        List<TimelyBalancedHost> serversUnregistered = new ArrayList<>();
+        while (System.currentTimeMillis() < now + 15000) {
+            try {
+                Thread.sleep(100);
+                for (int x = 0; x < 10000; x++) {
+                    int mNum = r.nextInt(numMetrics);
+                    balancedMetricResolver.getHostPortKeyIngest(String.format("metric%03d", mNum));
+                }
+                // if (cycle % 100 == 0) {
+                // randomServerDown(healthChecker, numHosts);
+                // }
+                // if (cycle % 2 == 0) {
+                // randomServerToggle(healthChecker, numHosts);
+                // }
+                if (cycle % 100 == 0) {
+                    serversUnregistered.add(randomServerUnregister(numHosts));
+                }
+                if (cycle % 500 == 0) {
+                    serversUnregistered.stream().forEach(tbh -> ServiceRegistry.registerService(curatorFramework, tbh));
+                    serversUnregistered.clear();
+                }
+
+            } catch (InterruptedException e) {
+
+            }
+            cycle++;
+        }
+        LOG.debug("Stop balancedMetricResolver");
+        balancedMetricResolver.stop();
+    }
+}

balancer/src/test/java/timely/balancer/resolver/TestHealthChecker.java

@@ -0,0 +1,89 @@
+package timely.balancer.resolver;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import timely.balancer.configuration.BalancerConfiguration;
+import timely.balancer.connection.TimelyBalancedHost;
+import timely.balancer.healthcheck.HealthChecker;
+import timely.balancer.healthcheck.TcpHealthChecker;
+
+public class TestHealthChecker implements HealthChecker {
+
+    private final Logger LOG = LoggerFactory.getLogger(TcpHealthChecker.class);
+    private List<TimelyBalancedHost> timelyHosts = new ArrayList<>();
+    private Map<TimelyBalancedHost, Boolean> serverUpMap = Collections.synchronizedMap(new HashMap<>());
+    private ScheduledExecutorService executorService = Executors.newScheduledThreadPool(1);
+
+    public TestHealthChecker(BalancerConfiguration config, List<TimelyBalancedHost> timelyHosts) {
+        setTimelyHosts(timelyHosts);
+        this.executorService.scheduleAtFixedRate(() -> {
+            synchronized (timelyHosts) {
+                for (TimelyBalancedHost h : timelyHosts) {
+                    try {
+                        if (TestHealthChecker.this.isServerHealthy(h)) {
+                            h.reportSuccess();
+                        } else {
+                            h.reportFailure();
+                        }
+                    } catch (Exception e) {
+                        LOG.error(e.getMessage(), e);
+                        h.reportFailure();
+                    }
+                }
+            }
+        }, 0, config.getCheckServerHealthInterval(), TimeUnit.MILLISECONDS);
+    }
+
+    @Override
+    public void setTimelyHosts(List<TimelyBalancedHost> timelyHosts) {
+        synchronized (timelyHosts) {
+            this.timelyHosts.clear();
+            this.timelyHosts.addAll(timelyHosts);
+            for (TimelyBalancedHost h : timelyHosts) {
+                serverUpMap.put(h, true);
+            }
+        }
+
+    }
+
+    public boolean isServerHealthy(TimelyBalancedHost h) {
+        return serverUpMap.get(h);
+    }
+
+    public void serverUp(TimelyBalancedHost h) {
+        serverUpMap.put(h, true);
+    }
+
+    public void serverDown(TimelyBalancedHost h) {
+        serverUpMap.put(h, false);
+    }
+
+    public void serverToggle(TimelyBalancedHost h) {
+        synchronized (serverUpMap) {
+            serverUpMap.put(h, !serverUpMap.get(h));
+        }
+    }
+
+    @Override
+    public void close() throws Exception {
+        this.executorService.shutdown();
+        try {
+            this.executorService.awaitTermination(5, TimeUnit.SECONDS);
+        } catch (InterruptedException e) {
+
+        } finally {
+            if (!this.executorService.isTerminated()) {
+                this.executorService.shutdownNow();
+            }
+        }
+    }
+}

balancer/src/test/resources/log4j2.xml

@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Configuration status="WARN">
+  <Appenders>
+    <Console name="CONSOLE" target="SYSTEM_OUT">
+      <PatternLayout pattern="%d{yyyyMMdd HHmmss.SSS} [%t] %-5level %logger{36} - %msg%n"/>
+    </Console>
+  </Appenders>
+  <Loggers>
+    <Logger name="org.apache.accumulo.core.client.impl.ThriftTransportPool" level="OFF" additivity="false" />
+    <Logger name="org.apache.accumulo.fate.zookeeper.ZooCache" level="OFF" additivity="false" />
+    <Logger name="org.apache.zookeeper" level="OFF" additivity="false" />
+    <Logger name="timely.balancer.resolver" level="TRACE" additivity="true" />
+    <Logger name="timely.balancer.resolver.TestBalancedMetricResolver" level="DEBUG" additivity="true" />
+    <Root level="ERROR">
+      <AppenderRef ref="CONSOLE"/>
+    </Root>
+  </Loggers>
+</Configuration>

client/pom.xml

@@ -4,13 +4,13 @@
   <parent>
     <groupId>timely</groupId>
     <artifactId>timely</artifactId>
-    <version>1.2.5-SNAPSHOT</version>
+    <version>1.2.8-SNAPSHOT</version>
   </parent>
   <artifactId>timely-client</artifactId>
   <properties>
     <!-- PEP 440 compliant version gets substituted into src/main/python/template/setup.py -->
     <!-- use .dev0 instead of -SNAPSHOT -->
-    <python.client.version>1.2.5.dev0</python.client.version>
+    <python.client.version>1.2.8.dev0</python.client.version>
   </properties>
   <dependencies>
     <dependency>

client/src/main/java/timely/client/tcp/TcpClient.java

@@ -1,7 +1,8 @@
 package timely.client.tcp;
 
+import java.io.BufferedWriter;
 import java.io.IOException;
-import java.io.PrintWriter;
+import java.io.OutputStreamWriter;
 import java.net.Socket;
 
 import org.slf4j.Logger;
@@ -14,7 +15,7 @@ public class TcpClient implements AutoCloseable {
     private final String host;
     private final int port;
     private Socket sock = null;
-    private PrintWriter out = null;
+    private BufferedWriter out = null;
     private long connectTime = 0L;
     private long backoff = 2000;
     private int bufferSize = -1;
@@ -59,7 +60,7 @@ public class TcpClient implements AutoCloseable {
         }
     }
 
-    public synchronized void flush() {
+    public synchronized void flush() throws IOException {
         if (null != out) {
             out.flush();
             writesSinceFlush = 0;
@@ -93,12 +94,12 @@ public class TcpClient implements AutoCloseable {
     }
 
     private synchronized int connect() {
-        if (null == sock || !sock.isConnected() || out.checkError()) {
+        if (null == sock || !sock.isConnected()) {
             if (System.currentTimeMillis() > (connectTime + backoff)) {
                 try {
                     connectTime = System.currentTimeMillis();
                     sock = new Socket(host, port);
-                    out = new PrintWriter(sock.getOutputStream(), false);
+                    out = new BufferedWriter(new OutputStreamWriter(sock.getOutputStream()));
                     backoff = 2000;
                     LOG.trace("Connected to Timely at {}:{}", host, port);
                 } catch (IOException e) {

client/src/main/spotbugs/excludes.xml

@@ -5,4 +5,9 @@
     <Method name="connect" />
     <Class name="timely.client.tcp.TcpClient"/>
   </Match>
+  <Match>
+    <!-- Must ignore these everywhere, because of a javac byte code generation bug -->
+    <!-- https://github.com/spotbugs/spotbugs/issues/756 -->
+    <Bug pattern="RCN_REDUNDANT_NULLCHECK_WOULD_HAVE_BEEN_A_NPE" />
+  </Match>
 </FindBugsFilter>

collectd-nsq-plugin/pom.xml

@@ -4,7 +4,7 @@
   <parent>
     <groupId>timely</groupId>
     <artifactId>timely</artifactId>
-    <version>1.2.5-SNAPSHOT</version>
+    <version>1.2.8-SNAPSHOT</version>
   </parent>
   <artifactId>collectd-nsq-plugin</artifactId>
   <dependencies>

collectd-plugin-parent/pom.xml

@@ -4,7 +4,7 @@
   <parent>
     <groupId>timely</groupId>
     <artifactId>timely</artifactId>
-    <version>1.2.5-SNAPSHOT</version>
+    <version>1.2.8-SNAPSHOT</version>
   </parent>
   <artifactId>collectd-plugin-parent</artifactId>
   <dependencies>

collectd-timely-plugin/pom.xml

@@ -4,7 +4,7 @@
   <parent>
     <groupId>timely</groupId>
     <artifactId>timely</artifactId>
-    <version>1.2.5-SNAPSHOT</version>
+    <version>1.2.8-SNAPSHOT</version>
   </parent>
   <artifactId>collectd-timely-plugin</artifactId>
   <dependencies>

common/pom.xml

@@ -4,7 +4,7 @@
   <parent>
     <groupId>timely</groupId>
     <artifactId>timely</artifactId>
-    <version>1.2.5-SNAPSHOT</version>
+    <version>1.2.8-SNAPSHOT</version>
   </parent>
   <artifactId>timely-common</artifactId>
   <dependencies>

grafana-auth/pom.xml

@@ -4,7 +4,7 @@
   <parent>
     <groupId>timely</groupId>
     <artifactId>timely</artifactId>
-    <version>1.2.5-SNAPSHOT</version>
+    <version>1.2.8-SNAPSHOT</version>
   </parent>
   <artifactId>grafana-auth</artifactId>
   <dependencies>

grafana-auth/src/main/java/timely/grafana/auth/GrafanaAuth.java

@@ -81,8 +81,11 @@ public class GrafanaAuth {
         this.grafanaAuthConfig = grafanaAuthConfig;
     }
 
-    protected static ConfigurableApplicationContext initializeConfiguration(String[] args) {
-        return new SpringApplicationBuilder(SpringBootstrap.class).web(WebApplicationType.NONE).run(args);
+    protected static ConfigurableApplicationContext initializeContext(String[] args) {
+        SpringApplicationBuilder builder = new SpringApplicationBuilder(SpringBootstrap.class);
+        builder.web(WebApplicationType.NONE);
+        builder.registerShutdownHook(false);
+        return builder.run(args);
     }
 
     private void shutdownHook() {
@@ -109,13 +112,11 @@ public class GrafanaAuth {
             try {
                 f.get();
             } catch (final Exception e) {
-                LOG.error("Error while shutting down channel: {}", f.channel().config());
-                LOG.error("{}", e.getMessage());
-                e.printStackTrace();
+                LOG.error("Channel:" + f.channel().config() + " -> " + e.getMessage(), e);
             }
         });
 
-        Integer quietPeriod = grafanaAuthConfig.getShutdownQuietPeriod();
+        int quietPeriod = grafanaAuthConfig.getShutdownQuietPeriod();
         List<Future<?>> groupFutures = new ArrayList<>();
 
         if (httpBossGroup != null) {
@@ -132,9 +133,7 @@ public class GrafanaAuth {
             try {
                 f.get();
             } catch (final Exception e) {
-                LOG.error("Error while shutting down group: {}", f.toString());
-                LOG.error("{}", e.getMessage());
-                e.printStackTrace();
+                LOG.error("Group:" + f.toString() + " -> " + e.getMessage(), e);
             }
         });
 
@@ -272,12 +271,13 @@ public class GrafanaAuth {
 
     public static void main(String[] args) throws Exception {
 
-        GrafanaAuth.applicationContext = GrafanaAuth.initializeConfiguration(args);
-        GrafanaAuthConfiguration balancerConf = applicationContext.getBean(GrafanaAuthConfiguration.class);
+        GrafanaAuth.applicationContext = GrafanaAuth.initializeContext(args);
+        GrafanaAuthConfiguration grafanaAuthConf = GrafanaAuth.applicationContext
+                .getBean(GrafanaAuthConfiguration.class);
 
-        GrafanaAuth b = new GrafanaAuth(balancerConf);
+        GrafanaAuth grafanaAuth = new GrafanaAuth(grafanaAuthConf);
         try {
-            b.run();
+            grafanaAuth.run();
             LATCH.await();
         } catch (final InterruptedException e) {
             LOG.info("Server shutting down.");
@@ -285,7 +285,7 @@ public class GrafanaAuth {
             LOG.error("Error running server.", e);
         } finally {
             try {
-                b.shutdown();
+                grafanaAuth.shutdown();
             } catch (Exception e) {
                 System.exit(1);
             }
@@ -314,7 +314,8 @@ public class GrafanaAuth {
                 ch.pipeline().addLast("fileServer", new HttpStaticFileServerHandler());
                 ch.pipeline().addLast("login", new X509LoginRequestHandler(config.getSecurity(), config.getHttp()));
                 ch.pipeline().addLast("httpRelay", new GrafanaRelayHandler(config, httpClientPool));
-                ch.pipeline().addLast("error", new TimelyExceptionHandler());
+                ch.pipeline().addLast("error", new TimelyExceptionHandler()
+                        .setIgnoreSslHandshakeErrors(config.getSecurity().getServerSsl().isUseGeneratedKeypair()));
             }
         };
     }

grafana-auth/src/main/java/timely/grafana/auth/netty/http/GrafanaRelayHandler.java

@@ -87,7 +87,7 @@ public class GrafanaRelayHandler extends SimpleChannelInboundHandler<HttpRequest
                 String protocol = config.getGrafana().getProtocol();
                 String host = config.getGrafana().getHost();
                 Integer port = config.getGrafana().getPort();
-                k = TimelyBalancedHost.of(host, -1, port, -1, -1);
+                k = TimelyBalancedHost.of(host, -1, port, -1, -1, null);
                 client = httpClientPool.borrowObject(k);
                 Multimap<String, String> headers = HttpHeaderUtils.toMultimap(request.headers());
 

grafana/pom.xml

@@ -4,7 +4,7 @@
   <parent>
     <groupId>timely</groupId>
     <artifactId>timely</artifactId>
-    <version>1.2.5-SNAPSHOT</version>
+    <version>1.2.8-SNAPSHOT</version>
   </parent>
   <artifactId>timely-app</artifactId>
   <packaging>pom</packaging>

grafana/timely-app/dist/datasource/datasource.js

@@ -509,7 +509,7 @@ System.register(['lodash', 'angular', '../../../app/core/utils/datemath'], funct
                 } else {
                   return target.metric === metricData.metric && _.every(target.tags, function (tagV, tagK) {
                     interpolatedTagValue = this.templateSrv.replace(tagV, options.scopedVars, 'pipe');
-                    return metricData.tags[tagK] === interpolatedTagValue || interpolatedTagValue === "*";
+                    return metricData.tags[tagK] === interpolatedTagValue || interpolatedTagValue === ".*";
                   }.bind(this));
                 }
               }.bind(this));

grafana/timely-app/dist/datasource/datasource.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/datasource/datasource.js"],"names":["_","angular","dateMath","TimelyDatasource","instanceSettings","$q","backendSrv","templateSrv","contextSrv","type","name","jsonData","httpsPort","timelyHost","wsPort","basicAuths","url","_makeHttpsUrl","withCredentials","basicAuth","supportMetrics","tagKeys","aggregatorsPromise","filterTypesPromise","options","start","convertToTSDBTime","rangeRaw","from","end","to","qs","each","targets","target","metric","push","convertTargetToQuery","bind","queries","compact","isEmpty","d","defer","resolve","data","promise","groupByTags","query","filters","length","val","tagk","tags","key","performTimeSeriesQuery","then","response","metricToTargetMapping","mapMetricsToTargets","result","map","metricData","index","_saveTagKeys","transformMetricData","datasourceRequest","method","status","message","title","eventList","aggregator","annotation","results","annotationObject","annotations","isGlobal","globalAnnotations","event","description","time","Math","floor","startTime","text","notes","when","interpolated","replace","err","reject","responseTransform","value","metrics_regex","tag_names_regex","tag_values_regex","tag_names_suggest_regex","tag_values_suggest_regex","metrics_query","match","_performSuggestQuery","tag_names_query","_performMetricKeyLookup","tag_values_query","_performMetricKeyValueLookup","tag_names_suggest_query","tag_values_suggest_query","msResolution","responsePromises","reqBody","showQuery","_addCredentialOptions","headers","Promise","all","queryResponses","combinedResponse","x","concat","Object","keys","aggregateTags","tag","_get","q","max","keysArray","split","trim","keysQuery","splice","join","m","limit","tagvs","r","indexOf","tagks","tagv","relativeUrl","params","isArray","sort","md","metricLabel","createMetricLabel","dps","v","k","datapoints","alias","scopedVars","clone","label","tagData","toPairs","has","hide","shouldComputeRate","rate","rateOptions","counter","isCounter","counterMax","parseInt","counterResetValue","resetValue","disableDownsampling","interval","downsampleInterval","parseFloat","downsample","downsampleAggregator","downsampleFillPolicy","copy","filter_key","filter","tag_key","metrics","interpolatedTagValue","findIndex","every","tagV","tagK","date","roundUp","parse","valueOf"],"mappings":";;;;;;;;;;;;;;;AAAOA,O;;AACAC,a;;AACKC,c;;;;;;;;;;;;;;;;;;;;;kCAECC,gB;;AAEX;AACA,kCAAYC,gBAAZ,EAA8BC,EAA9B,EAAkCC,UAAlC,EAA8CC,WAA9C,EAA2DC,UAA3D,EAAuE;AAAA;;AACrE,eAAKC,IAAL,GAAYL,iBAAiBK,IAA7B;AACA,eAAKC,IAAL,GAAYN,iBAAiBM,IAA7B;;AAEA,eAAKC,QAAL,GAAgBP,iBAAiBO,QAAjB,IAA6B,EAA7C;AACA,eAAKC,SAAL,GAAiBR,iBAAiBO,QAAjB,CAA0BC,SAA3C;AACA,eAAKC,UAAL,GAAkBT,iBAAiBO,QAAjB,CAA0BE,UAA5C;AACA,eAAKC,MAAL,GAAcV,iBAAiBO,QAAjB,CAA0BG,MAAxC;AACA,cAAI,CAACV,iBAAiBO,QAAjB,CAA0BI,UAA/B,EAA0C;AACxCX,6BAAiBO,QAAjB,CAA0BI,UAA1B,GAAuC,KAAvC;AACD;;AAED,eAAKC,GAAL,GAAW,KAAKC,aAAL,EAAX;AACA,eAAKC,eAAL,GAAuB,IAAvB;AACA,eAAKC,SAAL,GAAiB,KAAjB;;AAEA,eAAKC,cAAL,GAAsB,IAAtB;AACA,eAAKC,OAAL,GAAe,EAAf;;AAEA,eAAKhB,EAAL,GAAUA,EAAV;AACA,eAAKC,UAAL,GAAkBA,UAAlB;AACA,eAAKC,WAAL,GAAmBA,WAAnB;AACA,eAAKC,UAAL,GAAkBA,UAAlB;;AAEA,eAAKc,kBAAL,GAA0B,IAA1B;AACA,eAAKC,kBAAL,GAA0B,IAA1B;AAED;;AAED;;;;;gCACMC,O,EAAQ;AACZ,gBAAIC,QAAQ,KAAKC,iBAAL,CAAuBF,QAAQG,QAAR,CAAiBC,IAAxC,EAA8C,KAA9C,CAAZ;AACA,gBAAIC,MAAM,KAAKH,iBAAL,CAAuBF,QAAQG,QAAR,CAAiBG,EAAxC,EAA4C,IAA5C,CAAV;AACA,gBAAIC,KAAK,EAAT;;AAEA/B,cAAEgC,IAAF,CAAOR,QAAQS,OAAf,EAAwB,UAASC,MAAT,EAAiB;AACvC,kBAAI,CAACA,OAAOC,MAAZ,EAAoB;AAAE;AAAS;AAC/BJ,iBAAGK,IAAH,CAAQ,KAAKC,oBAAL,CAA0BH,MAA1B,EAAkCV,OAAlC,CAAR;AACD,aAHuB,CAGtBc,IAHsB,CAGjB,IAHiB,CAAxB;;AAKA,gBAAIC,UAAUvC,EAAEwC,OAAF,CAAUT,EAAV,CAAd;;AAEA;AACA,gBAAI/B,EAAEyC,OAAF,CAAUF,OAAV,CAAJ,EAAwB;AACtB,kBAAIG,IAAI,KAAKrC,EAAL,CAAQsC,KAAR,EAAR;AACAD,gBAAEE,OAAF,CAAU,EAAEC,MAAM,EAAR,EAAV;AACA,qBAAOH,EAAEI,OAAT;AACD;;AAED,gBAAIC,cAAc,EAAlB;AACA/C,cAAEgC,IAAF,CAAOO,OAAP,EAAgB,UAASS,KAAT,EAAgB;AAC9B,kBAAIA,MAAMC,OAAN,IAAiBD,MAAMC,OAAN,CAAcC,MAAd,GAAuB,CAA5C,EAA+C;AAC7ClD,kBAAEgC,IAAF,CAAOgB,MAAMC,OAAb,EAAsB,UAASE,GAAT,EAAc;AAClCJ,8BAAYI,IAAIC,IAAhB,IAAwB,IAAxB;AACD,iBAFD;AAGD,eAJD,MAIO;AACLpD,kBAAEgC,IAAF,CAAOgB,MAAMK,IAAb,EAAmB,UAASF,GAAT,EAAcG,GAAd,EAAmB;AACpCP,8BAAYO,GAAZ,IAAmB,IAAnB;AACD,iBAFD;AAGD;AACF,aAVD;;AAYA,mBAAO,KAAKC,sBAAL,CAA4BhB,OAA5B,EAAqCd,KAArC,EAA4CI,GAA5C,EAAiD2B,IAAjD,CAAsD,UAASC,QAAT,EAAmB;AAC9E,kBAAIC,wBAAwB,KAAKC,mBAAL,CAAyBF,SAASZ,IAAlC,EAAwCrB,OAAxC,CAA5B;AACA,kBAAIoC,SAAS5D,EAAE6D,GAAF,CAAMJ,SAASZ,IAAf,EAAqB,UAASiB,UAAT,EAAqBC,KAArB,EAA4B;AAC5DA,wBAAQL,sBAAsBK,KAAtB,CAAR;AACA,oBAAIA,UAAU,CAAC,CAAf,EAAkB;AAChBA,0BAAQ,CAAR;AACD;AACD,qBAAKC,YAAL,CAAkBF,UAAlB;;AAEA,uBAAO,KAAKG,mBAAL,CAAyBH,UAAzB,EAAqCf,WAArC,EAAkDvB,QAAQS,OAAR,CAAgB8B,KAAhB,CAAlD,EAA0EvC,OAA1E,CAAP;AACD,eARiC,CAQhCc,IARgC,CAQ3B,IAR2B,CAArB,CAAb;AASA,qBAAO,EAAEO,MAAMe,MAAR,EAAP;AACD,aAZ4D,CAY3DtB,IAZ2D,CAYtD,IAZsD,CAAtD,CAAP;AAaD;;;2CAGe;AACd,mBAAO,KAAKhC,UAAL,CAAgB4D,iBAAhB,CAAkC;AACvClD,mBAAK,KAAKA,GAAL,GAAW,UADuB;AAEvCmD,sBAAQ;AAF+B,aAAlC,EAGJX,IAHI,CAGC,oBAAY;AAClB,kBAAIC,SAASW,MAAT,KAAoB,GLine truncated
+{"version":3,"sources":["../../src/datasource/datasource.js"],"names":["_","angular","dateMath","TimelyDatasource","instanceSettings","$q","backendSrv","templateSrv","contextSrv","type","name","jsonData","httpsPort","timelyHost","wsPort","basicAuths","url","_makeHttpsUrl","withCredentials","basicAuth","supportMetrics","tagKeys","aggregatorsPromise","filterTypesPromise","options","start","convertToTSDBTime","rangeRaw","from","end","to","qs","each","targets","target","metric","push","convertTargetToQuery","bind","queries","compact","isEmpty","d","defer","resolve","data","promise","groupByTags","query","filters","length","val","tagk","tags","key","performTimeSeriesQuery","then","response","metricToTargetMapping","mapMetricsToTargets","result","map","metricData","index","_saveTagKeys","transformMetricData","datasourceRequest","method","status","message","title","eventList","aggregator","annotation","results","annotationObject","annotations","isGlobal","globalAnnotations","event","description","time","Math","floor","startTime","text","notes","when","interpolated","replace","err","reject","responseTransform","value","metrics_regex","tag_names_regex","tag_values_regex","tag_names_suggest_regex","tag_values_suggest_regex","metrics_query","match","_performSuggestQuery","tag_names_query","_performMetricKeyLookup","tag_values_query","_performMetricKeyValueLookup","tag_names_suggest_query","tag_values_suggest_query","msResolution","responsePromises","reqBody","showQuery","_addCredentialOptions","headers","Promise","all","queryResponses","combinedResponse","x","concat","Object","keys","aggregateTags","tag","_get","q","max","keysArray","split","trim","keysQuery","splice","join","m","limit","tagvs","r","indexOf","tagks","tagv","relativeUrl","params","isArray","sort","md","metricLabel","createMetricLabel","dps","v","k","datapoints","alias","scopedVars","clone","label","tagData","toPairs","has","hide","shouldComputeRate","rate","rateOptions","counter","isCounter","counterMax","parseInt","counterResetValue","resetValue","disableDownsampling","interval","downsampleInterval","parseFloat","downsample","downsampleAggregator","downsampleFillPolicy","copy","filter_key","filter","tag_key","metrics","interpolatedTagValue","findIndex","every","tagV","tagK","date","roundUp","parse","valueOf"],"mappings":";;;;;;;;;;;;;;;AAAOA,O;;AACAC,a;;AACKC,c;;;;;;;;;;;;;;;;;;;;;kCAECC,gB;;AAEX;AACA,kCAAYC,gBAAZ,EAA8BC,EAA9B,EAAkCC,UAAlC,EAA8CC,WAA9C,EAA2DC,UAA3D,EAAuE;AAAA;;AACrE,eAAKC,IAAL,GAAYL,iBAAiBK,IAA7B;AACA,eAAKC,IAAL,GAAYN,iBAAiBM,IAA7B;;AAEA,eAAKC,QAAL,GAAgBP,iBAAiBO,QAAjB,IAA6B,EAA7C;AACA,eAAKC,SAAL,GAAiBR,iBAAiBO,QAAjB,CAA0BC,SAA3C;AACA,eAAKC,UAAL,GAAkBT,iBAAiBO,QAAjB,CAA0BE,UAA5C;AACA,eAAKC,MAAL,GAAcV,iBAAiBO,QAAjB,CAA0BG,MAAxC;AACA,cAAI,CAACV,iBAAiBO,QAAjB,CAA0BI,UAA/B,EAA0C;AACxCX,6BAAiBO,QAAjB,CAA0BI,UAA1B,GAAuC,KAAvC;AACD;;AAED,eAAKC,GAAL,GAAW,KAAKC,aAAL,EAAX;AACA,eAAKC,eAAL,GAAuB,IAAvB;AACA,eAAKC,SAAL,GAAiB,KAAjB;;AAEA,eAAKC,cAAL,GAAsB,IAAtB;AACA,eAAKC,OAAL,GAAe,EAAf;;AAEA,eAAKhB,EAAL,GAAUA,EAAV;AACA,eAAKC,UAAL,GAAkBA,UAAlB;AACA,eAAKC,WAAL,GAAmBA,WAAnB;AACA,eAAKC,UAAL,GAAkBA,UAAlB;;AAEA,eAAKc,kBAAL,GAA0B,IAA1B;AACA,eAAKC,kBAAL,GAA0B,IAA1B;AAED;;AAED;;;;;gCACMC,O,EAAQ;AACZ,gBAAIC,QAAQ,KAAKC,iBAAL,CAAuBF,QAAQG,QAAR,CAAiBC,IAAxC,EAA8C,KAA9C,CAAZ;AACA,gBAAIC,MAAM,KAAKH,iBAAL,CAAuBF,QAAQG,QAAR,CAAiBG,EAAxC,EAA4C,IAA5C,CAAV;AACA,gBAAIC,KAAK,EAAT;;AAEA/B,cAAEgC,IAAF,CAAOR,QAAQS,OAAf,EAAwB,UAASC,MAAT,EAAiB;AACvC,kBAAI,CAACA,OAAOC,MAAZ,EAAoB;AAAE;AAAS;AAC/BJ,iBAAGK,IAAH,CAAQ,KAAKC,oBAAL,CAA0BH,MAA1B,EAAkCV,OAAlC,CAAR;AACD,aAHuB,CAGtBc,IAHsB,CAGjB,IAHiB,CAAxB;;AAKA,gBAAIC,UAAUvC,EAAEwC,OAAF,CAAUT,EAAV,CAAd;;AAEA;AACA,gBAAI/B,EAAEyC,OAAF,CAAUF,OAAV,CAAJ,EAAwB;AACtB,kBAAIG,IAAI,KAAKrC,EAAL,CAAQsC,KAAR,EAAR;AACAD,gBAAEE,OAAF,CAAU,EAAEC,MAAM,EAAR,EAAV;AACA,qBAAOH,EAAEI,OAAT;AACD;;AAED,gBAAIC,cAAc,EAAlB;AACA/C,cAAEgC,IAAF,CAAOO,OAAP,EAAgB,UAASS,KAAT,EAAgB;AAC9B,kBAAIA,MAAMC,OAAN,IAAiBD,MAAMC,OAAN,CAAcC,MAAd,GAAuB,CAA5C,EAA+C;AAC7ClD,kBAAEgC,IAAF,CAAOgB,MAAMC,OAAb,EAAsB,UAASE,GAAT,EAAc;AAClCJ,8BAAYI,IAAIC,IAAhB,IAAwB,IAAxB;AACD,iBAFD;AAGD,eAJD,MAIO;AACLpD,kBAAEgC,IAAF,CAAOgB,MAAMK,IAAb,EAAmB,UAASF,GAAT,EAAcG,GAAd,EAAmB;AACpCP,8BAAYO,GAAZ,IAAmB,IAAnB;AACD,iBAFD;AAGD;AACF,aAVD;;AAYA,mBAAO,KAAKC,sBAAL,CAA4BhB,OAA5B,EAAqCd,KAArC,EAA4CI,GAA5C,EAAiD2B,IAAjD,CAAsD,UAASC,QAAT,EAAmB;AAC9E,kBAAIC,wBAAwB,KAAKC,mBAAL,CAAyBF,SAASZ,IAAlC,EAAwCrB,OAAxC,CAA5B;AACA,kBAAIoC,SAAS5D,EAAE6D,GAAF,CAAMJ,SAASZ,IAAf,EAAqB,UAASiB,UAAT,EAAqBC,KAArB,EAA4B;AAC5DA,wBAAQL,sBAAsBK,KAAtB,CAAR;AACA,oBAAIA,UAAU,CAAC,CAAf,EAAkB;AAChBA,0BAAQ,CAAR;AACD;AACD,qBAAKC,YAAL,CAAkBF,UAAlB;;AAEA,uBAAO,KAAKG,mBAAL,CAAyBH,UAAzB,EAAqCf,WAArC,EAAkDvB,QAAQS,OAAR,CAAgB8B,KAAhB,CAAlD,EAA0EvC,OAA1E,CAAP;AACD,eARiC,CAQhCc,IARgC,CAQ3B,IAR2B,CAArB,CAAb;AASA,qBAAO,EAAEO,MAAMe,MAAR,EAAP;AACD,aAZ4D,CAY3DtB,IAZ2D,CAYtD,IAZsD,CAAtD,CAAP;AAaD;;;2CAGe;AACd,mBAAO,KAAKhC,UAAL,CAAgB4D,iBAAhB,CAAkC;AACvClD,mBAAK,KAAKA,GAAL,GAAW,UADuB;AAEvCmD,sBAAQ;AAF+B,aAAlC,EAGJX,IAHI,CAGC,oBAAY;AAClB,kBAAIC,SAASW,MAAT,KAAoB,GLine truncated

grafana/timely-app/dist/plugin.json

@@ -23,8 +23,8 @@
       {"name": "Project site", "url": "https://github.com/NationalSecurityAgency/timely"},
       {"name": "License & Terms", "url": "https://raw.githubusercontent.com/NationalSecurityAgency/timely/master/LICENSE"}
     ],
-    "version": "1.2.5-SNAPSHOT",
-    "updated": "2019-05-30"
+    "version": "1.2.8-SNAPSHOT",
+    "updated": "2020-02-21"
   },
 
   "includes": [

grafana/timely-app/package.json

@@ -1,7 +1,7 @@
 {
   "name": "timely-app",
   "private": true,
-  "version": "1.2.5-SNAPSHOT",
+  "version": "1.2.8-SNAPSHOT",
   "description": "",
   "main": "index.js",
   "scripts": {

grafana/timely-app/plugin.json

@@ -23,8 +23,8 @@
       {"name": "Project site", "url": "https://github.com/NationalSecurityAgency/timely"},
       {"name": "License & Terms", "url": "https://raw.githubusercontent.com/NationalSecurityAgency/timely/master/LICENSE"}
     ],
-    "version": "1.2.5-SNAPSHOT",
-    "updated": "2019-05-30"
+    "version": "1.2.8-SNAPSHOT",
+    "updated": "2020-02-21"
   },
 
   "includes": [

grafana/timely-app/src/datasource/datasource.js

@@ -446,7 +446,7 @@ export class TimelyDatasource {
         } else {
           return target.metric === metricData.metric && _.every(target.tags, function(tagV, tagK) {
                 interpolatedTagValue = this.templateSrv.replace(tagV, options.scopedVars, 'pipe');
-                return metricData.tags[tagK] === interpolatedTagValue || interpolatedTagValue === "*";
+                return metricData.tags[tagK] === interpolatedTagValue || interpolatedTagValue === ".*";
               }.bind(this));
         }
       }.bind(this));

pom.xml

@@ -3,7 +3,7 @@
   <modelVersion>4.0.0</modelVersion>
   <groupId>timely</groupId>
   <artifactId>timely</artifactId>
-  <version>1.2.5-SNAPSHOT</version>
+  <version>1.2.8-SNAPSHOT</version>
   <packaging>pom</packaging>
   <name>Timely</name>
   <description>A secure and scalable time series database based on Apache Accumulo</description>
@@ -50,37 +50,39 @@
     <version.commons-pool2>2.6.1</version.commons-pool2>
     <version.curator>4.1.0</version.curator>
     <version.disruptor>3.4.2</version.disruptor>
+    <version.easymock>4.0.2</version.easymock>
     <version.eclipse-lifecycle>1.0.0</version.eclipse-lifecycle>
     <version.fast-classpath-scanner>3.1.13</version.fast-classpath-scanner>
     <version.flink>1.1.4</version.flink>
     <version.google-flatbuffers>1.10.0</version.google-flatbuffers>
     <version.guava>27.1-jre</version.guava>
-    <version.hadoop>2.7.3</version.hadoop>
+    <version.hadoop>2.7.7</version.hadoop>
     <version.httpcomponents>4.5.6</version.httpcomponents>
-    <version.impsort>1.2.0</version.impsort>
-    <version.jackson>2.9.9</version.jackson>
-    <version.jacoco>0.7.6.201602180812</version.jacoco>
+    <version.impsort>1.3.0</version.impsort>
+    <version.jackson>2.10.0</version.jackson>
+    <version.jackson.databind>2.10.0</version.jackson.databind>
+    <version.jacoco>0.8.4</version.jacoco>
     <version.javacsv>2.0</version.javacsv>
     <version.javassist>3.24.1-GA</version.javassist>
     <version.jcip-annotations>1.0</version.jcip-annotations>
     <version.jsoup>1.11.3</version.jsoup>
     <version.junit>4.12</version.junit>
     <version.log4j>2.11.2</version.log4j>
-    <version.maven-assembly>2.6</version.maven-assembly>
-    <version.maven-build-helper>1.12</version.maven-build-helper>
-    <version.maven-compiler>3.3</version.maven-compiler>
-    <version.maven-dependency>2.10</version.maven-dependency>
-    <version.maven-jar>3.0.1</version.maven-jar>
-    <version.maven-pmd>3.6</version.maven-pmd>
-    <version.maven-rpm>2.1.4</version.maven-rpm>
-    <version.maven-shade>2.4.2</version.maven-shade>
-    <version.maven-surefire>2.19.1</version.maven-surefire>
-    <version.netty>4.1.35.Final</version.netty>
+    <version.maven-assembly>3.1.1</version.maven-assembly>
+    <version.maven-build-helper>3.0.0</version.maven-build-helper>
+    <version.maven-compiler>3.8.1</version.maven-compiler>
+    <version.maven-dependency>3.1.1</version.maven-dependency>
+    <version.maven-jar>3.1.2</version.maven-jar>
+    <version.maven-pmd>3.12.0</version.maven-pmd>
+    <version.maven-rpm>2.2.0</version.maven-rpm>
+    <version.maven-shade>3.2.1</version.maven-shade>
+    <version.maven-surefire>3.0.0-M3</version.maven-surefire>
+    <version.netty>4.1.42.Final</version.netty>
     <version.netty-tcnative>2.0.25.Final</version.netty-tcnative>
     <version.os-maven>1.5.0.Final</version.os-maven>
     <version.powermock>1.6.4</version.powermock>
     <version.slf4j>1.7.26</version.slf4j>
-    <version.spotbugs-maven>3.1.7</version.spotbugs-maven>
+    <version.spotbugs-maven>3.1.12.1</version.spotbugs-maven>
     <version.spring-boot>2.1.3.RELEASE</version.spring-boot>
     <version.tyrus>1.15</version.tyrus>
   </properties>
@@ -99,7 +101,7 @@
       <dependency>
         <groupId>com.fasterxml.jackson.core</groupId>
         <artifactId>jackson-databind</artifactId>
-        <version>${version.jackson}</version>
+        <version>${version.jackson.databind}</version>
       </dependency>
       <dependency>
         <groupId>com.fasterxml.jackson.datatype</groupId>
@@ -396,6 +398,12 @@
         <version>${version.jcip-annotations}</version>
         <scope>test</scope>
       </dependency>
+      <dependency>
+        <groupId>org.easymock</groupId>
+        <artifactId>easymock</artifactId>
+        <version>${version.easymock}</version>
+        <scope>test</scope>
+      </dependency>
       <dependency>
         <groupId>org.powermock</groupId>
         <artifactId>powermock-easymock-release-full</artifactId>
@@ -411,7 +419,7 @@
         <plugin>
           <groupId>net.revelc.code.formatter</groupId>
           <artifactId>formatter-maven-plugin</artifactId>
-          <version>2.8.0</version>
+          <version>2.10.0</version>
           <configuration>
             <compilerCompliance>${maven.compiler.source}</compilerCompliance>
             <compilerSource>${maven.compiler.source}</compilerSource>
@@ -429,7 +437,7 @@
         <plugin>
           <groupId>com.github.ekryd.sortpom</groupId>
           <artifactId>sortpom-maven-plugin</artifactId>
-          <version>2.8.0</version>
+          <version>2.10.0</version>
           <configuration>
             <predefinedSortOrder>recommended_2008_06</predefinedSortOrder>
             <createBackupFile>false</createBackupFile>
@@ -573,7 +581,7 @@
             <configuration>
               <rules>
                 <requireMavenVersion>
-                  <version>3.0.5</version>
+                  <version>3.5.0</version>
                 </requireMavenVersion>
                 <requireJavaVersion>
                   <version>[${maven.compiler.target},)</version>
@@ -744,6 +752,9 @@
           <name>m2e.version</name>
         </property>
       </activation>
+      <properties>
+        <maven.compiler.release>8</maven.compiler.release>
+      </properties>
       <build>
         <pluginManagement>
           <plugins>
@@ -804,5 +815,14 @@
         </pluginManagement>
       </build>
     </profile>
+    <profile>
+      <id>jdk-release-flag</id>
+      <activation>
+        <jdk>[9,)</jdk>
+      </activation>
+      <properties>
+        <maven.compiler.release>8</maven.compiler.release>
+      </properties>
+    </profile>
   </profiles>
 </project>

server/bin/get-tablet-summary.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+if [[ `uname` == "Darwin" ]]; then
+        THIS_SCRIPT=`python -c 'import os,sys; print os.path.realpath(sys.argv[1])' $0`
+else
+        THIS_SCRIPT=`readlink -f $0`
+fi
+
+THIS_DIR="${THIS_SCRIPT%/*}"
+BASE_DIR=${THIS_DIR}/..
+CONF_DIR="${BASE_DIR}/conf"
+LIB_DIR="${BASE_DIR}/lib"
+
+export CLASSPATH="${CONF_DIR}:${LIB_DIR}/*"
+JVM_ARGS="-Xmx1024m -Xms128m"
+JVM_ARGS="${JVM_ARGS} -Dlogging.config=${CONF_DIR}/log4j2-error-console.xml"
+JVM_ARGS="${JVM_ARGS} -Dlog4j.configurationFile=${CONF_DIR}/log4j2-error-console.xml"
+JVM_ARGS="${JVM_ARGS} -DLog4jContextSelector=org.apache.logging.log4j.core.async.AsyncLoggerContextSelector"
+
+echo "$JAVA_HOME/bin/java ${JVM_ARGS} timely.store.compaction.util.TabletMetadataConsole --spring.config.name=timely"
+$JAVA_HOME/bin/java ${JVM_ARGS} timely.store.compaction.util.TabletMetadataConsole --spring.config.name=timely

server/pom.xml

@@ -4,7 +4,7 @@
   <parent>
     <groupId>timely</groupId>
     <artifactId>timely</artifactId>
-    <version>1.2.5-SNAPSHOT</version>
+    <version>1.2.8-SNAPSHOT</version>
   </parent>
   <artifactId>timely-server</artifactId>
   <dependencies>
@@ -108,6 +108,10 @@
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-minicluster</artifactId>
     </dependency>
+    <dependency>
+      <groupId>org.glassfish.tyrus</groupId>
+      <artifactId>tyrus-client</artifactId>
+    </dependency>
     <dependency>
       <groupId>org.slf4j</groupId>
       <artifactId>slf4j-api</artifactId>
@@ -177,11 +181,22 @@
       <artifactId>junit</artifactId>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.easymock</groupId>
+      <artifactId>easymock</artifactId>
+      <scope>test</scope>
+    </dependency>
     <dependency>
       <groupId>org.jsoup</groupId>
       <artifactId>jsoup</artifactId>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.powermock</groupId>
+      <artifactId>powermock-easymock-release-full</artifactId>
+      <classifier>full</classifier>
+      <scope>test</scope>
+    </dependency>
     <dependency>
       <groupId>org.springframework.boot</groupId>
       <artifactId>spring-boot-test-autoconfigure</artifactId>

server/src/main/java/timely/Server.java

@@ -60,6 +60,7 @@ import io.netty.handler.stream.ChunkedWriteHandler;
 import io.netty.handler.timeout.IdleStateHandler;
 import io.netty.util.NettyRuntime;
 import io.netty.util.internal.SystemPropertyUtil;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
 import org.apache.curator.RetryPolicy;
 import org.apache.curator.framework.CuratorFramework;
 import org.apache.curator.framework.CuratorFrameworkFactory;
@@ -75,7 +76,6 @@ import org.slf4j.LoggerFactory;
 import org.springframework.boot.WebApplicationType;
 import org.springframework.boot.builder.SpringApplicationBuilder;
 import org.springframework.context.ConfigurableApplicationContext;
-import timely.api.response.TimelyException;
 import timely.auth.AuthCache;
 import timely.auth.VisibilityCache;
 import timely.configuration.Configuration;
@@ -233,11 +233,12 @@ public class Server {
 
     public static void main(String[] args) throws Exception {
 
-        Configuration conf = initializeConfiguration(args);
+        Server.applicationContext = initializeContext(args);
+        Configuration conf = Server.applicationContext.getBean(Configuration.class);
 
-        Server s = new Server(conf);
+        Server server = new Server(conf);
         try {
-            s.run();
+            server.run();
             LATCH.await();
         } catch (final InterruptedException e) {
             LOG.info("Server shutting down.");
@@ -245,16 +246,18 @@ public class Server {
             LOG.error("Error running server.", e);
         } finally {
             try {
-                s.shutdown();
+                server.shutdown();
             } catch (Exception e) {
                 System.exit(1);
             }
         }
     }
 
-    protected static Configuration initializeConfiguration(String[] args) {
-        applicationContext = new SpringApplicationBuilder(SpringBootstrap.class).web(WebApplicationType.NONE).run(args);
-        return applicationContext.getBean(Configuration.class);
+    protected static ConfigurableApplicationContext initializeContext(String[] args) {
+        SpringApplicationBuilder builder = new SpringApplicationBuilder(SpringBootstrap.class);
+        builder.web(WebApplicationType.NONE);
+        builder.registerShutdownHook(false);
+        return builder.run(args);
     }
 
     private void shutdownHook() {
@@ -269,6 +272,8 @@ public class Server {
     }
 
     public void shutdown() {
+        LOG.warn("Server shutdown 1");
+
         List<ChannelFuture> channelFutures = new ArrayList<>();
 
         if (tcpChannelHandle != null) {
@@ -286,8 +291,9 @@ public class Server {
             channelFutures.add(wsChannelHandle.close());
         }
 
+        int udpChannel = 1;
         for (Channel c : udpChannelHandleList) {
-            LOG.info("Closing udpChannelHandle");
+            LOG.info("Closing udpChannelHandle #" + udpChannel++);
             channelFutures.add(c.close());
         }
 
@@ -296,14 +302,15 @@ public class Server {
             try {
                 f.get();
             } catch (final Exception e) {
-                LOG.error("Error while shutting down channel: {}", f.channel().config());
-                LOG.error("{}", e.getMessage());
-                e.printStackTrace();
+                LOG.error("Channel:" + f.channel().config() + " -> " + e.getMessage(), e);
             }
         });
 
-        Integer quietPeriod = config.getServer().getShutdownQuietPeriod();
+        LOG.warn("Server shutdown 2");
+
+        int quietPeriod = config.getServer().getShutdownQuietPeriod();
         List<Future<?>> groupFutures = new ArrayList<>();
+
         if (tcpBossGroup != null) {
             LOG.info("Shutting down tcpBossGroup");
             groupFutures.add(tcpBossGroup.shutdownGracefully(quietPeriod, 10, TimeUnit.SECONDS));
@@ -339,6 +346,8 @@ public class Server {
             groupFutures.add(udpBossGroup.shutdownGracefully(quietPeriod, 10, TimeUnit.SECONDS));
         }
 
+        LOG.warn("Server shutdown 3");
+
         if (udpWorkerGroup != null) {
             LOG.info("Shutting down udpWorkerGroup");
             groupFutures.add(udpWorkerGroup.shutdownGracefully(quietPeriod, 10, TimeUnit.SECONDS));
@@ -348,41 +357,70 @@ public class Server {
             try {
                 f.get();
             } catch (final Exception e) {
-                LOG.error("Error while shutting down group: {}", f.toString());
-                LOG.error("{}", e.getMessage());
-                e.printStackTrace();
+                LOG.error("Group:" + f.toString() + " -> " + e.getMessage(), e);
             }
         });
 
-        try {
-            LOG.info("Flushing datastore.");
-            dataStore.flush();
-        } catch (TimelyException e) {
-            LOG.error("Error flushing to server during shutdown", e);
-        }
+        LOG.warn("Server shutdown 4");
 
         try {
-            LOG.info("Closing MetaCacheFactory");
+            LOG.info("Closing dataStore");
+            dataStore.close();
+        } catch (Exception e) {
+            LOG.error(e.getMessage(), e);
+        }
+
+        LOG.warn("Server shutdown 5");
+
+        if (dataStoreCache != null) {
+            try {
+                LOG.info("Closing dataStoreCache");
+                dataStoreCache.close();
+            } catch (Exception e) {
+                LOG.error(e.getMessage(), e);
+            }
+        }
+
+        LOG.warn("Server shutdown 6");
+
+        try {
+            LOG.info("Closing metaCacheFactory");
             MetaCacheFactory.close();
         } catch (Exception e) {
-            LOG.error("Error closing MetaCacheFactory during shutdown", e);
+            LOG.error(e.getMessage(), e);
         }
 
+        LOG.warn("Server shutdown 7");
+
         try {
-            LOG.info("Closing WebSocketRequestDecoder");
+            LOG.info("Closing webSocketRequestDecoder subscriptions");
             WebSocketRequestDecoder.close();
         } catch (Exception e) {
-            LOG.error("Error closing WebSocketRequestDecoder during shutdown", e);
+            LOG.error(e.getMessage(), e);
         }
 
+        LOG.warn("Server shutdown 8");
+
         if (curatorFramework != null) {
-            curatorFramework.close();
+            try {
+                LOG.info("Closing curatorFramework");
+                curatorFramework.close();
+            } catch (Exception e) {
+                LOG.error(e.getMessage(), e);
+            }
         }
 
+        LOG.warn("Server shutdown 9");
+
         if (applicationContext != null) {
-            LOG.info("Closing applicationContext");
-            applicationContext.close();
+            try {
+                LOG.info("Closing applicationContext");
+                applicationContext.close();
+            } catch (Exception e) {
+                LOG.error(e.getMessage(), e);
+            }
         }
+        LOG.warn("Server shutdown 10");
         this.shutdown = true;
         LOG.info("Server shut down.");
     }
@@ -417,7 +455,9 @@ public class Server {
     public void run() throws Exception {
 
         RetryPolicy retryPolicy = new RetryForever(1000);
-        curatorFramework = CuratorFrameworkFactory.newClient(config.getAccumulo().getZookeepers(), 30000, 10000,
+        int timeout = Long.valueOf(AccumuloConfiguration.getTimeInMillis(config.getAccumulo().getZookeeperTimeout()))
+                .intValue();
+        curatorFramework = CuratorFrameworkFactory.newClient(config.getAccumulo().getZookeepers(), timeout, 10000,
                 retryPolicy);
         curatorFramework.start();
         ensureZkPaths(curatorFramework, zkPaths);
@@ -544,7 +584,7 @@ public class Server {
         if (generate) {
             LOG.warn("Using generated self signed server certificate");
             Date begin = new Date();
-            Date end = new Date(begin.getTime() + 86400000);
+            Date end = new Date(begin.getTime() + TimeUnit.DAYS.toMillis(7));
             SelfSignedCertificate ssc = new SelfSignedCertificate("localhost", begin, end);
             ssl = SslContextBuilder.forServer(ssc.certificate(), ssc.privateKey());
         } else {
@@ -611,7 +651,8 @@ public class Server {
                 ch.pipeline().addLast("cors", new CorsHandler(cors));
                 ch.pipeline().addLast("queryDecoder",
                         new timely.netty.http.HttpRequestDecoder(config.getSecurity(), config.getHttp()));
-                ch.pipeline().addLast("fileServer", new HttpStaticFileServerHandler());
+                ch.pipeline().addLast("fileServer", new HttpStaticFileServerHandler()
+                        .setIgnoreSslHandshakeErrors(config.getSecurity().getServerSsl().isUseGeneratedKeypair()));
                 ch.pipeline().addLast("strict", new StrictTransportHandler(config));
                 ch.pipeline().addLast("login", new X509LoginRequestHandler(config.getSecurity(), config.getHttp()));
                 ch.pipeline().addLast("doLogin",
@@ -624,7 +665,9 @@ public class Server {
                 ch.pipeline().addLast("version", new HttpVersionRequestHandler());
                 ch.pipeline().addLast("cache", new HttpCacheRequestHandler(dataStoreCache));
                 ch.pipeline().addLast("put", new HttpMetricPutHandler(dataStore));
-                ch.pipeline().addLast("error", new TimelyExceptionHandler());
+                ch.pipeline().addLast("error", new TimelyExceptionHandler()
+                        .setIgnoreSslHandshakeErrors(config.getSecurity().getServerSsl().isUseGeneratedKeypair()));
+
             }
         };
     }

server/src/main/java/timely/StandaloneServer.java

@@ -52,7 +52,8 @@ public class StandaloneServer extends Server {
             System.exit(1);
         }
 
-        Configuration conf = initializeConfiguration(args);
+        StandaloneServer.applicationContext = initializeContext(args);
+        Configuration conf = StandaloneServer.applicationContext.getBean(Configuration.class);
 
         File accumuloDir = new File(tmp, "accumulo");
         MiniAccumuloConfig macConfig = new MiniAccumuloConfig(accumuloDir, "secret");

server/src/main/java/timely/api/model/Meta.java

@@ -5,6 +5,10 @@ import java.util.List;
 
 import org.apache.accumulo.core.data.Key;
 import org.apache.accumulo.core.data.Value;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.builder.CompareToBuilder;
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
 
 public class Meta implements Comparable<Meta> {
 
@@ -49,9 +53,15 @@ public class Meta implements Comparable<Meta> {
 
     public List<Key> toKeys() {
         List<Key> keys = new ArrayList<>();
-        keys.add(new Key(METRIC_PREFIX + metric));
-        keys.add(new Key(TAG_PREFIX + metric, tagKey));
-        keys.add(new Key(VALUE_PREFIX + metric, tagKey, tagValue));
+        if (StringUtils.isNotBlank(metric)) {
+            keys.add(new Key(METRIC_PREFIX + metric));
+        }
+        if (StringUtils.isNotBlank(tagKey)) {
+            keys.add(new Key(TAG_PREFIX + metric, tagKey));
+        }
+        if (StringUtils.isNotBlank(tagKey) && StringUtils.isNotBlank(tagValue)) {
+            keys.add(new Key(VALUE_PREFIX + metric, tagKey, tagValue));
+        }
         return keys;
     }
 
@@ -97,39 +107,25 @@ public class Meta implements Comparable<Meta> {
 
     @Override
     public int hashCode() {
-        final int prime = 31;
-        int result = 1;
-        result = prime * result + ((metric == null) ? 0 : metric.hashCode());
-        result = prime * result + ((tagKey == null) ? 0 : tagKey.hashCode());
-        result = prime * result + ((tagValue == null) ? 0 : tagValue.hashCode());
-        return result;
+        HashCodeBuilder builder = new HashCodeBuilder();
+        builder.append(metric);
+        builder.append(tagKey);
+        builder.append(tagValue);
+        return builder.toHashCode();
     }
 
     @Override
-    public boolean equals(Object obj) {
-        if (this == obj)
-            return true;
-        if (obj == null)
+    public boolean equals(Object o) {
+        if (o instanceof Meta) {
+            Meta obj = (Meta) o;
+            EqualsBuilder builder = new EqualsBuilder();
+            builder.append(this.metric, obj.metric);
+            builder.append(this.tagKey, obj.tagKey);
+            builder.append(this.tagValue, obj.tagValue);
+            return builder.isEquals();
+        } else {
             return false;
-        if (getClass() != obj.getClass())
-            return false;
-        Meta other = (Meta) obj;
-        if (metric == null) {
-            if (other.metric != null)
-                return false;
-        } else if (!metric.equals(other.metric))
-            return false;
-        if (tagKey == null) {
-            if (other.tagKey != null)
-                return false;
-        } else if (!tagKey.equals(other.tagKey))
-            return false;
-        if (tagValue == null) {
-            if (other.tagValue != null)
-                return false;
-        } else if (!tagValue.equals(other.tagValue))
-            return false;
-        return true;
+        }
     }
 
     @Override
@@ -139,15 +135,10 @@ public class Meta implements Comparable<Meta> {
 
     @Override
     public int compareTo(Meta o) {
-        int result = metric.compareTo(o.metric);
-        if (result != 0) {
-            return result;
-        }
-        result = tagKey.compareTo(o.tagKey);
-        if (result != 0) {
-            return result;
-        }
-        return tagValue.compareTo(o.tagValue);
+        CompareToBuilder builder = new CompareToBuilder();
+        builder.append(this.metric, o.metric);
+        builder.append(this.tagKey, o.tagKey);
+        builder.append(this.tagValue, o.tagValue);
+        return builder.toComparison();
     }
-
 }

server/src/main/java/timely/api/response/timeseries/MetricsResponse.java

@@ -156,8 +156,10 @@ public class MetricsResponse {
         b.append(TR_END);
         String prevMetric = null;
         StringBuilder tags = new StringBuilder();
+        Map<String, Long> tagMap = new HashMap<>();
         for (Meta m : tree) {
             if (prevMetric != null && !m.getMetric().equals(prevMetric)) {
+                tagMap.clear();
                 b.append(TR_START);
                 b.append(TD_START).append(prevMetric).append(TD_END);
                 b.append(TD_START).append(tags.toString()).append(TD_END);
@@ -168,8 +170,12 @@ public class MetricsResponse {
             if (prevMetric == null) {
                 prevMetric = m.getMetric();
             }
+            long numValues = tagMap.getOrDefault(m.getTagKey(), 0l);
             if (!(this.ignoredTags.contains(m.getTagKey()))) {
-                tags.append(m.getTagKey()).append("=").append(m.getTagValue()).append(" ");
+                if (numValues <= conf.getMetaCache().getMaxTagValues()) {
+                    tags.append(m.getTagKey()).append("=").append(m.getTagValue()).append(" ");
+                    tagMap.put(m.getTagKey(), ++numValues);
+                }
             }
         }
         b.append(TR_START);

server/src/main/java/timely/configuration/Accumulo.java

@@ -12,6 +12,8 @@ public class Accumulo {
     @NotBlank
     private String instanceName;
     @NotBlank
+    private String zookeeperTimeout = "120s";
+    @NotBlank
     private String username;
     @NotBlank
     private String password;
@@ -46,6 +48,14 @@ public class Accumulo {
         this.instanceName = instanceName;
     }
 
+    public String getZookeeperTimeout() {
+        return zookeeperTimeout;
+    }
+
+    public void setZookeeperTimeout(String zookeeperTimeout) {
+        this.zookeeperTimeout = zookeeperTimeout;
+    }
+
     public String getUsername() {
         return username;
     }

server/src/main/java/timely/configuration/Cache.java

@@ -13,6 +13,7 @@ public class Cache {
     private List<String> nonCachedMetrics = new ArrayList<>();
     private long maxUniqueTagSets = 50000;
     private long flushInterval = 5000;
+    private long staleCacheExpiration = 1800000;
 
     public HashMap<String, Integer> getMetricAgeOffHours() {
         return metricAgeOffHours;
@@ -57,4 +58,12 @@ public class Cache {
     public void setFlushInterval(long flushInterval) {
         this.flushInterval = flushInterval;
     }
+
+    public long getStaleCacheExpiration() {
+        return staleCacheExpiration;
+    }
+
+    public void setStaleCacheExpiration(long staleCacheExpiration) {
+        this.staleCacheExpiration = staleCacheExpiration;
+    }
 }

server/src/main/java/timely/netty/http/HttpStaticFileServerHandler.java

@@ -48,6 +48,7 @@ import java.util.TimeZone;
 import java.util.regex.Pattern;
 
 import javax.activation.MimetypesFileTypeMap;
+import javax.net.ssl.SSLHandshakeException;
 
 import io.netty.buffer.ByteBuf;
 import io.netty.buffer.Unpooled;
@@ -123,6 +124,7 @@ public class HttpStaticFileServerHandler extends SimpleChannelInboundHandler<Ful
     public static final String HTTP_DATE_FORMAT = "EEE, dd MMM yyyy HH:mm:ss zzz";
     public static final String HTTP_DATE_GMT_TIMEZONE = "GMT";
     public static final int HTTP_CACHE_SECONDS = 60;
+    private boolean ignoreSslHandshakeErrors = false;
 
     @Override
     public void channelRead0(ChannelHandlerContext ctx, FullHttpRequest request) throws Exception {
@@ -221,6 +223,10 @@ public class HttpStaticFileServerHandler extends SimpleChannelInboundHandler<Ful
 
     @Override
     public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) {
+        // ignore SSLHandshakeException when using a self-signed server certificate
+        if (ignoreSslHandshakeErrors && cause.getCause() instanceof SSLHandshakeException) {
+            return;
+        }
         LOG.error("Error serving file", cause);
         if (ctx.channel().isActive()) {
             sendError(ctx, INTERNAL_SERVER_ERROR);
@@ -379,4 +385,9 @@ public class HttpStaticFileServerHandler extends SimpleChannelInboundHandler<Ful
         MimetypesFileTypeMap mimeTypesMap = new MimetypesFileTypeMap();
         response.headers().set(CONTENT_TYPE, mimeTypesMap.getContentType(file.getPath()));
     }
+
+    public HttpStaticFileServerHandler setIgnoreSslHandshakeErrors(boolean ignoreSslHandshakeErrors) {
+        this.ignoreSslHandshakeErrors = ignoreSslHandshakeErrors;
+        return this;
+    }
 }

server/src/main/java/timely/netty/http/TimelyExceptionHandler.java

@@ -1,5 +1,7 @@
 package timely.netty.http;
 
+import javax.net.ssl.SSLHandshakeException;
+
 import io.netty.channel.ChannelHandler.Sharable;
 import io.netty.channel.ChannelHandlerContext;
 import io.netty.channel.SimpleChannelInboundHandler;
@@ -12,6 +14,7 @@ import timely.api.response.TimelyException;
 public class TimelyExceptionHandler extends SimpleChannelInboundHandler<TimelyException> implements TimelyHttpHandler {
 
     private static final Logger LOG = LoggerFactory.getLogger(TimelyExceptionHandler.class);
+    private boolean ignoreSslHandshakeErrors = false;
 
     @Override
     protected void channelRead0(ChannelHandlerContext ctx, TimelyException msg) throws Exception {
@@ -20,6 +23,10 @@ public class TimelyExceptionHandler extends SimpleChannelInboundHandler<TimelyEx
 
     @Override
     public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception {
+        // ignore SSLHandshakeException when using a self-signed server certificate
+        if (ignoreSslHandshakeErrors && cause.getCause() instanceof SSLHandshakeException) {
+            return;
+        }
         LOG.error("Unhandled exception in pipeline", cause);
         if (cause instanceof TimelyException) {
             this.sendHttpError(ctx, (TimelyException) cause);
@@ -32,4 +39,8 @@ public class TimelyExceptionHandler extends SimpleChannelInboundHandler<TimelyEx
         }
     }
 
+    public TimelyExceptionHandler setIgnoreSslHandshakeErrors(boolean ignoreSslHandshakeErrors) {
+        this.ignoreSslHandshakeErrors = ignoreSslHandshakeErrors;
+        return this;
+    }
 }

server/src/main/java/timely/netty/websocket/WebSocketRequestDecoder.java

@@ -109,7 +109,6 @@ public class WebSocketRequestDecoder extends MessageToMessageDecoder<WebSocketFr
     }
 
     public static void close() {
-        LOG.info("Closing subscriptions");
         SubscriptionRegistry.get().forEach((k, v) -> v.close());
     }
 

server/src/main/java/timely/store/DataStore.java

@@ -29,8 +29,6 @@ public interface DataStore {
 
     List<QueryResponse> query(QueryRequest msg) throws TimelyException;
 
-    void flush() throws TimelyException;
-
     long getAgeOffForMetric(String metricName);
 
     List<Range> getQueryRanges(String metric, long start, long end, Set<Tag> colFamValues);
@@ -45,4 +43,6 @@ public interface DataStore {
     DataStoreCache getCache();
 
     InternalMetrics getInternalMetrics();
+
+    void close() throws Exception;
 }

server/src/main/java/timely/store/DataStoreImpl.java

@@ -22,10 +22,10 @@ import java.util.Map.Entry;
 import java.util.Optional;
 import java.util.Set;
 import java.util.SortedMap;
-import java.util.Timer;
-import java.util.TimerTask;
 import java.util.TreeMap;
 import java.util.TreeSet;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicReference;
@@ -127,7 +127,7 @@ public class DataStoreImpl implements DataStore {
     private final String metricsTable;
     private final String metaTable;
     private final InternalMetrics internalMetrics;
-    private final Timer internalMetricsTimer = new Timer("InternalMetricsTimer", true);
+    private ScheduledExecutorService executorService = Executors.newScheduledThreadPool(1);
     private final int scannerThreads;
     private final long maxDownsampleMemory;
     private final BatchWriterConfig bwConfig;
@@ -147,6 +147,7 @@ public class DataStoreImpl implements DataStore {
             Accumulo accumuloConf = conf.getAccumulo();
             properties.put("instance.name", accumuloConf.getInstanceName());
             properties.put("instance.zookeeper.host", accumuloConf.getZookeepers());
+            properties.put("instance.zookeeper.timeout", accumuloConf.getZookeeperTimeout());
             final ClientConfiguration aconf = ClientConfiguration.fromMap(properties);
             final Instance instance = new ZooKeeperInstance(aconf);
             connector = instance.getConnector(accumuloConf.getUsername(),
@@ -243,14 +244,9 @@ public class DataStoreImpl implements DataStore {
             }
 
             internalMetrics = new InternalMetrics(conf);
-            internalMetricsTimer.schedule(new TimerTask() {
-
-                @Override
-                public void run() {
-                    internalMetrics.getMetricsAndReset().forEach(m -> store(m, false));
-                }
-
-            }, METRICS_PERIOD, METRICS_PERIOD);
+            executorService.scheduleAtFixedRate(
+                    () -> internalMetrics.getMetricsAndReset().forEach(m -> store(m, false)), METRICS_PERIOD,
+                    METRICS_PERIOD, TimeUnit.MILLISECONDS);
 
             this.metaCache = MetaCacheFactory.getCache(conf);
         } catch (Exception e2) {
@@ -354,10 +350,7 @@ public class DataStoreImpl implements DataStore {
         internalMetrics.incrementMetricsReceived(1);
         List<Meta> toCache = new ArrayList<>(metric.getTags().size());
         for (final Tag tag : metric.getTags()) {
-            Meta key = new Meta(metric.getName(), tag.getKey(), tag.getValue());
-            if (!metaCache.contains(key)) {
-                toCache.add(key);
-            }
+            toCache.add(new Meta(metric.getName(), tag.getKey(), tag.getValue()));
         }
         if (!toCache.isEmpty()) {
             final Set<Mutation> muts = new TreeSet<>(new Comparator<Mutation>() {
@@ -378,7 +371,7 @@ public class DataStoreImpl implements DataStore {
             MetaKeySet mks = new MetaKeySet();
             toCache.forEach(m -> mks.addAll(m.toKeys()));
             internalMetrics.incrementMetaKeysInserted(mks.size());
-            muts.addAll(mks.toMutations());
+            muts.addAll(mks.toMutations(metric.getValue().getTimestamp()));
             try {
                 metaWriter.get().addMutations(muts);
             } catch (MutationsRejectedException e) {
@@ -486,19 +479,6 @@ public class DataStoreImpl implements DataStore {
         return result;
     }
 
-    @Override
-    public void flush() {
-        internalMetricsTimer.cancel();
-        writers.forEach(w -> {
-            try {
-                w.close();
-            } catch (final Exception ex) {
-                LOG.warn("Error shutting down batchwriter", ex);
-            }
-
-        });
-    }
-
     @Override
     public SearchLookupResponse lookup(SearchLookupRequest msg) throws TimelyException {
         long startMillis = System.currentTimeMillis();
@@ -1056,4 +1036,24 @@ public class DataStoreImpl implements DataStore {
         }
     }
 
+    @Override
+    public void close() throws Exception {
+        this.executorService.shutdown();
+        try {
+            this.executorService.awaitTermination(5, TimeUnit.SECONDS);
+        } catch (InterruptedException e) {
+
+        } finally {
+            if (!this.executorService.isTerminated()) {
+                this.executorService.shutdownNow();
+            }
+        }
+        writers.forEach(w -> {
+            try {
+                w.close();
+            } catch (final Exception ex) {
+                LOG.warn("Error shutting down batchwriter: ", ex.getMessage());
+            }
+        });
+    }
 }

server/src/main/java/timely/store/MetaCacheImpl.java

@@ -5,16 +5,20 @@ import java.util.HashMap;
 import java.util.Iterator;
 import java.util.Map;
 import java.util.Set;
-import java.util.Timer;
-import java.util.TimerTask;
 import java.util.TreeSet;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
 
 import com.github.benmanes.caffeine.cache.Cache;
 import com.github.benmanes.caffeine.cache.Caffeine;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
 import org.apache.accumulo.core.client.ClientConfiguration;
 import org.apache.accumulo.core.client.Connector;
 import org.apache.accumulo.core.client.Instance;
 import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableNotFoundException;
 import org.apache.accumulo.core.client.ZooKeeperInstance;
 import org.apache.accumulo.core.client.security.tokens.PasswordToken;
 import org.apache.accumulo.core.data.Key;
@@ -34,7 +38,7 @@ public class MetaCacheImpl implements MetaCache {
     private volatile boolean closed = false;
     private Cache<Meta, Object> cache = null;
     private Configuration configuration;
-    private Timer timer = new Timer("MetaCacheImpl", true);
+    private ScheduledExecutorService executorService = Executors.newScheduledThreadPool(1);
 
     @Override
     public void init(Configuration config) {
@@ -42,13 +46,8 @@ public class MetaCacheImpl implements MetaCache {
         cache = Caffeine.newBuilder().initialCapacity(1000).build();
         long cacheRefreshMinutes = configuration.getMetaCache().getCacheRefreshMinutes();
         if (cacheRefreshMinutes > 0) {
-            timer.schedule(new TimerTask() {
-
-                @Override
-                public void run() {
-                    refreshCache(config.getMetaCache().getExpirationMinutes());
-                }
-            }, 60000, cacheRefreshMinutes * 60 * 1000);
+            executorService.scheduleAtFixedRate(() -> refreshCache(config.getMetaCache().getExpirationMinutes()), 1,
+                    cacheRefreshMinutes, TimeUnit.MINUTES);
         }
     }
 
@@ -63,12 +62,16 @@ public class MetaCacheImpl implements MetaCache {
         Map<String, Map<String, Long>> metricMap = new HashMap<>();
         Scanner scanner = null;
         try {
+            Map<Meta, Object> newCache = new HashMap<>();
             final Instance instance = new ZooKeeperInstance(aConf);
             Connector connector = instance.getConnector(accumuloConf.getUsername(),
                     new PasswordToken(accumuloConf.getPassword()));
             scanner = connector.createScanner(metaTable, Authorizations.EMPTY);
             LOG.debug("Begin scanning " + metaTable);
-            Range metricRange = new Range(new Key(Meta.METRIC_PREFIX), true, new Key(Meta.METRIC_PREFIX + '\0'), false);
+            Key metricPrefixBeginKey = new Key(Meta.METRIC_PREFIX);
+            int firstChar = Meta.METRIC_PREFIX.charAt(0);
+            Key metricPrefixEndKey = new Key(String.valueOf((char) (firstChar + 1)) + ':');
+            Range metricRange = new Range(metricPrefixBeginKey, true, metricPrefixEndKey, false);
             scanner.setRange(metricRange);
             Set<String> allMetrics = new TreeSet<>();
             for (Map.Entry<Key, Value> entry : scanner) {
@@ -96,20 +99,20 @@ public class MetaCacheImpl implements MetaCache {
                     if (entry.getKey().getTimestamp() > oldestTimestamp) {
                         tagMap.put(tagKey, ++numTagValues);
                         if (numTagValues <= configuration.getMetaCache().getMaxTagValues()) {
-                            cache.put(meta, DUMMY);
+                            newCache.put(meta, DUMMY);
                         } else {
-                            // found maxTagValues on this refresh, so remove this key
-                            cache.invalidate(meta);
+                            // found maxTagValues on this refresh
                             maxedOutValues = true;
                         }
-                    } else {
-                        // this entry is more than expirationMinutes old, so remove it
-                        cache.invalidate(meta);
                     }
                 }
             }
+            synchronized (cache) {
+                cache.invalidateAll();
+                cache.putAll(newCache);
+            }
             LOG.debug("Finished scanning " + metaTable);
-        } catch (Exception e) {
+        } catch (AccumuloException | AccumuloSecurityException | TableNotFoundException e) {
             LOG.error(e.getMessage(), e);
         } finally {
             if (scanner != null) {
@@ -141,6 +144,16 @@ public class MetaCacheImpl implements MetaCache {
     @Override
     public void close() {
         this.closed = true;
+        this.executorService.shutdown();
+        try {
+            this.executorService.awaitTermination(5, TimeUnit.SECONDS);
+        } catch (InterruptedException e) {
+
+        } finally {
+            if (!this.executorService.isTerminated()) {
+                this.executorService.shutdownNow();
+            }
+        }
     }
 
     @Override

server/src/main/java/timely/store/cache/DataStoreCache.java

@@ -11,9 +11,9 @@ import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
-import java.util.Timer;
-import java.util.TimerTask;
 import java.util.TreeSet;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.locks.StampedLock;
 
@@ -71,6 +71,7 @@ public class DataStoreCache {
     private Set<String> nonCachedMetrics = Collections.synchronizedSet(new HashSet<>());
     private DistributedAtomicValue nonCachedMetricsIP = null;
     private InterProcessReadWriteLock nonCachedMetricsIPRWLock;
+    private long staleCacheExpiration;
     private long maxUniqueTagSets;
     private final Security security;
     private Map<String, Long> minimumAgeOff;
@@ -78,7 +79,7 @@ public class DataStoreCache {
     private int flushBatch = 0;
     private int numBatches = 5;
     private InternalMetrics internalMetrics;
-    private Timer maintenanceTimer = new Timer("DataStoreCacheTimer");
+    private ScheduledExecutorService executorService = Executors.newScheduledThreadPool(6);
 
     public DataStoreCache(CuratorFramework curatorFramework, Configuration conf) {
         if (curatorFramework != null) {
@@ -89,6 +90,7 @@ public class DataStoreCache {
         LOG.info("Reading initial values from nonCachedMetricsIP");
         readNonCachedMetricsIP();
         maxUniqueTagSets = conf.getCache().getMaxUniqueTagSets();
+        staleCacheExpiration = conf.getCache().getStaleCacheExpiration();
         security = conf.getSecurity();
         Map<String, Integer> cacheAgeOff = conf.getCache().getMetricAgeOffHours();
         Map<String, Integer> accumuloAgeOff = conf.getMetricAgeOffDays();
@@ -101,70 +103,58 @@ public class DataStoreCache {
 
         minimumAgeOffForIterator = getAgeOffForIterator(minimumAgeOff);
 
-        maintenanceTimer.schedule(new TimerTask() {
-
-            @Override
-            public void run() {
-                try {
-                    flushCaches(flushBatch);
-                } catch (Exception e) {
-                    LOG.error(e.getMessage(), e);
-                } finally {
-                    flushBatch++;
-                    if (flushBatch == numBatches) {
-                        flushBatch = 0;
-                    }
+        executorService.scheduleAtFixedRate(() -> {
+            try {
+                flushCaches(flushBatch);
+            } catch (Exception e) {
+                LOG.error(e.getMessage(), e);
+            } finally {
+                flushBatch++;
+                if (flushBatch == numBatches) {
+                    flushBatch = 0;
                 }
             }
-        }, conf.getCache().getFlushInterval(), conf.getCache().getFlushInterval());
+        }, conf.getCache().getFlushInterval(), conf.getCache().getFlushInterval(), TimeUnit.MILLISECONDS);
 
-        maintenanceTimer.schedule(new TimerTask() {
-
-            @Override
-            public void run() {
-                try {
-                    ageOffGorillaStores();
-                } catch (Exception e) {
-                    LOG.error(e.getMessage(), e);
-                }
+        executorService.scheduleAtFixedRate(() -> {
+            try {
+                DataStoreCache.this.removeStaleMetrics();
+            } catch (Exception e) {
+                LOG.error(e.getMessage(), e);
             }
-        }, (600 * 1000), (600 * 1000));
+        }, 5, 5, TimeUnit.MINUTES);
 
-        maintenanceTimer.schedule(new TimerTask() {
-
-            @Override
-            public void run() {
-                try {
-                    archiveGorillaStoreCurrentCompressors();
-                } catch (Exception e) {
-                    LOG.error(e.getMessage(), e);
-                }
+        executorService.scheduleAtFixedRate(() -> {
+            try {
+                ageOffGorillaStores();
+            } catch (Exception e) {
+                LOG.error(e.getMessage(), e);
             }
-        }, (3600 * 1000), (3600 * 1000));
+        }, 10, 10, TimeUnit.MINUTES);
 
-        maintenanceTimer.schedule(new TimerTask() {
-
-            @Override
-            public void run() {
-                try {
-                    pruneStats();
-                } catch (Exception e) {
-                    LOG.error(e.getMessage(), e);
-                }
+        executorService.scheduleAtFixedRate(() -> {
+            try {
+                archiveGorillaStoreCurrentCompressors();
+            } catch (Exception e) {
+                LOG.error(e.getMessage(), e);
             }
-        }, (600 * 1000), (600 * 1000));
+        }, 60, 60, TimeUnit.MINUTES);
 
-        maintenanceTimer.schedule(new TimerTask() {
-
-            @Override
-            public void run() {
-                try {
-                    reportInternalMetrics();
-                } catch (Exception e) {
-                    LOG.error(e.getMessage(), e);
-                }
+        executorService.scheduleAtFixedRate(() -> {
+            try {
+                pruneStats();
+            } catch (Exception e) {
+                LOG.error(e.getMessage(), e);
             }
-        }, (60 * 1000), (60 * 1000));
+        }, 10, 10, TimeUnit.MINUTES);
+
+        executorService.scheduleAtFixedRate(() -> {
+            try {
+                reportInternalMetrics();
+            } catch (Exception e) {
+                LOG.error(e.getMessage(), e);
+            }
+        }, 1, 1, TimeUnit.MINUTES);
     }
 
     private void addNonCachedMetrics(Collection<String> nonCachedMetricsUpdate) {
@@ -216,6 +206,11 @@ public class DataStoreCache {
         nonCachedMetricsIPRWLock = new InterProcessReadWriteLock(curatorFramework, NON_CACHED_METRICS_LOCK_PATH);
         testIPRWLock(curatorFramework, nonCachedMetricsIPRWLock, NON_CACHED_METRICS_LOCK_PATH);
         nonCachedMetricsIP = new DistributedAtomicValue(curatorFramework, NON_CACHED_METRICS, new RetryForever(1000));
+        try {
+            nonCachedMetricsIP.initialize(SerializationUtils.serialize(new TreeSet<>()));
+        } catch (Exception e) {
+            LOG.error(e.getMessage(), e);
+        }
         TreeCacheListener nonCachedMetricsListener = new TreeCacheListener() {
 
             @Override
@@ -227,8 +222,7 @@ public class DataStoreCache {
             }
         };
 
-        try {
-            TreeCache nonCachedMetricsTreeCache = new TreeCache(curatorFramework, NON_CACHED_METRICS);
+        try (TreeCache nonCachedMetricsTreeCache = new TreeCache(curatorFramework, NON_CACHED_METRICS)) {
             nonCachedMetricsTreeCache.getListenable().addListener(nonCachedMetricsListener);
             nonCachedMetricsTreeCache.start();
         } catch (Exception e) {
@@ -437,6 +431,68 @@ public class DataStoreCache {
         }
     }
 
+    private void removeStaleMetrics() {
+
+        Set<String> metricsToRemove = new HashSet<>();
+        long stamp = gorillaMapLock.readLock();
+        try {
+            long now = System.currentTimeMillis();
+            for (Map.Entry<String, Map<TaggedMetric, GorillaStore>> entry : gorillaMap.entrySet()) {
+                String metricName = entry.getKey();
+                long totalMsecToMetric = 0;
+                long numStoresWithMultipleEntries = 0;
+                int tagVariations = entry.getValue().values().size();
+                for (GorillaStore store : entry.getValue().values()) {
+                    long numEntries = store.getNumEntries();
+                    long oldestTimestamp = store.getOldestTimestamp();
+                    long newestTimestamp = store.getNewestTimestamp();
+                    if (numEntries > 1) {
+                        long msecToMetric = (newestTimestamp - oldestTimestamp) / (numEntries - 1);
+                        totalMsecToMetric += msecToMetric;
+                        numStoresWithMultipleEntries++;
+                    }
+                }
+                if (numStoresWithMultipleEntries > 0) {
+                    long avgMsecToMetric = totalMsecToMetric / numStoresWithMultipleEntries;
+                    long ageOfNewest = now - getNewestTimestamp(metricName, stamp);
+                    // do not delete slow-arriving metrics based on this criteria
+                    // only consider metrics that should have arrived in staleCacheExpiration * 0.5
+                    LOG.trace("metric:{} tagVariations:{} avgMsecToMetric:{}", metricName, tagVariations,
+                            avgMsecToMetric);
+                    if (avgMsecToMetric < (staleCacheExpiration * 0.5)) {
+                        if (ageOfNewest > staleCacheExpiration) {
+                            LOG.info(
+                                    "Removing metric:{} tagVariations:{} avgMsecToMetric:{} ageOfNewestEntry:{} > staleCacheExpiration:{}",
+                                    metricName, tagVariations, avgMsecToMetric, ageOfNewest, staleCacheExpiration);
+                            metricsToRemove.add(metricName);
+                        }
+                    } else {
+                        LOG.trace(
+                                "Skipping staleness evaluation of metric:{} tagVariations:{} avgMsecToMetric:{} < (0.5 * staleCacheExpiration):{}",
+                                metricName, tagVariations, avgMsecToMetric, (staleCacheExpiration / 2));
+                    }
+                }
+            }
+        } catch (Exception e) {
+            LOG.error(e.getMessage(), e);
+        } finally {
+            gorillaMapLock.unlockRead(stamp);
+        }
+
+        if (metricsToRemove.size() > 0) {
+            stamp = gorillaMapLock.writeLock();
+            try {
+                for (String metricName : metricsToRemove) {
+                    gorillaMap.remove(metricName);
+                }
+            } catch (Exception e) {
+                LOG.error(e.getMessage(), e);
+            } finally {
+                gorillaMapLock.unlockWrite(stamp);
+            }
+        }
+    }
+
     public long getAgeOffForMetric(String metricName) {
         if (this.minimumAgeOff.containsKey(metricName)) {
             return this.minimumAgeOff.get(metricName);
@@ -512,6 +568,7 @@ public class DataStoreCache {
         try {
             metricMap = gorillaMap.get(metric);
             if (metricMap == null) {
+                LOG.info("Creating new cache for metric:{}", metric);
                 metricMap = new HashMap<>();
                 needWrite = true;
             }
@@ -769,9 +826,13 @@ public class DataStoreCache {
     }
 
     public long getNewestTimestamp(String metric) {
+        return getNewestTimestamp(metric, 0);
+    }
+
+    public long getNewestTimestamp(String metric, long lockStamp) {
 
         long newest = 0;
-        long stamp = gorillaMapLock.readLock();
+        long stamp = lockStamp == 0 ? gorillaMapLock.readLock() : lockStamp;
         try {
             Map<TaggedMetric, GorillaStore> gorillaStoreMap = gorillaMap.get(metric);
             for (Map.Entry<TaggedMetric, GorillaStore> entry : gorillaStoreMap.entrySet()) {
@@ -780,7 +841,9 @@ public class DataStoreCache {
                 }
             }
         } finally {
-            gorillaMapLock.unlockRead(stamp);
+            if (lockStamp == 0) {
+                gorillaMapLock.unlockRead(stamp);
+            }
         }
         return newest;
     }
@@ -848,4 +911,17 @@ public class DataStoreCache {
         }
         return response;
     }
+
+    public void close() {
+        this.executorService.shutdown();
+        try {
+            this.executorService.awaitTermination(5, TimeUnit.SECONDS);
+        } catch (InterruptedException e) {
+
+        } finally {
+            if (!this.executorService.isTerminated()) {
+                this.executorService.shutdownNow();
+            }
+        }
+    }
 }

server/src/main/java/timely/store/compaction/MetricCompactionStrategy.java

@@ -0,0 +1,189 @@
+package timely.store.compaction;
+
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Optional;
+import java.util.OptionalLong;
+
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.TabletId;
+import org.apache.accumulo.server.fs.FileRef;
+import org.apache.accumulo.tserver.compaction.CompactionPlan;
+import org.apache.accumulo.tserver.compaction.CompactionStrategy;
+import org.apache.accumulo.tserver.compaction.MajorCompactionRequest;
+import org.apache.commons.collections4.trie.PatriciaTrie;
+import org.apache.commons.configuration.Configuration;
+import org.apache.commons.configuration.MapConfiguration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import timely.store.MetricAgeOffIterator;
+
+public class MetricCompactionStrategy extends CompactionStrategy {
+
+    private static final Logger LOG = LoggerFactory.getLogger(MetricCompactionStrategy.class);
+
+    private boolean hasMinAgeOff;
+    private MetricAgeOffConfiguration minAgeOffConfig;
+    private boolean logOnly;
+    private String filterPrefix;
+    private String majcIteratorName;
+    private long explicitTimeMillis;
+    private boolean useExplicitTime;
+
+    public static final String LOG_ONLY_KEY = "logonly";
+    public static final String MAJC_ITERATOR_NAME_KEY = "majc.name";
+    public static final String DEFAULT_MAJC_ITERATOR_NAME = "ageoffmetrics";
+    public static final String AGE_OFF_PREFIX = MetricAgeOffIterator.AGE_OFF_PREFIX;
+    public static final String FILTER_PREFIX_KEY = AGE_OFF_PREFIX + "filter";
+    public static final String MIN_AGEOFF_KEY = AGE_OFF_PREFIX + "min";
+    public static final String DEFAULT_AGEOFF_KEY_SUFFIX = MetricAgeOffIterator.DEFAULT_AGEOFF_KEY;
+    public static final String DEFAULT_AGEOFF_KEY = AGE_OFF_PREFIX + DEFAULT_AGEOFF_KEY_SUFFIX;
+
+    @Override
+    public void init(Map<String, String> options) {
+        // options:
+        // logonly - disables compaction and will only log result
+        // majciterator - the iterator name to gather age-off values
+        // filter - limit the compaction to just a subset of prefixes
+        // min - minimum age-off which may override the metric age-off;
+        // allows more data to accumulate in tablets without being removed
+
+        logOnly = options.containsKey(LOG_ONLY_KEY) && Boolean.parseBoolean(options.get(LOG_ONLY_KEY));
+        majcIteratorName = options.getOrDefault(MAJC_ITERATOR_NAME_KEY, DEFAULT_MAJC_ITERATOR_NAME);
+        filterPrefix = options.get(FILTER_PREFIX_KEY);
+        if (options.containsKey(MIN_AGEOFF_KEY)) {
+            long minAgeOff = Long.parseLong(options.get(MIN_AGEOFF_KEY));
+            hasMinAgeOff = true;
+            minAgeOffConfig = MetricAgeOffConfiguration.newFromDefaultingMinimum(minAgeOff);
+        }
+    }
+
+    @Override
+    public boolean shouldCompact(MajorCompactionRequest request) {
+        TabletId tabletId = request.getTabletId();
+
+        if (null == tabletId.getEndRow() || null == tabletId.getPrevEndRow()) {
+            return false;
+        } else if (request.getFiles().isEmpty()) {
+            return false;
+        }
+
+        // check if prev/end key are both at age-off intervals
+        // skip tablet if there is no prev-end key, or previous age-off is not at max;
+        // other age-off will compact
+        Optional<String> endName = TabletRowAdapter.decodeRowPrefix(tabletId.getEndRow());
+        OptionalLong endOffset = TabletRowAdapter.decodeRowOffset(tabletId.getEndRow());
+        Optional<String> prevEndName = TabletRowAdapter.decodeRowPrefix(tabletId.getPrevEndRow());
+
+        if (!endName.isPresent() || !prevEndName.isPresent() || !endOffset.isPresent()) {
+            return false;
+        }
+
+        // compute age-off from majc iterator
+        // then maximize on the min-default-ageoff so that the
+        // configuration with largest age-off is used
+        MetricAgeOffConfiguration ageOffConfig = MetricAgeOffConfiguration.newFromRequest(request, majcIteratorName);
+        if (hasMinAgeOff) {
+            ageOffConfig = MetricAgeOffConfiguration.maximizeDefaultAgeOff(ageOffConfig, minAgeOffConfig);
+        }
+
+        long currentTime = useExplicitTime ? explicitTimeMillis : System.currentTimeMillis();
+        long ageOffComputed = currentTime - ageOffConfig.computeAgeOff(endName.get());
+        boolean shouldCompact = (ageOffComputed > endOffset.getAsLong() && endName.equals(prevEndName))
+                && (null == filterPrefix || endName.get().startsWith(filterPrefix));
+
+        if (LOG.isDebugEnabled() && logOnly && shouldCompact) {
+            LOG.debug("Tablet will be metric age-off compacted {threshold: {}, endKey: {}, offset: {}, prevKey: {}}",
+                    ageOffComputed, endName, endOffset.getAsLong(), prevEndName);
+        } else if (LOG.isTraceEnabled()) {
+            LOG.trace("Tablet check metric compaction {threshold: {}, endKey: {}, offset: {}, prevKey: {}, result: {}}",
+                    ageOffComputed, endName, endOffset.getAsLong(), prevEndName, shouldCompact);
+        }
+
+        return shouldCompact && !logOnly;
+    }
+
+    @Override
+    public CompactionPlan getCompactionPlan(MajorCompactionRequest request) {
+        CompactionPlan plan = null;
+        if (shouldCompact(request)) {
+            plan = new CompactionPlan();
+            for (FileRef ref : request.getFiles().keySet()) {
+                plan.inputFiles.add(ref);
+            }
+
+            if (LOG.isDebugEnabled()) {
+                LOG.debug("Tablet selected for age-off: " + TabletRowAdapter.toDebugOutput(request.getTabletId()));
+            }
+        }
+
+        return plan;
+    }
+
+    void setExplicitTime(long timeMillis) {
+        explicitTimeMillis = timeMillis;
+        useExplicitTime = true;
+    }
+
+    private static class MetricAgeOffConfiguration {
+
+        private final PatriciaTrie<Long> ageoffs;
+        private final long defaultAgeOff;
+
+        private MetricAgeOffConfiguration(PatriciaTrie<Long> ageoffs, long defaultAgeOff) {
+            this.ageoffs = ageoffs;
+            this.defaultAgeOff = defaultAgeOff;
+        }
+
+        public static MetricAgeOffConfiguration maximizeDefaultAgeOff(MetricAgeOffConfiguration x,
+                MetricAgeOffConfiguration y) {
+            return (x.defaultAgeOff > y.defaultAgeOff) ? x : y;
+        }
+
+        public static MetricAgeOffConfiguration newFromDefaultingMinimum(long minAgeOff) {
+            return new MetricAgeOffConfiguration(new PatriciaTrie<>(), minAgeOff);
+        }
+
+        public static MetricAgeOffConfiguration newFromRequest(MajorCompactionRequest request,
+                String majcIteratorName) {
+            Configuration config = new MapConfiguration(request.getTableProperties());
+            String majcIteratorKey = Property.TABLE_ITERATOR_MAJC_PREFIX.getKey() + majcIteratorName;
+
+            if (LOG.isTraceEnabled()) {
+                LOG.trace("Using key lookup for iterator: {}", majcIteratorKey);
+            }
+
+            Configuration configAgeOff = config.subset((majcIteratorKey + ".opt"));
+            if (null == configAgeOff.getString(DEFAULT_AGEOFF_KEY)) {
+                throw new IllegalArgumentException(
+                        DEFAULT_AGEOFF_KEY_SUFFIX + " must be configured for  " + majcIteratorKey);
+            }
+
+            PatriciaTrie<Long> ageoffs = new PatriciaTrie<>();
+            long configureMinAgeOff = Long.MAX_VALUE;
+            long configureMaxAgeOff = Long.MIN_VALUE;
+            @SuppressWarnings("unchecked")
+            Iterator<String> keys = configAgeOff.getKeys();
+            while (keys.hasNext()) {
+                String k = keys.next();
+                String v = configAgeOff.getString(k);
+                if (k.startsWith((AGE_OFF_PREFIX))) {
+                    String name = k.substring(AGE_OFF_PREFIX.length());
+                    if (LOG.isTraceEnabled()) {
+                        LOG.trace("Adding {} to Trie with value {}", name, Long.parseLong(v));
+                    }
+                    long ageoff = Long.parseLong(v);
+                    configureMinAgeOff = Math.min(configureMinAgeOff, ageoff);
+                    configureMaxAgeOff = Math.max(configureMaxAgeOff, ageoff);
+                    ageoffs.put(name, ageoff);
+                }
+            }
+            long defaultAgeOff = ageoffs.get(DEFAULT_AGEOFF_KEY_SUFFIX);
+            return new MetricAgeOffConfiguration(ageoffs, defaultAgeOff);
+        }
+
+        public long computeAgeOff(String metricName) {
+            return ageoffs.getOrDefault(metricName, defaultAgeOff);
+        }
+    }
+}

server/src/main/java/timely/store/compaction/TabletRowAdapter.java

@@ -0,0 +1,111 @@
+package timely.store.compaction;
+
+import java.time.Instant;
+import java.time.LocalDateTime;
+import java.time.ZoneId;
+import java.util.Arrays;
+import java.util.Optional;
+import java.util.OptionalLong;
+import java.util.StringJoiner;
+
+import org.apache.accumulo.core.client.lexicoder.Lexicoder;
+import org.apache.accumulo.core.client.lexicoder.LongLexicoder;
+import org.apache.accumulo.core.client.lexicoder.StringLexicoder;
+import org.apache.accumulo.core.client.lexicoder.impl.ByteUtils;
+import org.apache.accumulo.core.data.TabletId;
+import org.apache.commons.codec.binary.Hex;
+import org.apache.commons.lang3.ArrayUtils;
+import org.apache.hadoop.io.Text;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class TabletRowAdapter {
+
+    private static final Logger LOG = LoggerFactory.getLogger(TabletRowAdapter.class);
+    private static final Lexicoder<String> PREFIX_DECODER = new StringLexicoder();
+    private static final Lexicoder<Long> OFFSET_DECODER = new LongLexicoder();
+
+    private static final byte DELIMETER = 0x00;
+    private static final int MIN_OFFSET_LEN = 9;
+
+    public static OptionalLong decodeRowOffset(Text row) {
+        OptionalLong offset = OptionalLong.empty();
+        byte[] bytes = row.getBytes();
+        int delimidx = findRowDelimiterIndex(bytes);
+        if (delimidx > 0) {
+            try {
+                byte[] buffer = Arrays.copyOfRange(bytes, delimidx + 1, bytes.length);
+                buffer = ByteUtils.unescape(buffer);
+
+                // key might not have all bytes for long decoding
+                // ensure of sufficient length
+                int extendBy = (MIN_OFFSET_LEN - (buffer.length));
+                if (extendBy > 0) {
+                    buffer = ArrayUtils.addAll(buffer, new byte[extendBy]);
+                }
+
+                long decodedOffset = OFFSET_DECODER.decode(buffer);
+                if (decodedOffset > 0) {
+                    offset = OptionalLong.of(decodedOffset);
+                } else if (LOG.isTraceEnabled()) {
+                    Optional<String> prefix = decodeRowPrefix(row);
+                    LOG.trace(
+                            "Delimiter identified but offset could not parse { prefix: {}, byte-len: {}, "
+                                    + "offset-len: {}, offset-bytes: {}, row-bytes: {} }",
+                            prefix, bytes.length, buffer.length, Hex.encodeHex(buffer), Hex.encodeHex(bytes));
+                }
+            } catch (IllegalArgumentException e) {
+                if (LOG.isTraceEnabled()) {
+                    LOG.trace("Unable to parse offset: " + e.getMessage(), e);
+                }
+            }
+        }
+
+        return offset;
+    }
+
+    public static Optional<String> decodeRowPrefix(Text row) {
+        byte[] bytes = row.getBytes();
+        int delimidx = findRowDelimiterIndex(bytes);
+        byte[] prefixBytes = (delimidx >= 0) ? Arrays.copyOfRange(bytes, 0, delimidx) : bytes;
+        Optional<String> tabletName = Optional.empty();
+        try {
+            tabletName = Optional.of(PREFIX_DECODER.decode(ByteUtils.unescape(prefixBytes)));
+        } catch (IllegalArgumentException e) {
+            if (LOG.isTraceEnabled()) {
+                LOG.trace("Unable to parse offset: " + e.getMessage(), e);
+            }
+        }
+
+        return tabletName;
+    }
+
+    public static String toDebugOutput(TabletId tid) {
+        StringJoiner joiner = new StringJoiner(", ", "[", "]");
+        Text endRow = tid.getEndRow();
+        if (null != endRow) {
+            Optional<String> prefix = decodeRowPrefix(endRow);
+            OptionalLong offset = decodeRowOffset(endRow);
+            prefix.ifPresent(s -> joiner.add("prefix: " + s));
+            if (offset.isPresent()) {
+                LocalDateTime date = Instant.ofEpochMilli(offset.getAsLong()).atZone(ZoneId.of("UTC"))
+                        .toLocalDateTime();
+                joiner.add("date: " + date.toString());
+                joiner.add("millis: " + offset.getAsLong());
+            }
+        }
+
+        return joiner.toString();
+    }
+
+    private static int findRowDelimiterIndex(byte[] bytes) {
+        int delimidx = -1;
+        for (int i = bytes.length - 1; i >= 0; i--) {
+            if (bytes[i] == DELIMETER) {
+                delimidx = i;
+                break;
+            }
+        }
+        return delimidx;
+    }
+}

server/src/main/java/timely/store/compaction/TieredCompactionStrategy.java

@@ -0,0 +1,158 @@
+package timely.store.compaction;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader;
+import org.apache.accumulo.tserver.compaction.CompactionPlan;
+import org.apache.accumulo.tserver.compaction.CompactionStrategy;
+import org.apache.accumulo.tserver.compaction.MajorCompactionRequest;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class TieredCompactionStrategy extends CompactionStrategy {
+
+    public static final String TIERED_PREFIX = "tier.";
+    public static final String TIERED_CLASS = "class";
+    public static final String TIERED_OPTS = "opts";
+
+    protected final Collection<TieredCompactorSupplier> compactors;
+
+    public TieredCompactionStrategy() {
+        compactors = new ArrayList<>();
+    }
+
+    public Collection<TieredCompactorSupplier> getCompactors() {
+        return Collections.unmodifiableCollection(compactors);
+    }
+
+    @Override
+    public void init(Map<String, String> config) {
+        // example:
+        // <table-compaction-prefix>.tiered.0.class=<majcompaction-class>
+        // <table-compaction-prefix>.tiered.0.opts.key=value
+        // <table-compaction-prefix>.tiered.1.class=<majcompaction-class>
+        // <table-compaction-prefix>.tiered.1.opts.key_a=value
+        // <table-compaction-prefix>.tiered.1.opts.key_b=value
+        //
+        // the key=value pairs will be sent to the specific tiered compaction
+        // strategy as arguments
+
+        compactors.clear();
+        createSuppliers(config);
+    }
+
+    @Override
+    public void gatherInformation(MajorCompactionRequest request) throws IOException {
+        for (TieredCompactorSupplier compactor : compactors) {
+            CompactionStrategy strategy = compactor.get(request.getTableProperties());
+            strategy.gatherInformation(request);
+        }
+    }
+
+    @Override
+    public boolean shouldCompact(MajorCompactionRequest request) throws IOException {
+        boolean shouldCompact = false;
+        for (TieredCompactorSupplier supplier : compactors) {
+            CompactionStrategy strategy = supplier.get(request.getTableProperties());
+            shouldCompact = strategy.shouldCompact(request);
+            if (shouldCompact) {
+                break;
+            }
+        }
+
+        return shouldCompact;
+    }
+
+    @Override
+    public CompactionPlan getCompactionPlan(MajorCompactionRequest request) throws IOException {
+        CompactionPlan plan = null;
+        Iterator<TieredCompactorSupplier> iter = compactors.iterator();
+        while ((null == plan || plan.inputFiles.isEmpty()) && iter.hasNext()) {
+            CompactionStrategy strategy = iter.next().get(request.getTableProperties());
+            plan = strategy.getCompactionPlan(request);
+        }
+
+        return plan;
+    }
+
+    protected void createSuppliers(Map<String, String> config) {
+        int tier = 0;
+        boolean continueLoop = true;
+        while (continueLoop) {
+            String clazzKey = TIERED_PREFIX + tier + "." + TIERED_CLASS;
+            String clazzName = config.get(clazzKey);
+            if (null != clazzName) {
+                compactors.add(new TieredConfiguredSupplier(clazzName, tier, config));
+                tier++;
+            } else {
+                continueLoop = false;
+            }
+        }
+    }
+
+    static class TieredConfiguredSupplier implements TieredCompactorSupplier {
+
+        private static final Logger LOG = LoggerFactory.getLogger(TieredConfiguredSupplier.class);
+
+        private final String clazzName;
+        private final int tier;
+        private final Map<String, String> initConfig;
+
+        private CompactionStrategy strategy;
+        private Map<String, String> options;
+
+        TieredConfiguredSupplier(String clazzName, int tier, Map<String, String> initConfig) {
+            this.strategy = null;
+            this.clazzName = clazzName;
+            this.tier = tier;
+            this.initConfig = initConfig;
+        }
+
+        public Map<String, String> options() {
+            if (null == options) {
+                options = new HashMap<>();
+                String tierCompare = Integer.toString(tier);
+                int tierIdx = TIERED_PREFIX.length();
+                int optsIdx = TIERED_PREFIX.length() + tierCompare.length() + 1;
+                for (Map.Entry<String, String> entry : initConfig.entrySet()) {
+                    String key = entry.getKey();
+                    if (key.regionMatches(tierIdx, tierCompare, 0, tierCompare.length())
+                            && key.regionMatches(optsIdx, TIERED_OPTS, 0, TIERED_OPTS.length())) {
+                        options.put(key.substring(optsIdx + TIERED_OPTS.length() + 1), entry.getValue());
+                    }
+                }
+            }
+
+            return Collections.unmodifiableMap(options);
+        }
+
+        public CompactionStrategy get(Map<String, String> tableProperties) {
+            if (null == strategy) {
+                try {
+                    String tableContext = tableProperties.get(Property.TABLE_CLASSPATH.getKey());
+                    Class<? extends CompactionStrategy> clazz;
+                    if (null != tableContext && !tableContext.equals("")) {
+                        clazz = AccumuloVFSClassLoader.getContextManager().loadClass(tableContext, clazzName,
+                                CompactionStrategy.class);
+                    } else {
+                        clazz = AccumuloVFSClassLoader.loadClass(clazzName, CompactionStrategy.class);
+                    }
+                    strategy = clazz.newInstance();
+                    strategy.init(options());
+                } catch (Exception e) {
+                    LOG.debug("Failed to load class for tiered compaction: " + e.toString());
+                    throw new RuntimeException(e);
+                }
+            }
+
+            return strategy;
+        }
+    }
+}

server/src/main/java/timely/store/compaction/TieredCompactorSupplier.java

@@ -0,0 +1,12 @@
+package timely.store.compaction;
+
+import java.util.Map;
+
+import org.apache.accumulo.tserver.compaction.CompactionStrategy;
+
+public interface TieredCompactorSupplier {
+
+    Map<String, String> options();
+
+    CompactionStrategy get(Map<String, String> tableProperties);
+}

server/src/main/java/timely/store/compaction/util/MetadataAccumulator.java

@@ -0,0 +1,130 @@
+package timely.store.compaction.util;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Optional;
+import java.util.OptionalLong;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.hadoop.io.Text;
+import timely.store.compaction.TabletRowAdapter;
+
+public class MetadataAccumulator {
+
+    private final List<Entry> list;
+    private Entry currentEntry;
+
+    public MetadataAccumulator() {
+        this.list = new LinkedList<>();
+    }
+
+    public Entry checkEntryState(Text tablet) {
+        // entry may be null on initial invocation or after the pushOrDiscard operation
+        // check to see if the tablet has changed, if so then
+        // push or discard the current state
+        if (currentEntry != null && !tablet.equals(currentEntry.getTablet())) {
+            pushOrDiscardEntry();
+        }
+        if (currentEntry == null) {
+            currentEntry = new Entry(tablet);
+        }
+
+        return currentEntry;
+    }
+
+    public Collection<Entry> getEntries() {
+        return Collections.unmodifiableCollection(list);
+    }
+
+    public void ensureState() {
+        pushOrDiscardEntry();
+    }
+
+    private void pushOrDiscardEntry() {
+        if (currentEntry != null && currentEntry.hasFiles()) {
+            list.add(currentEntry);
+        }
+        currentEntry = null;
+    }
+
+    public static class Entry {
+
+        private final Text tablet;
+        private final String tabletPrefix;
+        private final boolean hasTabletOffset;
+        private final long tabletOffset;
+        private Text tabletPrev;
+        private long milliseconds;
+        private long totalBytes;
+        private int totalFiles;
+
+        public Entry(Text tablet) {
+            this.tablet = tablet;
+            Optional<String> prefixOptional = TabletRowAdapter.decodeRowPrefix(tablet);
+            OptionalLong offsetOptional = TabletRowAdapter.decodeRowOffset(tablet);
+            tabletPrefix = prefixOptional.orElseGet(tablet::toString);
+            if (offsetOptional.isPresent()) {
+                hasTabletOffset = true;
+                tabletOffset = offsetOptional.getAsLong();
+            } else {
+                hasTabletOffset = false;
+                tabletOffset = -1;
+            }
+        }
+
+        public String getTabletPrefix() {
+            return tabletPrefix;
+        }
+
+        public Text getTablet() {
+            return tablet;
+        }
+
+        public Text getTabletPrev() {
+            return tabletPrev;
+        }
+
+        public long getMilliseconds() {
+            return milliseconds;
+        }
+
+        public long getTotalFileBytes() {
+            return totalBytes;
+        }
+
+        public int getTotalFiles() {
+            return totalFiles;
+        }
+
+        public OptionalLong getTabletOffset() {
+            return hasTabletOffset ? OptionalLong.of(tabletOffset) : OptionalLong.empty();
+        }
+
+        public boolean hasFiles() {
+            return totalFiles > 0;
+        }
+
+        public void addFile(long size) {
+            totalFiles++;
+            totalBytes += size;
+        }
+
+        public void setTablePrev(Text prev) {
+            tabletPrev = prev;
+        }
+
+        public void setMilliseconds(long val) {
+            milliseconds = val;
+        }
+
+        @Override
+        public String toString() {
+            return String.format("metadata.entry {tablet: %s, millis: %d, offset-ms: %d, age-days: %d, files %d}",
+                    tablet, milliseconds, tabletOffset,
+                    (hasTabletOffset ? TimeUnit.MILLISECONDS.toDays(System.currentTimeMillis() - tabletOffset) : -1),
+                    totalFiles);
+        }
+    }
+}

server/src/main/java/timely/store/compaction/util/TabletMetadataConsole.java

@@ -0,0 +1,40 @@
+package timely.store.compaction.util;
+
+import java.util.HashMap;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.accumulo.core.client.ClientConfiguration;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.ZooKeeperInstance;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.springframework.boot.Banner;
+import org.springframework.boot.WebApplicationType;
+import org.springframework.boot.builder.SpringApplicationBuilder;
+import org.springframework.context.ConfigurableApplicationContext;
+import timely.configuration.Accumulo;
+import timely.configuration.Configuration;
+import timely.configuration.SpringBootstrap;
+
+public class TabletMetadataConsole {
+
+    public static void main(String[] args) throws Exception {
+        try (ConfigurableApplicationContext ctx = new SpringApplicationBuilder(SpringBootstrap.class)
+                .bannerMode(Banner.Mode.OFF).web(WebApplicationType.NONE).run(args)) {
+            Configuration conf = ctx.getBean(Configuration.class);
+            HashMap<String, String> apacheConf = new HashMap<>();
+            Accumulo accumuloConf = conf.getAccumulo();
+            apacheConf.put("instance.name", accumuloConf.getInstanceName());
+            apacheConf.put("instance.zookeeper.host", accumuloConf.getZookeepers());
+            ClientConfiguration aconf = ClientConfiguration.fromMap(apacheConf);
+            Instance instance = new ZooKeeperInstance(aconf);
+            Connector con = instance.getConnector(accumuloConf.getUsername(),
+                    new PasswordToken(accumuloConf.getPassword()));
+
+            TabletMetadataQuery query = new TabletMetadataQuery(con, conf.getMetricsTable());
+            TabletMetadataView view = query.run();
+
+            System.out.println(view.toText(TimeUnit.DAYS));
+        }
+    }
+}

server/src/main/java/timely/store/compaction/util/TabletMetadataFormatter.java

@@ -0,0 +1,31 @@
+package timely.store.compaction.util;
+
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.util.format.AggregatingFormatter;
+
+public class TabletMetadataFormatter extends AggregatingFormatter {
+
+    private final TabletMetadataView view;
+
+    public TabletMetadataFormatter() {
+        this(new TabletMetadataView());
+    }
+
+    public TabletMetadataFormatter(TabletMetadataView view) {
+        this.view = view;
+    }
+
+    @Override
+    public void aggregateStats(Map.Entry<Key, Value> entry) {
+        view.addEntry(entry);
+    }
+
+    @Override
+    protected String getStats() {
+        return view.toText(TimeUnit.DAYS);
+    }
+}

server/src/main/java/timely/store/compaction/util/TabletMetadataQuery.java

@@ -0,0 +1,44 @@
+package timely.store.compaction.util;
+
+import java.util.Map;
+
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.metadata.MetadataTable;
+
+public class TabletMetadataQuery {
+
+    private final Connector connector;
+    private final String tableName;
+
+    public TabletMetadataQuery(Connector connector, String tableName) {
+        this.connector = connector;
+        this.tableName = tableName;
+    }
+
+    public TabletMetadataView run() throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
+        TabletMetadataView view = new TabletMetadataView();
+        Map<String, String> tableMap = connector.tableOperations().tableIdMap();
+        String tableId = tableMap.get(tableName);
+        if (null == tableId) {
+            throw new IllegalStateException("Unable to find " + MetadataTable.NAME);
+        }
+
+        try (Scanner s = connector.createScanner(MetadataTable.NAME,
+                connector.securityOperations().getUserAuthorizations(connector.whoami()))) {
+            // s.setRange(new Range(tableId, true, tableId, true));
+            s.setRange(Range.prefix(tableId));
+            for (Map.Entry<Key, Value> e : s) {
+                view.addEntry(e);
+            }
+        }
+
+        return view;
+    }
+}

server/src/main/java/timely/store/compaction/util/TabletMetadataView.java

@@ -0,0 +1,144 @@
+package timely.store.compaction.util;
+
+import java.util.Collection;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.data.impl.KeyExtent;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.math3.stat.descriptive.StatisticalSummary;
+import org.apache.hadoop.io.Text;
+
+public class TabletMetadataView {
+
+    private final MetadataAccumulator accumulator;
+
+    private static final int DEFAULT_TOP_COUNT = 25;
+
+    public TabletMetadataView() {
+        accumulator = new MetadataAccumulator();
+    }
+
+    public void addEntry(Collection<Map.Entry<Key, Value>> entries) {
+        for (Map.Entry<Key, Value> entry : entries) {
+            addEntry(entry);
+        }
+    }
+
+    public void addEntry(Map.Entry<Key, Value> entry) {
+        KeyExtent ke = new KeyExtent(entry.getKey().getRow(), (Text) null);
+        if (ke.getEndRow() == null) {
+            return;
+        }
+
+        Text cf = entry.getKey().getColumnFamily();
+        Text cq = entry.getKey().getColumnQualifier();
+        MetadataAccumulator.Entry candidate = accumulator.checkEntryState(ke.getEndRow());
+
+        if (cf.equals(MetadataSchema.TabletsSection.DataFileColumnFamily.NAME)) {
+            String val = entry.getValue().toString();
+            String[] valSplit = val.split(",");
+            if (valSplit.length == 2) {
+                long size = Long.parseLong(valSplit[0]);
+                candidate.addFile(size);
+            }
+        } else if (cf.equals(MetadataSchema.TabletsSection.ServerColumnFamily.NAME)) {
+            if (cq.equals(MetadataSchema.TabletsSection.ServerColumnFamily.TIME_COLUMN.getColumnQualifier())) {
+                String timeVal = entry.getValue().toString();
+                if (timeVal.startsWith("M")) {
+                    long ms = Long.parseLong(timeVal.substring(1));
+                    candidate.setMilliseconds(ms);
+                }
+            }
+        } else if (cf.equals(MetadataSchema.TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.getColumnFamily())
+                && cq.equals(MetadataSchema.TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.getColumnQualifier())) {
+            ke = new KeyExtent(entry.getKey().getRow(), entry.getValue());
+            candidate.setTablePrev(ke.getPrevEndRow());
+        }
+    }
+
+    public Collection<MetadataAccumulator.Entry> getEntries() {
+        accumulator.ensureState();
+        return accumulator.getEntries();
+    }
+
+    public TabletSummary.Builder computeSummary() {
+        return TabletSummary.newBuilder(getEntries());
+    }
+
+    public String toText() {
+        return toText(TimeUnit.MILLISECONDS);
+    }
+
+    public String toText(TimeUnit unit) {
+        if (getEntries().isEmpty()) {
+            return "No results";
+        }
+
+        StringBuilder sb = new StringBuilder();
+        TabletSummary summary = computeSummary().build();
+        Map<TabletStatisticType, StatisticalSummary> tabletAggregate = summary.aggregateSummary();
+        StatisticalSummary tabletSizes = tabletAggregate.get(TabletStatisticType.SIZE);
+        StatisticalSummary tabletTime = tabletAggregate.get(TabletStatisticType.TIME);
+        long tabletMax = unit.convert((long) tabletTime.getMax(), TimeUnit.MILLISECONDS);
+        long tabletMean = unit.convert((long) tabletTime.getMean(), TimeUnit.MILLISECONDS);
+        Collection<TabletStatistic> topOldestMetrics = summary.findTabletPrefixesByOldest(DEFAULT_TOP_COUNT);
+        Collection<TabletStatistic> topSizeMetrics = summary.findTabletPrefixesByLargest(DEFAULT_TOP_COUNT);
+
+        sb.append(String.format("Tablets { prefix: %d, total: %d } %s", summary.totalTabletPrefixes(),
+                summary.totalTablets(), System.lineSeparator()));
+        sb.append(System.lineSeparator());
+        sb.append(String.format("Size: { sum: %s, avg: %s } %s",
+                FileUtils.byteCountToDisplaySize((long) tabletSizes.getSum()),
+                FileUtils.byteCountToDisplaySize((long) tabletSizes.getMean()), System.lineSeparator()));
+
+        sb.append(String.format("Age: { max: %d, avg: %d } %s", tabletMax, tabletMean, System.lineSeparator()));
+
+        sb.append(System.lineSeparator());
+        sb.append(System.lineSeparator());
+
+        sb.append(String.format("Top %d Tablets (by-age) %s", DEFAULT_TOP_COUNT, System.lineSeparator()));
+        sb.append("---");
+        sb.append(System.lineSeparator());
+
+        for (TabletStatistic entry : topOldestMetrics) {
+            sb.append(String.format("%s = { tablets: %d, max-age: %d } %s", entry.getKeyName(), entry.getKeyCount(),
+                    unit.convert(entry.maxAge(), TimeUnit.MILLISECONDS), System.lineSeparator()));
+        }
+
+        sb.append(System.lineSeparator());
+        sb.append(System.lineSeparator());
+
+        sb.append(String.format("Top %d Tablets (by-size) %s", DEFAULT_TOP_COUNT, System.lineSeparator()));
+        sb.append("---");
+        sb.append(System.lineSeparator());
+
+        for (TabletStatistic entry : topSizeMetrics) {
+            long totalSize = (long) entry.getSummary(TabletStatisticType.SIZE).getSum();
+            sb.append(String.format("%s = { tablets: %d, size: %s } %s", entry.getKeyName(), entry.getKeyCount(),
+                    FileUtils.byteCountToDisplaySize(totalSize), System.lineSeparator()));
+        }
+
+        sb.append(System.lineSeparator());
+        sb.append(System.lineSeparator());
+
+        sb.append(String.format("Tablets (by-name) %s", System.lineSeparator()));
+        sb.append("---");
+        sb.append(System.lineSeparator());
+
+        for (TabletStatistic entry : summary.getSummary()) {
+            long totalSize = (long) entry.getSummary(TabletStatisticType.SIZE).getSum();
+            sb.append(String.format("%s = { tablets: %d, size: %s, max-age: %d } %s", entry.getKeyName(),
+                    entry.getKeyCount(), FileUtils.byteCountToDisplaySize(totalSize),
+                    unit.convert(entry.maxAge(), TimeUnit.MILLISECONDS), System.lineSeparator()));
+        }
+
+        sb.append(System.lineSeparator());
+        sb.append(System.lineSeparator());
+
+        return sb.toString();
+    }
+}

server/src/main/java/timely/store/compaction/util/TabletStatistic.java

@@ -0,0 +1,53 @@
+package timely.store.compaction.util;
+
+import java.util.Map;
+
+import org.apache.commons.math3.stat.descriptive.StatisticalSummary;
+import org.apache.commons.math3.stat.descriptive.SummaryStatistics;
+
+public class TabletStatistic {
+
+    private final String keyName;
+    private final int keyCount;
+    private final Map<TabletStatisticType, SummaryStatistics> stats;
+    private final SummaryStatistics timeStat;
+    private final SummaryStatistics sizeStat;
+
+    TabletStatistic(String keyName, int keyCount, Map<TabletStatisticType, SummaryStatistics> stats) {
+        this.keyName = keyName;
+        this.keyCount = keyCount;
+        this.stats = stats;
+
+        timeStat = stats.getOrDefault(TabletStatisticType.TIME, new SummaryStatistics());
+        sizeStat = stats.getOrDefault(TabletStatisticType.SIZE, new SummaryStatistics());
+    }
+
+    public String getKeyName() {
+        return keyName;
+    }
+
+    public int getKeyCount() {
+        return keyCount;
+    }
+
+    public StatisticalSummary getSummary(TabletStatisticType statType) {
+        return this.stats.getOrDefault(statType, new SummaryStatistics());
+    }
+
+    public Map<TabletStatisticType, SummaryStatistics> getSummaryMap() {
+        return stats;
+    }
+
+    public long maxAge() {
+        return (long) timeStat.getMax();
+    }
+
+    public long totalSize() {
+        return (long) sizeStat.getSum();
+    }
+
+    @Override
+    public String toString() {
+        return String.format("key: %s, count: %d, age: %d, size %d", keyName, keyCount, maxAge(), totalSize());
+    }
+}

server/src/main/java/timely/store/compaction/util/TabletStatisticType.java

@@ -0,0 +1,5 @@
+package timely.store.compaction.util;
+
+public enum TabletStatisticType {
+    SIZE, TIME
+}

server/src/main/java/timely/store/compaction/util/TabletSummary.java

@@ -0,0 +1,155 @@
+package timely.store.compaction.util;
+
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Optional;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import org.apache.commons.math3.stat.descriptive.AggregateSummaryStatistics;
+import org.apache.commons.math3.stat.descriptive.StatisticalSummary;
+import org.apache.commons.math3.stat.descriptive.SummaryStatistics;
+import timely.store.compaction.TabletRowAdapter;
+
+public class TabletSummary {
+
+    private final Collection<TabletStatistic> tabletStats;
+
+    private TabletSummary(Collection<TabletStatistic> tabletStats) {
+        this.tabletStats = tabletStats;
+    }
+
+    public static Builder newBuilder(Collection<MetadataAccumulator.Entry> entries) {
+        return new Builder(entries);
+    }
+
+    public Map<TabletStatisticType, StatisticalSummary> aggregateSummary() {
+        // @formatter:off
+        return tabletStats.stream()
+                .flatMap(m -> m.getSummaryMap().entrySet().stream())
+                .collect(Collectors.groupingBy(Map.Entry::getKey,
+                        Collectors.mapping(Map.Entry::getValue, Collectors.toList())))
+                .entrySet().stream()
+                .collect(Collectors.toMap(Map.Entry::getKey, v -> AggregateSummaryStatistics.aggregate(v.getValue())));
+    }
+
+    public long totalTabletPrefixes() {
+        return tabletStats.size();
+    }
+
+    public long totalTablets() {
+        // @formatter:off
+        return tabletStats.stream()
+                .mapToLong(TabletStatistic::getKeyCount)
+                .sum();
+    }
+
+    public Collection<TabletStatistic> findTabletPrefixesByOldest(int count) {
+        // @formatter:off
+        return tabletStats.stream()
+                .sorted(Comparator.comparingLong(TabletStatistic::maxAge)
+                        .reversed()
+                        .thenComparing(TabletStatistic::getKeyName))
+                .limit(count)
+                .collect(Collectors.toList());
+    }
+
+    public Collection<TabletStatistic> findTabletPrefixesByLargest(int count) {
+        // @formatter:off
+        return tabletStats.stream()
+                .sorted(Comparator.comparingLong(TabletStatistic::totalSize)
+                        .reversed()
+                        .thenComparing(TabletStatistic::getKeyName))
+                .limit(count)
+                .collect(Collectors.toList());
+    }
+
+    public Collection<TabletStatistic> getSummary() {
+        return tabletStats;
+    }
+
+    public static class Builder {
+        private final Collection<MetadataAccumulator.Entry> entries;
+        private long explictTimeMillis;
+        private boolean useExplicitTime;
+        private boolean disableTabletRowCheckFilter;
+
+        public Builder(Collection<MetadataAccumulator.Entry> entries) {
+            this.entries = entries;
+        }
+
+        public Builder currentTime(long millis) {
+            explictTimeMillis = millis;
+            useExplicitTime = true;
+            return this;
+        }
+
+        public Builder disableTabletRowCheckFilter() {
+            disableTabletRowCheckFilter = true;
+            return this;
+        }
+
+        public TabletSummary build() {
+            Stream<MetadataAccumulator.Entry> stream = entries.stream();
+            if (!disableTabletRowCheckFilter) {
+                stream = stream.filter(Builder::tabletEndAnPrevAreSame);
+            }
+
+            // @formatter:off
+            Collection<TabletStatistic> tabletStats = stream
+                    .collect(Collectors.groupingBy(MetadataAccumulator.Entry::getTabletPrefix)).entrySet()
+                    .stream()
+                    .map(m -> computeStatistic(m.getValue(), m.getKey()))
+                    .sorted(Comparator.comparing(TabletStatistic::getKeyName))
+                    .collect(Collectors.toList());
+
+            // @formatter:on
+            return new TabletSummary(tabletStats);
+        }
+
+        private TabletStatistic computeStatistic(Collection<MetadataAccumulator.Entry> entries, String keyName) {
+            Map<TabletStatisticType, SummaryStatistics> stats = new HashMap<>(TabletStatisticType.values().length);
+            int maxSize = entries.size();
+            long timeMillis = useExplicitTime ? explictTimeMillis : System.currentTimeMillis();
+            for (TabletStatisticType statType : TabletStatisticType.values()) {
+                SummaryStatistics summaryStat = new SummaryStatistics();
+                Iterator<MetadataAccumulator.Entry> iterator = entries.iterator();
+                int idx = 0;
+                while (iterator.hasNext() && idx < maxSize) {
+                    MetadataAccumulator.Entry e = iterator.next();
+                    double d;
+                    switch (statType) {
+                        case SIZE:
+                            d = e.getTotalFileBytes();
+                            break;
+                        case TIME:
+                            // check the row key offset first; this was done to support
+                            // tests so that they age could function from the row-key
+                            // the tablets that do not have a key will use the modification time
+                            d = timeMillis - e.getTabletOffset().orElse(e.getMilliseconds());
+                            break;
+                        default:
+                            throw new IllegalStateException("Undefined switch case: " + statType.name());
+                    }
+                    summaryStat.addValue(d);
+                    idx++;
+                }
+                stats.put(statType, summaryStat);
+            }
+
+            return new TabletStatistic(keyName, entries.size(), stats);
+        }
+
+        private static boolean tabletEndAnPrevAreSame(MetadataAccumulator.Entry entry) {
+            if ((null == entry.getTablet()) || (null == entry.getTabletPrev())) {
+                return false;
+            }
+            Optional<String> o1 = TabletRowAdapter.decodeRowPrefix(entry.getTablet());
+            Optional<String> o2 = TabletRowAdapter.decodeRowPrefix(entry.getTabletPrev());
+            return o1.equals(o2);
+        }
+    }
+}

server/src/main/java/timely/util/MetaKeySet.java

@@ -41,7 +41,7 @@ public class MetaKeySet extends TreeSet<Key> {
         super(new MetaKeyComparator());
     }
 
-    public List<Mutation> toMutations() {
+    public List<Mutation> toMutations(long ts) {
         List<Mutation> results = new ArrayList<>();
         String prevRow = null;
         Iterator<Key> keys = this.iterator();
@@ -55,7 +55,7 @@ public class MetaKeySet extends TreeSet<Key> {
                 m = new Mutation(next.getRow());
                 prevRow = next.getRow().toString();
             }
-            m.put(next.getColumnFamily(), next.getColumnQualifier(), NULL_VALUE);
+            m.put(next.getColumnFamily(), next.getColumnQualifier(), ts, NULL_VALUE);
         }
         results.add(m);
         return results;

server/src/main/spotbugs/excludes.xml

@@ -13,4 +13,9 @@
     <Method name="getCompressorOutput" />
     <Bug pattern="EI_EXPOSE_REP" />
   </Match>
+  <Match>
+    <!-- Must ignore these everywhere, because of a javac byte code generation bug -->
+    <!-- https://github.com/spotbugs/spotbugs/issues/756 -->
+    <Bug pattern="RCN_REDUNDANT_NULLCHECK_WOULD_HAVE_BEEN_A_NPE" />
+  </Match>
 </FindBugsFilter>

server/src/test/java/timely/store/compaction/MetricCompactionStrategyTest.java

@@ -0,0 +1,290 @@
+package timely.store.compaction;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.Calendar;
+import java.util.GregorianCalendar;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.tserver.compaction.CompactionPlan;
+import org.apache.accumulo.tserver.compaction.CompactionStrategy;
+import org.apache.accumulo.tserver.compaction.MajorCompactionRequest;
+import org.junit.Test;
+import timely.test.CompactionRequestBuilder;
+
+public class MetricCompactionStrategyTest {
+
+    private static final long TIME_MILLIS = new GregorianCalendar(2019, Calendar.JANUARY, 1).getTimeInMillis();
+
+    @Test(expected = IllegalArgumentException.class)
+    public void initThrowsWhenNoDefaultAgeOff() {
+        Map<String, String> config = new HashMap<>();
+        MetricCompactionStrategy strategy = new MetricCompactionStrategy();
+        strategy.init(config);
+        CompactionRequestBuilder builder = new CompactionRequestBuilder()
+                .endKeyMetric("sys.cpu", TimeUnit.DAYS.toMillis(11))
+                .prevEndKeyMetric("sys.cpu", TimeUnit.DAYS.toMillis(12))
+                .file("hdfs://hdfs-name/accumulo/tables/2/default_tablet/C0000ao8.rf", 1, 1)
+                .file("hdfs://hdfs-name/accumulo/tables/2/default_tablet/C0000ao7.rf", 1, 1);
+
+        MajorCompactionRequest request = builder.build();
+        strategy.shouldCompact(request);
+    }
+
+    @Test
+    public void minAgeOffOverridesMajcDefaultWillUseMaximum() throws IOException {
+        Map<String, String> config = new HashMap<>();
+        config.put(MetricCompactionStrategy.MIN_AGEOFF_KEY, Long.toString(TimeUnit.DAYS.toMillis(10)));
+        CompactionStrategy strategy = newStrategy(config);
+        // @formatter:off
+        CompactionRequestBuilder builder = newRequestBuilder(13)
+                .endKeyMetric("sys.cpu", TimeUnit.DAYS.toMillis(11))
+                .prevEndKeyMetric("sys.cpu", TimeUnit.DAYS.toMillis(12))
+                .file("hdfs://hdfs-name/accumulo/tables/2/default_tablet/C0000ao8.rf", 1, 1)
+                .file("hdfs://hdfs-name/accumulo/tables/2/default_tablet/C0000ao7.rf", 1, 1);
+
+        // @formatter:on
+        MajorCompactionRequest request = builder.build();
+        boolean shouldCompact = strategy.shouldCompact(request);
+        CompactionPlan plan = strategy.getCompactionPlan(request);
+
+        assertFalse(shouldCompact);
+        assertNull(plan);
+    }
+
+    @Test
+    public void minAgeOffOverridesMajcDefaultNotCompact() throws IOException {
+        Map<String, String> config = new HashMap<>();
+        config.put(MetricCompactionStrategy.MIN_AGEOFF_KEY, Long.toString(TimeUnit.DAYS.toMillis(13)));
+        CompactionStrategy strategy = newStrategy(config);
+        // @formatter:off
+        CompactionRequestBuilder builder = newRequestBuilder(10)
+                .endKeyMetric("sys.cpu", TimeUnit.DAYS.toMillis(11))
+                .prevEndKeyMetric("sys.cpu", TimeUnit.DAYS.toMillis(12))
+                .file("hdfs://hdfs-name/accumulo/tables/2/default_tablet/C0000ao8.rf", 1, 1)
+                .file("hdfs://hdfs-name/accumulo/tables/2/default_tablet/C0000ao7.rf", 1, 1);
+
+        // @formatter:on
+        MajorCompactionRequest request = builder.build();
+        boolean shouldCompact = strategy.shouldCompact(request);
+        CompactionPlan plan = strategy.getCompactionPlan(request);
+
+        assertFalse(shouldCompact);
+        assertNull(plan);
+    }
+
+    @Test
+    public void compactFilesIfKeyIsPastAgeOff() throws IOException {
+        String f1 = "hdfs://hdfs-name/accumulo/tables/2/default_tablet/C0000ao8.rf";
+        String f2 = "hdfs://hdfs-name/accumulo/tables/2/default_tablet/C0000ao7.rf";
+        String f3 = "hdfs://hdfs-name/accumulo/tables/2/default_tablet/C0000ao6.rf";
+        CompactionStrategy strategy = newStrategy();
+
+        // @formatter:off
+        CompactionRequestBuilder builder = newRequestBuilder(10)
+                .endKeyMetric("sys", TimeUnit.DAYS.toMillis(11))
+                .prevEndKeyMetric("sys", TimeUnit.DAYS.toMillis(12))
+                .file(f1, 12, 1)
+                .file(f2, 11, 1)
+                .file(f3, 16, 1);
+
+        // @formatter:on
+        MajorCompactionRequest request = builder.build();
+        boolean shouldCompact = strategy.shouldCompact(request);
+        CompactionPlan plan = strategy.getCompactionPlan(request);
+
+        assertTrue(shouldCompact);
+        assertNotNull(plan);
+
+        assertEquals(3, plan.inputFiles.size());
+        assertEquals(f1, plan.inputFiles.get(0).path().toString());
+        assertEquals(f2, plan.inputFiles.get(1).path().toString());
+        assertEquals(f3, plan.inputFiles.get(2).path().toString());
+    }
+
+    @Test
+    public void noCompactionIfPrevKeyIsDifferent() throws IOException {
+        CompactionStrategy strategy = newStrategy();
+        // @formatter:off
+        CompactionRequestBuilder builder = newRequestBuilder(10)
+                .endKeyMetric("sys.mem", TimeUnit.DAYS.toMillis(11))
+                .prevEndKeyMetric("sys.cpu", TimeUnit.DAYS.toMillis(12))
+                .file("hdfs://hdfs-name/accumulo/tables/2/default_tablet/C0000ao8.rf", 1, 1)
+                .file("hdfs://hdfs-name/accumulo/tables/2/default_tablet/C0000ao7.rf", 1, 1);
+
+        // @formatter:on
+        MajorCompactionRequest request = builder.build();
+        boolean shouldCompact = strategy.shouldCompact(request);
+        CompactionPlan plan = strategy.getCompactionPlan(request);
+
+        assertFalse(shouldCompact);
+        assertNull(plan);
+    }
+
+    @Test
+    public void noCompactionIfPrevKeyIsNull() throws IOException {
+        CompactionStrategy strategy = newStrategy();
+        // @formatter:off
+        CompactionRequestBuilder builder = newRequestBuilder(10)
+                .endKeyMetric("sys", TimeUnit.DAYS.toMillis(11))
+                .file("hdfs://hdfs-name/accumulo/tables/2/default_tablet/C0000ao8.rf", 1, 1)
+                .file("hdfs://hdfs-name/accumulo/tables/2/default_tablet/C0000ao7.rf", 1, 1);
+
+        // @formatter:on
+        MajorCompactionRequest request = builder.build();
+        boolean shouldCompact = strategy.shouldCompact(request);
+        CompactionPlan plan = strategy.getCompactionPlan(request);
+
+        assertFalse(shouldCompact);
+        assertNull(plan);
+    }
+
+    @Test
+    public void noCompactionWhenExceedsButDoesNotHaveFiles() throws IOException {
+        CompactionStrategy strategy = newStrategy();
+        // @formatter:off
+        CompactionRequestBuilder builder = newRequestBuilder(10)
+                .endKeyMetric("sys", TimeUnit.DAYS.toMillis(11))
+                .prevEndKeyMetric("sys", TimeUnit.DAYS.toMillis(12));
+
+        // @formatter:on
+        MajorCompactionRequest request = builder.build();
+        boolean shouldCompact = strategy.shouldCompact(request);
+        CompactionPlan plan = strategy.getCompactionPlan(request);
+
+        assertFalse(shouldCompact);
+        assertNull(plan);
+    }
+
+    @Test
+    public void noCompactionIfKeyIsNotOffset() throws IOException {
+        CompactionStrategy strategy = newStrategy();
+        // @formatter:off
+        CompactionRequestBuilder builder = newRequestBuilder(10)
+                .endKeyMetricNonOffset("sys.mem", null)
+                .prevEndKeyMetricNonOffset("sys.cpu", null)
+                .file("hdfs://hdfs-name/accumulo/tables/2/default_tablet/C0000ao8.rf", 1, 1)
+                .file("hdfs://hdfs-name/accumulo/tables/2/default_tablet/C0000ao7.rf", 1, 1);
+
+        // @formatter:on
+        MajorCompactionRequest request = builder.build();
+        boolean shouldCompact = strategy.shouldCompact(request);
+        CompactionPlan plan = strategy.getCompactionPlan(request);
+
+        assertFalse(shouldCompact);
+        assertNull(plan);
+    }
+
+    @Test
+    public void noCompactionIfKeyIsWrongOffset() throws IOException {
+        CompactionStrategy strategy = newStrategy();
+        // @formatter:off
+        CompactionRequestBuilder builder = newRequestBuilder(10)
+                .endKeyMetricNonOffset("sys.cpu", "test2")
+                .prevEndKeyMetricNonOffset("sys.cpu", "test1")
+                .file("hdfs://hdfs-name/accumulo/tables/2/default_tablet/C0000ao8.rf", 1, 1)
+                .file("hdfs://hdfs-name/accumulo/tables/2/default_tablet/C0000ao7.rf", 1, 1);
+
+        // @formatter:on
+        MajorCompactionRequest request = builder.build();
+        boolean shouldCompact = strategy.shouldCompact(request);
+        CompactionPlan plan = strategy.getCompactionPlan(request);
+
+        assertFalse(shouldCompact);
+        assertNull(plan);
+    }
+
+    @Test
+    public void compactWithLogOnlyDoesNotCompact() throws IOException {
+        Map<String, String> defaults = new HashMap<>();
+        defaults.put(MetricCompactionStrategy.LOG_ONLY_KEY, "true");
+        CompactionStrategy strategy = newStrategy(defaults);
+        // @formatter:off
+        CompactionRequestBuilder builder = newRequestBuilder(10)
+                .endKeyMetric("sys.cpu", TimeUnit.DAYS.toMillis(11))
+                .prevEndKeyMetric("sys.cpu", TimeUnit.DAYS.toMillis(12))
+                .file("hdfs://hdfs-name/accumulo/tables/2/default_tablet/C0000ao8.rf", 1, 1)
+                .file("hdfs://hdfs-name/accumulo/tables/2/default_tablet/C0000ao7.rf", 1, 1);
+
+        // @formatter:on
+        MajorCompactionRequest request = builder.build();
+        boolean shouldCompact = strategy.shouldCompact(request);
+        CompactionPlan plan = strategy.getCompactionPlan(request);
+
+        assertFalse(shouldCompact);
+        assertNull(plan);
+    }
+
+    @Test
+    public void compactWithFilterPrefixWillCompact() throws IOException {
+        Map<String, String> defaults = new HashMap<>();
+        defaults.put(MetricCompactionStrategy.FILTER_PREFIX_KEY, "sys.cp");
+        CompactionStrategy strategy = newStrategy(defaults);
+        // @formatter:off
+        CompactionRequestBuilder builder = newRequestBuilder(10)
+                .endKeyMetric("sys.cpu", TimeUnit.DAYS.toMillis(11))
+                .prevEndKeyMetric("sys.cpu", TimeUnit.DAYS.toMillis(12))
+                .file("hdfs://hdfs-name/accumulo/tables/2/default_tablet/C0000ao8.rf", 1, 1)
+                .file("hdfs://hdfs-name/accumulo/tables/2/default_tablet/C0000ao7.rf", 1, 1);
+
+        // @formatter:on
+        MajorCompactionRequest request = builder.build();
+        boolean shouldCompact = strategy.shouldCompact(request);
+        CompactionPlan plan = strategy.getCompactionPlan(request);
+
+        assertTrue(shouldCompact);
+        assertNotNull(plan);
+
+        assertEquals(2, plan.inputFiles.size());
+    }
+
+    @Test
+    public void compactWithoutFilterPrefixWillNotCompact() throws IOException {
+        Map<String, String> defaults = new HashMap<>();
+        defaults.put(MetricCompactionStrategy.FILTER_PREFIX_KEY, "sys.mem");
+        CompactionStrategy strategy = newStrategy(defaults);
+        // @formatter:off
+        CompactionRequestBuilder builder = newRequestBuilder(10)
+                .endKeyMetric("sys.cpu", TimeUnit.DAYS.toMillis(11))
+                .prevEndKeyMetric("sys.cpu", TimeUnit.DAYS.toMillis(12))
+                .file("hdfs://hdfs-name/accumulo/tables/2/default_tablet/C0000ao8.rf", 1, 1)
+                .file("hdfs://hdfs-name/accumulo/tables/2/default_tablet/C0000ao7.rf", 1, 1);
+
+        // @formatter:on
+        MajorCompactionRequest request = builder.build();
+        boolean shouldCompact = strategy.shouldCompact(request);
+        CompactionPlan plan = strategy.getCompactionPlan(request);
+
+        assertFalse(shouldCompact);
+        assertNull(plan);
+    }
+
+    private static CompactionRequestBuilder newRequestBuilder(int defaultAgeOffDays) {
+        return new CompactionRequestBuilder(TIME_MILLIS).tableProperties(
+                Property.TABLE_ITERATOR_MAJC_PREFIX.getKey() + "ageoffmetrics.opt.ageoff.default",
+                Long.toString(TimeUnit.DAYS.toMillis(defaultAgeOffDays)));
+    }
+
+    private static MetricCompactionStrategy newStrategy() {
+        return newStrategy(new TreeMap<>());
+    }
+
+    private static MetricCompactionStrategy newStrategy(Map<String, String> config) {
+        if (!config.containsKey(MetricCompactionStrategy.LOG_ONLY_KEY)) {
+            config.put(MetricCompactionStrategy.LOG_ONLY_KEY, "false");
+        }
+        MetricCompactionStrategy strategy = new MetricCompactionStrategy();
+        strategy.setExplicitTime(TIME_MILLIS);
+        strategy.init(config);
+        return strategy;
+    }
+}

server/src/test/java/timely/store/compaction/TabletRowAdapterTest.java

@@ -0,0 +1,79 @@
+package timely.store.compaction;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
+
+import java.util.Arrays;
+import java.util.OptionalLong;
+
+import org.apache.accumulo.core.client.lexicoder.Lexicoder;
+import org.apache.accumulo.core.client.lexicoder.LongLexicoder;
+import org.apache.accumulo.core.client.lexicoder.PairLexicoder;
+import org.apache.accumulo.core.client.lexicoder.StringLexicoder;
+import org.apache.accumulo.core.data.TabletId;
+import org.apache.accumulo.core.util.ComparablePair;
+import org.apache.hadoop.io.Text;
+import org.easymock.EasyMock;
+import org.hamcrest.CoreMatchers;
+import org.junit.Test;
+
+public class TabletRowAdapterTest {
+
+    private static final Lexicoder<String> prefixCoder = new StringLexicoder();
+    private static final Lexicoder<Long> offsetCoder = new LongLexicoder();
+    private static final PairLexicoder<String, Long> pairCoder = new PairLexicoder<>(prefixCoder, offsetCoder);
+
+    @Test
+    public void decodePrefix() {
+        ComparablePair<String, Long> p = new ComparablePair<>("test", 1L);
+        byte[] bytes = pairCoder.encode(p);
+        OptionalLong decoded = TabletRowAdapter.decodeRowOffset(new Text(bytes));
+        assertTrue(decoded.isPresent());
+    }
+
+    @Test
+    public void decodeOffsetOnly() {
+        ComparablePair<String, Long> p = new ComparablePair<>("test", 1L);
+        byte[] bytes = pairCoder.encode(p);
+        OptionalLong decoded = TabletRowAdapter.decodeRowOffset(new Text(bytes));
+        assertTrue(decoded.isPresent());
+        assertEquals(1L, decoded.getAsLong());
+    }
+
+    @Test
+    public void decodeOffsetWithTruncatedOffset() {
+        // offset is truncated on tablet extents
+        long truncateOffset = 1554962382848L;
+        ComparablePair<String, Long> p = new ComparablePair<>("sys.disk.disk_octets", truncateOffset);
+        byte[] bytes = pairCoder.encode(p);
+        bytes = Arrays.copyOf(bytes, bytes.length - 4);
+        OptionalLong decoded = TabletRowAdapter.decodeRowOffset(new Text(bytes));
+        assertTrue(decoded.isPresent());
+        assertEquals(truncateOffset, decoded.getAsLong());
+    }
+
+    @Test
+    public void decodeOffsetWithBadDataWillSet() {
+        ComparablePair<String, String> p = new ComparablePair<>("test", "<");
+        byte[] bytes = new PairLexicoder<>(prefixCoder, prefixCoder).encode(p);
+        OptionalLong decoded = TabletRowAdapter.decodeRowOffset(new Text(bytes));
+        assertFalse(decoded.isPresent());
+    }
+
+    @Test
+    public void debugOutputTest() {
+        long offset = 1554962382848L;
+        ComparablePair<String, Long> p = new ComparablePair<>("sys.disk.disk_octets", offset);
+        byte[] bytes = pairCoder.encode(p);
+        TabletId tid = EasyMock.createMock(TabletId.class);
+        EasyMock.expect(tid.getEndRow()).andReturn(new Text(bytes));
+        EasyMock.replay(tid);
+
+        String debug = TabletRowAdapter.toDebugOutput(tid);
+        assertThat(debug, CoreMatchers.containsString("prefix: sys.disk.disk_octets"));
+        assertThat(debug, CoreMatchers.containsString("date: 2019-04-11T05:59:42.848"));
+        assertThat(debug, CoreMatchers.containsString("millis: 1554962382848"));
+    }
+}

server/src/test/java/timely/store/compaction/TieredCompactionStrategyTest.java

@@ -0,0 +1,236 @@
+package timely.store.compaction;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import com.google.common.collect.Lists;
+import org.apache.accumulo.server.fs.FileRef;
+import org.apache.accumulo.tserver.compaction.CompactionPlan;
+import org.apache.accumulo.tserver.compaction.CompactionStrategy;
+import org.apache.accumulo.tserver.compaction.DefaultCompactionStrategy;
+import org.apache.accumulo.tserver.compaction.MajorCompactionRequest;
+import org.easymock.EasyMock;
+import org.junit.Test;
+import timely.test.CompactionRequestBuilder;
+
+public class TieredCompactionStrategyTest {
+
+    @Test
+    public void initConfiguresWithMultipleTiers() {
+        Map<String, String> config = new HashMap<>();
+        config.put("tier.0.class", DefaultCompactionStrategy.class.getName());
+        config.put("tier.0.opts.arg1", "foo");
+        config.put("tier.0.opts.arg2", "sys");
+        config.put("tier.1.class", DefaultCompactionStrategy.class.getName());
+        config.put("tier.1.opts.arg1", "bar");
+        config.put("tier.2.class", DefaultCompactionStrategy.class.getName());
+
+        TieredCompactionStrategy strategy = new TieredCompactionStrategy();
+        strategy.init(config);
+        List<TieredCompactorSupplier> suppliers = Lists.newArrayList(strategy.getCompactors());
+        assertEquals(3, suppliers.size());
+        assertEquals(2, suppliers.get(0).options().size());
+        assertEquals(1, suppliers.get(1).options().size());
+        assertEquals(0, suppliers.get(2).options().size());
+        assertEquals("foo", suppliers.get(0).options().get("arg1"));
+        assertEquals("sys", suppliers.get(0).options().get("arg2"));
+        assertEquals("bar", suppliers.get(1).options().get("arg1"));
+    }
+
+    @Test
+    public void initConfiguresWithSingleTierNoArguments() {
+        Map<String, String> config = new HashMap<>();
+        config.put("tier.0.class", DefaultCompactionStrategy.class.getName());
+
+        TieredCompactionStrategy strategy = new TieredCompactionStrategy();
+        strategy.init(config);
+        List<TieredCompactorSupplier> suppliers = Lists.newArrayList(strategy.getCompactors());
+        assertEquals(1, suppliers.size());
+        assertEquals(0, suppliers.get(0).options().size());
+    }
+
+    @Test
+    public void gatherInformationAcrossTiers() throws IOException {
+        CompactionStrategy c1 = EasyMock.createMock(CompactionStrategy.class);
+        CompactionStrategy c2 = EasyMock.createMock(CompactionStrategy.class);
+
+        c1.gatherInformation(EasyMock.anyObject());
+        EasyMock.expectLastCall();
+
+        c2.gatherInformation(EasyMock.anyObject());
+        EasyMock.expectLastCall();
+
+        EasyMock.replay(c1, c2);
+
+        TieredCompactionStrategy strategy = newStrategy(c1, c2);
+        CompactionRequestBuilder builder = new CompactionRequestBuilder();
+        MajorCompactionRequest request = builder.build();
+
+        strategy.gatherInformation(request);
+
+        EasyMock.verify(c1, c2);
+    }
+
+    @Test
+    public void shouldCompactStopsAtTier() throws IOException {
+        CompactionStrategy c1 = EasyMock.createMock("c1", CompactionStrategy.class);
+        CompactionStrategy c2 = EasyMock.createMock("c2", CompactionStrategy.class);
+        CompactionStrategy c3 = EasyMock.createMock("c3", CompactionStrategy.class);
+
+        EasyMock.expect(c1.shouldCompact(EasyMock.anyObject())).andReturn(false);
+        EasyMock.expect(c2.shouldCompact(EasyMock.anyObject())).andReturn(true);
+
+        EasyMock.replay(c1, c2, c3);
+
+        TieredCompactionStrategy strategy = newStrategy(c1, c2, c3);
+        CompactionRequestBuilder builder = new CompactionRequestBuilder();
+        MajorCompactionRequest request = builder.build();
+
+        boolean result = strategy.shouldCompact(request);
+
+        assertTrue(result);
+        EasyMock.verify(c1, c2, c3);
+    }
+
+    @Test
+    public void shouldCompactWithSingleTier() throws Exception {
+        CompactionStrategy c1 = EasyMock.createMock("c1", CompactionStrategy.class);
+
+        EasyMock.expect(c1.shouldCompact(EasyMock.anyObject())).andReturn(false);
+        EasyMock.replay(c1);
+
+        TieredCompactionStrategy strategy = newStrategy(c1);
+        CompactionRequestBuilder builder = new CompactionRequestBuilder();
+        MajorCompactionRequest request = builder.build();
+
+        boolean result = strategy.shouldCompact(request);
+
+        assertFalse(result);
+        EasyMock.verify(c1);
+    }
+
+    @Test
+    public void getCompactionPlanSkipsAndStopsAtTier() throws IOException {
+        CompactionPlan p1 = new CompactionPlan();
+        p1.inputFiles.add(new FileRef("hdfs://hdfs-name/accumulo/tables/2/default_tablet/C0000ao8.rf"));
+
+        CompactionStrategy c1 = EasyMock.createMock("c1", CompactionStrategy.class);
+        CompactionStrategy c2 = EasyMock.createMock("c2", CompactionStrategy.class);
+        CompactionStrategy c3 = EasyMock.createMock("c3", CompactionStrategy.class);
+
+        EasyMock.expect(c1.getCompactionPlan(EasyMock.anyObject())).andReturn(null);
+        EasyMock.expect(c2.getCompactionPlan(EasyMock.anyObject())).andReturn(p1);
+
+        EasyMock.replay(c1, c2, c3);
+
+        TieredCompactionStrategy strategy = newStrategy(c1, c2, c3);
+        CompactionRequestBuilder builder = new CompactionRequestBuilder();
+        MajorCompactionRequest request = builder.build();
+
+        CompactionPlan actualPlan = strategy.getCompactionPlan(request);
+
+        assertNotNull(actualPlan);
+        assertSame(p1, actualPlan);
+        EasyMock.verify(c1, c2, c3);
+    }
+
+    @Test
+    public void getCompactionPlanSkipsWhenEmptyInputAndStopsAtTier() throws IOException {
+        CompactionPlan p1 = new CompactionPlan();
+        p1.inputFiles.add(new FileRef("hdfs://hdfs-name/accumulo/tables/2/default_tablet/C0000ao8.rf"));
+
+        CompactionStrategy c1 = EasyMock.createMock("c1", CompactionStrategy.class);
+        CompactionStrategy c2 = EasyMock.createMock("c2", CompactionStrategy.class);
+        CompactionStrategy c3 = EasyMock.createMock("c3", CompactionStrategy.class);
+
+        EasyMock.expect(c1.getCompactionPlan(EasyMock.anyObject())).andReturn(new CompactionPlan());
+        EasyMock.expect(c2.getCompactionPlan(EasyMock.anyObject())).andReturn(p1);
+
+        EasyMock.replay(c1, c2, c3);
+
+        TieredCompactionStrategy strategy = newStrategy(c1, c2, c3);
+        CompactionRequestBuilder builder = new CompactionRequestBuilder();
+        MajorCompactionRequest request = builder.build();
+
+        CompactionPlan actualPlan = strategy.getCompactionPlan(request);
+
+        assertNotNull(actualPlan);
+        assertSame(p1, actualPlan);
+        EasyMock.verify(c1, c2, c3);
+    }
+
+    @Test
+    public void getCompactionPlanWillReturnNullIfNonCompact() throws IOException {
+        CompactionStrategy c1 = EasyMock.createMock("c1", CompactionStrategy.class);
+        CompactionStrategy c2 = EasyMock.createMock("c2", CompactionStrategy.class);
+        CompactionStrategy c3 = EasyMock.createMock("c3", CompactionStrategy.class);
+
+        EasyMock.expect(c1.getCompactionPlan(EasyMock.anyObject())).andReturn(null);
+        EasyMock.expect(c2.getCompactionPlan(EasyMock.anyObject())).andReturn(null);
+        EasyMock.expect(c3.getCompactionPlan(EasyMock.anyObject())).andReturn(null);
+
+        EasyMock.replay(c1, c2, c3);
+
+        TieredCompactionStrategy strategy = newStrategy(c1, c2, c3);
+        CompactionRequestBuilder builder = new CompactionRequestBuilder();
+        MajorCompactionRequest request = builder.build();
+
+        CompactionPlan actualPlan = strategy.getCompactionPlan(request);
+
+        assertNull(actualPlan);
+        EasyMock.verify(c1, c2, c3);
+    }
+
+    private static TieredCompactionStrategy newStrategy(CompactionStrategy... strategies) {
+        Collection<TieredCompactorSupplier> mockSuppliers = new ArrayList<>();
+        for (CompactionStrategy s : strategies) {
+            mockSuppliers.add(new MockSupplier(s));
+        }
+        // @formatter:off
+        TieredCompactionStrategy strategy = EasyMock.partialMockBuilder(TieredCompactionStrategy.class)
+                .withConstructor()
+                .addMockedMethod("createSuppliers")
+                .createMock();
+
+        // @formatter:on
+        strategy.createSuppliers(EasyMock.anyObject());
+        EasyMock.expectLastCall().andAnswer(() -> strategy.compactors.addAll(mockSuppliers));
+
+        EasyMock.replay(strategy);
+
+        strategy.init(Collections.emptyMap());
+
+        return strategy;
+    }
+
+    static class MockSupplier implements TieredCompactorSupplier {
+
+        private final CompactionStrategy strategy;
+
+        MockSupplier(CompactionStrategy strategy) {
+            this.strategy = strategy;
+        }
+
+        @Override
+        public Map<String, String> options() {
+            return Collections.emptyMap();
+        }
+
+        @Override
+        public CompactionStrategy get(Map<String, String> tableProperites) {
+            return strategy;
+        }
+    }
+}

server/src/test/java/timely/store/compaction/util/TabletMetadataFormatterTest.java

@@ -0,0 +1,42 @@
+package timely.store.compaction.util;
+
+import java.util.Collection;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.util.format.FormatterConfig;
+import org.apache.hadoop.io.Text;
+import org.junit.Test;
+import timely.adapter.accumulo.MetricAdapter;
+import timely.test.TestTabletMetadata;
+
+public class TabletMetadataFormatterTest {
+
+    @Test
+    public void simpleTextOutput() {
+        Long a1 = System.currentTimeMillis() - TimeUnit.DAYS.toMillis(30);
+        Long a2 = System.currentTimeMillis() - TimeUnit.DAYS.toMillis(29);
+        Text r1 = new Text(MetricAdapter.encodeRowKey("sys.cpu", a1));
+        Text r2 = new Text(MetricAdapter.encodeRowKey("sys.cpu", a2));
+
+        // @formatter:off
+        Collection<Map.Entry<Key, Value>> metadata = new TestTabletMetadata("2")
+                .file(r1, "hdfs://hdfs-name/accumulo/tables/2/default_tablet/C0000ao8.rf", "800,3")
+                .file(r1, "hdfs://hdfs-name/accumulo/tables/2/default_tablet/C0000ao9.rf", "431,1")
+                .prev(r1, null)
+                .time(r1, a1)
+                .file(r2, "hdfs://hdfs-name/accumulo/tables/2/default_tablet/C0000ao8.rf", "800,3")
+                .file(r2, "hdfs://hdfs-name/accumulo/tables/2/default_tablet/C0000ao9.rf", "431,1")
+                .file(r2, "hdfs://hdfs-name/accumulo/tables/2/default_tablet/C0000ao0.rf", "451,2")
+                .time(r2, a2)
+                .prev(r2, r1)
+                .entries();
+
+        // @formatter:on
+        TabletMetadataFormatter f = new TabletMetadataFormatter();
+        f.initialize(metadata, new FormatterConfig());
+        System.out.println(f.next());
+    }
+}

server/src/test/java/timely/store/compaction/util/TabletMetadataViewTest.java

@@ -0,0 +1,106 @@
+package timely.store.compaction.util;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+import com.google.common.collect.Lists;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.hadoop.io.Text;
+import org.junit.Test;
+import timely.adapter.accumulo.MetricAdapter;
+import timely.test.TestTabletMetadata;
+
+public class TabletMetadataViewTest {
+
+    @Test
+    public void tabletAccumulate() {
+        Text r1 = new Text(MetricAdapter.encodeRowKey("sys.cpu", 1L));
+        Text r2 = new Text(MetricAdapter.encodeRowKey("sys.mem", 1L));
+        Text r3 = new Text(MetricAdapter.encodeRowKey("sys.disk", 1L));
+
+        // @formatter:off
+        Collection<Map.Entry<Key, Value>> metadata = new TestTabletMetadata("2")
+                .file(r1, "hdfs://hdfs-name/accumulo/tables/2/default_tablet/C0000ao8.rf", "800,3")
+                .file(r1, "hdfs://hdfs-name/accumulo/tables/2/default_tablet/C0000ao9.rf", "431,1")
+                .prev(r1, null)
+                .time(r1, 1)
+                .time(r2, 2)
+                .prev(r2, r3)
+                .file(r3, "hdfs://hdfs-name/accumulo/tables/2/default_tablet/C0000ao8.rf", "800,3")
+                .file(r3, "hdfs://hdfs-name/accumulo/tables/2/default_tablet/C0000ao9.rf", "431,1")
+                .file(r3, "hdfs://hdfs-name/accumulo/tables/2/default_tablet/C0000ao0.rf", "451,2")
+                .time(r3, 3)
+                .prev(r3, r1)
+                .entries();
+
+        // assert that r1/r2 are in the output correctly
+        // r2 does not have any files, it should not appear in the output
+        // @formatter:on
+        TabletMetadataView view = new TabletMetadataView();
+        view.addEntry(metadata);
+        List<MetadataAccumulator.Entry> accumulated = Lists.newArrayList(view.getEntries());
+        assertEquals(2, accumulated.size());
+
+        // validate that it skipped over non-file
+        assertEquals(r1, accumulated.get(0).getTablet());
+        assertEquals("sys.cpu", accumulated.get(0).getTabletPrefix());
+        assertEquals(1, accumulated.get(0).getMilliseconds());
+        assertEquals(2, accumulated.get(0).getTotalFiles());
+        assertEquals(r3, accumulated.get(1).getTablet());
+        assertEquals("sys.disk", accumulated.get(1).getTabletPrefix());
+        assertEquals(3, accumulated.get(1).getTotalFiles());
+        assertEquals(3, accumulated.get(1).getMilliseconds());
+
+        assertNull(accumulated.get(0).getTabletPrev());
+        assertEquals(r1, accumulated.get(1).getTabletPrev());
+    }
+
+    @Test
+    public void noRowsOnView() {
+        Collection<Map.Entry<Key, Value>> metadata = Collections.emptyList();
+        TabletMetadataView view = new TabletMetadataView();
+        view.addEntry(metadata);
+        List<MetadataAccumulator.Entry> accumulated = Lists.newArrayList(view.getEntries());
+        TabletSummary summary = view.computeSummary().build();
+        String text = view.toText();
+
+        assertEquals(0, accumulated.size());
+        assertEquals(summary.totalTablets(), 0);
+        assertNotNull(text);
+    }
+
+    @Test
+    public void simpleTextOutput() {
+        Long a1 = System.currentTimeMillis() - TimeUnit.DAYS.toMillis(30);
+        Long a2 = System.currentTimeMillis() - TimeUnit.DAYS.toMillis(29);
+        Text r1 = new Text(MetricAdapter.encodeRowKey("sys.cpu", a1));
+        Text r2 = new Text(MetricAdapter.encodeRowKey("sys.cpu", a2));
+
+        // @formatter:off
+        Collection<Map.Entry<Key, Value>> metadata = new TestTabletMetadata("2")
+                .file(r1, "hdfs://hdfs-name/accumulo/tables/2/default_tablet/C0000ao8.rf", "800,3")
+                .file(r1, "hdfs://hdfs-name/accumulo/tables/2/default_tablet/C0000ao9.rf", "431,1")
+                .prev(r1, null)
+                .time(r1, a1)
+                .file(r2, "hdfs://hdfs-name/accumulo/tables/2/default_tablet/C0000ao8.rf", "800,3")
+                .file(r2, "hdfs://hdfs-name/accumulo/tables/2/default_tablet/C0000ao9.rf", "431,1")
+                .file(r2, "hdfs://hdfs-name/accumulo/tables/2/default_tablet/C0000ao0.rf", "451,2")
+                .time(r2, a2)
+                .prev(r2, r1)
+                .entries();
+
+        // @formatter:on
+        TabletMetadataView view = new TabletMetadataView();
+        view.addEntry(metadata);
+        String output = view.toText(TimeUnit.DAYS);
+        System.out.println(output);
+    }
+}

server/src/test/java/timely/store/compaction/util/TabletSummaryTest.java

@@ -0,0 +1,171 @@
+package timely.store.compaction.util;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.util.Calendar;
+import java.util.GregorianCalendar;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.TimeUnit;
+
+import com.google.common.collect.Lists;
+import org.apache.commons.math3.stat.descriptive.StatisticalSummary;
+import org.apache.hadoop.io.Text;
+import org.junit.Test;
+import timely.adapter.accumulo.MetricAdapter;
+
+public class TabletSummaryTest {
+
+    private static final long DEFAULT_TIME_MILLIS = new GregorianCalendar(2019, Calendar.JANUARY, 1).getTimeInMillis();
+
+    @Test
+    public void totalsMatch() {
+        Text r1 = new Text(MetricAdapter.encodeRowKey("sys.cpu", DEFAULT_TIME_MILLIS));
+        Text r2 = new Text(MetricAdapter.encodeRowKey("sys.cpu", DEFAULT_TIME_MILLIS));
+        Text r3 = new Text(MetricAdapter.encodeRowKey("sys.mem", DEFAULT_TIME_MILLIS));
+
+        MetadataAccumulator.Entry e1 = new MetadataAccumulator.Entry(r1);
+        MetadataAccumulator.Entry e2 = new MetadataAccumulator.Entry(r2);
+        MetadataAccumulator.Entry e3 = new MetadataAccumulator.Entry(r3);
+
+        e1.addFile(800);
+        e1.addFile(100);
+        e2.addFile(100);
+        e2.addFile(200);
+        e3.addFile(500);
+
+        // @formatter:off
+        TabletSummary summary = TabletSummary.newBuilder(Lists.newArrayList(e1, e2, e3))
+                .disableTabletRowCheckFilter()
+                .build();
+
+        // @formatter:on
+        assertEquals(3, summary.totalTablets());
+        assertEquals(2, summary.totalTabletPrefixes());
+
+        List<TabletStatistic> largest = Lists.newArrayList(summary.findTabletPrefixesByLargest(3));
+        assertEquals(2, largest.size());
+        assertEquals(1200, largest.get(0).totalSize());
+        assertEquals(500, largest.get(1).totalSize());
+    }
+
+    @Test
+    public void statsComputedCorrectly() {
+        long a1 = TimeUnit.HOURS.toMillis(1);
+        long a2 = TimeUnit.HOURS.toMillis(2);
+
+        long ts1 = DEFAULT_TIME_MILLIS - a1;
+        long ts2 = DEFAULT_TIME_MILLIS - a2;
+
+        Text r1 = new Text(MetricAdapter.encodeRowKey("sys.cpu", ts1));
+        Text r2 = new Text(MetricAdapter.encodeRowKey("sys.cpu", ts2));
+        Text r3 = new Text(MetricAdapter.encodeRowKey("sys.mem", ts1));
+
+        MetadataAccumulator.Entry e1 = new MetadataAccumulator.Entry(r1);
+        MetadataAccumulator.Entry e2 = new MetadataAccumulator.Entry(r2);
+        MetadataAccumulator.Entry e3 = new MetadataAccumulator.Entry(r3);
+
+        e1.addFile(800);
+        e1.addFile(100);
+        e2.addFile(100);
+        e2.addFile(200);
+        e3.addFile(500);
+
+        // @formatter:off
+        TabletSummary summary = TabletSummary.newBuilder(Lists.newArrayList(e1, e2, e3))
+                .currentTime(DEFAULT_TIME_MILLIS)
+                .disableTabletRowCheckFilter()
+                .build();
+
+        // @formatter:on
+        List<TabletStatistic> largest = Lists.newArrayList(summary.findTabletPrefixesByLargest(3));
+        List<TabletStatistic> oldest = Lists.newArrayList(summary.findTabletPrefixesByOldest(3));
+
+        assertEquals(2, largest.size());
+        assertEquals(1200, largest.get(0).totalSize());
+        assertEquals(500, largest.get(1).totalSize());
+
+        assertEquals(2, oldest.size());
+        assertEquals(a2, oldest.get(0).maxAge());
+        assertEquals(a1, oldest.get(1).maxAge());
+    }
+
+    @Test
+    public void aggregateComputes() {
+        long a1 = TimeUnit.HOURS.toMillis(1);
+        long a2 = TimeUnit.HOURS.toMillis(2);
+
+        long ts1 = DEFAULT_TIME_MILLIS - a1;
+        long ts2 = DEFAULT_TIME_MILLIS - a2;
+
+        Text r1 = new Text(MetricAdapter.encodeRowKey("sys.cpu", ts1));
+        Text r2 = new Text(MetricAdapter.encodeRowKey("sys.cpu", ts2));
+        Text r3 = new Text(MetricAdapter.encodeRowKey("sys.mem", ts1));
+
+        MetadataAccumulator.Entry e1 = new MetadataAccumulator.Entry(r1);
+        MetadataAccumulator.Entry e2 = new MetadataAccumulator.Entry(r2);
+        MetadataAccumulator.Entry e3 = new MetadataAccumulator.Entry(r3);
+
+        e1.addFile(800);
+        e1.addFile(100);
+        e2.addFile(100);
+        e2.addFile(200);
+        e1.addFile(500);
+
+        // @formatter:off
+        TabletSummary summary = TabletSummary.newBuilder(Lists.newArrayList(e1, e2, e3))
+                .currentTime(DEFAULT_TIME_MILLIS)
+                .disableTabletRowCheckFilter()
+                .build();
+
+        // @formatter:on
+        Map<TabletStatisticType, StatisticalSummary> map = summary.aggregateSummary();
+        StatisticalSummary size = map.get(TabletStatisticType.SIZE);
+        StatisticalSummary time = map.get(TabletStatisticType.TIME);
+
+        assertEquals(1700, (long) size.getSum());
+        assertEquals(a2, (long) time.getMax());
+        assertEquals(a1, (long) time.getMin());
+    }
+
+    @Test
+    public void initialTabletsArePrunedByDefault() {
+        // test that the filter does not include
+        // the differing tablet rows; the compaction strategy
+        // should ignore
+
+        long a1 = TimeUnit.DAYS.toMillis(365);
+        long a2 = TimeUnit.HOURS.toMillis(2);
+
+        long ts1 = DEFAULT_TIME_MILLIS - a1;
+        long ts2 = DEFAULT_TIME_MILLIS - a2;
+
+        Text r1 = new Text(MetricAdapter.encodeRowKey("sys.cpu", ts1));
+        Text r2 = new Text(MetricAdapter.encodeRowKey("sys.cpu", ts2));
+
+        MetadataAccumulator.Entry e1 = new MetadataAccumulator.Entry(r1);
+        MetadataAccumulator.Entry e2 = new MetadataAccumulator.Entry(r2);
+
+        e1.addFile(800);
+        e2.addFile(100);
+        e2.setTablePrev(r1);
+
+        // @formatter:off
+        TabletSummary summary = TabletSummary.newBuilder(Lists.newArrayList(e1, e2))
+                .currentTime(DEFAULT_TIME_MILLIS)
+                .build();
+
+        // @formatter:on
+        assertEquals(1, summary.totalTablets());
+        assertEquals(1, summary.totalTabletPrefixes());
+
+        Optional<TabletStatistic> stat = summary.getSummary().stream().findAny();
+
+        // verify that initial stat was pruned from results
+        // this is the default behavior
+        assertTrue(stat.isPresent());
+        assertEquals(a2, stat.get().maxAge());
+    }
+}

server/src/test/java/timely/test/CompactionRequestBuilder.java

@@ -0,0 +1,119 @@
+package timely.test;
+
+import java.util.Calendar;
+import java.util.GregorianCalendar;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.TreeMap;
+
+import org.apache.accumulo.core.client.lexicoder.Lexicoder;
+import org.apache.accumulo.core.client.lexicoder.PairLexicoder;
+import org.apache.accumulo.core.client.lexicoder.StringLexicoder;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.TabletId;
+import org.apache.accumulo.core.data.impl.KeyExtent;
+import org.apache.accumulo.core.data.impl.TabletIdImpl;
+import org.apache.accumulo.core.metadata.schema.DataFileValue;
+import org.apache.accumulo.core.util.ComparablePair;
+import org.apache.accumulo.server.fs.FileRef;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.tserver.compaction.MajorCompactionReason;
+import org.apache.accumulo.tserver.compaction.MajorCompactionRequest;
+import org.easymock.EasyMock;
+import timely.adapter.accumulo.MetricAdapter;
+
+public class CompactionRequestBuilder {
+
+    private static final long DEFAULT_TIME_MILLIS = new GregorianCalendar(2019, Calendar.JANUARY, 1).getTimeInMillis();
+    private static final Lexicoder<String> stringEncoder = new StringLexicoder();
+    private static final PairLexicoder<String, String> badPairEncoder = new PairLexicoder<>(stringEncoder,
+            stringEncoder);
+
+    private final Map<FileRef, DataFileValue> refs = new LinkedHashMap<>();
+    private final Map<String, String> tableProperties;
+    private final long timeMillis;
+    private Key endKeyMetric;
+    private Key prevKeyMetric;
+
+    public CompactionRequestBuilder() {
+        this(DEFAULT_TIME_MILLIS);
+    }
+
+    public CompactionRequestBuilder(long timeMillis) {
+        this.timeMillis = timeMillis;
+        this.tableProperties = new TreeMap<>();
+    }
+
+    public CompactionRequestBuilder endKeyMetricNonOffset(String name, String offset) {
+        byte[] rowKey = offset != null ? badPairEncoder.encode(new ComparablePair<>(name, offset))
+                : stringEncoder.encode(name);
+        endKeyMetric = new Key(rowKey);
+        return this;
+    }
+
+    public CompactionRequestBuilder endKeyMetric(String name, long offset) {
+        long ts = timeMillis - offset;
+        byte[] rowKey = MetricAdapter.encodeRowKey(name, ts);
+        endKeyMetric = new Key(rowKey);
+        return this;
+    }
+
+    public CompactionRequestBuilder prevEndKeyMetric(String name, long offset) {
+        long ts = timeMillis - offset;
+        byte[] rowKey = MetricAdapter.encodeRowKey(name, ts);
+        prevKeyMetric = new Key(rowKey);
+        return this;
+    }
+
+    public CompactionRequestBuilder prevEndKeyMetricNonOffset(String name, String offset) {
+        byte[] rowKey = offset != null ? badPairEncoder.encode(new ComparablePair<>(name, offset))
+                : stringEncoder.encode(name);
+        prevKeyMetric = new Key(rowKey);
+        return this;
+    }
+
+    public CompactionRequestBuilder file(String path, long size, long entries) {
+        refs.put(new FileRef(path), new DataFileValue(size, entries));
+        return this;
+    }
+
+    public CompactionRequestBuilder tableProperties(String key, String value) {
+        tableProperties.put(key, value);
+        return this;
+    }
+
+    public MajorCompactionRequest build() {
+        String tableName = "1";
+        KeyExtent ke = new KeyExtent();
+        ke.setTableId(tableName);
+
+        if (null != endKeyMetric) {
+            ke.setEndRow(endKeyMetric.getRow());
+        }
+
+        if (null != prevKeyMetric) {
+            ke.setPrevEndRow(prevKeyMetric.getRow());
+        }
+
+        TabletId tid = new TabletIdImpl(ke);
+
+        // @formatter:off
+        MajorCompactionRequest req = EasyMock.partialMockBuilder(MajorCompactionRequest.class)
+                .withConstructor(KeyExtent.class, MajorCompactionReason.class, VolumeManager.class,
+                        AccumuloConfiguration.class)
+                .withArgs(ke, MajorCompactionReason.NORMAL, null, AccumuloConfiguration.getDefaultConfiguration())
+                .addMockedMethod("getTabletId")
+                .addMockedMethod("getTableProperties")
+                .createMock();
+
+        // @formatter:on
+        req.setFiles(refs);
+        EasyMock.expect(req.getTabletId()).andStubReturn(tid);
+        EasyMock.expect(req.getTableProperties()).andStubReturn(tableProperties);
+
+        EasyMock.replay(req);
+
+        return req;
+    }
+}

server/src/test/java/timely/test/TestMetricWriter.java

@@ -0,0 +1,62 @@
+package timely.test;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.data.Mutation;
+import timely.adapter.accumulo.MetricAdapter;
+import timely.model.Metric;
+
+public class TestMetricWriter {
+
+    private static final int DEFAULT_BATCHES = 10000;
+    private final Connector connector;
+    private final TestMetrics metrics;
+    private final String tableName;
+
+    public TestMetricWriter(String tableName, Connector connector, TestMetrics metrics) {
+        this.connector = connector;
+        this.metrics = metrics;
+        this.tableName = tableName;
+    }
+
+    public long ingestRandomDuration(long duration, TimeUnit durationUnit, long timestampInitial, int decrementMillis) {
+        long timestampCurrent = timestampInitial;
+        long runUntilMillis = System.currentTimeMillis() + TimeUnit.MILLISECONDS.convert(duration, durationUnit);
+        while (System.currentTimeMillis() < runUntilMillis) {
+            timestampCurrent = ingestRandom(timestampCurrent, decrementMillis);
+        }
+        return timestampCurrent;
+    }
+
+    private long ingestRandom(long timestampInitial, int decrementMillis) {
+        long ts = timestampInitial;
+        Collection<Mutation> mutations = new ArrayList<>(TestMetricWriter.DEFAULT_BATCHES);
+        int idx = 0;
+        while (idx++ < TestMetricWriter.DEFAULT_BATCHES) {
+            ts = ts - decrementMillis;
+            Metric m = metrics.randomMetric(ts);
+            Mutation mutation = MetricAdapter.toMutation(m);
+            mutations.add(mutation);
+        }
+
+        try {
+            BatchWriterConfig cfg = new BatchWriterConfig().setMaxWriteThreads(4);
+            try (BatchWriter writer = connector.createBatchWriter(tableName, cfg)) {
+                writer.addMutations(mutations);
+            }
+            connector.tableOperations().flush(tableName, null, null, true);
+        } catch (AccumuloException | AccumuloSecurityException | TableNotFoundException e) {
+            throw new RuntimeException(e);
+        }
+
+        return ts;
+    }
+}

server/src/test/java/timely/test/TestMetrics.java

@@ -0,0 +1,48 @@
+package timely.test;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import com.google.common.collect.Lists;
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.commons.lang3.RandomUtils;
+import timely.model.Metric;
+import timely.model.Tag;
+
+public class TestMetrics {
+
+    private final List<String> prefixes;
+    private final List<String> suffixes;
+    private final List<Tag> tags;
+
+    public TestMetrics() {
+        prefixes = Lists.newArrayList("sys", "statsd");
+        suffixes = createRandomSuffixes();
+        tags = new ArrayList<>(2);
+        tags.add(new Tag("hostname", "localhost"));
+        tags.add(new Tag("ip", "127.0.0.1"));
+    }
+
+    public Metric randomMetric(long timestamp) {
+        int prefixIdx = RandomUtils.nextInt(0, prefixes.size());
+        int suffixIdx = RandomUtils.nextInt(0, suffixes.size());
+        // @formatter:off
+        return Metric.newBuilder()
+                .name(prefixes.get(prefixIdx) + "." + suffixes.get(suffixIdx))
+                .tags(tags)
+                .value(timestamp, RandomUtils.nextDouble(0, 1.0))
+                .build();
+    }
+
+    private static List<String> createRandomSuffixes() {
+        int SUFFIX_MIN = 5;
+        int SUFFIX_MAX = 10;
+        int SUFFIX_NUM = 4;
+        List<String> list = new ArrayList<>(SUFFIX_NUM);
+        for (int i = 0; i < SUFFIX_NUM; i++) {
+            list.add(RandomStringUtils.randomAlphabetic(SUFFIX_MIN, SUFFIX_MAX).toLowerCase());
+        }
+        return list;
+    }
+
+}

server/src/test/java/timely/test/TestTabletMetadata.java

@@ -0,0 +1,58 @@
+package timely.test;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+
+import com.google.common.collect.Maps;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.data.impl.KeyExtent;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema;
+import org.apache.accumulo.core.util.ColumnFQ;
+import org.apache.hadoop.io.Text;
+
+public class TestTabletMetadata {
+
+    private final String tableId;
+    private final List<Map.Entry<Key, Value>> entries;
+
+    public TestTabletMetadata(String tableId) {
+        this.tableId = tableId;
+        entries = new ArrayList<>();
+    }
+
+    public TestTabletMetadata file(Text row, String path, String value) {
+        Text cq = new Text(path);
+        addEntry(row, MetadataSchema.TabletsSection.DataFileColumnFamily.NAME, cq, value);
+        return this;
+    }
+
+    public TestTabletMetadata prev(Text row, Text prevRow) {
+        ColumnFQ cfq = MetadataSchema.TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN;
+        addEntry(row, cfq.getColumnFamily(), cfq.getColumnQualifier(), KeyExtent.encodePrevEndRow(prevRow));
+        return this;
+    }
+
+    public TestTabletMetadata time(Text row, long millis) {
+        ColumnFQ cfq = MetadataSchema.TabletsSection.ServerColumnFamily.TIME_COLUMN;
+        addEntry(row, cfq.getColumnFamily(), cfq.getColumnQualifier(), "M" + millis);
+        return this;
+    }
+
+    public Collection<Map.Entry<Key, Value>> entries() {
+        return entries;
+    }
+
+    private void addEntry(Text row, Text cf, Text cq, String value) {
+        addEntry(row, cf, cq, new Value(value));
+    }
+
+    private void addEntry(Text row, Text cf, Text cq, Value value) {
+        Text rowEntry = KeyExtent.getMetadataEntry(tableId, row);
+        Value val = new Value(value);
+        Key k = new Key(rowEntry, cf, cq);
+        entries.add(Maps.immutableEntry(k, val));
+    }
+}

server/src/test/java/timely/test/integration/DataStoreIT.java

@@ -107,10 +107,23 @@ public class DataStoreIT extends OneWaySSLBase {
 
         final Server s = new Server(conf);
         s.run();
+        // @formatter:off
+        /*
+         * TEST_TIME = System.currentTimeMillis() - ONE_DAY
+         * Age off for all metrics is one day = 24 hours
+         *
+         * lines 1, 2, 3 will age off immediately as they are >= 2 days old
+         * line 4 is there because otherwise the meta tags would also age off and we would get a 400 - No Tags Found
+         * line 5 is one day old and will age off immediately
+         * lines 6 & 7 should be returned as they are not aged off and are within the query range
+         *
+         */
+        // @formatter:on
         try {
             put("sys.cpu.idle " + (TEST_TIME - ONE_DAY - (2 * ONE_HOUR)) + " 1.0 tag1=value1 tag2=value2",
                     "sys.cpu.idle " + (TEST_TIME - ONE_DAY - ONE_HOUR) + " 3.0 tag1=value1 tag2=value2",
                     "sys.cpu.idle " + (TEST_TIME - ONE_DAY) + " 2.0 tag1=value1 tag3=value3",
+                    "sys.cpu.idle " + (TEST_TIME + (ONE_DAY * 2)) + " 2.0 tag1=value1 tag3=value3",
                     "sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2",
                     "sys.cpu.user " + (TEST_TIME + ONE_HOUR) + " 3.0 tag1=value1 tag2=value2",
                     "sys.cpu.user " + (TEST_TIME + (ONE_HOUR * 2)) + " 2.0 tag1=value1 tag3=value3");

server/src/test/java/timely/test/integration/MetricCompactionIT.java

@@ -0,0 +1,254 @@
+package timely.test.integration;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.util.Collection;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
+import com.google.common.collect.Sets;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.iterators.IteratorUtil;
+import org.apache.accumulo.tserver.compaction.CompactionStrategy;
+import org.apache.accumulo.tserver.compaction.DefaultCompactionStrategy;
+import org.apache.hadoop.io.Text;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import timely.store.MetricAgeOffIterator;
+import timely.store.compaction.MetricCompactionStrategy;
+import timely.store.compaction.TabletRowAdapter;
+import timely.store.compaction.util.TabletMetadataQuery;
+import timely.store.compaction.util.TabletMetadataView;
+import timely.store.compaction.util.TabletStatistic;
+import timely.store.compaction.util.TabletSummary;
+import timely.test.IntegrationTest;
+import timely.test.TestMetricWriter;
+import timely.test.TestMetrics;
+
+@Category(IntegrationTest.class)
+public class MetricCompactionIT extends MacITBase {
+
+    private final static Logger LOG = LoggerFactory.getLogger(MetricCompactionIT.class);
+
+    private static final String AGE_OFF_ITERATOR_NAME = "ageoffmetrics";
+    private static final EnumSet<IteratorUtil.IteratorScope> AGEOFF_SCOPES = EnumSet
+            .allOf(IteratorUtil.IteratorScope.class);
+
+    private Connector connector;
+    private MetricConfigurationAdapter adapter;
+
+    @BeforeClass
+    public static void setupClass() throws Exception {
+        Connector connector = mac.getConnector(MAC_ROOT_USER, MAC_ROOT_PASSWORD);
+        connector.instanceOperations().setProperty(Property.TSERV_MAJC_DELAY.getKey(), "3s");
+    }
+
+    @Before
+    public void setup() throws Exception {
+        connector = mac.getConnector(MAC_ROOT_USER, MAC_ROOT_PASSWORD);
+        String metricsTable = conf.getMetricsTable();
+        if (!metricsTable.contains(".")) {
+            throw new IllegalArgumentException("Expected to find namespace in table name");
+        }
+        String[] parts = metricsTable.split("\\.", 2);
+        String namespace = parts[0];
+        if (!connector.namespaceOperations().exists(namespace)) {
+            connector.namespaceOperations().create(namespace);
+        }
+        connector.tableOperations().create(metricsTable);
+        adapter = new MetricConfigurationAdapter(connector, conf.getMetricsTable());
+        connector.tableOperations().setProperty(conf.getMetricsTable(), Property.TABLE_SPLIT_THRESHOLD.getKey(), "50K");
+        adapter.resetState();
+    }
+
+    @After
+    public void teardown() throws Exception {
+        adapter.resetState();
+    }
+
+    @AfterClass
+    public static void teardownClass() throws Exception {
+        Connector connector = mac.getConnector(MAC_ROOT_USER, MAC_ROOT_PASSWORD);
+        connector.instanceOperations().setProperty(Property.TSERV_MAJC_DELAY.getKey(), "30s");
+    }
+
+    @Test
+    public void tableMetadataEnumeratesTablets() throws Exception {
+        String metricsTable = conf.getMetricsTable();
+
+        TestMetricWriter writer = new TestMetricWriter(metricsTable, connector, new TestMetrics());
+        writer.ingestRandomDuration(60, TimeUnit.SECONDS, System.currentTimeMillis(), 250);
+
+        Collection<Text> splits = connector.tableOperations().listSplits(conf.getMetricsTable());
+        TabletMetadataQuery query = new TabletMetadataQuery(connector, metricsTable);
+        TabletMetadataView view = query.run();
+
+        // @formatter:off
+        TabletSummary summary = view.computeSummary()
+                .disableTabletRowCheckFilter()
+                .build();
+
+        Set<String> splitsPrefixed = splits.stream()
+                .map(row -> TabletRowAdapter.decodeRowPrefix(row).orElse("unknown"))
+                .collect(Collectors.toSet());
+
+        Set<String> tabletsPrefixed = summary.getSummary().stream()
+                .map(TabletStatistic::getKeyName)
+                .collect(Collectors.toSet());
+
+        // difference both sides
+        // should be empty
+        // @formatter:on
+        Set<String> i1 = Sets.difference(tabletsPrefixed, splitsPrefixed);
+        Set<String> i2 = Sets.difference(splitsPrefixed, tabletsPrefixed);
+        assertTrue("tablets-diff" + i1.toString(), i1.isEmpty());
+        assertTrue("splits-diff" + i2.toString(), i2.isEmpty());
+
+        // dump data to screen as summary
+        System.out.println(view.toText());
+    }
+
+    @Test
+    public void tabletsCompactedUntilAgeOff() throws Exception {
+        // test will replicated problem in time-series tablets where data is
+        // split automatically and the resulting tablets will not be cleaned up
+        // by the default compaction strategy
+        //
+        // metric compaction strategy will age-off tablets by date on tablets
+        // but ignore differing end/prev rows
+        //
+        // the mini-accumulo cluster should not have any age-off
+        // iterators on timely.metrics
+
+        String metricsTable = conf.getMetricsTable();
+
+        // check to make sure the table has only one iterator
+        Map<String, EnumSet<IteratorUtil.IteratorScope>> itrs = connector.tableOperations().listIterators(metricsTable);
+        assertEquals(1, itrs.size());
+        assertTrue(itrs.containsKey("vers"));
+
+        long timestampMax = System.currentTimeMillis();
+        TestMetricWriter writer = new TestMetricWriter(conf.getMetricsTable(), connector, new TestMetrics());
+        long timestampMin = writer.ingestRandomDuration(30, TimeUnit.SECONDS, timestampMax, 250);
+
+        // invoke compact operation to flush everything through
+        // without this call the compaction strategy later on
+        // was not compacting all of the tablets
+        adapter.runCompaction();
+
+        // figure out what should be targeted based on an ageoff date
+        // cut the delta by 1/3
+        long ageoff = (timestampMax - timestampMin) / 3;
+
+        // run check to verify there are pre-existing splits for a prefix
+        // @formatter:off
+        TabletMetadataQuery query = new TabletMetadataQuery(connector, metricsTable);
+        TabletSummary summary1 = query.run()
+                .computeSummary()
+                .build();
+
+        Set<String> tabletsBefore = summary1.getSummary().stream()
+                .filter(t -> t.maxAge() > ageoff)
+                .map(TabletStatistic::getKeyName)
+                .collect(Collectors.toSet());
+
+        // @formatter:on
+        assertFalse("Tablets were empty before compaction, expected test data", tabletsBefore.isEmpty());
+
+        long compactBeforeMillis = System.currentTimeMillis();
+
+        // invoke the age-off compactor
+        // after this runs it should have gotten rid o fdata
+        // have pre-age-off tablets
+        adapter.applyCompactionConfiguration(MetricCompactionStrategy.class, ageoff);
+        adapter.runCompaction();
+
+        long compactDeltaMillis = System.currentTimeMillis() - compactBeforeMillis;
+
+        // re-run check on tablet metadata
+        // check that there aren't new tablets past the predicted age-off
+        // any tablets with differing end/prev rows are filtered by the summary
+        // @formatter:off
+        TabletSummary summary2 = query.run()
+                .computeSummary()
+                .build();
+
+        List<TabletStatistic> tabletsFilterAfter = summary2.getSummary().stream()
+                .filter(t -> ((t.maxAge() - compactDeltaMillis) > ageoff) && t.getKeyCount() > 1)
+                .collect(Collectors.toList());
+
+        // @formatter:on
+        if (!tabletsFilterAfter.isEmpty()) {
+            LOG.info("tablets-before: {}", tabletsBefore.toString());
+            LOG.info("tablets-after: {}", tabletsFilterAfter.toString());
+        }
+
+        // should not have found tablets past the age-off and should also
+        // have found existing tablets still in system (i.e. did not compact everything)
+        assertTrue("Tablets still had unexpected after compaction", tabletsFilterAfter.isEmpty());
+        assertTrue("Some tablets should exist, but did not find any", summary2.totalTablets() > 0);
+    }
+
+    private static class MetricConfigurationAdapter {
+
+        private final Connector connector;
+        private final String tableName;
+
+        private static String[] CLEAR_PROPERTY_KEYS = {
+                Property.TABLE_COMPACTION_STRATEGY_PREFIX.getKey() + MetricCompactionStrategy.MIN_AGEOFF_KEY };
+
+        public MetricConfigurationAdapter(Connector connector, String tableName) {
+            this.connector = connector;
+            this.tableName = tableName;
+        }
+
+        public void applyCompactionConfiguration(Class<? extends CompactionStrategy> clazz, long ageOff)
+                throws Exception {
+            Map<String, String> ageOffs = new HashMap<>();
+            ageOffs.put("ageoff.default", Long.toString(ageOff));
+            IteratorSetting ageOffIteratorSettings = new IteratorSetting(100, AGE_OFF_ITERATOR_NAME,
+                    MetricAgeOffIterator.class, ageOffs);
+            connector.tableOperations().attachIterator(tableName, ageOffIteratorSettings, AGEOFF_SCOPES);
+            connector.tableOperations().setProperty(tableName, Property.TABLE_COMPACTION_STRATEGY.getKey(),
+                    clazz.getName());
+        }
+
+        public void runCompaction() throws Exception {
+            connector.tableOperations().compact(tableName, null, null, true, true);
+        }
+
+        public void resetState() throws Exception {
+            Map<String, EnumSet<IteratorUtil.IteratorScope>> iters = connector.tableOperations()
+                    .listIterators(tableName);
+            for (String name : iters.keySet()) {
+                if (name.startsWith("ageoff")) {
+                    connector.tableOperations().removeIterator(tableName, name, AGEOFF_SCOPES);
+                }
+            }
+
+            // reset compaction strategy to default
+            connector.tableOperations().setProperty(tableName, Property.TABLE_COMPACTION_STRATEGY.getKey(),
+                    DefaultCompactionStrategy.class.getName());
+
+            // reset any properties
+            for (String s : CLEAR_PROPERTY_KEYS) {
+                connector.tableOperations().removeProperty(tableName, s);
+            }
+        }
+    }
+}

server/src/test/java/timely/test/integration/client/TcpClientIT.java

@@ -177,7 +177,7 @@ public class TcpClientIT extends OneWaySSLBase {
             LOG.info("Meta no vers iter: " + entry);
             count++;
         }
-        assertEquals(10, count);
+        assertEquals(15, count);
     }
 
     @Test

server/src/test/java/timely/test/integration/client/UdpClientIT.java

@@ -187,7 +187,7 @@ public class UdpClientIT extends MacITBase {
             LOG.info("Meta no vers iter: " + entry);
             count++;
         }
-        assertEquals(10, count);
+        assertEquals(15, count);
     }
 
     @Test

server/src/test/java/timely/test/integration/http/HttpApiIT.java

@@ -62,8 +62,7 @@ public class HttpApiIT extends OneWaySSLBase {
         try {
             put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2",
                     "sys.cpu.idle " + (TEST_TIME + 1) + " 1.0 tag3=value3 tag4=value4",
-                    "sys.cpu.idle " + (TEST_TIME + 2) + " 1.0 tag3=value3 tag4=value4",
-                    "zzzz 1234567892 1.0 host=localhost");
+                    "sys.cpu.idle " + (TEST_TIME + 2) + " 1.0 tag3=value3 tag4=value4");
             // Latency in TestConfiguration is 2s, wait for it
             sleepUninterruptibly(TestConfiguration.WAIT_SECONDS, TimeUnit.SECONDS);
 
@@ -78,11 +77,7 @@ public class HttpApiIT extends OneWaySSLBase {
 
             // Test empty query
             result = query(suggest + "type=metrics&max=10");
-            assertEquals("[\"sys.cpu.idle\",\"sys.cpu.user\",\"zzzz\"]", result);
-
-            // Test skipping over initial metrics
-            result = query(suggest + "type=metrics&q=z&max=10");
-            assertEquals("[\"zzzz\"]", result);
+            assertEquals("[\"sys.cpu.idle\",\"sys.cpu.user\"]", result);
         } finally {
             s.shutdown();
         }

server/src/test/java/timely/test/integration/tcp/TimelyTcpIT.java

@@ -287,7 +287,7 @@ public class TimelyTcpIT extends MacITBase {
             LOG.info("Meta no vers iter: " + entry);
             count++;
         }
-        assertEquals(10, count);
+        assertEquals(15, count);
     }
 
     @Test

server/src/test/java/timely/test/integration/udp/TimelyUdpIT.java

@@ -283,7 +283,7 @@ public class TimelyUdpIT extends MacITBase {
             LOG.info("Meta no vers iter: " + entry);
             count++;
         }
-        assertEquals(10, count);
+        assertEquals(15, count);
     }
 
     @Test

server/src/test/java/timely/util/MetaKeySetTest.java

@@ -38,17 +38,18 @@ public class MetaKeySetTest {
         mks.addAll(one.toKeys());
         mks.addAll(two.toKeys());
         mks.addAll(three.toKeys());
-        List<Mutation> muts = mks.toMutations();
+        long ts = System.currentTimeMillis();
+        List<Mutation> muts = mks.toMutations(ts);
         Mutation e1 = new Mutation("m:sys.cpu.user");
-        e1.put("", "", MetaKeySet.NULL_VALUE);
+        e1.put("", "", ts, MetaKeySet.NULL_VALUE);
         Mutation e2 = new Mutation("t:sys.cpu.user");
-        e2.put("tag1", "", MetaKeySet.NULL_VALUE);
-        e2.put("tag2", "", MetaKeySet.NULL_VALUE);
-        e2.put("tag3", "", MetaKeySet.NULL_VALUE);
+        e2.put("tag1", "", ts, MetaKeySet.NULL_VALUE);
+        e2.put("tag2", "", ts, MetaKeySet.NULL_VALUE);
+        e2.put("tag3", "", ts, MetaKeySet.NULL_VALUE);
         Mutation e3 = new Mutation("v:sys.cpu.user");
-        e3.put("tag1", "value1", MetaKeySet.NULL_VALUE);
-        e3.put("tag2", "value2", MetaKeySet.NULL_VALUE);
-        e3.put("tag3", "value3", MetaKeySet.NULL_VALUE);
+        e3.put("tag1", "value1", ts, MetaKeySet.NULL_VALUE);
+        e3.put("tag2", "value2", ts, MetaKeySet.NULL_VALUE);
+        e3.put("tag3", "value3", ts, MetaKeySet.NULL_VALUE);
         Assert.assertEquals(3, muts.size());
         Assert.assertTrue(muts.contains(e1));
         Assert.assertTrue(muts.contains(e2));