0.0.4 release

fixes #108 - Implemented the aggregation across time series separate after downsampling

analytics/.gitignore

@@ -0,0 +1,2 @@
+/bin/
+/target

analytics/conf/summarization.properties

@@ -0,0 +1,20 @@
+timelyHostname=
+timelyTcpPort=
+timelyHttpsPort=
+timelyWssPort=
+doLogin=
+#timelyUsername=
+#timelyPassword=
+keyStoreFile=
+keyStoreType=
+keyStorePass=
+trustStoreFile=
+trustStoreType=
+trustStorePass=
+hostVerificationEnabled=false
+bufferSize=10485760
+metrics=
+startTime=0
+endTime=0
+interval=1
+intervalUnits=HOURS

analytics/pom.xml

@@ -0,0 +1,336 @@
+<?xml version="1.0"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <artifactId>timely-analytics</artifactId>
+    <parent>
+        <groupId>timely</groupId>
+        <artifactId>timely</artifactId>
+        <version>0.0.4</version>
+    </parent>
+    <properties>
+        <flink.version>1.1.4</flink.version>
+    </properties>
+    <!-- 
+		
+	Execute "mvn clean package -Pbuild-jar"
+	to build a jar file out of this project!
+
+	How to use the Flink Quickstart pom:
+
+	a) Adding new dependencies:
+		You can add dependencies to the list below.
+		Please check if the maven-shade-plugin below is filtering out your dependency
+		and remove the exclude from there.
+
+	b) Build a jar for running on the cluster:
+		There are two options for creating a jar from this project
+
+		b.1) "mvn clean package" -> this will create a fat jar which contains all
+				dependencies necessary for running the jar created by this pom in a cluster.
+				The "maven-shade-plugin" excludes everything that is provided on a running Flink cluster.
+
+		b.2) "mvn clean package -Pbuild-jar" -> This will also create a fat-jar, but with much
+				nicer dependency exclusion handling. This approach is preferred and leads to
+				much cleaner jar files.
+	-->
+    <dependencies>
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.flink</groupId>
+            <artifactId>flink-java</artifactId>
+            <version>${flink.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.flink</groupId>
+            <artifactId>flink-streaming-java_2.11</artifactId>
+            <version>${flink.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.flink</groupId>
+            <artifactId>flink-clients_2.11</artifactId>
+            <version>${flink.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>timely</groupId>
+            <artifactId>timely-client</artifactId>
+            <version>${parent.version}</version>
+            <exclusions>
+            	<exclusion>
+            		<groupId>org.apache.logging.log4j</groupId>
+            		<artifactId>*</artifactId>
+            	</exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>timely</groupId>
+            <artifactId>timely-common</artifactId>
+            <version>${parent.version}</version>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+    <profiles>
+        <profile>
+            <!-- Profile for packaging correct JAR files -->
+            <id>build-jar</id>
+            <activation>
+                <activeByDefault>false</activeByDefault>
+            </activation>
+            <dependencies>
+                <dependency>
+                    <groupId>org.apache.flink</groupId>
+                    <artifactId>flink-java</artifactId>
+                    <version>${flink.version}</version>
+                    <scope>provided</scope>
+                </dependency>
+                <dependency>
+                    <groupId>org.apache.flink</groupId>
+                    <artifactId>flink-streaming-java_2.11</artifactId>
+                    <version>${flink.version}</version>
+                    <scope>provided</scope>
+                </dependency>
+                <dependency>
+                    <groupId>org.apache.flink</groupId>
+                    <artifactId>flink-clients_2.11</artifactId>
+                    <version>${flink.version}</version>
+                    <scope>provided</scope>
+                </dependency>
+            </dependencies>
+            <build>
+                <plugins>
+                    <!-- disable the exclusion rules -->
+                    <plugin>
+                        <groupId>org.apache.maven.plugins</groupId>
+                        <artifactId>maven-shade-plugin</artifactId>
+                        <executions>
+                            <execution>
+                                <phase>package</phase>
+                                <goals>
+                                    <goal>shade</goal>
+                                </goals>
+                                <configuration>
+                                    <artifactSet>
+                                        <excludes combine.self="override"/>
+                                    </artifactSet>
+                                </configuration>
+                            </execution>
+                        </executions>
+                    </plugin>
+                </plugins>
+            </build>
+        </profile>
+    </profiles>
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>net.revelc.code</groupId>
+                <artifactId>formatter-maven-plugin</artifactId>
+            </plugin>
+            <plugin>
+                <groupId>org.jacoco</groupId>
+                <artifactId>jacoco-maven-plugin</artifactId>
+            </plugin>
+            <!-- We use the maven-shade plugin to create a fat jar that contains all dependencies
+			except flink and it's transitive dependencies. The resulting fat-jar can be executed
+			on a cluster. Change the value of Program-Class if your program entry point changes. -->
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-shade-plugin</artifactId>
+                <executions>
+                    <!-- Run shade goal on package phase -->
+                    <execution>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>shade</goal>
+                        </goals>
+                        <configuration>
+                            <artifactSet>
+                                <excludes>
+                                    <!-- This list contains all dependencies of flink-dist
+										Everything else will be packaged into the fat-jar
+									-->
+                                    <exclude>org.apache.flink:flink-annotations</exclude>
+                                    <exclude>org.apache.flink:flink-shaded-hadoop1_2.11</exclude>
+                                    <exclude>org.apache.flink:flink-shaded-hadoop2</exclude>
+                                    <exclude>org.apache.flink:flink-shaded-curator-recipes</exclude>
+                                    <exclude>org.apache.flink:flink-core</exclude>
+                                    <exclude>org.apache.flink:flink-java</exclude>
+                                    <exclude>org.apache.flink:flink-scala_2.11</exclude>
+                                    <exclude>org.apache.flink:flink-runtime_2.11</exclude>
+                                    <exclude>org.apache.flink:flink-optimizer_2.11</exclude>
+                                    <exclude>org.apache.flink:flink-clients_2.11</exclude>
+                                    <exclude>org.apache.flink:flink-avro_2.11</exclude>
+                                    <exclude>org.apache.flink:flink-examples-batch_2.11</exclude>
+                                    <exclude>org.apache.flink:flink-examples-streaming_2.11</exclude>
+                                    <exclude>org.apache.flink:flink-streaming-java_2.11</exclude>
+                                    <!-- Also exclude very big transitive dependencies of Flink
+										WARNING: You have to remove these excludes if your code relies on other
+										versions of these dependencies.
+ 				    				-->
+                                    <exclude>org.scala-lang:scala-library</exclude>
+                                    <exclude>org.scala-lang:scala-compiler</exclude>
+                                    <exclude>org.scala-lang:scala-reflect</exclude>
+                                    <exclude>com.typesafe.akka:akka-actor_*</exclude>
+                                    <exclude>com.typesafe.akka:akka-remote_*</exclude>
+                                    <exclude>com.typesafe.akka:akka-slf4j_*</exclude>
+                                    <exclude>io.netty:netty-all</exclude>
+                                    <exclude>io.netty:netty</exclude>
+                                    <exclude>commons-fileupload:commons-fileupload</exclude>
+                                    <exclude>org.apache.avro:avro</exclude>
+                                    <exclude>commons-collections:commons-collections</exclude>
+                                    <exclude>org.codehaus.jackson:jackson-core-asl</exclude>
+                                    <exclude>org.codehaus.jackson:jackson-mapper-asl</exclude>
+                                    <exclude>com.thoughtworks.paranamer:paranamer</exclude>
+                                    <exclude>org.xerial.snappy:snappy-java</exclude>
+                                    <exclude>org.apache.commons:commons-compress</exclude>
+                                    <exclude>org.tukaani:xz</exclude>
+                                    <exclude>com.esotericsoftware.kryo:kryo</exclude>
+                                    <exclude>com.esotericsoftware.minlog:minlog</exclude>
+                                    <exclude>org.objenesis:objenesis</exclude>
+                                    <exclude>com.twitter:chill_*</exclude>
+                                    <exclude>com.twitter:chill-java</exclude>
+                                    <exclude>commons-lang:commons-lang</exclude>
+                                    <exclude>junit:junit</exclude>
+                                    <exclude>org.apache.commons:commons-lang3</exclude>
+                                    <exclude>org.slf4j:slf4j-api</exclude>
+                                    <exclude>org.slf4j:slf4j-log4j12</exclude>
+                                    <exclude>log4j:log4j</exclude>
+                                    <exclude>org.apache.commons:commons-math</exclude>
+                                    <exclude>org.apache.sling:org.apache.sling.commons.json</exclude>
+                                    <exclude>commons-logging:commons-logging</exclude>
+                                    <exclude>commons-codec:commons-codec</exclude>
+                                    <exclude>com.fasterxml.jackson.core:jackson-core</exclude>
+                                    <exclude>com.fasterxml.jackson.core:jackson-databind</exclude>
+                                    <exclude>com.fasterxml.jackson.core:jackson-annotations</exclude>
+                                    <exclude>stax:stax-api</exclude>
+                                    <exclude>com.typesafe:config</exclude>
+                                    <exclude>org.uncommons.maths:uncommons-maths</exclude>
+                                    <exclude>com.github.scopt:scopt_*</exclude>
+                                    <exclude>commons-io:commons-io</exclude>
+                                    <exclude>commons-cli:commons-cli</exclude>
+                                </excludes>
+                            </artifactSet>
+                            <filters>
+                                <filter>
+                                    <artifact>org.apache.flink:*</artifact>
+                                    <excludes>
+                                        <!-- exclude shaded google but include shaded curator -->
+                                        <exclude>org/apache/flink/shaded/com/**</exclude>
+                                        <exclude>web-docs/**</exclude>
+                                    </excludes>
+                                </filter>
+                                <filter>
+                                    <!-- Do not copy the signatures in the META-INF folder.
+										Otherwise, this might cause SecurityExceptions when using the JAR.
+									-->
+                                    <artifact>*:*</artifact>
+                                    <excludes>
+                                        <exclude>META-INF/*.SF</exclude>
+                                        <exclude>META-INF/*.DSA</exclude>
+                                        <exclude>META-INF/*.RSA</exclude>
+                                    </excludes>
+                                </filter>
+                            </filters>
+                            <!-- If you want to use ./bin/flink run <quickstart jar> uncomment the following lines.
+							This will add a Main-Class entry to the manifest file -->
+                            <!--
+							<transformers>
+								<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+									<mainClass>jar.StreamingJob</mainClass>
+								</transformer>
+							</transformers>
+			    			-->
+                            <createDependencyReducedPom>false</createDependencyReducedPom>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+            </plugin>
+            <plugin>
+                <groupId>org.codehaus.mojo</groupId>
+                <artifactId>buildnumber-maven-plugin</artifactId>
+            </plugin>
+        </plugins>
+        <pluginManagement>
+            <plugins>
+                <plugin>
+                    <artifactId>maven-compiler-plugin</artifactId>
+                    <configuration>
+                        <source>1.8</source>
+                        <target>1.8</target>
+                        <compilerId>jdt</compilerId>
+                    </configuration>
+                    <dependencies>
+                        <dependency>
+                            <groupId>org.eclipse.tycho</groupId>
+                            <artifactId>tycho-compiler-jdt</artifactId>
+                            <version>0.21.0</version>
+                        </dependency>
+                    </dependencies>
+                </plugin>
+                <plugin>
+                    <groupId>org.eclipse.m2e</groupId>
+                    <artifactId>lifecycle-mapping</artifactId>
+                    <version>1.0.0</version>
+                    <configuration>
+                        <lifecycleMappingMetadata>
+                            <pluginExecutions>
+                                <pluginExecution>
+                                    <pluginExecutionFilter>
+                                        <groupId>org.apache.maven.plugins</groupId>
+                                        <artifactId>maven-assembly-plugin</artifactId>
+                                        <versionRange>[2.4,)</versionRange>
+                                        <goals>
+                                            <goal>single</goal>
+                                        </goals>
+                                    </pluginExecutionFilter>
+                                    <action>
+                                        <ignore/>
+                                    </action>
+                                </pluginExecution>
+                                <pluginExecution>
+                                    <pluginExecutionFilter>
+                                        <groupId>org.apache.maven.plugins</groupId>
+                                        <artifactId>maven-compiler-plugin</artifactId>
+                                        <versionRange>[3.1,)</versionRange>
+                                        <goals>
+                                            <goal>testCompile</goal>
+                                            <goal>compile</goal>
+                                        </goals>
+                                    </pluginExecutionFilter>
+                                    <action>
+                                        <ignore/>
+                                    </action>
+                                </pluginExecution>
+                            </pluginExecutions>
+                        </lifecycleMappingMetadata>
+                    </configuration>
+                </plugin>
+            </plugins>
+        </pluginManagement>
+    </build>
+</project>

analytics/src/main/java/timely/analytics/flink/MetricHistogram.java

@@ -0,0 +1,169 @@
+package timely.analytics.flink;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import org.apache.flink.streaming.util.serialization.SerializationSchema;
+
+import timely.model.Tag;
+
+public class MetricHistogram implements Serializable, SerializationSchema<MetricHistogram> {
+
+    private static final long serialVersionUID = 1L;
+    private static final String SPACE = " ";
+    private static final String AVG = " sample=avg\n";
+    private static final String COUNT = " sample=count\n";
+    private static final String MAX = " sample=max\n";
+    private static final String MIN = " sample=min\n";
+    private static final String SUM = " sample=sum\n";
+    private static final String p50 = " sample=50p\n";
+    private static final String p75 = " sample=75p\n";
+    private static final String p90 = " sample=90p\n";
+    private static final String p99 = " sample=99p\n";
+
+    private String metric = null;
+    private List<Tag> tags = null;
+    private long timestamp = 0L;
+    private boolean initialized = false;
+    private List<Double> values = new ArrayList<>();
+
+    public void initialize(String metric, List<Tag> tags) {
+        this.metric = metric;
+        this.tags = tags;
+        initialized = true;
+    }
+
+    public boolean isInitialized() {
+        return initialized;
+    }
+
+    public String getMetric() {
+        return metric;
+    }
+
+    public List<Tag> getTags() {
+        return tags;
+    }
+
+    public long getTimestamp() {
+        return timestamp;
+    }
+
+    public void update(Double d, long timestamp) {
+        values.add(d);
+        this.timestamp = timestamp;
+    }
+
+    public void done() {
+        Collections.sort(values);
+    }
+
+    public Double avg() {
+        if (values.size() == 0) {
+            return 0.0D;
+        }
+        return sum() / count();
+    }
+
+    public Double min() {
+        if (values.size() == 0) {
+            return 0.0D;
+        }
+        return values.get(0);
+    }
+
+    public Double max() {
+        if (values.size() == 0) {
+            return 0.0D;
+        }
+        return values.get(values.size() - 1);
+    }
+
+    public Double getPercentile(int p) {
+        if (values.size() == 0) {
+            return 0.0D;
+        }
+        int position = Math.round((p * 1.0f / 100) * values.size());
+        position = (0 == position) ? 0 : position - 1;
+        return values.get(position);
+    }
+
+    public Double count() {
+        return values.size() * 1.0D;
+    }
+
+    public Double sum() {
+        double sum = 0.0D;
+        for (Double d : values) {
+            sum += d;
+        }
+        return sum;
+    }
+
+    @Override
+    public String toString() {
+        done();
+        StringBuilder buf = new StringBuilder();
+        buf.append("metric: ").append(metric);
+        buf.append(" tags: ").append(tags);
+        buf.append(" timestamp: ").append(timestamp);
+        buf.append(" count: ").append(count());
+        buf.append(" min: ").append(min());
+        buf.append(" max: ").append(max());
+        buf.append(" sum: ").append(sum());
+        buf.append(" 50p: ").append(getPercentile(50));
+        buf.append(" 75p: ").append(getPercentile(75));
+        buf.append(" 99p: ").append(getPercentile(99));
+        return buf.toString();
+    }
+
+    public byte[] serialize(MetricHistogram histo) {
+        StringBuilder buf = new StringBuilder();
+
+        String tags = histo.getTags().stream().map(t -> t.join()).collect(Collectors.joining(" "));
+
+        // @formatter:off
+        buf.append("put ").append(histo.getMetric() + "_summarized")
+            .append(SPACE).append(histo.getTimestamp())
+            .append(SPACE).append(histo.min())
+            .append(SPACE).append(tags).append(MIN);
+        buf.append("put ").append(histo.getMetric() + "_summarized")
+            .append(SPACE).append(histo.getTimestamp())
+            .append(SPACE).append(histo.max())
+            .append(SPACE).append(tags).append(MAX);
+        buf.append("put ").append(histo.getMetric() + "_summarized")
+            .append(SPACE).append(histo.getTimestamp())
+            .append(SPACE).append(histo.count())
+            .append(SPACE).append(tags).append(COUNT);
+        buf.append("put ").append(histo.getMetric() + "_summarized")
+            .append(SPACE).append(histo.getTimestamp())
+            .append(SPACE).append(histo.sum())
+            .append(SPACE).append(tags).append(SUM);
+        buf.append("put ").append(histo.getMetric() + "_summarized")
+            .append(SPACE).append(histo.getTimestamp())
+            .append(SPACE).append(histo.avg())
+            .append(SPACE).append(tags).append(AVG);
+        buf.append("put ").append(histo.getMetric() + "_summarized")
+            .append(SPACE).append(histo.getTimestamp())
+            .append(SPACE).append(histo.getPercentile(50))
+            .append(SPACE).append(tags).append(p50);
+        buf.append("put ").append(histo.getMetric() + "_summarized")
+            .append(SPACE).append(histo.getTimestamp())
+            .append(SPACE).append(histo.getPercentile(75))
+            .append(SPACE).append(tags).append(p75);
+        buf.append("put ").append(histo.getMetric() + "_summarized")
+            .append(SPACE).append(histo.getTimestamp())
+            .append(SPACE).append(histo.getPercentile(90))
+            .append(SPACE).append(tags).append(p90);
+        buf.append("put ").append(histo.getMetric() + "_summarized")
+            .append(SPACE).append(histo.getTimestamp())
+            .append(SPACE).append(histo.getPercentile(99))
+            .append(SPACE).append(tags).append(p99);
+        // @formatter:off
+        return buf.toString().getBytes();
+    }
+
+}

analytics/src/main/java/timely/analytics/flink/MetricKeySelector.java

@@ -0,0 +1,20 @@
+package timely.analytics.flink;
+
+import org.apache.flink.api.java.functions.KeySelector;
+
+import timely.api.response.MetricResponse;
+
+/**
+ * Uniquely defines a time series in a Flink stream. A unique time series is the
+ * metric name and all of its tags, to include visibility if it exists.
+ */
+public class MetricKeySelector implements KeySelector<MetricResponse, String> {
+
+    private static final long serialVersionUID = 1L;
+
+    @Override
+    public String getKey(MetricResponse m) throws Exception {
+        return m.getMetric() + m.getTags().toString();
+    }
+
+}

analytics/src/main/java/timely/analytics/flink/SortedStringAccumulator.java

@@ -0,0 +1,55 @@
+package timely.analytics.flink;
+
+import java.util.concurrent.ConcurrentSkipListMap;
+
+import org.apache.commons.lang3.mutable.MutableLong;
+import org.apache.flink.api.common.accumulators.Accumulator;
+
+/**
+ * Custom accumulator that retains a sorted map of strings and their counts
+ *
+ */
+public class SortedStringAccumulator implements Accumulator<String, ConcurrentSkipListMap<String, MutableLong>> {
+
+    private static final long serialVersionUID = 1L;
+
+    private ConcurrentSkipListMap<String, MutableLong> values = new ConcurrentSkipListMap<>();
+
+    @Override
+    public void add(String value) {
+        if (values.containsKey(value)) {
+            values.get(value).increment();
+        } else {
+            values.put(value, new MutableLong(1));
+        }
+    }
+
+    @Override
+    public ConcurrentSkipListMap<String, MutableLong> getLocalValue() {
+        return values;
+    }
+
+    @Override
+    public void resetLocal() {
+        values.clear();
+    }
+
+    @Override
+    public void merge(Accumulator<String, ConcurrentSkipListMap<String, MutableLong>> other) {
+        other.getLocalValue().forEach((k, v) -> {
+            if (values.containsKey(k)) {
+                values.get(k).add(v.longValue());
+            } else {
+                values.put(k, v);
+            }
+        });
+    }
+
+    @Override
+    public Accumulator<String, ConcurrentSkipListMap<String, MutableLong>> clone() {
+        SortedStringAccumulator a = new SortedStringAccumulator();
+        a.getLocalValue().putAll(this.values);
+        return a;
+    }
+
+}

analytics/src/main/java/timely/analytics/flink/SubscriptionSource.java

@@ -0,0 +1,214 @@
+package timely.analytics.flink;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.RejectedExecutionException;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import javax.websocket.CloseReason;
+import javax.websocket.EndpointConfig;
+import javax.websocket.MessageHandler;
+import javax.websocket.Session;
+
+import org.apache.flink.api.common.accumulators.LongCounter;
+import org.apache.flink.api.common.functions.StoppableFunction;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.streaming.api.functions.source.RichSourceFunction;
+import org.apache.flink.streaming.api.watermark.Watermark;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import timely.api.response.MetricResponse;
+import timely.api.response.MetricResponses;
+import timely.client.websocket.ClientHandler;
+import timely.client.websocket.subscription.WebSocketSubscriptionClient;
+import timely.serialize.JsonSerializer;
+
+public class SubscriptionSource extends RichSourceFunction<MetricResponse> implements Serializable, StoppableFunction {
+
+    private static class SubscriptionThreadFactory implements ThreadFactory {
+
+        private static final String NAME = "Subscription Source";
+        private AtomicInteger threadNum = new AtomicInteger(1);
+
+        @Override
+        public Thread newThread(Runnable r) {
+            return new Thread(r, NAME + " " + threadNum.getAndIncrement());
+        }
+    }
+
+    private static final Logger LOG = LoggerFactory.getLogger(SubscriptionSource.class);
+    private static final long serialVersionUID = 1L;
+
+    private transient WebSocketSubscriptionClient client = null;
+    private long start = 0L;
+    private long end = 0L;
+    private final String[] metrics;
+    private final SummarizationJobParameters jp;
+    private final long window;
+
+    public SubscriptionSource(SummarizationJobParameters jp) {
+        this.jp = jp;
+        start = jp.getStartTime();
+        end = jp.getEndTime();
+        metrics = jp.getMetrics();
+        window = jp.getSummarizationInterval().toMilliseconds();
+    }
+
+    @Override
+    public void open(Configuration parameters) throws Exception {
+        LOG.info("Opening summarization job.");
+        super.open(parameters);
+        client = new WebSocketSubscriptionClient(jp.getTimelyHostname(), jp.getTimelyHttpsPort(),
+                jp.getTimelyWssPort(), jp.isDoLogin(), jp.getTimelyUsername(), jp.getTimelyPassword(),
+                jp.getKeyStoreFile(), jp.getKeyStoreType(), jp.getKeyStorePass(), jp.getTrustStoreFile(),
+                jp.getTrustStoreType(), jp.getTrustStorePass(), jp.isHostVerificationEnabled(), jp.getBufferSize());
+    }
+
+    @Override
+    public synchronized void close() throws Exception {
+        super.close();
+        if (null != client) {
+            client.close();
+        }
+    }
+
+    @Override
+    public void run(SourceContext<MetricResponse> ctx) throws Exception {
+        LOG.info("Running summarization job");
+        final SortedStringAccumulator dateTimeAccumulator = new SortedStringAccumulator();
+        this.getRuntimeContext().addAccumulator("source hourly count", dateTimeAccumulator);
+        final LongCounter sourceInputs = new LongCounter();
+        this.getRuntimeContext().addAccumulator("source input counter", sourceInputs);
+        final LongCounter inputProcessedInMainThread = new LongCounter();
+        this.getRuntimeContext().addAccumulator("input processed in main thread", inputProcessedInMainThread);
+
+        // Use only one thread to ensure that time is still ordered
+        final ExecutorService svc = Executors.newFixedThreadPool(1, new SubscriptionThreadFactory());
+
+        ClientHandler handler = new ClientHandler() {
+
+            @Override
+            public void onOpen(Session session, EndpointConfig config) {
+                session.addMessageHandler(new MessageHandler.Whole<String>() {
+
+                    private final SimpleDateFormat formatter = new SimpleDateFormat("yyyyMMdd-HH");
+                    private long lastWatermarkTime = 0L;
+
+                    @Override
+                    public void onMessage(final String message) {
+                        LOG.info("Message received on Websocket session {}: {}", session.getId(), message);
+                        try {
+                            // Deserialize in this thread
+                            final MetricResponses responses = JsonSerializer.getObjectMapper().readValue(message,
+                                    MetricResponses.class);
+                            // Process in the other
+                            final Runnable r = () -> {
+                                responses.getResponses().forEach(response -> {
+                                    if (response.isComplete()) {
+                                        LOG.info("Received last message.");
+                                        ctx.emitWatermark(new Watermark(Long.MAX_VALUE));
+                                        return;
+                                    }
+                                    LOG.trace("Sending metric: {}", response);
+                                    long time = response.getTimestamp();
+                                    ctx.collectWithTimestamp(response, time);
+                                    dateTimeAccumulator.add(formatter.format(new Date(response.getTimestamp())));
+                                    sourceInputs.add(1);
+                                    // Emit a watermark every second of event
+                                    // time
+                                        if (lastWatermarkTime == 0) {
+                                            lastWatermarkTime = time;
+                                        } else if ((time - lastWatermarkTime) > window) {
+                                            lastWatermarkTime = time;
+                                            ctx.emitWatermark(new Watermark(time - 1));
+                                        }
+                                    });
+                            };
+                            try {
+                                svc.execute(r);
+                            } catch (RejectedExecutionException e) {
+                                // Run it in this thread
+                                r.run();
+                                inputProcessedInMainThread.add(1);
+                            }
+                        } catch (IOException e) {
+                            LOG.error("Error deserializing metric response, closing", e);
+                            try {
+                                close();
+                            } catch (Exception e1) {
+                                LOG.error("Error closing client", e1);
+                            }
+                        }
+                    }
+                });
+            }
+
+            @Override
+            public void onClose(Session session, CloseReason reason) {
+                super.onClose(session, reason);
+                try {
+                    close();
+                } catch (Exception e1) {
+                    LOG.error("Error closing client", e1);
+                }
+                // Signal done sending data
+                ctx.emitWatermark(new Watermark(Long.MAX_VALUE));
+            }
+
+            @Override
+            public void onError(Session session, Throwable error) {
+                super.onError(session, error);
+                try {
+                    close();
+                } catch (Exception e1) {
+                    LOG.error("Error closing client", e1);
+                }
+            }
+
+        };
+        try {
+            client.open(handler);
+            for (String m : metrics) {
+                LOG.info("Adding subscription for {}", m);
+                client.addSubscription(m, null, start, end, 5000);
+            }
+            while (!client.isClosed()) {
+                try {
+                    Thread.sleep(5000);
+                } catch (InterruptedException e) {
+                    LOG.error("Sleep interrupted.");
+                }
+            }
+        } finally {
+            svc.shutdown();
+        }
+    }
+
+    @Override
+    public void cancel() {
+        LOG.info("Cancelling subscription source.");
+        try {
+            close();
+        } catch (Exception e) {
+            LOG.error("Error closing web socket client");
+        }
+    }
+
+    @Override
+    public void stop() {
+        LOG.info("Stopping subscription source.");
+        try {
+            close();
+        } catch (Exception e) {
+            LOG.error("Error closing web socket client");
+        }
+
+    }
+
+}

analytics/src/main/java/timely/analytics/flink/SummarizationJob.java

@@ -0,0 +1,109 @@
+package timely.analytics.flink;
+
+import java.text.SimpleDateFormat;
+import java.util.Date;
+
+import org.apache.flink.api.common.accumulators.LongCounter;
+import org.apache.flink.api.common.functions.FoldFunction;
+import org.apache.flink.api.java.utils.ParameterTool;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.streaming.api.TimeCharacteristic;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.functions.sink.SocketClientSink;
+import org.apache.flink.streaming.api.windowing.assigners.TumblingEventTimeWindows;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import timely.api.response.MetricResponse;
+
+public class SummarizationJob {
+
+    private static final Logger LOG = LoggerFactory.getLogger(SummarizationJob.class);
+
+    public static void main(String[] args) throws Exception {
+
+        if (args.length != 1) {
+            System.err.println("Usage: timely.analytics.flink.SummarizationJob <path_to_properties_file>");
+            System.exit(1);
+        }
+
+        LOG.info("Loading properties from: {}", args[0]);
+        ParameterTool params = ParameterTool.fromPropertiesFile(args[0]);
+
+        final SummarizationJobParameters jp = new SummarizationJobParameters(params);
+        LOG.info("Properties: {}", jp);
+
+        final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+        // @formatter:off
+        env.getConfig().setGlobalJobParameters(jp);
+        env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
+        env.addSource(new SubscriptionSource(jp), "Timely Metrics Subscription")
+        // partition by unique metric
+        .keyBy(new MetricKeySelector())
+        .window(TumblingEventTimeWindows.of(jp.getSummarizationInterval()))
+        .fold(new MetricHistogram(), new FoldFunction<MetricResponse, MetricHistogram>() {
+
+            private final Logger LOG = LoggerFactory
+                    .getLogger("timely.analytics.SummarizationJob.FoldFunction@");
+            private static final long serialVersionUID = 1L;
+
+            @Override
+            public MetricHistogram fold(MetricHistogram histo, MetricResponse metric) throws Exception {
+                if (!histo.isInitialized()) {
+                    histo.initialize(metric.getMetric(), metric.getTags());
+                } else {
+                    if (!histo.getMetric().equals(metric.getMetric())) {
+                        throw new RuntimeException("Metric name does not match - partitioning problem");
+                    }
+                    if (!histo.getTags().equals(metric.getTags())) {
+                        throw new RuntimeException("Tags don't not match - partitioning problem");
+                    }
+                }
+                LOG.trace("Updating histogram for {} with value {} at {} for metric {}", histo.getMetric()
+                        + histo.getTags(), metric.getValue(), metric.getTimestamp(), metric.getMetric()
+                        + metric.getTags());
+                histo.update(metric.getValue(), metric.getTimestamp());
+                return histo;
+            }
+        })
+        .name("Metric Windows")
+        .addSink(
+    		new SocketClientSink<MetricHistogram>(jp.getTimelyHostname(), jp.getTimelyTcpPort(),
+                    new MetricHistogram(), 2, true) {
+
+                private final Logger LOG = LoggerFactory
+                        .getLogger("timely.analytics.SummarizationJob.SocketClientSink");
+                private static final long serialVersionUID = 1L;
+                private final LongCounter sinkCounter = new LongCounter();
+                private final SortedStringAccumulator dateTimeAccumulator = new SortedStringAccumulator();
+                private final SimpleDateFormat formatter = new SimpleDateFormat("yyyyMMdd-HH");
+
+                @Override
+                public void open(Configuration parameters) throws Exception {
+                    super.open(parameters);
+                    this.getRuntimeContext().addAccumulator("sink counter", this.sinkCounter);
+                    this.getRuntimeContext().addAccumulator("sink hourly count", dateTimeAccumulator);
+                    LOG.info("Sink opened");
+                }
+
+                @Override
+                public void invoke(MetricHistogram histo) throws Exception {
+                    histo.done();
+                    super.invoke(histo);
+                    this.sinkCounter.add(1);
+                    this.dateTimeAccumulator.add(formatter.format(new Date(histo.getTimestamp())));
+                }
+
+                @Override
+                public void close() throws Exception {
+                    super.close();
+                    LOG.info("Sink closed");
+                }
+
+            }).name("Timely Metric Sink");
+        // @formatter:on
+        env.execute("Metric Summarization");
+        // Looks like there is no way to stop Flink job if underlying web socket
+        // is closed.
+    }
+}

analytics/src/main/java/timely/analytics/flink/SummarizationJobParameters.java

@@ -0,0 +1,208 @@
+package timely.analytics.flink;
+
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.flink.api.common.ExecutionConfig.GlobalJobParameters;
+import org.apache.flink.api.java.utils.ParameterTool;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.streaming.api.windowing.time.Time;
+
+public class SummarizationJobParameters extends GlobalJobParameters implements Serializable {
+
+    private static final long serialVersionUID = 1L;
+
+    private final String timelyHostname;
+    private final int timelyTcpPort;
+    private final int timelyHttpsPort;
+    private final int timelyWssPort;
+    private final boolean doLogin;
+    private final String timelyUsername;
+    private final String timelyPassword;
+    private final String keyStoreFile;
+    private final String keyStoreType;
+    private final String keyStorePass;
+    private final String trustStoreFile;
+    private final String trustStoreType;
+    private final String trustStorePass;
+    private final boolean hostVerificationEnabled;
+    private final int bufferSize;
+    private final String[] metrics;
+    private final long startTime;
+    private final long endTime;
+    private final String interval;
+    private final String intervalUnits;
+
+    public SummarizationJobParameters(ParameterTool params) {
+        timelyHostname = params.getRequired("timelyHostname");
+        timelyTcpPort = params.getInt("timelyTcpPort", 4241);
+        timelyHttpsPort = params.getInt("timelyHttpsPort", 4242);
+        timelyWssPort = params.getInt("timelyWssPort", 4243);
+        doLogin = params.getBoolean("doLogin", false);
+        timelyUsername = params.get("timelyUsername", null);
+        timelyPassword = params.get("timelyPassword", null);
+        keyStoreFile = params.getRequired("keyStoreFile");
+        keyStoreType = params.get("keyStoreType", "JKS");
+        keyStorePass = params.getRequired("keyStorePass");
+        trustStoreFile = params.getRequired("trustStoreFile");
+        trustStoreType = params.get("trustStoreType", "JKS");
+        trustStorePass = params.getRequired("trustStorePass");
+        hostVerificationEnabled = params.getBoolean("hostVerificationEnabled", true);
+        bufferSize = params.getInt("bufferSize", 10485760);
+        String metricNames = params.getRequired("metrics");
+        if (null != metricNames) {
+            metrics = metricNames.split(",");
+        } else {
+            metrics = null;
+        }
+        startTime = params.getLong("startTime", 0L);
+        endTime = params.getLong("endTime", 0L);
+        interval = params.getRequired("interval");
+        intervalUnits = params.getRequired("intervalUnits");
+    }
+
+    public SummarizationJobParameters(Configuration conf) {
+        System.out.println("Creating job parameters from configuration: " + conf);
+        timelyHostname = conf.getString("timelyHostname", null);
+        timelyTcpPort = conf.getInteger("timelyTcpPort", 4241);
+        timelyHttpsPort = conf.getInteger("timelyHttpsPort", 4242);
+        timelyWssPort = conf.getInteger("timelyWssPort", 4243);
+        doLogin = conf.getBoolean("doLogin", false);
+        timelyUsername = conf.getString("timelyUsername", null);
+        timelyPassword = conf.getString("timelyPassword", null);
+        keyStoreFile = conf.getString("keyStoreFile", null);
+        keyStoreType = conf.getString("keyStoreType", "JKS");
+        keyStorePass = conf.getString("keyStorePass", null);
+        trustStoreFile = conf.getString("trustStoreFile", null);
+        trustStoreType = conf.getString("trustStoreType", "JKS");
+        trustStorePass = conf.getString("trustStorePass", null);
+        hostVerificationEnabled = conf.getBoolean("hostVerificationEnabled", true);
+        bufferSize = conf.getInteger("bufferSize", 10485760);
+        String metricNames = conf.getString("metrics", null);
+        if (null != metricNames) {
+            metrics = metricNames.split(",");
+        } else {
+            metrics = null;
+        }
+        startTime = conf.getLong("startTime", 0L);
+        endTime = conf.getLong("endTime", 0L);
+        interval = conf.getString("interval", null);
+        intervalUnits = conf.getString("intervalUnits", null);
+    }
+
+    public Time getSummarizationInterval() {
+        return Time.of(Long.parseLong(interval), TimeUnit.valueOf(intervalUnits));
+    }
+
+    public String getTimelyHostname() {
+        return timelyHostname;
+    }
+
+    public int getTimelyTcpPort() {
+        return timelyTcpPort;
+    }
+
+    public int getTimelyHttpsPort() {
+        return timelyHttpsPort;
+    }
+
+    public int getTimelyWssPort() {
+        return timelyWssPort;
+    }
+
+    public boolean isDoLogin() {
+        return doLogin;
+    }
+
+    public String getTimelyUsername() {
+        return timelyUsername;
+    }
+
+    public String getTimelyPassword() {
+        return timelyPassword;
+    }
+
+    public String getKeyStoreFile() {
+        return keyStoreFile;
+    }
+
+    public String getKeyStoreType() {
+        return keyStoreType;
+    }
+
+    public String getKeyStorePass() {
+        return keyStorePass;
+    }
+
+    public String getTrustStoreFile() {
+        return trustStoreFile;
+    }
+
+    public String getTrustStoreType() {
+        return trustStoreType;
+    }
+
+    public String getTrustStorePass() {
+        return trustStorePass;
+    }
+
+    public boolean isHostVerificationEnabled() {
+        return hostVerificationEnabled;
+    }
+
+    public int getBufferSize() {
+        return bufferSize;
+    }
+
+    public String[] getMetrics() {
+        return metrics;
+    }
+
+    public long getStartTime() {
+        return startTime;
+    }
+
+    public long getEndTime() {
+        return endTime;
+    }
+
+    public String getInterval() {
+        return interval;
+    }
+
+    public String getIntervalUnits() {
+        return intervalUnits;
+    }
+
+    @Override
+    public Map<String, String> toMap() {
+        Map<String, String> m = new HashMap<>();
+        m.put("timelyHostname", this.timelyHostname);
+        m.put("timelyHttpsPort", Integer.toString(this.timelyHttpsPort));
+        m.put("timelyWssPort", Integer.toString(this.timelyWssPort));
+        m.put("doLogin", Boolean.toString(this.doLogin));
+        m.put("timelyUsername", this.timelyUsername);
+        m.put("timelyPassword", this.timelyPassword);
+        m.put("keyStoreFile", this.keyStoreFile);
+        m.put("keyStoreType", this.keyStoreType);
+        m.put("keyStorePass", this.keyStorePass);
+        m.put("trustStoreFile", this.trustStoreFile);
+        m.put("trustStoreType", this.trustStoreType);
+        m.put("trustStorePass", this.trustStorePass);
+        m.put("hostVerificationEnabled", Boolean.toString(this.hostVerificationEnabled));
+        m.put("bufferSize", Integer.toString(this.bufferSize));
+        m.put("metrics", String.join(",", this.metrics));
+        m.put("startTime", Long.toString(this.startTime));
+        m.put("endTime", Long.toString(this.endTime));
+        m.put("interval", this.interval);
+        m.put("intervalUnits", this.intervalUnits);
+        return m;
+    }
+
+    @Override
+    public String toString() {
+        return toMap().toString();
+    }
+}

analytics/src/test/java/timely/analytics/flink/MetricHistogramTest.java

@@ -0,0 +1,108 @@
+package timely.analytics.flink;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import timely.model.Metric;
+import timely.model.Tag;
+import timely.model.parse.MetricParser;
+
+public class MetricHistogramTest {
+
+    private MetricHistogram m = null;
+    private long timestamp = System.currentTimeMillis();
+
+    @Before
+    public void createMetric() throws Exception {
+        m = new MetricHistogram();
+        for (double d = 1.0D; d <= 100.0; d += 1.0) {
+            timestamp += 1000;
+            m.update(d, timestamp);
+        }
+        m.toString();
+        Assert.assertEquals(timestamp, m.getTimestamp());
+    }
+
+    @Test
+    public void testMin() throws Exception {
+        Assert.assertEquals(1.0D, m.min(), 0.0D);
+    }
+
+    @Test
+    public void testMax() throws Exception {
+        Assert.assertEquals(100.0D, m.max(), 0.0D);
+    }
+
+    @Test
+    public void testAvg() throws Exception {
+        int sum = 0;
+        for (int i = 1; i <= 100; i++) {
+            sum += i;
+        }
+        Assert.assertEquals((sum / 100.0D), m.avg(), 0.0D);
+    }
+
+    @Test
+    public void testCount() throws Exception {
+        Assert.assertEquals(100.0D, m.count(), 0.0D);
+    }
+
+    @Test
+    public void test50thPercentile() throws Exception {
+        Assert.assertEquals(50.0D, m.getPercentile(50), 0.0D);
+    }
+
+    @Test
+    public void test75thPercentile() throws Exception {
+        Assert.assertEquals(75.0D, m.getPercentile(75), 0.0D);
+    }
+
+    @Test
+    public void test90thPercentile() throws Exception {
+        Assert.assertEquals(90.0D, m.getPercentile(90), 0.0D);
+    }
+
+    @Test
+    public void test99thPercentile() throws Exception {
+        Assert.assertEquals(99.0D, m.getPercentile(99), 0.0D);
+    }
+
+    @Test
+    public void testSerialization() throws Exception {
+        MetricParser metricParser = new MetricParser();
+        Tag t1 = new Tag("tag1=value1");
+        Tag t2 = new Tag("tag2=value2");
+        Tag avg = new Tag("sample=avg");
+        Tag min = new Tag("sample=min");
+        Tag max = new Tag("sample=max");
+        Tag sum = new Tag("sample=sum");
+        Tag count = new Tag("sample=count");
+        Tag p50 = new Tag("sample=50p");
+        Tag p75 = new Tag("sample=75p");
+        Tag p90 = new Tag("sample=90p");
+        Tag p99 = new Tag("sample=99p");
+
+        List<Tag> tags = new ArrayList<>();
+        tags.add(t1);
+        tags.add(t2);
+        m.initialize("sys.cpu.user", tags);
+
+        byte[] bytes = m.serialize(m);
+        String puts = new String(bytes);
+        for (String put : puts.split("\n")) {
+            Metric metric = metricParser.parse(put);
+            Assert.assertEquals("sys.cpu.user_summarized", metric.getName());
+            metric.getTags().forEach(
+                    t -> {
+                        Assert.assertTrue(t.equals(t1) || t.equals(t2) || t.equals(avg) || t.equals(min)
+                                || t.equals(max) || t.equals(sum) || t.equals(count) || t.equals(p50) || t.equals(p75)
+                                || t.equals(p90) || t.equals(p99));
+                    });
+        }
+    }
+
+}

client/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>timely</groupId>
         <artifactId>timely</artifactId>
-        <version>0.0.3</version>
+        <version>0.0.4</version>
     </parent>
     <dependencies>
         <dependency>
@@ -39,6 +39,14 @@
             <artifactId>junit</artifactId>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-lang3</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.httpcomponents</groupId>
+            <artifactId>httpclient</artifactId>
+        </dependency>
         <dependency>
             <groupId>org.apache.logging.log4j</groupId>
             <artifactId>log4j-api</artifactId>
@@ -54,6 +62,16 @@
             <artifactId>log4j-slf4j-impl</artifactId>
             <scope>runtime</scope>
         </dependency>
+        <dependency>
+        	<groupId>org.glassfish.tyrus</groupId>
+        	<artifactId>tyrus-client</artifactId>
+        	<version>1.13</version>
+        </dependency>
+        <dependency>
+        	<groupId>org.glassfish.tyrus</groupId>
+        	<artifactId>tyrus-container-grizzly-client</artifactId>	
+        	<version>1.13</version>
+        </dependency>
         <dependency>
             <groupId>org.slf4j</groupId>
             <artifactId>slf4j-api</artifactId>
@@ -77,6 +95,10 @@
                 <groupId>net.revelc.code</groupId>
                 <artifactId>formatter-maven-plugin</artifactId>
             </plugin>
+            <plugin>
+                <groupId>org.codehaus.mojo</groupId>
+                <artifactId>buildnumber-maven-plugin</artifactId>
+            </plugin>
         </plugins>
     </build>
     <reporting>

client/src/main/java/timely/api/AddSubscription.java

@@ -0,0 +1,82 @@
+package timely.api;
+
+import java.util.Map;
+import java.util.Optional;
+
+public class AddSubscription extends SubscriptionRequest {
+
+    private String operation = "add";
+    private String metric = null;
+    private Optional<Map<String, String>> tags = Optional.empty();
+    private Optional<Long> startTime = Optional.empty();
+    private Optional<Long> endTime = Optional.empty();
+    private Optional<Long> delayTime = Optional.empty();
+
+    public String getOperation() {
+        return operation;
+    }
+
+    public String getMetric() {
+        return metric;
+    }
+
+    public Optional<Map<String, String>> getTags() {
+        return tags;
+    }
+
+    public Optional<Long> getStartTime() {
+        return startTime;
+    }
+
+    public Optional<Long> getDelayTime() {
+        return delayTime;
+    }
+
+    public void setMetric(String metric) {
+        this.metric = metric;
+    }
+
+    public void setTags(Optional<Map<String, String>> tags) {
+        this.tags = tags;
+    }
+
+    public void setStartTime(Optional<Long> startTime) {
+        this.startTime = startTime;
+    }
+
+    public void setDelayTime(Optional<Long> delayTime) {
+        this.delayTime = delayTime;
+    }
+
+    public Optional<Long> getEndTime() {
+        return endTime;
+    }
+
+    public void setEndTime(Optional<Long> endTime) {
+        this.endTime = endTime;
+    }
+
+    @Override
+    public int hashCode() {
+        return (operation + subscriptionId + metric + tags + startTime + endTime + delayTime).hashCode();
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (null == obj) {
+            return false;
+        }
+        if (obj == this) {
+            return true;
+        }
+        if (obj instanceof AddSubscription) {
+            AddSubscription other = (AddSubscription) obj;
+            return (this.subscriptionId.equals(other.subscriptionId) && this.metric.equals(other.metric)
+                    && this.tags.equals(other.tags) && this.startTime.equals(other.startTime)
+                    && this.endTime.equals(other.endTime) && this.delayTime.equals(other.delayTime));
+        } else {
+            return false;
+        }
+    }
+
+}

client/src/main/java/timely/api/BasicAuthLogin.java

@@ -0,0 +1,45 @@
+package timely.api;
+
+public class BasicAuthLogin {
+
+    protected String username;
+    protected String password;
+
+    public String getUsername() {
+        return username;
+    }
+
+    public void setUsername(String user) {
+        this.username = user;
+    }
+
+    public String getPassword() {
+        return password;
+    }
+
+    public void setPassword(String password) {
+        this.password = password;
+    }
+
+    @Override
+    public int hashCode() {
+        return (username + password).hashCode();
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (null == obj) {
+            return false;
+        }
+        if (obj == this) {
+            return true;
+        }
+        if (obj instanceof BasicAuthLogin) {
+            BasicAuthLogin other = (BasicAuthLogin) obj;
+            return (this.username.equals(other.username) && this.password.equals(other.password));
+        } else {
+            return false;
+        }
+    }
+
+}

client/src/main/java/timely/api/CloseSubscription.java

@@ -0,0 +1,32 @@
+package timely.api;
+
+public class CloseSubscription extends SubscriptionRequest {
+
+    private String operation = "close";
+
+    public String getOperation() {
+        return operation;
+    }
+
+    @Override
+    public int hashCode() {
+        return (operation + subscriptionId).hashCode();
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (null == obj) {
+            return false;
+        }
+        if (obj == this) {
+            return true;
+        }
+        if (obj instanceof CloseSubscription) {
+            CloseSubscription other = (CloseSubscription) obj;
+            return (this.subscriptionId.equals(other.subscriptionId));
+        } else {
+            return false;
+        }
+    }
+
+}

client/src/main/java/timely/api/CreateSubscription.java

@@ -0,0 +1,32 @@
+package timely.api;
+
+public class CreateSubscription extends SubscriptionRequest {
+
+    private String operation = "create";
+
+    public String getOperation() {
+        return operation;
+    }
+
+    @Override
+    public int hashCode() {
+        return (operation + subscriptionId).hashCode();
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (null == obj) {
+            return false;
+        }
+        if (obj == this) {
+            return true;
+        }
+        if (obj instanceof CreateSubscription) {
+            CreateSubscription other = (CreateSubscription) obj;
+            return (this.subscriptionId.equals(other.subscriptionId));
+        } else {
+            return false;
+        }
+    }
+
+}

client/src/main/java/timely/api/RemoveSubscription.java

@@ -0,0 +1,41 @@
+package timely.api;
+
+public class RemoveSubscription extends SubscriptionRequest {
+
+    private String operation = "remove";
+    private String metric = null;
+
+    public String getOperation() {
+        return operation;
+    }
+
+    public String getMetric() {
+        return metric;
+    }
+
+    public void setMetric(String metric) {
+        this.metric = metric;
+    }
+
+    @Override
+    public int hashCode() {
+        return (operation + subscriptionId + metric).hashCode();
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (null == obj) {
+            return false;
+        }
+        if (obj == this) {
+            return true;
+        }
+        if (obj instanceof RemoveSubscription) {
+            RemoveSubscription other = (RemoveSubscription) obj;
+            return (this.subscriptionId.equals(other.subscriptionId) && this.metric.equals(other.metric));
+        } else {
+            return false;
+        }
+    }
+
+}

client/src/main/java/timely/api/SubscriptionRequest.java

@@ -0,0 +1,15 @@
+package timely.api;
+
+public abstract class SubscriptionRequest {
+
+    protected String subscriptionId = null;
+
+    public final String getSubscriptionId() {
+        return subscriptionId;
+    }
+
+    public final void setSubscriptionId(String subscriptionId) {
+        this.subscriptionId = subscriptionId;
+    }
+
+}

client/src/main/java/timely/api/response/MetricResponse.java

@@ -16,6 +16,7 @@ public class MetricResponse {
     private double value;
     private List<Tag> tags;
     private String subscriptionId;
+    private boolean complete = false;
 
     public String getMetric() {
         return metric;
@@ -33,6 +34,10 @@ public class MetricResponse {
         return tags;
     }
 
+    public boolean isComplete() {
+        return complete;
+    }
+
     public void setMetric(String metric) {
         this.metric = metric;
     }
@@ -57,6 +62,10 @@ public class MetricResponse {
         this.subscriptionId = subscriptionId;
     }
 
+    public void setComplete(boolean closed) {
+        this.complete = closed;
+    }
+
     public static MetricResponse fromMetric(Metric metric, String subscriptionId) {
         MetricResponse response = new MetricResponse();
         response.setMetric(metric.getName());
@@ -64,6 +73,7 @@ public class MetricResponse {
         response.setTags(metric.getTags());
         response.setValue(metric.getValue().getMeasure());
         response.setSubscriptionId(subscriptionId);
+        response.setComplete(false);
         return response;
     }
 
@@ -75,6 +85,7 @@ public class MetricResponse {
         hash.append(this.value);
         hash.append(this.tags);
         hash.append(this.subscriptionId);
+        hash.append(this.complete);
         return hash.hashCode();
     }
 
@@ -96,6 +107,7 @@ public class MetricResponse {
         equals.append(this.value, other.value);
         equals.append(this.tags, other.tags);
         equals.append(this.subscriptionId, other.subscriptionId);
+        equals.append(this.complete, other.complete);
         return equals.isEquals();
     }
 
@@ -107,6 +119,7 @@ public class MetricResponse {
         tsb.append("value", this.value);
         tsb.append("tags", this.tags);
         tsb.append("subscriptionId", this.subscriptionId);
+        tsb.append("complete", this.complete);
         return tsb.toString();
     }
 

client/src/main/java/timely/api/response/MetricResponses.java

@@ -0,0 +1,29 @@
+package timely.api.response;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class MetricResponses {
+
+    private List<MetricResponse> responses = new ArrayList<>();
+
+    public synchronized List<MetricResponse> getResponses() {
+        return this.responses;
+    }
+
+    public synchronized void setResponses(List<MetricResponse> responses) {
+        this.responses = responses;
+    }
+
+    public synchronized void addResponse(MetricResponse r) {
+        this.responses.add(r);
+    }
+
+    public int size() {
+        return this.responses.size();
+    }
+
+    public synchronized void clear() {
+        this.responses.clear();
+    }
+}

client/src/main/java/timely/client/http/HttpClient.java

@@ -0,0 +1,55 @@
+package timely.client.http;
+
+import javax.net.ssl.SSLContext;
+
+import org.apache.http.client.CookieStore;
+import org.apache.http.client.config.CookieSpecs;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.conn.ssl.DefaultHostnameVerifier;
+import org.apache.http.conn.ssl.NoopHostnameVerifier;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClientBuilder;
+import org.apache.http.impl.client.HttpClients;
+import org.glassfish.tyrus.client.SslContextConfigurator;
+
+public class HttpClient {
+
+    public static SSLContext getSSLContext(String trustStoreFile, String trustStoreType, String trustStorePass,
+            String keyStoreFile, String keyStoreType, String keyStorePass) {
+        SslContextConfigurator scc = new SslContextConfigurator();
+        scc.setTrustStoreFile(trustStoreFile);
+        scc.setTrustStoreType(trustStoreType);
+        if (null != trustStorePass) {
+            scc.setTrustStorePassword(trustStorePass);
+        }
+        scc.setKeyStoreFile(keyStoreFile);
+        scc.setKeyStoreType(keyStoreType);
+        if (null != keyStorePass) {
+            scc.setKeyStorePassword(keyStorePass);
+        }
+        return scc.createSSLContext();
+    }
+
+    public static CloseableHttpClient get(String trustStoreFile, String trustStoreType, String trustStorePass,
+            String keyStoreFile, String keyStoreType, String keyStorePass, CookieStore cookieStore,
+            boolean hostVerificationEnabled) {
+
+        SSLContext ssl = getSSLContext(trustStoreFile, trustStoreType, trustStorePass, keyStoreFile, keyStoreType,
+                keyStorePass);
+        return get(ssl, cookieStore, hostVerificationEnabled);
+    }
+
+    public static CloseableHttpClient get(SSLContext ssl, CookieStore cookieStore, boolean hostVerificationEnabled) {
+        RequestConfig defaultRequestConfig = RequestConfig.custom().setCookieSpec(CookieSpecs.STANDARD).build();
+
+        HttpClientBuilder builder = HttpClients.custom().setSSLContext(ssl).setDefaultCookieStore(cookieStore)
+                .setDefaultRequestConfig(defaultRequestConfig);
+        if (hostVerificationEnabled) {
+            builder.setSSLHostnameVerifier(new DefaultHostnameVerifier());
+        } else {
+            builder.setSSLHostnameVerifier(new NoopHostnameVerifier());
+        }
+        return builder.build();
+    }
+
+}

client/src/main/java/timely/client/tcp/TcpClient.java

@@ -0,0 +1,106 @@
+package timely.client.tcp;
+
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.net.Socket;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class TcpClient implements AutoCloseable {
+
+    private static final Logger LOG = LoggerFactory.getLogger(TcpClient.class);
+
+    private final String host;
+    private final int port;
+    private Socket sock = null;
+    private PrintWriter out = null;
+    private long connectTime = 0L;
+    private long backoff = 2000;
+
+    public TcpClient(String hostname, int port) {
+        this.host = hostname;
+        this.port = port;
+    }
+
+    /**
+     * Opens a TCP connection to the specified host and port
+     * 
+     * @throws IOException
+     *             if an error occurs
+     */
+    public void open() throws IOException {
+        if (connect() != 0) {
+            throw new IOException();
+        }
+    }
+
+    /**
+     * Write a metric to Timely
+     * 
+     * @param metric
+     *            newline terminated string representation of Timely metric
+     * @throws IOException
+     */
+    public synchronized void write(String metric) throws IOException {
+        if (connect() != 0) {
+            throw new IOException();
+        }
+        out.write(metric);
+    }
+
+    public synchronized void flush() {
+        if (null != out) {
+            out.flush();
+        }
+    }
+
+    /**
+     * Closes the tcp connection to Timely
+     * 
+     * @throws IOException
+     *             if an error occurs
+     */
+    @Override
+    public void close() throws IOException {
+        LOG.info("Shutting down connection to Timely at {}:{}", host, port);
+        if (null != sock) {
+            try {
+                if (null != out) {
+                    out.close();
+                }
+                sock.close();
+            } catch (IOException e) {
+                LOG.error("Error closing connection to Timely at " + host + ":" + port + ". Error: " + e.getMessage());
+            } finally {
+                sock = null;
+            }
+        }
+    }
+
+    private synchronized int connect() {
+        if (null == sock || !sock.isConnected() || out.checkError()) {
+            if (System.currentTimeMillis() > (connectTime + backoff)) {
+                try {
+                    connectTime = System.currentTimeMillis();
+                    sock = new Socket(host, port);
+                    out = new PrintWriter(sock.getOutputStream(), false);
+                    backoff = 2000;
+                    LOG.info("Connected to Timely at {}:{}", host, port);
+                } catch (IOException e) {
+                    LOG.error("Error connecting to Timely at {}:" + host + ":" + port + ". Error: " + e.getMessage());
+                    backoff = backoff * 2;
+                    sock = null;
+                    out = null;
+                    LOG.warn("Will retry connection in {} ms.", backoff);
+                    return -1;
+                }
+            } else {
+                LOG.warn("Not writing to Timely, waiting to reconnect");
+                return -1;
+            }
+        }
+        return 0;
+    }
+
+}

client/src/main/java/timely/client/udp/UdpClient.java

@@ -0,0 +1,44 @@
+package timely.client.udp;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import java.io.IOException;
+import java.net.DatagramPacket;
+import java.net.DatagramSocket;
+import java.net.InetSocketAddress;
+
+public class UdpClient implements AutoCloseable {
+
+    private final InetSocketAddress address;
+    private final DatagramPacket packet;
+    private DatagramSocket sock;
+
+    public UdpClient(String hostname, int port) {
+        this.address = new InetSocketAddress(hostname, port);
+        this.packet = new DatagramPacket("".getBytes(UTF_8), 0, 0, address.getAddress(), port);
+    }
+
+    public void open() throws IOException {
+        if (null == sock) {
+            this.sock = new DatagramSocket();
+        }
+    }
+
+    public void write(String metric) throws IOException {
+        if (null == this.sock) {
+            throw new IllegalStateException("Must call open first");
+        }
+        this.packet.setData(metric.getBytes(UTF_8));
+        this.sock.send(packet);
+    }
+
+    public void flush() throws IOException {
+    }
+
+    public void close() throws IOException {
+        if (null != sock) {
+            sock.close();
+        }
+    }
+
+}

client/src/main/java/timely/client/websocket/ClientHandler.java

@@ -0,0 +1,38 @@
+package timely.client.websocket;
+
+import javax.websocket.CloseReason;
+import javax.websocket.Endpoint;
+import javax.websocket.EndpointConfig;
+import javax.websocket.MessageHandler;
+import javax.websocket.Session;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public abstract class ClientHandler extends Endpoint {
+
+    private final static Logger LOG = LoggerFactory.getLogger(ClientHandler.class);
+
+    @Override
+    public void onOpen(Session session, EndpointConfig config) {
+        LOG.info("Websocket session {} opened.", session.getId());
+        session.addMessageHandler(new MessageHandler.Whole<String>() {
+
+            @Override
+            public void onMessage(String message) {
+                LOG.info("Message received on Websocket session {}: {}", session.getId(), message);
+            }
+        });
+    }
+
+    @Override
+    public void onClose(Session session, CloseReason reason) {
+        LOG.info("Websocket session {} closed.", session.getId());
+    }
+
+    @Override
+    public void onError(Session session, Throwable error) {
+        LOG.error("Error occurred on Websocket session" + session.getId(), error);
+    }
+
+}

client/src/main/java/timely/client/websocket/TimelyEndpointConfig.java

@@ -0,0 +1,65 @@
+package timely.client.websocket;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import javax.websocket.ClientEndpointConfig;
+import javax.websocket.Decoder;
+import javax.websocket.Encoder;
+import javax.websocket.Extension;
+
+import org.apache.http.cookie.Cookie;
+
+public class TimelyEndpointConfig implements ClientEndpointConfig {
+
+    private String sessionCookie = null;
+
+    private static final String FORMAT = "%s=%s;";
+
+    public TimelyEndpointConfig(Cookie sessionCookie) {
+        if (null != sessionCookie) {
+            this.sessionCookie = String.format(FORMAT, sessionCookie.getName(), sessionCookie.getValue());
+        }
+    }
+
+    @Override
+    public List<Class<? extends Encoder>> getEncoders() {
+        return Collections.emptyList();
+    }
+
+    @Override
+    public List<Class<? extends Decoder>> getDecoders() {
+        return Collections.emptyList();
+    }
+
+    @Override
+    public Map<String, Object> getUserProperties() {
+        return Collections.emptyMap();
+    }
+
+    @Override
+    public List<String> getPreferredSubprotocols() {
+        return Collections.emptyList();
+    }
+
+    @Override
+    public List<Extension> getExtensions() {
+        return Collections.emptyList();
+    }
+
+    @Override
+    public Configurator getConfigurator() {
+        return new Configurator() {
+
+            @Override
+            public void beforeRequest(Map<String, List<String>> headers) {
+                super.beforeRequest(headers);
+                if (null != sessionCookie) {
+                    headers.put("Cookie", Collections.singletonList(sessionCookie));
+                }
+            }
+        };
+    }
+
+}

client/src/main/java/timely/client/websocket/WebSocketClient.java

@@ -0,0 +1,179 @@
+package timely.client.websocket;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.nio.ByteBuffer;
+import java.util.concurrent.TimeUnit;
+
+import javax.net.ssl.SSLContext;
+import javax.websocket.DeploymentException;
+import javax.websocket.Session;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.http.HttpResponse;
+import org.apache.http.HttpStatus;
+import org.apache.http.client.HttpResponseException;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.methods.HttpRequestBase;
+import org.apache.http.client.utils.HttpClientUtils;
+import org.apache.http.cookie.Cookie;
+import org.apache.http.entity.StringEntity;
+import org.apache.http.impl.client.BasicCookieStore;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.glassfish.tyrus.client.ClientManager;
+import org.glassfish.tyrus.client.ClientProperties;
+import org.glassfish.tyrus.client.SslEngineConfigurator;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import timely.api.BasicAuthLogin;
+import timely.client.http.HttpClient;
+import timely.serialize.JsonSerializer;
+
+import com.google.common.base.Preconditions;
+
+public class WebSocketClient implements AutoCloseable {
+
+    private static final Logger LOG = LoggerFactory.getLogger(WebSocketClient.class);
+
+    private final String timelyHostname;
+    private final int timelyHttpsPort;
+    private final int timelyWssPort;
+    private final boolean doLogin;
+    private final String timelyUsername;
+    private final String timelyPassword;
+    private final boolean hostVerificationEnabled;
+    private final int bufferSize;
+    private final SSLContext ssl;
+
+    private ClientManager webSocketClient = null;
+    protected Session session = null;
+    protected volatile boolean closed = true;
+
+    protected WebSocketClient(SSLContext ssl, String timelyHostname, int timelyHttpsPort, int timelyWssPort,
+            boolean doLogin, String timelyUsername, String timelyPassword, boolean hostVerificationEnabled,
+            int bufferSize) {
+        this.ssl = ssl;
+        this.timelyHostname = timelyHostname;
+        this.timelyHttpsPort = timelyHttpsPort;
+        this.timelyWssPort = timelyWssPort;
+        this.doLogin = doLogin;
+        this.timelyUsername = timelyUsername;
+        this.timelyPassword = timelyPassword;
+        this.hostVerificationEnabled = hostVerificationEnabled;
+        this.bufferSize = bufferSize;
+
+        Preconditions.checkNotNull(timelyHostname, "%s must be supplied", "Timely host name");
+        Preconditions.checkNotNull(timelyHttpsPort, "%s must be supplied", "Timely HTTPS port");
+        Preconditions.checkNotNull(timelyWssPort, "%s must be supplied", "Timely WSS port");
+
+        if (doLogin
+                && ((StringUtils.isEmpty(timelyUsername) && !StringUtils.isEmpty(timelyPassword) || (!StringUtils
+                        .isEmpty(timelyUsername) && StringUtils.isEmpty(timelyPassword))))) {
+            throw new IllegalArgumentException("Both Timely username and password must be empty or non-empty");
+        }
+
+    }
+
+    protected WebSocketClient(String timelyHostname, int timelyHttpsPort, int timelyWssPort, boolean doLogin,
+            String timelyUsername, String timelyPassword, String keyStoreFile, String keyStoreType,
+            String keyStorePass, String trustStoreFile, String trustStoreType, String trustStorePass,
+            boolean hostVerificationEnabled, int bufferSize) {
+        this(HttpClient.getSSLContext(trustStoreFile, trustStoreType, trustStorePass, keyStoreFile, keyStoreType,
+                keyStorePass), timelyHostname, timelyHttpsPort, timelyWssPort, doLogin, timelyUsername, timelyPassword,
+                hostVerificationEnabled, bufferSize);
+    }
+
+    public void open(ClientHandler clientEndpoint) throws IOException, DeploymentException, URISyntaxException {
+
+        Cookie sessionCookie = null;
+        if (doLogin) {
+            BasicCookieStore cookieJar = new BasicCookieStore();
+            try (CloseableHttpClient client = HttpClient.get(ssl, cookieJar, hostVerificationEnabled)) {
+
+                String target = "https://" + timelyHostname + ":" + timelyHttpsPort + "/login";
+
+                HttpRequestBase request = null;
+                if (StringUtils.isEmpty(timelyUsername)) {
+                    // HTTP GET to /login to use certificate based login
+                    request = new HttpGet(target);
+                    LOG.trace("Performing client certificate login");
+                } else {
+                    // HTTP POST to /login to use username/password
+                    BasicAuthLogin login = new BasicAuthLogin();
+                    login.setUsername(timelyUsername);
+                    login.setPassword(timelyPassword);
+                    String payload = JsonSerializer.getObjectMapper().writeValueAsString(login);
+                    HttpPost post = new HttpPost(target);
+                    post.setEntity(new StringEntity(payload));
+                    request = post;
+                    LOG.trace("Performing BasicAuth login");
+                }
+
+                HttpResponse response = null;
+                try {
+                    response = client.execute(request);
+                    if (response.getStatusLine().getStatusCode() != HttpStatus.SC_OK) {
+                        throw new HttpResponseException(response.getStatusLine().getStatusCode(), response
+                                .getStatusLine().getReasonPhrase());
+                    }
+                    for (Cookie c : cookieJar.getCookies()) {
+                        if (c.getName().equals("TSESSIONID")) {
+                            sessionCookie = c;
+                            break;
+                        }
+                    }
+                    if (null == sessionCookie) {
+                        throw new IllegalStateException(
+                                "Unable to find TSESSIONID cookie header in Timely login response");
+                    }
+                } finally {
+                    HttpClientUtils.closeQuietly(response);
+                }
+            }
+        }
+
+        SslEngineConfigurator sslEngine = new SslEngineConfigurator(ssl);
+        sslEngine.setClientMode(true);
+        sslEngine.setHostVerificationEnabled(hostVerificationEnabled);
+
+        webSocketClient = ClientManager.createClient();
+        webSocketClient.getProperties().put(ClientProperties.SSL_ENGINE_CONFIGURATOR, sslEngine);
+        webSocketClient.getProperties().put(ClientProperties.INCOMING_BUFFER_SIZE, bufferSize);
+        String wssPath = "wss://" + timelyHostname + ":" + timelyWssPort + "/websocket";
+        session = webSocketClient.connectToServer(clientEndpoint, new TimelyEndpointConfig(sessionCookie), new URI(
+                wssPath));
+
+        final ByteBuffer pingData = ByteBuffer.allocate(0);
+        webSocketClient.getScheduledExecutorService().scheduleAtFixedRate(() -> {
+            try {
+                session.getBasicRemote().sendPing(pingData);
+            } catch (Exception e) {
+                LOG.error("Error sending ping", e);
+            }
+        }, 30, 60, TimeUnit.SECONDS);
+        closed = false;
+    }
+
+    public synchronized void close() throws IOException {
+        if (closed) {
+            return;
+        }
+        try {
+            if (null != webSocketClient) {
+                webSocketClient.shutdown();
+            }
+        } finally {
+            session = null;
+            webSocketClient = null;
+            closed = true;
+        }
+    }
+
+    public boolean isClosed() {
+        return closed;
+    }
+
+}

client/src/main/java/timely/client/websocket/subscription/WebSocketSubscriptionClient.java

@@ -0,0 +1,92 @@
+package timely.client.websocket.subscription;
+
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.util.Map;
+import java.util.Optional;
+import java.util.UUID;
+import java.util.concurrent.Future;
+
+import javax.net.ssl.SSLContext;
+import javax.websocket.CloseReason;
+import javax.websocket.DeploymentException;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import timely.api.AddSubscription;
+import timely.api.CloseSubscription;
+import timely.api.CreateSubscription;
+import timely.api.RemoveSubscription;
+import timely.client.websocket.ClientHandler;
+import timely.client.websocket.WebSocketClient;
+import timely.serialize.JsonSerializer;
+
+public class WebSocketSubscriptionClient extends WebSocketClient {
+
+    private static final Logger LOG = LoggerFactory.getLogger(WebSocketSubscriptionClient.class);
+    private final String subscriptionId;
+
+    public WebSocketSubscriptionClient(SSLContext ssl, String timelyHostname, int timelyHttpsPort, int timelyWssPort,
+            boolean doLogin, String timelyUsername, String timelyPassword, boolean hostVerificationEnabled,
+            int bufferSize) {
+        super(ssl, timelyHostname, timelyHttpsPort, timelyWssPort, doLogin, timelyUsername, timelyPassword,
+                hostVerificationEnabled, bufferSize);
+        subscriptionId = UUID.randomUUID().toString();
+        LOG.trace("Created WebSocketClient with subscriptionId {}", this.subscriptionId);
+    }
+
+    public WebSocketSubscriptionClient(String timelyHostname, int timelyHttpsPort, int timelyWssPort, boolean doLogin,
+            String timelyUsername, String timelyPassword, String keyStoreFile, String keyStoreType,
+            String keyStorePass, String trustStoreFile, String trustStoreType, String trustStorePass,
+            boolean hostVerificationEnabled, int bufferSize) {
+        super(timelyHostname, timelyHttpsPort, timelyWssPort, doLogin, timelyUsername, timelyPassword, keyStoreFile,
+                keyStoreType, keyStorePass, trustStoreFile, trustStoreType, trustStorePass, hostVerificationEnabled,
+                bufferSize);
+        subscriptionId = UUID.randomUUID().toString();
+        LOG.trace("Created WebSocketClient with subscriptionId {}", this.subscriptionId);
+    }
+
+    @Override
+    public void open(ClientHandler clientEndpoint) throws IOException, DeploymentException, URISyntaxException {
+        super.open(clientEndpoint);
+        CreateSubscription create = new CreateSubscription();
+        create.setSubscriptionId(subscriptionId);
+        session.getBasicRemote().sendText(JsonSerializer.getObjectMapper().writeValueAsString(create));
+    }
+
+    @Override
+    public synchronized void close() throws IOException {
+        if (null != session) {
+            CloseSubscription close = new CloseSubscription();
+            close.setSubscriptionId(subscriptionId);
+            try {
+                session.getBasicRemote().sendText(JsonSerializer.getObjectMapper().writeValueAsString(close));
+                session.close(new CloseReason(CloseReason.CloseCodes.NORMAL_CLOSURE, "Client called close."));
+            } catch (Exception e) {
+                LOG.info("Unable to send close message to server: {}", e.getMessage());
+            }
+        }
+        super.close();
+    }
+
+    public Future<Void> addSubscription(String metric, Map<String, String> tags, long startTime, long endTime,
+            long delayTime) throws IOException {
+        AddSubscription add = new AddSubscription();
+        add.setSubscriptionId(subscriptionId);
+        add.setMetric(metric);
+        add.setTags(Optional.ofNullable(tags));
+        add.setStartTime(Optional.ofNullable(startTime));
+        add.setEndTime(Optional.ofNullable(endTime));
+        add.setDelayTime(Optional.ofNullable(delayTime));
+        return session.getAsyncRemote().sendText(JsonSerializer.getObjectMapper().writeValueAsString(add));
+    }
+
+    public Future<Void> removeSubscription(String metric) throws Exception {
+        RemoveSubscription remove = new RemoveSubscription();
+        remove.setSubscriptionId(subscriptionId);
+        remove.setMetric(metric);
+        return session.getAsyncRemote().sendText(JsonSerializer.getObjectMapper().writeValueAsString(remove));
+    }
+
+}

client/src/main/java/timely/model/Metric.java

@@ -1,12 +1,18 @@
 package timely.model;
 
-import com.fasterxml.jackson.annotation.*;
-import timely.model.parse.TagParser;
-
+import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 
+import timely.model.parse.TagParser;
+
+import com.fasterxml.jackson.annotation.JsonGetter;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonPropertyOrder;
+import com.fasterxml.jackson.annotation.JsonRootName;
+import com.fasterxml.jackson.annotation.JsonUnwrapped;
+
 /**
  * A Metric consists of a metric name, tags, and a Value
  *
@@ -17,7 +23,9 @@ import java.util.List;
 @JsonRootName("metric")
 @JsonInclude(JsonInclude.Include.NON_NULL)
 @JsonPropertyOrder({ "name" })
-public class Metric {
+public class Metric implements Serializable {
+
+    private static final long serialVersionUID = 1L;
 
     private String name;
     private List<Tag> tags;

client/src/main/java/timely/model/Tag.java

@@ -6,6 +6,8 @@ import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonRootName;
 import com.google.common.base.Joiner;
 import com.google.common.base.Preconditions;
+import org.apache.commons.lang3.builder.CompareToBuilder;
+import org.apache.commons.lang3.builder.EqualsBuilder;
 import timely.model.parse.TagParser;
 
 import java.io.Serializable;
@@ -21,7 +23,6 @@ public class Tag implements Comparable<Tag>, Serializable {
 
     private static final long serialVersionUID = 1L;
 
-    private static final Joiner equalJoiner = Joiner.on("=");
     private static final TagParser tagParser = new TagParser();
 
     private String key;
@@ -81,12 +82,12 @@ public class Tag implements Comparable<Tag>, Serializable {
      * @return
      */
     public String join() {
-        return equalJoiner.join(key, value);
+        return tagParser.combine(this);
     }
 
     @Override
     public String toString() {
-        return "Tag{" + equalJoiner.join(this.getKey(), this.getValue()) + "}";
+        return "Tag{" + join() + "}";
     }
 
     @Override
@@ -100,12 +101,12 @@ public class Tag implements Comparable<Tag>, Serializable {
             return true;
         if (null == o || getClass() != o.getClass())
             return false;
-
-        return this.toString().equals(o.toString());
+        Tag other = (Tag) o;
+        return new EqualsBuilder().append(this.key, other.key).append(this.value, other.value).isEquals();
     }
 
     @Override
     public int compareTo(Tag other) {
-        return this.toString().compareTo(other.toString());
+        return new CompareToBuilder().append(this.key, other.key).append(this.value, other.value).toComparison();
     }
 }

client/src/main/java/timely/model/parse/Combiner.java

@@ -0,0 +1,6 @@
+package timely.model.parse;
+
+public interface Combiner<T> {
+
+    String combine(T o);
+}

client/src/main/java/timely/model/parse/TagListParser.java

@@ -4,12 +4,17 @@ import com.google.common.base.Splitter;
 import timely.model.Tag;
 
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.List;
+import java.util.Map;
 
 /**
  * Parses tag1=val1,tag2=val2,tag3=val3 into a {@code List} of {@code Tag}
+ * Combines {@code List} of {@code Tag} into {@code String} of
+ * tag1=val1,tag2=val2,tag3=val3 NOTE: the tag and the value are both URL
+ * encoded and subsequently URL decoded.
  */
-public class TagListParser implements Parser<List<Tag>> {
+public class TagListParser implements Parser<List<Tag>>, Combiner<Collection<Tag>> {
 
     private static final Splitter commaSplitter = Splitter.on(",").omitEmptyStrings();
     private static final TagParser tagParser = new TagParser();
@@ -20,4 +25,30 @@ public class TagListParser implements Parser<List<Tag>> {
         commaSplitter.splitToList(t).forEach(p -> tags.add(tagParser.parse(p)));
         return tags;
     }
+
+    @Override
+    public String combine(Collection<Tag> tags) {
+        StringBuilder builder = new StringBuilder();
+        if (!tags.isEmpty()) {
+            tags.forEach(p -> builder.append(tagParser.combine(p)).append(','));
+            builder.setLength(builder.length() - 1);
+        }
+        return builder.toString();
+    }
+
+    /**
+     * An alternative combiner for a map of string to string instead of a list
+     * of tags
+     * 
+     * @param tags
+     * @return string representation
+     */
+    public String combine(Map<String, String> tags) {
+        StringBuilder builder = new StringBuilder();
+        if (!tags.isEmpty()) {
+            tags.forEach((k, v) -> builder.append(tagParser.combine(new Tag(k, v))).append(','));
+            builder.setLength(builder.length() - 1);
+        }
+        return builder.toString();
+    }
 }

client/src/main/java/timely/model/parse/TagParser.java

@@ -1,13 +1,24 @@
 package timely.model.parse;
 
+import com.google.common.base.Joiner;
 import com.google.common.base.Splitter;
 import timely.model.Tag;
 
+import java.io.UnsupportedEncodingException;
+import java.net.URLDecoder;
+import java.net.URLEncoder;
 import java.util.List;
+import java.util.Map;
 
-public class TagParser implements Parser<Tag> {
+/**
+ * Parses tag1=val1 into a {@code Tag} Combines {@code Tag} into {@code String}
+ * of tag1=val1 NOTE: the tag and the value are both URL encoded and
+ * subsequently URL decoded.
+ */
+public class TagParser implements Parser<Tag>, Combiner<Tag> {
 
     private static final Splitter equalSplitter = Splitter.on("=").trimResults();
+    private static final Joiner equalsJoiner = Joiner.on("=");
 
     @Override
     public Tag parse(String t) {
@@ -15,6 +26,28 @@ public class TagParser implements Parser<Tag> {
         if (parts.size() != 2) {
             throw new IllegalArgumentException("Invalid tag format: " + t);
         }
-        return new Tag(parts.get(0), parts.get(1));
+        return new Tag(unescape(parts.get(0)), unescape(parts.get(1)));
     }
+
+    @Override
+    public String combine(Tag t) {
+        return equalsJoiner.join(escape(t.getKey()), escape(t.getValue()));
+    }
+
+    private String escape(String tag) {
+        try {
+            return URLEncoder.encode(tag, "UTF8");
+        } catch (UnsupportedEncodingException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    private String unescape(String tag) {
+        try {
+            return URLDecoder.decode(tag, "UTF8");
+        } catch (UnsupportedEncodingException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
 }

client/src/main/java/timely/serialize/JsonSerializer.java

@@ -0,0 +1,22 @@
+package timely.serialize;
+
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.datatype.jdk8.Jdk8Module;
+
+public class JsonSerializer {
+
+    private static ObjectMapper mapper = new ObjectMapper();
+
+    static {
+        mapper.registerModule(new Jdk8Module());
+        mapper.configure(DeserializationFeature.ACCEPT_EMPTY_ARRAY_AS_NULL_OBJECT, true);
+        mapper.configure(DeserializationFeature.ACCEPT_EMPTY_STRING_AS_NULL_OBJECT, true);
+        mapper.configure(JsonParser.Feature.ALLOW_NON_NUMERIC_NUMBERS, true);
+    }
+
+    public static ObjectMapper getObjectMapper() {
+        return mapper;
+    }
+}

client/src/main/python/DataOperations.py

@@ -0,0 +1,128 @@
+import time
+from datetime import datetime
+from TimeDateRange import UTC
+import pandas
+import matplotlib.dates as mdates
+import plotly.graph_objs as go
+import plotly.offline as py
+
+utc = UTC()
+
+def pivot(df, metric, groupByColumn=None):
+
+    dataFrame = pandas.DataFrame(df, copy=True)
+    if dataFrame is not None:
+        if groupByColumn is not None:
+            dataFrame = dataFrame.pivot_table(index="date", columns=groupByColumn, values=metric)
+    return dataFrame
+
+
+def resample(df, sample, how='mean', fill_method='ffill'):
+
+    dataFrame = pandas.DataFrame(df, copy=True)
+    if dataFrame is not None:
+        if sample is not None:
+            dataFrame = dataFrame.resample(sample, how=how, fill_method=fill_method)
+    return dataFrame
+
+
+def unpivot(df, metric, groupByColumn=None):
+    dataFrame = pandas.DataFrame(df, copy=True)
+    if dataFrame is not None:
+        dataFrame['date'] = dataFrame.index.values
+        dataFrame = pandas.melt(dataFrame, id_vars=['date'], value_name=metric, var_name=groupByColumn)
+        dataFrame = dataFrame.set_index(dataFrame['date'])
+    return dataFrame
+
+
+def rolling_average(df, metric, rolling_average=None):
+
+    dataFrame = pandas.DataFrame(df, copy=True)
+    if dataFrame is not None:
+        if rolling_average is not None:
+            dataFrame[metric] = pandas.rolling_mean(dataFrame[metric], rolling_average)
+    return dataFrame
+
+
+def graph(analyticConfig, df, metric, seriesConfig={}, graphConfig={}, notebook=False):
+
+    dataFrame = pandas.DataFrame(df, copy=True)
+    if dataFrame is not None:
+        dataFrame['colFromIndex'] = dataFrame.index
+        if analyticConfig.groupByColumn is not None:
+            # sort on date and the groupByColumn so that the series legend is sorted
+            dataFrame = dataFrame.sort_values(['colFromIndex', analyticConfig.groupByColumn])
+        else:
+            dataFrame = dataFrame.sort_values(['colFromIndex'])
+
+        title = None
+        if analyticConfig.sample is None:
+            title = metric
+        else:
+            title = metric + '\rsample ' + analyticConfig.sample
+
+        dataFrame['date'] = dataFrame.index
+
+        layoutConfig = dict(
+            title=title,
+            autosize=False,
+            width=1000,
+            height=700,
+            showlegend=True,
+            xaxis=dict(
+                autorange=True,
+                tickangle=-45
+            ),
+            yaxis=dict(
+                autorange=True,
+                title=title
+            )
+        )
+
+        layoutConfig.update(graphConfig)
+
+        layout = go.Layout(layoutConfig)
+
+        data = None
+        for col in df[analyticConfig.groupByColumn].unique():
+            tempDataFrame = dataFrame.loc[dataFrame[analyticConfig.groupByColumn] == col]
+
+            addlColConfig = graphConfig.get(col, graphConfig.get('default', dict()))
+
+            config = dict(
+                name=col,
+                x=tempDataFrame.index,
+                y=tempDataFrame[metric],
+                hoverinfo='x+y+text',
+                text=col
+            )
+
+            config.update(addlColConfig)
+
+            s = go.Scatter(config)
+
+            if data is None:
+                data = [s]
+            else:
+                data.append(s)
+
+        now = datetime.now(tz=utc)
+        format = "%Y%m%d%H%M%S"
+        nowStr = time.strftime(format, now.timetuple())
+
+        fileNameBase = analyticConfig.output_dir + "/" + metric
+        if analyticConfig.sample is not None:
+            fileNameBase += "-" + analyticConfig.sample
+        if analyticConfig.how is not None:
+            fileNameBase += "-" + analyticConfig.how
+
+        fig = go.Figure(data=data, layout=layout)
+        if notebook:
+            py.iplot(fig, filename=fileNameBase+'-'+nowStr, show_link=False)
+        else:
+            py.plot(fig, output_type='file', filename=fileNameBase+'-'+nowStr+'.html', auto_open=False, show_link=False)
+
+
+def datetime2graphdays(dt):
+
+    return mdates.epoch2num((time.mktime(dt.timetuple()) / 86400) * mdates.SEC_PER_DAY)

client/src/main/python/TimeDateRange.py

@@ -0,0 +1,124 @@
+import time
+from datetime import datetime
+from datetime import timedelta
+from datetime import tzinfo
+
+class TimeDateError(ValueError):
+
+    def __init__(self, message, *args):
+        self.message = message
+        super(TimeDateError, self).__init__(message, *args)
+
+
+ZERO = timedelta(0)
+
+# A UTC class.
+class UTC(tzinfo):
+    """UTC"""
+
+    def __str__(self):
+        return "UTC"
+
+    def utcoffset(self, dt):
+        return ZERO
+
+    def tzname(self, dt):
+        return "UTC"
+
+    def dst(self, dt):
+        return ZERO
+
+utc = UTC()
+
+class TimeDateRange:
+
+    beginDt = None
+    endDt = None
+    format = "%Y%m%d %H%M%S"
+
+    def getBeginMs(self):
+        if self.beginDt == None:
+            return None
+        return int(time.mktime(self.beginDt.timetuple()) * 1000)
+
+    def getBeginStr(self):
+        if self.beginDt == None:
+            return None
+        return time.strftime(self.format, self.beginDt.timetuple())
+
+    def getEndMs(self):
+        if self.endDt == None:
+            return None
+        return int(time.mktime(self.endDt.timetuple()) * 1000)
+
+    def getEndStr(self):
+        if self.endDt == None:
+            return None
+        return time.strftime(self.format, self.endDt.timetuple())
+
+    def __init__(self, beginStr, endStr, hours, format="%Y%m%d-%H%M%S"):
+
+        now = datetime.now(tz=utc)
+        nowMs = TimeDateRange.unix_time_millis(now)
+
+        if beginStr != None and endStr != None and hours != None:
+            raise TimeDateError("You can not privide a begin, end, and hours")
+        elif (beginStr == None and endStr == None and hours == None) or (hours == None and beginStr == None):
+            raise TimeDateError("You must provide a time range")
+
+        if hours == None:
+            # calculate begin time
+            beginTuple = time.strptime(beginStr, self.format)
+            self.beginDt = datetime(*beginTuple[0:6], tzinfo=utc)
+
+            # calculate end time
+            # end is either specified or now
+            if endStr == None or endStr == 'now':
+                self.endDt = now
+            else:
+
+                self.endDt = datetime(*time.strptime(endStr, self.format)[0:6], tzinfo=utc)
+                if self.endDt > now:
+                    self.endDt = now
+        else:
+            # hours must be specified
+            rangeInSec = hours * 3600
+            if beginStr == None:
+                # calculate begin time from end tme and hours
+
+                # calculate end time
+                # end is either specified or now
+                if endStr == None or endStr == 'now':
+                    self.endDt = now
+                else:
+                    self.endDt = datetime(*time.strptime(endStr, self.format)[0:6], tzinfo=utc)
+                    if self.endDt > now:
+                        self.endDt = now
+
+                # calculate begin time
+                self.beginDt = self.endDt - timedelta(milliseconds=rangeInSec * 1000)
+
+            else:
+                # calculate end time from begin tme and hours
+                # calculate begin time
+                self.beginDt = datetime(*time.strptime(beginStr, self.format)[0:6], tzinfo=utc)
+
+                # calculate end time
+                self.endDt = self.beginDt + timedelta(milliseconds=rangeInSec * 1000)
+
+                if self.endDt > now:
+                    self.endDt = now
+
+        if self.beginDt > now:
+            raise TimeDateError("begin date/time can not be in the future")
+
+        if self.beginDt > self.endDt:
+            raise TimeDateError("begin date/time can not be past end date/time")
+
+    epoch = datetime.utcfromtimestamp(0).replace(tzinfo=utc)
+
+    @classmethod
+    def unix_time_millis(cls, dt):
+        return (dt - TimeDateRange.epoch).total_seconds() * 1000.0
+
+

client/src/main/python/TimelyAlert.py

@@ -0,0 +1,123 @@
+import smtplib
+from os.path import basename
+from email.mime.application import MIMEApplication
+from email.mime.multipart import MIMEMultipart
+from email.mime.text import MIMEText
+from email.utils import COMMASPACE, formatdate
+from TimelyMetric import graph
+import syslog
+
+syslog.openlog("TimelyNotifications")
+
+#def __init__(self, file, send_to, subject, text, analyticConfig, notebook):
+
+
+class TimelyAlert():
+
+    def __init__(self, timelyMetric, dataFrame, seriesConfig, analyticConfig, notebook):
+        self.timelyMetric = timelyMetric
+        self.dataFrame = dataFrame
+        self.seriesConfig = seriesConfig
+        self.analyticConfig = analyticConfig
+        self.notebook = notebook
+
+    def createFilename(self, metric, config):
+
+        metricTitle = metric
+        if config.sample is not None:
+            sampleTitle = config.sample + "-" + config.how
+
+        filename = metricTitle
+        if sampleTitle != "":
+            filename += "-" + sampleTitle
+
+        return filename
+
+    def createTitle(self, metric, config):
+
+        if config.orCondition:
+            condition = "OR"
+        else:
+            condition = "AND"
+
+        metricTitle = metric
+        sampleTitle = ""
+        alertTitle = ""
+        if config.sample is not None:
+            sampleTitle = config.sample + " " + config.how + " samples"
+
+        if config.min_threshold is not None:
+            if alertTitle != "":
+                alertTitle += " " + condition + " "
+            alertTitle += "(y < " + str(config.min_threshold) + ")"
+
+        if config.max_threshold is not None:
+            if alertTitle != "":
+                alertTitle += " " + condition + " "
+            alertTitle += "(y > " + str(config.max_threshold) + ")"
+
+        if config.alert_percentage is not None:
+            if alertTitle != "":
+                alertTitle += " " + condition + " "
+            if config.alert_percentage > 0:
+                alertTitle += "(y > " + str(config.alert_percentage) + "% above " + str(config.rolling_average) + " sample average)"
+            else:
+                alertTitle += "(y < " + str(-config.alert_percentage) + "% below " + str(config.rolling_average) + " sample average)"
+
+        title = metricTitle
+        if sampleTitle != "":
+            title += "\n" + sampleTitle
+
+        if alertTitle != "":
+            title += "\n" + alertTitle
+
+        return title
+
+    def notify(self, send_from, send_to, server="127.0.0.1"):
+
+        self.graph()
+
+        return
+
+    def email(self, send_from, send_to, subject, text, files=None, server="127.0.0.1"):
+        assert isinstance(send_to, list)
+
+        msg = MIMEMultipart()
+        msg['From'] = send_from
+        msg['To'] = COMMASPACE.join(send_to)
+        msg['Date'] = formatdate(localtime=True)
+        msg['Subject'] = subject
+
+        msg.attach(MIMEText(text))
+
+        for f in files or []:
+            with open(f, "rb") as fil:
+                part = MIMEApplication(
+                    fil.read(),
+                    Name=basename(f)
+                )
+                part['Content-Disposition'] = 'attachment; filename="%s"' % basename(f)
+                msg.attach(part)
+
+
+        smtp = smtplib.SMTP(server)
+        smtp.sendmail(send_from, send_to, msg.as_string())
+        smtp.close()
+
+    def graph(self):
+
+        graphConfig = {}
+        graphConfig["title"] = self.createTitle(self.timelyMetric.metric, self.analyticConfig)
+        graph(self.analyticConfig, self.dataFrame, self.timelyMetric.metric, seriesConfig=self.seriesConfig, graphConfig=graphConfig, notebook=self.notebook)
+
+
+def main():
+    #syslog.syslog(syslog.LOG_ALERT, "Example message")
+    a = TimelyAlert(None, None, None, None, False)
+
+    a.email_alert("wwoley@localhost", ["wwoley@localhost"], "Test1", "This is a sendmail test")
+
+
+
+if __name__ == '__main__':
+    main()

client/src/main/python/TimelyAnalytic.py

@@ -0,0 +1,105 @@
+import TimelyMetric
+from TimeDateRange import TimeDateRange
+from TimeDateRange import UTC
+from datetime import datetime
+import pandas
+import re
+import plotly
+from TimelyAlert import TimelyAlert
+
+utc = UTC()
+plotly.offline.init_notebook_mode()
+
+def logTiming(timelyMetric, startTime, stopTime):
+    df = timelyMetric.getDataFrame()
+    print("start: " + str(startTime))
+    print("stop: " + str(stopTime))
+    print("num records: " + str(len(df)))
+    elapsedSec = (TimeDateRange.unix_time_millis(stopTime) - TimeDateRange.unix_time_millis(startTime)) / 1000
+    print("elapsed seconds: " + str(elapsedSec))
+    recPerSec = len(df) / elapsedSec
+    print("records per second: " + recPerSec)
+
+def addCondition(orCondition, currResult, condition):
+    if orCondition:
+        currResult = currResult | condition
+    else:
+        currResult = currResult & condition
+    return currResult
+
+def find_alerts(timelyMetric, analyticConfig, notebook=False):
+
+    if analyticConfig.groupByColumn is None:
+        return
+
+    df = timelyMetric.getDataFrame()
+
+    graphDF = TimelyMetric.pivot(df, timelyMetric.metric, groupByColumn=analyticConfig.groupByColumn)
+
+    if analyticConfig.sample is not None:
+        graphDF = TimelyMetric.resample(graphDF, analyticConfig.sample, how=analyticConfig.how)
+
+    if analyticConfig.excludeColRegex is not None:
+        graphDF = graphDF.select(lambda x : not (re.search(analyticConfig.excludeColRegex, x)), axis=1)
+    if analyticConfig.includeColRegex is not None:
+        graphDF = graphDF.select(lambda x : re.search(analyticConfig.includeColRegex, x), axis=1)
+
+    graphDF_avg = pandas.DataFrame(graphDF, copy=True)
+
+    combined = pandas.DataFrame()
+
+    seriesConfig = {}
+    for i in graphDF_avg.columns:
+        col = str(i)
+
+        result = True
+        if analyticConfig.orCondition:
+            result = False
+
+        if analyticConfig.min_threshold is not None:
+            currCondition = graphDF[col].astype(float) < analyticConfig.min_threshold
+            result = addCondition(analyticConfig.orCondition, result, currCondition)
+
+        if analyticConfig.max_threshold is not None:
+            currCondition = graphDF[col].astype(float) > analyticConfig.max_threshold
+            result = addCondition(analyticConfig.orCondition, result, currCondition)
+
+        graphDF_avg = TimelyMetric.rolling_average(graphDF_avg, str(i), rolling_average=analyticConfig.rolling_average)
+        if (analyticConfig.alert_percentage is not None) and (analyticConfig.rolling_average is not None):
+            if analyticConfig.alert_percentage > 0:
+                multiple = 1.0 + (float(abs(analyticConfig.alert_percentage)) / float(100))
+                currCondition = graphDF[col].astype(float) > (graphDF_avg[col].astype(float) * multiple)
+                result = addCondition(analyticConfig.orCondition, result, currCondition)
+            if analyticConfig.alert_percentage < 0:
+                multiple = 1.0 - (float(abs(analyticConfig.alert_percentage)) / float(100))
+                if multiple > 0:
+                    currCondition = graphDF[col].astype(float) < (graphDF_avg[col].astype(float) * multiple)
+                    result = addCondition(analyticConfig.orCondition, result, currCondition)
+
+        exceptional = graphDF.loc[result, col]
+        if (analyticConfig.display.lower() == "all") or (analyticConfig.display.lower() == "alerts" and exceptional.size > 0):
+            combined[col] = graphDF[col]
+
+        if (analyticConfig.display.lower() == "all") or (analyticConfig.display.lower() == "alerts" and exceptional.size > 0 and analyticConfig.alert_percentage is not None and analyticConfig.rolling_average is not None):
+            combined[col+'_avg'] = graphDF_avg[col]
+
+        if (exceptional.size > 0):
+            combined[col+'_warn'] = exceptional.dropna()
+            seriesConfig[col+'_warn'] = {
+                "mode" : "markers",
+                "marker" : {
+                    "symbol" : "hash-open",
+                    "color" : "red"
+                }
+            }
+
+
+
+    timelyAlert = None
+    if not combined.empty:
+        combined = TimelyMetric.unpivot(combined, timelyMetric.metric, groupByColumn=analyticConfig.groupByColumn)
+        combined = combined.sort_index()
+        combined = combined.sort_values(['date', analyticConfig.groupByColumn])
+        timelyAlert = TimelyAlert(timelyMetric, combined, seriesConfig, analyticConfig, notebook)
+
+    return timelyAlert

client/src/main/python/TimelyAnalyticConfiguration.py

@@ -0,0 +1,25 @@
+
+
+
+class TimelyAnalyticConfiguration():
+
+
+    def __init__(self, analyticConfig):
+
+        self.groupByColumn = analyticConfig.get('groupByColumn', None)
+        self.includeColRegex = analyticConfig.get('includeColRegex', None)
+        self.excludeColRegex = analyticConfig.get('excludeColRegex', None)
+        self.sample = analyticConfig.get('sample', None)
+        self.how = analyticConfig.get('how', 'mean')
+        self.rolling_average = analyticConfig.get('rolling_average', None)
+        self.min_threshold = analyticConfig.get('min_threshold', None)
+        self.max_threshold = analyticConfig.get('max_threshold', None)
+        self.alert_percentage = analyticConfig.get('alert_percentage', None)
+        boolean = analyticConfig.get('boolean', 'or')
+        self.orCondition = boolean == 'or' or boolean == 'OR'
+        # alerts or all
+        self.display = analyticConfig.get('display', 'alerts')
+        self.send_alerts_to = analyticConfig.get('send_alerts_to', [])
+        self.output_dir = analyticConfig.get('output_dir', '/tmp')
+
+

client/src/main/python/TimelyMetric.py

@@ -0,0 +1,212 @@
+from TimeDateRange import TimeDateRange
+from TimeDateRange import TimeDateError
+from TimeDateRange import UTC
+from DataOperations import *
+import json
+import pandas
+import getopt
+import sys
+import random
+
+from WebSocketClient import WebSocketClient
+from tornado import ioloop
+
+utc = UTC()
+
+class TimelyWebSocketClient(WebSocketClient):
+
+    def __init__(self, hostport, metric, tags, startTime, endTime, connect_timeout=WebSocketClient.DEFAULT_CONNECT_TIMEOUT,
+                 request_timeout=WebSocketClient.DEFAULT_REQUEST_TIMEOUT):
+
+        self.metric = metric
+        self.tags = tags
+        self.startTime = startTime
+        self.endTime = endTime
+        self.subscriptionId = random.randint(1000000000, 9999999999)
+        WebSocketClient.__init__(self, hostport, connect_timeout, request_timeout)
+
+    def _on_message(self, msg):
+        # implement in subclass
+        pass
+
+    def _on_connection_success(self):
+        print('Connected.')
+
+        create = {
+            "operation": "create",
+            "subscriptionId": self.subscriptionId,
+        }
+        self.send(create)
+        self.send(create)
+
+        m1 = {
+            "operation": "add",
+            "subscriptionId": self.subscriptionId,
+            "metric": self.metric,
+            "startTime": self.startTime,
+            "endTime" : self.endTime
+        }
+
+        if self.tags is not None:
+            t = self.tags.strip().split(',')
+            tagDict = {}
+            for pair in t:
+                k = pair.split('=')[0].strip()
+                v = pair.split('=')[1].strip()
+                tagDict = dict(tagDict.items() + dict({k : v}).items())
+            m1["tags"] = tagDict
+
+        self.send(m1)
+
+    def _on_connection_close(self):
+        print('Connection closed!')
+        exit()
+
+    def _on_connection_error(self, exception):
+        print('Connection error: %s', exception)
+
+
+
+
+class TimelyMetric(TimelyWebSocketClient):
+
+    client = None
+    endtime = None
+
+    def __init__(self, hostport, metric, tags, beginStr, endStr, hours, sample):
+        self.series = pandas.Series()
+        self.metric = metric
+        self.tags = tags
+        self.sample = sample
+        self.debug = False
+        self.dataFrame = None
+        self.data = []
+
+        timeDateRange = TimeDateRange(beginStr, endStr, hours)
+
+        TimelyWebSocketClient.__init__(self, hostport, metric, tags, timeDateRange.getBeginMs(), timeDateRange.getEndMs())
+
+    def debugOn(self):
+        self.debug = True
+
+    def debugOff(self):
+        self.debug = False
+
+    def fetch(self):
+
+        ioloop.IOLoop().make_current()
+        self.connect(ioloop.IOLoop.current(instance=False))
+        ioloop.IOLoop.current(instance=False).start()
+        return self
+
+
+    def _on_message(self, msg):
+
+        global df
+        global endtime
+
+        responsesObj = json.loads(msg)
+        responses = responsesObj.get("responses")
+
+        for obj in responses:
+            complete = bool(obj.get("complete"))
+            if complete:
+                self._on_connection_close()
+                return
+            else:
+                date = int(obj.get("timestamp")/1000)
+                dt = pandas.datetime.utcfromtimestamp(date)
+                metricName = str(obj.get("metric"))
+                metricValue = obj.get("value")
+
+                newData = {}
+                newData["date"] = dt
+                newData[metricName] = metricValue
+
+                tags = obj.get("tags")
+                for d in tags:
+                    for k,v in d.items():
+                        newData[k] = v
+
+                self.data.append(newData)
+
+
+    def _on_connection_close(self):
+        global client
+
+        self.dataFrame = pandas.DataFrame(self.data)
+        self.dataFrame = self.dataFrame.set_index(self.dataFrame['date'].apply(lambda x: pandas.DatetimeIndex([x])))
+
+
+        self.close()
+        ioloop.IOLoop.current(instance=False).stop()
+        print("Exiting.")
+
+    def getDataFrame(self):
+
+        return self.dataFrame;
+
+    def print_debug(self):
+        if self.debug:
+            print(self.dataFrame)
+
+    def graph(self, groupByColumn=None):
+        graph(self.dataFrame, self.metric, sample=self.sample, how=self.how, groupByColumn=groupByColumn)
+
+
+
+def main():
+
+    hours = None
+    beginStr = None
+    endStr = None
+
+    metric = "timely.metrics.received"
+    sample = None
+    how = None
+    tags = None
+    groupByColumn = None
+    try:
+        argv = sys.argv
+        options, remainder = getopt.getopt(argv[1:], 'h', ['metric=', 'tags=', 'hours=', 'groupByColumn=', 'begin=', 'end=', 'sample=', 'how=', 'hostport='])
+    except getopt.GetoptError:
+        print('TimelyMetric.py ---hostport=<host:port> --metric=<metric> --tags=<tag1=val1,tag2=val2> --hours=<hours>, --groupByColumn=<groupByColumn>, --sample=<sample_period> --how=<mean|min|max>')
+        sys.exit(2)
+    for opt, arg in options:
+        if opt == '-h':
+            print('TimelyMetric.py --hostport=<host:port> --metric=<metric> --tags=<tag1=val1,tag2=val2> --hours=<hours>, --groupByColumn=<groupByColumn>, --sample=<sample_period> --how=<mean|min|max>')
+            sys.exit()
+        elif opt in ("--metric"):
+            metric = arg
+        elif opt in ("--hours"):
+            hours = int(arg)
+        elif opt in ("--begin"):
+            beginStr = arg
+        elif opt in ("--end"):
+            endStr = arg
+        elif opt in ("--groupByColumn"):
+            groupByColumn = arg
+        elif opt in ("--sample"):
+            sample = arg
+        elif opt in ("--how"):
+            how = arg
+        elif opt in ("--hostport"):
+            hostport = arg
+    try:
+
+        timelyMetric = TimelyMetric(hostport, metric, tags, beginStr, endStr, hours, sample).fetch()
+        dataFrame = timelyMetric.getDataFrame()
+
+        if dataFrame is not None:
+            dataFrame = resample(dataFrame, metric, sample, how)
+            dataFrame = pivot(dataFrame, metric, groupByColumn=groupByColumn)
+            graph(dataFrame, metric, sample=sample, groupByColumn=groupByColumn, graphConfig={}, notebook=False)
+
+    except TimeDateError as e:
+        print(e.message)
+
+if __name__ == '__main__':
+    main()
+
+
+

client/src/main/python/WebSocketClient.py

@@ -0,0 +1,91 @@
+from tornado import escape
+from tornado import gen
+from tornado import ioloop
+from tornado import websocket
+
+import json
+
+
+class WebSocketClient():
+    """Base for web socket clients.
+    """
+
+    DEFAULT_CONNECT_TIMEOUT = 60
+    DEFAULT_REQUEST_TIMEOUT = 60
+    APPLICATION_JSON = 'application/json'
+
+    def __init__(self, hostport, connect_timeout=DEFAULT_CONNECT_TIMEOUT,
+                 request_timeout=DEFAULT_REQUEST_TIMEOUT):
+
+        self.connect_timeout = connect_timeout
+        self.request_timeout = request_timeout
+        self.url = 'wss://' + hostport + '/websocket'
+
+    def connect(self, ioloop=ioloop.IOLoop.current(instance=False)):
+        """Connect to the server.
+        :param str url: server URL.
+        """
+        request = websocket.httpclient.HTTPRequest(self.url, connect_timeout=self.connect_timeout, request_timeout=self.request_timeout, validate_cert=False)
+        websocket.websocket_connect(request, io_loop=ioloop, callback=self._connect_callback)
+
+    def send(self, data):
+        """Send message to the server
+        :param str data: message.
+        """
+        if not self._ws_connection:
+            raise RuntimeError('Web socket connection is closed.')
+
+        self._ws_connection.write_message(escape.utf8(json.dumps(data)))
+
+    def close(self):
+        """Close connection.
+        """
+        if not self._ws_connection:
+            raise RuntimeError('Web socket connection is already closed.')
+
+        self._ws_connection.close()
+        self._ws_connection = None
+
+    def _connect_callback(self, future):
+        if future.exception() is None:
+            self._ws_connection = future.result()
+            self._on_connection_success()
+            self._read_messages()
+        else:
+            self._on_connection_error(future.exception())
+
+    @gen.coroutine
+    def _read_messages(self):
+        while True and not self._ws_connection is None:
+            msg = yield self._ws_connection.read_message()
+            if msg is None:
+                self._on_connection_close()
+                break
+
+            self._on_message(msg)
+
+    def _on_message(self, msg):
+        """This is called when new message is available from the server.
+        :param str msg: server message.
+        """
+
+        pass
+
+    def _on_connection_success(self):
+        """This is called on successful connection ot the server.
+        """
+
+        pass
+
+    def _on_connection_close(self):
+        """This is called when server closed the connection.
+        """
+        pass
+
+    def _on_connection_error(self, exception):
+        """This is called in case if connection to the server could
+        not established.
+        """
+
+        pass
+

client/src/main/python/sampleAnalytic.py

@@ -0,0 +1,23 @@
+import TimelyMetric
+import TimelyAnalytic
+
+hostport = "127.0.0.1:4244"
+timelyMetric = TimelyMetric.TimelyMetric(hostport, '', None, None, None, 48, None).fetch()
+TimelyAnalytic.find_alerts(timelyMetric, TimelyAnalyticConfiguration({
+    'includeColRegex' : None,
+    'excludeColRegex' : None,
+    'groupByColumn' : 'instance',
+    'sample' : '10min',
+    'how' : 'mean',
+    'rolling_average' : 12,
+    'min_threshold' : None,
+    'max_threshold' : None,
+    'alert_percentage' : 25,
+    'boolean' : 'and'
+    'display' : 'all',
+    'output_dir' : '/path/to/output'
+}))
+
+if alert is not None:
+    alert.graph()
+

client/src/main/resources/findbugs-excludes.xml

@@ -1,3 +1,8 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <FindBugsFilter>
+  <Or>
+    <Bug pattern="DM_DEFAULT_ENCODING"/>
+    <Method name="connect" />
+    <Class name="timely.client.tcp.TcpClient"/>
+  </Or>
 </FindBugsFilter>

client/src/test/java/timely/api/SerializationTest.java

@@ -0,0 +1,58 @@
+package timely.api;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import timely.serialize.JsonSerializer;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+public class SerializationTest {
+
+    private ObjectMapper mapper = JsonSerializer.getObjectMapper();
+
+    @Test
+    public void testBasicAuth() throws Exception {
+        BasicAuthLogin login = new BasicAuthLogin();
+        login.setUsername("test");
+        login.setPassword("pass");
+        testSerialization(login);
+    }
+
+    @Test
+    public void testCreateSubscription() throws Exception {
+        CreateSubscription create = new CreateSubscription();
+        create.setSubscriptionId("1234");
+        testSerialization(create);
+    }
+
+    @Test
+    public void testCloseSubscription() throws Exception {
+        CloseSubscription close = new CloseSubscription();
+        close.setSubscriptionId("1234");
+        testSerialization(close);
+    }
+
+    @Test
+    public void testAddSubscription() throws Exception {
+        AddSubscription add = new AddSubscription();
+        add.setSubscriptionId("1234");
+        add.setMetric("sys.cpu.user");
+        testSerialization(add);
+    }
+
+    @Test
+    public void testRemoveSubscription() throws Exception {
+        RemoveSubscription remove = new RemoveSubscription();
+        remove.setSubscriptionId("1234");
+        remove.setMetric("sys.cpu.user");
+        testSerialization(remove);
+    }
+
+    @SuppressWarnings("unchecked")
+    private <T> void testSerialization(T o) throws Exception {
+        String ser = mapper.writeValueAsString(o);
+        T des = mapper.readValue(ser, (Class<T>) o.getClass());
+        Assert.assertEquals(o, des);
+    }
+}

client/src/test/java/timely/model/TagListParserTest.java

@@ -0,0 +1,58 @@
+package timely.model;
+
+import org.junit.Assert;
+import org.junit.Test;
+import timely.model.parse.TagListParser;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+
+public class TagListParserTest {
+
+    @Test
+    public void testListParse() {
+        String value = "tag1=value1,tag2=value2";
+        List<Tag> tags = new TagListParser().parse(value);
+        Assert.assertEquals(2, tags.size());
+        Assert.assertEquals(new Tag("tag1", "value1"), tags.get(0));
+        Assert.assertEquals(new Tag("tag2", "value2"), tags.get(1));
+    }
+
+    @Test
+    public void testListParseEncoded() {
+        String value = "tag%2C%3D%31=value%2C%3D%31,tag%2C%3D%32=value%2C%3D%32";
+        List<Tag> tags = new TagListParser().parse(value);
+        Assert.assertEquals(2, tags.size());
+        Assert.assertEquals(new Tag("tag,=1", "value,=1"), tags.get(0));
+        Assert.assertEquals(new Tag("tag,=2", "value,=2"), tags.get(1));
+    }
+
+    @Test
+    public void testListCombine() {
+        List<Tag> tags = new ArrayList<>();
+        tags.add(new Tag("tag1", "value1"));
+        tags.add(new Tag("tag2", "value2"));
+        String combined = new TagListParser().combine(tags);
+        Assert.assertEquals("tag1=value1,tag2=value2", combined);
+    }
+
+    @Test
+    public void testListCombineEncoded() {
+        List<Tag> tags = new ArrayList<>();
+        tags.add(new Tag("tag,=1", "value,=1"));
+        tags.add(new Tag("tag,=2", "value,=2"));
+        String combined = new TagListParser().combine(tags);
+        Assert.assertEquals("tag%2C%3D1=value%2C%3D1,tag%2C%3D2=value%2C%3D2", combined);
+    }
+
+    @Test
+    public void testListCombineMap() {
+        Map<String, String> map = new TreeMap<>();
+        map.put("tag1", "value1");
+        map.put("tag2", "value2");
+        String combined = new TagListParser().combine(map);
+        Assert.assertEquals("tag1=value1,tag2=value2", combined);
+    }
+}

collectd-nsq-plugin/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>timely</groupId>
         <artifactId>timely</artifactId>
-        <version>0.0.3</version>
+        <version>0.0.4</version>
     </parent>
     <artifactId>collectd-nsq-plugin</artifactId>
     <dependencies>
@@ -60,6 +60,10 @@
                 <groupId>net.revelc.code</groupId>
                 <artifactId>formatter-maven-plugin</artifactId>
             </plugin>
+            <plugin>
+                <groupId>org.codehaus.mojo</groupId>
+                <artifactId>buildnumber-maven-plugin</artifactId>
+            </plugin>
         </plugins>
     </build>
 </project>

collectd-plugin-parent/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>timely</groupId>
         <artifactId>timely</artifactId>
-        <version>0.0.3</version>
+        <version>0.0.4</version>
     </parent>
     <artifactId>collectd-plugin-parent</artifactId>
     <dependencies>
@@ -55,6 +55,10 @@
                 <groupId>net.revelc.code</groupId>
                 <artifactId>formatter-maven-plugin</artifactId>
             </plugin>
+            <plugin>
+                <groupId>org.codehaus.mojo</groupId>
+                <artifactId>buildnumber-maven-plugin</artifactId>
+            </plugin>
         </plugins>
     </build>
 </project>

collectd-plugin-parent/src/main/java/timely/collectd/plugin/CollectDPluginParent.java

@@ -32,7 +32,7 @@ public abstract class CollectDPluginParent {
     private static final Pattern ETHSTAT_PATTERN = Pattern.compile("([\\w-_]+)_queue_([\\w-_]+)_([\\w-_]+)");
     private static final String STATSD_PREFIX = "statsd";
     private static final String NSQ_PREFIX = STATSD_PREFIX + ".nsq.";
-    private static final Pattern HAPROXY_PATTERN = Pattern.compile("([\\w-_]+)\\.([\\w-_]+)\\.([\\w-_]+)");
+    private static final Pattern HAPROXY_PATTERN = Pattern.compile("\\[([\\w-_=]+),([\\w-_=]+)\\]");
 
     private final SMARTCodeMapping smart = new SMARTCodeMapping();
 
@@ -158,13 +158,11 @@ public abstract class CollectDPluginParent {
             metric.append("sys.sensors.").append(vl.getType()).append(PERIOD).append(vl.getPluginInstance());
             tags.append(INSTANCE).append(instance);
         } else if (vl.getPlugin().equals("haproxy")) {
-            Matcher m = HAPROXY_PATTERN.matcher(vl.getTypeInstance());
+            metric.append("sys.haproxy.").append(vl.getTypeInstance());
+            Matcher m = HAPROXY_PATTERN.matcher(vl.getPluginInstance());
             if (m.matches()) {
-                metric.append("sys.haproxy.").append(m.group(3));
-                tags.append(" proxy=").append(m.group(2));
-                tags.append(" server=").append(m.group(1));
-            } else {
-                metric.append("sys.haproxy.").append(vl.getTypeInstance());
+                tags.append(" ").append(m.group(1));
+                tags.append(" ").append(m.group(2));
             }
         } else if (vl.getPlugin().equals("ipmi")) {
             metric.append("sys.ipmi.").append(vl.getType());

collectd-plugin-parent/src/test/java/timely/collectd/plugin/PluginParentTest.java

@@ -253,9 +253,10 @@ public class PluginParentTest {
         ValueList vl = new ValueList();
         vl.setHost(HOST);
         vl.setPlugin("haproxy");
+        vl.setPluginInstance("[proxy_name=proxy1,service_name=server1]");
         vl.setTime(TIME);
         vl.setType("gauge");
-        vl.setTypeInstance("server1.proxy1.queue_current");
+        vl.setTypeInstance("queue_current");
         vl.setValues(Collections.singletonList((Number) 0.0D));
         DataSet ds = new DataSet("GAUGE");
         ds.addDataSource(new DataSource("value", 1, 0.0D, 100.0D));
@@ -264,7 +265,7 @@ public class PluginParentTest {
         TestPlugin test = new TestPlugin();
         test.process(vl);
         assertEquals(
-                "put sys.haproxy.queue_current 1456156976840 0.0 host=r01n01 rack=r01 addl1=foo proxy=proxy1 server=server1 sample=value sampleType=GAUGE\n",
+                "put sys.haproxy.queue_current 1456156976840 0.0 host=r01n01 rack=r01 addl1=foo proxy_name=proxy1 service_name=server1 sample=value sampleType=GAUGE\n",
                 result);
     }
 

collectd-timely-plugin/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>timely</groupId>
         <artifactId>timely</artifactId>
-        <version>0.0.3</version>
+        <version>0.0.4</version>
     </parent>
     <artifactId>collectd-timely-plugin</artifactId>
     <dependencies>
@@ -85,6 +85,10 @@
                 <groupId>net.revelc.code</groupId>
                 <artifactId>formatter-maven-plugin</artifactId>
             </plugin>
+            <plugin>
+                <groupId>org.codehaus.mojo</groupId>
+                <artifactId>buildnumber-maven-plugin</artifactId>
+            </plugin>
         </plugins>
     </build>
 </project>

common/.gitignore

@@ -0,0 +1 @@
+/target/

common/pom.xml

@@ -3,7 +3,7 @@
   <parent>
     <groupId>timely</groupId>
     <artifactId>timely</artifactId>
-    <version>0.0.3</version>
+    <version>0.0.4</version>
   </parent>
   <artifactId>timely-common</artifactId>
   <dependencies>
@@ -13,4 +13,12 @@
 	    <scope>test</scope>
 	</dependency>
   </dependencies>
-</project>
+  <build>
+        <plugins>
+            <plugin>
+                <groupId>org.codehaus.mojo</groupId>
+                <artifactId>buildnumber-maven-plugin</artifactId>
+            </plugin>
+        </plugins>
+  </build>
+</project>

grafana/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>timely</groupId>
         <artifactId>timely</artifactId>
-        <version>0.0.3</version>
+        <version>0.0.4</version>
     </parent>
     <build>
         <plugins>
@@ -27,6 +27,10 @@
                     </execution>
                 </executions>
             </plugin>
+            <plugin>
+                <groupId>org.codehaus.mojo</groupId>
+                <artifactId>buildnumber-maven-plugin</artifactId>
+            </plugin>
         </plugins>
     </build>
-</project>
+</project>

grafana/timely-app/dist/plugin.json

@@ -23,8 +23,8 @@
       {"name": "Project site", "url": "https://github.com/NationalSecurityAgency/timely"},
       {"name": "License & Terms", "url": "https://raw.githubusercontent.com/NationalSecurityAgency/timely/master/LICENSE"}
     ],
-    "version": "0.0.3",
-    "updated": "2016-06-30"
+    "version": "0.0.4-SNAPSHOT",
+    "updated": "2016-10-27"
   },
 
   "includes": [

grafana/timely-app/plugin.json

@@ -23,8 +23,8 @@
       {"name": "Project site", "url": "https://github.com/NationalSecurityAgency/timely"},
       {"name": "License & Terms", "url": "https://raw.githubusercontent.com/NationalSecurityAgency/timely/master/LICENSE"}
     ],
-    "version": "0.0.3",
-    "updated": "2016-06-30"
+    "version": "0.0.4-SNAPSHOT",
+    "updated": "2016-10-27"
   },
 
   "includes": [

pom.xml

@@ -1,9 +1,9 @@
-<?xml version="1.0"?>
+<?xml version="1.0" encoding="UTF-8"?>
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <modelVersion>4.0.0</modelVersion>
     <groupId>timely</groupId>
     <artifactId>timely</artifactId>
-    <version>0.0.3</version>
+    <version>0.0.4</version>
     <packaging>pom</packaging>
     <name>Timely</name>
     <description>A secure and scalable time series database based on Apache Accumulo</description>
@@ -30,6 +30,7 @@
         <module>collectd-nsq-plugin</module>
         <module>collectd-plugin-parent</module>
         <module>grafana</module>
+        <module>analytics</module>
     </modules>
     <properties>
         <!-- Configure the os-maven-plugin extension to expand the classifier on -->
@@ -41,29 +42,28 @@
         <collectd-api-jar>/usr/share/collectd/java/collectd-api.jar</collectd-api-jar>
         <maven.compiler.source>1.8</maven.compiler.source>
         <maven.compiler.target>1.8</maven.compiler.target>
-        <version.accumulo>1.7.0</version.accumulo>
-        <version.caffeine>2.2.0</version.caffeine>
+        <version.accumulo>1.7.2</version.accumulo>
+        <version.caffeine>2.3.3</version.caffeine>
         <version.collectd>5.5.1</version.collectd>
-        <version.commons-configuration>1.10</version.commons-configuration>
         <version.commons-collections4>4.1</version.commons-collections4>
-        <version.commons-lang3>3.1</version.commons-lang3>
-        <version.disruptor>3.3.2</version.disruptor>
-        <version.fast-classpath-scanner>1.10.4</version.fast-classpath-scanner>
+        <version.commons-lang3>3.5</version.commons-lang3>
+        <version.disruptor>3.3.6</version.disruptor>
+        <version.fast-classpath-scanner>2.0.5</version.fast-classpath-scanner>
         <version.google-flatbuffers>1.4.0</version.google-flatbuffers>
         <version.guava>19.0</version.guava>
-        <version.hadoop>2.6.2</version.hadoop>
-        <version.httpcomponents>4.5.1</version.httpcomponents>
-        <version.jackson>2.6.1</version.jackson>
-        <version.javassist>3.20.0-GA</version.javassist>
+        <version.hadoop>2.7.3</version.hadoop>
+        <version.httpcomponents>4.5.2</version.httpcomponents>
+        <version.jackson>2.8.4</version.jackson>
+        <version.javassist>3.21.0-GA</version.javassist>
         <version.jcip-annotations>1.0</version.jcip-annotations>
         <version.jsoup>1.9.2</version.jsoup>
         <version.junit>4.12</version.junit>
-        <version.log4j>2.5</version.log4j>
+        <version.log4j>2.6.2</version.log4j>
         <version.netty>4.0.38.Final</version.netty>
         <version.netty-tcnative>1.1.33.Fork18</version.netty-tcnative>
         <version.powermock>1.6.4</version.powermock>
-        <version.slf4j>1.7.12</version.slf4j>
-        <version.spring-boot>1.4.0.RELEASE</version.spring-boot>
+        <version.slf4j>1.7.21</version.slf4j>
+        <version.spring-boot>1.4.1.RELEASE</version.spring-boot>
         <!-- plugin versions -->
         <version.eclipse-lifecycle>1.0.0</version.eclipse-lifecycle>
         <version.findbugs-maven>3.0.3</version.findbugs-maven>
@@ -74,6 +74,7 @@
         <version.maven-compiler>3.3</version.maven-compiler>
         <version.maven-dependency>2.10</version.maven-dependency>
         <version.maven-jar>3.0.1</version.maven-jar>
+        <version.buildnumber-maven>1.4</version.buildnumber-maven>
         <version.os-maven>1.5.0.Final</version.os-maven>
         <version.maven-pmd>3.6</version.maven-pmd>
         <version.maven-rpm>2.1.4</version.maven-rpm>
@@ -136,21 +137,10 @@
                 <version>LATEST</version>
             </dependency>
             <dependency>
-                <groupId>commons-configuration</groupId>
-                <artifactId>commons-configuration</artifactId>
-                <version>${version.commons-configuration}</version>
-                <exclusions>
-                    <exclusion>
-                        <groupId>*</groupId>
-                        <artifactId>*</artifactId>
-                    </exclusion>
-                </exclusions>
-            </dependency>
-	        <dependency>
-	            <groupId>io.github.lukehutch</groupId>
-	            <artifactId>fast-classpath-scanner</artifactId>
-	            <version>${version.fast-classpath-scanner}</version>
-	        </dependency>
+	        <groupId>io.github.lukehutch</groupId>
+	        <artifactId>fast-classpath-scanner</artifactId>
+	        <version>${version.fast-classpath-scanner}</version>
+	    </dependency>
             <dependency>
                 <groupId>io.netty</groupId>
                 <artifactId>netty-all</artifactId>
@@ -379,7 +369,12 @@
 	                    <archive>
 	                        <manifest>
 	                            <addDefaultImplementationEntries>true</addDefaultImplementationEntries>
+	                            <addDefaultSpecificationEntries>true</addDefaultSpecificationEntries>
 	                        </manifest>
+                                <manifestEntries>
+				    <Implementation-Build>${buildNumber}</Implementation-Build>
+                                    <Implementation-Timestamp>${timestamp}</Implementation-Timestamp>
+                                </manifestEntries>
 	                    </archive>
 	                </configuration>
                 </plugin>
@@ -403,6 +398,23 @@
 	                <artifactId>build-helper-maven-plugin</artifactId>
 	                <version>${version.maven-build-helper}</version>
 	            </plugin>
+                <plugin>
+                    <groupId>org.codehaus.mojo</groupId>
+                    <artifactId>buildnumber-maven-plugin</artifactId>
+                    <version>${version.buildnumber-maven}</version>
+                    <executions>
+                        <execution>
+                           <phase>validate</phase>
+                           <goals>
+                              <goal>create</goal>
+                           </goals>
+                        </execution>
+                    </executions>
+                    <configuration>
+                        <doCheck>false</doCheck>
+                        <doUpdate>false</doUpdate>
+                    </configuration>
+                </plugin>
                 <plugin>
                     <groupId>org.codehaus.mojo</groupId>
                     <artifactId>findbugs-maven-plugin</artifactId>

server/.gitignore

@@ -0,0 +1 @@
+/target/

server/conf/log4j2-spring.xml

@@ -13,11 +13,22 @@
       </Policies>
       <DefaultRolloverStrategy max="10" />
     </RollingRandomAccessFile>
+    <RollingRandomAccessFile name="Subscription" fileName="logs/timely-subscriptions.log"
+    						 filePattern="logs/$${date:yyyy-MM}/timely-subscriptions-%d{MM-dd-yyyy}-%i.log.gz">
+      <PatternLayout pattern="${LOG_PATTERN}"/>
+      <Policies>
+      	<SizeBasedTriggeringPolicy size="1024 MB" />
+      </Policies>
+      <DefaultRolloverStrategy max="10" />
+    </RollingRandomAccessFile>
     <Console name="Console" target="SYSTEM_OUT">
       <PatternLayout pattern="${LOG_PATTERN}"/>
     </Console>
   </Appenders>
   <Loggers>
+    <Logger name="timely.subscription" level="trace" additivity="false">
+      <AppenderRef ref="Subscription" />
+    </Logger>
     <Root level="trace" includeLocation="false">
       <AppenderRef ref="Console" level="warn" />
       <AppenderRef ref="Rolling" level="trace" />

server/conf/timely-standalone.yml

@@ -47,6 +47,10 @@ timely:
     port: 54323
     timeout: 60
     subscription-lag: 120
+    scanner-batch-size: 10000
+    flush-interval-seconds: 30
+    scanner-read-ahead: 1
+    subscription-batch-size: 5000
   meta-cache:
     expiration-minutes: 60
     initial-capacity: 2000

server/conf/timely.yml

@@ -48,6 +48,10 @@ timely:
     port:
     timeout: 60
     subscription-lag: 120
+    scanner-batch-size: 10000
+    flush-interval-seconds: 30
+    scanner-read-ahead: 1
+    subscription-batch-size: 5000
   meta-cache:
     expiration-minutes: 60
     initial-capacity: 2000

server/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>timely</groupId>
         <artifactId>timely</artifactId>
-        <version>0.0.3</version>
+        <version>0.0.4</version>
     </parent>
     <dependencies>
         <dependency>
@@ -45,10 +45,6 @@
             <scope>system</scope>
             <systemPath>${project.basedir}/lib/flatbuffers-java-1.4.0.jar</systemPath>
         </dependency>
-        <dependency>
-            <groupId>commons-configuration</groupId>
-            <artifactId>commons-configuration</artifactId>
-        </dependency>
         <dependency>
             <groupId>commons-beanutils</groupId>
             <artifactId>commons-beanutils</artifactId>
@@ -71,7 +67,7 @@
             <artifactId>netty-tcnative</artifactId>
             <version>${version.netty-tcnative}</version>
             <classifier>${os.detected.classifier}</classifier>
-            <!-- Eclipse doesn't like this, but it works on the cmd line -->
+            <scope>runtime</scope>
         </dependency>
         <dependency>
             <groupId>junit</groupId>
@@ -287,6 +283,10 @@
                 <groupId>net.revelc.code</groupId>
                 <artifactId>formatter-maven-plugin</artifactId>
             </plugin>
+            <plugin>
+                <groupId>org.codehaus.mojo</groupId>
+                <artifactId>buildnumber-maven-plugin</artifactId>
+            </plugin>
         </plugins>
     </build>
     <reporting>
@@ -334,4 +334,37 @@
             </plugin>
         </plugins>
     </reporting>
+    <profiles>
+        <profile>
+            <id>native-epoll-linux</id>
+            <activation>
+                <os>
+                    <family>!mac</family>
+                </os>
+            </activation>
+            <dependencies>
+		        <dependency>
+		            <groupId>io.netty</groupId>
+		            <artifactId>netty-transport-native-epoll</artifactId>
+		            <version>${version.netty}</version>
+		            <classifier>linux-x86_64</classifier>
+		        </dependency>
+            </dependencies>
+        </profile>
+        <profile>
+            <id>native-epoll</id>
+            <activation>
+                <os>
+                    <family>mac</family>
+                </os>
+            </activation>
+            <dependencies>
+		        <dependency>
+		            <groupId>io.netty</groupId>
+		            <artifactId>netty-transport-native-epoll</artifactId>
+		            <version>${version.netty}</version>
+		        </dependency>
+            </dependencies>
+        </profile>
+    </profiles>
 </project>

server/src/main/java/timely/Configuration.java

@@ -530,6 +530,10 @@ public class Configuration {
         private Integer port;
         public int timeout = 60;
         public int subscriptionLag = 120;
+        public int scannerBatchSize = 5000;
+        public int flushIntervalSeconds = 30;
+        public int scannerReadAhead = 1;
+        public int subscriptionBatchSize = 1000;
 
         public String getIp() {
             return ip;
@@ -566,6 +570,42 @@ public class Configuration {
             this.subscriptionLag = subscriptionLag;
             return Configuration.this;
         }
+
+        public int getScannerBatchSize() {
+            return this.scannerBatchSize;
+        }
+
+        public Configuration setScannerBatchSize(int batchSize) {
+            this.scannerBatchSize = batchSize;
+            return Configuration.this;
+        }
+
+        public int getFlushIntervalSeconds() {
+            return this.flushIntervalSeconds;
+        }
+
+        public Configuration setFlushIntervalSeconds(int flushInterval) {
+            this.flushIntervalSeconds = flushInterval;
+            return Configuration.this;
+        }
+
+        public int getScannerReadAhead() {
+            return this.scannerReadAhead;
+        }
+
+        public Configuration setScannerReadAhead(int readAhead) {
+            this.scannerReadAhead = readAhead;
+            return Configuration.this;
+        }
+
+        public int getSubscriptionBatchSize() {
+            return this.subscriptionBatchSize;
+        }
+
+        public Configuration setSubscriptionBatchSize(int batchSize) {
+            this.subscriptionBatchSize = batchSize;
+            return Configuration.this;
+        }
     }
 
     public class MetaCache {

server/src/main/java/timely/Server.java

@@ -3,7 +3,12 @@ package timely;
 import io.netty.bootstrap.Bootstrap;
 import io.netty.bootstrap.ServerBootstrap;
 import io.netty.buffer.PooledByteBufAllocator;
-import io.netty.channel.*;
+import io.netty.channel.Channel;
+import io.netty.channel.ChannelFuture;
+import io.netty.channel.ChannelHandler;
+import io.netty.channel.ChannelInitializer;
+import io.netty.channel.ChannelOption;
+import io.netty.channel.EventLoopGroup;
 import io.netty.channel.epoll.EpollDatagramChannel;
 import io.netty.channel.epoll.EpollEventLoopGroup;
 import io.netty.channel.epoll.EpollServerSocketChannel;
@@ -82,6 +87,7 @@ import timely.netty.websocket.subscription.WSAddSubscriptionRequestHandler;
 import timely.netty.websocket.subscription.WSCloseSubscriptionRequestHandler;
 import timely.netty.websocket.subscription.WSCreateSubscriptionRequestHandler;
 import timely.netty.websocket.subscription.WSRemoveSubscriptionRequestHandler;
+import timely.netty.websocket.subscription.WSTimelyExceptionHandler;
 import timely.netty.websocket.timeseries.WSAggregatorsRequestHandler;
 import timely.netty.websocket.timeseries.WSMetricsRequestHandler;
 import timely.netty.websocket.timeseries.WSQueryRequestHandler;
@@ -396,6 +402,10 @@ public class Server {
         wsServer.option(ChannelOption.ALLOCATOR, PooledByteBufAllocator.DEFAULT);
         wsServer.option(ChannelOption.SO_BACKLOG, 128);
         wsServer.option(ChannelOption.SO_KEEPALIVE, true);
+        /* Not sure if next three lines are necessary */
+        wsServer.option(ChannelOption.SO_SNDBUF, 1048576);
+        wsServer.option(ChannelOption.WRITE_BUFFER_HIGH_WATER_MARK, 838860);
+        wsServer.option(ChannelOption.WRITE_BUFFER_LOW_WATER_MARK, 620145);
         wsChannelHandle = wsServer.bind(wsIp, wsPort).sync().channel();
         final String wsAddress = ((InetSocketAddress) wsChannelHandle.localAddress()).getAddress().getHostAddress();
 
@@ -558,7 +568,7 @@ public class Server {
                 ch.pipeline().addLast("add", new WSAddSubscriptionRequestHandler());
                 ch.pipeline().addLast("remove", new WSRemoveSubscriptionRequestHandler());
                 ch.pipeline().addLast("close", new WSCloseSubscriptionRequestHandler());
-
+                ch.pipeline().addLast("error", new WSTimelyExceptionHandler());
             }
         };
 

server/src/main/java/timely/adapter/accumulo/MetricAdapter.java

@@ -82,7 +82,7 @@ public class MetricAdapter {
         // @formatter:on
     }
 
-    public static Metric parse(Key k, Value v) {
+    public static Metric parse(Key k, Value v, boolean includeVizTag) {
         ComparablePair<String, Long> row = rowCoder.decode(k.getRow().getBytes());
         // @formatter:off
         Metric.Builder builder = Metric.newBuilder()
@@ -91,9 +91,16 @@ public class MetricAdapter {
                 .tag(tagParser.parse(k.getColumnFamily().toString()));
         // @formatter:on
         tagListParser.parse(k.getColumnQualifier().toString()).forEach(builder::tag);
+        if (includeVizTag && k.getColumnVisibility().getLength() > 0) {
+            tagListParser.parse("viz=" + k.getColumnVisibility().toString()).forEach(builder::tag);
+        }
         return builder.build();
     }
 
+    public static Metric parse(Key k, Value v) {
+        return parse(k, v, false);
+    }
+
     public static byte[] encodeRowKey(String metricName, Long timestamp) {
         return rowCoder.encode(new ComparablePair<>(metricName, timestamp));
     }

server/src/main/java/timely/api/annotation/AnnotationResolver.java

@@ -1,6 +1,7 @@
 package timely.api.annotation;
 
 import io.github.lukehutch.fastclasspathscanner.FastClasspathScanner;
+import io.github.lukehutch.fastclasspathscanner.scanner.ScanResult;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -17,7 +18,7 @@ import timely.api.request.WebSocketRequest;
 public class AnnotationResolver {
 
     private static final Logger LOG = LoggerFactory.getLogger(AnnotationResolver.class);
-    private static final FastClasspathScanner scanner = new FastClasspathScanner("timely.api").scan();
+    private static final ScanResult result = new FastClasspathScanner("timely.api").scan();
     private static List<String> tcpClassNames;
     private static List<String> httpClassNames;
     private static List<String> wsClassNames;
@@ -31,7 +32,7 @@ public class AnnotationResolver {
     }
 
     static {
-        tcpClassNames = scanner.getNamesOfClassesWithAnnotation(Tcp.class);
+        tcpClassNames = result.getNamesOfClassesWithAnnotation(Tcp.class);
         LOG.trace("Found tcp classes: {}", tcpClassNames);
         if (null != tcpClassNames) {
             for (String cls : tcpClassNames) {
@@ -43,7 +44,7 @@ public class AnnotationResolver {
             }
         }
         LOG.trace("Loaded tcp classes: {}", tcpClasses);
-        httpClassNames = scanner.getNamesOfClassesWithAnnotation(Http.class);
+        httpClassNames = result.getNamesOfClassesWithAnnotation(Http.class);
         LOG.trace("Found http class names: {}", httpClassNames);
         if (null != httpClassNames) {
             for (String cls : httpClassNames) {
@@ -55,7 +56,7 @@ public class AnnotationResolver {
             }
         }
         LOG.trace("Loaded http classes: {}", httpClasses);
-        wsClassNames = scanner.getNamesOfClassesWithAnnotation(WebSocket.class);
+        wsClassNames = result.getNamesOfClassesWithAnnotation(WebSocket.class);
         LOG.trace("Found web socket class names: {}", wsClassNames);
         if (null != wsClassNames) {
             for (String cls : wsClassNames) {
@@ -67,7 +68,7 @@ public class AnnotationResolver {
             }
         }
         LOG.trace("Loaded web socket classes: {}", wsClasses);
-        udpClassNames = scanner.getNamesOfClassesWithAnnotation(Udp.class);
+        udpClassNames = result.getNamesOfClassesWithAnnotation(Udp.class);
         LOG.trace("Found udp classes: {}", udpClassNames);
         if (null != udpClassNames) {
             for (String cls : udpClassNames) {

server/src/main/java/timely/api/request/auth/BasicAuthLoginRequest.java

@@ -1,30 +1,12 @@
 package timely.api.request.auth;
 
+import timely.api.BasicAuthLogin;
 import timely.api.annotation.Http;
 import timely.api.request.HttpPostRequest;
 import timely.util.JsonUtil;
 
 @Http(path = "/login")
-public class BasicAuthLoginRequest implements HttpPostRequest {
-
-    private String username;
-    private String password;
-
-    public String getUsername() {
-        return username;
-    }
-
-    public void setUsername(String user) {
-        this.username = user;
-    }
-
-    public String getPassword() {
-        return password;
-    }
-
-    public void setPassword(String password) {
-        this.password = password;
-    }
+public class BasicAuthLoginRequest extends BasicAuthLogin implements HttpPostRequest {
 
     @Override
     public void validate() {

server/src/main/java/timely/api/request/subscription/AddSubscription.java

@@ -11,6 +11,7 @@ public class AddSubscription extends SubscriptionRequest {
     private String metric = null;
     private Optional<Map<String, String>> tags = Optional.empty();
     private Optional<Long> startTime = Optional.empty();
+    private Optional<Long> endTime = Optional.empty();
     private Optional<Long> delayTime = Optional.empty();
 
     public String getMetric() {
@@ -37,6 +38,14 @@ public class AddSubscription extends SubscriptionRequest {
         this.startTime = Optional.ofNullable(startTime);
     }
 
+    public Optional<Long> getEndTime() {
+        return endTime;
+    }
+
+    public void setEndTime(long endTime) {
+        this.endTime = Optional.ofNullable(endTime);
+    }
+
     public Optional<Long> getDelayTime() {
         return delayTime;
     }

server/src/main/java/timely/api/request/timeseries/QueryRequest.java

@@ -30,7 +30,7 @@ public class QueryRequest extends AuthenticatedRequest implements HttpGetRequest
     public static class RateOption {
 
         private boolean counter = false;
-        private long counterMax = Long.MAX_VALUE;
+        private long counterMax = 0;
         private long resetValue = 0;
 
         public RateOption() {

server/src/main/java/timely/api/response/timeseries/AggregatorsResponse.java

@@ -6,6 +6,7 @@ import java.util.List;
 
 import timely.api.response.timeseries.AggregatorsResponse.AggregatorsResponseDeserializer;
 import timely.api.response.timeseries.AggregatorsResponse.AggregatorsResponseSerializer;
+import timely.sample.Aggregator;
 import timely.sample.aggregators.Avg;
 import timely.sample.aggregators.Count;
 import timely.sample.aggregators.Dev;
@@ -57,6 +58,7 @@ public class AggregatorsResponse {
         RESPONSE.addAggregator(Min.class.getSimpleName().toLowerCase());
         RESPONSE.addAggregator(Sum.class.getSimpleName().toLowerCase());
         RESPONSE.addAggregator(Count.class.getSimpleName().toLowerCase());
+        RESPONSE.addAggregator(Aggregator.NONE);
     }
 
     private List<String> aggregators = new ArrayList<>();

server/src/main/java/timely/netty/websocket/subscription/WSAddSubscriptionRequestHandler.java

@@ -28,6 +28,7 @@ public class WSAddSubscriptionRequestHandler extends SimpleChannelInboundHandler
             }
             Map<String, String> tags = null;
             Long startTime = 0L;
+            Long endTime = 0L;
             Long delayTime = 5000L;
             if (add.getTags().isPresent()) {
                 tags = add.getTags().get();
@@ -35,10 +36,13 @@ public class WSAddSubscriptionRequestHandler extends SimpleChannelInboundHandler
             if (add.getStartTime().isPresent()) {
                 startTime = add.getStartTime().get();
             }
+            if (add.getEndTime().isPresent()) {
+                endTime = add.getEndTime().get();
+            }
             if (add.getDelayTime().isPresent()) {
                 delayTime = add.getDelayTime().get();
             }
-            s.addMetric(metric, tags, startTime, delayTime);
+            s.addMetric(metric, tags, startTime, endTime, delayTime);
         } else {
             LOG.error("Unknown subscription id, create subscription first");
             ctx.writeAndFlush(new CloseWebSocketFrame(1003, "Unknown subscription id, create subscription first"));

server/src/main/java/timely/netty/websocket/subscription/WSCloseSubscriptionRequestHandler.java

@@ -2,6 +2,7 @@ package timely.netty.websocket.subscription;
 
 import io.netty.channel.ChannelHandlerContext;
 import io.netty.channel.SimpleChannelInboundHandler;
+import io.netty.handler.codec.http.websocketx.CloseWebSocketFrame;
 import timely.api.request.subscription.CloseSubscription;
 import timely.subscription.Subscription;
 import timely.subscription.SubscriptionRegistry;
@@ -14,6 +15,7 @@ public class WSCloseSubscriptionRequestHandler extends SimpleChannelInboundHandl
         if (null != s) {
             s.close();
         }
+        ctx.writeAndFlush(new CloseWebSocketFrame(1000, "Client requested close."));
     }
 
 }

server/src/main/java/timely/netty/websocket/subscription/WSTimelyExceptionHandler.java

@@ -0,0 +1,20 @@
+package timely.netty.websocket.subscription;
+
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.channel.SimpleChannelInboundHandler;
+import io.netty.handler.codec.http.websocketx.CloseWebSocketFrame;
+import timely.api.response.TimelyException;
+
+public class WSTimelyExceptionHandler extends SimpleChannelInboundHandler<TimelyException> {
+
+    @Override
+    protected void channelRead0(ChannelHandlerContext ctx, TimelyException e) throws Exception {
+        ctx.writeAndFlush(new CloseWebSocketFrame(1008, e.getMessage()));
+    }
+
+    @Override
+    public void exceptionCaught(ChannelHandlerContext ctx, Throwable e) throws Exception {
+        ctx.writeAndFlush(new CloseWebSocketFrame(1008, e.getMessage()));
+    }
+
+}

server/src/main/java/timely/sample/Aggregation.java

@@ -0,0 +1,143 @@
+package timely.sample;
+
+import com.google.common.base.Preconditions;
+import org.apache.commons.lang3.builder.ToStringBuilder;
+import timely.api.request.timeseries.QueryRequest;
+
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.Serializable;
+import java.util.*;
+
+public class Aggregation implements Iterable<Sample>, Serializable {
+
+    public static class AggregatedValue implements Serializable {
+
+        private static final long serialVersionUID = 1L;
+
+        private int count = 0;
+        private double value = 0.0D;
+
+        public void incrementCount() {
+            count++;
+        }
+
+        public int getCount() {
+            return count;
+        }
+
+        public void setCount(int count) {
+            this.count = count;
+        }
+
+        public double getValue() {
+            return value;
+        }
+
+        public void setValue(double value) {
+            this.value = value;
+        }
+
+        @Override
+        public String toString() {
+            StringBuilder buf = new StringBuilder();
+            buf.append("{count: ").append(this.count);
+            buf.append(" value: ").append(this.value).append("}");
+            return buf.toString();
+        }
+
+    }
+
+    private static final long serialVersionUID = 1L;
+
+    protected transient Aggregator aggregator;
+    protected final String aggregatorClassname;
+
+    protected final TreeMap<Long, AggregatedValue> buckets = new TreeMap<>();
+
+    public Aggregation(Aggregator agg) {
+        Preconditions.checkNotNull(agg, "Aggregator object cannot be null");
+        this.aggregator = agg;
+        this.aggregatorClassname = agg.getClass().getName();
+    }
+
+    private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException {
+        in.defaultReadObject();
+        try {
+            aggregator = (Aggregator) getClass().getClassLoader().loadClass(aggregatorClassname).newInstance();
+        } catch (Exception ex) {
+            throw new RuntimeException(ex);
+        }
+    }
+
+    public void add(long ts, double value) {
+        AggregatedValue val = buckets.get(ts);
+        if (null == val) {
+            buckets.put(ts, val = new AggregatedValue());
+        }
+        val.setValue(aggregator.aggregate(val.getValue(), val.getCount(), value));
+        val.incrementCount();
+    }
+
+    public void merge(Aggregation other) {
+        for (Map.Entry<Long, AggregatedValue> e : other.buckets.entrySet()) {
+            AggregatedValue thisVal = buckets.get(e.getKey());
+            if (null == thisVal) {
+                buckets.put(e.getKey(), e.getValue());
+            } else {
+                thisVal.setCount(thisVal.getCount() + e.getValue().getCount());
+                thisVal.setValue(aggregator.aggregate(thisVal.getValue(), thisVal.getCount(), e.getValue().getValue()));
+            }
+        }
+    }
+
+    @Override
+    public Iterator<Sample> iterator() {
+        return new Iterator<Sample>() {
+
+            Iterator<Long> keys = buckets.keySet().iterator();
+
+            @Override
+            public boolean hasNext() {
+                return keys.hasNext();
+            }
+
+            @Override
+            public Sample next() {
+                long time = keys.next();
+                AggregatedValue val = buckets.get(time);
+                double value = aggregator.last(val.getValue(), val.getCount());
+                return new Sample(time, value);
+            }
+        };
+    }
+
+    @Override
+    public String toString() {
+        ToStringBuilder tsb = new ToStringBuilder(this);
+        tsb.append("aggregator", this.aggregatorClassname);
+        tsb.append("values={");
+        for (Map.Entry<Long, AggregatedValue> e : this.buckets.entrySet()) {
+            tsb.append(e.getKey()).append("=").append(e.getValue()).append(", ");
+        }
+        tsb.append("}");
+        return tsb.toString();
+    }
+
+    public static Aggregation combineAggregation(Collection<Aggregation> samples, QueryRequest.RateOption rateOption) {
+        List<Aggregation> sampleList = new ArrayList<>(samples.size());
+        if (samples.isEmpty()) {
+            throw new IllegalArgumentException("Empty samples to combine");
+        }
+        for (Aggregation a : samples) {
+            sampleList.add(a);
+        }
+        Iterator<Aggregation> iter = sampleList.iterator();
+        Aggregation result = iter.next();
+        while (iter.hasNext()) {
+            result.merge(iter.next());
+        }
+        return result;
+    }
+
+}

server/src/main/java/timely/sample/Aggregator.java

@@ -9,6 +9,8 @@ import timely.sample.aggregators.Sum;
 
 public interface Aggregator {
 
+    String NONE = "none";
+
     /**
      * Given the current value, and the current count, incorporate the new value
      * into current and return the result.
@@ -36,6 +38,9 @@ public interface Aggregator {
     double last(double current, int count);
 
     static Class<? extends Aggregator> getAggregator(String aggregatorName) {
+        if (aggregatorName == null) {
+            return null;
+        }
         switch (aggregatorName) {
             case "sum":
                 return Sum.class;
@@ -50,7 +55,7 @@ public interface Aggregator {
             case "count":
                 return Count.class;
             default:
-                return Avg.class;
+                return null;
         }
     }
 }

server/src/main/java/timely/sample/Downsample.java

@@ -1,88 +1,31 @@
 package timely.sample;
 
-import java.io.IOException;
-import java.io.ObjectInputStream;
+import com.google.common.base.Preconditions;
+import org.apache.commons.lang3.builder.ToStringBuilder;
+import timely.api.request.timeseries.QueryRequest.RateOption;
+
 import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Iterator;
 import java.util.List;
-import java.util.Map.Entry;
-import java.util.TreeMap;
 
-import org.apache.commons.lang3.builder.ToStringBuilder;
-
-import timely.api.request.timeseries.QueryRequest.RateOption;
-
-import com.google.common.base.Preconditions;
-
-public class Downsample implements Iterable<Sample>, Serializable {
-
-    public static class AggregatedValue implements Serializable {
-
-        private static final long serialVersionUID = 1L;
-
-        private int count = 0;
-        private double value = 0.0D;
-
-        public void incrementCount() {
-            count++;
-        }
-
-        public int getCount() {
-            return count;
-        }
-
-        public void setCount(int count) {
-            this.count = count;
-        }
-
-        public double getValue() {
-            return value;
-        }
-
-        public void setValue(double value) {
-            this.value = value;
-        }
-
-        @Override
-        public String toString() {
-            StringBuilder buf = new StringBuilder();
-            buf.append("{count: ").append(this.count);
-            buf.append(" value: ").append(this.value).append("}");
-            return buf.toString();
-        }
-
-    }
+public class Downsample extends Aggregation implements Iterable<Sample>, Serializable {
 
     private static final long serialVersionUID = 1L;
 
-    private transient Aggregator aggregator;
-    private final String aggregatorClassname;
-    private final long start;
-    private final long period;
-
-    private final TreeMap<Long, AggregatedValue> buckets = new TreeMap<>();
+    protected final long start;
+    protected final long period;
 
     public Downsample(long start, long end, long period, Aggregator agg) {
+        super(agg);
         Preconditions.checkArgument(start < end, "Start must be < end");
         Preconditions.checkArgument(period >= 1, "period cannot be < 1");
-        Preconditions.checkNotNull(agg, "Aggregator object cannot be null");
         this.start = start;
         this.period = period;
-        this.aggregator = agg;
-        this.aggregatorClassname = agg.getClass().getName();
-    }
-
-    private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException {
-        in.defaultReadObject();
-        try {
-            aggregator = (Aggregator) getClass().getClassLoader().loadClass(aggregatorClassname).newInstance();
-        } catch (Exception ex) {
-            throw new RuntimeException(ex);
-        }
     }
 
+    @Override
     public void add(long ts, double value) {
         if (ts < start) {
             throw new IllegalArgumentException("timestamp is out of range");
@@ -97,54 +40,16 @@ public class Downsample implements Iterable<Sample>, Serializable {
         val.incrementCount();
     }
 
-    public void merge(Downsample other) {
-        for (Entry<Long, AggregatedValue> e : other.buckets.entrySet()) {
-            AggregatedValue thisVal = buckets.get(e.getKey());
-            if (null == thisVal) {
-                buckets.put(e.getKey(), e.getValue());
-            } else {
-                thisVal.setCount(thisVal.getCount() + e.getValue().getCount());
-                thisVal.setValue(aggregator.aggregate(thisVal.getValue(), thisVal.getCount(), e.getValue().getValue()));
-            }
-        }
-    }
-
-    @Override
-    public Iterator<Sample> iterator() {
-        return new Iterator<Sample>() {
-
-            Iterator<Long> keys = buckets.keySet().iterator();
-
-            @Override
-            public boolean hasNext() {
-                return keys.hasNext();
-            }
-
-            @Override
-            public Sample next() {
-                long time = keys.next();
-                AggregatedValue val = buckets.get(time);
-                double value = aggregator.last(val.getValue(), val.getCount());
-                return new Sample(time, value);
-            }
-        };
-    }
-
     @Override
     public String toString() {
         ToStringBuilder tsb = new ToStringBuilder(this);
-        tsb.append("aggregator", this.aggregatorClassname);
         tsb.append("period", this.period);
         tsb.append("start", this.start);
-        tsb.append("values={");
-        for (Entry<Long, AggregatedValue> e : this.buckets.entrySet()) {
-            tsb.append(e.getKey()).append("=").append(e.getValue()).append(", ");
-        }
-        tsb.append("}");
+        tsb.append(super.toString());
         return tsb.toString();
     }
 
-    public static Downsample combine(Collection<Downsample> samples, RateOption rateOption) {
+    public static Downsample combineDownsample(Collection<Downsample> samples, RateOption rateOption) {
         List<Downsample> sampleList = new ArrayList<>(samples.size());
         if (samples.isEmpty()) {
             throw new IllegalArgumentException("Empty samples to combine");

server/src/main/java/timely/sample/iterators/AggregationIterator.java

@@ -0,0 +1,126 @@
+package timely.sample.iterators;
+
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.IteratorEnvironment;
+import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.accumulo.core.iterators.WrappingIterator;
+import timely.model.Tag;
+import timely.model.parse.TagListParser;
+import timely.sample.Aggregation;
+import timely.sample.Aggregator;
+import timely.sample.Downsample;
+import timely.sample.Sample;
+
+import java.io.*;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * This iterator will aggregate across series as output from the
+ * DownsampleIterator. The same set of aggregator functions are supported as in
+ * the DownsampleIterator.
+ */
+public class AggregationIterator extends WrappingIterator {
+
+    // To keep the output of the aggregation iterator consistent with that of
+    // the downsample iterator, we
+    // supply a static set of tags to be returned with the Aggregation (subclass
+    // of Downsample)
+    private static final String TAGS = "aggregation.tags";
+    private static final String AGGCLASS = "aggregation.aggclass";
+
+    private Aggregation aggregation;
+    private Set<Tag> tags;
+    private Key last;
+
+    @SuppressWarnings("unchecked")
+    @Override
+    public void init(SortedKeyValueIterator<Key, Value> source, Map<String, String> options, IteratorEnvironment env)
+            throws IOException {
+        super.init(source, options, env);
+        String aggClassname = options.get(AGGCLASS);
+        Class<? extends Aggregator> aggClass = null;
+        try {
+            aggClass = (Class<? extends Aggregator>) this.getClass().getClassLoader().loadClass(aggClassname);
+            tags = new HashSet<>(new TagListParser().parse(options.get(TAGS)));
+            aggregation = new Aggregation(aggClass.newInstance());
+        } catch (ClassNotFoundException e) {
+            throw new RuntimeException(e);
+        } catch (InstantiationException | IllegalAccessException e) {
+            throw new RuntimeException("Error creating aggregator class: " + aggClass, e);
+        }
+    }
+
+    @Override
+    public boolean hasTop() {
+        // for each set of downsamples (one per subquery)
+        while (super.hasTop()) {
+            last = super.getTopKey();
+
+            // decode the set of samples (or series)
+            Map<Set<Tag>, Downsample> samples = null;
+            try {
+                samples = DownsampleIterator.decodeValue(super.getTopValue());
+            } catch (IOException e) {
+                throw new RuntimeException("Unable to decode upstream value as a Downsample", e);
+            } catch (ClassNotFoundException e) {
+                throw new RuntimeException("Unable to find class for value (expected to be a Downsample)", e);
+            }
+
+            // add the downsampled values to the aggregation
+            for (Map.Entry<Set<Tag>, Downsample> entry : samples.entrySet()) {
+                for (Sample sample : entry.getValue()) {
+                    aggregation.add(sample.timestamp, sample.value);
+                }
+            }
+
+            try {
+                super.next();
+            } catch (IOException e) {
+                throw new RuntimeException("Downstream next() failed", e);
+            }
+        }
+        return last != null;
+    }
+
+    @Override
+    public Key getTopKey() {
+        return last;
+    }
+
+    @Override
+    public Value getTopValue() {
+        // return a value which is consistent with Map<Set<Tag>, Downsample>
+        try (ByteArrayOutputStream bos = new ByteArrayOutputStream();
+                ObjectOutputStream out = new ObjectOutputStream(bos)) {
+            Map<Set<Tag>, Aggregation> aggregationMap = new HashMap<>();
+            aggregationMap.put(tags, aggregation);
+            out.writeObject(aggregationMap);
+            out.flush();
+            return new Value(bos.toByteArray());
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    @Override
+    public void next() throws IOException {
+        last = null;
+    }
+
+    public static void setAggregationOptions(IteratorSetting is, Map<String, String> tags, String classname) {
+        is.addOption(TAGS, new TagListParser().combine(tags));
+        is.addOption(AGGCLASS, classname);
+    }
+
+    @SuppressWarnings("unchecked")
+    public static Map<Set<Tag>, Aggregation> decodeValue(Value value) throws IOException, ClassNotFoundException {
+        ByteArrayInputStream bis = new ByteArrayInputStream(value.get());
+        ObjectInputStream ois = new ObjectInputStream(bis);
+        return (Map<Set<Tag>, Aggregation>) ois.readObject();
+    }
+}

server/src/main/java/timely/sample/iterators/DownsampleIterator.java

@@ -72,7 +72,7 @@ public class DownsampleIterator extends WrappingIterator {
             try {
                 super.next();
             } catch (IOException e) {
-                throw new RuntimeException(e);
+                throw new RuntimeException("Downstream next() failed", e);
             }
         }
         return last != null;

server/src/main/java/timely/store/DataStore.java

@@ -26,7 +26,7 @@ public interface DataStore {
 
     void flush() throws TimelyException;
 
-    Scanner createScannerForMetric(String sessionId, String metric, Map<String, String> tags, long startTime, int lag)
-            throws TimelyException;
+    Scanner createScannerForMetric(String sessionId, String metric, Map<String, String> tags, long startTime,
+            long endTime, int lag, int scannerBatchSize, int scannerReadAhead) throws TimelyException;
 
 }

server/src/main/java/timely/store/DataStoreImpl.java

@@ -78,9 +78,12 @@ import timely.api.response.timeseries.SuggestResponse;
 import timely.auth.AuthCache;
 import timely.model.Metric;
 import timely.model.Tag;
+import timely.sample.Aggregation;
 import timely.sample.Aggregator;
 import timely.sample.Downsample;
 import timely.sample.Sample;
+import timely.sample.aggregators.Avg;
+import timely.sample.iterators.AggregationIterator;
 import timely.sample.iterators.DownsampleIterator;
 import timely.store.iterators.RateIterator;
 import timely.util.MetaKeySet;
@@ -90,7 +93,8 @@ public class DataStoreImpl implements DataStore {
     private static final Logger LOG = LoggerFactory.getLogger(DataStoreImpl.class);
 
     private static final long METRICS_PERIOD = 30000;
-    private static final long DEFAULT_DOWNSAMPLE_MS = 60000;
+    private static final long DEFAULT_DOWNSAMPLE_MS = 1;
+    private static final String DEFAULT_DOWNSAMPLE_AGGREGATOR = Avg.class.getSimpleName().toLowerCase();
     private static final Pattern REGEX_TEST = Pattern.compile("^\\w+$");
 
     /*
@@ -127,6 +131,8 @@ public class DataStoreImpl implements DataStore {
     private final ThreadLocal<BatchWriter> metaWriter = new ThreadLocal<>();
     private final ThreadLocal<BatchWriter> batchWriter = new ThreadLocal<>();
     private boolean anonAccessAllowed = false;
+    private final Map<String, String> ageOff;
+    private final long defaultAgeOff;
 
     public DataStoreImpl(Configuration conf, int numWriteThreads) throws TimelyException {
 
@@ -159,6 +165,9 @@ public class DataStoreImpl implements DataStore {
                     }
                 }
             }
+            ageOff = getAgeOff(conf);
+            defaultAgeOff = this.getAgeOffForMetric(MetricAgeOffIterator.DEFAULT_AGEOFF_KEY);
+
             final Map<String, String> tableIdMap = connector.tableOperations().tableIdMap();
             if (!tableIdMap.containsKey(metricsTable)) {
                 try {
@@ -169,7 +178,7 @@ public class DataStoreImpl implements DataStore {
                 }
             }
             this.removeAgeOffIterators(connector, metricsTable);
-            this.applyAgeOffIterator(connector, metricsTable, conf);
+            this.applyAgeOffIterator(connector, metricsTable, true);
 
             metaTable = conf.getMetaTable();
             if (!tableIdMap.containsKey(metaTable)) {
@@ -181,7 +190,7 @@ public class DataStoreImpl implements DataStore {
                 }
             }
             this.removeAgeOffIterators(connector, metaTable);
-            this.applyAgeOffIterator(connector, metaTable, conf);
+            this.applyAgeOffIterator(connector, metaTable, false);
 
             internalMetricsTimer.schedule(new TimerTask() {
 
@@ -209,18 +218,26 @@ public class DataStoreImpl implements DataStore {
         }
     }
 
-    private void applyAgeOffIterator(Connector con, String tableName, Configuration config) throws Exception {
-        int priority = 100;
-        Map<String, String> ageOffOptions = new HashMap<>();
-        for (Entry<String, Integer> e : config.getMetricAgeOffDays().entrySet()) {
-            String ageoff = Long.toString(e.getValue() * 86400000L);
-            ageOffOptions.put(MetricAgeOffFilter.AGE_OFF_PREFIX + e.getKey(), ageoff);
+    private void applyAgeOffIterator(Connector con, String tableName, boolean useIterator) throws Exception {
+        IteratorSetting ageOffIteratorSettings = null;
+        if (useIterator) {
+            ageOffIteratorSettings = new IteratorSetting(100, "ageoff", MetricAgeOffIterator.class, this.ageOff);
+        } else {
+            ageOffIteratorSettings = new IteratorSetting(100, "ageoff", MetricAgeOffFilter.class, this.ageOff);
         }
-        IteratorSetting ageOffIteratorSettings = new IteratorSetting(priority, "ageoff", MetricAgeOffFilter.class,
-                ageOffOptions);
         connector.tableOperations().attachIterator(tableName, ageOffIteratorSettings, AGEOFF_SCOPES);
     }
 
+    private Map<String, String> getAgeOff(Configuration conf) {
+        Map<String, String> ageOffOptions = new HashMap<>();
+        conf.getMetricAgeOffDays().forEach((k, v) -> {
+            String ageoff = Long.toString(v * 86400000L);
+            LOG.trace("Adding age off for metric: {} of {} days", k, v);
+            ageOffOptions.put(MetricAgeOffIterator.AGE_OFF_PREFIX + k, ageoff);
+        });
+        return ageOffOptions;
+    }
+
     @Override
     public void store(Metric metric) {
         LOG.trace("Received Store Request for: {}", metric);
@@ -453,12 +470,22 @@ public class DataStoreImpl implements DataStore {
         try {
             long now = System.currentTimeMillis();
             for (SubQuery query : msg.getQueries()) {
-                Map<Set<Tag>, List<Downsample>> allSeries = new HashMap<>();
+                Map<Set<Tag>, List<Aggregation>> allSeries = new HashMap<>();
                 String metric = query.getMetric();
                 BatchScanner scanner = connector.createBatchScanner(metricsTable, getSessionAuthorizations(msg),
                         scannerThreads);
                 try {
-                    setQueryRange(scanner, metric, startTs, endTs);
+
+                    // Reset the start timestamp for the query to the
+                    // beginning of the downsample period based on the epoch
+                    long downsample = getDownsamplePeriod(query);
+                    LOG.trace("Downsample period {}", downsample);
+                    long startOfFirstPeriod = startTs - (startTs % downsample);
+                    long endDistanceFromDownSample = endTs % downsample;
+                    long endOfLastPeriod = (endDistanceFromDownSample > 0 ? endTs + downsample
+                            - endDistanceFromDownSample : endTs);
+
+                    setQueryRange(scanner, metric, startOfFirstPeriod, endOfLastPeriod);
                     List<String> tagOrder = prioritizeTags(query);
                     Map<String, String> orderedTags = orderTags(tagOrder, query.getTags());
                     setQueryColumns(scanner, metric, orderedTags);
@@ -466,29 +493,50 @@ public class DataStoreImpl implements DataStore {
                     if (query.isRate()) {
                         LOG.trace("Adding rate iterator");
                         IteratorSetting rate = new IteratorSetting(499, RateIterator.class);
+                        RateIterator.setRateOptions(rate, query.getRateOptions());
                         scanner.addScanIterator(rate);
                     }
-                    long downsample = getDownsamplePeriod(query);
-                    LOG.trace("Downsample period {}", downsample);
+
+                    Class<? extends Aggregator> daggClass = getDownsampleAggregator(query);
+                    if (daggClass == null) {
+                        // we should always have a downsample iterator in the
+                        // stack.
+                        throw new TimelyException(HttpResponseStatus.INTERNAL_SERVER_ERROR.code(),
+                                "Error during query: programming error", "daggClass == null");
+                    } else {
+                        LOG.trace("Downsample Aggregator type {}", daggClass.getSimpleName());
+                        IteratorSetting is = new IteratorSetting(500, DownsampleIterator.class);
+                        DownsampleIterator.setDownsampleOptions(is, startOfFirstPeriod, endOfLastPeriod, downsample,
+                                daggClass.getName());
+                        scanner.addScanIterator(is);
+                    }
+
                     Class<? extends Aggregator> aggClass = getAggregator(query);
-                    LOG.trace("Aggregator type {}", aggClass.getSimpleName());
-                    IteratorSetting is = new IteratorSetting(500, DownsampleIterator.class);
-                    DownsampleIterator.setDownsampleOptions(is, startTs, endTs, downsample, aggClass.getName());
-                    scanner.addScanIterator(is);
+                    // the aggregation iterator is optional
+                    if (aggClass != null) {
+                        LOG.trace("Aggregator type {}", aggClass.getSimpleName());
+                        IteratorSetting is = new IteratorSetting(501, AggregationIterator.class);
+                        AggregationIterator.setAggregationOptions(is, query.getTags(), aggClass.getName());
+                        scanner.addScanIterator(is);
+                    }
 
                     // tag -> array of results by period starting at start
                     for (Entry<Key, Value> encoded : scanner) {
-                        Map<Set<Tag>, Downsample> samples = DownsampleIterator.decodeValue(encoded.getValue());
-                        for (Entry<Set<Tag>, Downsample> entry : samples.entrySet()) {
+                        // we can decode the value as a Map<Set<Tag>,
+                        // Aggregation> even if an AggregationIterator
+                        // is not used because Downsample is a subclass of
+                        // Aggregation
+                        Map<Set<Tag>, Aggregation> samples = AggregationIterator.decodeValue(encoded.getValue());
+                        for (Entry<Set<Tag>, Aggregation> entry : samples.entrySet()) {
                             Set<Tag> key = new HashSet<>();
                             for (Tag tag : entry.getKey()) {
                                 if (query.getTags().keySet().contains(tag.getKey())) {
                                     key.add(tag);
                                 }
                             }
-                            List<Downsample> downsamples = allSeries.getOrDefault(key, new ArrayList<>());
-                            downsamples.add(entry.getValue());
-                            allSeries.put(key, downsamples);
+                            List<Aggregation> aggregations = allSeries.getOrDefault(key, new ArrayList<>());
+                            aggregations.add(entry.getValue());
+                            allSeries.put(key, aggregations);
                         }
                     }
                     LOG.trace("allSeries: {}", allSeries);
@@ -498,7 +546,7 @@ public class DataStoreImpl implements DataStore {
 
                 // TODO groupby here?
                 long tsDivisor = msg.isMsResolution() ? 1 : 1000;
-                for (Entry<Set<Tag>, List<Downsample>> entry : allSeries.entrySet()) {
+                for (Entry<Set<Tag>, List<Aggregation>> entry : allSeries.entrySet()) {
                     result.add(convertToQueryResponse(query, entry.getKey(), entry.getValue(), tsDivisor));
                 }
             }
@@ -575,7 +623,7 @@ public class DataStoreImpl implements DataStore {
         return result;
     }
 
-    private QueryResponse convertToQueryResponse(SubQuery query, Set<Tag> tags, Collection<Downsample> values,
+    private QueryResponse convertToQueryResponse(SubQuery query, Set<Tag> tags, Collection<Aggregation> values,
             long tsDivisor) {
         QueryResponse response = new QueryResponse();
         response.setMetric(query.getMetric());
@@ -583,7 +631,7 @@ public class DataStoreImpl implements DataStore {
             response.putTag(tag.getKey(), tag.getValue());
         }
         RateOption rateOptions = query.getRateOptions();
-        Downsample combined = Downsample.combine(values, rateOptions);
+        Aggregation combined = Aggregation.combineAggregation(values, rateOptions);
         for (Sample entry : combined) {
             long ts = entry.timestamp / tsDivisor;
             response.putDps(Long.toString(ts), entry.value);
@@ -731,16 +779,29 @@ public class DataStoreImpl implements DataStore {
     }
 
     private Class<? extends Aggregator> getAggregator(SubQuery query) {
-        String aggregatorName = "avg";
+        return Aggregator.getAggregator(query.getAggregator());
+    }
+
+    private Class<? extends Aggregator> getDownsampleAggregator(SubQuery query) {
+        String aggregatorName = Aggregator.NONE;
         if (query.getDownsample().isPresent()) {
             String parts[] = query.getDownsample().get().split("-");
             aggregatorName = parts[1];
         }
+        // disabling the downsampling OR setting the aggregation to none are
+        // both considered to be disabling
+        if (aggregatorName.equals(Aggregator.NONE)) {
+            // we need a downsampling iterator, so default to max to ensure we
+            // return something
+            aggregatorName = DEFAULT_DOWNSAMPLE_AGGREGATOR;
+        }
         return Aggregator.getAggregator(aggregatorName);
     }
 
     private long getDownsamplePeriod(SubQuery query) {
-        if (!query.getDownsample().isPresent()) {
+        // disabling the downsampling OR setting the aggregation to none are
+        // both considered to be disabling
+        if (!query.getDownsample().isPresent() || query.getDownsample().get().endsWith("-none")) {
             return DEFAULT_DOWNSAMPLE_MS;
         }
         String parts[] = query.getDownsample().get().split("-");
@@ -775,16 +836,24 @@ public class DataStoreImpl implements DataStore {
         }
     }
 
+    private long getAgeOffForMetric(String metricName) {
+        String age = this.ageOff.get(MetricAgeOffIterator.AGE_OFF_PREFIX + metricName);
+        if (null == age) {
+            return this.defaultAgeOff;
+        } else {
+            return Long.parseLong(age);
+        }
+    }
+
     public Scanner createScannerForMetric(String sessionId, String metric, Map<String, String> tags, long startTime,
-            int lag) throws TimelyException {
+            long endTime, int lag, int scannerBatchSize, int scannerReadAhead) throws TimelyException {
         try {
             Authorizations auths = null;
             try {
                 auths = getSessionAuthorizations(sessionId);
             } catch (NullPointerException npe) {
                 // Session id being used for metric scanner, but session Id does
-                // not
-                // exist in Auth Cache. Use Empty auths if anonymous access
+                // not exist in Auth Cache. Use Empty auths if anonymous access
                 // allowed
                 if (anonAccessAllowed) {
                     auths = Authorizations.EMPTY;
@@ -792,13 +861,18 @@ public class DataStoreImpl implements DataStore {
                     throw npe;
                 }
             }
+            LOG.debug("Creating metric scanner for session: {} with auths: {}", sessionId, auths);
             Scanner s = connector.createScanner(this.metricsTable, auths);
             if (null == metric) {
                 throw new IllegalArgumentException("metric name must be specified");
             }
+            if (0 == startTime) {
+                startTime = (System.currentTimeMillis() - getAgeOffForMetric(metric) - 1000);
+                LOG.debug("Overriding zero start time to {} due to age off configuration", startTime);
+            }
             byte[] start = MetricAdapter.encodeRowKey(metric, startTime);
-            long endTime = (System.currentTimeMillis() - (lag * 1000));
-            byte[] end = MetricAdapter.encodeRowKey(metric, endTime);
+            long endTimeStamp = (endTime == 0) ? (System.currentTimeMillis() - (lag * 1000)) : endTime;
+            byte[] end = MetricAdapter.encodeRowKey(metric, endTimeStamp);
             s.setRange(new Range(new Text(start), true, new Text(end), false));
             SubQuery query = new SubQuery();
             query.setMetric(metric);
@@ -809,6 +883,8 @@ public class DataStoreImpl implements DataStore {
             List<String> tagOrder = prioritizeTags(query);
             Map<String, String> orderedTags = orderTags(tagOrder, query.getTags());
             setQueryColumns(s, metric, orderedTags);
+            s.setBatchSize(scannerBatchSize);
+            s.setReadaheadThreshold(scannerReadAhead);
             return s;
         } catch (IllegalArgumentException | TableNotFoundException ex) {
             LOG.error("Error during lookup: " + ex.getMessage(), ex);

server/src/main/java/timely/store/MetricAgeOffIterator.java

@@ -0,0 +1,200 @@
+package timely.store;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Map;
+
+import org.apache.accumulo.core.data.*;
+import org.apache.accumulo.core.iterators.IteratorEnvironment;
+import org.apache.accumulo.core.iterators.OptionDescriber;
+import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.accumulo.core.iterators.WrappingIterator;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.commons.collections4.trie.PatriciaTrie;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.Text;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import timely.adapter.accumulo.MetricAdapter;
+
+public class MetricAgeOffIterator extends WrappingIterator implements OptionDescriber {
+
+    private static final Logger LOG = LoggerFactory.getLogger(MetricAgeOffIterator.class);
+    public static final String AGE_OFF_PREFIX = "ageoff.";
+    public static final String DEFAULT_AGEOFF_KEY = "default";
+
+    /* set in init */
+    private PatriciaTrie<Long> ageoffs = null;
+    private Long currentTime = 0L;
+    private Long defaultAgeOff = 0L;
+    private Long minAgeOff = Long.MAX_VALUE;
+    private Long maxAgeOff = Long.MIN_VALUE;
+    /* changes at runtime */
+    private final BytesWritable prevMetricBytes = new BytesWritable();
+    private Long prevAgeOff = null;
+    private Range range;
+    private Collection<ByteSequence> columnFamilies;
+    private boolean inclusive;
+
+    @Override
+    public SortedKeyValueIterator<Key, Value> deepCopy(IteratorEnvironment env) {
+        MetricAgeOffIterator iter = new MetricAgeOffIterator();
+        iter.ageoffs = this.ageoffs;
+        iter.defaultAgeOff = this.defaultAgeOff;
+        iter.currentTime = this.currentTime;
+        iter.minAgeOff = this.minAgeOff;
+        iter.maxAgeOff = this.maxAgeOff;
+        return iter;
+    }
+
+    @Override
+    public void init(SortedKeyValueIterator<Key, Value> source, Map<String, String> options, IteratorEnvironment env)
+            throws IOException {
+        super.init(source, options, env);
+        validateOptions(options);
+        ageoffs = new PatriciaTrie<>();
+        options.forEach((k, v) -> {
+            if (k.startsWith(AGE_OFF_PREFIX)) {
+                String name = k.substring(AGE_OFF_PREFIX.length());
+                LOG.trace("Adding {} to Trie with value {}", name, Long.parseLong(v));
+                long ageoff = Long.parseLong(v);
+                this.minAgeOff = Math.min(this.minAgeOff, ageoff);
+                this.maxAgeOff = Math.max(this.maxAgeOff, ageoff);
+                ageoffs.put(name, ageoff);
+            }
+        });
+        defaultAgeOff = ageoffs.get(DEFAULT_AGEOFF_KEY);
+        currentTime = System.currentTimeMillis();
+    }
+
+    private void seekIfNecessary() {
+        if (super.hasTop()) {
+            Key top = super.getTopKey();
+            // If less than any configured ageoff, then we want to return this
+            // K,V
+            if ((this.currentTime - top.getTimestamp()) < this.minAgeOff) {
+                return;
+            }
+            // If greater than any configured ageoff, then drop it
+            if ((this.currentTime - top.getTimestamp()) > this.maxAgeOff) {
+                LOG.trace("Current key is older than max age off, seeking to start of valid data");
+                String metricName = MetricAdapter.decodeRowKey(top).getFirst();
+                handleNewMetricName(metricName);
+                seekPastAgedOffMetricData(metricName, this.maxAgeOff);
+                return;
+            }
+            if (isNextMetricTheSame(top.getRow())) {
+                // this metric name is the same as previous
+                if ((currentTime - top.getTimestamp()) > prevAgeOff) {
+                    // We are in the same metric, seek to data we want to
+                    // process
+                    String metricName = new String(prevMetricBytes.copyBytes(), UTF_8);
+                    LOG.trace("Current metric is older than age off for metric {}, seeking to start of valid data",
+                            metricName);
+                    seekPastAgedOffMetricData(metricName, prevAgeOff);
+                    return;
+                }
+            } else {
+                // Metric name is different or prev information is not set
+                String metricName = MetricAdapter.decodeRowKey(top).getFirst();
+                handleNewMetricName(metricName);
+                if (currentTime - top.getTimestamp() > prevAgeOff) {
+                    LOG.trace("New metric found, but older than age off for metric {}, seeking to start of valid data",
+                            metricName);
+                    seekPastAgedOffMetricData(metricName, prevAgeOff);
+                }
+            }
+        }
+    }
+
+    private boolean isNextMetricTheSame(Text nextRow) {
+        byte[] next = nextRow.getBytes();
+        if (next.length > prevMetricBytes.getLength()
+                && 0 == prevMetricBytes.compareTo(next, 0, prevMetricBytes.getLength())
+                && nextRow.charAt(prevMetricBytes.getLength()) == 0x00) {
+            return true;
+        } else {
+            return false;
+        }
+    }
+
+    private void handleNewMetricName(String metricName) {
+        byte[] b = metricName.getBytes(UTF_8);
+        prevMetricBytes.set(b, 0, b.length);
+        prevAgeOff = ageoffs.get(metricName);
+        if (null == prevAgeOff) {
+            // no specific ageoff for this metric name, use default
+            prevAgeOff = defaultAgeOff;
+        }
+    }
+
+    private void seekPastAgedOffMetricData(String metricName, long ageOffTime) {
+        long timeTarget = currentTime - ageOffTime;
+        Range newRange;
+        if (null == this.range) {
+            byte[] newStartRow = MetricAdapter.encodeRowKey(metricName, timeTarget);
+            newRange = new Range(new Key(new Text(newStartRow), timeTarget), null);
+        } else {
+            Key startKey = new Key(new Text(MetricAdapter.encodeRowKey(metricName, timeTarget)), timeTarget);
+            Pair<String, Long> start = MetricAdapter.decodeRowKey(startKey);
+            byte[] newStartRow = MetricAdapter.encodeRowKey(start.getFirst(), timeTarget);
+            // @formatter:off
+            Key newStartKey = new Key(newStartRow, 
+            		null != startKey.getColumnFamily() ? startKey.getColumnFamilyData().getBackingArray() : null, 
+            		null != startKey.getColumnQualifier() ? startKey.getColumnQualifierData().getBackingArray() : null,
+                    null != startKey.getColumnVisibility() ? startKey.getColumnVisibility().getBytes() : null,
+                    timeTarget, startKey.isDeleted());
+            // @formatter:on
+            if (this.range.getEndKey() == null || this.range.getEndKey().compareTo(newStartKey) >= 0) {
+                newRange = new Range(newStartKey, true, this.range.getEndKey(), this.range.isEndKeyInclusive());
+            } else {
+                newRange = new Range(this.range.getEndKey(), false, this.range.getEndKey().followingKey(
+                        PartialKey.ROW_COLFAM_COLQUAL_COLVIS_TIME), false);
+            }
+        }
+        try {
+            LOG.trace("Seeking to: {}", newRange);
+            this.seek(newRange, columnFamilies, inclusive);
+        } catch (IOException e) {
+            LOG.error("Error seeking to new range: " + newRange, e);
+        }
+
+    }
+
+    @Override
+    public void next() throws IOException {
+        super.next();
+        seekIfNecessary();
+    }
+
+    @Override
+    public void seek(Range range, Collection<ByteSequence> columnFamilies, boolean inclusive) throws IOException {
+        this.range = range;
+        this.columnFamilies = columnFamilies;
+        this.inclusive = inclusive;
+        super.seek(range, columnFamilies, inclusive);
+        seekIfNecessary();
+    }
+
+    @Override
+    public IteratorOptions describeOptions() {
+        return new IteratorOptions("metric-age-off", "Iterator that ages off time series data for Timely metrics",
+                Collections.singletonMap(AGE_OFF_PREFIX + DEFAULT_AGEOFF_KEY, "default age off days"),
+                Collections.singletonList("Additional metric age off properties where the metric is specified as "
+                        + AGE_OFF_PREFIX
+                        + " the metric name and the value is an integer representing the number of days to keep"));
+    }
+
+    @Override
+    public boolean validateOptions(Map<String, String> options) {
+        if (null == options.get(AGE_OFF_PREFIX + DEFAULT_AGEOFF_KEY)) {
+            throw new IllegalArgumentException(DEFAULT_AGEOFF_KEY + " must be configured for MetricAgeOffFilter");
+        }
+        return true;
+    }
+
+}

server/src/main/java/timely/store/iterators/RateIterator.java

@@ -6,6 +6,7 @@ import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 
+import org.apache.accumulo.core.client.IteratorSetting;
 import org.apache.accumulo.core.data.Key;
 import org.apache.accumulo.core.data.Value;
 import org.apache.accumulo.core.iterators.IteratorEnvironment;
@@ -13,14 +14,35 @@ import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
 import org.apache.accumulo.core.util.Pair;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import timely.api.request.timeseries.QueryRequest;
 
 public class RateIterator extends TimeSeriesGroupingIterator {
 
     public static final Logger LOG = LoggerFactory.getLogger(RateIterator.class);
+    private static final String COUNTER_MAX = "rate.counter.max";
+    private static final String RATE_RESET_VALUE = "rate.reset.value";
+
+    private boolean isCounter = false;
+    private long maxCounter = 0;
+    private long resetValue = 0;
+
+    public static void setRateOptions(IteratorSetting is, QueryRequest.RateOption options) {
+        if (options != null && options.isCounter()) {
+            LOG.trace("Setting rate counter options cm:{}, rv:{}", options.getCounterMax(), options.getResetValue());
+            is.addOption(COUNTER_MAX, Long.toString(options.getCounterMax()));
+            is.addOption(RATE_RESET_VALUE, Long.toString(options.getResetValue()));
+        }
+    }
 
     @Override
     public void init(SortedKeyValueIterator<Key, Value> source, Map<String, String> options, IteratorEnvironment env)
             throws IOException {
+        if (options.containsKey(COUNTER_MAX)) {
+            this.isCounter = true;
+            this.maxCounter = Long.parseLong(options.get(COUNTER_MAX));
+            this.resetValue = Long.parseLong(options.get(RATE_RESET_VALUE));
+            LOG.trace("Setting rate counter options cm:{}, rv:{}", this.maxCounter, this.resetValue);
+        }
         Map<String, String> opts = new HashMap<>(options);
         opts.put(TimeSeriesGroupingIterator.FILTER, "-1,1");
         super.init(source, opts, env);
@@ -32,7 +54,7 @@ public class RateIterator extends TimeSeriesGroupingIterator {
         Iterator<Pair<Key, Double>> iter = values.iterator();
         Pair<Key, Double> first = iter.next();
         Long firstTs = first.getFirst().getTimestamp();
-        Double firstVal = first.getSecond() * -1;
+        Double firstVal = first.getSecond();
         LOG.trace("first ts:{}, value:{}", firstTs, firstVal);
 
         Pair<Key, Double> second = iter.next();
@@ -40,12 +62,28 @@ public class RateIterator extends TimeSeriesGroupingIterator {
         Double secondVal = second.getSecond();
         LOG.trace("second ts:{}, value:{}", secondTs, secondVal);
 
+        if (isCounter && (secondVal < firstVal)) {
+            if (maxCounter > 0) {
+                secondVal += maxCounter;
+                LOG.trace("second counter reset based on max:{} to ts:{}, value:{}", maxCounter, secondTs, secondVal);
+            } else {
+                secondVal += firstVal;
+                LOG.trace("second counter reset based on first:{} to ts:{}, value:{}", firstVal, secondTs, secondVal);
+            }
+        }
+
         long timeDiff = secondTs - firstTs;
         if (timeDiff == 0) {
             return 0.0D;
         }
-        Double result = ((firstVal + secondVal) / timeDiff);
+        Double result = ((secondVal - firstVal) / timeDiff);
         LOG.trace("compute - result: {}", result);
+
+        if (isCounter && resetValue > 0 && result > resetValue) {
+            result = 0.0D;
+            LOG.trace("compute - reset result base on {} to {}", resetValue, result);
+        }
+
         return result;
     }
 

server/src/main/java/timely/subscription/MetricScanner.java

@@ -3,6 +3,7 @@ package timely.subscription;
 import static com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly;
 import io.netty.channel.ChannelHandlerContext;
 import io.netty.handler.codec.http.websocketx.TextWebSocketFrame;
+import io.netty.util.concurrent.ScheduledFuture;
 
 import java.lang.Thread.UncaughtExceptionHandler;
 import java.util.Iterator;
@@ -21,8 +22,9 @@ import org.slf4j.LoggerFactory;
 
 import timely.adapter.accumulo.MetricAdapter;
 import timely.api.response.MetricResponse;
-import timely.model.Metric;
+import timely.api.response.MetricResponses;
 import timely.api.response.TimelyException;
+import timely.model.Metric;
 import timely.store.DataStore;
 import timely.util.JsonUtil;
 
@@ -32,6 +34,8 @@ import com.fasterxml.jackson.databind.ObjectMapper;
 public class MetricScanner extends Thread implements UncaughtExceptionHandler {
 
     private static final Logger LOG = LoggerFactory.getLogger(MetricScanner.class);
+    private static final ObjectMapper om = JsonUtil.getObjectMapper();
+
     private final Scanner scanner;
     private Iterator<Entry<Key, Value>> iter = null;
     private final ChannelHandlerContext ctx;
@@ -41,78 +45,115 @@ public class MetricScanner extends Thread implements UncaughtExceptionHandler {
     private final int lag;
     private final String subscriptionId;
     private final String metric;
+    private final long endTime;
+    private final Subscription subscription;
+    private MetricResponses responses = new MetricResponses();
+    private final ScheduledFuture<?> flusher;
+    private final int subscriptionBatchSize;
 
-    public MetricScanner(String subscriptionId, String sessionId, DataStore store, String metric,
-            Map<String, String> tags, long startTime, long delay, int lag, ChannelHandlerContext ctx)
+    public MetricScanner(Subscription sub, String subscriptionId, String sessionId, DataStore store, String metric,
+            Map<String, String> tags, long startTime, long endTime, long delay, int lag, ChannelHandlerContext ctx,
+            int scannerBatchSize, int flushIntervalSeconds, int scannerReadAhead, int subscriptionBatchSize)
             throws TimelyException {
         this.setDaemon(true);
         this.setUncaughtExceptionHandler(this);
+        this.subscription = sub;
         this.ctx = ctx;
         this.lag = lag;
         this.metric = metric;
-        this.scanner = store.createScannerForMetric(sessionId, metric, tags, startTime, lag);
+        this.endTime = endTime;
+        this.scanner = store.createScannerForMetric(sessionId, metric, tags, startTime, endTime, lag, scannerBatchSize,
+                scannerReadAhead);
         this.iter = scanner.iterator();
         this.delay = delay;
         this.subscriptionId = subscriptionId;
+        this.subscriptionBatchSize = subscriptionBatchSize;
         ToStringBuilder buf = new ToStringBuilder(this);
         buf.append("sessionId", sessionId);
         buf.append("metric", metric);
         buf.append("startTime", startTime);
+        buf.append("endTime", endTime);
         buf.append("delayTime", delay);
         if (null != tags) {
             buf.append("tags", tags.toString());
         }
         name = buf.toString();
-        LOG.trace("Created MetricScanner: {}", name);
         this.setName("Metric Scanner " + name);
+        this.flusher = this.ctx.executor().scheduleAtFixedRate(() -> {
+            flush();
+        }, flushIntervalSeconds, flushIntervalSeconds, TimeUnit.SECONDS);
+        LOG.trace("Created MetricScanner: {}", name);
+    }
+
+    public synchronized void flush() {
+        if (responses.size() > 0) {
+            try {
+                String json = om.writeValueAsString(responses);
+                this.ctx.writeAndFlush(new TextWebSocketFrame(json));
+                responses.clear();
+            } catch (JsonProcessingException e) {
+                LOG.error("Error serializing metrics: " + responses, e);
+            }
+        }
     }
 
     @Override
     public void run() {
         Metric m = null;
         try {
-            ObjectMapper om = JsonUtil.getObjectMapper();
             while (!closed) {
 
                 if (this.iter.hasNext()) {
                     Entry<Key, Value> e = this.iter.next();
-                    m = MetricAdapter.parse(e.getKey(), e.getValue());
-                    try {
-                        String json = om.writeValueAsString(MetricResponse.fromMetric(m, this.subscriptionId));
-                        LOG.trace("Returning {} for subscription", json);
-                        this.ctx.writeAndFlush(new TextWebSocketFrame(json));
-                    } catch (JsonProcessingException e1) {
-                        LOG.error("Error serializing metric: " + m, e1);
+                    m = MetricAdapter.parse(e.getKey(), e.getValue(), true);
+                    if (responses.size() >= this.subscriptionBatchSize) {
+                        flush();
                     }
-                } else {
-                    long endTime = (System.currentTimeMillis() - (lag * 1000));
-                    byte[] end = MetricAdapter.encodeRowKey(this.metric, endTime);
+                    this.responses.addResponse(MetricResponse.fromMetric(m, this.subscriptionId));
+                } else if (this.endTime == 0) {
+                    flush();
+                    long endTimeStamp = (System.currentTimeMillis() - (lag * 1000));
+                    byte[] end = MetricAdapter.encodeRowKey(this.metric, endTimeStamp);
                     Text endRow = new Text(end);
                     this.scanner.close();
                     Range prevRange = this.scanner.getRange();
                     if (null == m) {
-                        LOG.debug("No results found, waiting {}ms to retry.", delay);
+                        LOG.debug("No results found, waiting {}ms to retry with new end time {}.", delay, endTimeStamp);
                         sleepUninterruptibly(delay, TimeUnit.MILLISECONDS);
                         this.scanner.setRange(new Range(prevRange.getStartKey().getRow(), prevRange
                                 .isStartKeyInclusive(), endRow, false));
                         this.iter = this.scanner.iterator();
                     } else {
                         // Reset the starting range to the last key returned
-                        LOG.debug("Exhausted scanner, waiting {}ms to retry.", delay);
+                        LOG.debug("Exhausted scanner, last metric returned was {}", m);
+                        LOG.debug("Waiting {}ms to retry with new end time {}.", delay, endTimeStamp);
                         sleepUninterruptibly(delay, TimeUnit.MILLISECONDS);
                         this.scanner.setRange(new Range(new Text(MetricAdapter.encodeRowKey(m)), false, endRow,
                                 prevRange.isEndKeyInclusive()));
                         this.iter = this.scanner.iterator();
                     }
+                } else {
+                    LOG.debug("Exhausted scanner, sending completed message for subscription {}", this.subscriptionId);
+                    final MetricResponse last = new MetricResponse();
+                    last.setSubscriptionId(this.subscriptionId);
+                    last.setComplete(true);
+                    this.responses.addResponse(last);
+                    flush();
+                    break;
                 }
             }
+        } catch (Exception e) {
+            LOG.error("Error in metric scanner, closing.", e);
         } finally {
+            close();
             this.scanner.close();
+            subscription.scannerComplete(metric);
         }
     }
 
     public void close() {
         LOG.info("Marking metric scanner closed: {}", name);
+        this.flusher.cancel(false);
         this.closed = true;
     }
 

server/src/main/java/timely/subscription/Subscription.java

@@ -4,6 +4,7 @@ import io.netty.channel.ChannelHandlerContext;
 import io.netty.handler.codec.http.websocketx.PingWebSocketFrame;
 import io.netty.util.concurrent.ScheduledFuture;
 
+import java.util.LinkedList;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.TimeUnit;
@@ -24,8 +25,13 @@ public class Subscription {
     private final DataStore store;
     private final ChannelHandlerContext ctx;
     private final ScheduledFuture<?> ping;
-    private final Integer lag;
+    private final int lag;
+    private final int scannerBatchSize;
+    private final int flushIntervalSeconds;
+    private final int scannerReadAhead;
+    private final int subscriptionBatchSize;
     private final String subscriptionId;
+    private LinkedList<String> deadMetricScannerQueue = new LinkedList<>();
 
     public Subscription(String subscriptionId, String sessionId, DataStore store, ChannelHandlerContext ctx,
             Configuration conf) {
@@ -34,22 +40,41 @@ public class Subscription {
         this.store = store;
         this.ctx = ctx;
         this.lag = conf.getWebsocket().getSubscriptionLag();
+        this.scannerBatchSize = conf.getWebsocket().getScannerBatchSize();
+        this.flushIntervalSeconds = conf.getWebsocket().getFlushIntervalSeconds();
+        this.scannerReadAhead = conf.getWebsocket().getScannerReadAhead();
+        this.subscriptionBatchSize = conf.getWebsocket().getSubscriptionBatchSize();
         // send a websocket ping at half the timeout interval.
         int rate = conf.getWebsocket().getTimeout() / 2;
         this.ping = this.ctx.executor().scheduleAtFixedRate(() -> {
             LOG.trace("Sending ping on channel {}", ctx.channel());
             ctx.writeAndFlush(new PingWebSocketFrame());
+            cleanupCompletedMetrics();
         }, rate, rate, TimeUnit.SECONDS);
     }
 
-    public void addMetric(String metric, Map<String, String> tags, long startTime, long delay) throws TimelyException {
+    public void addMetric(String metric, Map<String, String> tags, long startTime, long endTime, long delay)
+            throws TimelyException {
         LOG.debug("Adding metric scanner for subscription {}", this.subscriptionId);
-        MetricScanner m = new MetricScanner(this.subscriptionId, this.sessionId, store, metric, tags, startTime, delay,
-                lag, ctx);
+        MetricScanner m = new MetricScanner(this, this.subscriptionId, this.sessionId, store, metric, tags, startTime,
+                endTime, delay, lag, ctx, scannerBatchSize, flushIntervalSeconds, scannerReadAhead,
+                subscriptionBatchSize);
         METRICS.put(metric, m);
         m.start();
     }
 
+    void scannerComplete(String metric) {
+        deadMetricScannerQueue.add(metric);
+    }
+
+    private void cleanupCompletedMetrics() {
+        String metric = this.deadMetricScannerQueue.poll();
+        while (null != metric) {
+            this.removeMetric(metric);
+            metric = this.deadMetricScannerQueue.poll();
+        }
+    }
+
     public void removeMetric(String metric) {
         MetricScanner m = METRICS.remove(metric);
         if (null != m) {

server/src/test/java/timely/api/model/MetricAdapterTest.java

@@ -8,7 +8,9 @@ import org.apache.accumulo.core.client.lexicoder.LongLexicoder;
 import org.apache.accumulo.core.client.lexicoder.PairLexicoder;
 import org.apache.accumulo.core.client.lexicoder.StringLexicoder;
 import org.apache.accumulo.core.data.ColumnUpdate;
+import org.apache.accumulo.core.data.Key;
 import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
 import org.apache.accumulo.core.util.ComparablePair;
 import org.junit.Assert;
 import org.junit.Before;
@@ -98,7 +100,49 @@ public class MetricAdapterTest {
         Metric m = Metric.newBuilder().name("sys.cpu.user").value(ts, 2.0D).tags(tags)
                 .tag(MetricAdapter.VISIBILITY_TAG, "(a&b)|(c&d)").build();
         String json = JsonUtil.getObjectMapper().writeValueAsString(MetricResponse.fromMetric(m, subscriptionId));
-        String expected = "{\"metric\":\"sys.cpu.user\",\"timestamp\":1000,\"value\":2.0,\"tags\":[{\"tag1\":\"value1\"},{\"viz\":\"(a&b)|(c&d)\"}],\"subscriptionId\":\"12345\"}";
+        String expected = "{\"metric\":\"sys.cpu.user\",\"timestamp\":1000,\"value\":2.0,\"tags\":[{\"tag1\":\"value1\"},{\"viz\":\"(a&b)|(c&d)\"}],\"subscriptionId\":\"12345\",\"complete\":false}";
         Assert.assertEquals(expected, json);
     }
+
+    @Test
+    public void testParse() throws Exception {
+        PairLexicoder<String, Long> rowCoder = new PairLexicoder<>(new StringLexicoder(), new LongLexicoder());
+        byte[] row = rowCoder.encode(new ComparablePair<>("sys.cpu.user", 1000L));
+        byte[] value = new byte[Double.BYTES];
+        ByteBuffer.wrap(value).putDouble(2.0D);
+        Key k = new Key(row, "tag1=value1".getBytes(), "tag2=value2,tag3=value3".getBytes(), "(a&b)|(c&d)".getBytes(),
+                1000);
+        Value v = new Value(value);
+        Metric m = MetricAdapter.parse(k, v);
+        Assert.assertEquals("sys.cpu.user", m.getName());
+        List<Tag> tags = new ArrayList<>();
+        tags.add(new Tag("tag1=value1"));
+        tags.add(new Tag("tag2=value2"));
+        tags.add(new Tag("tag3=value3"));
+        Assert.assertEquals(tags, m.getTags());
+        Assert.assertEquals(new Long(1000), m.getValue().getTimestamp());
+        Assert.assertEquals(2.0D, m.getValue().getMeasure(), 0.0D);
+    }
+
+    @Test
+    public void testParseWithViz() throws Exception {
+        PairLexicoder<String, Long> rowCoder = new PairLexicoder<>(new StringLexicoder(), new LongLexicoder());
+        byte[] row = rowCoder.encode(new ComparablePair<>("sys.cpu.user", 1000L));
+        byte[] value = new byte[Double.BYTES];
+        ByteBuffer.wrap(value).putDouble(2.0D);
+        Key k = new Key(row, "tag1=value1".getBytes(), "tag2=value2,tag3=value3".getBytes(), "(a&b)|(c&d)".getBytes(),
+                1000);
+        Value v = new Value(value);
+        Metric m = MetricAdapter.parse(k, v, true);
+        Assert.assertEquals("sys.cpu.user", m.getName());
+        List<Tag> tags = new ArrayList<>();
+        tags.add(new Tag("tag1=value1"));
+        tags.add(new Tag("tag2=value2"));
+        tags.add(new Tag("tag3=value3"));
+        tags.add(new Tag("viz=(a&b)|(c&d)"));
+        Assert.assertEquals(tags, m.getTags());
+        Assert.assertEquals(new Long(1000), m.getValue().getTimestamp());
+        Assert.assertEquals(2.0D, m.getValue().getMeasure(), 0.0D);
+    }
+
 }

server/src/test/java/timely/api/request/subscription/WebSocketRequestDeserializationTest.java

@@ -108,7 +108,8 @@ public class WebSocketRequestDeserializationTest {
 					       "\"tag2\" : \"value2\"," +
 					       "\"tag1\" : \"value1\"" +
 					    "}," +
-						"\"startTime\" : \"1000\"" +
+						"\"startTime\" : \"1000\"," +
+					    "\"endTime\" : \"2000\""+
 					  "}";
 		// @formatter:on
         WebSocketRequest request = JsonUtil.getObjectMapper().readValue(json.getBytes(), WebSocketRequest.class);
@@ -123,8 +124,11 @@ public class WebSocketRequestDeserializationTest {
         Assert.assertTrue(tags.containsKey("tag2"));
         Assert.assertEquals("value2", tags.get("tag2"));
         Assert.assertEquals(true, ((AddSubscription) request).getStartTime().isPresent());
-        long time = ((AddSubscription) request).getStartTime().get();
-        Assert.assertEquals(1000L, time);
+        long start = ((AddSubscription) request).getStartTime().get();
+        Assert.assertEquals(1000L, start);
+        Assert.assertEquals(true, ((AddSubscription) request).getEndTime().isPresent());
+        long end = ((AddSubscription) request).getEndTime().get();
+        Assert.assertEquals(2000L, end);
     }
 
     @Test

server/src/test/java/timely/sample/AggregationTest.java

@@ -0,0 +1,44 @@
+package timely.sample;
+
+import org.junit.Test;
+import timely.sample.aggregators.Avg;
+import timely.sample.aggregators.Sum;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class AggregationTest {
+
+    @Test
+    public void simple() {
+        Aggregation asample = new Aggregation(new Avg());
+        for (int i = 10; i < 30; i++) {
+            asample.add(i, i - 10);
+        }
+        for (int i = 10; i < 30; i++) {
+            asample.add(i, i);
+        }
+        int i = 0;
+        for (Sample sample : asample) {
+            assertEquals(10 + i, sample.timestamp);
+            assertTrue(sample.timestamp < 30);
+            assertEquals(i + 5, (int) sample.value);
+            i++;
+        }
+        assertEquals(20, i);
+        asample = new Aggregation(new Sum());
+        for (int j = 0; j < 5; j++) {
+            for (int k = 10; k < 100; k++) {
+                asample.add(k, j + 0.);
+            }
+        }
+        i = 0;
+        for (Sample sample : asample) {
+            assertEquals(10 + i, sample.timestamp);
+            assertEquals((1 + 2 + 3 + 4), sample.value, 0.0D);
+            i++;
+        }
+        assertEquals(100 - 10, i);
+    }
+
+}

server/src/test/java/timely/sample/DownsampleTest.java

@@ -69,7 +69,7 @@ public class DownsampleTest {
         for (int i = 0; i < 1000; i += 100) {
             ds.add(i, .2);
         }
-        Downsample result = Downsample.combine(Collections.singleton(ds), null);
+        Downsample result = Downsample.combineDownsample(Collections.singleton(ds), null);
         int count = 0;
         for (Sample s : result) {
             assertEquals(.2, s.value, 0.0D);
@@ -86,7 +86,7 @@ public class DownsampleTest {
                 ds.add(i, .2);
             }
         }
-        Downsample result = Downsample.combine(Collections.singleton(ds), null);
+        Downsample result = Downsample.combineDownsample(Collections.singleton(ds), null);
         int count = 0;
         for (Sample s : result) {
             assertEquals(.2, s.value, 0.0D);

server/src/test/java/timely/sample/iterators/AggregationIteratorTest.java

@@ -0,0 +1,158 @@
+package timely.sample.iterators;
+
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.data.*;
+import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.accumulo.core.iterators.SortedMapIterator;
+import org.junit.Before;
+import org.junit.Test;
+import timely.Configuration;
+import timely.adapter.accumulo.MetricAdapter;
+import timely.auth.VisibilityCache;
+import timely.model.Metric;
+import timely.model.Tag;
+import timely.sample.Aggregation;
+import timely.sample.Downsample;
+import timely.sample.Sample;
+import timely.sample.aggregators.Avg;
+
+import java.io.ByteArrayOutputStream;
+import java.io.ObjectOutputStream;
+import java.util.*;
+import java.util.Map.Entry;
+
+import static org.junit.Assert.*;
+
+public class AggregationIteratorTest {
+
+    final private SortedMap<Key, Value> testData1 = new TreeMap<>();
+    final private SortedMap<Key, Value> testData2 = new TreeMap<>();
+
+    @Before
+    public void before() {
+        VisibilityCache.init(new Configuration());
+    }
+
+    @Before
+    public void createTestData() throws Exception {
+        createTestData1();
+        createTestData2();
+    }
+
+    private void createTestData1() throws Exception {
+        List<Tag> tags = Collections.singletonList(new Tag("host", "host1"));
+
+        Downsample sample = new Downsample(0L, 10000L, 1L, new Avg());
+        Metric m = null;
+        for (long i = 0; i < 1000; i += 100) {
+            m = new Metric("sys.loadAvg", i, .2, tags);
+            sample.add(i, .2);
+        }
+
+        put(testData1, m, sample);
+    }
+
+    /**
+     * This will add key, values to the test data as output from the
+     * DownsampleIterator
+     * 
+     * @param testData
+     * @param m
+     * @param sample
+     * @throws Exception
+     */
+    void put(Map<Key, Value> testData, Metric m, Downsample sample) throws Exception {
+        Mutation mutation = MetricAdapter.toMutation(m);
+        for (ColumnUpdate cu : mutation.getUpdates()) {
+            Key key = new Key(mutation.getRow(), cu.getColumnFamily(), cu.getColumnQualifier(),
+                    cu.getColumnVisibility(), cu.getTimestamp());
+            Map<Set<Tag>, Downsample> samples = new HashMap<>();
+            samples.put(new HashSet<Tag>(m.getTags()), sample);
+            ByteArrayOutputStream bos = new ByteArrayOutputStream();
+            ObjectOutputStream out = new ObjectOutputStream(bos);
+            out.writeObject(samples);
+            out.flush();
+            testData.put(key, new Value(bos.toByteArray()));
+        }
+    }
+
+    private void createTestData2() throws Exception {
+        List<Tag> tags = Collections.singletonList(new Tag("host", "host1"));
+        List<Tag> tags2 = Collections.singletonList(new Tag("host", "host2"));
+
+        Downsample sample = new Downsample(0L, 10000L, 1L, new Avg());
+        Metric m = null;
+        for (long i = 0; i < 1000; i += 100) {
+            m = new Metric("sys.loadAvg", i, .2, tags);
+            sample.add(i, .2);
+        }
+        put(testData2, m, sample);
+
+        sample = new Downsample(0L, 10000L, 1L, new Avg());
+        m = null;
+        for (long i = 0; i < 1000; i += 100) {
+            m = new Metric("sys.loadAvg", i + 100, .5, tags2);
+            sample.add(i + 100, .5);
+        }
+        put(testData2, m, sample);
+    }
+
+    @Test
+    public void simpleGetOneSample() throws Exception {
+        // check that data gets pulled out
+        AggregationIterator iter = new AggregationIterator();
+        Map<Set<Tag>, Aggregation> samples = runQuery(iter, testData1, 100);
+        assertEquals(1, samples.size());
+        for (Entry<Set<Tag>, Aggregation> entry : samples.entrySet()) {
+            Set<Tag> tags = entry.getKey();
+            assertEquals(1, tags.size());
+            assertEquals(Collections.singleton(new Tag("host", ".*")), tags);
+            long ts = 0;
+            int count = 0;
+            for (Sample sample : entry.getValue()) {
+                assertEquals(ts, sample.timestamp);
+                ts += 100;
+                assertEquals(0.2, sample.value, 0.0001);
+                count++;
+            }
+            assertEquals(1000, ts);
+            assertEquals(10, count);
+        }
+    }
+
+    @Test
+    public void simpleAggregatedSample() throws Exception {
+        AggregationIterator iter = new AggregationIterator();
+        Map<Set<Tag>, Aggregation> samples = runQuery(iter, testData2, 100);
+        assertEquals(1, samples.size());
+        for (Entry<Set<Tag>, Aggregation> entry : samples.entrySet()) {
+            Set<Tag> tags = entry.getKey();
+            assertEquals(1, tags.size());
+            assertEquals(Collections.singleton(new Tag("host", ".*")), tags);
+            long ts = 0;
+            int count = 0;
+            for (Sample sample : entry.getValue()) {
+                assertEquals(ts, sample.timestamp);
+                ts += 100;
+                assertEquals(count == 0 ? 0.2 : (count == 10 ? 0.5 : 0.35), sample.value, 0.0001);
+                count++;
+            }
+            assertEquals(11, count);
+        }
+    }
+
+    private Map<Set<Tag>, Aggregation> runQuery(SortedKeyValueIterator<Key, Value> iter,
+            SortedMap<Key, Value> testData, long period) throws Exception {
+        IteratorSetting is = new IteratorSetting(100, AggregationIterator.class);
+        AggregationIterator.setAggregationOptions(is, Collections.singletonMap("host", ".*"), Avg.class.getName());
+        SortedKeyValueIterator<Key, Value> source = new SortedMapIterator(testData);
+        iter.init(source, is.getOptions(), null);
+        iter.seek(new Range(), Collections.emptyList(), true);
+        assertTrue(iter.hasTop());
+        Key key = iter.getTopKey();
+        assertEquals(testData.lastKey(), key);
+        Map<Set<Tag>, Aggregation> samples = AggregationIterator.decodeValue(iter.getTopValue());
+        return samples;
+    }
+
+}

server/src/test/java/timely/sample/iterators/DownsampleIteratorTest.java

@@ -48,7 +48,7 @@ public class DownsampleIteratorTest {
         createTestData2();
     }
 
-    private void createTestData2() {
+    private void createTestData1() {
         List<Tag> tags = Collections.singletonList(new Tag("host", "host1"));
 
         for (long i = 0; i < 1000; i += 100) {
@@ -71,7 +71,7 @@ public class DownsampleIteratorTest {
         }
     }
 
-    private void createTestData1() {
+    private void createTestData2() {
         List<Tag> tags = Collections.singletonList(new Tag("host", "host1"));
         List<Tag> tags2 = Collections.singletonList(new Tag("host", "host2"));
 

server/src/test/java/timely/store/MetricAgeOffIteratorTest.java

@@ -0,0 +1,194 @@
+package timely.store;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.SortedMap;
+import java.util.TreeMap;
+
+import org.apache.accumulo.core.data.ByteSequence;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.accumulo.core.iterators.SortedMapIterator;
+import org.junit.Assert;
+import org.junit.Test;
+
+import timely.adapter.accumulo.MetricAdapter;
+
+public class MetricAgeOffIteratorTest {
+
+    private static final Long TEST_TIME = System.currentTimeMillis() - 1000;
+    private static final Integer ONE_DAY = 86400000;
+    private static final Value EMPTY_VALUE = new Value(new byte[0]);
+    private static final Collection<ByteSequence> columnFamilies = new ArrayList<>();
+
+    @Test(expected = IllegalArgumentException.class)
+    public void testDefaultMissing() throws Exception {
+        SortedMap<Key, Value> table = new TreeMap<>();
+        SortedKeyValueIterator<Key, Value> source = new SortedMapIterator(table);
+        MetricAgeOffIterator iter = new MetricAgeOffIterator();
+        HashMap<String, String> options = new HashMap<>();
+        iter.init(source, options, null);
+    }
+
+    @Test
+    public void testDefault() throws Exception {
+        SortedMap<Key, Value> table = new TreeMap<>();
+        table.put(new Key(MetricAdapter.encodeRowKey("sys.cpu.user", TEST_TIME), new byte[0], new byte[0], new byte[0],
+                TEST_TIME), EMPTY_VALUE);
+        table.put(new Key(MetricAdapter.encodeRowKey("sys.cpu.user", TEST_TIME + 1), new byte[0], new byte[0],
+                new byte[0], TEST_TIME + 1), EMPTY_VALUE);
+        table.put(new Key(MetricAdapter.encodeRowKey("sys.cpu.user", TEST_TIME + 2), new byte[0], new byte[0],
+                new byte[0], TEST_TIME + 2), EMPTY_VALUE);
+        table.put(new Key(MetricAdapter.encodeRowKey("sys.cpu.user", TEST_TIME + 3), new byte[0], new byte[0],
+                new byte[0], TEST_TIME + 3), EMPTY_VALUE);
+        table.put(new Key(MetricAdapter.encodeRowKey("sys.cpu.user", TEST_TIME + 4), new byte[0], new byte[0],
+                new byte[0], TEST_TIME + 4), EMPTY_VALUE);
+        table.put(new Key(MetricAdapter.encodeRowKey("sys.cpu.user", TEST_TIME + 5), new byte[0], new byte[0],
+                new byte[0], TEST_TIME + 5), EMPTY_VALUE);
+
+        SortedKeyValueIterator<Key, Value> source = new SortedMapIterator(table);
+        MetricAgeOffIterator iter = new MetricAgeOffIterator();
+        HashMap<String, String> options = new HashMap<>();
+        options.put(MetricAgeOffIterator.AGE_OFF_PREFIX + "default", Integer.toString(1 * ONE_DAY));
+        iter.init(source, options, null);
+        iter.seek(new Range(), columnFamilies, true);
+        int seen = 0;
+        while (iter.hasTop()) {
+            Key k = iter.getTopKey();
+            Assert.assertTrue(k.getTimestamp() >= TEST_TIME && k.getTimestamp() <= TEST_TIME + 5);
+            seen++;
+            iter.next();
+        }
+        Assert.assertEquals(6, seen);
+    }
+
+    @Test
+    public void testMixed() throws Exception {
+        SortedMap<Key, Value> table = new TreeMap<>();
+        table.put(new Key(MetricAdapter.encodeRowKey("sys.cpu.idle", TEST_TIME), new byte[0], new byte[0], new byte[0],
+                TEST_TIME), EMPTY_VALUE);
+        table.put(new Key(MetricAdapter.encodeRowKey("sys.cpu.idle", TEST_TIME + 1), new byte[0], new byte[0],
+                new byte[0], TEST_TIME + 1), EMPTY_VALUE);
+        table.put(new Key(MetricAdapter.encodeRowKey("sys.cpu.idle", TEST_TIME + 2), new byte[0], new byte[0],
+                new byte[0], TEST_TIME + 2), EMPTY_VALUE);
+        table.put(new Key(MetricAdapter.encodeRowKey("sys.cpu.idle", TEST_TIME + 3), new byte[0], new byte[0],
+                new byte[0], TEST_TIME + 3), EMPTY_VALUE);
+        table.put(new Key(MetricAdapter.encodeRowKey("sys.cpu.idle", TEST_TIME + 4), new byte[0], new byte[0],
+                new byte[0], TEST_TIME + 4), EMPTY_VALUE);
+        table.put(new Key(MetricAdapter.encodeRowKey("sys.cpu.idle", TEST_TIME + 5), new byte[0], new byte[0],
+                new byte[0], TEST_TIME + 5), EMPTY_VALUE);
+        table.put(new Key(MetricAdapter.encodeRowKey("sys.cpu.user", TEST_TIME), new byte[0], new byte[0], new byte[0],
+                TEST_TIME), EMPTY_VALUE);
+        table.put(new Key(MetricAdapter.encodeRowKey("sys.cpu.user", TEST_TIME + 1), new byte[0], new byte[0],
+                new byte[0], TEST_TIME + 1), EMPTY_VALUE);
+        table.put(new Key(MetricAdapter.encodeRowKey("sys.cpu.user", TEST_TIME + 2), new byte[0], new byte[0],
+                new byte[0], TEST_TIME + 2), EMPTY_VALUE);
+        table.put(new Key(MetricAdapter.encodeRowKey("sys.cpu.user", TEST_TIME + 3), new byte[0], new byte[0],
+                new byte[0], TEST_TIME + 3), EMPTY_VALUE);
+        table.put(new Key(MetricAdapter.encodeRowKey("sys.cpu.user", TEST_TIME + 4), new byte[0], new byte[0],
+                new byte[0], TEST_TIME + 4), EMPTY_VALUE);
+        table.put(new Key(MetricAdapter.encodeRowKey("sys.cpu.user", TEST_TIME + 5), new byte[0], new byte[0],
+                new byte[0], TEST_TIME + 5), EMPTY_VALUE);
+
+        SortedKeyValueIterator<Key, Value> source = new SortedMapIterator(table);
+        MetricAgeOffIterator iter = new MetricAgeOffIterator();
+        HashMap<String, String> options = new HashMap<>();
+        options.put(MetricAgeOffIterator.AGE_OFF_PREFIX + "default", Integer.toString(1 * ONE_DAY));
+        iter.init(source, options, null);
+        iter.seek(new Range(), columnFamilies, true);
+        int seen = 0;
+        while (iter.hasTop()) {
+            Key k = iter.getTopKey();
+            Assert.assertTrue(k.getTimestamp() >= TEST_TIME && k.getTimestamp() <= TEST_TIME + 5);
+            seen++;
+            iter.next();
+        }
+        Assert.assertEquals(12, seen);
+    }
+
+    @Test
+    public void testAgeoffMixed() throws Exception {
+        SortedMap<Key, Value> table = new TreeMap<>();
+        table.put(new Key(MetricAdapter.encodeRowKey("sys.cpu.idle", TEST_TIME - (3 * ONE_DAY)), new byte[0],
+                new byte[0], new byte[0], TEST_TIME - (3 * ONE_DAY)), EMPTY_VALUE);
+        table.put(new Key(MetricAdapter.encodeRowKey("sys.cpu.idle", TEST_TIME - (2 * ONE_DAY)), new byte[0],
+                new byte[0], new byte[0], TEST_TIME - (2 * ONE_DAY)), EMPTY_VALUE);
+        table.put(new Key(MetricAdapter.encodeRowKey("sys.cpu.idle", TEST_TIME - (1 * ONE_DAY)), new byte[0],
+                new byte[0], new byte[0], TEST_TIME - (1 * ONE_DAY)), EMPTY_VALUE);
+        table.put(new Key(MetricAdapter.encodeRowKey("sys.cpu.idle", TEST_TIME), new byte[0], new byte[0], new byte[0],
+                TEST_TIME), EMPTY_VALUE);
+        table.put(new Key(MetricAdapter.encodeRowKey("sys.cpu.idle", TEST_TIME + ONE_DAY), new byte[0], new byte[0],
+                new byte[0], TEST_TIME + ONE_DAY), EMPTY_VALUE);
+        table.put(new Key(MetricAdapter.encodeRowKey("sys.cpu.idle", TEST_TIME + (2 * ONE_DAY)), new byte[0],
+                new byte[0], new byte[0], TEST_TIME + (2 * ONE_DAY)), EMPTY_VALUE);
+        table.put(new Key(MetricAdapter.encodeRowKey("sys.cpu.user", TEST_TIME - (3 * ONE_DAY)), new byte[0],
+                new byte[0], new byte[0], TEST_TIME - (3 * ONE_DAY)), EMPTY_VALUE);
+        table.put(new Key(MetricAdapter.encodeRowKey("sys.cpu.user", TEST_TIME - (2 * ONE_DAY)), new byte[0],
+                new byte[0], new byte[0], TEST_TIME - (2 * ONE_DAY)), EMPTY_VALUE);
+        table.put(new Key(MetricAdapter.encodeRowKey("sys.cpu.user", TEST_TIME - (1 * ONE_DAY)), new byte[0],
+                new byte[0], new byte[0], TEST_TIME - (1 * ONE_DAY)), EMPTY_VALUE);
+        table.put(new Key(MetricAdapter.encodeRowKey("sys.cpu.user", TEST_TIME), new byte[0], new byte[0], new byte[0],
+                TEST_TIME), EMPTY_VALUE);
+        table.put(new Key(MetricAdapter.encodeRowKey("sys.cpu.user", TEST_TIME + ONE_DAY), new byte[0], new byte[0],
+                new byte[0], TEST_TIME + ONE_DAY), EMPTY_VALUE);
+        table.put(new Key(MetricAdapter.encodeRowKey("sys.cpu.user", TEST_TIME + (2 * ONE_DAY)), new byte[0],
+                new byte[0], new byte[0], TEST_TIME + (2 * ONE_DAY)), EMPTY_VALUE);
+
+        SortedKeyValueIterator<Key, Value> source = new SortedMapIterator(table);
+        MetricAgeOffIterator iter = new MetricAgeOffIterator();
+        HashMap<String, String> options = new HashMap<>();
+        options.put(MetricAgeOffIterator.AGE_OFF_PREFIX + "default", Integer.toString(1 * ONE_DAY));
+        options.put(MetricAgeOffIterator.AGE_OFF_PREFIX + "sys.cpu.user", Integer.toString(2 * ONE_DAY));
+        iter.init(source, options, null);
+        iter.seek(new Range(), columnFamilies, true);
+        int seen = 0;
+        while (iter.hasTop()) {
+            Key k = iter.getTopKey();
+            Assert.assertTrue(k.getTimestamp() >= (TEST_TIME - (2 * ONE_DAY))
+                    && k.getTimestamp() <= TEST_TIME + (2 * ONE_DAY));
+            seen++;
+            iter.next();
+        }
+        Assert.assertEquals(7, seen);
+
+    }
+
+    @Test
+    public void testSeekPastEndKey() throws Exception {
+        SortedMap<Key, Value> table = new TreeMap<>();
+        table.put(new Key(MetricAdapter.encodeRowKey("sys.cpu.user", TEST_TIME), new byte[0], new byte[0], new byte[0],
+                TEST_TIME), EMPTY_VALUE);
+        table.put(new Key(MetricAdapter.encodeRowKey("sys.cpu.user", TEST_TIME + 1), new byte[0], new byte[0],
+                new byte[0], TEST_TIME + 1), EMPTY_VALUE);
+        table.put(new Key(MetricAdapter.encodeRowKey("sys.cpu.user", TEST_TIME + 2), new byte[0], new byte[0],
+                new byte[0], TEST_TIME + 2), EMPTY_VALUE);
+        table.put(new Key(MetricAdapter.encodeRowKey("sys.cpu.user", TEST_TIME + 3), new byte[0], new byte[0],
+                new byte[0], TEST_TIME + 3), EMPTY_VALUE);
+        table.put(new Key(MetricAdapter.encodeRowKey("sys.cpu.user", TEST_TIME + 4), new byte[0], new byte[0],
+                new byte[0], TEST_TIME + 4), EMPTY_VALUE);
+        table.put(new Key(MetricAdapter.encodeRowKey("sys.cpu.user", TEST_TIME + 5), new byte[0], new byte[0],
+                new byte[0], TEST_TIME + 5), EMPTY_VALUE);
+
+        SortedKeyValueIterator<Key, Value> source = new SortedMapIterator(table);
+        MetricAgeOffIterator iter = new MetricAgeOffIterator();
+        HashMap<String, String> options = new HashMap<>();
+        options.put(MetricAgeOffIterator.AGE_OFF_PREFIX + "default", Integer.toString(1));
+        iter.init(source, options, null);
+        iter.seek(
+                new Range(new Key("sys.cpu.user"), true, new Key(MetricAdapter.encodeRowKey("sys.cpu.user",
+                        TEST_TIME + 3), new byte[0], new byte[0], new byte[0], TEST_TIME + 3), true), columnFamilies,
+                true);
+        int seen = 0;
+        while (iter.hasTop()) {
+            Key k = iter.getTopKey();
+            Assert.assertTrue(k.getTimestamp() >= TEST_TIME && k.getTimestamp() <= TEST_TIME + 5);
+            seen++;
+            iter.next();
+        }
+        Assert.assertEquals(0, seen);
+    }
+
+}

server/src/test/java/timely/store/iterators/RateIteratorTest.java

@@ -19,6 +19,7 @@ import org.junit.Before;
 import org.junit.Test;
 
 import timely.adapter.accumulo.MetricAdapter;
+import timely.api.request.timeseries.QueryRequest;
 import timely.model.Metric;
 import timely.model.Tag;
 
@@ -103,4 +104,107 @@ public class RateIteratorTest extends IteratorTestBase {
         assertFalse(iter.hasTop());
     }
 
+    @Test
+    public void testCounterRate() throws Exception {
+        table.clear();
+        long ts = System.currentTimeMillis();
+        for (int j = 0; j < 10; j++) {
+            for (int i = 1; i <= 10; i++) {
+                ts += 1000;
+                Metric m = new Metric("sys.cpu.user", ts, i * 1.0D, tags);
+                byte[] row = MetricAdapter.encodeRowKey(m);
+                Key k = new Key(row, tags.get(0).join().getBytes(StandardCharsets.UTF_8), new byte[0], new byte[0], ts);
+                Value v = new Value(MetricAdapter.encodeValue(m.getValue().getMeasure()));
+                table.put(k, v);
+            }
+        }
+
+        SortedMapIterator source = new SortedMapIterator(table);
+        RateIterator iter = new RateIterator();
+        IteratorSetting settings = new IteratorSetting(100, RateIterator.class);
+
+        QueryRequest.RateOption option = new QueryRequest.RateOption();
+        option.setCounter(true);
+        option.setCounterMax(0);
+        RateIterator.setRateOptions(settings, option);
+
+        iter.init(source, settings.getOptions(), SCAN_IE);
+        iter.seek(new Range(), EMPTY_COL_FAMS, true);
+        for (int i = 0; i < 99; i++) {
+            assertTrue(iter.hasTop());
+            assertEquals(0.001D, MetricAdapter.decodeValue(iter.getTopValue().get()), 0.0D);
+            iter.next();
+        }
+        assertFalse(iter.hasTop());
+    }
+
+    @Test
+    public void testCounterRateWithMax() throws Exception {
+        table.clear();
+        long ts = System.currentTimeMillis();
+        for (int j = 0; j < 10; j++) {
+            for (int i = 0; i < 10; i++) {
+                ts += 1000;
+                Metric m = new Metric("sys.cpu.user", ts, i * 1.0D, tags);
+                byte[] row = MetricAdapter.encodeRowKey(m);
+                Key k = new Key(row, tags.get(0).join().getBytes(StandardCharsets.UTF_8), new byte[0], new byte[0], ts);
+                Value v = new Value(MetricAdapter.encodeValue(m.getValue().getMeasure()));
+                table.put(k, v);
+            }
+        }
+
+        SortedMapIterator source = new SortedMapIterator(table);
+        RateIterator iter = new RateIterator();
+        IteratorSetting settings = new IteratorSetting(100, RateIterator.class);
+
+        QueryRequest.RateOption option = new QueryRequest.RateOption();
+        option.setCounter(true);
+        option.setCounterMax(10);
+        RateIterator.setRateOptions(settings, option);
+
+        iter.init(source, settings.getOptions(), SCAN_IE);
+        iter.seek(new Range(), EMPTY_COL_FAMS, true);
+        for (int i = 0; i < 99; i++) {
+            assertTrue(iter.hasTop());
+            assertEquals(0.001D, MetricAdapter.decodeValue(iter.getTopValue().get()), 0.0D);
+            iter.next();
+        }
+        assertFalse(iter.hasTop());
+    }
+
+    @Test
+    public void testCounterRateWithReset() throws Exception {
+        table.clear();
+        long ts = System.currentTimeMillis();
+        for (int j = 0; j < 10; j++) {
+            for (int i = 0; i < 10; i++) {
+                ts += 1000;
+                Metric m = new Metric("sys.cpu.user", ts, i * 1.0D, tags);
+                byte[] row = MetricAdapter.encodeRowKey(m);
+                Key k = new Key(row, tags.get(0).join().getBytes(StandardCharsets.UTF_8), new byte[0], new byte[0], ts);
+                Value v = new Value(MetricAdapter.encodeValue(m.getValue().getMeasure()));
+                table.put(k, v);
+            }
+        }
+
+        SortedMapIterator source = new SortedMapIterator(table);
+        RateIterator iter = new RateIterator();
+        IteratorSetting settings = new IteratorSetting(100, RateIterator.class);
+
+        QueryRequest.RateOption option = new QueryRequest.RateOption();
+        option.setCounter(true);
+        option.setCounterMax(Long.MAX_VALUE);
+        option.setResetValue(1);
+        RateIterator.setRateOptions(settings, option);
+
+        iter.init(source, settings.getOptions(), SCAN_IE);
+        iter.seek(new Range(), EMPTY_COL_FAMS, true);
+        for (int i = 0; i < 99; i++) {
+            assertTrue(iter.hasTop());
+            assertEquals(((i + 1) % 10 == 0 ? 0.0D : 0.001D), MetricAdapter.decodeValue(iter.getTopValue().get()), 0.0D);
+            iter.next();
+        }
+        assertFalse(iter.hasTop());
+    }
+
 }

server/src/test/java/timely/test/integration/DataStoreIT.java

@@ -16,6 +16,7 @@ import timely.Server;
 import timely.api.request.timeseries.QueryRequest;
 import timely.api.request.timeseries.QueryRequest.SubQuery;
 import timely.api.response.timeseries.QueryResponse;
+import timely.store.MetricAgeOffIterator;
 import timely.test.IntegrationTest;
 
 @Category(IntegrationTest.class)
@@ -28,7 +29,7 @@ public class DataStoreIT extends OneWaySSLBase {
     @Test
     public void testDefaultAgeOff() throws Exception {
         HashMap<String, Integer> ageOffSettings = new HashMap<>();
-        ageOffSettings.put("default", 1);
+        ageOffSettings.put(MetricAgeOffIterator.DEFAULT_AGEOFF_KEY, 1);
         conf.setMetricAgeOffDays(ageOffSettings);
 
         final Server s = new Server(conf);
@@ -62,7 +63,7 @@ public class DataStoreIT extends OneWaySSLBase {
     @Test
     public void testMultipleAgeOff() throws Exception {
         HashMap<String, Integer> ageOffSettings = new HashMap<>();
-        ageOffSettings.put("default", 1);
+        ageOffSettings.put(MetricAgeOffIterator.DEFAULT_AGEOFF_KEY, 1);
         ageOffSettings.put("sys.cpu.user", 1);
         conf.setMetricAgeOffDays(ageOffSettings);
 

server/src/test/java/timely/test/integration/MacITBase.java

@@ -1,5 +1,10 @@
 package timely.test.integration;
 
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.util.HashMap;
+
 import org.apache.accumulo.core.client.Connector;
 import org.apache.accumulo.minicluster.MiniAccumuloCluster;
 import org.apache.accumulo.minicluster.MiniAccumuloConfig;
@@ -7,12 +12,10 @@ import org.junit.Before;
 import org.junit.BeforeClass;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import timely.Configuration;
-import timely.test.TestConfiguration;
 
-import java.io.File;
-import java.io.IOException;
-import java.nio.file.Files;
+import timely.Configuration;
+import timely.store.MetricAgeOffIterator;
+import timely.test.TestConfiguration;
 
 /**
  * Base class for integration tests using mini accumulo cluster.
@@ -49,6 +52,10 @@ public class MacITBase {
             conf.getAccumulo().setZookeepers(mac.getZooKeepers());
             conf.getSecurity().getSsl().setUseOpenssl(false);
             conf.getSecurity().getSsl().setUseGeneratedKeypair(true);
+            conf.getWebsocket().setFlushIntervalSeconds(TestConfiguration.WAIT_SECONDS);
+            HashMap<String, Integer> ageOffSettings = new HashMap<>();
+            ageOffSettings.put(MetricAgeOffIterator.DEFAULT_AGEOFF_KEY, 7);
+            conf.setMetricAgeOffDays(ageOffSettings);
         } else {
             LOG.info("Mini Accumulo already running.");
         }
@@ -71,6 +78,10 @@ public class MacITBase {
         conf.getAccumulo().setZookeepers(mac.getZooKeepers());
         conf.getSecurity().getSsl().setUseOpenssl(false);
         conf.getSecurity().getSsl().setUseGeneratedKeypair(true);
+        conf.getWebsocket().setFlushIntervalSeconds(TestConfiguration.WAIT_SECONDS);
+        HashMap<String, Integer> ageOffSettings = new HashMap<>();
+        ageOffSettings.put(MetricAgeOffIterator.DEFAULT_AGEOFF_KEY, 7);
+        conf.setMetricAgeOffDays(ageOffSettings);
     }
 
 }

server/src/test/java/timely/test/integration/client/TcpClientIT.java

@@ -0,0 +1,242 @@
+package timely.test.integration.client;
+
+import static com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly;
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static timely.test.TestConfiguration.WAIT_SECONDS;
+
+import java.nio.ByteBuffer;
+import java.util.EnumSet;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.ZooKeeperInstance;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
+import org.apache.accumulo.core.security.Authorizations;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import timely.Server;
+import timely.TestServer;
+import timely.api.request.MetricRequest;
+import timely.auth.AuthCache;
+import timely.client.tcp.TcpClient;
+import timely.model.Metric;
+import timely.model.Tag;
+import timely.test.IntegrationTest;
+import timely.test.integration.OneWaySSLBase;
+
+@Category(IntegrationTest.class)
+public class TcpClientIT extends OneWaySSLBase {
+
+    private static final Logger LOG = LoggerFactory.getLogger(TcpClientIT.class);
+    private static final Long TEST_TIME = System.currentTimeMillis();
+
+    @After
+    public void tearDown() throws Exception {
+        AuthCache.resetSessionMaxAge();
+    }
+
+    @Test
+    public void testPut() throws Exception {
+        final TestServer m = new TestServer(conf);
+        m.run();
+        try (TcpClient client = new TcpClient("127.0.0.1", 54321)) {
+            client.open();
+            client.write("put sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2\n");
+            client.flush();
+            while (1 != m.getTcpRequests().getCount()) {
+                Thread.sleep(5);
+            }
+            Assert.assertEquals(1, m.getTcpRequests().getResponses().size());
+            Assert.assertEquals(MetricRequest.class, m.getTcpRequests().getResponses().get(0).getClass());
+            final MetricRequest actual = (MetricRequest) m.getTcpRequests().getResponses().get(0);
+            // @formatter:off
+            final MetricRequest expected = new MetricRequest(
+                    Metric.newBuilder()
+                            .name("sys.cpu.user")
+                            .value(TEST_TIME, 1.0D)
+                            .tag(new Tag("tag1", "value1"))
+                            .tag(new Tag("tag2", "value2"))
+                            .build()
+            );
+            // @formatter:on
+            Assert.assertEquals(expected, actual);
+        } finally {
+            m.shutdown();
+        }
+    }
+
+    @Test
+    public void testPutMultiple() throws Exception {
+
+        final TestServer m = new TestServer(conf);
+        m.run();
+        try (TcpClient client = new TcpClient("127.0.0.1", 54321)) {
+            client.open();
+            // @formatter:off
+        	client.write("put sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2\n"
+                       + "put sys.cpu.idle " + (TEST_TIME + 1) + " 1.0 tag3=value3 tag4=value4\n");
+        	client.flush();
+            while (2 != m.getTcpRequests().getCount()) {
+                Thread.sleep(5);
+            }
+            Assert.assertEquals(2, m.getTcpRequests().getResponses().size());
+            Assert.assertEquals(MetricRequest.class, m.getTcpRequests().getResponses().get(0).getClass());
+            MetricRequest actual = (MetricRequest) m.getTcpRequests().getResponses().get(0);
+            MetricRequest expected = new MetricRequest(
+                    Metric.newBuilder()
+                            .name("sys.cpu.user")
+                            .value(TEST_TIME, 1.0D)
+                            .tag(new Tag("tag1", "value1"))
+                            .tag(new Tag("tag2", "value2"))
+                            .build()
+            );
+            Assert.assertEquals(expected, actual);
+
+            Assert.assertEquals(MetricRequest.class, m.getTcpRequests().getResponses().get(1).getClass());
+            actual = (MetricRequest) m.getTcpRequests().getResponses().get(1);
+            expected = new MetricRequest(
+                    Metric.newBuilder()
+                        .name("sys.cpu.idle")
+                        .value(TEST_TIME + 1, 1.0D)
+                        .tag(new Tag("tag3", "value3"))
+                        .tag(new Tag("tag4", "value4"))
+                        .build()
+            );
+            // @formatter:on
+            Assert.assertEquals(expected, actual);
+
+        } finally {
+            m.shutdown();
+        }
+    }
+
+    @Test
+    public void testPutInvalidTimestamp() throws Exception {
+        final TestServer m = new TestServer(conf);
+        m.run();
+        try (TcpClient client = new TcpClient("127.0.0.1", 54321)) {
+            client.open();
+            client.write("put sys.cpu.user " + TEST_TIME + "Z" + " 1.0 tag1=value1 tag2=value2\n");
+            client.flush();
+            sleepUninterruptibly(WAIT_SECONDS, TimeUnit.SECONDS);
+            Assert.assertEquals(0, m.getTcpRequests().getCount());
+        } finally {
+            m.shutdown();
+        }
+    }
+
+    @Test
+    public void testPersistence() throws Exception {
+        final Server s = new Server(conf);
+        s.run();
+        try {
+            put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2", "sys.cpu.idle " + (TEST_TIME + 1)
+                    + " 1.0 tag3=value3 tag4=value4", "sys.cpu.idle " + (TEST_TIME + 2)
+                    + " 1.0 tag3=value3 tag4=value4");
+            sleepUninterruptibly(WAIT_SECONDS, TimeUnit.SECONDS);
+        } finally {
+            s.shutdown();
+        }
+        final ZooKeeperInstance inst = new ZooKeeperInstance(mac.getClientConfig());
+        final Connector connector = inst.getConnector("root", new PasswordToken("secret".getBytes(UTF_8)));
+        assertTrue(connector.namespaceOperations().exists("timely"));
+        assertTrue(connector.tableOperations().exists("timely.metrics"));
+        assertTrue(connector.tableOperations().exists("timely.meta"));
+        int count = 0;
+        for (final Entry<Key, Value> entry : connector.createScanner("timely.metrics", Authorizations.EMPTY)) {
+            LOG.info("Entry: " + entry);
+            final double value = ByteBuffer.wrap(entry.getValue().get()).getDouble();
+            assertEquals(1.0, value, 1e-9);
+            count++;
+        }
+        assertEquals(6, count);
+        count = 0;
+        for (final Entry<Key, Value> entry : connector.createScanner("timely.meta", Authorizations.EMPTY)) {
+            LOG.info("Meta entry: " + entry);
+            count++;
+        }
+        assertEquals(10, count);
+        // count w/out versioning iterator to make sure that the optimization
+        // for writing is working
+        connector.tableOperations().removeIterator("timely.meta", "vers", EnumSet.of(IteratorScope.scan));
+        // wait for zookeeper propagation
+        sleepUninterruptibly(WAIT_SECONDS, TimeUnit.SECONDS);
+        count = 0;
+        for (final Entry<Key, Value> entry : connector.createScanner("timely.meta", Authorizations.EMPTY)) {
+            LOG.info("Meta no vers iter: " + entry);
+            count++;
+        }
+        assertEquals(10, count);
+    }
+
+    @Test
+    public void testPersistenceWithVisibility() throws Exception {
+        final Server s = new Server(conf);
+        s.run();
+        try {
+            put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2", "sys.cpu.idle " + (TEST_TIME + 1)
+                    + " 1.0 tag3=value3 tag4=value4 viz=(a|b)", "sys.cpu.idle " + (TEST_TIME + 2)
+                    + " 1.0 tag3=value3 tag4=value4 viz=(c&b)");
+            sleepUninterruptibly(WAIT_SECONDS, TimeUnit.SECONDS);
+        } finally {
+            s.shutdown();
+        }
+        final ZooKeeperInstance inst = new ZooKeeperInstance(mac.getClientConfig());
+        final Connector connector = inst.getConnector("root", new PasswordToken("secret".getBytes(UTF_8)));
+        connector.securityOperations().changeUserAuthorizations("root", new Authorizations("a", "b", "c"));
+
+        int count = 0;
+        for (final Map.Entry<Key, Value> entry : connector.createScanner("timely.metrics", Authorizations.EMPTY)) {
+            LOG.info("Entry: " + entry);
+            final double value = ByteBuffer.wrap(entry.getValue().get()).getDouble();
+            assertEquals(1.0, value, 1e-9);
+            count++;
+        }
+        assertEquals(2, count);
+        count = 0;
+        Authorizations auth1 = new Authorizations("a");
+        for (final Map.Entry<Key, Value> entry : connector.createScanner("timely.metrics", auth1)) {
+            LOG.info("Entry: " + entry);
+            final double value = ByteBuffer.wrap(entry.getValue().get()).getDouble();
+            assertEquals(1.0, value, 1e-9);
+            count++;
+        }
+        assertEquals(4, count);
+        count = 0;
+        Authorizations auth2 = new Authorizations("b", "c");
+        for (final Map.Entry<Key, Value> entry : connector.createScanner("timely.metrics", auth2)) {
+            LOG.info("Entry: " + entry);
+            final double value = ByteBuffer.wrap(entry.getValue().get()).getDouble();
+            assertEquals(1.0, value, 1e-9);
+            count++;
+        }
+        assertEquals(6, count);
+    }
+
+    protected void put(String... lines) throws Exception {
+        StringBuffer format = new StringBuffer();
+        for (String line : lines) {
+            format.append("put ");
+            format.append(line);
+            format.append("\n");
+        }
+        try (TcpClient client = new TcpClient("127.0.0.1", 54321)) {
+            client.open();
+            client.write(format.toString());
+            client.flush();
+        }
+    }
+
+}

server/src/test/java/timely/test/integration/client/UdpClientIT.java

@@ -0,0 +1,251 @@
+package timely.test.integration.client;
+
+import static com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly;
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.nio.ByteBuffer;
+import java.util.EnumSet;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.ZooKeeperInstance;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
+import org.apache.accumulo.core.security.Authorizations;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import timely.Server;
+import timely.TestServer;
+import timely.api.request.MetricRequest;
+import timely.auth.AuthCache;
+import timely.client.udp.UdpClient;
+import timely.model.Metric;
+import timely.model.Tag;
+import timely.test.IntegrationTest;
+import timely.test.TestConfiguration;
+import timely.test.integration.MacITBase;
+
+@Category(IntegrationTest.class)
+public class UdpClientIT extends MacITBase {
+
+    private static final Logger LOG = LoggerFactory.getLogger(UdpClientIT.class);
+    private static final Long TEST_TIME = System.currentTimeMillis();
+
+    @Before
+    public void setup() throws Exception {
+        Connector con = mac.getConnector("root", "secret");
+        con.tableOperations().list().forEach(t -> {
+            if (t.startsWith("timely")) {
+                try {
+                    con.tableOperations().delete(t);
+                } catch (Exception e) {
+                }
+            }
+        });
+    }
+
+    @After
+    public void tearDown() throws Exception {
+        AuthCache.resetSessionMaxAge();
+    }
+
+    @Test
+    public void testPut() throws Exception {
+        final TestServer m = new TestServer(conf);
+        m.run();
+        try (UdpClient client = new UdpClient("127.0.0.1", 54325)) {
+            client.open();
+            client.write(("put sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2\n"));
+            while (1 != m.getUdpRequests().getCount()) {
+                Thread.sleep(5);
+            }
+            Assert.assertEquals(1, m.getUdpRequests().getResponses().size());
+            Assert.assertEquals(MetricRequest.class, m.getUdpRequests().getResponses().get(0).getClass());
+            final MetricRequest actual = (MetricRequest) m.getUdpRequests().getResponses().get(0);
+            // @formatter:off
+            final MetricRequest expected = new MetricRequest(
+                    Metric.newBuilder()
+                            .name("sys.cpu.user")
+                            .value(TEST_TIME, 1.0D)
+                            .tag(new Tag("tag1", "value1"))
+                            .tag(new Tag("tag2", "value2"))
+                            .build()
+            );
+            // @formatter:on
+            Assert.assertEquals(expected, actual);
+        } finally {
+            m.shutdown();
+        }
+    }
+
+    @Test
+    public void testPutMultiple() throws Exception {
+
+        final TestServer m = new TestServer(conf);
+        m.run();
+        try (UdpClient client = new UdpClient("127.0.0.1", 54325)) {
+            client.open();
+            client.write(("put sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2\n" + "put sys.cpu.idle "
+                    + (TEST_TIME + 1) + " 1.0 tag3=value3 tag4=value4\n"));
+            while (2 != m.getUdpRequests().getCount()) {
+                Thread.sleep(5);
+            }
+            Assert.assertEquals(2, m.getUdpRequests().getResponses().size());
+            Assert.assertEquals(MetricRequest.class, m.getUdpRequests().getResponses().get(0).getClass());
+            MetricRequest actual = (MetricRequest) m.getUdpRequests().getResponses().get(0);
+            // @formatter:off
+            MetricRequest expected = new MetricRequest (
+                    Metric.newBuilder()
+                            .name("sys.cpu.user")
+                            .value(TEST_TIME, 1.0D)
+                            .tag(new Tag("tag1", "value1"))
+                            .tag(new Tag("tag2", "value2"))
+                            .build()
+            );
+            Assert.assertEquals(expected, actual);
+
+            Assert.assertEquals(MetricRequest.class, m.getUdpRequests().getResponses().get(1).getClass());
+            actual = (MetricRequest) m.getUdpRequests().getResponses().get(1);
+            expected = new MetricRequest(
+                    Metric.newBuilder()
+                            .name("sys.cpu.idle")
+                            .value(TEST_TIME + 1, 1.0D)
+                            .tag(new Tag("tag3", "value3"))
+                            .tag(new Tag("tag4", "value4"))
+                            .build()
+            );
+            Assert.assertEquals(expected, actual);
+            // @formatter:on
+        } finally {
+            m.shutdown();
+        }
+    }
+
+    @Test
+    public void testPutInvalidTimestamp() throws Exception {
+        final TestServer m = new TestServer(conf);
+        m.run();
+        try (UdpClient client = new UdpClient("127.0.0.1", 54325)) {
+            client.open();
+            client.write(("put sys.cpu.user " + TEST_TIME + "Z" + " 1.0 tag1=value1 tag2=value2\n"));
+            sleepUninterruptibly(TestConfiguration.WAIT_SECONDS, TimeUnit.SECONDS);
+            Assert.assertEquals(0, m.getUdpRequests().getCount());
+        } finally {
+            m.shutdown();
+        }
+    }
+
+    @Test
+    public void testPersistence() throws Exception {
+        final Server s = new Server(conf);
+        s.run();
+        try {
+            put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2", "sys.cpu.idle " + (TEST_TIME + 1)
+                    + " 1.0 tag3=value3 tag4=value4", "sys.cpu.idle " + (TEST_TIME + 2)
+                    + " 1.0 tag3=value3 tag4=value4");
+            sleepUninterruptibly(TestConfiguration.WAIT_SECONDS, TimeUnit.SECONDS);
+        } finally {
+            s.shutdown();
+        }
+        final ZooKeeperInstance inst = new ZooKeeperInstance(mac.getClientConfig());
+        final Connector connector = inst.getConnector("root", new PasswordToken("secret".getBytes(UTF_8)));
+        assertTrue(connector.namespaceOperations().exists("timely"));
+        assertTrue(connector.tableOperations().exists("timely.metrics"));
+        assertTrue(connector.tableOperations().exists("timely.meta"));
+        int count = 0;
+        for (final Entry<Key, Value> entry : connector.createScanner("timely.metrics", Authorizations.EMPTY)) {
+            LOG.info("Entry: " + entry);
+            final double value = ByteBuffer.wrap(entry.getValue().get()).getDouble();
+            assertEquals(1.0, value, 1e-9);
+            count++;
+        }
+        assertEquals(6, count);
+        count = 0;
+        for (final Entry<Key, Value> entry : connector.createScanner("timely.meta", Authorizations.EMPTY)) {
+            LOG.info("Meta entry: " + entry);
+            count++;
+        }
+        assertEquals(10, count);
+        // count w/out versioning iterator to make sure that the optimization
+        // for writing is working
+        connector.tableOperations().removeIterator("timely.meta", "vers", EnumSet.of(IteratorScope.scan));
+        // wait for zookeeper propagation
+        sleepUninterruptibly(TestConfiguration.WAIT_SECONDS, TimeUnit.SECONDS);
+        count = 0;
+        for (final Entry<Key, Value> entry : connector.createScanner("timely.meta", Authorizations.EMPTY)) {
+            LOG.info("Meta no vers iter: " + entry);
+            count++;
+        }
+        assertEquals(10, count);
+    }
+
+    @Test
+    public void testPersistenceWithVisibility() throws Exception {
+        final Server s = new Server(conf);
+        s.run();
+        try {
+            put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2", "sys.cpu.idle " + (TEST_TIME + 1)
+                    + " 1.0 tag3=value3 tag4=value4 viz=(a|b)", "sys.cpu.idle " + (TEST_TIME + 2)
+                    + " 1.0 tag3=value3 tag4=value4 viz=(c&b)");
+            sleepUninterruptibly(TestConfiguration.WAIT_SECONDS, TimeUnit.SECONDS);
+        } finally {
+            s.shutdown();
+        }
+        final ZooKeeperInstance inst = new ZooKeeperInstance(mac.getClientConfig());
+        final Connector connector = inst.getConnector("root", new PasswordToken("secret".getBytes(UTF_8)));
+        connector.securityOperations().changeUserAuthorizations("root", new Authorizations("a", "b", "c"));
+
+        int count = 0;
+        for (final Map.Entry<Key, Value> entry : connector.createScanner("timely.metrics", Authorizations.EMPTY)) {
+            LOG.info("Entry: " + entry);
+            final double value = ByteBuffer.wrap(entry.getValue().get()).getDouble();
+            assertEquals(1.0, value, 1e-9);
+            count++;
+        }
+        assertEquals(2, count);
+        count = 0;
+        Authorizations auth1 = new Authorizations("a");
+        for (final Map.Entry<Key, Value> entry : connector.createScanner("timely.metrics", auth1)) {
+            LOG.info("Entry: " + entry);
+            final double value = ByteBuffer.wrap(entry.getValue().get()).getDouble();
+            assertEquals(1.0, value, 1e-9);
+            count++;
+        }
+        assertEquals(4, count);
+        count = 0;
+        Authorizations auth2 = new Authorizations("b", "c");
+        for (final Map.Entry<Key, Value> entry : connector.createScanner("timely.metrics", auth2)) {
+            LOG.info("Entry: " + entry);
+            final double value = ByteBuffer.wrap(entry.getValue().get()).getDouble();
+            assertEquals(1.0, value, 1e-9);
+            count++;
+        }
+        assertEquals(6, count);
+    }
+
+    private void put(String... lines) throws Exception {
+        StringBuffer format = new StringBuffer();
+        for (String line : lines) {
+            format.append("put ");
+            format.append(line);
+            format.append("\n");
+        }
+        try (UdpClient client = new UdpClient("127.0.0.1", 54325)) {
+            client.open();
+            client.write(format.toString());
+        }
+    }
+
+}

server/src/test/java/timely/test/integration/client/WebSocketClientIT.java

@@ -0,0 +1,176 @@
+package timely.test.integration.client;
+
+import static com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly;
+import io.netty.handler.ssl.ApplicationProtocolConfig;
+import io.netty.handler.ssl.JdkSslClientContext;
+import io.netty.handler.ssl.JdkSslContext;
+import io.netty.handler.ssl.SslContext;
+import io.netty.handler.ssl.SslContextBuilder;
+import io.netty.handler.ssl.SslProvider;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+import javax.net.ssl.SSLContext;
+import javax.websocket.CloseReason;
+import javax.websocket.EndpointConfig;
+import javax.websocket.MessageHandler;
+import javax.websocket.Session;
+
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.http.client.HttpResponseException;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import timely.Server;
+import timely.api.response.MetricResponses;
+import timely.auth.AuthCache;
+import timely.client.websocket.ClientHandler;
+import timely.client.websocket.subscription.WebSocketSubscriptionClient;
+import timely.serialize.JsonSerializer;
+import timely.test.IntegrationTest;
+import timely.test.TestConfiguration;
+import timely.test.integration.OneWaySSLBase;
+
+@SuppressWarnings("deprecation")
+@Category(IntegrationTest.class)
+public class WebSocketClientIT extends OneWaySSLBase {
+
+    private final static Logger LOG = LoggerFactory.getLogger(WebSocketClientIT.class);
+    private static final Long TEST_TIME = System.currentTimeMillis() - (240 * 1000);
+
+    private static SSLContext sslCtx = null;
+    private Server s = null;
+
+    private void setupSslCtx() throws Exception {
+        Assert.assertNotNull(clientTrustStoreFile);
+        SslContextBuilder builder = SslContextBuilder.forClient();
+        builder.applicationProtocolConfig(ApplicationProtocolConfig.DISABLED);
+        builder.sslProvider(SslProvider.JDK);
+        builder.trustManager(clientTrustStoreFile); // Trust the server cert
+        SslContext ctx = builder.build();
+        Assert.assertEquals(JdkSslClientContext.class, ctx.getClass());
+        JdkSslContext jdk = (JdkSslContext) ctx;
+        sslCtx = jdk.context();
+    }
+
+    @Before
+    public void setup() throws Exception {
+        Connector con = mac.getConnector("root", "secret");
+        con.securityOperations().changeUserAuthorizations("root", new Authorizations("A", "B", "C", "D", "E", "F"));
+        s = new Server(conf);
+        s.run();
+        setupSslCtx();
+    }
+
+    @After
+    public void tearDown() throws Exception {
+        s.shutdown();
+        AuthCache.resetSessionMaxAge();
+    }
+
+    public void testWorkflow(WebSocketSubscriptionClient client) throws Exception {
+        // Add some data
+        // @formatter:off
+        put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2 rack=r1",
+        	"sys.cpu.user " + TEST_TIME + " 1.0 tag3=value3 rack=r2", 
+        	"sys.cpu.idle " + (TEST_TIME + 2) + " 1.0 tag3=value3 tag4=value4 rack=r1", 
+        	"sys.cpu.idle " + (TEST_TIME + 2) + " 3.0 tag3=value3 tag4=value4 rack=r2");
+		// @formatter:on
+
+        // Latency in TestConfiguration is 2s, wait for it
+        sleepUninterruptibly(TestConfiguration.WAIT_SECONDS, TimeUnit.SECONDS);
+
+        List<String> messages = new ArrayList<>();
+        ClientHandler handler = new ClientHandler() {
+
+            @Override
+            public void onOpen(Session session, EndpointConfig config) {
+                session.addMessageHandler(new MessageHandler.Whole<String>() {
+
+                    @Override
+                    public void onMessage(String message) {
+                        messages.add(message);
+                        LOG.debug("Message received on Websocket session {}: {}", session.getId(), message);
+                    }
+                });
+            }
+
+            @Override
+            public void onClose(Session session, CloseReason reason) {
+                super.onClose(session, reason);
+                try {
+                    client.close();
+                } catch (IOException e) {
+                    Assert.fail("Error calling close on client: " + e.getMessage());
+                }
+            }
+
+            @Override
+            public void onError(Session session, Throwable error) {
+                super.onError(session, error);
+                try {
+                    client.close();
+                } catch (IOException e) {
+                    Assert.fail("Error calling close on client: " + e.getMessage());
+                }
+            }
+        };
+
+        try {
+            client.open(handler);
+            client.addSubscription("sys.cpu.user", null, TEST_TIME, TEST_TIME + 1000, 5000);
+            sleepUninterruptibly(2, TimeUnit.SECONDS);
+
+            Assert.assertEquals(1, messages.size());
+            MetricResponses responses = JsonSerializer.getObjectMapper().readValue(messages.get(0),
+                    MetricResponses.class);
+            responses.getResponses().forEach(metric -> {
+                try {
+                    Assert.assertTrue("sys.cpu.user".equals(metric.getMetric()) || metric.isComplete());
+                } catch (Exception e) {
+                    Assert.fail(e.getMessage());
+                }
+            });
+        } finally {
+            client.close();
+        }
+    }
+
+    @Test
+    public void testClientAnonymousAccess() throws Exception {
+        WebSocketSubscriptionClient client = new WebSocketSubscriptionClient(sslCtx, "localhost", 54322, 54323, false,
+                null, null, false, 65536);
+        testWorkflow(client);
+    }
+
+    @Test
+    public void testClientBasicAuthAccess() throws Exception {
+        WebSocketSubscriptionClient client = new WebSocketSubscriptionClient(sslCtx, "localhost", 54322, 54323, true,
+                "test", "test1", false, 65536);
+        testWorkflow(client);
+    }
+
+    @Test(expected = HttpResponseException.class)
+    public void testClientBasicAuthAccessFailure() throws Exception {
+        WebSocketSubscriptionClient client = new WebSocketSubscriptionClient(sslCtx, "localhost", 54322, 54323, true,
+                "test", "test2", false, 65536);
+        testWorkflow(client);
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void testClientBasicAuthParameterMismatch() throws Exception {
+        WebSocketSubscriptionClient client = new WebSocketSubscriptionClient(sslCtx, "localhost", 54322, 54323, true,
+                "test", null, false, 65536);
+        testWorkflow(client);
+    }
+
+}

server/src/test/java/timely/test/integration/http/HttpApiIT.java

@@ -112,6 +112,38 @@ public class HttpApiIT extends OneWaySSLBase {
         }
     }
 
+    @Test
+    public void testMetricsURLEncoded() throws Exception {
+        final Server s = new Server(conf);
+        s.run();
+        try {
+            // @formatter:off
+            put("sys.cpu.user " + TEST_TIME + " 1.0 tag%31=value%31 tag%32=value%32",
+                    "sys.cpu.idle " + (TEST_TIME + 1) + " 1.0 tag%33=value%33 tag%34=value%34",
+                    "sys.cpu.idle " + (TEST_TIME + 2) + " 1.0 tag%33=value%33 tag%34=value%34 viz=(a|b|c)",
+                    "zzzz 1234567892 1.0 host=localhost");
+
+            // Latency in TestConfiguration is 2s, wait for it
+            sleepUninterruptibly(TestConfiguration.WAIT_SECONDS, TimeUnit.SECONDS);
+
+            String metrics = "https://localhost:54322/api/metrics";
+            // Test prefix matching
+            String result = query(metrics);
+            Document doc = Jsoup.parse(result);
+            Elements tableData = doc.select("td");
+
+            assertEquals(1, tableData.select(":contains(sys.cpu.user)").size());
+            assertEquals(1, tableData.select(":contains(tag1=value1 tag2=value2)").size());
+            assertEquals(1, tableData.select(":contains(sys.cpu.idle)").size());
+            assertEquals(1, tableData.select(":contains(tag3=value3 tag4=value4)").size());
+            assertEquals(1, tableData.select(":contains(zzzz)").size());
+            assertEquals(1, tableData.select(":contains(host=localhost)").size());
+            // @formatter:on
+        } finally {
+            s.shutdown();
+        }
+    }
+
     @Test
     public void testMetricsJson() throws Exception {
         // @formatter:off
@@ -204,16 +236,22 @@ public class HttpApiIT extends OneWaySSLBase {
             Entry<String, Object> entry = entries.next();
             // We are downsampling to 1 second by default, which is why the
             // times in the results end with ms of the start parameter.
-            assertEquals(TEST_TIME.toString(), entry.getKey());
+            assertEquals(getBaselineStart(TEST_TIME), entry.getKey());
             assertEquals(1.0, entry.getValue());
             entry = entries.next();
-            assertEquals(Long.toString(TEST_TIME + 1000), entry.getKey());
+            assertEquals(getBaselineStart(TEST_TIME + 1000), entry.getKey());
             assertEquals(3.0, entry.getValue());
         } finally {
             s.shutdown();
         }
     }
 
+    private String getBaselineStart(Long time) {
+        String timeString = time.toString();
+        return timeString.substring(0, timeString.length() - 3) + "000";
+
+    }
+
     @Test
     public void testQueryWithoutMsResolution() throws Exception {
         final Server s = new Server(conf);
@@ -515,18 +553,19 @@ public class HttpApiIT extends OneWaySSLBase {
                            Map<String, Object> dps = response.getDps();
                            assertEquals(1,dps.size());
                            Entry<String, Object> dpsEntry = dps.entrySet().iterator().next();
-                           long ts = Long.parseLong(dpsEntry.getKey());
+                           Long ts = Long.parseLong(dpsEntry.getKey());
                            double measure = (Double)dpsEntry.getValue();
                            switch(tagv){
                                case "r1":
                                    assertEquals(2.0D, measure, 0.0);
-                                   // TODO re-evaluate this when #81 is resolved
-                                   // assertEquals("TEST_TIME: " + TEST_TIME, TEST_TIME + 2L, ts);
+                                   assertEquals("TEST_TIME: " + getBaselineStart(TEST_TIME + 2), 
+                                		   getBaselineStart(TEST_TIME + 2), ts.toString());
                                    rack1Count.getAndIncrement();
                                    break;
                                case "r2":
                                    assertEquals(3.0D, measure, 0.0);
-                                   assertEquals("TEST_TIME: " + TEST_TIME, TEST_TIME + 1000L, ts);
+                                   assertEquals("TEST_TIME: " + getBaselineStart(TEST_TIME + 1000), 
+                                		   getBaselineStart(TEST_TIME + 1000), ts.toString());
                                    rack2Count.getAndIncrement();
                                    break;
                                default:
@@ -724,6 +763,116 @@ public class HttpApiIT extends OneWaySSLBase {
         }
     }
 
+    @Test
+    public void testRateCounterQuery() throws Exception {
+        final Server s = new Server(conf);
+        s.run();
+        try {
+            // @formatter:off
+            put("sys.cpu.user " + (TEST_TIME) + " 1.0 tag1=value1 tag2=value2 rack=r1",
+                    "sys.cpu.user " + (TEST_TIME+1) + " 2.0 tag1=value1 tag2=value2 rack=r1",
+                    "sys.cpu.user " + (TEST_TIME+2) + " 3.0 tag1=value1 tag2=value2 rack=r1",
+                    "sys.cpu.user " + (TEST_TIME+3) + " 4.0 tag1=value1 tag2=value2 rack=r1",
+                    "sys.cpu.user " + (TEST_TIME+4) + " 5.0 tag1=value1 tag2=value2 rack=r1",
+                    "sys.cpu.user " + (TEST_TIME+5) + " 6.0 tag1=value1 tag2=value2 rack=r1",
+                    "sys.cpu.user " + (TEST_TIME+6) + " 7.0 tag1=value1 tag2=value2 rack=r1",
+                    "sys.cpu.user " + (TEST_TIME+7) + " 8.0 tag1=value1 tag2=value2 rack=r1",
+                    "sys.cpu.user " + (TEST_TIME+8) + " 9.0 tag1=value1 tag2=value2 rack=r1",
+                    "sys.cpu.user " + (TEST_TIME+9) + " 10.0 tag1=value1 tag2=value2 rack=r1",
+                    "sys.cpu.user " + (TEST_TIME+10) + " 1.0 tag1=value1 tag2=value2 rack=r1",
+                    "sys.cpu.user " + (TEST_TIME+11) + " 2.0 tag1=value1 tag2=value2 rack=r1",
+                    "sys.cpu.user " + (TEST_TIME+12) + " 3.0 tag1=value1 tag2=value2 rack=r1",
+                    "sys.cpu.user " + (TEST_TIME+13) + " 4.0 tag1=value1 tag2=value2 rack=r1",
+                    "sys.cpu.user " + (TEST_TIME+14) + " 5.0 tag1=value1 tag2=value2 rack=r1");
+            // @formatter:on
+            // Latency in TestConfiguration is 2s, wait for it
+            sleepUninterruptibly(TestConfiguration.WAIT_SECONDS, TimeUnit.SECONDS);
+            QueryRequest request = new QueryRequest();
+            request.setStart(TEST_TIME);
+            request.setEnd(TEST_TIME + 15);
+            request.setMsResolution(true);
+            SubQuery subQuery = new SubQuery();
+            subQuery.setMetric("sys.cpu.user");
+            Map<String, String> t = new LinkedHashMap<>();
+            subQuery.setTags(t);
+            subQuery.setDownsample(Optional.of("1ms-max"));
+            subQuery.setRate(true);
+            subQuery.setRateOptions(new QueryRequest.RateOption());
+            subQuery.getRateOptions().setCounter(true);
+            request.addQuery(subQuery);
+            List<QueryResponse> response = query("https://127.0.0.1:54322/api/query", request);
+            assertEquals(1, response.size());
+            QueryResponse response1 = response.get(0);
+            Map<String, Object> dps = response1.getDps();
+            assertEquals(14, dps.size());
+            int i = 1;
+            for (Entry<String, Object> e : dps.entrySet()) {
+                assertEquals(Long.toString(TEST_TIME + i), e.getKey());
+                assertEquals(1.0, e.getValue());
+                i++;
+            }
+
+        } finally {
+            s.shutdown();
+        }
+    }
+
+    @Test
+    public void testRateCounterWithOptions() throws Exception {
+        final Server s = new Server(conf);
+        s.run();
+        try {
+            // @formatter:off
+            put("sys.cpu.user " + (TEST_TIME) + " 1.0 tag1=value1 tag2=value2 rack=r1",
+                    "sys.cpu.user " + (TEST_TIME+1) + " 2.0 tag1=value1 tag2=value2 rack=r1",
+                    "sys.cpu.user " + (TEST_TIME+2) + " 3.0 tag1=value1 tag2=value2 rack=r1",
+                    "sys.cpu.user " + (TEST_TIME+3) + " 4.0 tag1=value1 tag2=value2 rack=r1",
+                    "sys.cpu.user " + (TEST_TIME+4) + " 5.0 tag1=value1 tag2=value2 rack=r1",
+                    "sys.cpu.user " + (TEST_TIME+5) + " 6.0 tag1=value1 tag2=value2 rack=r1",
+                    "sys.cpu.user " + (TEST_TIME+6) + " 7.0 tag1=value1 tag2=value2 rack=r1",
+                    "sys.cpu.user " + (TEST_TIME+7) + " 8.0 tag1=value1 tag2=value2 rack=r1",
+                    "sys.cpu.user " + (TEST_TIME+8) + " 9.0 tag1=value1 tag2=value2 rack=r1",
+                    "sys.cpu.user " + (TEST_TIME+9) + " 10.0 tag1=value1 tag2=value2 rack=r1",
+                    "sys.cpu.user " + (TEST_TIME+10) + " 0.0 tag1=value1 tag2=value2 rack=r1",
+                    "sys.cpu.user " + (TEST_TIME+11) + " 1.0 tag1=value1 tag2=value2 rack=r1",
+                    "sys.cpu.user " + (TEST_TIME+12) + " 2.0 tag1=value1 tag2=value2 rack=r1",
+                    "sys.cpu.user " + (TEST_TIME+13) + " 3.0 tag1=value1 tag2=value2 rack=r1",
+                    "sys.cpu.user " + (TEST_TIME+14) + " 4.0 tag1=value1 tag2=value2 rack=r1");
+            // @formatter:on
+            // Latency in TestConfiguration is 2s, wait for it
+            sleepUninterruptibly(TestConfiguration.WAIT_SECONDS, TimeUnit.SECONDS);
+            QueryRequest request = new QueryRequest();
+            request.setStart(TEST_TIME);
+            request.setEnd(TEST_TIME + 15);
+            request.setMsResolution(true);
+            SubQuery subQuery = new SubQuery();
+            subQuery.setMetric("sys.cpu.user");
+            Map<String, String> t = new LinkedHashMap<>();
+            subQuery.setTags(t);
+            subQuery.setDownsample(Optional.of("1ms-max"));
+            subQuery.setRate(true);
+            subQuery.setRateOptions(new QueryRequest.RateOption());
+            subQuery.getRateOptions().setCounter(true);
+            subQuery.getRateOptions().setCounterMax(Long.MAX_VALUE);
+            subQuery.getRateOptions().setResetValue(2);
+            request.addQuery(subQuery);
+            List<QueryResponse> response = query("https://127.0.0.1:54322/api/query", request);
+            assertEquals(1, response.size());
+            QueryResponse response1 = response.get(0);
+            Map<String, Object> dps = response1.getDps();
+            assertEquals(14, dps.size());
+            int i = 1;
+            for (Entry<String, Object> e : dps.entrySet()) {
+                assertEquals(Long.toString(TEST_TIME + i), e.getKey());
+                assertEquals((i == 10 ? 0.0 : 1.0), e.getValue());
+                i++;
+            }
+
+        } finally {
+            s.shutdown();
+        }
+    }
+
     @Test
     public void testGetVersion() throws Exception {
         final Server s = new Server(conf);

server/src/test/java/timely/test/integration/websocket/WebSocketIT.java

@@ -1,18 +1,46 @@
 package timely.test.integration.websocket;
 
-import com.fasterxml.jackson.databind.JavaType;
+import static com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
 import io.netty.bootstrap.Bootstrap;
-import io.netty.channel.*;
+import io.netty.channel.Channel;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.channel.ChannelInitializer;
+import io.netty.channel.ChannelPromise;
+import io.netty.channel.EventLoopGroup;
+import io.netty.channel.SimpleChannelInboundHandler;
 import io.netty.channel.nio.NioEventLoopGroup;
 import io.netty.channel.socket.SocketChannel;
 import io.netty.channel.socket.nio.NioSocketChannel;
-import io.netty.handler.codec.http.*;
+import io.netty.handler.codec.http.DefaultHttpHeaders;
+import io.netty.handler.codec.http.FullHttpResponse;
+import io.netty.handler.codec.http.HttpClientCodec;
+import io.netty.handler.codec.http.HttpHeaders;
 import io.netty.handler.codec.http.HttpHeaders.Names;
+import io.netty.handler.codec.http.HttpObjectAggregator;
 import io.netty.handler.codec.http.cookie.ClientCookieEncoder;
-import io.netty.handler.codec.http.websocketx.*;
+import io.netty.handler.codec.http.websocketx.CloseWebSocketFrame;
+import io.netty.handler.codec.http.websocketx.PingWebSocketFrame;
+import io.netty.handler.codec.http.websocketx.PongWebSocketFrame;
+import io.netty.handler.codec.http.websocketx.TextWebSocketFrame;
+import io.netty.handler.codec.http.websocketx.WebSocketClientHandshaker;
+import io.netty.handler.codec.http.websocketx.WebSocketClientHandshakerFactory;
+import io.netty.handler.codec.http.websocketx.WebSocketFrame;
+import io.netty.handler.codec.http.websocketx.WebSocketVersion;
 import io.netty.handler.ssl.SslContext;
 import io.netty.handler.ssl.SslContextBuilder;
 import io.netty.handler.ssl.util.InsecureTrustManagerFactory;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.security.Authorizations;
 import org.junit.AfterClass;
 import org.junit.Assert;
 import org.junit.Before;
@@ -21,6 +49,8 @@ import org.junit.experimental.categories.Category;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.security.authentication.UsernamePasswordAuthenticationToken;
+import org.springframework.security.core.Authentication;
+
 import timely.Server;
 import timely.api.request.VersionRequest;
 import timely.api.request.subscription.AddSubscription;
@@ -30,12 +60,14 @@ import timely.api.request.subscription.RemoveSubscription;
 import timely.api.request.timeseries.AggregatorsRequest;
 import timely.api.request.timeseries.MetricsRequest;
 import timely.api.response.MetricResponse;
+import timely.api.response.MetricResponses;
 import timely.api.response.timeseries.AggregatorsResponse;
 import timely.api.response.timeseries.QueryResponse;
 import timely.api.response.timeseries.SearchLookupResponse;
 import timely.api.response.timeseries.SearchLookupResponse.Result;
 import timely.api.response.timeseries.SuggestResponse;
 import timely.auth.AuthCache;
+import timely.auth.AuthenticationService;
 import timely.model.Metric;
 import timely.model.Tag;
 import timely.netty.Constants;
@@ -44,16 +76,7 @@ import timely.test.TestConfiguration;
 import timely.test.integration.OneWaySSLBase;
 import timely.util.JsonUtil;
 
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.UUID;
-import java.util.concurrent.TimeUnit;
-
-import static com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
+import com.fasterxml.jackson.databind.JavaType;
 
 @Category(IntegrationTest.class)
 public class WebSocketIT extends OneWaySSLBase {
@@ -163,13 +186,18 @@ public class WebSocketIT extends OneWaySSLBase {
     private ClientHandler handler = null;
     private Server s = null;
     private String sessionId = null;
+    private UsernamePasswordAuthenticationToken token = new UsernamePasswordAuthenticationToken("test", "test1");
 
     @Before
     public void setup() throws Exception {
         s = new Server(conf);
         s.run();
+
+        Connector con = mac.getConnector("root", "secret");
+        con.securityOperations().changeUserAuthorizations("root", new Authorizations("A", "B", "C", "D", "E", "F"));
+
         this.sessionId = UUID.randomUUID().toString();
-        AuthCache.getCache().put(sessionId, new UsernamePasswordAuthenticationToken("test", "test1"));
+        AuthCache.getCache().put(sessionId, token);
         group = new NioEventLoopGroup();
         SslContext ssl = SslContextBuilder.forClient().trustManager(InsecureTrustManagerFactory.INSTANCE).build();
 
@@ -208,11 +236,12 @@ public class WebSocketIT extends OneWaySSLBase {
             ch.writeAndFlush(new TextWebSocketFrame(JsonUtil.getObjectMapper().writeValueAsString(c)));
 
             // Add some data
-            put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2 rack=r1", "sys.cpu.user " + TEST_TIME
-                    + " 1.0 tag3=value3 rack=r2", "sys.cpu.idle " + (TEST_TIME + 2)
-                    + " 1.0 tag3=value3 tag4=value4 rack=r1", "sys.cpu.idle " + (TEST_TIME + 2)
-                    + " 3.0 tag3=value3 tag4=value4 rack=r2");
-
+            // @formatter:off
+            put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2 rack=r1",
+            	"sys.cpu.user " + TEST_TIME + " 1.0 tag3=value3 rack=r2",
+            	"sys.cpu.idle " + (TEST_TIME + 2) + " 1.0 tag3=value3 tag4=value4 rack=r1",
+            	"sys.cpu.idle " + (TEST_TIME + 2) + " 3.0 tag3=value3 tag4=value4 rack=r2");
+            // @formatter:on
             // Latency in TestConfiguration is 2s, wait for it
             sleepUninterruptibly(TestConfiguration.WAIT_SECONDS, TimeUnit.SECONDS);
 
@@ -234,8 +263,11 @@ public class WebSocketIT extends OneWaySSLBase {
                     .tag(new Tag("tag1", "value1")).tag(new Tag("tag2", "value2")).build();
             Metric second = Metric.newBuilder().name("sys.cpu.user").value(TEST_TIME, 1.0D).tag(new Tag("rack", "r2"))
                     .tag(new Tag("tag3", "value3")).build();
-            for (String metrics : response) {
-                MetricResponse m = JsonUtil.getObjectMapper().readValue(metrics, MetricResponse.class);
+
+            Assert.assertEquals(1, response.size());
+            MetricResponses responses = JsonUtil.getObjectMapper().readValue(response.get(0), MetricResponses.class);
+            Assert.assertEquals(2, responses.size());
+            for (MetricResponse m : responses.getResponses()) {
                 Assert.assertTrue(m.equals(MetricResponse.fromMetric(first, subscriptionId))
                         || m.equals(MetricResponse.fromMetric(second, subscriptionId)));
             }
@@ -255,11 +287,12 @@ public class WebSocketIT extends OneWaySSLBase {
             ch.writeAndFlush(new TextWebSocketFrame(JsonUtil.getObjectMapper().writeValueAsString(c)));
 
             // Add some data
-            put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2 rack=r1", "sys.cpu.user " + TEST_TIME
-                    + " 1.0 tag3=value3 rack=r2", "sys.cpu.idle " + (TEST_TIME + 2)
-                    + " 1.0 tag3=value3 tag4=value4 rack=r1", "sys.cpu.idle " + (TEST_TIME + 2)
-                    + " 3.0 tag3=value3 tag4=value4 rack=r2");
-
+            // @formatter:off
+            put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2 rack=r1",
+            	"sys.cpu.user " + TEST_TIME + " 1.0 tag3=value3 rack=r2",
+            	"sys.cpu.idle " + (TEST_TIME + 2) + " 1.0 tag3=value3 tag4=value4 rack=r1",
+            	"sys.cpu.idle " + (TEST_TIME + 2) + " 3.0 tag3=value3 tag4=value4 rack=r2");
+            // @formatter:on
             // Latency in TestConfiguration is 2s, wait for it
             sleepUninterruptibly(TestConfiguration.WAIT_SECONDS, TimeUnit.SECONDS);
 
@@ -281,18 +314,22 @@ public class WebSocketIT extends OneWaySSLBase {
                     .tag(new Tag("tag1", "value1")).tag(new Tag("tag2", "value2")).build();
             Metric second = Metric.newBuilder().name("sys.cpu.user").value(TEST_TIME, 1.0D).tag(new Tag("rack", "r2"))
                     .tag(new Tag("tag3", "value3")).build();
-            for (String metrics : response) {
-                MetricResponse m = JsonUtil.getObjectMapper().readValue(metrics, MetricResponse.class);
+
+            Assert.assertEquals(1, response.size());
+            MetricResponses responses = JsonUtil.getObjectMapper().readValue(response.get(0), MetricResponses.class);
+            Assert.assertEquals(2, responses.size());
+            for (MetricResponse m : responses.getResponses()) {
                 Assert.assertTrue(m.equals(MetricResponse.fromMetric(first, subscriptionId))
                         || m.equals(MetricResponse.fromMetric(second, subscriptionId)));
             }
 
             // Add some more data
-            put("sys.cpu.user " + (TEST_TIME + 500) + " 6.0 tag1=value1 tag2=value2 rack=r1", "sys.cpu.user "
-                    + (TEST_TIME + 500) + " 7.0 tag3=value3 rack=r2", "sys.cpu.idle " + (TEST_TIME + 1000)
-                    + " 1.0 tag3=value3 tag4=value4 rack=r1", "sys.cpu.idle " + (TEST_TIME + 1000)
-                    + " 3.0 tag3=value3 tag4=value4 rack=r2");
-
+            // @formatter:off
+            put("sys.cpu.user " + (TEST_TIME + 500) + " 6.0 tag1=value1 tag2=value2 rack=r1",
+            	"sys.cpu.user " + (TEST_TIME + 500) + " 7.0 tag3=value3 rack=r2",
+            	"sys.cpu.idle " + (TEST_TIME + 1000) + " 1.0 tag3=value3 tag4=value4 rack=r1",
+            	"sys.cpu.idle " + (TEST_TIME + 1000) + " 3.0 tag3=value3 tag4=value4 rack=r2");
+            // @formatter:on
             // Latency in TestConfiguration is 2s, wait for it
             sleepUninterruptibly(TestConfiguration.WAIT_SECONDS, TimeUnit.SECONDS);
 
@@ -308,8 +345,11 @@ public class WebSocketIT extends OneWaySSLBase {
                     .tag(new Tag("tag1", "value1")).tag(new Tag("tag2", "value2")).build();
             second = Metric.newBuilder().name("sys.cpu.user").value(TEST_TIME + 500, 7.0D).tag(new Tag("rack", "r2"))
                     .tag(new Tag("tag3", "value3")).build();
-            for (String metrics : response) {
-                MetricResponse m = JsonUtil.getObjectMapper().readValue(metrics, MetricResponse.class);
+
+            Assert.assertEquals(1, response.size());
+            responses = JsonUtil.getObjectMapper().readValue(response.get(0), MetricResponses.class);
+            Assert.assertEquals(2, responses.size());
+            for (MetricResponse m : responses.getResponses()) {
                 Assert.assertTrue(m.equals(MetricResponse.fromMetric(first, subscriptionId))
                         || m.equals(MetricResponse.fromMetric(second, subscriptionId)));
             }
@@ -339,8 +379,11 @@ public class WebSocketIT extends OneWaySSLBase {
                     .tag(new Tag("rack", "r1")).tag(new Tag("tag3", "value3")).tag(new Tag("tag4", "value4")).build();
             Metric fourth = Metric.newBuilder().name("sys.cpu.idle").value(TEST_TIME + 1000, 3.0D)
                     .tag(new Tag("rack", "r2")).tag(new Tag("tag3", "value3")).tag(new Tag("tag4", "value4")).build();
-            for (String metrics : response) {
-                MetricResponse m = JsonUtil.getObjectMapper().readValue(metrics, MetricResponse.class);
+
+            Assert.assertEquals(1, response.size());
+            responses = JsonUtil.getObjectMapper().readValue(response.get(0), MetricResponses.class);
+            Assert.assertEquals(4, responses.size());
+            for (MetricResponse m : responses.getResponses()) {
                 Assert.assertTrue(m.equals(MetricResponse.fromMetric(first, subscriptionId))
                         || m.equals(MetricResponse.fromMetric(second, subscriptionId))
                         || m.equals(MetricResponse.fromMetric(third, subscriptionId))
@@ -368,6 +411,178 @@ public class WebSocketIT extends OneWaySSLBase {
         }
     }
 
+    @Test
+    public void testSubscriptionWorkflowWithEndTime() throws Exception {
+        try {
+            final String subscriptionId = "1234";
+            CreateSubscription c = new CreateSubscription();
+            c.setSubscriptionId(subscriptionId);
+            ch.writeAndFlush(new TextWebSocketFrame(JsonUtil.getObjectMapper().writeValueAsString(c)));
+
+            // Add some data
+            // @formatter:off
+            put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2 rack=r1",
+            	"sys.cpu.user " + TEST_TIME + " 1.0 tag3=value3 rack=r2",
+            	"sys.cpu.idle " + (TEST_TIME + 2) + " 1.0 tag3=value3 tag4=value4 rack=r1",
+            	"sys.cpu.idle " + (TEST_TIME + 2) + " 3.0 tag3=value3 tag4=value4 rack=r2");
+            // @formatter:on
+            // Latency in TestConfiguration is 2s, wait for it
+            sleepUninterruptibly(TestConfiguration.WAIT_SECONDS, TimeUnit.SECONDS);
+
+            // Add subscription, confirm data
+            AddSubscription add = new AddSubscription();
+            add.setSubscriptionId(subscriptionId);
+            add.setMetric("sys.cpu.user");
+            add.setEndTime(TEST_TIME + 1000);
+            add.setDelayTime(1000L);
+            ch.writeAndFlush(new TextWebSocketFrame(JsonUtil.getObjectMapper().writeValueAsString(add)));
+
+            List<String> response = handler.getResponses();
+            while (response.size() == 0 && handler.isConnected()) {
+                LOG.info("Waiting for web socket response");
+                sleepUninterruptibly(500, TimeUnit.MILLISECONDS);
+                response = handler.getResponses();
+            }
+
+            Metric first = Metric.newBuilder().name("sys.cpu.user").value(TEST_TIME, 1.0D).tag(new Tag("rack", "r1"))
+                    .tag(new Tag("tag1", "value1")).tag(new Tag("tag2", "value2")).build();
+            Metric second = Metric.newBuilder().name("sys.cpu.user").value(TEST_TIME, 1.0D).tag(new Tag("rack", "r2"))
+                    .tag(new Tag("tag3", "value3")).build();
+
+            Assert.assertEquals(1, response.size());
+            MetricResponses responses = JsonUtil.getObjectMapper().readValue(response.get(0), MetricResponses.class);
+            Assert.assertEquals(3, responses.size());
+            for (MetricResponse m : responses.getResponses()) {
+                Assert.assertTrue(m.equals(MetricResponse.fromMetric(first, subscriptionId))
+                        || m.equals(MetricResponse.fromMetric(second, subscriptionId)) || m.isComplete());
+            }
+
+            // Add some more data
+            // @formatter:off
+            put("sys.cpu.user " + (TEST_TIME + 500) + " 6.0 tag1=value1 tag2=value2 rack=r1",
+            	"sys.cpu.user " + (TEST_TIME + 500) + " 7.0 tag3=value3 rack=r2",
+            	"sys.cpu.idle " + (TEST_TIME + 1000) + " 1.0 tag3=value3 tag4=value4 rack=r1",
+            	"sys.cpu.idle " + (TEST_TIME + 1000) + " 3.0 tag3=value3 tag4=value4 rack=r2");
+            // @formatter:on
+            // Latency in TestConfiguration is 2s, wait for it
+            sleepUninterruptibly(TestConfiguration.WAIT_SECONDS, TimeUnit.SECONDS);
+
+            // Delay time on the metric scanner is 1 second, the metric
+            // scanner should be exhausted on the backend due to the presence
+            // of the end time. Wait 2 seconds then exit.
+            LOG.info("Waiting for web socket response");
+            sleepUninterruptibly(2, TimeUnit.SECONDS);
+            response = handler.getResponses();
+
+            // confirm no data - the metric scanner ran once on the back end
+            Assert.assertEquals(0, response.size());
+
+            // Remove subscriptions to metric
+            RemoveSubscription remove1 = new RemoveSubscription();
+            remove1.setSubscriptionId(subscriptionId);
+            remove1.setMetric("sys.cpu.user");
+            ch.writeAndFlush(new TextWebSocketFrame(JsonUtil.getObjectMapper().writeValueAsString(remove1)));
+
+            // Close subscription
+            CloseSubscription close = new CloseSubscription();
+            close.setSubscriptionId(subscriptionId);
+            ch.writeAndFlush(new TextWebSocketFrame(JsonUtil.getObjectMapper().writeValueAsString(close)));
+        } finally {
+            ch.close().sync();
+            s.shutdown();
+            group.shutdownGracefully();
+        }
+    }
+
+    @Test
+    public void testSubscriptionWorkflowWithEndTimeAndVisibilities() throws Exception {
+
+        Authentication auth = AuthenticationService.getAuthenticationManager().authenticate(token);
+        AuthCache.getCache().put(sessionId, auth);
+
+        try {
+            final String subscriptionId = "1234";
+            CreateSubscription c = new CreateSubscription();
+            c.setSubscriptionId(subscriptionId);
+            ch.writeAndFlush(new TextWebSocketFrame(JsonUtil.getObjectMapper().writeValueAsString(c)));
+
+            // Add some data
+            // @formatter:off
+            put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2 rack=r1 viz=A",
+            	"sys.cpu.user " + TEST_TIME + " 1.0 tag3=value3 rack=r2 viz=A",
+            	"sys.cpu.idle " + (TEST_TIME + 2) + " 1.0 tag3=value3 tag4=value4 rack=r1 viz=A",
+            	"sys.cpu.idle " + (TEST_TIME + 2) + " 3.0 tag3=value3 tag4=value4 rack=r2 viz=A");
+            // @formatter:on
+            // Latency in TestConfiguration is 2s, wait for it
+            sleepUninterruptibly(TestConfiguration.WAIT_SECONDS, TimeUnit.SECONDS);
+
+            // Add subscription, confirm data
+            AddSubscription add = new AddSubscription();
+            add.setSubscriptionId(subscriptionId);
+            add.setMetric("sys.cpu.user");
+            add.setDelayTime(1000L);
+            add.setEndTime(TEST_TIME + 1000);
+
+            ch.writeAndFlush(new TextWebSocketFrame(JsonUtil.getObjectMapper().writeValueAsString(add)));
+
+            List<String> response = handler.getResponses();
+            while (response.size() == 0 && handler.isConnected()) {
+                LOG.info("Waiting for web socket response");
+                sleepUninterruptibly(500, TimeUnit.MILLISECONDS);
+                response = handler.getResponses();
+            }
+
+            Metric first = Metric.newBuilder().name("sys.cpu.user").value(TEST_TIME, 1.0D).tag(new Tag("rack", "r1"))
+                    .tag(new Tag("tag1", "value1")).tag(new Tag("tag2", "value2")).tag(new Tag("viz", "A")).build();
+            Metric second = Metric.newBuilder().name("sys.cpu.user").value(TEST_TIME, 1.0D).tag(new Tag("rack", "r2"))
+                    .tag(new Tag("tag3", "value3")).tag(new Tag("viz", "A")).build();
+
+            Assert.assertEquals(1, response.size());
+            MetricResponses responses = JsonUtil.getObjectMapper().readValue(response.get(0), MetricResponses.class);
+            Assert.assertEquals(3, responses.size());
+            for (MetricResponse m : responses.getResponses()) {
+                Assert.assertTrue(m.equals(MetricResponse.fromMetric(first, subscriptionId))
+                        || m.equals(MetricResponse.fromMetric(second, subscriptionId)) || m.isComplete());
+            }
+
+            // Add some more data
+            // @formatter:off
+            put("sys.cpu.user " + (TEST_TIME + 500) + " 6.0 tag1=value1 tag2=value2 rack=r1, viz=A",
+            	"sys.cpu.user " + (TEST_TIME + 500) + " 7.0 tag3=value3 rack=r2, viz=A",
+            	"sys.cpu.idle " + (TEST_TIME + 1000) + " 1.0 tag3=value3 tag4=value4 rack=r1, viz=A",
+            	"sys.cpu.idle " + (TEST_TIME + 1000) + " 3.0 tag3=value3 tag4=value4 rack=r2, viz=A");
+            // @formatter:on
+            // Latency in TestConfiguration is 2s, wait for it
+            sleepUninterruptibly(TestConfiguration.WAIT_SECONDS, TimeUnit.SECONDS);
+
+            // Delay time on the metric scanner is 1 second, the metric
+            // scanner should be exhausted on the backend due to the presence
+            // of the end time. Wait 2 seconds then exit.
+            LOG.info("Waiting for web socket response");
+            sleepUninterruptibly(2, TimeUnit.SECONDS);
+            response = handler.getResponses();
+
+            // confirm no data - the metric scanner ran once on the back end
+            Assert.assertEquals(0, response.size());
+
+            // Remove subscriptions to metric
+            RemoveSubscription remove1 = new RemoveSubscription();
+            remove1.setSubscriptionId(subscriptionId);
+            remove1.setMetric("sys.cpu.user");
+            ch.writeAndFlush(new TextWebSocketFrame(JsonUtil.getObjectMapper().writeValueAsString(remove1)));
+
+            // Close subscription
+            CloseSubscription close = new CloseSubscription();
+            close.setSubscriptionId(subscriptionId);
+            ch.writeAndFlush(new TextWebSocketFrame(JsonUtil.getObjectMapper().writeValueAsString(close)));
+        } finally {
+            AuthCache.getCache().asMap().remove(sessionId, auth);
+            ch.close().sync();
+            s.shutdown();
+            group.shutdownGracefully();
+        }
+    }
+
     @Test
     public void testWSAggregators() throws Exception {
         try {
@@ -419,9 +634,9 @@ public class WebSocketIT extends OneWaySSLBase {
         try {
             // @formatter:off
             put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2",
-            		"sys.cpu.user " + (TEST_TIME + 1000) + " 3.0 tag1=value1 tag2=value2",
-            		"sys.cpu.user " + (TEST_TIME + 2000) + " 2.0 tag1=value1 tag3=value3 viz=secret");
-         // @formatter:on
+            	"sys.cpu.user " + (TEST_TIME + 1000) + " 3.0 tag1=value1 tag2=value2",
+            	"sys.cpu.user " + (TEST_TIME + 2000) + " 2.0 tag1=value1 tag3=value3 viz=secret");
+            // @formatter:on
             // Latency in TestConfiguration is 2s, wait for it
             sleepUninterruptibly(TestConfiguration.WAIT_SECONDS, TimeUnit.SECONDS);
 
@@ -466,7 +681,7 @@ public class WebSocketIT extends OneWaySSLBase {
             assertEquals("sys.cpu.user", query.getMetric());
             assertEquals(1, query.getTags().size());
             assertTrue(query.getTags().containsKey("tag1"));
-            assertEquals("value1", query.getTags().get("tag1"));
+            assertEquals(".*", query.getTags().get("tag1"));
             assertEquals(1, query.getDps().size()); // this is a rate query and
                                                     // we can only see 2 dps, we
                                                     // will only get 1 response.
@@ -481,9 +696,12 @@ public class WebSocketIT extends OneWaySSLBase {
     @Test
     public void testWSLookup() throws Exception {
         try {
-            put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2", "sys.cpu.user " + (TEST_TIME + 1)
-                    + " 1.0 tag3=value3", "sys.cpu.idle " + (TEST_TIME + 1) + " 1.0 tag3=value3 tag4=value4",
-                    "sys.cpu.idle " + (TEST_TIME + 2) + " 1.0 tag3=value3 tag4=value4");
+            // @formatter:off
+            put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2",
+            	"sys.cpu.user " + (TEST_TIME + 1) + " 1.0 tag3=value3",
+            	"sys.cpu.idle " + (TEST_TIME + 1) + " 1.0 tag3=value3 tag4=value4",
+                "sys.cpu.idle " + (TEST_TIME + 2) + " 1.0 tag3=value3 tag4=value4");
+            // @formatter:on
             // Latency in TestConfiguration is 2s, wait for it
             sleepUninterruptibly(TestConfiguration.WAIT_SECONDS, TimeUnit.SECONDS);
 
@@ -523,9 +741,12 @@ public class WebSocketIT extends OneWaySSLBase {
     @Test
     public void testWSSuggest() throws Exception {
         try {
-            put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2", "sys.cpu.idle " + (TEST_TIME + 1)
-                    + " 1.0 tag3=value3 tag4=value4", "sys.cpu.idle " + (TEST_TIME + 2)
-                    + " 1.0 tag3=value3 tag4=value4", "zzzz 1234567892 1.0 host=localhost");
+            // @formatter:off
+            put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2",
+            	"sys.cpu.idle " + (TEST_TIME + 1) + " 1.0 tag3=value3 tag4=value4",
+            	"sys.cpu.idle " + (TEST_TIME + 2) + " 1.0 tag3=value3 tag4=value4",
+            	"zzzz 1234567892 1.0 host=localhost");
+            // @formatter:off
             // Latency in TestConfiguration is 2s, wait for it
             sleepUninterruptibly(TestConfiguration.WAIT_SECONDS, TimeUnit.SECONDS);
 

server/src/test/resources/log4j2.xml

@@ -9,6 +9,7 @@
     <Logger name="org.apache.accumulo.core.client.impl.ThriftTransportPool" level="OFF" additivity="false" />
     <Logger name="org.apache.accumulo.fate.zookeeper.ZooCache" level="OFF" additivity="false" />
     <Logger name="org.apache.zookeeper" level="OFF" additivity="false" />
+<!--     <Logger name="javax.net.debug" level="ALL" additivity="false"/> -->
 <!--     <Logger name="sun.net.www.protocol.http.HttpURLConnection" level="ALL" additivity="false" /> -->
     <Root level="TRACE">
       <AppenderRef ref="CONSOLE"/>