Add setup.py to .gitignore, add logging for metrics that fail to parse

analytics/pom.xml

@@ -23,7 +23,7 @@ under the License.
     <parent>
         <groupId>timely</groupId>
         <artifactId>timely</artifactId>
-        <version>1.0.0</version>
+        <version>0.0.7-SNAPSHOT</version>
     </parent>
     <properties>
         <flink.version>1.1.4</flink.version>

client/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>timely</groupId>
         <artifactId>timely</artifactId>
-        <version>1.0.0</version>
+        <version>0.0.7-SNAPSHOT</version>
     </parent>
     <properties>
         <!-- PEP 440 compliant version gets supstituted into src/main/python/template/setup.py -->

client/src/main/java/timely/model/parse/TagListParser.java

@@ -1,5 +1,6 @@
 package timely.model.parse;
 
+import com.google.common.base.Splitter;
 import timely.model.Tag;
 
 import java.util.ArrayList;
@@ -14,22 +15,13 @@ import java.util.Map;
  */
 public class TagListParser implements Parser<List<Tag>>, Combiner<Collection<Tag>> {
 
+    private static final Splitter commaSplitter = Splitter.on(",").omitEmptyStrings();
     private static final TagParser tagParser = new TagParser();
 
     @Override
     public List<Tag> parse(String t) {
         ArrayList<Tag> tags = new ArrayList<>();
-        // only split on commas that are not preceded with a backslash
-        String[] parts = t.split("(?<!\\\\),");
-        for (String p : parts) {
-            if (!p.isEmpty()) {
-                Tag tag = tagParser.parse(p);
-                // unescape commas in the key and value of the tag
-                tag.setKey(tag.getKey().replaceAll("\\\\,", ","));
-                tag.setValue(tag.getValue().replaceAll("\\\\,", ","));
-                tags.add(tag);
-            }
-        }
+        commaSplitter.splitToList(t).forEach(p -> tags.add(tagParser.parse(p)));
         return tags;
     }
 

client/src/main/java/timely/model/parse/TagParser.java

@@ -12,7 +12,7 @@ import java.util.List;
  */
 public class TagParser implements Parser<Tag>, Combiner<Tag> {
 
-    private static final Splitter equalSplitter = Splitter.on("=").limit(2).trimResults();
+    private static final Splitter equalSplitter = Splitter.on("=").trimResults();
     private static final Joiner equalsJoiner = Joiner.on("=");
 
     @Override

client/src/test/java/timely/model/TagListParserTest.java

@@ -1,8 +1,7 @@
-package timely.model.parse;
+package timely.model;
 
 import org.junit.Assert;
 import org.junit.Test;
-import timely.model.Tag;
 import timely.model.parse.TagListParser;
 
 import java.util.ArrayList;
@@ -38,15 +37,4 @@ public class TagListParserTest {
         String combined = new TagListParser().combine(map);
         Assert.assertEquals("tag1=value1,tag2=value2", combined);
     }
-
-    @Test
-    public void testParseTagsWithCommas() {
-
-        try {
-            String s = "tag1=value1,tag2=3.4.3_(default\\,_Date\\,_Time)_";
-            new TagListParser().parse(s);
-        } catch (Exception e) {
-            Assert.fail(e.getMessage());
-        }
-    }
 }

client/src/test/java/timely/model/parse/MetricParserTest.java

@@ -1,40 +0,0 @@
-package timely.model.parse;
-
-import org.junit.Assert;
-import org.junit.Test;
-import timely.model.Metric;
-import timely.model.Tag;
-
-import java.util.ArrayList;
-import java.util.List;
-
-public class MetricParserTest {
-
-    @Test
-    public void testParseWithEscapedCharacters() {
-
-        MetricParser parser = new MetricParser();
-        Metric m = parser.parse("put mymetric 12341234 5.0 tag1=value1,value1 tag2=value2=value2");
-
-        Assert.assertEquals("mymetric", m.getName());
-        Assert.assertEquals(12341234, (long) m.getValue().getTimestamp());
-        Assert.assertEquals(5.0, (double) m.getValue().getMeasure(), 0);
-        List<Tag> expected = new ArrayList<>();
-        expected.add(new Tag("tag1", "value1,value1"));
-        expected.add(new Tag("tag2", "value2=value2"));
-        Assert.assertEquals(expected, m.getTags());
-    }
-
-    @Test
-    public void testParseMalformatted() {
-
-        MetricParser parser = new MetricParser();
-        try {
-            // parser should throw an exception
-            parser.parse("put mymetric 12341234 5.0 tag1 tag2=value2");
-            Assert.fail();
-        } catch (IllegalArgumentException e) {
-
-        }
-    }
-}

collectd-nsq-plugin/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>timely</groupId>
         <artifactId>timely</artifactId>
-        <version>1.0.0</version>
+        <version>0.0.7-SNAPSHOT</version>
     </parent>
     <artifactId>collectd-nsq-plugin</artifactId>
     <dependencies>

collectd-plugin-parent/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>timely</groupId>
         <artifactId>timely</artifactId>
-        <version>1.0.0</version>
+        <version>0.0.7-SNAPSHOT</version>
     </parent>
     <artifactId>collectd-plugin-parent</artifactId>
     <dependencies>

collectd-timely-plugin/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>timely</groupId>
         <artifactId>timely</artifactId>
-        <version>1.0.0</version>
+        <version>0.0.7-SNAPSHOT</version>
     </parent>
     <artifactId>collectd-timely-plugin</artifactId>
     <dependencies>

common/pom.xml

@@ -3,7 +3,7 @@
   <parent>
     <groupId>timely</groupId>
     <artifactId>timely</artifactId>
-    <version>1.0.0</version>
+    <version>0.0.7-SNAPSHOT</version>
   </parent>
   <artifactId>timely-common</artifactId>
   <dependencies>

grafana/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>timely</groupId>
         <artifactId>timely</artifactId>
-        <version>1.0.0</version>
+        <version>0.0.7-SNAPSHOT</version>
     </parent>
     <build>
         <plugins>

pom.xml

@@ -3,7 +3,7 @@
     <modelVersion>4.0.0</modelVersion>
     <groupId>timely</groupId>
     <artifactId>timely</artifactId>
-    <version>1.0.0</version>
+    <version>0.0.7-SNAPSHOT</version>
     <packaging>pom</packaging>
     <name>Timely</name>
     <description>A secure and scalable time series database based on Apache Accumulo</description>

server/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>timely</groupId>
         <artifactId>timely</artifactId>
-        <version>1.0.0</version>
+        <version>0.0.7-SNAPSHOT</version>
     </parent>
     <dependencies>
         <dependency>

server/src/main/java/timely/adapter/accumulo/MetricAdapter.java

@@ -17,7 +17,6 @@ import timely.model.parse.TagParser;
 
 import java.nio.ByteBuffer;
 import java.nio.charset.Charset;
-import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import java.util.Optional;
@@ -48,26 +47,10 @@ public class MetricAdapter {
         return timestamp - (timestamp % 3600000) + 3600000;
     }
 
-    private static List<Tag> escapeDelimiters(List<Tag> tags) {
-        List<Tag> newTags = new ArrayList<>();
-        for (Tag t : tags) {
-            // escape all commas and equals in the tag key and value since they
-            // are used as a delimiter
-            String k = t.getKey();
-            String v = t.getValue();
-            k = k.replaceAll("=", "\\=");
-            k = k.replaceAll(",", "\\,");
-            v = v.replaceAll("=", "\\=");
-            v = v.replaceAll(",", "\\,");
-            newTags.add(new Tag(k, v));
-        }
-        return newTags;
-    }
-
     public static Mutation toMutation(Metric metric) {
         final Mutation mutation = new Mutation(encodeRowKey(metric));
+
         List<Tag> tags = metric.getTags();
-        tags = escapeDelimiters(tags);
         Collections.sort(tags);
 
         for (final Tag entry : tags) {

server/src/main/java/timely/api/request/MetricRequest.java

@@ -58,7 +58,6 @@ public class MetricRequest implements TcpRequest, HttpPostRequest, WebSocketRequ
             metric = metricParser.parse(line);
         } catch (Exception e) {
             LOG.error("Error parsing metric: {}", line);
-            throw e;
         }
     }
 

server/src/main/java/timely/api/response/timeseries/QueryResponse.java

@@ -2,9 +2,9 @@ package timely.api.response.timeseries;
 
 import java.util.ArrayList;
 import java.util.HashMap;
-import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.TreeMap;
 
 import org.apache.commons.lang3.builder.ToStringBuilder;
 
@@ -13,7 +13,7 @@ public class QueryResponse {
     private String metric;
     private Map<String, String> tags = new HashMap<>();
     private List<String> aggregatedTags = new ArrayList<>();
-    private Map<String, Object> dps = new LinkedHashMap<>();
+    private Map<String, Object> dps = new TreeMap<>();
 
     public String getMetric() {
         return metric;

server/src/main/java/timely/sample/iterators/DownsampleIterator.java

@@ -72,39 +72,30 @@ public class DownsampleIterator extends WrappingIterator {
         if (super.hasTop()) {
             Key topKey = super.getTopKey();
             Value topValue = super.getTopValue();
-            Metric metric = null;
-            try {
-                metric = MetricAdapter.parse(topKey, topValue);
-                LOG.trace("entering hasTop() for metric=" + metric.toString());
-            } catch (Exception e) {
-                LOG.error("Error: {} parsing metric at key: {}", e.getMessage(), topKey.toString());
-            }
+            Metric metric = MetricAdapter.parse(topKey, topValue);
+            LOG.trace("entering hasTop() for metric=" + metric.toString());
             while (super.hasTop()) {
                 topKey = super.getTopKey();
                 topValue = super.getTopValue();
-                try {
-                    metric = MetricAdapter.parse(topKey, topValue);
-                    long timestamp = metric.getValue().getTimestamp();
-                    if (memoryEstimator.shouldReturnBasedOnMemoryUsage(timestamp, value)) {
-                        LOG.trace("returning current values - memory usage > " + memoryEstimator.maxDownsampleMemory
-                                + " for metric=" + metric.toString());
-                        break;
-                    }
-                    last = topKey;
-
-                    Set<Tag> tags = new HashSet<Tag>(metric.getTags());
-                    Downsample sample = value.get(tags);
-                    if (sample == null) {
-                        try {
-                            value.put(tags, sample = factory.create());
-                        } catch (TimelyException e) {
-                            throw new RuntimeException(e);
-                        }
-                    }
-                    sample.add(metric.getValue().getTimestamp(), metric.getValue().getMeasure());
-                } catch (Exception e) {
-                    LOG.error("Error: {} parsing metric at key: {}", e.getMessage(), topKey.toString());
+                metric = MetricAdapter.parse(topKey, topValue);
+                long timestamp = metric.getValue().getTimestamp();
+                if (memoryEstimator.shouldReturnBasedOnMemoryUsage(timestamp, value)) {
+                    LOG.trace("returning current values - memory usage > " + memoryEstimator.maxDownsampleMemory
+                            + " for metric=" + metric.toString());
+                    break;
                 }
+                last = topKey;
+
+                Set<Tag> tags = new HashSet<Tag>(metric.getTags());
+                Downsample sample = value.get(tags);
+                if (sample == null) {
+                    try {
+                        value.put(tags, sample = factory.create());
+                    } catch (TimelyException e) {
+                        throw new RuntimeException(e);
+                    }
+                }
+                sample.add(metric.getValue().getTimestamp(), metric.getValue().getMeasure());
                 try {
                     super.next();
                 } catch (IOException e) {

server/src/main/java/timely/store/DataStoreImpl.java

@@ -6,25 +6,8 @@ import io.netty.handler.codec.http.HttpResponseStatus;
 
 import java.io.IOException;
 import java.nio.charset.Charset;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.EnumSet;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
 import java.util.Map.Entry;
-import java.util.Set;
-import java.util.SortedMap;
-import java.util.Timer;
-import java.util.TimerTask;
-import java.util.TreeMap;
-import java.util.TreeSet;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicReference;

server/src/main/java/timely/store/iterators/TimeSeriesGroupingIterator.java

@@ -237,27 +237,23 @@ public class TimeSeriesGroupingIterator extends WrappingIterator {
             }
             if (time.equals(newTime)) {
 
-                try {
-                    Metric m = MetricAdapter.parse(k, super.getTopValue());
-                    timely.model.Value v = m.getValue();
-                    m.setValue(null);
+                Metric m = MetricAdapter.parse(k, super.getTopValue());
+                timely.model.Value v = m.getValue();
+                m.setValue(null);
 
-                    TimeSeries values = series.get(m);
-                    if (null == values) {
-                        LOG.trace("Creating new time series {}", m);
-                        values = new TimeSeries(this, filters.length);
-                    }
-
-                    LOG.trace("Adding value {} to series {}", v.getMeasure(), m);
-                    values.add(k, v.getMeasure());
-
-                    // always re-put the metric back into the TimeSeriesGroup as
-                    // it
-                    // maintains a list of prepared answers
-                    series.put(m, values);
-                } catch (Exception e) {
-                    LOG.error("Error: {} parsing metric at key: {}", e.getMessage(), k.toString());
+                TimeSeries values = series.get(m);
+                if (null == values) {
+                    LOG.trace("Creating new time series {}", m);
+                    values = new TimeSeries(this, filters.length);
                 }
+
+                LOG.trace("Adding value {} to series {}", v.getMeasure(), m);
+                values.add(k, v.getMeasure());
+
+                // always re-put the metric back into the TimeSeriesGroup as it
+                // maintains a list of prepared answers
+                series.put(m, values);
+
                 super.next();
             } else {
                 break;

server/src/main/java/timely/subscription/MetricScanner.java

@@ -140,15 +140,11 @@ public class MetricScanner extends Thread implements UncaughtExceptionHandler {
 
                 if (this.iter.hasNext()) {
                     Entry<Key, Value> e = this.iter.next();
-                    try {
-                        m = MetricAdapter.parse(e.getKey(), e.getValue(), true);
-                        if (responses.size() >= this.subscriptionBatchSize) {
-                            flush();
-                        }
-                        this.responses.addResponse(MetricResponse.fromMetric(m, this.subscriptionId));
-                    } catch (Exception e1) {
-                        LOG.error("Error {} parsing metric at key: {}", e1.getMessage(), e.getKey().toString());
+                    m = MetricAdapter.parse(e.getKey(), e.getValue(), true);
+                    if (responses.size() >= this.subscriptionBatchSize) {
+                        flush();
                     }
+                    this.responses.addResponse(MetricResponse.fromMetric(m, this.subscriptionId));
                 } else if (rangeItr.hasNext()) {
                     // set next range on the scanner
                     Range r = rangeItr.next();

server/src/test/java/timely/api/model/MetricAdapterTest.java

@@ -132,27 +132,6 @@ public class MetricAdapterTest {
         Assert.assertEquals(2.0D, m.getValue().getMeasure(), 0.0D);
     }
 
-    @Test
-    public void testParseWithEscapes() throws Exception {
-        PairLexicoder<String, Long> rowCoder = new PairLexicoder<>(new StringLexicoder(), new LongLexicoder());
-        byte[] row = rowCoder.encode(new ComparablePair<>("sys.cpu.user", 1000L));
-        byte[] value = new byte[Double.BYTES];
-        ByteBuffer.wrap(value).putDouble(2.0D);
-        PairLexicoder<Long, String> colQualCoder = new PairLexicoder<>(new LongLexicoder(), new StringLexicoder());
-        Key k = new Key(row, "tag1=value1".getBytes(), colQualCoder.encode(new ComparablePair<>(new Long(1000),
-                "tag2=test\\,value2,tag3=val\\=ue3")), "(a&b)|(c&d)".getBytes(), 1000);
-        Value v = new Value(value);
-        Metric m = MetricAdapter.parse(k, v);
-        Assert.assertEquals("sys.cpu.user", m.getName());
-        List<Tag> tags = new ArrayList<>();
-        tags.add(new Tag("tag1=value1"));
-        tags.add(new Tag("tag2=test,value2"));
-        tags.add(new Tag("tag3=val\\=ue3"));
-        Assert.assertEquals(tags, m.getTags());
-        Assert.assertEquals(new Long(1000), m.getValue().getTimestamp());
-        Assert.assertEquals(2.0D, m.getValue().getMeasure(), 0.0D);
-    }
-
     @Test
     public void testParseWithViz() throws Exception {
         PairLexicoder<String, Long> rowCoder = new PairLexicoder<>(new StringLexicoder(), new LongLexicoder());