Compare commits

...
27 changed files with 540 additions and 481 deletions

No files matched your search

@@ -22,7 +22,7 @@ import timely.model.parse.TagParser;
@JsonRootName("metric")
@JsonInclude(JsonInclude.Include.NON_NULL)
@JsonPropertyOrder({ "name" })
public class Metric implements Serializable {
public class Metric implements Serializable, ObjectSizeOf {
private static final long serialVersionUID = 1L;
@@ -186,4 +186,9 @@ public class Metric implements Serializable {
return this.metric;
}
}
@Override
public long sizeInBytes() {
return 0;
}
}
@@ -3,11 +3,15 @@ package timely.model;
import java.lang.reflect.Array;
import java.lang.reflect.Field;
import java.lang.reflect.Modifier;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.Stack;
import java.util.logging.Logger;
import org.apache.commons.lang3.StringUtils;
/**
* A simple interface that objects can implement to return the object size.
*/
@@ -21,15 +25,29 @@ public interface ObjectSizeOf {
public static class ObjectInstance {
private Object o;
private String name = "";
public ObjectInstance(Object _o) {
this.o = _o;
}
public ObjectInstance(Object _o, String name) {
this.o = _o;
this.name = name;
}
public Object getObject() {
return o;
}
public void setName(String name) {
this.name = name;
}
public String getName() {
return name;
}
@Override
public int hashCode() {
return System.identityHashCode(o);
@@ -63,103 +81,134 @@ public interface ObjectSizeOf {
*
* @return an approximation of the object size
*/
public static long getObjectSize(Object o, boolean useSizeInBytesMethod, boolean debug) {
long estSize = getObjectSize(o, new HashSet<ObjectInstance>(), new Stack<ObjectInstance>(),
useSizeInBytesMethod, debug);
return estSize;
}
public static long getObjectSize(Object o) {
long estSize = getObjectSize(o, new HashSet<ObjectInstance>(), new Stack<ObjectInstance>(), true, false);
return estSize;
return getObjectSize(o, true, false);
}
public static long getObjectSize(Object o, Set<ObjectInstance> visited, Stack<ObjectInstance> stack,
boolean useSizeInBytesMethod, boolean debug) {
long totalSize = 0;
stack.add(new ObjectInstance(o));
while (!stack.isEmpty()) {
long size = 0;
ObjectInstance oi = stack.pop();
o = oi.getObject();
if (o != null && !visited.contains(oi)) {
visited.add(oi);
try {
if (useSizeInBytesMethod) {
if (o instanceof ObjectSizeOf) {
size = ((ObjectSizeOf) o).sizeInBytes();
public static long getObjectSize(Object o, boolean useSizeInBytesMethod, boolean debug) {
List<String> output = null;
if (debug) {
output = new ArrayList<>();
}
long size = getObjectSize(new ObjectInstance(o, o.getClass().getSimpleName()), new HashSet<>(), 0,
useSizeInBytesMethod, output);
if (debug) {
Collections.reverse(output);
for (String s : output)
System.out.println(s);
}
return size;
}
public static long getObjectSize(ObjectInstance oi, Set<ObjectInstance> visited, int indent,
boolean useSizeInBytesMethod, List<String> output) {
long size = 0;
Object o = oi.getObject();
int arrayOverhead = 0;
int objectOverhead = 0;
int reference = 0;
boolean roundUp = false;
if (o != null && !visited.contains(oi)) {
visited.add(oi);
try {
if (useSizeInBytesMethod) {
if (o instanceof ObjectSizeOf) {
size = ((ObjectSizeOf) o).sizeInBytes();
if (output != null && size > 0) {
output.add(StringUtils.repeat(' ', indent) + oi.getName() + " [" + size + "] ("
+ o.getClass().getCanonicalName() + ") [ObjectSizeOf]");
}
}
if (size == 0) {
// the hard way...
// do not include Class related objects or
// reflection objects
if ((o instanceof Class) || (o instanceof ClassLoader) || (o.getClass().getPackage() != null
&& o.getClass().getPackage().getName().startsWith("java.lang.reflect"))) {
size = 0;
} else if (o instanceof Number || o instanceof Boolean || o instanceof Character) {
size = NUMBER_SIZE;
} else {
// lets do a simple sizing
Class<?> c = o.getClass();
if (c.isArray()) {
size = ARRAY_OVERHEAD;
int length = Array.getLength(o);
if (c.getComponentType().isPrimitive()) {
size += length * getPrimitiveObjectSize(c.getComponentType());
} else {
size += length * REFERENCE;
for (int i = 0; i < length; i++) {
Object element = Array.get(o, i);
if (element != null) {
stack.add(new ObjectInstance(element));
}
}
}
}
if (size == 0) {
// the hard way...
// do not include Class related objects or
// reflection objects
if ((o instanceof Class) || (o instanceof ClassLoader) || (o.getClass().getPackage() != null
&& o.getClass().getPackage().getName().startsWith("java.lang.reflect"))) {
size = 0;
} else if (o instanceof Number || o instanceof Boolean || o instanceof Character) {
size = NUMBER_SIZE;
} else {
// lets do a simple sizing
Class<?> c = o.getClass();
if (c.isArray()) {
arrayOverhead++;
int length = Array.getLength(o);
if (c.getComponentType().isPrimitive()) {
size += length * getPrimitiveObjectSize(c.getComponentType());
} else {
size += OBJECT_OVERHEAD;
while (c != null) {
for (Field field : c.getDeclaredFields()) {
if (Modifier.isStatic(field.getModifiers())) {
continue;
}
if (field.getType().isPrimitive()) {
size += getPrimitiveObjectSize(field.getType());
} else {
size += REFERENCE;
boolean accessible = field.isAccessible();
field.setAccessible(true);
try {
Object fieldObject = field.get(o);
if (fieldObject != null) {
stack.push(new ObjectInstance(fieldObject));
}
} catch (Exception e) {
// cannot get to field, so
// ignore it in this size
// calculation
e.printStackTrace();
}
field.setAccessible(accessible);
}
reference += length;
for (int i = 0, index = length - 1; i < length; i++, index--) {
Object element = Array.get(o, index);
if (element != null) {
size += getObjectSize(
new ObjectInstance(element, oi.getName() + "[" + index + "]"),
visited, indent + 2, useSizeInBytesMethod, output);
}
c = c.getSuperclass();
}
}
size = roundUp(size);
} else {
objectOverhead++;
while (c != null) {
for (Field field : c.getDeclaredFields()) {
if (Modifier.isStatic(field.getModifiers())) {
continue;
}
if (field.getType().isPrimitive()) {
long primSize = getPrimitiveObjectSize(field.getType());
if (output != null) {
output.add(StringUtils.repeat(' ', indent + 2) + field.getName() + " ["
+ primSize + "] (" + field.getType().getCanonicalName() + ")");
}
size += primSize;
} else {
reference++;
boolean accessible = field.isAccessible();
field.setAccessible(true);
try {
Object fieldObject = field.get(o);
if (fieldObject != null) {
size += getObjectSize(
new ObjectInstance(fieldObject, field.getName()), visited,
indent + 2, useSizeInBytesMethod, output);
}
} catch (Exception e) {
// cannot get to field, so
// ignore it in this size
// calculation
e.printStackTrace();
}
field.setAccessible(accessible);
}
}
c = c.getSuperclass();
}
}
roundUp = true;
}
} catch (Throwable t) {
log.warning("Unable to determine object size for " + o);
}
if (debug) {
System.out.println(o.getClass().getCanonicalName() + "=" + size);
long underlyingObjects = size;
long overheads = REFERENCE * reference + OBJECT_OVERHEAD * objectOverhead
+ ARRAY_OVERHEAD * arrayOverhead;
long addedOverheads = size + overheads;
if (roundUp) {
size = roundUp(addedOverheads);
} else {
size = addedOverheads;
}
if (output != null) {
output.add(StringUtils.repeat(' ', indent) + oi.getName() + " [" + size + "] ("
+ o.getClass().getCanonicalName() + ") = " + underlyingObjects + "+" + overheads + "+"
+ (size - addedOverheads) + " REF=" + reference + " OBJ=" + objectOverhead + " ARRAY="
+ arrayOverhead + " ROUND=" + (size - addedOverheads));
}
} catch (Throwable t) {
log.warning("Unable to determine object size for " + o);
}
totalSize += size;
}
return totalSize;
return size;
}
public static long roundUp(long size) {
@@ -183,6 +232,42 @@ public interface ObjectSizeOf {
return 0;
}
}
public static long getStringSize(String s) {
long size = 0;
size += ObjectSizeOf.Sizer.OBJECT_OVERHEAD;
size += ObjectSizeOf.Sizer.REFERENCE;
size += ObjectSizeOf.Sizer.getPrimitiveObjectSize(int.class);
try {
// change with Java 9
if (String.class.getDeclaredField("coder") != null) {
size += ObjectSizeOf.Sizer.getPrimitiveObjectSize(byte.class);
}
} catch (Exception e) {
}
size += roundUp(ARRAY_OVERHEAD + (s.length() * ObjectSizeOf.Sizer.getPrimitiveObjectSize(byte.class)));
return roundUp(size);
}
}
public static void main(String[] args) {
Metric m = new Metric("mymetric", 100000l, 32);
m.addTag(new Tag("key1", "value1"));
m.addTag(new Tag("key2", "value2"));
m.addTag(new Tag("key3", "value3"));
m.addTag(new Tag("key4", "value4"));
m.addTag(new Tag("key5", "value5"));
long newSize1 = ObjectSizeOf.Sizer.getObjectSize(m, false, true);
System.out.println("new size #1: " + newSize1);
System.out.println();
System.out.println();
System.out.println();
long newSize2 = ObjectSizeOf.Sizer.getObjectSize(m, true, true);
System.out.println("new size #2: " + newSize2);
}
}
+32 -1
View File
@@ -18,7 +18,7 @@ import timely.model.parse.TagParser;
* Tag consists of key value pair
*/
@JsonRootName("tag")
public class Tag implements Comparable<Tag>, Serializable {
public class Tag implements Comparable<Tag>, ObjectSizeOf, Serializable {
private static final long serialVersionUID = 1L;
@@ -27,6 +27,12 @@ public class Tag implements Comparable<Tag>, Serializable {
private String key;
private String value;
@JsonIgnore
private boolean mustRecalculateLength = true;
@JsonIgnore
private long length;
public Tag() {
}
@@ -55,6 +61,7 @@ public class Tag implements Comparable<Tag>, Serializable {
public void set(String key, String value) {
this.key = key;
this.value = value;
this.mustRecalculateLength = true;
}
@JsonIgnore
@@ -64,6 +71,7 @@ public class Tag implements Comparable<Tag>, Serializable {
public void setKey(String key) {
this.key = key;
this.mustRecalculateLength = true;
}
@JsonIgnore
@@ -73,6 +81,7 @@ public class Tag implements Comparable<Tag>, Serializable {
public void setValue(String value) {
this.value = value;
this.mustRecalculateLength = true;
}
/**
@@ -106,4 +115,26 @@ public class Tag implements Comparable<Tag>, Serializable {
public int compareTo(Tag other) {
return new CompareToBuilder().append(this.key, other.key).append(this.value, other.value).toComparison();
}
@JsonIgnore
@Override
public long sizeInBytes() {
if (this.mustRecalculateLength) {
calculateLength();
this.mustRecalculateLength = false;
}
return this.length;
}
private void calculateLength() {
long l = 0;
l += 2 * Sizer.REFERENCE;
l += Sizer.OBJECT_OVERHEAD;
l += Sizer.getPrimitiveObjectSize(long.class);
l += Sizer.getPrimitiveObjectSize(boolean.class);
l += Sizer.getStringSize(key);
l += Sizer.getStringSize(value);
this.length = Sizer.roundUp(l);
}
}
+13 -1
View File
@@ -5,13 +5,14 @@ import java.util.Comparator;
import java.util.concurrent.TimeUnit;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Objects;
/**
* Value consists of a long timestamp and a double measurement
*/
public class Value implements Comparator<Value>, Serializable {
public class Value implements Comparator<Value>, ObjectSizeOf, Serializable {
private static final long serialVersionUID = 1L;
@@ -90,4 +91,15 @@ public class Value implements Comparator<Value>, Serializable {
public String toString() {
return "Value{" + "timestamp=" + timestamp + ", measure=" + measure + "}";
}
@JsonIgnore
@Override
public long sizeInBytes() {
long l = 0;
l += Sizer.OBJECT_OVERHEAD;
l += 2 * Sizer.REFERENCE;
l += Sizer.NUMBER_SIZE;
l += Sizer.NUMBER_SIZE;
return Sizer.roundUp(l);
}
}
+18 -14
View File
@@ -132,6 +132,7 @@ public class Server {
private static final String OS_VERSION = "os.version";
private static final String WS_PATH = "/websocket";
private SslContext sslCtx = null;
private CuratorFramework curatorFramework;
protected static final CountDownLatch LATCH = new CountDownLatch(1);
static ConfigurableApplicationContext applicationContext;
@@ -238,7 +239,7 @@ public class Server {
Server server = new Server(conf);
try {
server.run();
server.run(createSSLContext(conf));
LATCH.await();
} catch (final InterruptedException e) {
LOG.info("Server shutting down.");
@@ -355,18 +356,22 @@ public class Server {
}
});
try {
LOG.info("Closing dataStore");
dataStore.close();
} catch (Exception e) {
LOG.error(e.getMessage(), e);
if (dataStore != null) {
try {
LOG.info("Closing dataStore");
dataStore.close();
} catch (Exception e) {
LOG.error(e.getMessage(), e);
}
}
try {
LOG.info("Closing dataStoreCache");
dataStoreCache.close();
} catch (Exception e) {
LOG.error(e.getMessage(), e);
if (dataStoreCache != null) {
try {
LOG.info("Closing dataStoreCache");
dataStoreCache.close();
} catch (Exception e) {
LOG.error(e.getMessage(), e);
}
}
try {
@@ -431,7 +436,7 @@ public class Server {
}
}
public void run() throws Exception {
public void run(SslContext sslCtx) throws Exception {
RetryPolicy retryPolicy = new RetryForever(1000);
int timeout = Long.valueOf(AccumuloConfiguration.getTimeInMillis(config.getAccumulo().getZookeeperTimeout()))
@@ -498,7 +503,6 @@ public class Server {
final int httpPort = config.getHttp().getPort();
final String httpIp = config.getHttp().getIp();
SslContext sslCtx = createSSLContext(config);
if (sslCtx instanceof OpenSslServerContext) {
OpenSslServerContext openssl = (OpenSslServerContext) sslCtx;
String application = "Timely_" + httpPort;
@@ -555,7 +559,7 @@ public class Server {
tcpAddress, tcpPort, httpAddress, httpPort, wsAddress, wsPort, wsAddress, udpPort);
}
protected SslContext createSSLContext(Configuration config) throws Exception {
public static SslContext createSSLContext(Configuration config) throws Exception {
ServerSsl sslCfg = config.getSecurity().getServerSsl();
Boolean generate = sslCfg.isUseGeneratedKeypair();
@@ -85,7 +85,7 @@ public class StandaloneServer extends Server {
try {
LOG.info("Starting StandaloneServer");
StandaloneServer s = new StandaloneServer(conf);
s.run();
s.run(createSSLContext(conf));
} catch (Exception e) {
System.err.println("Error starting server");
e.printStackTrace();
@@ -8,6 +8,7 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import com.google.common.base.Joiner;
@@ -39,6 +40,8 @@ public class MetricAdapter {
private static final TagParser tagParser = new TagParser();
private static final TagListParser tagListParser = new TagListParser();
private static Pattern equalsPattern = Pattern.compile("=");
private static Pattern commaPattern = Pattern.compile(",");
public static final ColumnVisibility EMPTY_VISIBILITY = new ColumnVisibility();
public static final String VISIBILITY_TAG = "viz";
@@ -59,10 +62,10 @@ public class MetricAdapter {
// are used as a delimiter
String k = t.getKey();
String v = t.getValue();
k = k.replaceAll("=", "\\=");
k = k.replaceAll(",", "\\,");
v = v.replaceAll("=", "\\=");
v = v.replaceAll(",", "\\,");
k = equalsPattern.matcher(k).replaceAll("\\=");
k = commaPattern.matcher(k).replaceAll("\\,");
v = equalsPattern.matcher(v).replaceAll("\\=");
v = commaPattern.matcher(v).replaceAll("\\,");
newTags.add(new Tag(k, v));
}
return newTags;
@@ -75,10 +78,10 @@ public class MetricAdapter {
// are used as a delimiter
String k = t.getKey();
String v = t.getValue();
k = k.replaceAll("=", "\\=");
k = k.replaceAll(",", "\\,");
v = v.replaceAll("=", "\\=");
v = v.replaceAll(",", "\\,");
k = equalsPattern.matcher(k).replaceAll("\\=");
k = commaPattern.matcher(k).replaceAll("\\,");
v = equalsPattern.matcher(v).replaceAll("\\=");
v = commaPattern.matcher(v).replaceAll("\\,");
newTags.put(k, v);
}
return newTags;
@@ -1,6 +1,7 @@
package timely.configuration;
import java.util.List;
import java.util.Objects;
import com.google.common.collect.Lists;
import timely.validator.NotEmptyIfFieldSet;
@@ -76,4 +77,24 @@ public class ServerSsl {
public void setUseCiphers(List<String> useCiphers) {
this.useCiphers = useCiphers;
}
@Override
public boolean equals(Object o) {
if (this == o)
return true;
if (o == null || getClass() != o.getClass())
return false;
ServerSsl serverSsl = (ServerSsl) o;
return useGeneratedKeypair == serverSsl.useGeneratedKeypair && useOpenssl == serverSsl.useOpenssl
&& Objects.equals(certificateFile, serverSsl.certificateFile)
&& Objects.equals(keyFile, serverSsl.keyFile) && Objects.equals(keyPassword, serverSsl.keyPassword)
&& Objects.equals(trustStoreFile, serverSsl.trustStoreFile)
&& Objects.equals(useCiphers, serverSsl.useCiphers);
}
@Override
public int hashCode() {
return Objects.hash(certificateFile, keyFile, keyPassword, trustStoreFile, useGeneratedKeypair, useOpenssl,
useCiphers);
}
}
@@ -4,8 +4,15 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import java.util.*;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.SortedMap;
import java.util.TreeMap;
import org.apache.accumulo.core.client.IteratorSetting;
import org.apache.accumulo.core.data.ColumnUpdate;
@@ -251,6 +258,52 @@ public class DownsampleIteratorTest {
assertEquals((elapsedTime / sampleInterval) * numTagVariations, totalBuckets);
}
@Test
public void test() {
String hello = new String("this is a longer string");
long maybe = ObjectSizeOf.Sizer.OBJECT_OVERHEAD + // for String
ObjectSizeOf.Sizer.REFERENCE + // for array of char
ObjectSizeOf.Sizer.getPrimitiveObjectSize(int.class) +
ObjectSizeOf.Sizer.ARRAY_OVERHEAD + // for array of char
hello.length() * ObjectSizeOf.Sizer.getPrimitiveObjectSize(char.class);
maybe = ObjectSizeOf.Sizer.roundUp(maybe);
long size = ObjectSizeOf.Sizer.getObjectSize(hello, true, true);
System.out.println("size = " + size);
System.out.println("maybe = " + maybe);
}
@Test
public void test2() {
System.out.println(ObjectSizeOf.Sizer.getObjectSize(new Long(100)));
}
@Test
public void testDownsampleMemoryCalculation() throws Exception {
int numTagVariations = 1;
int sampleInterval = 50;
int elapsedTime = 100;
int skipInterval = 10;
SortedMap<Key, Value> testData3 = createTestData3(elapsedTime, skipInterval, numTagVariations);
DownsampleIterator iter = new DownsampleIterator();
Map<Set<Tag>, Downsample> samples = runQuery(iter, testData3, sampleInterval, 1000);
assertEquals(numTagVariations, samples.size());
long totalBuckets = 0;
for (Entry<Set<Tag>, Downsample> entry : samples.entrySet()) {
Set<Tag> tagSet = entry.getKey();
Downsample downsample = entry.getValue();
for (Tag tag : tagSet) {
assertEquals(ObjectSizeOf.Sizer.getObjectSize(tag, true, true),
ObjectSizeOf.Sizer.getObjectSize(tag, false, true));
}
}
}
private Map<Set<Tag>, Downsample> runQuery(SortedKeyValueIterator<Key, Value> iter, SortedMap<Key, Value> testData,
long period, long maxDownsampleMemory) throws Exception {
IteratorSetting is = new IteratorSetting(100, DownsampleIterator.class);
@@ -59,7 +59,7 @@ public class DataStoreIT extends OneWaySSLBase {
conf.setMetricAgeOffDays(ageOffSettings);
final Server s = new Server(conf);
s.run();
s.run(getSslContext());
try {
put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2",
"sys.cpu.user " + (TEST_TIME + ONE_HOUR) + " 3.0 tag1=value1 tag2=value2",
@@ -106,7 +106,7 @@ public class DataStoreIT extends OneWaySSLBase {
conf.setMetricAgeOffDays(ageOffSettings);
final Server s = new Server(conf);
s.run();
s.run(getSslContext());
// @formatter:off
/*
* TEST_TIME = System.currentTimeMillis() - ONE_DAY
@@ -5,6 +5,8 @@ import java.io.IOException;
import java.nio.file.Files;
import java.util.HashMap;
import io.netty.handler.ssl.SslContext;
import io.netty.handler.ssl.util.SelfSignedCertificate;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.minicluster.MiniAccumuloCluster;
import org.apache.accumulo.minicluster.MiniAccumuloConfig;
@@ -12,8 +14,10 @@ import org.junit.Before;
import org.junit.BeforeClass;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import timely.Server;
import timely.auth.AuthCache;
import timely.configuration.Configuration;
import timely.configuration.ServerSsl;
import timely.store.MetricAgeOffIterator;
import timely.test.TestConfiguration;
@@ -24,6 +28,11 @@ public class MacITBase {
private static final Logger LOG = LoggerFactory.getLogger(MacITBase.class);
protected static ServerSsl serverSsl = null;
protected static File clientTrustStoreFile = null;
protected static SslContext sslCtx = null;
protected static SelfSignedCertificate serverCert = null;
private static final File tempDir;
protected static final String MAC_ROOT_USER = "root";
@@ -50,8 +59,6 @@ public class MacITBase {
conf = TestConfiguration.createMinimalConfigurationForTest();
conf.getAccumulo().setInstanceName(mac.getInstanceName());
conf.getAccumulo().setZookeepers(mac.getZooKeepers());
conf.getSecurity().getServerSsl().setUseOpenssl(false);
conf.getSecurity().getServerSsl().setUseGeneratedKeypair(true);
conf.getWebsocket().setFlushIntervalSeconds(TestConfiguration.WAIT_SECONDS);
HashMap<String, Integer> ageOffSettings = new HashMap<>();
ageOffSettings.put(MetricAgeOffIterator.DEFAULT_AGEOFF_KEY, 7);
@@ -88,4 +95,52 @@ public class MacITBase {
public void initializeAuthCache() {
AuthCache.clear();
}
protected static void setupSSL() throws Exception {
setupSSL(TestConfiguration.createMinimalConfigurationForTest(), false);
}
protected static void setupSSL(Configuration config, boolean twoWaySsl) throws Exception {
setupSSL(config, twoWaySsl, null);
}
protected static void setupSSL(Configuration config, boolean twoWaySsl, SelfSignedCertificate serverCert)
throws Exception {
ServerSsl serverSsl = config.getSecurity().getServerSsl();
boolean generateNewSsl = MacITBase.serverSsl == null || serverCert != null
|| !MacITBase.serverSsl.equals(serverSsl);
if (generateNewSsl) {
// new serverCert and trustStoreFile
if (serverCert == null) {
MacITBase.serverCert = new SelfSignedCertificate();
} else {
MacITBase.serverCert = serverCert;
}
MacITBase.clientTrustStoreFile = MacITBase.serverCert.certificate().getAbsoluteFile();
MacITBase.serverSsl = serverSsl;
MacITBase.serverSsl.setCertificateFile(MacITBase.serverCert.certificate().getAbsolutePath());
MacITBase.serverSsl.setKeyFile(MacITBase.serverCert.privateKey().getAbsolutePath());
}
config.getSecurity().getServerSsl().setCertificateFile(MacITBase.serverSsl.getCertificateFile());
config.getSecurity().getServerSsl().setKeyFile(MacITBase.serverSsl.getKeyFile());
config.getSecurity().getServerSsl().setUseOpenssl(false);
config.getSecurity().getServerSsl().setUseGeneratedKeypair(false);
if (twoWaySsl) {
// Needed for 2way SSL
config.getSecurity().getServerSsl().setTrustStoreFile(MacITBase.serverSsl.getCertificateFile());
config.getSecurity().setAllowAnonymousHttpAccess(false);
} else {
config.getSecurity().setAllowAnonymousHttpAccess(true);
}
if (generateNewSsl) {
// new SslContext
OneWaySSLBase.sslCtx = Server.createSSLContext(config);
}
}
protected static SslContext getSslContext() {
return MacITBase.sslCtx;
}
}
@@ -30,7 +30,7 @@ public class OneWaySSLAnonAccessIT extends OneWaySSLBase {
@Test
public void testQueryWithVisibility() throws Exception {
final Server s = new Server(conf);
s.run();
s.run(getSslContext());
try {
put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2",
"sys.cpu.user " + (TEST_TIME + 1000) + " 3.0 tag1=value1 tag2=value2",
@@ -1,6 +1,5 @@
package timely.test.integration;
import java.io.File;
import java.net.URL;
import javax.net.ssl.HttpsURLConnection;
@@ -12,25 +11,21 @@ import io.netty.handler.ssl.JdkSslContext;
import io.netty.handler.ssl.SslContext;
import io.netty.handler.ssl.SslContextBuilder;
import io.netty.handler.ssl.SslProvider;
import io.netty.handler.ssl.util.SelfSignedCertificate;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import timely.auth.AuthCache;
import timely.configuration.Configuration;
/**
* Base test class for SSL with anonymous access
*/
public class OneWaySSLBase extends QueryBase {
protected static File clientTrustStoreFile = null;
protected SSLSocketFactory getSSLSocketFactory() throws Exception {
SslContextBuilder builder = SslContextBuilder.forClient();
builder.applicationProtocolConfig(ApplicationProtocolConfig.DISABLED);
builder.sslProvider(SslProvider.JDK);
builder.trustManager(clientTrustStoreFile); // Trust the server cert
builder.trustManager(MacITBase.clientTrustStoreFile); // Trust the server cert
SslContext ctx = builder.build();
Assert.assertTrue(ctx.isClient());
Assert.assertTrue(ctx instanceof JdkSslContext);
@@ -39,19 +34,9 @@ public class OneWaySSLBase extends QueryBase {
return jdkSslContext.getSocketFactory();
}
protected static void setupSSL(Configuration config) throws Exception {
SelfSignedCertificate serverCert = new SelfSignedCertificate();
config.getSecurity().getServerSsl().setCertificateFile(serverCert.certificate().getAbsolutePath());
clientTrustStoreFile = serverCert.certificate().getAbsoluteFile();
config.getSecurity().getServerSsl().setKeyFile(serverCert.privateKey().getAbsolutePath());
config.getSecurity().getServerSsl().setUseOpenssl(false);
config.getSecurity().getServerSsl().setUseGeneratedKeypair(false);
config.getSecurity().setAllowAnonymousHttpAccess(true);
}
@Before
public void configureSSL() throws Exception {
setupSSL(conf);
setupSSL(conf, false);
}
@Override
@@ -93,7 +93,7 @@ public class OneWaySSLBasicAuthAccessIT extends OneWaySSLBase {
@Test
public void testBasicAuthLogin() throws Exception {
final Server s = new Server(conf);
s.run();
s.run(getSslContext());
try {
String metrics = "https://localhost:54322/api/metrics";
query(metrics);
@@ -105,7 +105,7 @@ public class OneWaySSLBasicAuthAccessIT extends OneWaySSLBase {
@Test(expected = UnauthorizedUserException.class)
public void testBasicAuthLoginFailure() throws Exception {
final Server s = new Server(conf);
s.run();
s.run(getSslContext());
try {
String metrics = "https://localhost:54322/api/metrics";
query("test", "test2", metrics);
@@ -117,7 +117,7 @@ public class OneWaySSLBasicAuthAccessIT extends OneWaySSLBase {
@Test
public void testQueryWithVisibility() throws Exception {
final Server s = new Server(conf);
s.run();
s.run(getSslContext());
try {
put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2",
"sys.cpu.user " + (TEST_TIME + 1000) + " 3.0 tag1=value1 tag2=value2",
@@ -13,7 +13,7 @@ public class StaticFileServerIT extends OneWaySSLBase {
@Before
public void before() throws Exception {
s = new Server(conf);
s.run();
s.run(getSslContext());
}
@After
@@ -0,0 +1,92 @@
package timely.test.integration;
import java.net.URL;
import java.util.List;
import javax.net.ssl.HostnameVerifier;
import javax.net.ssl.HttpsURLConnection;
import javax.net.ssl.SSLContext;
import javax.net.ssl.SSLSession;
import javax.net.ssl.SSLSocketFactory;
import io.netty.handler.codec.http.HttpHeaderNames;
import io.netty.handler.codec.http.cookie.ClientCookieDecoder;
import io.netty.handler.codec.http.cookie.Cookie;
import io.netty.handler.ssl.ApplicationProtocolConfig;
import io.netty.handler.ssl.JdkSslContext;
import io.netty.handler.ssl.SslContext;
import io.netty.handler.ssl.SslContextBuilder;
import io.netty.handler.ssl.SslProvider;
import org.junit.After;
import org.junit.Assert;
import timely.auth.AuthCache;
import timely.netty.Constants;
/**
* Base test class for SSL with anonymous access
*/
public class TwoWaySSLBase extends QueryBase {
@After
public void tearDown() throws Exception {
AuthCache.resetConfiguration();
}
protected SSLSocketFactory getSSLSocketFactory() throws Exception {
SslContextBuilder builder = SslContextBuilder.forClient();
builder.applicationProtocolConfig(ApplicationProtocolConfig.DISABLED);
// Use server cert / key on client side
builder.keyManager(MacITBase.serverCert.key(), (String) null, MacITBase.serverCert.cert());
builder.sslProvider(SslProvider.JDK);
builder.trustManager(MacITBase.clientTrustStoreFile); // Trust the server cert
SslContext ctx = builder.build();
Assert.assertTrue(ctx.isClient());
Assert.assertTrue(ctx instanceof JdkSslContext);
JdkSslContext jdk = (JdkSslContext) ctx;
SSLContext jdkSslContext = jdk.context();
return jdkSslContext.getSocketFactory();
}
@Override
protected HttpsURLConnection getUrlConnection(URL url) throws Exception {
// Username and password not used in 2way SSL case
return getUrlConnection(null, null, url);
}
@Override
protected HttpsURLConnection getUrlConnection(String username, String password, URL url) throws Exception {
HttpsURLConnection.setDefaultSSLSocketFactory(getSSLSocketFactory());
URL loginURL = new URL(url.getProtocol() + "://" + url.getHost() + ":" + url.getPort() + "/login");
HttpsURLConnection con = (HttpsURLConnection) loginURL.openConnection();
con.setHostnameVerifier(new HostnameVerifier() {
@Override
public boolean verify(String arg0, SSLSession arg1) {
return true;
}
});
con.setRequestMethod("GET");
con.setDoOutput(true);
con.setRequestProperty("Content-Type", "application/json");
con.connect();
int responseCode = con.getResponseCode();
if (401 == responseCode) {
throw new UnauthorizedUserException();
}
Assert.assertEquals(200, responseCode);
List<String> cookies = con.getHeaderFields().get(HttpHeaderNames.SET_COOKIE.toString());
Assert.assertEquals(1, cookies.size());
Cookie sessionCookie = ClientCookieDecoder.STRICT.decode(cookies.get(0));
Assert.assertEquals(Constants.COOKIE_NAME, sessionCookie.name());
con = (HttpsURLConnection) url.openConnection();
con.setRequestProperty(HttpHeaderNames.COOKIE.toString(), sessionCookie.name() + "=" + sessionCookie.value());
con.setHostnameVerifier(new HostnameVerifier() {
@Override
public boolean verify(String arg0, SSLSession arg1) {
return true;
}
});
return con;
}
}
@@ -1,30 +1,11 @@
package timely.test.integration;
import java.io.File;
import java.net.URL;
import java.util.List;
import javax.net.ssl.HostnameVerifier;
import javax.net.ssl.HttpsURLConnection;
import javax.net.ssl.SSLContext;
import javax.net.ssl.SSLSession;
import javax.net.ssl.SSLSocketFactory;
import io.netty.handler.codec.http.HttpHeaderNames;
import io.netty.handler.codec.http.cookie.ClientCookieDecoder;
import io.netty.handler.codec.http.cookie.Cookie;
import io.netty.handler.ssl.ApplicationProtocolConfig;
import io.netty.handler.ssl.JdkSslContext;
import io.netty.handler.ssl.SslContext;
import io.netty.handler.ssl.SslContextBuilder;
import io.netty.handler.ssl.SslProvider;
import io.netty.handler.ssl.util.SelfSignedCertificate;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.minicluster.MiniAccumuloCluster;
import org.apache.accumulo.minicluster.MiniAccumuloConfig;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.ClassRule;
@@ -33,10 +14,9 @@ import org.junit.rules.TemporaryFolder;
import timely.Server;
import timely.auth.AuthCache;
import timely.configuration.Configuration;
import timely.netty.Constants;
import timely.test.TestConfiguration;
public class TwoWaySSLFailureIT extends QueryBase {
public class TwoWaySSLFailureIT extends TwoWaySSLBase {
@ClassRule
public static final TemporaryFolder temp = new TemporaryFolder();
@@ -44,87 +24,6 @@ public class TwoWaySSLFailureIT extends QueryBase {
private static MiniAccumuloCluster mac = null;
private static Configuration conf = null;
private static SelfSignedCertificate serverCert = null;
private static File clientTrustStoreFile = null;
static {
try {
// This fqdn does not match what is in security.xml
serverCert = new SelfSignedCertificate("CN=bad.example.com");
clientTrustStoreFile = serverCert.certificate().getAbsoluteFile();
} catch (Exception e) {
throw new RuntimeException("Error creating self signed certificate", e);
}
}
protected SSLSocketFactory getSSLSocketFactory() throws Exception {
SslContextBuilder builder = SslContextBuilder.forClient();
builder.applicationProtocolConfig(ApplicationProtocolConfig.DISABLED);
// Use server cert / key on client side
builder.keyManager(serverCert.key(), (String) null, serverCert.cert());
builder.sslProvider(SslProvider.JDK);
builder.trustManager(clientTrustStoreFile); // Trust the server cert
SslContext ctx = builder.build();
Assert.assertTrue(ctx.isClient());
Assert.assertTrue(ctx instanceof JdkSslContext);
JdkSslContext jdk = (JdkSslContext) ctx;
SSLContext jdkSslContext = jdk.context();
return jdkSslContext.getSocketFactory();
}
protected static void setupSSL(Configuration config) throws Exception {
config.getSecurity().getServerSsl().setCertificateFile(serverCert.certificate().getAbsolutePath());
config.getSecurity().getServerSsl().setKeyFile(serverCert.privateKey().getAbsolutePath());
// Needed for 2way SSL
config.getSecurity().getServerSsl().setTrustStoreFile(serverCert.certificate().getAbsolutePath());
config.getSecurity().getServerSsl().setUseOpenssl(false);
config.getSecurity().getServerSsl().setUseGeneratedKeypair(false);
config.getSecurity().setAllowAnonymousHttpAccess(false);
}
@Override
protected HttpsURLConnection getUrlConnection(URL url) throws Exception {
// Username and password not used in 2way SSL case
return getUrlConnection(null, null, url);
}
@Override
protected HttpsURLConnection getUrlConnection(String username, String password, URL url) throws Exception {
HttpsURLConnection.setDefaultSSLSocketFactory(getSSLSocketFactory());
URL loginURL = new URL(url.getProtocol() + "://" + url.getHost() + ":" + url.getPort() + "/login");
HttpsURLConnection con = (HttpsURLConnection) loginURL.openConnection();
con.setHostnameVerifier(new HostnameVerifier() {
@Override
public boolean verify(String arg0, SSLSession arg1) {
return true;
}
});
con.setRequestMethod("GET");
con.setDoOutput(true);
con.setRequestProperty("Content-Type", "application/json");
con.connect();
int responseCode = con.getResponseCode();
if (401 == responseCode) {
throw new UnauthorizedUserException();
}
Assert.assertEquals(200, responseCode);
List<String> cookies = con.getHeaderFields().get(HttpHeaderNames.SET_COOKIE.toString());
Assert.assertEquals(1, cookies.size());
Cookie sessionCookie = ClientCookieDecoder.STRICT.decode(cookies.get(0));
Assert.assertEquals(Constants.COOKIE_NAME, sessionCookie.name());
con = (HttpsURLConnection) url.openConnection();
con.setRequestProperty(HttpHeaderNames.COOKIE.toString(), sessionCookie.name() + "=" + sessionCookie.value());
con.setHostnameVerifier(new HostnameVerifier() {
@Override
public boolean verify(String arg0, SSLSession arg1) {
return true;
}
});
return con;
}
@BeforeClass
public static void beforeClass() throws Exception {
temp.create();
@@ -134,7 +33,8 @@ public class TwoWaySSLFailureIT extends QueryBase {
conf = TestConfiguration.createMinimalConfigurationForTest();
conf.getAccumulo().setInstanceName(mac.getInstanceName());
conf.getAccumulo().setZookeepers(mac.getZooKeepers());
setupSSL(conf);
// This fqdn does not match what is in security.xml
setupSSL(conf, true, new SelfSignedCertificate("CN=bad.example.com"));
}
@AfterClass
@@ -163,7 +63,7 @@ public class TwoWaySSLFailureIT extends QueryBase {
@Test(expected = UnauthorizedUserException.class)
public void testBasicAuthLoginFailure() throws Exception {
final Server s = new Server(conf);
s.run();
s.run(getSslContext());
try {
String metrics = "https://localhost:54322/api/metrics";
query(metrics);
@@ -3,33 +3,17 @@ package timely.test.integration;
import static com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly;
import static org.junit.Assert.assertEquals;
import java.io.File;
import java.net.URL;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.TimeUnit;
import javax.net.ssl.HttpsURLConnection;
import javax.net.ssl.SSLContext;
import javax.net.ssl.SSLSocketFactory;
import io.netty.handler.codec.http.HttpHeaderNames;
import io.netty.handler.codec.http.cookie.ClientCookieDecoder;
import io.netty.handler.codec.http.cookie.Cookie;
import io.netty.handler.ssl.ApplicationProtocolConfig;
import io.netty.handler.ssl.JdkSslContext;
import io.netty.handler.ssl.SslContext;
import io.netty.handler.ssl.SslContextBuilder;
import io.netty.handler.ssl.SslProvider;
import io.netty.handler.ssl.util.SelfSignedCertificate;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.minicluster.MiniAccumuloCluster;
import org.apache.accumulo.minicluster.MiniAccumuloConfig;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.ClassRule;
@@ -41,7 +25,6 @@ import timely.api.request.timeseries.QueryRequest;
import timely.api.response.timeseries.QueryResponse;
import timely.auth.AuthCache;
import timely.configuration.Configuration;
import timely.netty.Constants;
import timely.test.IntegrationTest;
import timely.test.TestConfiguration;
@@ -51,7 +34,7 @@ import timely.test.TestConfiguration;
*
*/
@Category(IntegrationTest.class)
public class TwoWaySSLIT extends QueryBase {
public class TwoWaySSLIT extends TwoWaySSLBase {
private static final Long TEST_TIME = System.currentTimeMillis();
@@ -61,74 +44,6 @@ public class TwoWaySSLIT extends QueryBase {
private static MiniAccumuloCluster mac = null;
private static Configuration conf = null;
protected static SelfSignedCertificate serverCert = null;
protected static File clientTrustStoreFile = null;
static {
try {
serverCert = new SelfSignedCertificate();
clientTrustStoreFile = serverCert.certificate().getAbsoluteFile();
} catch (Exception e) {
throw new RuntimeException("Error creating self signed certificate", e);
}
}
protected SSLSocketFactory getSSLSocketFactory() throws Exception {
SslContextBuilder builder = SslContextBuilder.forClient();
builder.applicationProtocolConfig(ApplicationProtocolConfig.DISABLED);
// Use server cert / key on client side.
builder.keyManager(serverCert.key(), (String) null, serverCert.cert());
builder.sslProvider(SslProvider.JDK);
builder.trustManager(clientTrustStoreFile); // Trust the server cert
SslContext ctx = builder.build();
Assert.assertTrue(ctx.isClient());
Assert.assertTrue(ctx instanceof JdkSslContext);
JdkSslContext jdk = (JdkSslContext) ctx;
SSLContext jdkSslContext = jdk.context();
return jdkSslContext.getSocketFactory();
}
protected static void setupSSL(Configuration config) throws Exception {
config.getSecurity().getServerSsl().setCertificateFile(serverCert.certificate().getAbsolutePath());
config.getSecurity().getServerSsl().setKeyFile(serverCert.privateKey().getAbsolutePath());
// Needed for 2way SSL
config.getSecurity().getServerSsl().setTrustStoreFile(serverCert.certificate().getAbsolutePath());
config.getSecurity().getServerSsl().setUseOpenssl(false);
config.getSecurity().getServerSsl().setUseGeneratedKeypair(false);
config.getSecurity().setAllowAnonymousHttpAccess(false);
}
@Override
protected HttpsURLConnection getUrlConnection(URL url) throws Exception {
// Username and password not used in 2way SSL case
return getUrlConnection(null, null, url);
}
@Override
protected HttpsURLConnection getUrlConnection(String username, String password, URL url) throws Exception {
HttpsURLConnection.setDefaultSSLSocketFactory(getSSLSocketFactory());
URL loginURL = new URL(url.getProtocol() + "://" + url.getHost() + ":" + url.getPort() + "/login");
HttpsURLConnection con = (HttpsURLConnection) loginURL.openConnection();
con.setHostnameVerifier((host, session) -> true);
con.setRequestMethod("GET");
con.setDoOutput(true);
con.setRequestProperty("Content-Type", "application/json");
con.connect();
int responseCode = con.getResponseCode();
if (401 == responseCode) {
throw new UnauthorizedUserException();
}
Assert.assertEquals(200, responseCode);
List<String> cookies = con.getHeaderFields().get(HttpHeaderNames.SET_COOKIE.toString());
Assert.assertEquals(1, cookies.size());
Cookie sessionCookie = ClientCookieDecoder.STRICT.decode(cookies.get(0));
Assert.assertEquals(Constants.COOKIE_NAME, sessionCookie.name());
con = (HttpsURLConnection) url.openConnection();
con.setRequestProperty(HttpHeaderNames.COOKIE.toString(), sessionCookie.name() + "=" + sessionCookie.value());
con.setHostnameVerifier((host, session) -> true);
return con;
}
@BeforeClass
public static void beforeClass() throws Exception {
final MiniAccumuloConfig macConfig = new MiniAccumuloConfig(temp.newFolder("mac"), "secret");
@@ -137,7 +52,6 @@ public class TwoWaySSLIT extends QueryBase {
conf = TestConfiguration.createMinimalConfigurationForTest();
conf.getAccumulo().setInstanceName(mac.getInstanceName());
conf.getAccumulo().setZookeepers(mac.getZooKeepers());
setupSSL(conf);
}
@AfterClass
@@ -157,6 +71,7 @@ public class TwoWaySSLIT extends QueryBase {
}
}
});
setupSSL(conf, true);
}
@After
@@ -167,7 +82,7 @@ public class TwoWaySSLIT extends QueryBase {
@Test
public void testBasicAuthLogin() throws Exception {
final Server s = new Server(conf);
s.run();
s.run(getSslContext());
try {
String metrics = "https://localhost:54322/api/metrics";
query(metrics);
@@ -190,7 +105,7 @@ public class TwoWaySSLIT extends QueryBase {
public void testQueryWithVisibility(Configuration conf) throws Exception {
final Server s = new Server(conf);
s.run();
s.run(getSslContext());
try {
// @formatter:off
put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2",
@@ -3,32 +3,14 @@ package timely.test.integration;
import static com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly;
import static org.junit.Assert.assertEquals;
import java.io.File;
import java.net.URL;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.TimeUnit;
import javax.net.ssl.HostnameVerifier;
import javax.net.ssl.HttpsURLConnection;
import javax.net.ssl.SSLContext;
import javax.net.ssl.SSLSession;
import javax.net.ssl.SSLSocketFactory;
import io.netty.handler.codec.http.HttpHeaderNames;
import io.netty.handler.codec.http.cookie.ClientCookieDecoder;
import io.netty.handler.codec.http.cookie.Cookie;
import io.netty.handler.ssl.ApplicationProtocolConfig;
import io.netty.handler.ssl.JdkSslContext;
import io.netty.handler.ssl.SslContext;
import io.netty.handler.ssl.SslContextBuilder;
import io.netty.handler.ssl.SslProvider;
import io.netty.handler.ssl.util.SelfSignedCertificate;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.security.Authorizations;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@@ -36,8 +18,6 @@ import timely.Server;
import timely.api.request.timeseries.QueryRequest;
import timely.api.response.timeseries.QueryResponse;
import timely.auth.AuthCache;
import timely.configuration.Configuration;
import timely.netty.Constants;
import timely.test.IntegrationTest;
import timely.test.TestConfiguration;
@@ -48,97 +28,13 @@ import timely.test.TestConfiguration;
*
*/
@Category(IntegrationTest.class)
public class TwoWaySSLOpenSSLIT extends QueryBase {
public class TwoWaySSLOpenSSLIT extends TwoWaySSLBase {
private static final Long TEST_TIME = System.currentTimeMillis();
protected static SelfSignedCertificate serverCert = null;
protected static File clientTrustStoreFile = null;
static {
try {
serverCert = new SelfSignedCertificate();
clientTrustStoreFile = serverCert.certificate().getAbsoluteFile();
} catch (Exception e) {
throw new RuntimeException("Error creating self signed certificate", e);
}
}
protected SSLSocketFactory getSSLSocketFactory() throws Exception {
SslContextBuilder builder = SslContextBuilder.forClient();
builder.applicationProtocolConfig(ApplicationProtocolConfig.DISABLED);
// Use server cert / key on client side.
builder.keyManager(serverCert.key(), (String) null, serverCert.cert());
builder.sslProvider(SslProvider.JDK);
builder.trustManager(clientTrustStoreFile); // Trust the server cert
SslContext ctx = builder.build();
Assert.assertTrue(ctx.isClient());
Assert.assertTrue(ctx instanceof JdkSslContext);
JdkSslContext jdk = (JdkSslContext) ctx;
SSLContext jdkSslContext = jdk.context();
return jdkSslContext.getSocketFactory();
}
protected static void setupSSL(Configuration config) throws Exception {
config.getSecurity().getServerSsl().setCertificateFile(serverCert.certificate().getAbsolutePath());
config.getSecurity().getServerSsl().setKeyFile(serverCert.privateKey().getAbsolutePath());
// Needed for 2way SSL
config.getSecurity().getServerSsl().setTrustStoreFile(serverCert.certificate().getAbsolutePath());
config.getSecurity().getServerSsl().setUseOpenssl(false);
config.getSecurity().getServerSsl().setUseGeneratedKeypair(false);
config.getSecurity().setAllowAnonymousHttpAccess(false);
}
@Before
public void configureSSL() throws Exception {
setupSSL(conf);
}
@Override
protected HttpsURLConnection getUrlConnection(URL url) throws Exception {
// Username and password not used in 2way SSL case
return getUrlConnection(null, null, url);
}
@Override
protected HttpsURLConnection getUrlConnection(String username, String password, URL url) throws Exception {
HttpsURLConnection.setDefaultSSLSocketFactory(getSSLSocketFactory());
URL loginURL = new URL(url.getProtocol() + "://" + url.getHost() + ":" + url.getPort() + "/login");
HttpsURLConnection con = (HttpsURLConnection) loginURL.openConnection();
con.setHostnameVerifier(new HostnameVerifier() {
@Override
public boolean verify(String arg0, SSLSession arg1) {
return true;
}
});
con.setRequestMethod("GET");
con.setDoOutput(true);
con.setRequestProperty("Content-Type", "application/json");
con.connect();
int responseCode = con.getResponseCode();
if (401 == responseCode) {
throw new UnauthorizedUserException();
}
Assert.assertEquals(200, responseCode);
List<String> cookies = con.getHeaderFields().get(HttpHeaderNames.SET_COOKIE.toString());
Assert.assertEquals(1, cookies.size());
Cookie sessionCookie = ClientCookieDecoder.STRICT.decode(cookies.get(0));
Assert.assertEquals(Constants.COOKIE_NAME, sessionCookie.name());
con = (HttpsURLConnection) url.openConnection();
con.setRequestProperty(HttpHeaderNames.COOKIE.toString(), sessionCookie.name() + "=" + sessionCookie.value());
con.setHostnameVerifier(new HostnameVerifier() {
@Override
public boolean verify(String arg0, SSLSession arg1) {
return true;
}
});
return con;
}
@Before
public void setup() throws Exception {
setupSSL(conf, true);
Connector con = mac.getConnector("root", "secret");
con.securityOperations().changeUserAuthorizations("root", new Authorizations("A", "B", "C", "D", "E", "F"));
}
@@ -151,7 +47,7 @@ public class TwoWaySSLOpenSSLIT extends QueryBase {
@Test
public void testBasicAuthLogin() throws Exception {
final Server s = new Server(conf);
s.run();
s.run(getSslContext());
try {
String metrics = "https://localhost:54322/api/metrics";
query(metrics);
@@ -163,7 +59,7 @@ public class TwoWaySSLOpenSSLIT extends QueryBase {
@Test
public void testQueryWithVisibility() throws Exception {
final Server s = new Server(conf);
s.run();
s.run(getSslContext());
try {
put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2",
"sys.cpu.user " + (TEST_TIME + 1000) + " 3.0 tag1=value1 tag2=value2",
@@ -49,7 +49,7 @@ public class TcpClientIT extends OneWaySSLBase {
@Test
public void testPut() throws Exception {
final TestServer m = new TestServer(conf);
m.run();
m.run(getSslContext());
try (TcpClient client = new TcpClient("127.0.0.1", 54321)) {
client.open();
client.write("put sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2\n");
@@ -80,7 +80,7 @@ public class TcpClientIT extends OneWaySSLBase {
public void testPutMultiple() throws Exception {
final TestServer m = new TestServer(conf);
m.run();
m.run(getSslContext());
try (TcpClient client = new TcpClient("127.0.0.1", 54321)) {
client.open();
// @formatter:off
@@ -124,7 +124,7 @@ public class TcpClientIT extends OneWaySSLBase {
@Test
public void testPutInvalidTimestamp() throws Exception {
final TestServer m = new TestServer(conf);
m.run();
m.run(getSslContext());
try (TcpClient client = new TcpClient("127.0.0.1", 54321)) {
client.open();
client.write("put sys.cpu.user " + TEST_TIME + "Z" + " 1.0 tag1=value1 tag2=value2\n");
@@ -139,7 +139,7 @@ public class TcpClientIT extends OneWaySSLBase {
@Test
public void testPersistence() throws Exception {
final Server s = new Server(conf);
s.run();
s.run(getSslContext());
try {
put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2",
"sys.cpu.idle " + (TEST_TIME + 1) + " 1.0 tag3=value3 tag4=value4",
@@ -183,7 +183,7 @@ public class TcpClientIT extends OneWaySSLBase {
@Test
public void testPersistenceWithVisibility() throws Exception {
final Server s = new Server(conf);
s.run();
s.run(getSslContext());
try {
put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2",
"sys.cpu.idle " + (TEST_TIME + 1) + " 1.0 tag3=value3 tag4=value4 viz=(a|b)",
@@ -53,6 +53,7 @@ public class UdpClientIT extends MacITBase {
}
}
});
setupSSL();
}
@After
@@ -63,7 +64,7 @@ public class UdpClientIT extends MacITBase {
@Test
public void testPut() throws Exception {
final TestServer m = new TestServer(conf);
m.run();
m.run(getSslContext());
try (UdpClient client = new UdpClient("127.0.0.1", 54325)) {
client.open();
client.write(("put sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2\n"));
@@ -93,7 +94,7 @@ public class UdpClientIT extends MacITBase {
public void testPutMultiple() throws Exception {
final TestServer m = new TestServer(conf);
m.run();
m.run(getSslContext());
try (UdpClient client = new UdpClient("127.0.0.1", 54325)) {
client.open();
client.write(("put sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2\n" + "put sys.cpu.idle "
@@ -135,7 +136,7 @@ public class UdpClientIT extends MacITBase {
@Test
public void testPutInvalidTimestamp() throws Exception {
final TestServer m = new TestServer(conf);
m.run();
m.run(getSslContext());
try (UdpClient client = new UdpClient("127.0.0.1", 54325)) {
client.open();
client.write(("put sys.cpu.user " + TEST_TIME + "Z" + " 1.0 tag1=value1 tag2=value2\n"));
@@ -149,7 +150,7 @@ public class UdpClientIT extends MacITBase {
@Test
public void testPersistence() throws Exception {
final Server s = new Server(conf);
s.run();
s.run(getSslContext());
try {
put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2",
"sys.cpu.idle " + (TEST_TIME + 1) + " 1.0 tag3=value3 tag4=value4",
@@ -193,7 +194,7 @@ public class UdpClientIT extends MacITBase {
@Test
public void testPersistenceWithVisibility() throws Exception {
final Server s = new Server(conf);
s.run();
s.run(getSslContext());
try {
put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2",
"sys.cpu.idle " + (TEST_TIME + 1) + " 1.0 tag3=value3 tag4=value4 viz=(a|b)",
@@ -65,7 +65,7 @@ public class WebSocketClientIT extends OneWaySSLBase {
Connector con = mac.getConnector("root", "secret");
con.securityOperations().changeUserAuthorizations("root", new Authorizations("A", "B", "C", "D", "E", "F"));
s = new Server(conf);
s.run();
s.run(getSslContext());
setupSslCtx();
}
@@ -23,7 +23,7 @@ public class HTTPStrictTransportSecurityIT extends OneWaySSLBase {
@Before
public void before() throws Exception {
s = new Server(conf);
s.run();
s.run(getSslContext());
}
@After
@@ -58,7 +58,7 @@ public class HttpApiIT extends OneWaySSLBase {
@Test
public void testSuggest() throws Exception {
final Server s = new Server(conf);
s.run();
s.run(getSslContext());
try {
put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2",
"sys.cpu.idle " + (TEST_TIME + 1) + " 1.0 tag3=value3 tag4=value4",
@@ -86,7 +86,7 @@ public class HttpApiIT extends OneWaySSLBase {
@Test
public void testMetrics() throws Exception {
final Server s = new Server(conf);
s.run();
s.run(getSslContext());
try {
// @formatter:off
put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2",
@@ -120,7 +120,7 @@ public class HttpApiIT extends OneWaySSLBase {
// @formatter:off
final Server s = new Server(conf);
s.run();
s.run(getSslContext());
try {
put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2",
"sys.cpu.idle " + (TEST_TIME + 1) + " 1.0 tag3=value3 tag4=value4",
@@ -175,7 +175,7 @@ public class HttpApiIT extends OneWaySSLBase {
@Test
public void testLookup() throws Exception {
final Server s = new Server(conf);
s.run();
s.run(getSslContext());
try {
put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2",
"sys.cpu.user " + (TEST_TIME + 1) + " 1.0 tag3=value3",
@@ -211,7 +211,7 @@ public class HttpApiIT extends OneWaySSLBase {
@Test
public void testQueryWithMsResolution() throws Exception {
final Server s = new Server(conf);
s.run();
s.run(getSslContext());
try {
put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2",
"sys.cpu.user " + (TEST_TIME + 1) + " 1.0 tag3=value3",
@@ -259,7 +259,7 @@ public class HttpApiIT extends OneWaySSLBase {
@Test
public void testQueryWithoutMsResolution() throws Exception {
final Server s = new Server(conf);
s.run();
s.run(getSslContext());
try {
put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2",
"sys.cpu.user " + (TEST_TIME + 1) + " 1.0 tag3=value3",
@@ -298,7 +298,7 @@ public class HttpApiIT extends OneWaySSLBase {
@Test
public void testQueryWithNoTags() throws Exception {
final Server s = new Server(conf);
s.run();
s.run(getSslContext());
try {
put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2",
"sys.cpu.user " + (TEST_TIME + 1) + " 1.0 tag3=value3",
@@ -334,7 +334,7 @@ public class HttpApiIT extends OneWaySSLBase {
@Test
public void testQueryWithNoTagsMultipleSeries() throws Exception {
final Server s = new Server(conf);
s.run();
s.run(getSslContext());
try {
put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2 host=h1",
"sys.cpu.user " + TEST_TIME + " 2.0 tag1=value1 tag2=value2 host=h2",
@@ -375,7 +375,7 @@ public class HttpApiIT extends OneWaySSLBase {
@Test(expected = NotSuccessfulException.class)
public void testQueryWithNoMatchingTags() throws Exception {
final Server s = new Server(conf);
s.run();
s.run(getSslContext());
try {
put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2 rack=r1",
"sys.cpu.user " + (TEST_TIME + 1) + " 1.0 tag3=value3 rack=r2",
@@ -399,7 +399,7 @@ public class HttpApiIT extends OneWaySSLBase {
@Test
public void testQueryWithTagWildcard() throws Exception {
final Server s = new Server(conf);
s.run();
s.run(getSslContext());
// @formatter:off
try {
put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2 rack=r1",
@@ -464,7 +464,7 @@ public class HttpApiIT extends OneWaySSLBase {
@Test
public void testQueryWithTagWildcard2() throws Exception {
final Server s = new Server(conf);
s.run();
s.run(getSslContext());
// @formatter:off
try {
put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2 rack=r1",
@@ -524,7 +524,7 @@ public class HttpApiIT extends OneWaySSLBase {
@Test
public void testQueryWithTagOr() throws Exception {
final Server s = new Server(conf);
s.run();
s.run(getSslContext());
try {
// @formatter:off
put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2 rack=r1",
@@ -608,7 +608,7 @@ public class HttpApiIT extends OneWaySSLBase {
@Test
public void testQueryWithTagRegex() throws Exception {
final Server s = new Server(conf);
s.run();
s.run(getSslContext());
try {
System.out.println("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2 rack=r1");
System.out.println("sys.cpu.user " + (TEST_TIME + 1) + " 1.0 tag3=value3 rack=r2");
@@ -674,7 +674,7 @@ public class HttpApiIT extends OneWaySSLBase {
@Test
public void testQueryWithTagRegex2() throws Exception {
final Server s = new Server(conf);
s.run();
s.run(getSslContext());
try {
put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2 rack=r1",
"sys.cpu.user " + (TEST_TIME + 1) + " 1.0 tag3=value3 rack=r2",
@@ -730,7 +730,7 @@ public class HttpApiIT extends OneWaySSLBase {
@Test
public void testRateQuery() throws Exception {
final Server s = new Server(conf);
s.run();
s.run(getSslContext());
try {
// @formatter:off
put("sys.cpu.user " + (TEST_TIME) + " 1.0 tag1=value1 tag2=value2 rack=r1",
@@ -782,7 +782,7 @@ public class HttpApiIT extends OneWaySSLBase {
@Test
public void testRateCounterQuery() throws Exception {
final Server s = new Server(conf);
s.run();
s.run(getSslContext());
try {
// @formatter:off
put("sys.cpu.user " + (TEST_TIME) + " 1.0 tag1=value1 tag2=value2 rack=r1",
@@ -836,7 +836,7 @@ public class HttpApiIT extends OneWaySSLBase {
@Test
public void testRateCounterWithOptions() throws Exception {
final Server s = new Server(conf);
s.run();
s.run(getSslContext());
try {
// @formatter:off
put("sys.cpu.user " + (TEST_TIME) + " 1.0 tag1=value1 tag2=value2 rack=r1",
@@ -892,7 +892,7 @@ public class HttpApiIT extends OneWaySSLBase {
@Test
public void testGetVersion() throws Exception {
final Server s = new Server(conf);
s.run();
s.run(getSslContext());
try {
String response = query("https://127.0.0.1:54322/version", "application/json");
assertNotNull(response);
@@ -905,7 +905,7 @@ public class HttpApiIT extends OneWaySSLBase {
@Test
public void testPutMetric() throws Exception {
final Server s = new Server(conf);
s.run();
s.run(getSslContext());
try {
Metric m = Metric.newBuilder().name("sys.cpu.user").value(TEST_TIME, 1.0D).tag(new Tag("tag1", "value1"))
.build();
@@ -56,7 +56,7 @@ public class TimelyTcpIT extends MacITBase {
@Test
public void testVersion() throws Exception {
final TestServer m = new TestServer(conf);
m.run();
m.run(getSslContext());
try (Socket sock = new Socket("127.0.0.1", 54321);
PrintWriter writer = new PrintWriter(sock.getOutputStream(), true);) {
writer.write("version\n");
@@ -76,7 +76,7 @@ public class TimelyTcpIT extends MacITBase {
@Test
public void testPut() throws Exception {
final TestServer m = new TestServer(conf);
m.run();
m.run(getSslContext());
try (Socket sock = new Socket("127.0.0.1", 54321);
PrintWriter writer = new PrintWriter(sock.getOutputStream(), true);) {
writer.write("put sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2\n");
@@ -107,7 +107,7 @@ public class TimelyTcpIT extends MacITBase {
public void testPutMultiple() throws Exception {
final TestServer m = new TestServer(conf);
m.run();
m.run(getSslContext());
try (Socket sock = new Socket("127.0.0.1", 54321);
PrintWriter writer = new PrintWriter(sock.getOutputStream(), true)) {
// @formatter:off
@@ -190,7 +190,7 @@ public class TimelyTcpIT extends MacITBase {
LOG.debug("Sending {} bytes", data.length);
final TestServer m = new TestServer(conf);
m.run();
m.run(getSslContext());
try (Socket sock = new Socket("127.0.0.1", 54321);) {
sock.getOutputStream().write(data);
sock.getOutputStream().flush();
@@ -233,7 +233,7 @@ public class TimelyTcpIT extends MacITBase {
@Test
public void testPutInvalidTimestamp() throws Exception {
final TestServer m = new TestServer(conf);
m.run();
m.run(getSslContext());
try (Socket sock = new Socket("127.0.0.1", 54321);
PrintWriter writer = new PrintWriter(sock.getOutputStream(), true);
BufferedReader reader = new BufferedReader(new InputStreamReader(sock.getInputStream()));) {
@@ -249,7 +249,7 @@ public class TimelyTcpIT extends MacITBase {
@Test
public void testPersistence() throws Exception {
final Server s = new Server(conf);
s.run();
s.run(getSslContext());
try {
put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2",
"sys.cpu.idle " + (TEST_TIME + 1) + " 1.0 tag3=value3 tag4=value4",
@@ -293,7 +293,7 @@ public class TimelyTcpIT extends MacITBase {
@Test
public void testPersistenceWithVisibility() throws Exception {
final Server s = new Server(conf);
s.run();
s.run(getSslContext());
try {
put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2",
"sys.cpu.idle " + (TEST_TIME + 1) + " 1.0 tag3=value3 tag4=value4 viz=(a|b)",
@@ -60,6 +60,7 @@ public class TimelyUdpIT extends MacITBase {
}
}
});
setupSSL();
}
@After
@@ -70,7 +71,7 @@ public class TimelyUdpIT extends MacITBase {
@Test
public void testPut() throws Exception {
final TestServer m = new TestServer(conf);
m.run();
m.run(getSslContext());
InetSocketAddress address = new InetSocketAddress("127.0.0.1", 54325);
DatagramPacket packet = new DatagramPacket("".getBytes(UTF_8), 0, 0, address.getAddress(), 54325);
try (DatagramSocket sock = new DatagramSocket()) {
@@ -102,7 +103,7 @@ public class TimelyUdpIT extends MacITBase {
public void testPutMultiple() throws Exception {
final TestServer m = new TestServer(conf);
m.run();
m.run(getSslContext());
InetSocketAddress address = new InetSocketAddress("127.0.0.1", 54325);
DatagramPacket packet = new DatagramPacket("".getBytes(UTF_8), 0, 0, address.getAddress(), 54325);
// @formatter:off
@@ -185,7 +186,7 @@ public class TimelyUdpIT extends MacITBase {
LOG.debug("Sending {} bytes", data.length);
final TestServer m = new TestServer(conf);
m.run();
m.run(getSslContext());
InetSocketAddress address = new InetSocketAddress("127.0.0.1", 54325);
DatagramPacket packet = new DatagramPacket("".getBytes(UTF_8), 0, 0, address.getAddress(), 54325);
try (DatagramSocket sock = new DatagramSocket()) {
@@ -229,7 +230,7 @@ public class TimelyUdpIT extends MacITBase {
@Test
public void testPutInvalidTimestamp() throws Exception {
final TestServer m = new TestServer(conf);
m.run();
m.run(getSslContext());
InetSocketAddress address = new InetSocketAddress("127.0.0.1", 54325);
DatagramPacket packet = new DatagramPacket("".getBytes(UTF_8), 0, 0, address.getAddress(), 54325);
try (DatagramSocket sock = new DatagramSocket();) {
@@ -245,7 +246,7 @@ public class TimelyUdpIT extends MacITBase {
@Test
public void testPersistence() throws Exception {
final Server s = new Server(conf);
s.run();
s.run(getSslContext());
try {
put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2",
"sys.cpu.idle " + (TEST_TIME + 1) + " 1.0 tag3=value3 tag4=value4",
@@ -289,7 +290,7 @@ public class TimelyUdpIT extends MacITBase {
@Test
public void testPersistenceWithVisibility() throws Exception {
final Server s = new Server(conf);
s.run();
s.run(getSslContext());
try {
put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2",
"sys.cpu.idle " + (TEST_TIME + 1) + " 1.0 tag3=value3 tag4=value4 viz=(a|b)",
@@ -189,7 +189,7 @@ public class WebSocketIT extends OneWaySSLBase {
@Before
public void setup() throws Exception {
s = new Server(conf);
s.run();
s.run(getSslContext());
Connector con = mac.getConnector("root", "secret");
con.securityOperations().changeUserAuthorizations("root", new Authorizations("A", "B", "C", "D", "E", "F"));