Compare commits

..
Author SHA1 Message Date
Bill Oley f582e8ca1c Refactor SSL integration tests to use the same SslContext when possible 2020-04-27 09:29:05 -04:00
Bill Oley 6972c46a1a Merge pull request #204 from matthpeterson/issue-203
Closes issue #203 - avoid calling sizeInBytes if it's not expected to exist
2020-04-17 10:42:14 -04:00
matthpeterson 4d7bee7566 Closes issue #203 - Stop calling sizeInBytes if it's not expected to exist 2020-04-16 15:17:08 +00:00
Bill Oley 4b11b3d50f closes #201 Create balancer assignment file if it doesn't exist 2020-04-10 10:27:18 -04:00
Bill Oley 84b9856ec7 closes #199 Fix issues when using a generated (self-signed) server certificate 2020-04-10 10:20:25 -04:00
Bill Oley c9548a9b9a closes #198 Cleaner shutdown of resources for Timely server, balancer, grafana-auth 2020-04-10 10:01:47 -04:00
Bill Oley aec7bd4c38 closes #197 Fix scan range in MetaCacheImpl 2020-04-09 11:58:30 -04:00
Bill Oley e2b8b643f2 closes #200 Initialize non-cached metrics stored in zookeeper 2020-04-09 11:32:03 -04:00
Bill Oley c19e1d8b2f Set version to 1.2.8-SNAPSHOT 2020-02-21 10:20:48 -05:00
Bill Oley cb73e8e75c Tag version 1.2.7 2020-02-21 10:02:31 -05:00
Bill Oley 4c508c1a7d Merge pull request #195 from dtspence/TIMELY-194
Fixed #194, corrected datasource.js to look at wildcard tag value
2020-02-21 08:55:37 -05:00
dtspence c6c2c0d41e Fixed #194, corrected datasource.js to look at wildcard tag value
The alignment process to match tag query results with the metric definition
looks at the tag k/v pairs and makes a special consideration for a wildcard tag value.
The change adjusts datasource.js so that the '.*' notation will match a query result
and metric definition.
2020-02-05 19:56:03 +00:00
Christopher Tubbs a24cef78b3 Update jackson (multiple CVEs) (#193) 2019-11-07 23:05:38 -05:00
Bill Oley 08f99f6540 Set version to 1.2.7-SNAPSHOT 2019-10-30 12:33:49 -04:00
59 changed files with 931 additions and 783 deletions

No files matched your search

+1 -1
View File
@@ -4,7 +4,7 @@
<parent>
<groupId>timely</groupId>
<artifactId>timely</artifactId>
<version>1.2.6</version>
<version>1.2.8-SNAPSHOT</version>
</parent>
<artifactId>timely-analytics</artifactId>
<!--
+1 -1
View File
@@ -4,7 +4,7 @@
<parent>
<groupId>timely</groupId>
<artifactId>timely</artifactId>
<version>1.2.6</version>
<version>1.2.8-SNAPSHOT</version>
</parent>
<artifactId>timely-balancer</artifactId>
<dependencies>
@@ -1,7 +1,7 @@
package timely.balancer;
import java.util.Timer;
import java.util.TimerTask;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
import com.google.common.util.concurrent.AtomicDouble;
@@ -20,38 +20,26 @@ public class ArrivalRate {
private AtomicDouble longTermRate = new AtomicDouble(0);
private AtomicLong longTermLastReset = new AtomicLong(0);
public ArrivalRate(Timer timer) {
public ArrivalRate(ScheduledExecutorService arrivalRateExecutor) {
long now = System.currentTimeMillis();
shortTermLastReset.set(now);
mediumTermLastReset.set(now);
longTermLastReset.set(now);
// 2 minute
timer.schedule(new TimerTask() {
@Override
public void run() {
resetShort();
}
}, 120000, 120000);
arrivalRateExecutor.scheduleAtFixedRate(() -> {
resetShort();
}, 2, 2, TimeUnit.MINUTES);
// 5 minute
timer.schedule(new TimerTask() {
@Override
public void run() {
resetMedium();
}
}, 300000, 300000);
arrivalRateExecutor.scheduleAtFixedRate(() -> {
resetMedium();
}, 5, 5, TimeUnit.MINUTES);
// 15 minute
timer.schedule(new TimerTask() {
@Override
public void run() {
resetLong();
}
}, 900000, 900000);
arrivalRateExecutor.scheduleAtFixedRate(() -> {
resetLong();
}, 15, 15, TimeUnit.MINUTES);
}
private void resetShort() {
@@ -145,8 +145,11 @@ public class Balancer {
this.balancerConfig = balancerConf;
}
protected static ConfigurableApplicationContext initializeConfiguration(String[] args) {
return new SpringApplicationBuilder(SpringBootstrap.class).web(WebApplicationType.NONE).run(args);
protected static ConfigurableApplicationContext initializeContext(String[] args) {
SpringApplicationBuilder builder = new SpringApplicationBuilder(SpringBootstrap.class);
builder.web(WebApplicationType.NONE);
builder.registerShutdownHook(false);
return builder.run(args);
}
private void shutdownHook() {
@@ -178,8 +181,9 @@ public class Balancer {
channelFutures.add(wsChannelHandle.close());
}
int udpChannel = 1;
for (Channel c : udpChannelHandleList) {
LOG.info("Closing udpChannelHandle");
LOG.info("Closing udpChannelHandle #" + udpChannel++);
channelFutures.add(c.close());
}
@@ -188,14 +192,13 @@ public class Balancer {
try {
f.get();
} catch (final Exception e) {
LOG.error("Error while shutting down channel: {}", f.channel().config());
LOG.error("{}", e.getMessage());
e.printStackTrace();
LOG.error("Channel:" + f.channel().config() + " -> " + e.getMessage(), e);
}
});
Integer quietPeriod = balancerConfig.getServer().getShutdownQuietPeriod();
int quietPeriod = balancerConfig.getServer().getShutdownQuietPeriod();
List<Future<?>> groupFutures = new ArrayList<>();
if (tcpBossGroup != null) {
LOG.info("Shutting down tcpBossGroup");
groupFutures.add(tcpBossGroup.shutdownGracefully(quietPeriod, 10, TimeUnit.SECONDS));
@@ -240,9 +243,7 @@ public class Balancer {
try {
f.get();
} catch (final Exception e) {
LOG.error("Error while shutting down group: {}", f.toString());
LOG.error("{}", e.getMessage());
e.printStackTrace();
LOG.error("Group:" + f.toString() + " -> " + e.getMessage(), e);
}
});
@@ -253,14 +254,29 @@ public class Balancer {
LOG.error("Error closing WebSocketRequestDecoder during shutdown", e);
}
if (this.metricResolver != null) {
try {
LOG.info("Closing metricResolver");
this.metricResolver.close();
} catch (Exception e) {
LOG.error(e.getMessage(), e);
}
}
if (curatorFramework != null) {
try {
LOG.info("Closing curatorFramework");
curatorFramework.close();
} catch (Exception e) {
LOG.error(e.getMessage(), e);
}
}
if (applicationContext != null) {
LOG.info("Closing applicationContext");
applicationContext.close();
}
if (curatorFramework != null) {
curatorFramework.close();
}
this.shutdown = true;
LOG.info("Server shut down.");
}
@@ -481,12 +497,12 @@ public class Balancer {
public static void main(String[] args) throws Exception {
Balancer.applicationContext = Balancer.initializeConfiguration(args);
BalancerConfiguration balancerConf = applicationContext.getBean(BalancerConfiguration.class);
Balancer.applicationContext = Balancer.initializeContext(args);
BalancerConfiguration balancerConf = Balancer.applicationContext.getBean(BalancerConfiguration.class);
Balancer b = new Balancer(balancerConf);
Balancer balancer = new Balancer(balancerConf);
try {
b.run();
balancer.run();
LATCH.await();
} catch (final InterruptedException e) {
LOG.info("Server shutting down.");
@@ -494,7 +510,7 @@ public class Balancer {
LOG.error("Error running server.", e);
} finally {
try {
b.shutdown();
balancer.shutdown();
} catch (Exception e) {
System.exit(1);
}
@@ -518,11 +534,13 @@ public class Balancer {
ch.pipeline().addLast("chunker", new ChunkedWriteHandler());
ch.pipeline().addLast("queryDecoder", new timely.netty.http.HttpRequestDecoder(
balancerConfig.getSecurity(), balancerConfig.getHttp()));
ch.pipeline().addLast("fileServer", new HttpStaticFileServerHandler());
ch.pipeline().addLast("fileServer", new HttpStaticFileServerHandler().setIgnoreSslHandshakeErrors(
balancerConfig.getSecurity().getServerSsl().isUseGeneratedKeypair()));
ch.pipeline().addLast("login",
new X509LoginRequestHandler(balancerConfig.getSecurity(), balancerConfig.getHttp()));
ch.pipeline().addLast("httpRelay", new HttpRelayHandler(metricResolver, httpClientPool));
ch.pipeline().addLast("error", new TimelyExceptionHandler());
ch.pipeline().addLast("error", new TimelyExceptionHandler().setIgnoreSslHandshakeErrors(
balancerConfig.getSecurity().getServerSsl().isUseGeneratedKeypair()));
}
};
}
@@ -1,7 +1,5 @@
package timely.balancer.connection;
import java.util.Timer;
import org.apache.commons.lang3.builder.EqualsBuilder;
import org.apache.commons.lang3.builder.HashCodeBuilder;
import org.apache.commons.lang3.builder.ToStringBuilder;
@@ -13,7 +11,6 @@ import timely.balancer.configuration.BalancerConfiguration;
public class TimelyBalancedHost {
private static final Logger LOG = LoggerFactory.getLogger(TimelyBalancedHost.class);
static private Timer arrivalRateTimer = new Timer("ArrivalRateTimerTBH", true);
private String host;
private int tcpPort;
@@ -24,34 +21,25 @@ public class TimelyBalancedHost {
private BalancerConfiguration balancerConfig;
private int failures = 0;
private int successes = 0;
private ArrivalRate arrivalRate = new ArrivalRate(arrivalRateTimer);
private ArrivalRate arrivalRate = null;
public TimelyBalancedHost() {
private TimelyBalancedHost() {
}
public TimelyBalancedHost(String host, int tcpPort, int httpPort, int wsPort, int udpPort) {
private TimelyBalancedHost(String host, int tcpPort, int httpPort, int wsPort, int udpPort,
ArrivalRate arrivalRate) {
this.host = host;
this.tcpPort = tcpPort;
this.httpPort = httpPort;
this.wsPort = wsPort;
this.udpPort = udpPort;
this.arrivalRate = arrivalRate;
}
public TimelyBalancedHost(String host, int basePort) {
this.host = host;
this.tcpPort = basePort;
this.httpPort = basePort + 1;
this.wsPort = basePort + 2;
this.udpPort = basePort + 3;
}
static public TimelyBalancedHost of(String host, int tcpPort, int httpPort, int wsPort, int udpPort) {
return new TimelyBalancedHost(host, tcpPort, httpPort, wsPort, udpPort);
}
static public TimelyBalancedHost of(String host, int basePort) {
return new TimelyBalancedHost(host, basePort);
static public TimelyBalancedHost of(String host, int tcpPort, int httpPort, int wsPort, int udpPort,
ArrivalRate arrivalRate) {
return new TimelyBalancedHost(host, tcpPort, httpPort, wsPort, udpPort, arrivalRate);
}
public void setBalancerConfig(BalancerConfiguration balancerConfig) {
@@ -141,11 +129,17 @@ public class TimelyBalancedHost {
}
public void arrived() {
arrivalRate.arrived();
if (this.arrivalRate != null) {
this.arrivalRate.arrived();
}
}
public double getArrivalRate() {
return arrivalRate.getRate();
if (this.arrivalRate == null) {
return 0;
} else {
return this.arrivalRate.getRate();
}
}
@Override
@@ -2,8 +2,9 @@ package timely.balancer.healthcheck;
import java.util.ArrayList;
import java.util.List;
import java.util.Timer;
import java.util.TimerTask;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -15,12 +16,12 @@ public class HealthChecker {
private static final Logger LOG = LoggerFactory.getLogger(HealthChecker.class);
private List<TimelyBalancedHost> timelyHosts = new ArrayList<>();
private Timer timer;
private ScheduledExecutorService executorService = Executors.newScheduledThreadPool(1);
final private TCPServerHealthCheck tcpServerHealthCheck;
private TimerTask check = new TimerTask() {
@Override
public void run() {
public HealthChecker(BalancerConfiguration config, TcpClientPool tcpClientPool) {
this.tcpServerHealthCheck = new TCPServerHealthCheck(tcpClientPool);
this.executorService.scheduleAtFixedRate(() -> {
synchronized (timelyHosts) {
for (TimelyBalancedHost h : timelyHosts) {
try {
@@ -35,13 +36,7 @@ public class HealthChecker {
}
}
}
}
};
public HealthChecker(BalancerConfiguration config, TcpClientPool tcpClientPool) {
this.tcpServerHealthCheck = new TCPServerHealthCheck(tcpClientPool);
this.timer = new Timer("HealthCheckerThread", true);
this.timer.scheduleAtFixedRate(check, 0, config.getCheckServerHealthInterval());
}, 0, config.getCheckServerHealthInterval(), TimeUnit.MILLISECONDS);
}
public void setTimelyHosts(List<TimelyBalancedHost> timelyHosts) {
@@ -50,4 +45,17 @@ public class HealthChecker {
this.timelyHosts.addAll(timelyHosts);
}
}
public void close() throws Exception {
this.executorService.shutdown();
try {
this.executorService.awaitTermination(5, TimeUnit.SECONDS);
} catch (InterruptedException e) {
} finally {
if (!this.executorService.isTerminated()) {
this.executorService.shutdownNow();
}
}
}
}
@@ -7,6 +7,7 @@ import static timely.balancer.Balancer.LEADER_LATCH_PATH;
import static timely.store.cache.DataStoreCache.NON_CACHED_METRICS;
import static timely.store.cache.DataStoreCache.NON_CACHED_METRICS_LOCK_PATH;
import java.io.FileNotFoundException;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Collection;
@@ -19,10 +20,10 @@ import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.Set;
import java.util.Timer;
import java.util.TimerTask;
import java.util.TreeMap;
import java.util.TreeSet;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
@@ -71,8 +72,8 @@ public class BalancedMetricResolver implements MetricResolver {
private List<TimelyBalancedHost> serverList = new ArrayList<>();
private Random r = new Random();
final private HealthChecker healthChecker;
private Timer timer = new Timer("RebalanceTimer", true);
private Timer arrivalRateTimer = new Timer("ArrivalRateTimerResolver", true);
private ScheduledExecutorService assignmentExecutor = Executors.newScheduledThreadPool(2);
private ScheduledExecutorService arrivalRateExecutor = Executors.newScheduledThreadPool(2);
private int roundRobinCounter = 0;
private Set<String> nonCachedMetrics = new HashSet<>();
private DistributedAtomicValue nonCachedMetricsIP;
@@ -151,45 +152,36 @@ public class BalancedMetricResolver implements MetricResolver {
readAssignmentsFromHdfs(false);
timer.schedule(new TimerTask() {
@Override
public void run() {
if (isLeader.get()) {
try {
balance();
} catch (Exception e) {
LOG.error(e.getMessage(), e);
}
}
}
}, 900000, 900000);
timer.schedule(new TimerTask() {
@Override
public void run() {
assignmentExecutor.scheduleAtFixedRate(() -> {
if (isLeader.get()) {
try {
if (isLeader.get()) {
long lastLocalUpdate = assignmentsLastUpdatedLocal.get();
long lastHdfsUpdate = assignmentsLastUpdatedInHdfs.get().postValue();
if (lastLocalUpdate > lastHdfsUpdate) {
LOG.debug("Leader writing assignments to hdfs lastLocalUpdate ({}) > lastHdfsUpdate ({})",
new Date(lastLocalUpdate), new Date(lastHdfsUpdate));
writeAssignmentsToHdfs();
} else {
LOG.trace(
"Leader not writing assignments to hdfs lastLocalUpdate ({}) <= lastHdfsUpdate ({})",
new Date(lastLocalUpdate), new Date(lastHdfsUpdate));
}
} else {
readAssignmentsFromHdfs(true);
}
balance();
} catch (Exception e) {
LOG.error(e.getMessage(), e);
}
}
}, 10000, 60000);
}, 15, 15, TimeUnit.MINUTES);
assignmentExecutor.scheduleAtFixedRate(() -> {
try {
if (isLeader.get()) {
long lastLocalUpdate = assignmentsLastUpdatedLocal.get();
long lastHdfsUpdate = assignmentsLastUpdatedInHdfs.get().postValue();
if (lastLocalUpdate > lastHdfsUpdate) {
LOG.debug("Leader writing assignments to hdfs lastLocalUpdate ({}) > lastHdfsUpdate ({})",
new Date(lastLocalUpdate), new Date(lastHdfsUpdate));
writeAssignmentsToHdfs();
} else {
LOG.trace("Leader not writing assignments to hdfs lastLocalUpdate ({}) <= lastHdfsUpdate ({})",
new Date(lastLocalUpdate), new Date(lastHdfsUpdate));
}
} else {
readAssignmentsFromHdfs(true);
}
} catch (Exception e) {
LOG.error(e.getMessage(), e);
}
}, 10, 60, TimeUnit.SECONDS);
}
private void readNonCachedMetricsIP() {
@@ -273,7 +265,7 @@ public class BalancedMetricResolver implements MetricResolver {
for (ServiceInstance<ServerDetails> si : instances) {
ServerDetails pl = si.getPayload();
TimelyBalancedHost tbh = TimelyBalancedHost.of(pl.getHost(), pl.getTcpPort(), pl.getHttpPort(),
pl.getWsPort(), pl.getUdpPort());
pl.getWsPort(), pl.getUdpPort(), new ArrivalRate(arrivalRateExecutor));
LOG.info("adding service {} host:{} tcpPort:{} httpPort:{} wsPort:{} udpPort:{}", si.getId(),
pl.getHost(), pl.getTcpPort(), pl.getHttpPort(), pl.getWsPort(), pl.getUdpPort());
tbh.setBalancerConfig(balancerConfig);
@@ -298,7 +290,8 @@ public class BalancedMetricResolver implements MetricResolver {
for (ServiceInstance<ServerDetails> si : instances) {
ServerDetails pl = (ServerDetails) si.getPayload();
TimelyBalancedHost tbh = TimelyBalancedHost.of(pl.getHost(), pl.getTcpPort(),
pl.getHttpPort(), pl.getWsPort(), pl.getUdpPort());
pl.getHttpPort(), pl.getWsPort(), pl.getUdpPort(),
new ArrivalRate(arrivalRateExecutor));
tbh.setBalancerConfig(balancerConfig);
availableHosts.add(tbh);
}
@@ -747,7 +740,7 @@ public class BalancedMetricResolver implements MetricResolver {
if (chooseMetricSpecificHost) {
ArrivalRate rate = metricMap.get(metric);
if (rate == null) {
rate = new ArrivalRate(arrivalRateTimer);
rate = new ArrivalRate(arrivalRateExecutor);
if (!balancerLock.isWriteLockedByCurrentThread()) {
balancerLock.readLock().unlock();
balancerLock.writeLock().lock();
@@ -878,6 +871,18 @@ public class BalancedMetricResolver implements MetricResolver {
return tbh;
}
private void createAssignmentFile() {
try {
LOG.info("Creating assignment file: " + balancerConfig.getAssignmentFile());
Configuration configuration = new Configuration();
FileSystem fs = FileSystem.get(configuration);
Path assignmentFile = new Path(balancerConfig.getAssignmentFile());
fs.create(assignmentFile);
} catch (Exception e) {
LOG.error(e.getMessage(), e);
}
}
private void readAssignmentsFromHdfs(boolean checkIfNecessary) {
try {
@@ -962,6 +967,9 @@ public class BalancedMetricResolver implements MetricResolver {
assignmentsLastUpdatedLocal.set(assignmentsLastUpdatedInHdfs.get().postValue());
LOG.info("Read {} assignments from hdfs lastHdfsUpdate = lastLocalUpdate ({})", metricToHostMap.size(),
new Date(assignmentsLastUpdatedLocal.get()));
} catch (FileNotFoundException e) {
createAssignmentFile();
metricToHostMap.clear();
} catch (Exception e) {
LOG.error(e.getMessage(), e);
} finally {
@@ -1053,4 +1061,36 @@ public class BalancedMetricResolver implements MetricResolver {
}
}
}
@Override
public void close() throws Exception {
this.assignmentExecutor.shutdown();
try {
this.assignmentExecutor.awaitTermination(5, TimeUnit.SECONDS);
} catch (InterruptedException e) {
} finally {
if (!this.assignmentExecutor.isTerminated()) {
this.assignmentExecutor.shutdownNow();
}
}
this.arrivalRateExecutor.shutdown();
try {
this.arrivalRateExecutor.awaitTermination(5, TimeUnit.SECONDS);
} catch (InterruptedException e) {
} finally {
if (!this.arrivalRateExecutor.isTerminated()) {
this.arrivalRateExecutor.shutdownNow();
}
}
try {
this.healthChecker.close();
} catch (Exception e) {
LOG.error(e.getMessage(), e);
}
}
}
@@ -7,4 +7,6 @@ public interface MetricResolver {
public TimelyBalancedHost getHostPortKeyIngest(String metric);
public TimelyBalancedHost getHostPortKey(String metric);
public void close() throws Exception;
}
+2 -2
View File
@@ -4,13 +4,13 @@
<parent>
<groupId>timely</groupId>
<artifactId>timely</artifactId>
<version>1.2.6</version>
<version>1.2.8-SNAPSHOT</version>
</parent>
<artifactId>timely-client</artifactId>
<properties>
<!-- PEP 440 compliant version gets substituted into src/main/python/template/setup.py -->
<!-- use .dev0 instead of -SNAPSHOT -->
<python.client.version>1.2.6</python.client.version>
<python.client.version>1.2.8.dev0</python.client.version>
</properties>
<dependencies>
<dependency>
@@ -22,7 +22,7 @@ import timely.model.parse.TagParser;
@JsonRootName("metric")
@JsonInclude(JsonInclude.Include.NON_NULL)
@JsonPropertyOrder({ "name" })
public class Metric implements Serializable {
public class Metric implements Serializable, ObjectSizeOf {
private static final long serialVersionUID = 1L;
@@ -186,4 +186,9 @@ public class Metric implements Serializable {
return this.metric;
}
}
@Override
public long sizeInBytes() {
return 0;
}
}
@@ -2,13 +2,16 @@ package timely.model;
import java.lang.reflect.Array;
import java.lang.reflect.Field;
import java.lang.reflect.Method;
import java.lang.reflect.Modifier;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.Stack;
import java.util.logging.Logger;
import org.apache.commons.lang3.StringUtils;
/**
* A simple interface that objects can implement to return the object size.
*/
@@ -22,15 +25,29 @@ public interface ObjectSizeOf {
public static class ObjectInstance {
private Object o;
private String name = "";
public ObjectInstance(Object _o) {
this.o = _o;
}
public ObjectInstance(Object _o, String name) {
this.o = _o;
this.name = name;
}
public Object getObject() {
return o;
}
public void setName(String name) {
this.name = name;
}
public String getName() {
return name;
}
@Override
public int hashCode() {
return System.identityHashCode(o);
@@ -64,110 +81,134 @@ public interface ObjectSizeOf {
*
* @return an approximation of the object size
*/
public static long getObjectSize(Object o, boolean useSizeInBytesMethod, boolean debug) {
long estSize = getObjectSize(o, new HashSet<ObjectInstance>(), new Stack<ObjectInstance>(),
useSizeInBytesMethod, debug);
return estSize;
}
public static long getObjectSize(Object o) {
long estSize = getObjectSize(o, new HashSet<ObjectInstance>(), new Stack<ObjectInstance>(), true, false);
return estSize;
return getObjectSize(o, true, false);
}
public static long getObjectSize(Object o, Set<ObjectInstance> visited, Stack<ObjectInstance> stack,
boolean useSizeInBytesMethod, boolean debug) {
long totalSize = 0;
stack.add(new ObjectInstance(o));
while (!stack.isEmpty()) {
long size = 0;
ObjectInstance oi = stack.pop();
o = oi.getObject();
if (o != null && !visited.contains(oi)) {
visited.add(oi);
try {
if (useSizeInBytesMethod) {
try {
if (o instanceof ObjectSizeOf) {
size = ((ObjectSizeOf) o).sizeInBytes();
} else {
Method sizeInBytes = o.getClass().getMethod("sizeInBytes", (Class<?>[]) null);
size = (Long) sizeInBytes.invoke(o);
}
} catch (Throwable t) {
// ok, lets do this the hard way...
}
}
if (size == 0) {
// the hard way...
// do not include Class related objects or
// reflection objects
if ((o instanceof Class) || (o instanceof ClassLoader) || (o.getClass().getPackage() != null
&& o.getClass().getPackage().getName().startsWith("java.lang.reflect"))) {
size = 0;
} else if (o instanceof Number || o instanceof Boolean || o instanceof Character) {
size = NUMBER_SIZE;
} else {
// lets do a simple sizing
Class<?> c = o.getClass();
if (c.isArray()) {
size = ARRAY_OVERHEAD;
int length = Array.getLength(o);
if (c.getComponentType().isPrimitive()) {
size += length * getPrimitiveObjectSize(c.getComponentType());
} else {
size += length * REFERENCE;
for (int i = 0; i < length; i++) {
Object element = Array.get(o, i);
if (element != null) {
stack.add(new ObjectInstance(element));
}
}
}
} else {
size += OBJECT_OVERHEAD;
while (c != null) {
for (Field field : c.getDeclaredFields()) {
if (Modifier.isStatic(field.getModifiers())) {
continue;
}
if (field.getType().isPrimitive()) {
size += getPrimitiveObjectSize(field.getType());
} else {
size += REFERENCE;
boolean accessible = field.isAccessible();
field.setAccessible(true);
try {
Object fieldObject = field.get(o);
if (fieldObject != null) {
stack.push(new ObjectInstance(fieldObject));
}
} catch (Exception e) {
// cannot get to field, so
// ignore it in this size
// calculation
e.printStackTrace();
}
field.setAccessible(accessible);
}
}
c = c.getSuperclass();
}
}
size = roundUp(size);
}
}
} catch (Throwable t) {
log.warning("Unable to determine object size for " + o);
}
if (debug) {
System.out.println(o.getClass().getCanonicalName() + "=" + size);
}
}
totalSize += size;
public static long getObjectSize(Object o, boolean useSizeInBytesMethod, boolean debug) {
List<String> output = null;
if (debug) {
output = new ArrayList<>();
}
long size = getObjectSize(new ObjectInstance(o, o.getClass().getSimpleName()), new HashSet<>(), 0,
useSizeInBytesMethod, output);
if (debug) {
Collections.reverse(output);
for (String s : output)
System.out.println(s);
}
return size;
}
return totalSize;
public static long getObjectSize(ObjectInstance oi, Set<ObjectInstance> visited, int indent,
boolean useSizeInBytesMethod, List<String> output) {
long size = 0;
Object o = oi.getObject();
int arrayOverhead = 0;
int objectOverhead = 0;
int reference = 0;
boolean roundUp = false;
if (o != null && !visited.contains(oi)) {
visited.add(oi);
try {
if (useSizeInBytesMethod) {
if (o instanceof ObjectSizeOf) {
size = ((ObjectSizeOf) o).sizeInBytes();
if (output != null && size > 0) {
output.add(StringUtils.repeat(' ', indent) + oi.getName() + " [" + size + "] ("
+ o.getClass().getCanonicalName() + ") [ObjectSizeOf]");
}
}
}
if (size == 0) {
// the hard way...
// do not include Class related objects or
// reflection objects
if ((o instanceof Class) || (o instanceof ClassLoader) || (o.getClass().getPackage() != null
&& o.getClass().getPackage().getName().startsWith("java.lang.reflect"))) {
size = 0;
} else if (o instanceof Number || o instanceof Boolean || o instanceof Character) {
size = NUMBER_SIZE;
} else {
// lets do a simple sizing
Class<?> c = o.getClass();
if (c.isArray()) {
arrayOverhead++;
int length = Array.getLength(o);
if (c.getComponentType().isPrimitive()) {
size += length * getPrimitiveObjectSize(c.getComponentType());
} else {
reference += length;
for (int i = 0, index = length - 1; i < length; i++, index--) {
Object element = Array.get(o, index);
if (element != null) {
size += getObjectSize(
new ObjectInstance(element, oi.getName() + "[" + index + "]"),
visited, indent + 2, useSizeInBytesMethod, output);
}
}
}
} else {
objectOverhead++;
while (c != null) {
for (Field field : c.getDeclaredFields()) {
if (Modifier.isStatic(field.getModifiers())) {
continue;
}
if (field.getType().isPrimitive()) {
long primSize = getPrimitiveObjectSize(field.getType());
if (output != null) {
output.add(StringUtils.repeat(' ', indent + 2) + field.getName() + " ["
+ primSize + "] (" + field.getType().getCanonicalName() + ")");
}
size += primSize;
} else {
reference++;
boolean accessible = field.isAccessible();
field.setAccessible(true);
try {
Object fieldObject = field.get(o);
if (fieldObject != null) {
size += getObjectSize(
new ObjectInstance(fieldObject, field.getName()), visited,
indent + 2, useSizeInBytesMethod, output);
}
} catch (Exception e) {
// cannot get to field, so
// ignore it in this size
// calculation
e.printStackTrace();
}
field.setAccessible(accessible);
}
}
c = c.getSuperclass();
}
}
roundUp = true;
}
}
long underlyingObjects = size;
long overheads = REFERENCE * reference + OBJECT_OVERHEAD * objectOverhead
+ ARRAY_OVERHEAD * arrayOverhead;
long addedOverheads = size + overheads;
if (roundUp) {
size = roundUp(addedOverheads);
} else {
size = addedOverheads;
}
if (output != null) {
output.add(StringUtils.repeat(' ', indent) + oi.getName() + " [" + size + "] ("
+ o.getClass().getCanonicalName() + ") = " + underlyingObjects + "+" + overheads + "+"
+ (size - addedOverheads) + " REF=" + reference + " OBJ=" + objectOverhead + " ARRAY="
+ arrayOverhead + " ROUND=" + (size - addedOverheads));
}
} catch (Throwable t) {
log.warning("Unable to determine object size for " + o);
}
}
return size;
}
public static long roundUp(long size) {
@@ -191,6 +232,42 @@ public interface ObjectSizeOf {
return 0;
}
}
public static long getStringSize(String s) {
long size = 0;
size += ObjectSizeOf.Sizer.OBJECT_OVERHEAD;
size += ObjectSizeOf.Sizer.REFERENCE;
size += ObjectSizeOf.Sizer.getPrimitiveObjectSize(int.class);
try {
// change with Java 9
if (String.class.getDeclaredField("coder") != null) {
size += ObjectSizeOf.Sizer.getPrimitiveObjectSize(byte.class);
}
} catch (Exception e) {
}
size += roundUp(ARRAY_OVERHEAD + (s.length() * ObjectSizeOf.Sizer.getPrimitiveObjectSize(byte.class)));
return roundUp(size);
}
}
public static void main(String[] args) {
Metric m = new Metric("mymetric", 100000l, 32);
m.addTag(new Tag("key1", "value1"));
m.addTag(new Tag("key2", "value2"));
m.addTag(new Tag("key3", "value3"));
m.addTag(new Tag("key4", "value4"));
m.addTag(new Tag("key5", "value5"));
long newSize1 = ObjectSizeOf.Sizer.getObjectSize(m, false, true);
System.out.println("new size #1: " + newSize1);
System.out.println();
System.out.println();
System.out.println();
long newSize2 = ObjectSizeOf.Sizer.getObjectSize(m, true, true);
System.out.println("new size #2: " + newSize2);
}
}
+32 -1
View File
@@ -18,7 +18,7 @@ import timely.model.parse.TagParser;
* Tag consists of key value pair
*/
@JsonRootName("tag")
public class Tag implements Comparable<Tag>, Serializable {
public class Tag implements Comparable<Tag>, ObjectSizeOf, Serializable {
private static final long serialVersionUID = 1L;
@@ -27,6 +27,12 @@ public class Tag implements Comparable<Tag>, Serializable {
private String key;
private String value;
@JsonIgnore
private boolean mustRecalculateLength = true;
@JsonIgnore
private long length;
public Tag() {
}
@@ -55,6 +61,7 @@ public class Tag implements Comparable<Tag>, Serializable {
public void set(String key, String value) {
this.key = key;
this.value = value;
this.mustRecalculateLength = true;
}
@JsonIgnore
@@ -64,6 +71,7 @@ public class Tag implements Comparable<Tag>, Serializable {
public void setKey(String key) {
this.key = key;
this.mustRecalculateLength = true;
}
@JsonIgnore
@@ -73,6 +81,7 @@ public class Tag implements Comparable<Tag>, Serializable {
public void setValue(String value) {
this.value = value;
this.mustRecalculateLength = true;
}
/**
@@ -106,4 +115,26 @@ public class Tag implements Comparable<Tag>, Serializable {
public int compareTo(Tag other) {
return new CompareToBuilder().append(this.key, other.key).append(this.value, other.value).toComparison();
}
@JsonIgnore
@Override
public long sizeInBytes() {
if (this.mustRecalculateLength) {
calculateLength();
this.mustRecalculateLength = false;
}
return this.length;
}
private void calculateLength() {
long l = 0;
l += 2 * Sizer.REFERENCE;
l += Sizer.OBJECT_OVERHEAD;
l += Sizer.getPrimitiveObjectSize(long.class);
l += Sizer.getPrimitiveObjectSize(boolean.class);
l += Sizer.getStringSize(key);
l += Sizer.getStringSize(value);
this.length = Sizer.roundUp(l);
}
}
+13 -1
View File
@@ -5,13 +5,14 @@ import java.util.Comparator;
import java.util.concurrent.TimeUnit;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Objects;
/**
* Value consists of a long timestamp and a double measurement
*/
public class Value implements Comparator<Value>, Serializable {
public class Value implements Comparator<Value>, ObjectSizeOf, Serializable {
private static final long serialVersionUID = 1L;
@@ -90,4 +91,15 @@ public class Value implements Comparator<Value>, Serializable {
public String toString() {
return "Value{" + "timestamp=" + timestamp + ", measure=" + measure + "}";
}
@JsonIgnore
@Override
public long sizeInBytes() {
long l = 0;
l += Sizer.OBJECT_OVERHEAD;
l += 2 * Sizer.REFERENCE;
l += Sizer.NUMBER_SIZE;
l += Sizer.NUMBER_SIZE;
return Sizer.roundUp(l);
}
}
+1 -1
View File
@@ -4,7 +4,7 @@
<parent>
<groupId>timely</groupId>
<artifactId>timely</artifactId>
<version>1.2.6</version>
<version>1.2.8-SNAPSHOT</version>
</parent>
<artifactId>collectd-nsq-plugin</artifactId>
<dependencies>
+1 -1
View File
@@ -4,7 +4,7 @@
<parent>
<groupId>timely</groupId>
<artifactId>timely</artifactId>
<version>1.2.6</version>
<version>1.2.8-SNAPSHOT</version>
</parent>
<artifactId>collectd-plugin-parent</artifactId>
<dependencies>
+1 -1
View File
@@ -4,7 +4,7 @@
<parent>
<groupId>timely</groupId>
<artifactId>timely</artifactId>
<version>1.2.6</version>
<version>1.2.8-SNAPSHOT</version>
</parent>
<artifactId>collectd-timely-plugin</artifactId>
<dependencies>
+1 -1
View File
@@ -4,7 +4,7 @@
<parent>
<groupId>timely</groupId>
<artifactId>timely</artifactId>
<version>1.2.6</version>
<version>1.2.8-SNAPSHOT</version>
</parent>
<artifactId>timely-common</artifactId>
<dependencies>
+1 -1
View File
@@ -4,7 +4,7 @@
<parent>
<groupId>timely</groupId>
<artifactId>timely</artifactId>
<version>1.2.6</version>
<version>1.2.8-SNAPSHOT</version>
</parent>
<artifactId>grafana-auth</artifactId>
<dependencies>
@@ -81,8 +81,11 @@ public class GrafanaAuth {
this.grafanaAuthConfig = grafanaAuthConfig;
}
protected static ConfigurableApplicationContext initializeConfiguration(String[] args) {
return new SpringApplicationBuilder(SpringBootstrap.class).web(WebApplicationType.NONE).run(args);
protected static ConfigurableApplicationContext initializeContext(String[] args) {
SpringApplicationBuilder builder = new SpringApplicationBuilder(SpringBootstrap.class);
builder.web(WebApplicationType.NONE);
builder.registerShutdownHook(false);
return builder.run(args);
}
private void shutdownHook() {
@@ -109,13 +112,11 @@ public class GrafanaAuth {
try {
f.get();
} catch (final Exception e) {
LOG.error("Error while shutting down channel: {}", f.channel().config());
LOG.error("{}", e.getMessage());
e.printStackTrace();
LOG.error("Channel:" + f.channel().config() + " -> " + e.getMessage(), e);
}
});
Integer quietPeriod = grafanaAuthConfig.getShutdownQuietPeriod();
int quietPeriod = grafanaAuthConfig.getShutdownQuietPeriod();
List<Future<?>> groupFutures = new ArrayList<>();
if (httpBossGroup != null) {
@@ -132,9 +133,7 @@ public class GrafanaAuth {
try {
f.get();
} catch (final Exception e) {
LOG.error("Error while shutting down group: {}", f.toString());
LOG.error("{}", e.getMessage());
e.printStackTrace();
LOG.error("Group:" + f.toString() + " -> " + e.getMessage(), e);
}
});
@@ -272,12 +271,13 @@ public class GrafanaAuth {
public static void main(String[] args) throws Exception {
GrafanaAuth.applicationContext = GrafanaAuth.initializeConfiguration(args);
GrafanaAuthConfiguration balancerConf = applicationContext.getBean(GrafanaAuthConfiguration.class);
GrafanaAuth.applicationContext = GrafanaAuth.initializeContext(args);
GrafanaAuthConfiguration grafanaAuthConf = GrafanaAuth.applicationContext
.getBean(GrafanaAuthConfiguration.class);
GrafanaAuth b = new GrafanaAuth(balancerConf);
GrafanaAuth grafanaAuth = new GrafanaAuth(grafanaAuthConf);
try {
b.run();
grafanaAuth.run();
LATCH.await();
} catch (final InterruptedException e) {
LOG.info("Server shutting down.");
@@ -285,7 +285,7 @@ public class GrafanaAuth {
LOG.error("Error running server.", e);
} finally {
try {
b.shutdown();
grafanaAuth.shutdown();
} catch (Exception e) {
System.exit(1);
}
@@ -314,7 +314,8 @@ public class GrafanaAuth {
ch.pipeline().addLast("fileServer", new HttpStaticFileServerHandler());
ch.pipeline().addLast("login", new X509LoginRequestHandler(config.getSecurity(), config.getHttp()));
ch.pipeline().addLast("httpRelay", new GrafanaRelayHandler(config, httpClientPool));
ch.pipeline().addLast("error", new TimelyExceptionHandler());
ch.pipeline().addLast("error", new TimelyExceptionHandler()
.setIgnoreSslHandshakeErrors(config.getSecurity().getServerSsl().isUseGeneratedKeypair()));
}
};
}
@@ -87,7 +87,7 @@ public class GrafanaRelayHandler extends SimpleChannelInboundHandler<HttpRequest
String protocol = config.getGrafana().getProtocol();
String host = config.getGrafana().getHost();
Integer port = config.getGrafana().getPort();
k = TimelyBalancedHost.of(host, -1, port, -1, -1);
k = TimelyBalancedHost.of(host, -1, port, -1, -1, null);
client = httpClientPool.borrowObject(k);
Multimap<String, String> headers = HttpHeaderUtils.toMultimap(request.headers());
+1 -1
View File
@@ -4,7 +4,7 @@
<parent>
<groupId>timely</groupId>
<artifactId>timely</artifactId>
<version>1.2.6</version>
<version>1.2.8-SNAPSHOT</version>
</parent>
<artifactId>timely-app</artifactId>
<packaging>pom</packaging>
+1 -1
View File
@@ -509,7 +509,7 @@ System.register(['lodash', 'angular', '../../../app/core/utils/datemath'], funct
} else {
return target.metric === metricData.metric && _.every(target.tags, function (tagV, tagK) {
interpolatedTagValue = this.templateSrv.replace(tagV, options.scopedVars, 'pipe');
return metricData.tags[tagK] === interpolatedTagValue || interpolatedTagValue === "*";
return metricData.tags[tagK] === interpolatedTagValue || interpolatedTagValue === ".*";
}.bind(this));
}
}.bind(this));
+1 -1
View File
@@ -1 +1 @@
{"version":3,"sources":["../../src/datasource/datasource.js"],"names":["_","angular","dateMath","TimelyDatasource","instanceSettings","$q","backendSrv","templateSrv","contextSrv","type","name","jsonData","httpsPort","timelyHost","wsPort","basicAuths","url","_makeHttpsUrl","withCredentials","basicAuth","supportMetrics","tagKeys","aggregatorsPromise","filterTypesPromise","options","start","convertToTSDBTime","rangeRaw","from","end","to","qs","each","targets","target","metric","push","convertTargetToQuery","bind","queries","compact","isEmpty","d","defer","resolve","data","promise","groupByTags","query","filters","length","val","tagk","tags","key","performTimeSeriesQuery","then","response","metricToTargetMapping","mapMetricsToTargets","result","map","metricData","index","_saveTagKeys","transformMetricData","datasourceRequest","method","status","message","title","eventList","aggregator","annotation","results","annotationObject","annotations","isGlobal","globalAnnotations","event","description","time","Math","floor","startTime","text","notes","when","interpolated","replace","err","reject","responseTransform","value","metrics_regex","tag_names_regex","tag_values_regex","tag_names_suggest_regex","tag_values_suggest_regex","metrics_query","match","_performSuggestQuery","tag_names_query","_performMetricKeyLookup","tag_values_query","_performMetricKeyValueLookup","tag_names_suggest_query","tag_values_suggest_query","msResolution","responsePromises","reqBody","showQuery","_addCredentialOptions","headers","Promise","all","queryResponses","combinedResponse","x","concat","Object","keys","aggregateTags","tag","_get","q","max","keysArray","split","trim","keysQuery","splice","join","m","limit","tagvs","r","indexOf","tagks","tagv","relativeUrl","params","isArray","sort","md","metricLabel","createMetricLabel","dps","v","k","datapoints","alias","scopedVars","clone","label","tagData","toPairs","has","hide","shouldComputeRate","rate","rateOptions","counter","isCounter","counterMax","parseInt","counterResetValue","resetValue","disableDownsampling","interval","downsampleInterval","parseFloat","downsample","downsampleAggregator","downsampleFillPolicy","copy","filter_key","filter","tag_key","metrics","interpolatedTagValue","findIndex","every","tagV","tagK","date","roundUp","parse","valueOf"],"mappings":";;;;;;;;;;;;;;;AAAOA,O;;AACAC,a;;AACKC,c;;;;;;;;;;;;;;;;;;;;;kCAECC,gB;;AAEX;AACA,kCAAYC,gBAAZ,EAA8BC,EAA9B,EAAkCC,UAAlC,EAA8CC,WAA9C,EAA2DC,UAA3D,EAAuE;AAAA;;AACrE,eAAKC,IAAL,GAAYL,iBAAiBK,IAA7B;AACA,eAAKC,IAAL,GAAYN,iBAAiBM,IAA7B;;AAEA,eAAKC,QAAL,GAAgBP,iBAAiBO,QAAjB,IAA6B,EAA7C;AACA,eAAKC,SAAL,GAAiBR,iBAAiBO,QAAjB,CAA0BC,SAA3C;AACA,eAAKC,UAAL,GAAkBT,iBAAiBO,QAAjB,CAA0BE,UAA5C;AACA,eAAKC,MAAL,GAAcV,iBAAiBO,QAAjB,CAA0BG,MAAxC;AACA,cAAI,CAACV,iBAAiBO,QAAjB,CAA0BI,UAA/B,EAA0C;AACxCX,6BAAiBO,QAAjB,CAA0BI,UAA1B,GAAuC,KAAvC;AACD;;AAED,eAAKC,GAAL,GAAW,KAAKC,aAAL,EAAX;AACA,eAAKC,eAAL,GAAuB,IAAvB;AACA,eAAKC,SAAL,GAAiB,KAAjB;;AAEA,eAAKC,cAAL,GAAsB,IAAtB;AACA,eAAKC,OAAL,GAAe,EAAf;;AAEA,eAAKhB,EAAL,GAAUA,EAAV;AACA,eAAKC,UAAL,GAAkBA,UAAlB;AACA,eAAKC,WAAL,GAAmBA,WAAnB;AACA,eAAKC,UAAL,GAAkBA,UAAlB;;AAEA,eAAKc,kBAAL,GAA0B,IAA1B;AACA,eAAKC,kBAAL,GAA0B,IAA1B;AAED;;AAED;;;;;gCACMC,O,EAAQ;AACZ,gBAAIC,QAAQ,KAAKC,iBAAL,CAAuBF,QAAQG,QAAR,CAAiBC,IAAxC,EAA8C,KAA9C,CAAZ;AACA,gBAAIC,MAAM,KAAKH,iBAAL,CAAuBF,QAAQG,QAAR,CAAiBG,EAAxC,EAA4C,IAA5C,CAAV;AACA,gBAAIC,KAAK,EAAT;;AAEA/B,cAAEgC,IAAF,CAAOR,QAAQS,OAAf,EAAwB,UAASC,MAAT,EAAiB;AACvC,kBAAI,CAACA,OAAOC,MAAZ,EAAoB;AAAE;AAAS;AAC/BJ,iBAAGK,IAAH,CAAQ,KAAKC,oBAAL,CAA0BH,MAA1B,EAAkCV,OAAlC,CAAR;AACD,aAHuB,CAGtBc,IAHsB,CAGjB,IAHiB,CAAxB;;AAKA,gBAAIC,UAAUvC,EAAEwC,OAAF,CAAUT,EAAV,CAAd;;AAEA;AACA,gBAAI/B,EAAEyC,OAAF,CAAUF,OAAV,CAAJ,EAAwB;AACtB,kBAAIG,IAAI,KAAKrC,EAAL,CAAQsC,KAAR,EAAR;AACAD,gBAAEE,OAAF,CAAU,EAAEC,MAAM,EAAR,EAAV;AACA,qBAAOH,EAAEI,OAAT;AACD;;AAED,gBAAIC,cAAc,EAAlB;AACA/C,cAAEgC,IAAF,CAAOO,OAAP,EAAgB,UAASS,KAAT,EAAgB;AAC9B,kBAAIA,MAAMC,OAAN,IAAiBD,MAAMC,OAAN,CAAcC,MAAd,GAAuB,CAA5C,EAA+C;AAC7ClD,kBAAEgC,IAAF,CAAOgB,MAAMC,OAAb,EAAsB,UAASE,GAAT,EAAc;AAClCJ,8BAAYI,IAAIC,IAAhB,IAAwB,IAAxB;AACD,iBAFD;AAGD,eAJD,MAIO;AACLpD,kBAAEgC,IAAF,CAAOgB,MAAMK,IAAb,EAAmB,UAASF,GAAT,EAAcG,GAAd,EAAmB;AACpCP,8BAAYO,GAAZ,IAAmB,IAAnB;AACD,iBAFD;AAGD;AACF,aAVD;;AAYA,mBAAO,KAAKC,sBAAL,CAA4BhB,OAA5B,EAAqCd,KAArC,EAA4CI,GAA5C,EAAiD2B,IAAjD,CAAsD,UAASC,QAAT,EAAmB;AAC9E,kBAAIC,wBAAwB,KAAKC,mBAAL,CAAyBF,SAASZ,IAAlC,EAAwCrB,OAAxC,CAA5B;AACA,kBAAIoC,SAAS5D,EAAE6D,GAAF,CAAMJ,SAASZ,IAAf,EAAqB,UAASiB,UAAT,EAAqBC,KAArB,EAA4B;AAC5DA,wBAAQL,sBAAsBK,KAAtB,CAAR;AACA,oBAAIA,UAAU,CAAC,CAAf,EAAkB;AAChBA,0BAAQ,CAAR;AACD;AACD,qBAAKC,YAAL,CAAkBF,UAAlB;;AAEA,uBAAO,KAAKG,mBAAL,CAAyBH,UAAzB,EAAqCf,WAArC,EAAkDvB,QAAQS,OAAR,CAAgB8B,KAAhB,CAAlD,EAA0EvC,OAA1E,CAAP;AACD,eARiC,CAQhCc,IARgC,CAQ3B,IAR2B,CAArB,CAAb;AASA,qBAAO,EAAEO,MAAMe,MAAR,EAAP;AACD,aAZ4D,CAY3DtB,IAZ2D,CAYtD,IAZsD,CAAtD,CAAP;AAaD;;;2CAGe;AACd,mBAAO,KAAKhC,UAAL,CAAgB4D,iBAAhB,CAAkC;AACvClD,mBAAK,KAAKA,GAAL,GAAW,UADuB;AAEvCmD,sBAAQ;AAF+B,aAAlC,EAGJX,IAHI,CAGC,oBAAY;AAClB,kBAAIC,SAASW,MAAT,KAAoB,GLine truncated
{"version":3,"sources":["../../src/datasource/datasource.js"],"names":["_","angular","dateMath","TimelyDatasource","instanceSettings","$q","backendSrv","templateSrv","contextSrv","type","name","jsonData","httpsPort","timelyHost","wsPort","basicAuths","url","_makeHttpsUrl","withCredentials","basicAuth","supportMetrics","tagKeys","aggregatorsPromise","filterTypesPromise","options","start","convertToTSDBTime","rangeRaw","from","end","to","qs","each","targets","target","metric","push","convertTargetToQuery","bind","queries","compact","isEmpty","d","defer","resolve","data","promise","groupByTags","query","filters","length","val","tagk","tags","key","performTimeSeriesQuery","then","response","metricToTargetMapping","mapMetricsToTargets","result","map","metricData","index","_saveTagKeys","transformMetricData","datasourceRequest","method","status","message","title","eventList","aggregator","annotation","results","annotationObject","annotations","isGlobal","globalAnnotations","event","description","time","Math","floor","startTime","text","notes","when","interpolated","replace","err","reject","responseTransform","value","metrics_regex","tag_names_regex","tag_values_regex","tag_names_suggest_regex","tag_values_suggest_regex","metrics_query","match","_performSuggestQuery","tag_names_query","_performMetricKeyLookup","tag_values_query","_performMetricKeyValueLookup","tag_names_suggest_query","tag_values_suggest_query","msResolution","responsePromises","reqBody","showQuery","_addCredentialOptions","headers","Promise","all","queryResponses","combinedResponse","x","concat","Object","keys","aggregateTags","tag","_get","q","max","keysArray","split","trim","keysQuery","splice","join","m","limit","tagvs","r","indexOf","tagks","tagv","relativeUrl","params","isArray","sort","md","metricLabel","createMetricLabel","dps","v","k","datapoints","alias","scopedVars","clone","label","tagData","toPairs","has","hide","shouldComputeRate","rate","rateOptions","counter","isCounter","counterMax","parseInt","counterResetValue","resetValue","disableDownsampling","interval","downsampleInterval","parseFloat","downsample","downsampleAggregator","downsampleFillPolicy","copy","filter_key","filter","tag_key","metrics","interpolatedTagValue","findIndex","every","tagV","tagK","date","roundUp","parse","valueOf"],"mappings":";;;;;;;;;;;;;;;AAAOA,O;;AACAC,a;;AACKC,c;;;;;;;;;;;;;;;;;;;;;kCAECC,gB;;AAEX;AACA,kCAAYC,gBAAZ,EAA8BC,EAA9B,EAAkCC,UAAlC,EAA8CC,WAA9C,EAA2DC,UAA3D,EAAuE;AAAA;;AACrE,eAAKC,IAAL,GAAYL,iBAAiBK,IAA7B;AACA,eAAKC,IAAL,GAAYN,iBAAiBM,IAA7B;;AAEA,eAAKC,QAAL,GAAgBP,iBAAiBO,QAAjB,IAA6B,EAA7C;AACA,eAAKC,SAAL,GAAiBR,iBAAiBO,QAAjB,CAA0BC,SAA3C;AACA,eAAKC,UAAL,GAAkBT,iBAAiBO,QAAjB,CAA0BE,UAA5C;AACA,eAAKC,MAAL,GAAcV,iBAAiBO,QAAjB,CAA0BG,MAAxC;AACA,cAAI,CAACV,iBAAiBO,QAAjB,CAA0BI,UAA/B,EAA0C;AACxCX,6BAAiBO,QAAjB,CAA0BI,UAA1B,GAAuC,KAAvC;AACD;;AAED,eAAKC,GAAL,GAAW,KAAKC,aAAL,EAAX;AACA,eAAKC,eAAL,GAAuB,IAAvB;AACA,eAAKC,SAAL,GAAiB,KAAjB;;AAEA,eAAKC,cAAL,GAAsB,IAAtB;AACA,eAAKC,OAAL,GAAe,EAAf;;AAEA,eAAKhB,EAAL,GAAUA,EAAV;AACA,eAAKC,UAAL,GAAkBA,UAAlB;AACA,eAAKC,WAAL,GAAmBA,WAAnB;AACA,eAAKC,UAAL,GAAkBA,UAAlB;;AAEA,eAAKc,kBAAL,GAA0B,IAA1B;AACA,eAAKC,kBAAL,GAA0B,IAA1B;AAED;;AAED;;;;;gCACMC,O,EAAQ;AACZ,gBAAIC,QAAQ,KAAKC,iBAAL,CAAuBF,QAAQG,QAAR,CAAiBC,IAAxC,EAA8C,KAA9C,CAAZ;AACA,gBAAIC,MAAM,KAAKH,iBAAL,CAAuBF,QAAQG,QAAR,CAAiBG,EAAxC,EAA4C,IAA5C,CAAV;AACA,gBAAIC,KAAK,EAAT;;AAEA/B,cAAEgC,IAAF,CAAOR,QAAQS,OAAf,EAAwB,UAASC,MAAT,EAAiB;AACvC,kBAAI,CAACA,OAAOC,MAAZ,EAAoB;AAAE;AAAS;AAC/BJ,iBAAGK,IAAH,CAAQ,KAAKC,oBAAL,CAA0BH,MAA1B,EAAkCV,OAAlC,CAAR;AACD,aAHuB,CAGtBc,IAHsB,CAGjB,IAHiB,CAAxB;;AAKA,gBAAIC,UAAUvC,EAAEwC,OAAF,CAAUT,EAAV,CAAd;;AAEA;AACA,gBAAI/B,EAAEyC,OAAF,CAAUF,OAAV,CAAJ,EAAwB;AACtB,kBAAIG,IAAI,KAAKrC,EAAL,CAAQsC,KAAR,EAAR;AACAD,gBAAEE,OAAF,CAAU,EAAEC,MAAM,EAAR,EAAV;AACA,qBAAOH,EAAEI,OAAT;AACD;;AAED,gBAAIC,cAAc,EAAlB;AACA/C,cAAEgC,IAAF,CAAOO,OAAP,EAAgB,UAASS,KAAT,EAAgB;AAC9B,kBAAIA,MAAMC,OAAN,IAAiBD,MAAMC,OAAN,CAAcC,MAAd,GAAuB,CAA5C,EAA+C;AAC7ClD,kBAAEgC,IAAF,CAAOgB,MAAMC,OAAb,EAAsB,UAASE,GAAT,EAAc;AAClCJ,8BAAYI,IAAIC,IAAhB,IAAwB,IAAxB;AACD,iBAFD;AAGD,eAJD,MAIO;AACLpD,kBAAEgC,IAAF,CAAOgB,MAAMK,IAAb,EAAmB,UAASF,GAAT,EAAcG,GAAd,EAAmB;AACpCP,8BAAYO,GAAZ,IAAmB,IAAnB;AACD,iBAFD;AAGD;AACF,aAVD;;AAYA,mBAAO,KAAKC,sBAAL,CAA4BhB,OAA5B,EAAqCd,KAArC,EAA4CI,GAA5C,EAAiD2B,IAAjD,CAAsD,UAASC,QAAT,EAAmB;AAC9E,kBAAIC,wBAAwB,KAAKC,mBAAL,CAAyBF,SAASZ,IAAlC,EAAwCrB,OAAxC,CAA5B;AACA,kBAAIoC,SAAS5D,EAAE6D,GAAF,CAAMJ,SAASZ,IAAf,EAAqB,UAASiB,UAAT,EAAqBC,KAArB,EAA4B;AAC5DA,wBAAQL,sBAAsBK,KAAtB,CAAR;AACA,oBAAIA,UAAU,CAAC,CAAf,EAAkB;AAChBA,0BAAQ,CAAR;AACD;AACD,qBAAKC,YAAL,CAAkBF,UAAlB;;AAEA,uBAAO,KAAKG,mBAAL,CAAyBH,UAAzB,EAAqCf,WAArC,EAAkDvB,QAAQS,OAAR,CAAgB8B,KAAhB,CAAlD,EAA0EvC,OAA1E,CAAP;AACD,eARiC,CAQhCc,IARgC,CAQ3B,IAR2B,CAArB,CAAb;AASA,qBAAO,EAAEO,MAAMe,MAAR,EAAP;AACD,aAZ4D,CAY3DtB,IAZ2D,CAYtD,IAZsD,CAAtD,CAAP;AAaD;;;2CAGe;AACd,mBAAO,KAAKhC,UAAL,CAAgB4D,iBAAhB,CAAkC;AACvClD,mBAAK,KAAKA,GAAL,GAAW,UADuB;AAEvCmD,sBAAQ;AAF+B,aAAlC,EAGJX,IAHI,CAGC,oBAAY;AAClB,kBAAIC,SAASW,MAAT,KAAoB,GLine truncated
+2 -2
View File
@@ -23,8 +23,8 @@
{"name": "Project site", "url": "https://github.com/NationalSecurityAgency/timely"},
{"name": "License & Terms", "url": "https://raw.githubusercontent.com/NationalSecurityAgency/timely/master/LICENSE"}
],
"version": "1.2.6",
"updated": "2019-10-30"
"version": "1.2.8-SNAPSHOT",
"updated": "2020-02-21"
},
"includes": [
+1 -1
View File
@@ -1,7 +1,7 @@
{
"name": "timely-app",
"private": true,
"version": "1.2.6",
"version": "1.2.8-SNAPSHOT",
"description": "",
"main": "index.js",
"scripts": {
+2 -2
View File
@@ -23,8 +23,8 @@
{"name": "Project site", "url": "https://github.com/NationalSecurityAgency/timely"},
{"name": "License & Terms", "url": "https://raw.githubusercontent.com/NationalSecurityAgency/timely/master/LICENSE"}
],
"version": "1.2.6",
"updated": "2019-10-30"
"version": "1.2.8-SNAPSHOT",
"updated": "2020-02-21"
},
"includes": [
@@ -446,7 +446,7 @@ export class TimelyDatasource {
} else {
return target.metric === metricData.metric && _.every(target.tags, function(tagV, tagK) {
interpolatedTagValue = this.templateSrv.replace(tagV, options.scopedVars, 'pipe');
return metricData.tags[tagK] === interpolatedTagValue || interpolatedTagValue === "*";
return metricData.tags[tagK] === interpolatedTagValue || interpolatedTagValue === ".*";
}.bind(this));
}
}.bind(this));
+3 -3
View File
@@ -3,7 +3,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>timely</groupId>
<artifactId>timely</artifactId>
<version>1.2.6</version>
<version>1.2.8-SNAPSHOT</version>
<packaging>pom</packaging>
<name>Timely</name>
<description>A secure and scalable time series database based on Apache Accumulo</description>
@@ -59,8 +59,8 @@
<version.hadoop>2.7.7</version.hadoop>
<version.httpcomponents>4.5.6</version.httpcomponents>
<version.impsort>1.3.0</version.impsort>
<version.jackson>2.9.10</version.jackson>
<version.jackson.databind>2.9.10</version.jackson.databind>
<version.jackson>2.10.0</version.jackson>
<version.jackson.databind>2.10.0</version.jackson.databind>
<version.jacoco>0.8.4</version.jacoco>
<version.javacsv>2.0</version.javacsv>
<version.javassist>3.24.1-GA</version.javassist>
+1 -1
View File
@@ -4,7 +4,7 @@
<parent>
<groupId>timely</groupId>
<artifactId>timely</artifactId>
<version>1.2.6</version>
<version>1.2.8-SNAPSHOT</version>
</parent>
<artifactId>timely-server</artifactId>
<dependencies>
+57 -34
View File
@@ -76,7 +76,6 @@ import org.slf4j.LoggerFactory;
import org.springframework.boot.WebApplicationType;
import org.springframework.boot.builder.SpringApplicationBuilder;
import org.springframework.context.ConfigurableApplicationContext;
import timely.api.response.TimelyException;
import timely.auth.AuthCache;
import timely.auth.VisibilityCache;
import timely.configuration.Configuration;
@@ -133,6 +132,7 @@ public class Server {
private static final String OS_VERSION = "os.version";
private static final String WS_PATH = "/websocket";
private SslContext sslCtx = null;
private CuratorFramework curatorFramework;
protected static final CountDownLatch LATCH = new CountDownLatch(1);
static ConfigurableApplicationContext applicationContext;
@@ -234,11 +234,12 @@ public class Server {
public static void main(String[] args) throws Exception {
Configuration conf = initializeConfiguration(args);
Server.applicationContext = initializeContext(args);
Configuration conf = Server.applicationContext.getBean(Configuration.class);
Server s = new Server(conf);
Server server = new Server(conf);
try {
s.run();
server.run(createSSLContext(conf));
LATCH.await();
} catch (final InterruptedException e) {
LOG.info("Server shutting down.");
@@ -246,16 +247,18 @@ public class Server {
LOG.error("Error running server.", e);
} finally {
try {
s.shutdown();
server.shutdown();
} catch (Exception e) {
System.exit(1);
}
}
}
protected static Configuration initializeConfiguration(String[] args) {
applicationContext = new SpringApplicationBuilder(SpringBootstrap.class).web(WebApplicationType.NONE).run(args);
return applicationContext.getBean(Configuration.class);
protected static ConfigurableApplicationContext initializeContext(String[] args) {
SpringApplicationBuilder builder = new SpringApplicationBuilder(SpringBootstrap.class);
builder.web(WebApplicationType.NONE);
builder.registerShutdownHook(false);
return builder.run(args);
}
private void shutdownHook() {
@@ -287,8 +290,9 @@ public class Server {
channelFutures.add(wsChannelHandle.close());
}
int udpChannel = 1;
for (Channel c : udpChannelHandleList) {
LOG.info("Closing udpChannelHandle");
LOG.info("Closing udpChannelHandle #" + udpChannel++);
channelFutures.add(c.close());
}
@@ -297,14 +301,13 @@ public class Server {
try {
f.get();
} catch (final Exception e) {
LOG.error("Error while shutting down channel: {}", f.channel().config());
LOG.error("{}", e.getMessage());
e.printStackTrace();
LOG.error("Channel:" + f.channel().config() + " -> " + e.getMessage(), e);
}
});
Integer quietPeriod = config.getServer().getShutdownQuietPeriod();
int quietPeriod = config.getServer().getShutdownQuietPeriod();
List<Future<?>> groupFutures = new ArrayList<>();
if (tcpBossGroup != null) {
LOG.info("Shutting down tcpBossGroup");
groupFutures.add(tcpBossGroup.shutdownGracefully(quietPeriod, 10, TimeUnit.SECONDS));
@@ -349,40 +352,58 @@ public class Server {
try {
f.get();
} catch (final Exception e) {
LOG.error("Error while shutting down group: {}", f.toString());
LOG.error("{}", e.getMessage());
e.printStackTrace();
LOG.error("Group:" + f.toString() + " -> " + e.getMessage(), e);
}
});
try {
LOG.info("Flushing datastore.");
dataStore.flush();
} catch (TimelyException e) {
LOG.error("Error flushing to server during shutdown", e);
if (dataStore != null) {
try {
LOG.info("Closing dataStore");
dataStore.close();
} catch (Exception e) {
LOG.error(e.getMessage(), e);
}
}
if (dataStoreCache != null) {
try {
LOG.info("Closing dataStoreCache");
dataStoreCache.close();
} catch (Exception e) {
LOG.error(e.getMessage(), e);
}
}
try {
LOG.info("Closing MetaCacheFactory");
LOG.info("Closing metaCacheFactory");
MetaCacheFactory.close();
} catch (Exception e) {
LOG.error("Error closing MetaCacheFactory during shutdown", e);
LOG.error(e.getMessage(), e);
}
try {
LOG.info("Closing WebSocketRequestDecoder");
LOG.info("Closing webSocketRequestDecoder subscriptions");
WebSocketRequestDecoder.close();
} catch (Exception e) {
LOG.error("Error closing WebSocketRequestDecoder during shutdown", e);
LOG.error(e.getMessage(), e);
}
if (curatorFramework != null) {
curatorFramework.close();
try {
LOG.info("Closing curatorFramework");
curatorFramework.close();
} catch (Exception e) {
LOG.error(e.getMessage(), e);
}
}
if (applicationContext != null) {
LOG.info("Closing applicationContext");
applicationContext.close();
try {
LOG.info("Closing applicationContext");
applicationContext.close();
} catch (Exception e) {
LOG.error(e.getMessage(), e);
}
}
this.shutdown = true;
LOG.info("Server shut down.");
@@ -415,7 +436,7 @@ public class Server {
}
}
public void run() throws Exception {
public void run(SslContext sslCtx) throws Exception {
RetryPolicy retryPolicy = new RetryForever(1000);
int timeout = Long.valueOf(AccumuloConfiguration.getTimeInMillis(config.getAccumulo().getZookeeperTimeout()))
@@ -482,7 +503,6 @@ public class Server {
final int httpPort = config.getHttp().getPort();
final String httpIp = config.getHttp().getIp();
SslContext sslCtx = createSSLContext(config);
if (sslCtx instanceof OpenSslServerContext) {
OpenSslServerContext openssl = (OpenSslServerContext) sslCtx;
String application = "Timely_" + httpPort;
@@ -539,7 +559,7 @@ public class Server {
tcpAddress, tcpPort, httpAddress, httpPort, wsAddress, wsPort, wsAddress, udpPort);
}
protected SslContext createSSLContext(Configuration config) throws Exception {
public static SslContext createSSLContext(Configuration config) throws Exception {
ServerSsl sslCfg = config.getSecurity().getServerSsl();
Boolean generate = sslCfg.isUseGeneratedKeypair();
@@ -547,7 +567,7 @@ public class Server {
if (generate) {
LOG.warn("Using generated self signed server certificate");
Date begin = new Date();
Date end = new Date(begin.getTime() + 86400000);
Date end = new Date(begin.getTime() + TimeUnit.DAYS.toMillis(7));
SelfSignedCertificate ssc = new SelfSignedCertificate("localhost", begin, end);
ssl = SslContextBuilder.forServer(ssc.certificate(), ssc.privateKey());
} else {
@@ -614,7 +634,8 @@ public class Server {
ch.pipeline().addLast("cors", new CorsHandler(cors));
ch.pipeline().addLast("queryDecoder",
new timely.netty.http.HttpRequestDecoder(config.getSecurity(), config.getHttp()));
ch.pipeline().addLast("fileServer", new HttpStaticFileServerHandler());
ch.pipeline().addLast("fileServer", new HttpStaticFileServerHandler()
.setIgnoreSslHandshakeErrors(config.getSecurity().getServerSsl().isUseGeneratedKeypair()));
ch.pipeline().addLast("strict", new StrictTransportHandler(config));
ch.pipeline().addLast("login", new X509LoginRequestHandler(config.getSecurity(), config.getHttp()));
ch.pipeline().addLast("doLogin",
@@ -627,7 +648,9 @@ public class Server {
ch.pipeline().addLast("version", new HttpVersionRequestHandler());
ch.pipeline().addLast("cache", new HttpCacheRequestHandler(dataStoreCache));
ch.pipeline().addLast("put", new HttpMetricPutHandler(dataStore));
ch.pipeline().addLast("error", new TimelyExceptionHandler());
ch.pipeline().addLast("error", new TimelyExceptionHandler()
.setIgnoreSslHandshakeErrors(config.getSecurity().getServerSsl().isUseGeneratedKeypair()));
}
};
}
@@ -52,7 +52,8 @@ public class StandaloneServer extends Server {
System.exit(1);
}
Configuration conf = initializeConfiguration(args);
StandaloneServer.applicationContext = initializeContext(args);
Configuration conf = StandaloneServer.applicationContext.getBean(Configuration.class);
File accumuloDir = new File(tmp, "accumulo");
MiniAccumuloConfig macConfig = new MiniAccumuloConfig(accumuloDir, "secret");
@@ -84,7 +85,7 @@ public class StandaloneServer extends Server {
try {
LOG.info("Starting StandaloneServer");
StandaloneServer s = new StandaloneServer(conf);
s.run();
s.run(createSSLContext(conf));
} catch (Exception e) {
System.err.println("Error starting server");
e.printStackTrace();
@@ -8,6 +8,7 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import com.google.common.base.Joiner;
@@ -39,6 +40,8 @@ public class MetricAdapter {
private static final TagParser tagParser = new TagParser();
private static final TagListParser tagListParser = new TagListParser();
private static Pattern equalsPattern = Pattern.compile("=");
private static Pattern commaPattern = Pattern.compile(",");
public static final ColumnVisibility EMPTY_VISIBILITY = new ColumnVisibility();
public static final String VISIBILITY_TAG = "viz";
@@ -59,10 +62,10 @@ public class MetricAdapter {
// are used as a delimiter
String k = t.getKey();
String v = t.getValue();
k = k.replaceAll("=", "\\=");
k = k.replaceAll(",", "\\,");
v = v.replaceAll("=", "\\=");
v = v.replaceAll(",", "\\,");
k = equalsPattern.matcher(k).replaceAll("\\=");
k = commaPattern.matcher(k).replaceAll("\\,");
v = equalsPattern.matcher(v).replaceAll("\\=");
v = commaPattern.matcher(v).replaceAll("\\,");
newTags.add(new Tag(k, v));
}
return newTags;
@@ -75,10 +78,10 @@ public class MetricAdapter {
// are used as a delimiter
String k = t.getKey();
String v = t.getValue();
k = k.replaceAll("=", "\\=");
k = k.replaceAll(",", "\\,");
v = v.replaceAll("=", "\\=");
v = v.replaceAll(",", "\\,");
k = equalsPattern.matcher(k).replaceAll("\\=");
k = commaPattern.matcher(k).replaceAll("\\,");
v = equalsPattern.matcher(v).replaceAll("\\=");
v = commaPattern.matcher(v).replaceAll("\\,");
newTags.put(k, v);
}
return newTags;
@@ -1,6 +1,7 @@
package timely.configuration;
import java.util.List;
import java.util.Objects;
import com.google.common.collect.Lists;
import timely.validator.NotEmptyIfFieldSet;
@@ -76,4 +77,24 @@ public class ServerSsl {
public void setUseCiphers(List<String> useCiphers) {
this.useCiphers = useCiphers;
}
@Override
public boolean equals(Object o) {
if (this == o)
return true;
if (o == null || getClass() != o.getClass())
return false;
ServerSsl serverSsl = (ServerSsl) o;
return useGeneratedKeypair == serverSsl.useGeneratedKeypair && useOpenssl == serverSsl.useOpenssl
&& Objects.equals(certificateFile, serverSsl.certificateFile)
&& Objects.equals(keyFile, serverSsl.keyFile) && Objects.equals(keyPassword, serverSsl.keyPassword)
&& Objects.equals(trustStoreFile, serverSsl.trustStoreFile)
&& Objects.equals(useCiphers, serverSsl.useCiphers);
}
@Override
public int hashCode() {
return Objects.hash(certificateFile, keyFile, keyPassword, trustStoreFile, useGeneratedKeypair, useOpenssl,
useCiphers);
}
}
@@ -48,6 +48,7 @@ import java.util.TimeZone;
import java.util.regex.Pattern;
import javax.activation.MimetypesFileTypeMap;
import javax.net.ssl.SSLHandshakeException;
import io.netty.buffer.ByteBuf;
import io.netty.buffer.Unpooled;
@@ -123,6 +124,7 @@ public class HttpStaticFileServerHandler extends SimpleChannelInboundHandler<Ful
public static final String HTTP_DATE_FORMAT = "EEE, dd MMM yyyy HH:mm:ss zzz";
public static final String HTTP_DATE_GMT_TIMEZONE = "GMT";
public static final int HTTP_CACHE_SECONDS = 60;
private boolean ignoreSslHandshakeErrors = false;
@Override
public void channelRead0(ChannelHandlerContext ctx, FullHttpRequest request) throws Exception {
@@ -221,6 +223,10 @@ public class HttpStaticFileServerHandler extends SimpleChannelInboundHandler<Ful
@Override
public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) {
// ignore SSLHandshakeException when using a self-signed server certificate
if (ignoreSslHandshakeErrors && cause.getCause() instanceof SSLHandshakeException) {
return;
}
LOG.error("Error serving file", cause);
if (ctx.channel().isActive()) {
sendError(ctx, INTERNAL_SERVER_ERROR);
@@ -379,4 +385,9 @@ public class HttpStaticFileServerHandler extends SimpleChannelInboundHandler<Ful
MimetypesFileTypeMap mimeTypesMap = new MimetypesFileTypeMap();
response.headers().set(CONTENT_TYPE, mimeTypesMap.getContentType(file.getPath()));
}
public HttpStaticFileServerHandler setIgnoreSslHandshakeErrors(boolean ignoreSslHandshakeErrors) {
this.ignoreSslHandshakeErrors = ignoreSslHandshakeErrors;
return this;
}
}
@@ -1,5 +1,7 @@
package timely.netty.http;
import javax.net.ssl.SSLHandshakeException;
import io.netty.channel.ChannelHandler.Sharable;
import io.netty.channel.ChannelHandlerContext;
import io.netty.channel.SimpleChannelInboundHandler;
@@ -12,6 +14,7 @@ import timely.api.response.TimelyException;
public class TimelyExceptionHandler extends SimpleChannelInboundHandler<TimelyException> implements TimelyHttpHandler {
private static final Logger LOG = LoggerFactory.getLogger(TimelyExceptionHandler.class);
private boolean ignoreSslHandshakeErrors = false;
@Override
protected void channelRead0(ChannelHandlerContext ctx, TimelyException msg) throws Exception {
@@ -20,6 +23,10 @@ public class TimelyExceptionHandler extends SimpleChannelInboundHandler<TimelyEx
@Override
public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception {
// ignore SSLHandshakeException when using a self-signed server certificate
if (ignoreSslHandshakeErrors && cause.getCause() instanceof SSLHandshakeException) {
return;
}
LOG.error("Unhandled exception in pipeline", cause);
if (cause instanceof TimelyException) {
this.sendHttpError(ctx, (TimelyException) cause);
@@ -32,4 +39,8 @@ public class TimelyExceptionHandler extends SimpleChannelInboundHandler<TimelyEx
}
}
public TimelyExceptionHandler setIgnoreSslHandshakeErrors(boolean ignoreSslHandshakeErrors) {
this.ignoreSslHandshakeErrors = ignoreSslHandshakeErrors;
return this;
}
}
@@ -109,7 +109,6 @@ public class WebSocketRequestDecoder extends MessageToMessageDecoder<WebSocketFr
}
public static void close() {
LOG.info("Closing subscriptions");
SubscriptionRegistry.get().forEach((k, v) -> v.close());
}
@@ -29,8 +29,6 @@ public interface DataStore {
List<QueryResponse> query(QueryRequest msg) throws TimelyException;
void flush() throws TimelyException;
long getAgeOffForMetric(String metricName);
List<Range> getQueryRanges(String metric, long start, long end, Set<Tag> colFamValues);
@@ -45,4 +43,6 @@ public interface DataStore {
DataStoreCache getCache();
InternalMetrics getInternalMetrics();
void close() throws Exception;
}
@@ -22,10 +22,10 @@ import java.util.Map.Entry;
import java.util.Optional;
import java.util.Set;
import java.util.SortedMap;
import java.util.Timer;
import java.util.TimerTask;
import java.util.TreeMap;
import java.util.TreeSet;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;
@@ -127,7 +127,7 @@ public class DataStoreImpl implements DataStore {
private final String metricsTable;
private final String metaTable;
private final InternalMetrics internalMetrics;
private final Timer internalMetricsTimer = new Timer("InternalMetricsTimer", true);
private ScheduledExecutorService executorService = Executors.newScheduledThreadPool(1);
private final int scannerThreads;
private final long maxDownsampleMemory;
private final BatchWriterConfig bwConfig;
@@ -244,14 +244,9 @@ public class DataStoreImpl implements DataStore {
}
internalMetrics = new InternalMetrics(conf);
internalMetricsTimer.schedule(new TimerTask() {
@Override
public void run() {
internalMetrics.getMetricsAndReset().forEach(m -> store(m, false));
}
}, METRICS_PERIOD, METRICS_PERIOD);
executorService.scheduleAtFixedRate(
() -> internalMetrics.getMetricsAndReset().forEach(m -> store(m, false)), METRICS_PERIOD,
METRICS_PERIOD, TimeUnit.MILLISECONDS);
this.metaCache = MetaCacheFactory.getCache(conf);
} catch (Exception e2) {
@@ -484,19 +479,6 @@ public class DataStoreImpl implements DataStore {
return result;
}
@Override
public void flush() {
internalMetricsTimer.cancel();
writers.forEach(w -> {
try {
w.close();
} catch (final Exception ex) {
LOG.warn("Error shutting down batchwriter", ex);
}
});
}
@Override
public SearchLookupResponse lookup(SearchLookupRequest msg) throws TimelyException {
long startMillis = System.currentTimeMillis();
@@ -1054,4 +1036,24 @@ public class DataStoreImpl implements DataStore {
}
}
@Override
public void close() throws Exception {
this.executorService.shutdown();
try {
this.executorService.awaitTermination(5, TimeUnit.SECONDS);
} catch (InterruptedException e) {
} finally {
if (!this.executorService.isTerminated()) {
this.executorService.shutdownNow();
}
}
writers.forEach(w -> {
try {
w.close();
} catch (final Exception ex) {
LOG.warn("Error shutting down batchwriter: ", ex.getMessage());
}
});
}
}
@@ -5,9 +5,10 @@ import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;
import java.util.Timer;
import java.util.TimerTask;
import java.util.TreeSet;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import com.github.benmanes.caffeine.cache.Cache;
import com.github.benmanes.caffeine.cache.Caffeine;
@@ -37,7 +38,7 @@ public class MetaCacheImpl implements MetaCache {
private volatile boolean closed = false;
private Cache<Meta, Object> cache = null;
private Configuration configuration;
private Timer timer = new Timer("MetaCacheImpl", true);
private ScheduledExecutorService executorService = Executors.newScheduledThreadPool(1);
@Override
public void init(Configuration config) {
@@ -45,13 +46,8 @@ public class MetaCacheImpl implements MetaCache {
cache = Caffeine.newBuilder().initialCapacity(1000).build();
long cacheRefreshMinutes = configuration.getMetaCache().getCacheRefreshMinutes();
if (cacheRefreshMinutes > 0) {
timer.schedule(new TimerTask() {
@Override
public void run() {
refreshCache(config.getMetaCache().getExpirationMinutes());
}
}, 60000, cacheRefreshMinutes * 60 * 1000);
executorService.scheduleAtFixedRate(() -> refreshCache(config.getMetaCache().getExpirationMinutes()), 1,
cacheRefreshMinutes, TimeUnit.MINUTES);
}
}
@@ -72,7 +68,10 @@ public class MetaCacheImpl implements MetaCache {
new PasswordToken(accumuloConf.getPassword()));
scanner = connector.createScanner(metaTable, Authorizations.EMPTY);
LOG.debug("Begin scanning " + metaTable);
Range metricRange = new Range(new Key(Meta.METRIC_PREFIX), true, new Key(Meta.METRIC_PREFIX + '\0'), false);
Key metricPrefixBeginKey = new Key(Meta.METRIC_PREFIX);
int firstChar = Meta.METRIC_PREFIX.charAt(0);
Key metricPrefixEndKey = new Key(String.valueOf((char) (firstChar + 1)) + ':');
Range metricRange = new Range(metricPrefixBeginKey, true, metricPrefixEndKey, false);
scanner.setRange(metricRange);
Set<String> allMetrics = new TreeSet<>();
for (Map.Entry<Key, Value> entry : scanner) {
@@ -145,6 +144,16 @@ public class MetaCacheImpl implements MetaCache {
@Override
public void close() {
this.closed = true;
this.executorService.shutdown();
try {
this.executorService.awaitTermination(5, TimeUnit.SECONDS);
} catch (InterruptedException e) {
} finally {
if (!this.executorService.isTerminated()) {
this.executorService.shutdownNow();
}
}
}
@Override
+61 -67
View File
@@ -11,9 +11,9 @@ import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.Timer;
import java.util.TimerTask;
import java.util.TreeSet;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.StampedLock;
@@ -79,7 +79,7 @@ public class DataStoreCache {
private int flushBatch = 0;
private int numBatches = 5;
private InternalMetrics internalMetrics;
private Timer maintenanceTimer = new Timer("DataStoreCacheTimer");
private ScheduledExecutorService executorService = Executors.newScheduledThreadPool(6);
public DataStoreCache(CuratorFramework curatorFramework, Configuration conf) {
if (curatorFramework != null) {
@@ -103,82 +103,58 @@ public class DataStoreCache {
minimumAgeOffForIterator = getAgeOffForIterator(minimumAgeOff);
maintenanceTimer.schedule(new TimerTask() {
@Override
public void run() {
try {
flushCaches(flushBatch);
} catch (Exception e) {
LOG.error(e.getMessage(), e);
} finally {
flushBatch++;
if (flushBatch == numBatches) {
flushBatch = 0;
}
executorService.scheduleAtFixedRate(() -> {
try {
flushCaches(flushBatch);
} catch (Exception e) {
LOG.error(e.getMessage(), e);
} finally {
flushBatch++;
if (flushBatch == numBatches) {
flushBatch = 0;
}
}
}, conf.getCache().getFlushInterval(), conf.getCache().getFlushInterval());
}, conf.getCache().getFlushInterval(), conf.getCache().getFlushInterval(), TimeUnit.MILLISECONDS);
maintenanceTimer.schedule(new TimerTask() {
@Override
public void run() {
try {
removeStaleMetrics();
} catch (Exception e) {
LOG.error(e.getMessage(), e);
}
executorService.scheduleAtFixedRate(() -> {
try {
DataStoreCache.this.removeStaleMetrics();
} catch (Exception e) {
LOG.error(e.getMessage(), e);
}
}, (300 * 1000), (300 * 1000));
}, 5, 5, TimeUnit.MINUTES);
maintenanceTimer.schedule(new TimerTask() {
@Override
public void run() {
try {
ageOffGorillaStores();
} catch (Exception e) {
LOG.error(e.getMessage(), e);
}
executorService.scheduleAtFixedRate(() -> {
try {
ageOffGorillaStores();
} catch (Exception e) {
LOG.error(e.getMessage(), e);
}
}, (600 * 1000), (600 * 1000));
}, 10, 10, TimeUnit.MINUTES);
maintenanceTimer.schedule(new TimerTask() {
@Override
public void run() {
try {
archiveGorillaStoreCurrentCompressors();
} catch (Exception e) {
LOG.error(e.getMessage(), e);
}
executorService.scheduleAtFixedRate(() -> {
try {
archiveGorillaStoreCurrentCompressors();
} catch (Exception e) {
LOG.error(e.getMessage(), e);
}
}, (3600 * 1000), (3600 * 1000));
}, 60, 60, TimeUnit.MINUTES);
maintenanceTimer.schedule(new TimerTask() {
@Override
public void run() {
try {
pruneStats();
} catch (Exception e) {
LOG.error(e.getMessage(), e);
}
executorService.scheduleAtFixedRate(() -> {
try {
pruneStats();
} catch (Exception e) {
LOG.error(e.getMessage(), e);
}
}, (600 * 1000), (600 * 1000));
}, 10, 10, TimeUnit.MINUTES);
maintenanceTimer.schedule(new TimerTask() {
@Override
public void run() {
try {
reportInternalMetrics();
} catch (Exception e) {
LOG.error(e.getMessage(), e);
}
executorService.scheduleAtFixedRate(() -> {
try {
reportInternalMetrics();
} catch (Exception e) {
LOG.error(e.getMessage(), e);
}
}, (60 * 1000), (60 * 1000));
}, 1, 1, TimeUnit.MINUTES);
}
private void addNonCachedMetrics(Collection<String> nonCachedMetricsUpdate) {
@@ -230,6 +206,11 @@ public class DataStoreCache {
nonCachedMetricsIPRWLock = new InterProcessReadWriteLock(curatorFramework, NON_CACHED_METRICS_LOCK_PATH);
testIPRWLock(curatorFramework, nonCachedMetricsIPRWLock, NON_CACHED_METRICS_LOCK_PATH);
nonCachedMetricsIP = new DistributedAtomicValue(curatorFramework, NON_CACHED_METRICS, new RetryForever(1000));
try {
nonCachedMetricsIP.initialize(SerializationUtils.serialize(new TreeSet<>()));
} catch (Exception e) {
LOG.error(e.getMessage(), e);
}
TreeCacheListener nonCachedMetricsListener = new TreeCacheListener() {
@Override
@@ -930,4 +911,17 @@ public class DataStoreCache {
}
return response;
}
public void close() {
this.executorService.shutdown();
try {
this.executorService.awaitTermination(5, TimeUnit.SECONDS);
} catch (InterruptedException e) {
} finally {
if (!this.executorService.isTerminated()) {
this.executorService.shutdownNow();
}
}
}
}
@@ -4,8 +4,15 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import java.util.*;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.SortedMap;
import java.util.TreeMap;
import org.apache.accumulo.core.client.IteratorSetting;
import org.apache.accumulo.core.data.ColumnUpdate;
@@ -251,6 +258,52 @@ public class DownsampleIteratorTest {
assertEquals((elapsedTime / sampleInterval) * numTagVariations, totalBuckets);
}
@Test
public void test() {
String hello = new String("this is a longer string");
long maybe = ObjectSizeOf.Sizer.OBJECT_OVERHEAD + // for String
ObjectSizeOf.Sizer.REFERENCE + // for array of char
ObjectSizeOf.Sizer.getPrimitiveObjectSize(int.class) +
ObjectSizeOf.Sizer.ARRAY_OVERHEAD + // for array of char
hello.length() * ObjectSizeOf.Sizer.getPrimitiveObjectSize(char.class);
maybe = ObjectSizeOf.Sizer.roundUp(maybe);
long size = ObjectSizeOf.Sizer.getObjectSize(hello, true, true);
System.out.println("size = " + size);
System.out.println("maybe = " + maybe);
}
@Test
public void test2() {
System.out.println(ObjectSizeOf.Sizer.getObjectSize(new Long(100)));
}
@Test
public void testDownsampleMemoryCalculation() throws Exception {
int numTagVariations = 1;
int sampleInterval = 50;
int elapsedTime = 100;
int skipInterval = 10;
SortedMap<Key, Value> testData3 = createTestData3(elapsedTime, skipInterval, numTagVariations);
DownsampleIterator iter = new DownsampleIterator();
Map<Set<Tag>, Downsample> samples = runQuery(iter, testData3, sampleInterval, 1000);
assertEquals(numTagVariations, samples.size());
long totalBuckets = 0;
for (Entry<Set<Tag>, Downsample> entry : samples.entrySet()) {
Set<Tag> tagSet = entry.getKey();
Downsample downsample = entry.getValue();
for (Tag tag : tagSet) {
assertEquals(ObjectSizeOf.Sizer.getObjectSize(tag, true, true),
ObjectSizeOf.Sizer.getObjectSize(tag, false, true));
}
}
}
private Map<Set<Tag>, Downsample> runQuery(SortedKeyValueIterator<Key, Value> iter, SortedMap<Key, Value> testData,
long period, long maxDownsampleMemory) throws Exception {
IteratorSetting is = new IteratorSetting(100, DownsampleIterator.class);
@@ -59,7 +59,7 @@ public class DataStoreIT extends OneWaySSLBase {
conf.setMetricAgeOffDays(ageOffSettings);
final Server s = new Server(conf);
s.run();
s.run(getSslContext());
try {
put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2",
"sys.cpu.user " + (TEST_TIME + ONE_HOUR) + " 3.0 tag1=value1 tag2=value2",
@@ -106,7 +106,7 @@ public class DataStoreIT extends OneWaySSLBase {
conf.setMetricAgeOffDays(ageOffSettings);
final Server s = new Server(conf);
s.run();
s.run(getSslContext());
// @formatter:off
/*
* TEST_TIME = System.currentTimeMillis() - ONE_DAY
@@ -5,6 +5,8 @@ import java.io.IOException;
import java.nio.file.Files;
import java.util.HashMap;
import io.netty.handler.ssl.SslContext;
import io.netty.handler.ssl.util.SelfSignedCertificate;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.minicluster.MiniAccumuloCluster;
import org.apache.accumulo.minicluster.MiniAccumuloConfig;
@@ -12,8 +14,10 @@ import org.junit.Before;
import org.junit.BeforeClass;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import timely.Server;
import timely.auth.AuthCache;
import timely.configuration.Configuration;
import timely.configuration.ServerSsl;
import timely.store.MetricAgeOffIterator;
import timely.test.TestConfiguration;
@@ -24,6 +28,11 @@ public class MacITBase {
private static final Logger LOG = LoggerFactory.getLogger(MacITBase.class);
protected static ServerSsl serverSsl = null;
protected static File clientTrustStoreFile = null;
protected static SslContext sslCtx = null;
protected static SelfSignedCertificate serverCert = null;
private static final File tempDir;
protected static final String MAC_ROOT_USER = "root";
@@ -50,8 +59,6 @@ public class MacITBase {
conf = TestConfiguration.createMinimalConfigurationForTest();
conf.getAccumulo().setInstanceName(mac.getInstanceName());
conf.getAccumulo().setZookeepers(mac.getZooKeepers());
conf.getSecurity().getServerSsl().setUseOpenssl(false);
conf.getSecurity().getServerSsl().setUseGeneratedKeypair(true);
conf.getWebsocket().setFlushIntervalSeconds(TestConfiguration.WAIT_SECONDS);
HashMap<String, Integer> ageOffSettings = new HashMap<>();
ageOffSettings.put(MetricAgeOffIterator.DEFAULT_AGEOFF_KEY, 7);
@@ -88,4 +95,52 @@ public class MacITBase {
public void initializeAuthCache() {
AuthCache.clear();
}
protected static void setupSSL() throws Exception {
setupSSL(TestConfiguration.createMinimalConfigurationForTest(), false);
}
protected static void setupSSL(Configuration config, boolean twoWaySsl) throws Exception {
setupSSL(config, twoWaySsl, null);
}
protected static void setupSSL(Configuration config, boolean twoWaySsl, SelfSignedCertificate serverCert)
throws Exception {
ServerSsl serverSsl = config.getSecurity().getServerSsl();
boolean generateNewSsl = MacITBase.serverSsl == null || serverCert != null
|| !MacITBase.serverSsl.equals(serverSsl);
if (generateNewSsl) {
// new serverCert and trustStoreFile
if (serverCert == null) {
MacITBase.serverCert = new SelfSignedCertificate();
} else {
MacITBase.serverCert = serverCert;
}
MacITBase.clientTrustStoreFile = MacITBase.serverCert.certificate().getAbsoluteFile();
MacITBase.serverSsl = serverSsl;
MacITBase.serverSsl.setCertificateFile(MacITBase.serverCert.certificate().getAbsolutePath());
MacITBase.serverSsl.setKeyFile(MacITBase.serverCert.privateKey().getAbsolutePath());
}
config.getSecurity().getServerSsl().setCertificateFile(MacITBase.serverSsl.getCertificateFile());
config.getSecurity().getServerSsl().setKeyFile(MacITBase.serverSsl.getKeyFile());
config.getSecurity().getServerSsl().setUseOpenssl(false);
config.getSecurity().getServerSsl().setUseGeneratedKeypair(false);
if (twoWaySsl) {
// Needed for 2way SSL
config.getSecurity().getServerSsl().setTrustStoreFile(MacITBase.serverSsl.getCertificateFile());
config.getSecurity().setAllowAnonymousHttpAccess(false);
} else {
config.getSecurity().setAllowAnonymousHttpAccess(true);
}
if (generateNewSsl) {
// new SslContext
OneWaySSLBase.sslCtx = Server.createSSLContext(config);
}
}
protected static SslContext getSslContext() {
return MacITBase.sslCtx;
}
}
@@ -30,7 +30,7 @@ public class OneWaySSLAnonAccessIT extends OneWaySSLBase {
@Test
public void testQueryWithVisibility() throws Exception {
final Server s = new Server(conf);
s.run();
s.run(getSslContext());
try {
put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2",
"sys.cpu.user " + (TEST_TIME + 1000) + " 3.0 tag1=value1 tag2=value2",
@@ -1,6 +1,5 @@
package timely.test.integration;
import java.io.File;
import java.net.URL;
import javax.net.ssl.HttpsURLConnection;
@@ -12,25 +11,21 @@ import io.netty.handler.ssl.JdkSslContext;
import io.netty.handler.ssl.SslContext;
import io.netty.handler.ssl.SslContextBuilder;
import io.netty.handler.ssl.SslProvider;
import io.netty.handler.ssl.util.SelfSignedCertificate;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import timely.auth.AuthCache;
import timely.configuration.Configuration;
/**
* Base test class for SSL with anonymous access
*/
public class OneWaySSLBase extends QueryBase {
protected static File clientTrustStoreFile = null;
protected SSLSocketFactory getSSLSocketFactory() throws Exception {
SslContextBuilder builder = SslContextBuilder.forClient();
builder.applicationProtocolConfig(ApplicationProtocolConfig.DISABLED);
builder.sslProvider(SslProvider.JDK);
builder.trustManager(clientTrustStoreFile); // Trust the server cert
builder.trustManager(MacITBase.clientTrustStoreFile); // Trust the server cert
SslContext ctx = builder.build();
Assert.assertTrue(ctx.isClient());
Assert.assertTrue(ctx instanceof JdkSslContext);
@@ -39,19 +34,9 @@ public class OneWaySSLBase extends QueryBase {
return jdkSslContext.getSocketFactory();
}
protected static void setupSSL(Configuration config) throws Exception {
SelfSignedCertificate serverCert = new SelfSignedCertificate();
config.getSecurity().getServerSsl().setCertificateFile(serverCert.certificate().getAbsolutePath());
clientTrustStoreFile = serverCert.certificate().getAbsoluteFile();
config.getSecurity().getServerSsl().setKeyFile(serverCert.privateKey().getAbsolutePath());
config.getSecurity().getServerSsl().setUseOpenssl(false);
config.getSecurity().getServerSsl().setUseGeneratedKeypair(false);
config.getSecurity().setAllowAnonymousHttpAccess(true);
}
@Before
public void configureSSL() throws Exception {
setupSSL(conf);
setupSSL(conf, false);
}
@Override
@@ -93,7 +93,7 @@ public class OneWaySSLBasicAuthAccessIT extends OneWaySSLBase {
@Test
public void testBasicAuthLogin() throws Exception {
final Server s = new Server(conf);
s.run();
s.run(getSslContext());
try {
String metrics = "https://localhost:54322/api/metrics";
query(metrics);
@@ -105,7 +105,7 @@ public class OneWaySSLBasicAuthAccessIT extends OneWaySSLBase {
@Test(expected = UnauthorizedUserException.class)
public void testBasicAuthLoginFailure() throws Exception {
final Server s = new Server(conf);
s.run();
s.run(getSslContext());
try {
String metrics = "https://localhost:54322/api/metrics";
query("test", "test2", metrics);
@@ -117,7 +117,7 @@ public class OneWaySSLBasicAuthAccessIT extends OneWaySSLBase {
@Test
public void testQueryWithVisibility() throws Exception {
final Server s = new Server(conf);
s.run();
s.run(getSslContext());
try {
put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2",
"sys.cpu.user " + (TEST_TIME + 1000) + " 3.0 tag1=value1 tag2=value2",
@@ -13,7 +13,7 @@ public class StaticFileServerIT extends OneWaySSLBase {
@Before
public void before() throws Exception {
s = new Server(conf);
s.run();
s.run(getSslContext());
}
@After
@@ -0,0 +1,92 @@
package timely.test.integration;
import java.net.URL;
import java.util.List;
import javax.net.ssl.HostnameVerifier;
import javax.net.ssl.HttpsURLConnection;
import javax.net.ssl.SSLContext;
import javax.net.ssl.SSLSession;
import javax.net.ssl.SSLSocketFactory;
import io.netty.handler.codec.http.HttpHeaderNames;
import io.netty.handler.codec.http.cookie.ClientCookieDecoder;
import io.netty.handler.codec.http.cookie.Cookie;
import io.netty.handler.ssl.ApplicationProtocolConfig;
import io.netty.handler.ssl.JdkSslContext;
import io.netty.handler.ssl.SslContext;
import io.netty.handler.ssl.SslContextBuilder;
import io.netty.handler.ssl.SslProvider;
import org.junit.After;
import org.junit.Assert;
import timely.auth.AuthCache;
import timely.netty.Constants;
/**
* Base test class for SSL with anonymous access
*/
public class TwoWaySSLBase extends QueryBase {
@After
public void tearDown() throws Exception {
AuthCache.resetConfiguration();
}
protected SSLSocketFactory getSSLSocketFactory() throws Exception {
SslContextBuilder builder = SslContextBuilder.forClient();
builder.applicationProtocolConfig(ApplicationProtocolConfig.DISABLED);
// Use server cert / key on client side
builder.keyManager(MacITBase.serverCert.key(), (String) null, MacITBase.serverCert.cert());
builder.sslProvider(SslProvider.JDK);
builder.trustManager(MacITBase.clientTrustStoreFile); // Trust the server cert
SslContext ctx = builder.build();
Assert.assertTrue(ctx.isClient());
Assert.assertTrue(ctx instanceof JdkSslContext);
JdkSslContext jdk = (JdkSslContext) ctx;
SSLContext jdkSslContext = jdk.context();
return jdkSslContext.getSocketFactory();
}
@Override
protected HttpsURLConnection getUrlConnection(URL url) throws Exception {
// Username and password not used in 2way SSL case
return getUrlConnection(null, null, url);
}
@Override
protected HttpsURLConnection getUrlConnection(String username, String password, URL url) throws Exception {
HttpsURLConnection.setDefaultSSLSocketFactory(getSSLSocketFactory());
URL loginURL = new URL(url.getProtocol() + "://" + url.getHost() + ":" + url.getPort() + "/login");
HttpsURLConnection con = (HttpsURLConnection) loginURL.openConnection();
con.setHostnameVerifier(new HostnameVerifier() {
@Override
public boolean verify(String arg0, SSLSession arg1) {
return true;
}
});
con.setRequestMethod("GET");
con.setDoOutput(true);
con.setRequestProperty("Content-Type", "application/json");
con.connect();
int responseCode = con.getResponseCode();
if (401 == responseCode) {
throw new UnauthorizedUserException();
}
Assert.assertEquals(200, responseCode);
List<String> cookies = con.getHeaderFields().get(HttpHeaderNames.SET_COOKIE.toString());
Assert.assertEquals(1, cookies.size());
Cookie sessionCookie = ClientCookieDecoder.STRICT.decode(cookies.get(0));
Assert.assertEquals(Constants.COOKIE_NAME, sessionCookie.name());
con = (HttpsURLConnection) url.openConnection();
con.setRequestProperty(HttpHeaderNames.COOKIE.toString(), sessionCookie.name() + "=" + sessionCookie.value());
con.setHostnameVerifier(new HostnameVerifier() {
@Override
public boolean verify(String arg0, SSLSession arg1) {
return true;
}
});
return con;
}
}
@@ -1,30 +1,11 @@
package timely.test.integration;
import java.io.File;
import java.net.URL;
import java.util.List;
import javax.net.ssl.HostnameVerifier;
import javax.net.ssl.HttpsURLConnection;
import javax.net.ssl.SSLContext;
import javax.net.ssl.SSLSession;
import javax.net.ssl.SSLSocketFactory;
import io.netty.handler.codec.http.HttpHeaderNames;
import io.netty.handler.codec.http.cookie.ClientCookieDecoder;
import io.netty.handler.codec.http.cookie.Cookie;
import io.netty.handler.ssl.ApplicationProtocolConfig;
import io.netty.handler.ssl.JdkSslContext;
import io.netty.handler.ssl.SslContext;
import io.netty.handler.ssl.SslContextBuilder;
import io.netty.handler.ssl.SslProvider;
import io.netty.handler.ssl.util.SelfSignedCertificate;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.minicluster.MiniAccumuloCluster;
import org.apache.accumulo.minicluster.MiniAccumuloConfig;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.ClassRule;
@@ -33,10 +14,9 @@ import org.junit.rules.TemporaryFolder;
import timely.Server;
import timely.auth.AuthCache;
import timely.configuration.Configuration;
import timely.netty.Constants;
import timely.test.TestConfiguration;
public class TwoWaySSLFailureIT extends QueryBase {
public class TwoWaySSLFailureIT extends TwoWaySSLBase {
@ClassRule
public static final TemporaryFolder temp = new TemporaryFolder();
@@ -44,87 +24,6 @@ public class TwoWaySSLFailureIT extends QueryBase {
private static MiniAccumuloCluster mac = null;
private static Configuration conf = null;
private static SelfSignedCertificate serverCert = null;
private static File clientTrustStoreFile = null;
static {
try {
// This fqdn does not match what is in security.xml
serverCert = new SelfSignedCertificate("CN=bad.example.com");
clientTrustStoreFile = serverCert.certificate().getAbsoluteFile();
} catch (Exception e) {
throw new RuntimeException("Error creating self signed certificate", e);
}
}
protected SSLSocketFactory getSSLSocketFactory() throws Exception {
SslContextBuilder builder = SslContextBuilder.forClient();
builder.applicationProtocolConfig(ApplicationProtocolConfig.DISABLED);
// Use server cert / key on client side
builder.keyManager(serverCert.key(), (String) null, serverCert.cert());
builder.sslProvider(SslProvider.JDK);
builder.trustManager(clientTrustStoreFile); // Trust the server cert
SslContext ctx = builder.build();
Assert.assertTrue(ctx.isClient());
Assert.assertTrue(ctx instanceof JdkSslContext);
JdkSslContext jdk = (JdkSslContext) ctx;
SSLContext jdkSslContext = jdk.context();
return jdkSslContext.getSocketFactory();
}
protected static void setupSSL(Configuration config) throws Exception {
config.getSecurity().getServerSsl().setCertificateFile(serverCert.certificate().getAbsolutePath());
config.getSecurity().getServerSsl().setKeyFile(serverCert.privateKey().getAbsolutePath());
// Needed for 2way SSL
config.getSecurity().getServerSsl().setTrustStoreFile(serverCert.certificate().getAbsolutePath());
config.getSecurity().getServerSsl().setUseOpenssl(false);
config.getSecurity().getServerSsl().setUseGeneratedKeypair(false);
config.getSecurity().setAllowAnonymousHttpAccess(false);
}
@Override
protected HttpsURLConnection getUrlConnection(URL url) throws Exception {
// Username and password not used in 2way SSL case
return getUrlConnection(null, null, url);
}
@Override
protected HttpsURLConnection getUrlConnection(String username, String password, URL url) throws Exception {
HttpsURLConnection.setDefaultSSLSocketFactory(getSSLSocketFactory());
URL loginURL = new URL(url.getProtocol() + "://" + url.getHost() + ":" + url.getPort() + "/login");
HttpsURLConnection con = (HttpsURLConnection) loginURL.openConnection();
con.setHostnameVerifier(new HostnameVerifier() {
@Override
public boolean verify(String arg0, SSLSession arg1) {
return true;
}
});
con.setRequestMethod("GET");
con.setDoOutput(true);
con.setRequestProperty("Content-Type", "application/json");
con.connect();
int responseCode = con.getResponseCode();
if (401 == responseCode) {
throw new UnauthorizedUserException();
}
Assert.assertEquals(200, responseCode);
List<String> cookies = con.getHeaderFields().get(HttpHeaderNames.SET_COOKIE.toString());
Assert.assertEquals(1, cookies.size());
Cookie sessionCookie = ClientCookieDecoder.STRICT.decode(cookies.get(0));
Assert.assertEquals(Constants.COOKIE_NAME, sessionCookie.name());
con = (HttpsURLConnection) url.openConnection();
con.setRequestProperty(HttpHeaderNames.COOKIE.toString(), sessionCookie.name() + "=" + sessionCookie.value());
con.setHostnameVerifier(new HostnameVerifier() {
@Override
public boolean verify(String arg0, SSLSession arg1) {
return true;
}
});
return con;
}
@BeforeClass
public static void beforeClass() throws Exception {
temp.create();
@@ -134,7 +33,8 @@ public class TwoWaySSLFailureIT extends QueryBase {
conf = TestConfiguration.createMinimalConfigurationForTest();
conf.getAccumulo().setInstanceName(mac.getInstanceName());
conf.getAccumulo().setZookeepers(mac.getZooKeepers());
setupSSL(conf);
// This fqdn does not match what is in security.xml
setupSSL(conf, true, new SelfSignedCertificate("CN=bad.example.com"));
}
@AfterClass
@@ -163,7 +63,7 @@ public class TwoWaySSLFailureIT extends QueryBase {
@Test(expected = UnauthorizedUserException.class)
public void testBasicAuthLoginFailure() throws Exception {
final Server s = new Server(conf);
s.run();
s.run(getSslContext());
try {
String metrics = "https://localhost:54322/api/metrics";
query(metrics);
@@ -3,33 +3,17 @@ package timely.test.integration;
import static com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly;
import static org.junit.Assert.assertEquals;
import java.io.File;
import java.net.URL;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.TimeUnit;
import javax.net.ssl.HttpsURLConnection;
import javax.net.ssl.SSLContext;
import javax.net.ssl.SSLSocketFactory;
import io.netty.handler.codec.http.HttpHeaderNames;
import io.netty.handler.codec.http.cookie.ClientCookieDecoder;
import io.netty.handler.codec.http.cookie.Cookie;
import io.netty.handler.ssl.ApplicationProtocolConfig;
import io.netty.handler.ssl.JdkSslContext;
import io.netty.handler.ssl.SslContext;
import io.netty.handler.ssl.SslContextBuilder;
import io.netty.handler.ssl.SslProvider;
import io.netty.handler.ssl.util.SelfSignedCertificate;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.minicluster.MiniAccumuloCluster;
import org.apache.accumulo.minicluster.MiniAccumuloConfig;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.ClassRule;
@@ -41,7 +25,6 @@ import timely.api.request.timeseries.QueryRequest;
import timely.api.response.timeseries.QueryResponse;
import timely.auth.AuthCache;
import timely.configuration.Configuration;
import timely.netty.Constants;
import timely.test.IntegrationTest;
import timely.test.TestConfiguration;
@@ -51,7 +34,7 @@ import timely.test.TestConfiguration;
*
*/
@Category(IntegrationTest.class)
public class TwoWaySSLIT extends QueryBase {
public class TwoWaySSLIT extends TwoWaySSLBase {
private static final Long TEST_TIME = System.currentTimeMillis();
@@ -61,74 +44,6 @@ public class TwoWaySSLIT extends QueryBase {
private static MiniAccumuloCluster mac = null;
private static Configuration conf = null;
protected static SelfSignedCertificate serverCert = null;
protected static File clientTrustStoreFile = null;
static {
try {
serverCert = new SelfSignedCertificate();
clientTrustStoreFile = serverCert.certificate().getAbsoluteFile();
} catch (Exception e) {
throw new RuntimeException("Error creating self signed certificate", e);
}
}
protected SSLSocketFactory getSSLSocketFactory() throws Exception {
SslContextBuilder builder = SslContextBuilder.forClient();
builder.applicationProtocolConfig(ApplicationProtocolConfig.DISABLED);
// Use server cert / key on client side.
builder.keyManager(serverCert.key(), (String) null, serverCert.cert());
builder.sslProvider(SslProvider.JDK);
builder.trustManager(clientTrustStoreFile); // Trust the server cert
SslContext ctx = builder.build();
Assert.assertTrue(ctx.isClient());
Assert.assertTrue(ctx instanceof JdkSslContext);
JdkSslContext jdk = (JdkSslContext) ctx;
SSLContext jdkSslContext = jdk.context();
return jdkSslContext.getSocketFactory();
}
protected static void setupSSL(Configuration config) throws Exception {
config.getSecurity().getServerSsl().setCertificateFile(serverCert.certificate().getAbsolutePath());
config.getSecurity().getServerSsl().setKeyFile(serverCert.privateKey().getAbsolutePath());
// Needed for 2way SSL
config.getSecurity().getServerSsl().setTrustStoreFile(serverCert.certificate().getAbsolutePath());
config.getSecurity().getServerSsl().setUseOpenssl(false);
config.getSecurity().getServerSsl().setUseGeneratedKeypair(false);
config.getSecurity().setAllowAnonymousHttpAccess(false);
}
@Override
protected HttpsURLConnection getUrlConnection(URL url) throws Exception {
// Username and password not used in 2way SSL case
return getUrlConnection(null, null, url);
}
@Override
protected HttpsURLConnection getUrlConnection(String username, String password, URL url) throws Exception {
HttpsURLConnection.setDefaultSSLSocketFactory(getSSLSocketFactory());
URL loginURL = new URL(url.getProtocol() + "://" + url.getHost() + ":" + url.getPort() + "/login");
HttpsURLConnection con = (HttpsURLConnection) loginURL.openConnection();
con.setHostnameVerifier((host, session) -> true);
con.setRequestMethod("GET");
con.setDoOutput(true);
con.setRequestProperty("Content-Type", "application/json");
con.connect();
int responseCode = con.getResponseCode();
if (401 == responseCode) {
throw new UnauthorizedUserException();
}
Assert.assertEquals(200, responseCode);
List<String> cookies = con.getHeaderFields().get(HttpHeaderNames.SET_COOKIE.toString());
Assert.assertEquals(1, cookies.size());
Cookie sessionCookie = ClientCookieDecoder.STRICT.decode(cookies.get(0));
Assert.assertEquals(Constants.COOKIE_NAME, sessionCookie.name());
con = (HttpsURLConnection) url.openConnection();
con.setRequestProperty(HttpHeaderNames.COOKIE.toString(), sessionCookie.name() + "=" + sessionCookie.value());
con.setHostnameVerifier((host, session) -> true);
return con;
}
@BeforeClass
public static void beforeClass() throws Exception {
final MiniAccumuloConfig macConfig = new MiniAccumuloConfig(temp.newFolder("mac"), "secret");
@@ -137,7 +52,6 @@ public class TwoWaySSLIT extends QueryBase {
conf = TestConfiguration.createMinimalConfigurationForTest();
conf.getAccumulo().setInstanceName(mac.getInstanceName());
conf.getAccumulo().setZookeepers(mac.getZooKeepers());
setupSSL(conf);
}
@AfterClass
@@ -157,6 +71,7 @@ public class TwoWaySSLIT extends QueryBase {
}
}
});
setupSSL(conf, true);
}
@After
@@ -167,7 +82,7 @@ public class TwoWaySSLIT extends QueryBase {
@Test
public void testBasicAuthLogin() throws Exception {
final Server s = new Server(conf);
s.run();
s.run(getSslContext());
try {
String metrics = "https://localhost:54322/api/metrics";
query(metrics);
@@ -190,7 +105,7 @@ public class TwoWaySSLIT extends QueryBase {
public void testQueryWithVisibility(Configuration conf) throws Exception {
final Server s = new Server(conf);
s.run();
s.run(getSslContext());
try {
// @formatter:off
put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2",
@@ -3,32 +3,14 @@ package timely.test.integration;
import static com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly;
import static org.junit.Assert.assertEquals;
import java.io.File;
import java.net.URL;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.TimeUnit;
import javax.net.ssl.HostnameVerifier;
import javax.net.ssl.HttpsURLConnection;
import javax.net.ssl.SSLContext;
import javax.net.ssl.SSLSession;
import javax.net.ssl.SSLSocketFactory;
import io.netty.handler.codec.http.HttpHeaderNames;
import io.netty.handler.codec.http.cookie.ClientCookieDecoder;
import io.netty.handler.codec.http.cookie.Cookie;
import io.netty.handler.ssl.ApplicationProtocolConfig;
import io.netty.handler.ssl.JdkSslContext;
import io.netty.handler.ssl.SslContext;
import io.netty.handler.ssl.SslContextBuilder;
import io.netty.handler.ssl.SslProvider;
import io.netty.handler.ssl.util.SelfSignedCertificate;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.security.Authorizations;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@@ -36,8 +18,6 @@ import timely.Server;
import timely.api.request.timeseries.QueryRequest;
import timely.api.response.timeseries.QueryResponse;
import timely.auth.AuthCache;
import timely.configuration.Configuration;
import timely.netty.Constants;
import timely.test.IntegrationTest;
import timely.test.TestConfiguration;
@@ -48,97 +28,13 @@ import timely.test.TestConfiguration;
*
*/
@Category(IntegrationTest.class)
public class TwoWaySSLOpenSSLIT extends QueryBase {
public class TwoWaySSLOpenSSLIT extends TwoWaySSLBase {
private static final Long TEST_TIME = System.currentTimeMillis();
protected static SelfSignedCertificate serverCert = null;
protected static File clientTrustStoreFile = null;
static {
try {
serverCert = new SelfSignedCertificate();
clientTrustStoreFile = serverCert.certificate().getAbsoluteFile();
} catch (Exception e) {
throw new RuntimeException("Error creating self signed certificate", e);
}
}
protected SSLSocketFactory getSSLSocketFactory() throws Exception {
SslContextBuilder builder = SslContextBuilder.forClient();
builder.applicationProtocolConfig(ApplicationProtocolConfig.DISABLED);
// Use server cert / key on client side.
builder.keyManager(serverCert.key(), (String) null, serverCert.cert());
builder.sslProvider(SslProvider.JDK);
builder.trustManager(clientTrustStoreFile); // Trust the server cert
SslContext ctx = builder.build();
Assert.assertTrue(ctx.isClient());
Assert.assertTrue(ctx instanceof JdkSslContext);
JdkSslContext jdk = (JdkSslContext) ctx;
SSLContext jdkSslContext = jdk.context();
return jdkSslContext.getSocketFactory();
}
protected static void setupSSL(Configuration config) throws Exception {
config.getSecurity().getServerSsl().setCertificateFile(serverCert.certificate().getAbsolutePath());
config.getSecurity().getServerSsl().setKeyFile(serverCert.privateKey().getAbsolutePath());
// Needed for 2way SSL
config.getSecurity().getServerSsl().setTrustStoreFile(serverCert.certificate().getAbsolutePath());
config.getSecurity().getServerSsl().setUseOpenssl(false);
config.getSecurity().getServerSsl().setUseGeneratedKeypair(false);
config.getSecurity().setAllowAnonymousHttpAccess(false);
}
@Before
public void configureSSL() throws Exception {
setupSSL(conf);
}
@Override
protected HttpsURLConnection getUrlConnection(URL url) throws Exception {
// Username and password not used in 2way SSL case
return getUrlConnection(null, null, url);
}
@Override
protected HttpsURLConnection getUrlConnection(String username, String password, URL url) throws Exception {
HttpsURLConnection.setDefaultSSLSocketFactory(getSSLSocketFactory());
URL loginURL = new URL(url.getProtocol() + "://" + url.getHost() + ":" + url.getPort() + "/login");
HttpsURLConnection con = (HttpsURLConnection) loginURL.openConnection();
con.setHostnameVerifier(new HostnameVerifier() {
@Override
public boolean verify(String arg0, SSLSession arg1) {
return true;
}
});
con.setRequestMethod("GET");
con.setDoOutput(true);
con.setRequestProperty("Content-Type", "application/json");
con.connect();
int responseCode = con.getResponseCode();
if (401 == responseCode) {
throw new UnauthorizedUserException();
}
Assert.assertEquals(200, responseCode);
List<String> cookies = con.getHeaderFields().get(HttpHeaderNames.SET_COOKIE.toString());
Assert.assertEquals(1, cookies.size());
Cookie sessionCookie = ClientCookieDecoder.STRICT.decode(cookies.get(0));
Assert.assertEquals(Constants.COOKIE_NAME, sessionCookie.name());
con = (HttpsURLConnection) url.openConnection();
con.setRequestProperty(HttpHeaderNames.COOKIE.toString(), sessionCookie.name() + "=" + sessionCookie.value());
con.setHostnameVerifier(new HostnameVerifier() {
@Override
public boolean verify(String arg0, SSLSession arg1) {
return true;
}
});
return con;
}
@Before
public void setup() throws Exception {
setupSSL(conf, true);
Connector con = mac.getConnector("root", "secret");
con.securityOperations().changeUserAuthorizations("root", new Authorizations("A", "B", "C", "D", "E", "F"));
}
@@ -151,7 +47,7 @@ public class TwoWaySSLOpenSSLIT extends QueryBase {
@Test
public void testBasicAuthLogin() throws Exception {
final Server s = new Server(conf);
s.run();
s.run(getSslContext());
try {
String metrics = "https://localhost:54322/api/metrics";
query(metrics);
@@ -163,7 +59,7 @@ public class TwoWaySSLOpenSSLIT extends QueryBase {
@Test
public void testQueryWithVisibility() throws Exception {
final Server s = new Server(conf);
s.run();
s.run(getSslContext());
try {
put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2",
"sys.cpu.user " + (TEST_TIME + 1000) + " 3.0 tag1=value1 tag2=value2",
@@ -49,7 +49,7 @@ public class TcpClientIT extends OneWaySSLBase {
@Test
public void testPut() throws Exception {
final TestServer m = new TestServer(conf);
m.run();
m.run(getSslContext());
try (TcpClient client = new TcpClient("127.0.0.1", 54321)) {
client.open();
client.write("put sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2\n");
@@ -80,7 +80,7 @@ public class TcpClientIT extends OneWaySSLBase {
public void testPutMultiple() throws Exception {
final TestServer m = new TestServer(conf);
m.run();
m.run(getSslContext());
try (TcpClient client = new TcpClient("127.0.0.1", 54321)) {
client.open();
// @formatter:off
@@ -124,7 +124,7 @@ public class TcpClientIT extends OneWaySSLBase {
@Test
public void testPutInvalidTimestamp() throws Exception {
final TestServer m = new TestServer(conf);
m.run();
m.run(getSslContext());
try (TcpClient client = new TcpClient("127.0.0.1", 54321)) {
client.open();
client.write("put sys.cpu.user " + TEST_TIME + "Z" + " 1.0 tag1=value1 tag2=value2\n");
@@ -139,7 +139,7 @@ public class TcpClientIT extends OneWaySSLBase {
@Test
public void testPersistence() throws Exception {
final Server s = new Server(conf);
s.run();
s.run(getSslContext());
try {
put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2",
"sys.cpu.idle " + (TEST_TIME + 1) + " 1.0 tag3=value3 tag4=value4",
@@ -183,7 +183,7 @@ public class TcpClientIT extends OneWaySSLBase {
@Test
public void testPersistenceWithVisibility() throws Exception {
final Server s = new Server(conf);
s.run();
s.run(getSslContext());
try {
put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2",
"sys.cpu.idle " + (TEST_TIME + 1) + " 1.0 tag3=value3 tag4=value4 viz=(a|b)",
@@ -53,6 +53,7 @@ public class UdpClientIT extends MacITBase {
}
}
});
setupSSL();
}
@After
@@ -63,7 +64,7 @@ public class UdpClientIT extends MacITBase {
@Test
public void testPut() throws Exception {
final TestServer m = new TestServer(conf);
m.run();
m.run(getSslContext());
try (UdpClient client = new UdpClient("127.0.0.1", 54325)) {
client.open();
client.write(("put sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2\n"));
@@ -93,7 +94,7 @@ public class UdpClientIT extends MacITBase {
public void testPutMultiple() throws Exception {
final TestServer m = new TestServer(conf);
m.run();
m.run(getSslContext());
try (UdpClient client = new UdpClient("127.0.0.1", 54325)) {
client.open();
client.write(("put sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2\n" + "put sys.cpu.idle "
@@ -135,7 +136,7 @@ public class UdpClientIT extends MacITBase {
@Test
public void testPutInvalidTimestamp() throws Exception {
final TestServer m = new TestServer(conf);
m.run();
m.run(getSslContext());
try (UdpClient client = new UdpClient("127.0.0.1", 54325)) {
client.open();
client.write(("put sys.cpu.user " + TEST_TIME + "Z" + " 1.0 tag1=value1 tag2=value2\n"));
@@ -149,7 +150,7 @@ public class UdpClientIT extends MacITBase {
@Test
public void testPersistence() throws Exception {
final Server s = new Server(conf);
s.run();
s.run(getSslContext());
try {
put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2",
"sys.cpu.idle " + (TEST_TIME + 1) + " 1.0 tag3=value3 tag4=value4",
@@ -193,7 +194,7 @@ public class UdpClientIT extends MacITBase {
@Test
public void testPersistenceWithVisibility() throws Exception {
final Server s = new Server(conf);
s.run();
s.run(getSslContext());
try {
put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2",
"sys.cpu.idle " + (TEST_TIME + 1) + " 1.0 tag3=value3 tag4=value4 viz=(a|b)",
@@ -65,7 +65,7 @@ public class WebSocketClientIT extends OneWaySSLBase {
Connector con = mac.getConnector("root", "secret");
con.securityOperations().changeUserAuthorizations("root", new Authorizations("A", "B", "C", "D", "E", "F"));
s = new Server(conf);
s.run();
s.run(getSslContext());
setupSslCtx();
}
@@ -23,7 +23,7 @@ public class HTTPStrictTransportSecurityIT extends OneWaySSLBase {
@Before
public void before() throws Exception {
s = new Server(conf);
s.run();
s.run(getSslContext());
}
@After
@@ -58,7 +58,7 @@ public class HttpApiIT extends OneWaySSLBase {
@Test
public void testSuggest() throws Exception {
final Server s = new Server(conf);
s.run();
s.run(getSslContext());
try {
put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2",
"sys.cpu.idle " + (TEST_TIME + 1) + " 1.0 tag3=value3 tag4=value4",
@@ -86,7 +86,7 @@ public class HttpApiIT extends OneWaySSLBase {
@Test
public void testMetrics() throws Exception {
final Server s = new Server(conf);
s.run();
s.run(getSslContext());
try {
// @formatter:off
put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2",
@@ -120,7 +120,7 @@ public class HttpApiIT extends OneWaySSLBase {
// @formatter:off
final Server s = new Server(conf);
s.run();
s.run(getSslContext());
try {
put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2",
"sys.cpu.idle " + (TEST_TIME + 1) + " 1.0 tag3=value3 tag4=value4",
@@ -175,7 +175,7 @@ public class HttpApiIT extends OneWaySSLBase {
@Test
public void testLookup() throws Exception {
final Server s = new Server(conf);
s.run();
s.run(getSslContext());
try {
put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2",
"sys.cpu.user " + (TEST_TIME + 1) + " 1.0 tag3=value3",
@@ -211,7 +211,7 @@ public class HttpApiIT extends OneWaySSLBase {
@Test
public void testQueryWithMsResolution() throws Exception {
final Server s = new Server(conf);
s.run();
s.run(getSslContext());
try {
put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2",
"sys.cpu.user " + (TEST_TIME + 1) + " 1.0 tag3=value3",
@@ -259,7 +259,7 @@ public class HttpApiIT extends OneWaySSLBase {
@Test
public void testQueryWithoutMsResolution() throws Exception {
final Server s = new Server(conf);
s.run();
s.run(getSslContext());
try {
put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2",
"sys.cpu.user " + (TEST_TIME + 1) + " 1.0 tag3=value3",
@@ -298,7 +298,7 @@ public class HttpApiIT extends OneWaySSLBase {
@Test
public void testQueryWithNoTags() throws Exception {
final Server s = new Server(conf);
s.run();
s.run(getSslContext());
try {
put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2",
"sys.cpu.user " + (TEST_TIME + 1) + " 1.0 tag3=value3",
@@ -334,7 +334,7 @@ public class HttpApiIT extends OneWaySSLBase {
@Test
public void testQueryWithNoTagsMultipleSeries() throws Exception {
final Server s = new Server(conf);
s.run();
s.run(getSslContext());
try {
put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2 host=h1",
"sys.cpu.user " + TEST_TIME + " 2.0 tag1=value1 tag2=value2 host=h2",
@@ -375,7 +375,7 @@ public class HttpApiIT extends OneWaySSLBase {
@Test(expected = NotSuccessfulException.class)
public void testQueryWithNoMatchingTags() throws Exception {
final Server s = new Server(conf);
s.run();
s.run(getSslContext());
try {
put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2 rack=r1",
"sys.cpu.user " + (TEST_TIME + 1) + " 1.0 tag3=value3 rack=r2",
@@ -399,7 +399,7 @@ public class HttpApiIT extends OneWaySSLBase {
@Test
public void testQueryWithTagWildcard() throws Exception {
final Server s = new Server(conf);
s.run();
s.run(getSslContext());
// @formatter:off
try {
put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2 rack=r1",
@@ -464,7 +464,7 @@ public class HttpApiIT extends OneWaySSLBase {
@Test
public void testQueryWithTagWildcard2() throws Exception {
final Server s = new Server(conf);
s.run();
s.run(getSslContext());
// @formatter:off
try {
put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2 rack=r1",
@@ -524,7 +524,7 @@ public class HttpApiIT extends OneWaySSLBase {
@Test
public void testQueryWithTagOr() throws Exception {
final Server s = new Server(conf);
s.run();
s.run(getSslContext());
try {
// @formatter:off
put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2 rack=r1",
@@ -608,7 +608,7 @@ public class HttpApiIT extends OneWaySSLBase {
@Test
public void testQueryWithTagRegex() throws Exception {
final Server s = new Server(conf);
s.run();
s.run(getSslContext());
try {
System.out.println("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2 rack=r1");
System.out.println("sys.cpu.user " + (TEST_TIME + 1) + " 1.0 tag3=value3 rack=r2");
@@ -674,7 +674,7 @@ public class HttpApiIT extends OneWaySSLBase {
@Test
public void testQueryWithTagRegex2() throws Exception {
final Server s = new Server(conf);
s.run();
s.run(getSslContext());
try {
put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2 rack=r1",
"sys.cpu.user " + (TEST_TIME + 1) + " 1.0 tag3=value3 rack=r2",
@@ -730,7 +730,7 @@ public class HttpApiIT extends OneWaySSLBase {
@Test
public void testRateQuery() throws Exception {
final Server s = new Server(conf);
s.run();
s.run(getSslContext());
try {
// @formatter:off
put("sys.cpu.user " + (TEST_TIME) + " 1.0 tag1=value1 tag2=value2 rack=r1",
@@ -782,7 +782,7 @@ public class HttpApiIT extends OneWaySSLBase {
@Test
public void testRateCounterQuery() throws Exception {
final Server s = new Server(conf);
s.run();
s.run(getSslContext());
try {
// @formatter:off
put("sys.cpu.user " + (TEST_TIME) + " 1.0 tag1=value1 tag2=value2 rack=r1",
@@ -836,7 +836,7 @@ public class HttpApiIT extends OneWaySSLBase {
@Test
public void testRateCounterWithOptions() throws Exception {
final Server s = new Server(conf);
s.run();
s.run(getSslContext());
try {
// @formatter:off
put("sys.cpu.user " + (TEST_TIME) + " 1.0 tag1=value1 tag2=value2 rack=r1",
@@ -892,7 +892,7 @@ public class HttpApiIT extends OneWaySSLBase {
@Test
public void testGetVersion() throws Exception {
final Server s = new Server(conf);
s.run();
s.run(getSslContext());
try {
String response = query("https://127.0.0.1:54322/version", "application/json");
assertNotNull(response);
@@ -905,7 +905,7 @@ public class HttpApiIT extends OneWaySSLBase {
@Test
public void testPutMetric() throws Exception {
final Server s = new Server(conf);
s.run();
s.run(getSslContext());
try {
Metric m = Metric.newBuilder().name("sys.cpu.user").value(TEST_TIME, 1.0D).tag(new Tag("tag1", "value1"))
.build();
@@ -56,7 +56,7 @@ public class TimelyTcpIT extends MacITBase {
@Test
public void testVersion() throws Exception {
final TestServer m = new TestServer(conf);
m.run();
m.run(getSslContext());
try (Socket sock = new Socket("127.0.0.1", 54321);
PrintWriter writer = new PrintWriter(sock.getOutputStream(), true);) {
writer.write("version\n");
@@ -76,7 +76,7 @@ public class TimelyTcpIT extends MacITBase {
@Test
public void testPut() throws Exception {
final TestServer m = new TestServer(conf);
m.run();
m.run(getSslContext());
try (Socket sock = new Socket("127.0.0.1", 54321);
PrintWriter writer = new PrintWriter(sock.getOutputStream(), true);) {
writer.write("put sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2\n");
@@ -107,7 +107,7 @@ public class TimelyTcpIT extends MacITBase {
public void testPutMultiple() throws Exception {
final TestServer m = new TestServer(conf);
m.run();
m.run(getSslContext());
try (Socket sock = new Socket("127.0.0.1", 54321);
PrintWriter writer = new PrintWriter(sock.getOutputStream(), true)) {
// @formatter:off
@@ -190,7 +190,7 @@ public class TimelyTcpIT extends MacITBase {
LOG.debug("Sending {} bytes", data.length);
final TestServer m = new TestServer(conf);
m.run();
m.run(getSslContext());
try (Socket sock = new Socket("127.0.0.1", 54321);) {
sock.getOutputStream().write(data);
sock.getOutputStream().flush();
@@ -233,7 +233,7 @@ public class TimelyTcpIT extends MacITBase {
@Test
public void testPutInvalidTimestamp() throws Exception {
final TestServer m = new TestServer(conf);
m.run();
m.run(getSslContext());
try (Socket sock = new Socket("127.0.0.1", 54321);
PrintWriter writer = new PrintWriter(sock.getOutputStream(), true);
BufferedReader reader = new BufferedReader(new InputStreamReader(sock.getInputStream()));) {
@@ -249,7 +249,7 @@ public class TimelyTcpIT extends MacITBase {
@Test
public void testPersistence() throws Exception {
final Server s = new Server(conf);
s.run();
s.run(getSslContext());
try {
put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2",
"sys.cpu.idle " + (TEST_TIME + 1) + " 1.0 tag3=value3 tag4=value4",
@@ -293,7 +293,7 @@ public class TimelyTcpIT extends MacITBase {
@Test
public void testPersistenceWithVisibility() throws Exception {
final Server s = new Server(conf);
s.run();
s.run(getSslContext());
try {
put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2",
"sys.cpu.idle " + (TEST_TIME + 1) + " 1.0 tag3=value3 tag4=value4 viz=(a|b)",
@@ -60,6 +60,7 @@ public class TimelyUdpIT extends MacITBase {
}
}
});
setupSSL();
}
@After
@@ -70,7 +71,7 @@ public class TimelyUdpIT extends MacITBase {
@Test
public void testPut() throws Exception {
final TestServer m = new TestServer(conf);
m.run();
m.run(getSslContext());
InetSocketAddress address = new InetSocketAddress("127.0.0.1", 54325);
DatagramPacket packet = new DatagramPacket("".getBytes(UTF_8), 0, 0, address.getAddress(), 54325);
try (DatagramSocket sock = new DatagramSocket()) {
@@ -102,7 +103,7 @@ public class TimelyUdpIT extends MacITBase {
public void testPutMultiple() throws Exception {
final TestServer m = new TestServer(conf);
m.run();
m.run(getSslContext());
InetSocketAddress address = new InetSocketAddress("127.0.0.1", 54325);
DatagramPacket packet = new DatagramPacket("".getBytes(UTF_8), 0, 0, address.getAddress(), 54325);
// @formatter:off
@@ -185,7 +186,7 @@ public class TimelyUdpIT extends MacITBase {
LOG.debug("Sending {} bytes", data.length);
final TestServer m = new TestServer(conf);
m.run();
m.run(getSslContext());
InetSocketAddress address = new InetSocketAddress("127.0.0.1", 54325);
DatagramPacket packet = new DatagramPacket("".getBytes(UTF_8), 0, 0, address.getAddress(), 54325);
try (DatagramSocket sock = new DatagramSocket()) {
@@ -229,7 +230,7 @@ public class TimelyUdpIT extends MacITBase {
@Test
public void testPutInvalidTimestamp() throws Exception {
final TestServer m = new TestServer(conf);
m.run();
m.run(getSslContext());
InetSocketAddress address = new InetSocketAddress("127.0.0.1", 54325);
DatagramPacket packet = new DatagramPacket("".getBytes(UTF_8), 0, 0, address.getAddress(), 54325);
try (DatagramSocket sock = new DatagramSocket();) {
@@ -245,7 +246,7 @@ public class TimelyUdpIT extends MacITBase {
@Test
public void testPersistence() throws Exception {
final Server s = new Server(conf);
s.run();
s.run(getSslContext());
try {
put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2",
"sys.cpu.idle " + (TEST_TIME + 1) + " 1.0 tag3=value3 tag4=value4",
@@ -289,7 +290,7 @@ public class TimelyUdpIT extends MacITBase {
@Test
public void testPersistenceWithVisibility() throws Exception {
final Server s = new Server(conf);
s.run();
s.run(getSslContext());
try {
put("sys.cpu.user " + TEST_TIME + " 1.0 tag1=value1 tag2=value2",
"sys.cpu.idle " + (TEST_TIME + 1) + " 1.0 tag3=value3 tag4=value4 viz=(a|b)",
@@ -189,7 +189,7 @@ public class WebSocketIT extends OneWaySSLBase {
@Before
public void setup() throws Exception {
s = new Server(conf);
s.run();
s.run(getSslContext());
Connector con = mac.getConnector("root", "secret");
con.securityOperations().changeUserAuthorizations("root", new Authorizations("A", "B", "C", "D", "E", "F"));