mirror of
https://github.com/NationalSecurityAgency/timely.git
synced 2026-11-09 17:36:01 -10:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
cb875819ff | ||
|
|
fcc5f0f716 | ||
|
|
c87543f7d7 | ||
|
|
d7f5a3b20b | ||
|
|
9601cdc927 | ||
|
|
60c2588c79 |
No files matched your search
+1
-1
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>timely</groupId>
|
||||
<artifactId>timely</artifactId>
|
||||
<version>2.0.1</version>
|
||||
<version>2.0.2</version>
|
||||
</parent>
|
||||
<artifactId>timely-analytics</artifactId>
|
||||
<!--
|
||||
|
||||
@@ -7,6 +7,7 @@ import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.apache.flink.streaming.util.serialization.SerializationSchema;
|
||||
|
||||
import timely.model.Tag;
|
||||
|
||||
public class MetricHistogram implements Serializable, SerializationSchema<MetricHistogram> {
|
||||
|
||||
@@ -1,13 +1,13 @@
|
||||
package timely.analytics.flink;
|
||||
|
||||
import org.apache.flink.api.java.functions.KeySelector;
|
||||
|
||||
import timely.api.response.MetricResponse;
|
||||
|
||||
/**
|
||||
* Uniquely defines a time series in a Flink stream. A unique time series is the
|
||||
* metric name and all of its tags, to include visibility if it exists.
|
||||
* Uniquely defines a time series in a Flink stream. A unique time series is the metric name and all of its tags, to include visibility if it exists.
|
||||
*/
|
||||
public class MetricKeySelector implements KeySelector<MetricResponse, String> {
|
||||
public class MetricKeySelector implements KeySelector<MetricResponse,String> {
|
||||
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
|
||||
@@ -9,11 +9,11 @@ import org.apache.flink.api.common.accumulators.Accumulator;
|
||||
* Custom accumulator that retains a sorted map of strings and their counts
|
||||
*
|
||||
*/
|
||||
public class SortedStringAccumulator implements Accumulator<String, ConcurrentSkipListMap<String, MutableLong>> {
|
||||
public class SortedStringAccumulator implements Accumulator<String,ConcurrentSkipListMap<String,MutableLong>> {
|
||||
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
private ConcurrentSkipListMap<String, MutableLong> values = new ConcurrentSkipListMap<>();
|
||||
private ConcurrentSkipListMap<String,MutableLong> values = new ConcurrentSkipListMap<>();
|
||||
|
||||
@Override
|
||||
public void add(String value) {
|
||||
@@ -25,7 +25,7 @@ public class SortedStringAccumulator implements Accumulator<String, ConcurrentSk
|
||||
}
|
||||
|
||||
@Override
|
||||
public ConcurrentSkipListMap<String, MutableLong> getLocalValue() {
|
||||
public ConcurrentSkipListMap<String,MutableLong> getLocalValue() {
|
||||
return values;
|
||||
}
|
||||
|
||||
@@ -35,7 +35,7 @@ public class SortedStringAccumulator implements Accumulator<String, ConcurrentSk
|
||||
}
|
||||
|
||||
@Override
|
||||
public void merge(Accumulator<String, ConcurrentSkipListMap<String, MutableLong>> other) {
|
||||
public void merge(Accumulator<String,ConcurrentSkipListMap<String,MutableLong>> other) {
|
||||
other.getLocalValue().forEach((k, v) -> {
|
||||
if (values.containsKey(k)) {
|
||||
values.get(k).add(v.longValue());
|
||||
@@ -46,7 +46,7 @@ public class SortedStringAccumulator implements Accumulator<String, ConcurrentSk
|
||||
}
|
||||
|
||||
@Override
|
||||
public Accumulator<String, ConcurrentSkipListMap<String, MutableLong>> clone() {
|
||||
public Accumulator<String,ConcurrentSkipListMap<String,MutableLong>> clone() {
|
||||
SortedStringAccumulator a = new SortedStringAccumulator();
|
||||
a.getLocalValue().putAll(this.values);
|
||||
return a;
|
||||
|
||||
@@ -22,6 +22,7 @@ import org.apache.flink.streaming.api.functions.source.RichSourceFunction;
|
||||
import org.apache.flink.streaming.api.watermark.Watermark;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import timely.api.response.MetricResponse;
|
||||
import timely.api.response.MetricResponses;
|
||||
import timely.client.websocket.ClientHandler;
|
||||
@@ -65,10 +66,9 @@ public class SubscriptionSource extends RichSourceFunction<MetricResponse> imple
|
||||
public void open(Configuration parameters) throws Exception {
|
||||
LOG.info("Opening summarization job.");
|
||||
super.open(parameters);
|
||||
client = new WebSocketSubscriptionClient(jp.getTimelyHostname(), jp.getTimelyHttpsPort(), jp.getTimelyWssPort(),
|
||||
jp.isClientAuth(), jp.isDoLogin(), jp.getTimelyUsername(), jp.getTimelyPassword(), jp.getKeyStoreFile(),
|
||||
jp.getKeyStoreType(), jp.getKeyStorePass(), jp.getTrustStoreFile(), jp.getTrustStoreType(),
|
||||
jp.getTrustStorePass(), jp.isHostVerificationEnabled(), jp.getBufferSize());
|
||||
client = new WebSocketSubscriptionClient(jp.getTimelyHostname(), jp.getTimelyHttpsPort(), jp.getTimelyWssPort(), jp.isClientAuth(), jp.isDoLogin(),
|
||||
jp.getTimelyUsername(), jp.getTimelyPassword(), jp.getKeyStoreFile(), jp.getKeyStoreType(), jp.getKeyStorePass(),
|
||||
jp.getTrustStoreFile(), jp.getTrustStoreType(), jp.getTrustStorePass(), jp.isHostVerificationEnabled(), jp.getBufferSize());
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -103,12 +103,10 @@ public class SubscriptionSource extends RichSourceFunction<MetricResponse> imple
|
||||
|
||||
@Override
|
||||
public void onMessage(final String message) {
|
||||
LOG.info("Message received on Websocket session {}, length: {}", session.getId(),
|
||||
message.length());
|
||||
LOG.info("Message received on Websocket session {}, length: {}", session.getId(), message.length());
|
||||
try {
|
||||
// Deserialize in this thread
|
||||
final MetricResponses responses = JsonSerializer.getObjectMapper().readValue(message,
|
||||
MetricResponses.class);
|
||||
final MetricResponses responses = JsonSerializer.getObjectMapper().readValue(message, MetricResponses.class);
|
||||
// Process in the other
|
||||
final Runnable r = () -> {
|
||||
responses.getResponses().forEach(response -> {
|
||||
|
||||
@@ -13,6 +13,7 @@ import org.apache.flink.streaming.api.functions.sink.SocketClientSink;
|
||||
import org.apache.flink.streaming.api.windowing.assigners.TumblingEventTimeWindows;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import timely.api.response.MetricResponse;
|
||||
|
||||
public class SummarizationJob {
|
||||
|
||||
@@ -184,8 +184,8 @@ public class SummarizationJobParameters extends GlobalJobParameters implements S
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<String, String> toMap() {
|
||||
Map<String, String> m = new HashMap<>();
|
||||
public Map<String,String> toMap() {
|
||||
Map<String,String> m = new HashMap<>();
|
||||
m.put("timelyHostname", this.timelyHostname);
|
||||
m.put("timelyHttpsPort", Integer.toString(this.timelyHttpsPort));
|
||||
m.put("timelyWssPort", Integer.toString(this.timelyWssPort));
|
||||
|
||||
@@ -6,6 +6,7 @@ import java.util.List;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
|
||||
import timely.model.Metric;
|
||||
import timely.model.Tag;
|
||||
import timely.model.parse.MetricParser;
|
||||
@@ -96,9 +97,8 @@ public class MetricHistogramTest {
|
||||
Metric metric = metricParser.parse(put);
|
||||
Assert.assertEquals("sys.cpu.user_summarized", metric.getName());
|
||||
metric.getTags().forEach(t -> {
|
||||
Assert.assertTrue(
|
||||
t.equals(t1) || t.equals(t2) || t.equals(avg) || t.equals(min) || t.equals(max) || t.equals(sum)
|
||||
|| t.equals(count) || t.equals(p50) || t.equals(p75) || t.equals(p90) || t.equals(p99));
|
||||
Assert.assertTrue(t.equals(t1) || t.equals(t2) || t.equals(avg) || t.equals(min) || t.equals(max) || t.equals(sum) || t.equals(count)
|
||||
|| t.equals(p50) || t.equals(p75) || t.equals(p90) || t.equals(p99));
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
+1
-1
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>timely</groupId>
|
||||
<artifactId>timely</artifactId>
|
||||
<version>2.0.1</version>
|
||||
<version>2.0.2</version>
|
||||
</parent>
|
||||
<artifactId>timely-balancer</artifactId>
|
||||
<dependencies>
|
||||
|
||||
@@ -16,6 +16,18 @@ import java.util.concurrent.TimeUnit;
|
||||
|
||||
import javax.net.ssl.SSLContext;
|
||||
|
||||
import org.apache.curator.RetryPolicy;
|
||||
import org.apache.curator.framework.CuratorFramework;
|
||||
import org.apache.curator.framework.CuratorFrameworkFactory;
|
||||
import org.apache.curator.retry.RetryForever;
|
||||
import org.apache.zookeeper.CreateMode;
|
||||
import org.apache.zookeeper.data.Stat;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.boot.WebApplicationType;
|
||||
import org.springframework.boot.builder.SpringApplicationBuilder;
|
||||
import org.springframework.context.ConfigurableApplicationContext;
|
||||
|
||||
import io.netty.bootstrap.Bootstrap;
|
||||
import io.netty.bootstrap.ServerBootstrap;
|
||||
import io.netty.buffer.PooledByteBufAllocator;
|
||||
@@ -56,17 +68,6 @@ import io.netty.handler.stream.ChunkedWriteHandler;
|
||||
import io.netty.handler.timeout.IdleStateHandler;
|
||||
import io.netty.util.NettyRuntime;
|
||||
import io.netty.util.internal.SystemPropertyUtil;
|
||||
import org.apache.curator.RetryPolicy;
|
||||
import org.apache.curator.framework.CuratorFramework;
|
||||
import org.apache.curator.framework.CuratorFrameworkFactory;
|
||||
import org.apache.curator.retry.RetryForever;
|
||||
import org.apache.zookeeper.CreateMode;
|
||||
import org.apache.zookeeper.data.Stat;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.boot.WebApplicationType;
|
||||
import org.springframework.boot.builder.SpringApplicationBuilder;
|
||||
import org.springframework.context.ConfigurableApplicationContext;
|
||||
import timely.auth.AuthCache;
|
||||
import timely.auth.JWTTokenHandler;
|
||||
import timely.auth.util.SslHelper;
|
||||
@@ -133,12 +134,11 @@ public class Balancer {
|
||||
protected volatile boolean shutdown = false;
|
||||
private static final int DEFAULT_EVENT_LOOP_THREADS;
|
||||
|
||||
private String[] zkPaths = new String[] { LEADER_LATCH_PATH, ASSIGNMENTS_LAST_UPDATED_PATH, ASSIGNMENTS_LOCK_PATH,
|
||||
SERVICE_DISCOVERY_PATH, NON_CACHED_METRICS, NON_CACHED_METRICS_LOCK_PATH };
|
||||
private String[] zkPaths = new String[] {LEADER_LATCH_PATH, ASSIGNMENTS_LAST_UPDATED_PATH, ASSIGNMENTS_LOCK_PATH, SERVICE_DISCOVERY_PATH,
|
||||
NON_CACHED_METRICS, NON_CACHED_METRICS_LOCK_PATH};
|
||||
|
||||
static {
|
||||
DEFAULT_EVENT_LOOP_THREADS = Math.max(1,
|
||||
SystemPropertyUtil.getInt("io.netty.eventLoopThreads", NettyRuntime.availableProcessors() * 2));
|
||||
DEFAULT_EVENT_LOOP_THREADS = Math.max(1, SystemPropertyUtil.getInt("io.netty.eventLoopThreads", NettyRuntime.availableProcessors() * 2));
|
||||
}
|
||||
|
||||
public Balancer(BalancerConfiguration balancerConf) throws Exception {
|
||||
@@ -335,8 +335,8 @@ public class Balancer {
|
||||
protected SSLContext createSSLClientContext(BalancerConfiguration config) throws Exception {
|
||||
|
||||
ClientSsl ssl = config.getSecurity().getClientSsl();
|
||||
return HttpClient.getSSLContext(ssl.getTrustStoreFile(), ssl.getTrustStoreType(), ssl.getTrustStorePassword(),
|
||||
ssl.getKeyStoreFile(), ssl.getKeyStoreType(), ssl.getKeyStorePassword());
|
||||
return HttpClient.getSSLContext(ssl.getTrustStoreFile(), ssl.getTrustStoreType(), ssl.getTrustStorePassword(), ssl.getKeyStoreFile(),
|
||||
ssl.getKeyStoreType(), ssl.getKeyStorePassword());
|
||||
}
|
||||
|
||||
private void ensureZkPaths(CuratorFramework curatorFramework, String[] paths) {
|
||||
@@ -344,8 +344,7 @@ public class Balancer {
|
||||
try {
|
||||
Stat stat = curatorFramework.checkExists().forPath(s);
|
||||
if (stat == null) {
|
||||
curatorFramework.create().creatingParentContainersIfNeeded().withMode(CreateMode.PERSISTENT)
|
||||
.forPath(s);
|
||||
curatorFramework.create().creatingParentContainersIfNeeded().withMode(CreateMode.PERSISTENT).forPath(s);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
LOG.info(e.getMessage());
|
||||
@@ -359,8 +358,7 @@ public class Balancer {
|
||||
// start curator framework
|
||||
RetryPolicy retryPolicy = new RetryForever(1000);
|
||||
int timeout = Long.valueOf(getTimeInMillis(balancerConfig.getZooKeeper().getTimeout())).intValue();
|
||||
curatorFramework = CuratorFrameworkFactory.newClient(balancerConfig.getZooKeeper().getServers(), timeout, 10000,
|
||||
retryPolicy);
|
||||
curatorFramework = CuratorFrameworkFactory.newClient(balancerConfig.getZooKeeper().getServers(), timeout, 10000, retryPolicy);
|
||||
curatorFramework.start();
|
||||
ensureZkPaths(curatorFramework, zkPaths);
|
||||
|
||||
@@ -431,8 +429,7 @@ public class Balancer {
|
||||
httpServer.group(httpBossGroup, httpWorkerGroup);
|
||||
httpServer.channel(channelClass);
|
||||
httpServer.handler(new LoggingHandler());
|
||||
HttpClientPool httpClientPool = new HttpClientPool(balancerConfig.getSecurity(), balancerConfig.getHttp(),
|
||||
clientSSLContext);
|
||||
HttpClientPool httpClientPool = new HttpClientPool(balancerConfig.getSecurity(), balancerConfig.getHttp(), clientSSLContext);
|
||||
httpServer.childHandler(setupHttpChannel(balancerConfig, sslCtx, metricResolver, httpClientPool));
|
||||
httpServer.option(ChannelOption.ALLOCATOR, PooledByteBufAllocator.DEFAULT);
|
||||
httpServer.option(ChannelOption.SO_BACKLOG, 128);
|
||||
@@ -471,9 +468,8 @@ public class Balancer {
|
||||
udpChannelHandleList.add(udpServer.bind(udpIp, udpPort).sync().channel());
|
||||
}
|
||||
shutdownHook();
|
||||
LOG.info(
|
||||
"Server started. Listening on {}:{} for TCP traffic, {}:{} for HTTP traffic, {}:{} for WebSocket traffic, and {}:{} for UDP traffic",
|
||||
tcpAddress, tcpPort, httpAddress, httpPort, wsAddress, wsPort, wsAddress, udpPort);
|
||||
LOG.info("Server started. Listening on {}:{} for TCP traffic, {}:{} for HTTP traffic, {}:{} for WebSocket traffic, and {}:{} for UDP traffic",
|
||||
tcpAddress, tcpPort, httpAddress, httpPort, wsAddress, wsPort, wsAddress, udpPort);
|
||||
}
|
||||
|
||||
public static void main(String[] args) throws Exception {
|
||||
@@ -498,8 +494,8 @@ public class Balancer {
|
||||
}
|
||||
}
|
||||
|
||||
protected ChannelHandler setupHttpChannel(BalancerConfiguration balancerConfig, SslContext sslCtx,
|
||||
MetricResolver metricResolver, HttpClientPool httpClientPool) {
|
||||
protected ChannelHandler setupHttpChannel(BalancerConfiguration balancerConfig, SslContext sslCtx, MetricResolver metricResolver,
|
||||
HttpClientPool httpClientPool) {
|
||||
|
||||
return new ChannelInitializer<SocketChannel>() {
|
||||
|
||||
@@ -513,15 +509,13 @@ public class Balancer {
|
||||
ch.pipeline().addLast("decompressor", new HttpContentDecompressor());
|
||||
ch.pipeline().addLast("aggregator", new HttpObjectAggregator(65536));
|
||||
ch.pipeline().addLast("chunker", new ChunkedWriteHandler());
|
||||
ch.pipeline().addLast("queryDecoder", new timely.netty.http.HttpRequestDecoder(
|
||||
balancerConfig.getSecurity(), balancerConfig.getHttp()));
|
||||
ch.pipeline().addLast("fileServer", new HttpStaticFileServerHandler().setIgnoreSslHandshakeErrors(
|
||||
balancerConfig.getSecurity().getServerSsl().isIgnoreSslHandshakeErrors()));
|
||||
ch.pipeline().addLast("login",
|
||||
new X509LoginRequestHandler(balancerConfig.getSecurity(), balancerConfig.getHttp()));
|
||||
ch.pipeline().addLast("queryDecoder", new timely.netty.http.HttpRequestDecoder(balancerConfig.getSecurity(), balancerConfig.getHttp()));
|
||||
ch.pipeline().addLast("fileServer", new HttpStaticFileServerHandler()
|
||||
.setIgnoreSslHandshakeErrors(balancerConfig.getSecurity().getServerSsl().isIgnoreSslHandshakeErrors()));
|
||||
ch.pipeline().addLast("login", new X509LoginRequestHandler(balancerConfig.getSecurity(), balancerConfig.getHttp()));
|
||||
ch.pipeline().addLast("httpRelay", new HttpRelayHandler(metricResolver, httpClientPool));
|
||||
ch.pipeline().addLast("error", new TimelyExceptionHandler().setIgnoreSslHandshakeErrors(
|
||||
balancerConfig.getSecurity().getServerSsl().isIgnoreSslHandshakeErrors()));
|
||||
ch.pipeline().addLast("error", new TimelyExceptionHandler()
|
||||
.setIgnoreSslHandshakeErrors(balancerConfig.getSecurity().getServerSsl().isIgnoreSslHandshakeErrors()));
|
||||
}
|
||||
};
|
||||
}
|
||||
@@ -555,8 +549,7 @@ public class Balancer {
|
||||
};
|
||||
}
|
||||
|
||||
protected ChannelHandler setupWSChannel(BalancerConfiguration balancerConfig, SslContext sslCtx,
|
||||
MetricResolver metricResolver, WsClientPool wsClientPool) {
|
||||
protected ChannelHandler setupWSChannel(BalancerConfiguration balancerConfig, SslContext sslCtx, MetricResolver metricResolver, WsClientPool wsClientPool) {
|
||||
return new ChannelInitializer<SocketChannel>() {
|
||||
|
||||
@Override
|
||||
@@ -566,10 +559,8 @@ public class Balancer {
|
||||
ch.pipeline().addLast("aggregator", new HttpObjectAggregator(65536));
|
||||
ch.pipeline().addLast("sessionExtractor", new WebSocketFullRequestHandler());
|
||||
|
||||
ch.pipeline().addLast("idle-handler",
|
||||
new IdleStateHandler(balancerConfig.getWebsocket().getTimeout(), 0, 0));
|
||||
ch.pipeline().addLast("ws-protocol",
|
||||
new WebSocketServerProtocolHandler(WS_PATH, null, true, 65536, false, true));
|
||||
ch.pipeline().addLast("idle-handler", new IdleStateHandler(balancerConfig.getWebsocket().getTimeout(), 0, 0));
|
||||
ch.pipeline().addLast("ws-protocol", new WebSocketServerProtocolHandler(WS_PATH, null, true, 65536, false, true));
|
||||
ch.pipeline().addLast("wsDecoder", new WebSocketRequestDecoder(balancerConfig.getSecurity()));
|
||||
ch.pipeline().addLast("httpRelay", new WsRelayHandler(balancerConfig, metricResolver, wsClientPool));
|
||||
ch.pipeline().addLast("error", new WSTimelyExceptionHandler());
|
||||
|
||||
@@ -6,6 +6,7 @@ import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
import org.springframework.boot.context.properties.NestedConfigurationProperty;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.validation.annotation.Validated;
|
||||
|
||||
import timely.configuration.Cache;
|
||||
|
||||
@Validated
|
||||
|
||||
@@ -5,6 +5,7 @@ import javax.validation.Valid;
|
||||
import org.apache.commons.pool2.impl.GenericKeyedObjectPoolConfig;
|
||||
import org.apache.http.impl.client.CloseableHttpClient;
|
||||
import org.springframework.boot.context.properties.NestedConfigurationProperty;
|
||||
|
||||
import timely.configuration.Http;
|
||||
|
||||
public class BalancerHttp extends Http {
|
||||
|
||||
@@ -3,6 +3,7 @@ package timely.balancer.configuration;
|
||||
import javax.validation.Valid;
|
||||
|
||||
import org.springframework.boot.context.properties.NestedConfigurationProperty;
|
||||
|
||||
import timely.configuration.ClientSsl;
|
||||
import timely.configuration.Security;
|
||||
|
||||
|
||||
@@ -4,6 +4,7 @@ import javax.validation.Valid;
|
||||
|
||||
import org.apache.commons.pool2.impl.GenericKeyedObjectPoolConfig;
|
||||
import org.springframework.boot.context.properties.NestedConfigurationProperty;
|
||||
|
||||
import timely.client.tcp.TcpClient;
|
||||
import timely.client.udp.UdpClient;
|
||||
import timely.configuration.Server;
|
||||
|
||||
@@ -4,6 +4,7 @@ import javax.validation.Valid;
|
||||
|
||||
import org.apache.commons.pool2.impl.GenericKeyedObjectPoolConfig;
|
||||
import org.springframework.boot.context.properties.NestedConfigurationProperty;
|
||||
|
||||
import timely.client.websocket.subscription.WebSocketSubscriptionClient;
|
||||
import timely.configuration.Websocket;
|
||||
|
||||
|
||||
@@ -2,6 +2,5 @@ package timely.balancer.configuration;
|
||||
|
||||
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||
|
||||
@SpringBootApplication(scanBasePackageClasses = { SpringBootstrap.class })
|
||||
public class SpringBootstrap {
|
||||
}
|
||||
@SpringBootApplication(scanBasePackageClasses = {SpringBootstrap.class})
|
||||
public class SpringBootstrap {}
|
||||
@@ -5,6 +5,7 @@ import org.apache.commons.lang3.builder.HashCodeBuilder;
|
||||
import org.apache.commons.lang3.builder.ToStringBuilder;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import timely.balancer.ArrivalRate;
|
||||
import timely.balancer.configuration.BalancerConfiguration;
|
||||
|
||||
@@ -27,8 +28,7 @@ public class TimelyBalancedHost {
|
||||
|
||||
}
|
||||
|
||||
private TimelyBalancedHost(String host, int tcpPort, int httpPort, int wsPort, int udpPort,
|
||||
ArrivalRate arrivalRate) {
|
||||
private TimelyBalancedHost(String host, int tcpPort, int httpPort, int wsPort, int udpPort, ArrivalRate arrivalRate) {
|
||||
this.host = host;
|
||||
this.tcpPort = tcpPort;
|
||||
this.httpPort = httpPort;
|
||||
@@ -37,8 +37,7 @@ public class TimelyBalancedHost {
|
||||
this.arrivalRate = arrivalRate;
|
||||
}
|
||||
|
||||
static public TimelyBalancedHost of(String host, int tcpPort, int httpPort, int wsPort, int udpPort,
|
||||
ArrivalRate arrivalRate) {
|
||||
static public TimelyBalancedHost of(String host, int tcpPort, int httpPort, int wsPort, int udpPort, ArrivalRate arrivalRate) {
|
||||
return new TimelyBalancedHost(host, tcpPort, httpPort, wsPort, udpPort, arrivalRate);
|
||||
}
|
||||
|
||||
@@ -56,8 +55,8 @@ public class TimelyBalancedHost {
|
||||
int p = tcpPort;
|
||||
synchronized (this) {
|
||||
if (LOG.isTraceEnabled()) {
|
||||
LOG.trace("success reported host:{} port:{} isUp:{} successes:{} serverSuccessesBeforeUp:{}", h, p,
|
||||
isUp, (successes + 1), serverSuccessesBeforeUp);
|
||||
LOG.trace("success reported host:{} port:{} isUp:{} successes:{} serverSuccessesBeforeUp:{}", h, p, isUp, (successes + 1),
|
||||
serverSuccessesBeforeUp);
|
||||
}
|
||||
if (!isUp) {
|
||||
if (++successes >= serverSuccessesBeforeUp) {
|
||||
@@ -75,8 +74,8 @@ public class TimelyBalancedHost {
|
||||
int p = tcpPort;
|
||||
synchronized (this) {
|
||||
if (LOG.isTraceEnabled()) {
|
||||
LOG.trace("failure reported host:{} port:{} isUp:{} failures:{} serverFailuresBeforeUp:{}", h, p, isUp,
|
||||
(failures + 1), serverFailuresBeforeDown);
|
||||
LOG.trace("failure reported host:{} port:{} isUp:{} failures:{} serverFailuresBeforeUp:{}", h, p, isUp, (failures + 1),
|
||||
serverFailuresBeforeDown);
|
||||
}
|
||||
if (isUp) {
|
||||
if (++failures >= serverFailuresBeforeDown) {
|
||||
|
||||
@@ -6,10 +6,11 @@ import org.apache.commons.pool2.KeyedPooledObjectFactory;
|
||||
import org.apache.commons.pool2.PooledObject;
|
||||
import org.apache.commons.pool2.impl.DefaultPooledObject;
|
||||
import org.apache.http.impl.client.CloseableHttpClient;
|
||||
|
||||
import timely.balancer.configuration.BalancerSecurity;
|
||||
import timely.balancer.connection.TimelyBalancedHost;
|
||||
|
||||
public class HttpClientFactory implements KeyedPooledObjectFactory<TimelyBalancedHost, CloseableHttpClient> {
|
||||
public class HttpClientFactory implements KeyedPooledObjectFactory<TimelyBalancedHost,CloseableHttpClient> {
|
||||
|
||||
private final SSLContext sslContext;
|
||||
private final BalancerSecurity security;
|
||||
@@ -22,8 +23,8 @@ public class HttpClientFactory implements KeyedPooledObjectFactory<TimelyBalance
|
||||
@Override
|
||||
public PooledObject<CloseableHttpClient> makeObject(TimelyBalancedHost k) throws Exception {
|
||||
// disable cookie management because we are sharing connections
|
||||
return new DefaultPooledObject<>(timely.client.http.HttpClient.get(this.sslContext, null,
|
||||
security.getClientSsl().isHostVerificationEnabled(), security.getClientSsl().isUseClientCert()));
|
||||
return new DefaultPooledObject<>(timely.client.http.HttpClient.get(this.sslContext, null, security.getClientSsl().isHostVerificationEnabled(),
|
||||
security.getClientSsl().isUseClientCert()));
|
||||
}
|
||||
|
||||
@Override
|
||||
|
||||
@@ -4,11 +4,12 @@ import javax.net.ssl.SSLContext;
|
||||
|
||||
import org.apache.commons.pool2.impl.GenericKeyedObjectPool;
|
||||
import org.apache.http.impl.client.CloseableHttpClient;
|
||||
|
||||
import timely.balancer.configuration.BalancerHttp;
|
||||
import timely.balancer.configuration.BalancerSecurity;
|
||||
import timely.balancer.connection.TimelyBalancedHost;
|
||||
|
||||
public class HttpClientPool extends GenericKeyedObjectPool<TimelyBalancedHost, CloseableHttpClient> {
|
||||
public class HttpClientPool extends GenericKeyedObjectPool<TimelyBalancedHost,CloseableHttpClient> {
|
||||
|
||||
public HttpClientPool(BalancerSecurity balancerSecurity, BalancerHttp http, SSLContext sslContext) {
|
||||
super(new HttpClientFactory(balancerSecurity, sslContext), http.getHttpClientPool());
|
||||
|
||||
@@ -5,10 +5,11 @@ import java.io.IOException;
|
||||
import org.apache.commons.pool2.KeyedPooledObjectFactory;
|
||||
import org.apache.commons.pool2.PooledObject;
|
||||
import org.apache.commons.pool2.impl.DefaultPooledObject;
|
||||
|
||||
import timely.balancer.connection.TimelyBalancedHost;
|
||||
import timely.client.tcp.TcpClient;
|
||||
|
||||
public class TcpClientFactory implements KeyedPooledObjectFactory<TimelyBalancedHost, TcpClient> {
|
||||
public class TcpClientFactory implements KeyedPooledObjectFactory<TimelyBalancedHost,TcpClient> {
|
||||
|
||||
private int bufferSize;
|
||||
|
||||
|
||||
@@ -1,13 +1,14 @@
|
||||
package timely.balancer.connection.tcp;
|
||||
|
||||
import org.apache.commons.pool2.impl.GenericKeyedObjectPool;
|
||||
|
||||
import timely.balancer.configuration.BalancerConfiguration;
|
||||
import timely.balancer.connection.TimelyBalancedHost;
|
||||
import timely.client.tcp.TcpClient;
|
||||
|
||||
public class TcpClientPool extends GenericKeyedObjectPool<TimelyBalancedHost, TcpClient> {
|
||||
public class TcpClientPool extends GenericKeyedObjectPool<TimelyBalancedHost,TcpClient> {
|
||||
|
||||
public TcpClientPool(BalancerConfiguration config) {
|
||||
super(new TcpClientFactory(config.getServer().getTcpBufferSize()), config.getServer().getTcpClientPool());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -5,13 +5,13 @@ import java.io.IOException;
|
||||
import org.apache.commons.pool2.KeyedPooledObjectFactory;
|
||||
import org.apache.commons.pool2.PooledObject;
|
||||
import org.apache.commons.pool2.impl.DefaultPooledObject;
|
||||
|
||||
import timely.balancer.connection.TimelyBalancedHost;
|
||||
import timely.client.udp.UdpClient;
|
||||
|
||||
public class UdpClientFactory implements KeyedPooledObjectFactory<TimelyBalancedHost, UdpClient> {
|
||||
public class UdpClientFactory implements KeyedPooledObjectFactory<TimelyBalancedHost,UdpClient> {
|
||||
|
||||
public UdpClientFactory() {
|
||||
}
|
||||
public UdpClientFactory() {}
|
||||
|
||||
@Override
|
||||
public PooledObject<UdpClient> makeObject(TimelyBalancedHost k) throws Exception {
|
||||
|
||||
@@ -1,11 +1,12 @@
|
||||
package timely.balancer.connection.udp;
|
||||
|
||||
import org.apache.commons.pool2.impl.GenericKeyedObjectPool;
|
||||
|
||||
import timely.balancer.configuration.BalancerConfiguration;
|
||||
import timely.balancer.connection.TimelyBalancedHost;
|
||||
import timely.client.udp.UdpClient;
|
||||
|
||||
public class UdpClientPool extends GenericKeyedObjectPool<TimelyBalancedHost, UdpClient> {
|
||||
public class UdpClientPool extends GenericKeyedObjectPool<TimelyBalancedHost,UdpClient> {
|
||||
|
||||
public UdpClientPool(BalancerConfiguration config) {
|
||||
super(new UdpClientFactory(), config.getServer().getUdpClientPool());
|
||||
|
||||
@@ -5,11 +5,12 @@ import javax.net.ssl.SSLContext;
|
||||
import org.apache.commons.pool2.KeyedPooledObjectFactory;
|
||||
import org.apache.commons.pool2.PooledObject;
|
||||
import org.apache.commons.pool2.impl.DefaultPooledObject;
|
||||
|
||||
import timely.balancer.configuration.BalancerConfiguration;
|
||||
import timely.balancer.connection.TimelyBalancedHost;
|
||||
import timely.client.websocket.subscription.WebSocketSubscriptionClient;
|
||||
|
||||
public class WsClientFactory implements KeyedPooledObjectFactory<TimelyBalancedHost, WebSocketSubscriptionClient> {
|
||||
public class WsClientFactory implements KeyedPooledObjectFactory<TimelyBalancedHost,WebSocketSubscriptionClient> {
|
||||
|
||||
private final SSLContext sslContext;
|
||||
private BalancerConfiguration balancerConfig;
|
||||
@@ -23,10 +24,9 @@ public class WsClientFactory implements KeyedPooledObjectFactory<TimelyBalancedH
|
||||
public PooledObject<WebSocketSubscriptionClient> makeObject(TimelyBalancedHost k) throws Exception {
|
||||
|
||||
int bufferSize = balancerConfig.getWebsocket().getIncomingBufferSize();
|
||||
WebSocketSubscriptionClient client = new WebSocketSubscriptionClient(sslContext, k.getHost(), k.getHttpPort(),
|
||||
k.getWsPort(), balancerConfig.getSecurity().getClientSsl().isUseClientCert(),
|
||||
balancerConfig.isLoginRequired(), "", "",
|
||||
balancerConfig.getSecurity().getClientSsl().isHostVerificationEnabled(), bufferSize);
|
||||
WebSocketSubscriptionClient client = new WebSocketSubscriptionClient(sslContext, k.getHost(), k.getHttpPort(), k.getWsPort(),
|
||||
balancerConfig.getSecurity().getClientSsl().isUseClientCert(), balancerConfig.isLoginRequired(), "", "",
|
||||
balancerConfig.getSecurity().getClientSsl().isHostVerificationEnabled(), bufferSize);
|
||||
return new DefaultPooledObject<>(client);
|
||||
}
|
||||
|
||||
|
||||
@@ -3,11 +3,12 @@ package timely.balancer.connection.ws;
|
||||
import javax.net.ssl.SSLContext;
|
||||
|
||||
import org.apache.commons.pool2.impl.GenericKeyedObjectPool;
|
||||
|
||||
import timely.balancer.configuration.BalancerConfiguration;
|
||||
import timely.balancer.connection.TimelyBalancedHost;
|
||||
import timely.client.websocket.subscription.WebSocketSubscriptionClient;
|
||||
|
||||
public class WsClientPool extends GenericKeyedObjectPool<TimelyBalancedHost, WebSocketSubscriptionClient> {
|
||||
public class WsClientPool extends GenericKeyedObjectPool<TimelyBalancedHost,WebSocketSubscriptionClient> {
|
||||
|
||||
public WsClientPool(BalancerConfiguration balancerConfig, SSLContext sslContext) {
|
||||
super(new WsClientFactory(balancerConfig, sslContext), balancerConfig.getWebsocket().getWsClientPool());
|
||||
|
||||
@@ -8,6 +8,7 @@ import java.util.concurrent.TimeUnit;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import timely.balancer.configuration.BalancerConfiguration;
|
||||
import timely.balancer.connection.TimelyBalancedHost;
|
||||
import timely.balancer.connection.tcp.TcpClientPool;
|
||||
|
||||
@@ -11,17 +11,6 @@ import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import com.google.common.collect.Multimap;
|
||||
import io.netty.buffer.Unpooled;
|
||||
import io.netty.channel.ChannelHandlerContext;
|
||||
import io.netty.channel.SimpleChannelInboundHandler;
|
||||
import io.netty.handler.codec.http.DefaultFullHttpResponse;
|
||||
import io.netty.handler.codec.http.FullHttpRequest;
|
||||
import io.netty.handler.codec.http.FullHttpResponse;
|
||||
import io.netty.handler.codec.http.HttpMethod;
|
||||
import io.netty.handler.codec.http.HttpResponseStatus;
|
||||
import io.netty.handler.codec.http.HttpVersion;
|
||||
import io.netty.util.ReferenceCountUtil;
|
||||
import org.apache.http.Header;
|
||||
import org.apache.http.client.methods.CloseableHttpResponse;
|
||||
import org.apache.http.client.methods.HttpGet;
|
||||
@@ -33,6 +22,19 @@ import org.apache.http.message.BasicHeader;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.util.StreamUtils;
|
||||
|
||||
import com.google.common.collect.Multimap;
|
||||
|
||||
import io.netty.buffer.Unpooled;
|
||||
import io.netty.channel.ChannelHandlerContext;
|
||||
import io.netty.channel.SimpleChannelInboundHandler;
|
||||
import io.netty.handler.codec.http.DefaultFullHttpResponse;
|
||||
import io.netty.handler.codec.http.FullHttpRequest;
|
||||
import io.netty.handler.codec.http.FullHttpResponse;
|
||||
import io.netty.handler.codec.http.HttpMethod;
|
||||
import io.netty.handler.codec.http.HttpResponseStatus;
|
||||
import io.netty.handler.codec.http.HttpVersion;
|
||||
import io.netty.util.ReferenceCountUtil;
|
||||
import timely.api.request.AuthenticatedRequest;
|
||||
import timely.api.request.MetricRequest;
|
||||
import timely.api.request.timeseries.HttpRequest;
|
||||
@@ -88,7 +90,7 @@ public class HttpRelayHandler extends SimpleChannelInboundHandler<HttpRequest> i
|
||||
}
|
||||
client = httpClientPool.borrowObject(k);
|
||||
|
||||
Multimap<String, String> headers = HttpHeaderUtils.toMultimap(request.headers());
|
||||
Multimap<String,String> headers = HttpHeaderUtils.toMultimap(request.headers());
|
||||
if (msg instanceof AuthenticatedRequest) {
|
||||
AuthenticatedRequest authenticatedRequest = (AuthenticatedRequest) msg;
|
||||
TimelyAuthenticationToken token = authenticatedRequest.getToken();
|
||||
@@ -100,7 +102,7 @@ public class HttpRelayHandler extends SimpleChannelInboundHandler<HttpRequest> i
|
||||
}
|
||||
|
||||
List<Header> relayedHeaderList = new ArrayList<>();
|
||||
for (Map.Entry<String, String> h : headers.entries()) {
|
||||
for (Map.Entry<String,String> h : headers.entries()) {
|
||||
if (!h.getKey().equals(CONTENT_LENGTH.toString())) {
|
||||
relayedHeaderList.add(new BasicHeader(h.getKey(), h.getValue()));
|
||||
}
|
||||
@@ -138,8 +140,7 @@ public class HttpRelayHandler extends SimpleChannelInboundHandler<HttpRequest> i
|
||||
LOG.error(message, e);
|
||||
}
|
||||
}
|
||||
this.sendHttpError(ctx, new TimelyException(HttpResponseStatus.INTERNAL_SERVER_ERROR.code(),
|
||||
e.getMessage(), e.getLocalizedMessage(), e));
|
||||
this.sendHttpError(ctx, new TimelyException(HttpResponseStatus.INTERNAL_SERVER_ERROR.code(), e.getMessage(), e.getLocalizedMessage(), e));
|
||||
return;
|
||||
} finally {
|
||||
if (request != null) {
|
||||
@@ -148,15 +149,13 @@ public class HttpRelayHandler extends SimpleChannelInboundHandler<HttpRequest> i
|
||||
}
|
||||
FullHttpResponse response = null;
|
||||
if (relayedResponse == null) {
|
||||
this.sendHttpError(ctx, new TimelyException(HttpResponseStatus.METHOD_NOT_ALLOWED.code(),
|
||||
"Method not allowed", "Method not allowed"));
|
||||
this.sendHttpError(ctx, new TimelyException(HttpResponseStatus.METHOD_NOT_ALLOWED.code(), "Method not allowed", "Method not allowed"));
|
||||
} else {
|
||||
ByteArrayOutputStream baos = new MyByteArrayOutputStream();
|
||||
StreamUtils.copy(relayedResponse.getEntity().getContent(), baos);
|
||||
|
||||
response = new DefaultFullHttpResponse(HttpVersion.HTTP_1_1,
|
||||
HttpResponseStatus.valueOf(relayedResponse.getStatusLine().getStatusCode()),
|
||||
Unpooled.copiedBuffer(baos.toByteArray()));
|
||||
response = new DefaultFullHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.valueOf(relayedResponse.getStatusLine().getStatusCode()),
|
||||
Unpooled.copiedBuffer(baos.toByteArray()));
|
||||
for (Header h : relayedResponse.getAllHeaders()) {
|
||||
response.headers().add(h.getName(), h.getValue());
|
||||
}
|
||||
@@ -177,8 +176,7 @@ public class HttpRelayHandler extends SimpleChannelInboundHandler<HttpRequest> i
|
||||
|
||||
static public class MyByteArrayOutputStream extends ByteArrayOutputStream {
|
||||
|
||||
public MyByteArrayOutputStream() {
|
||||
}
|
||||
public MyByteArrayOutputStream() {}
|
||||
|
||||
public MyByteArrayOutputStream(int size) {
|
||||
super(size);
|
||||
|
||||
@@ -4,13 +4,14 @@ import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.commons.lang3.tuple.Pair;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import io.netty.buffer.Unpooled;
|
||||
import io.netty.channel.ChannelFuture;
|
||||
import io.netty.channel.ChannelHandlerContext;
|
||||
import io.netty.channel.SimpleChannelInboundHandler;
|
||||
import org.apache.commons.lang3.tuple.Pair;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import timely.api.request.MetricRequest;
|
||||
import timely.api.request.TcpRequest;
|
||||
import timely.balancer.connection.TimelyBalancedHost;
|
||||
@@ -44,7 +45,7 @@ public class TcpRelayHandler extends SimpleChannelInboundHandler<TcpRequest> {
|
||||
LOG.trace("Received {}", msg);
|
||||
try {
|
||||
String line;
|
||||
Pair<TimelyBalancedHost, TcpClient> keyClientPair = null;
|
||||
Pair<TimelyBalancedHost,TcpClient> keyClientPair = null;
|
||||
try {
|
||||
if (msg instanceof MetricRequest) {
|
||||
String metricName = ((MetricRequest) msg).getMetric().getName();
|
||||
@@ -64,22 +65,20 @@ public class TcpRelayHandler extends SimpleChannelInboundHandler<TcpRequest> {
|
||||
|
||||
} catch (IOException e) {
|
||||
LOG.error(LOG_ERR_MSG, msg, e);
|
||||
ChannelFuture cf = ctx.writeAndFlush(
|
||||
Unpooled.copiedBuffer((ERR_MSG + e.getMessage() + "\n").getBytes(StandardCharsets.UTF_8)));
|
||||
ChannelFuture cf = ctx.writeAndFlush(Unpooled.copiedBuffer((ERR_MSG + e.getMessage() + "\n").getBytes(StandardCharsets.UTF_8)));
|
||||
if (!cf.isSuccess()) {
|
||||
LOG.error(Constants.ERR_WRITING_RESPONSE, cf.cause());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private Pair<TimelyBalancedHost, TcpClient> getClient(String metric, boolean metricRequest) {
|
||||
private Pair<TimelyBalancedHost,TcpClient> getClient(String metric, boolean metricRequest) {
|
||||
TcpClient client = null;
|
||||
TimelyBalancedHost k = null;
|
||||
int failures = 0;
|
||||
while (client == null) {
|
||||
try {
|
||||
k = (metricRequest == true) ? metricResolver.getHostPortKeyIngest(metric)
|
||||
: metricResolver.getHostPortKey(metric);
|
||||
k = (metricRequest == true) ? metricResolver.getHostPortKeyIngest(metric) : metricResolver.getHostPortKey(metric);
|
||||
client = tcpClientPool().borrowObject(k);
|
||||
} catch (Exception e1) {
|
||||
failures++;
|
||||
|
||||
@@ -2,13 +2,14 @@ package timely.balancer.netty.udp;
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
|
||||
import org.apache.commons.lang3.tuple.Pair;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import io.netty.buffer.Unpooled;
|
||||
import io.netty.channel.ChannelFuture;
|
||||
import io.netty.channel.ChannelHandlerContext;
|
||||
import io.netty.channel.SimpleChannelInboundHandler;
|
||||
import org.apache.commons.lang3.tuple.Pair;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import timely.api.request.MetricRequest;
|
||||
import timely.api.request.UdpRequest;
|
||||
import timely.balancer.connection.TimelyBalancedHost;
|
||||
@@ -37,7 +38,7 @@ public class UdpRelayHandler extends SimpleChannelInboundHandler<UdpRequest> {
|
||||
try {
|
||||
MetricRequest metricRequest = (MetricRequest) msg;
|
||||
String metricName = metricRequest.getMetric().getName();
|
||||
Pair<TimelyBalancedHost, UdpClient> keyClientPair = null;
|
||||
Pair<TimelyBalancedHost,UdpClient> keyClientPair = null;
|
||||
try {
|
||||
keyClientPair = getClient(metricName, true);
|
||||
keyClientPair.getRight().write(metricRequest.getLine() + "\n");
|
||||
@@ -50,22 +51,20 @@ public class UdpRelayHandler extends SimpleChannelInboundHandler<UdpRequest> {
|
||||
|
||||
} catch (Exception e) {
|
||||
LOG.error(LOG_ERR_MSG, msg, e);
|
||||
ChannelFuture cf = ctx.writeAndFlush(
|
||||
Unpooled.copiedBuffer((ERR_MSG + e.getMessage() + "\n").getBytes(StandardCharsets.UTF_8)));
|
||||
ChannelFuture cf = ctx.writeAndFlush(Unpooled.copiedBuffer((ERR_MSG + e.getMessage() + "\n").getBytes(StandardCharsets.UTF_8)));
|
||||
if (!cf.isSuccess()) {
|
||||
LOG.error(Constants.ERR_WRITING_RESPONSE, cf.cause());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private Pair<TimelyBalancedHost, UdpClient> getClient(String metric, boolean metricRequest) {
|
||||
private Pair<TimelyBalancedHost,UdpClient> getClient(String metric, boolean metricRequest) {
|
||||
UdpClient client = null;
|
||||
TimelyBalancedHost k = null;
|
||||
int failures = 0;
|
||||
while (client == null) {
|
||||
try {
|
||||
k = (metricRequest == true) ? metricResolver.getHostPortKeyIngest(metric)
|
||||
: metricResolver.getHostPortKey(metric);
|
||||
k = (metricRequest == true) ? metricResolver.getHostPortKeyIngest(metric) : metricResolver.getHostPortKey(metric);
|
||||
client = udpClientPool.borrowObject(k);
|
||||
} catch (Exception e1) {
|
||||
failures++;
|
||||
|
||||
@@ -9,15 +9,17 @@ import javax.websocket.CloseReason;
|
||||
import javax.websocket.EndpointConfig;
|
||||
import javax.websocket.Session;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import com.google.common.collect.HashMultimap;
|
||||
import com.google.common.collect.Multimap;
|
||||
|
||||
import io.netty.channel.ChannelHandlerContext;
|
||||
import io.netty.handler.codec.http.HttpResponseStatus;
|
||||
import io.netty.handler.codec.http.websocketx.PingWebSocketFrame;
|
||||
import io.netty.handler.codec.http.websocketx.TextWebSocketFrame;
|
||||
import io.netty.util.concurrent.ScheduledFuture;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import timely.api.response.TimelyException;
|
||||
import timely.auth.util.ProxiedEntityUtils;
|
||||
import timely.client.websocket.ClientHandler;
|
||||
@@ -40,16 +42,16 @@ public class WsClientHandler extends ClientHandler {
|
||||
}
|
||||
|
||||
@Override
|
||||
public void beforeRequest(Map<String, List<String>> headers) {
|
||||
public void beforeRequest(Map<String,List<String>> headers) {
|
||||
if (token.getClientCert() != null) {
|
||||
Multimap<String, String> proxyRequestHeaders = HashMultimap.create();
|
||||
Multimap<String,String> proxyRequestHeaders = HashMultimap.create();
|
||||
ProxiedEntityUtils.addProxyHeaders(proxyRequestHeaders, token.getClientCert());
|
||||
for (String s : proxyRequestHeaders.keySet()) {
|
||||
headers.put(s, new ArrayList<>(proxyRequestHeaders.get(s)));
|
||||
}
|
||||
}
|
||||
|
||||
Multimap<String, String> originalRequestHeaders = token.getHttpHeaders();
|
||||
Multimap<String,String> originalRequestHeaders = token.getHttpHeaders();
|
||||
for (String s : originalRequestHeaders.keySet()) {
|
||||
if (!headers.containsKey(s)) {
|
||||
// add pre-existing values if key does not exist in proxyRequest
|
||||
@@ -71,8 +73,7 @@ public class WsClientHandler extends ClientHandler {
|
||||
super.onClose(session, reason);
|
||||
if (!reason.getCloseCode().equals(CloseReason.CloseCodes.NORMAL_CLOSURE)) {
|
||||
LOG.error("Abnormal close: " + reason.getReasonPhrase());
|
||||
Exception e = new TimelyException(HttpResponseStatus.INTERNAL_SERVER_ERROR.code(), reason.getReasonPhrase(),
|
||||
"");
|
||||
Exception e = new TimelyException(HttpResponseStatus.INTERNAL_SERVER_ERROR.code(), reason.getReasonPhrase(), "");
|
||||
WsRelayHandler.sendErrorResponse(ctx, HttpResponseStatus.INTERNAL_SERVER_ERROR, e);
|
||||
}
|
||||
if (!ping.isCancelled()) {
|
||||
|
||||
@@ -4,6 +4,7 @@ import java.io.IOException;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import timely.balancer.connection.TimelyBalancedHost;
|
||||
import timely.balancer.connection.ws.WsClientPool;
|
||||
import timely.client.websocket.subscription.WebSocketSubscriptionClient;
|
||||
|
||||
@@ -5,11 +5,15 @@ import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.fasterxml.jackson.databind.SerializationFeature;
|
||||
import com.fasterxml.jackson.datatype.jdk8.Jdk8Module;
|
||||
import com.google.common.collect.Multimap;
|
||||
|
||||
import io.netty.channel.ChannelFuture;
|
||||
import io.netty.channel.ChannelFutureListener;
|
||||
import io.netty.channel.ChannelHandlerContext;
|
||||
@@ -17,8 +21,6 @@ import io.netty.channel.SimpleChannelInboundHandler;
|
||||
import io.netty.handler.codec.http.HttpResponseStatus;
|
||||
import io.netty.handler.codec.http.websocketx.CloseWebSocketFrame;
|
||||
import io.netty.handler.codec.http.websocketx.TextWebSocketFrame;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import timely.api.request.subscription.AddSubscription;
|
||||
import timely.api.request.subscription.CloseSubscription;
|
||||
import timely.api.request.subscription.CreateSubscription;
|
||||
@@ -41,8 +43,8 @@ public class WsRelayHandler extends SimpleChannelInboundHandler<SubscriptionRequ
|
||||
private final WsClientPool wsClientPool;
|
||||
private MetricResolver metricResolver;
|
||||
static private ObjectMapper mapper;
|
||||
Map<String, Map<String, WsClientHolder>> wsClients = new ConcurrentHashMap<>();
|
||||
Map<String, ChannelHandlerContext> wsSubscriptions = new ConcurrentHashMap<>();
|
||||
Map<String,Map<String,WsClientHolder>> wsClients = new ConcurrentHashMap<>();
|
||||
Map<String,ChannelHandlerContext> wsSubscriptions = new ConcurrentHashMap<>();
|
||||
|
||||
static {
|
||||
mapper = new ObjectMapper();
|
||||
@@ -52,8 +54,7 @@ public class WsRelayHandler extends SimpleChannelInboundHandler<SubscriptionRequ
|
||||
mapper.configure(SerializationFeature.WRAP_ROOT_VALUE, true);
|
||||
}
|
||||
|
||||
public WsRelayHandler(BalancerConfiguration balancerConfig, MetricResolver metricResolver,
|
||||
WsClientPool wsClientPool) {
|
||||
public WsRelayHandler(BalancerConfiguration balancerConfig, MetricResolver metricResolver, WsClientPool wsClientPool) {
|
||||
this.balancerConfig = balancerConfig;
|
||||
this.metricResolver = metricResolver;
|
||||
this.wsClientPool = wsClientPool;
|
||||
@@ -67,7 +68,7 @@ public class WsRelayHandler extends SimpleChannelInboundHandler<SubscriptionRequ
|
||||
try {
|
||||
String metric = null;
|
||||
|
||||
Multimap<String, String> headers = msg.getRequestHeaders();
|
||||
Multimap<String,String> headers = msg.getRequestHeaders();
|
||||
ProxiedEntityUtils.addProxyHeaders(headers, msg.getToken().getClientCert());
|
||||
|
||||
if (msg instanceof CreateSubscription) {
|
||||
@@ -83,14 +84,13 @@ public class WsRelayHandler extends SimpleChannelInboundHandler<SubscriptionRequ
|
||||
|
||||
@Override
|
||||
public void operationComplete(ChannelFuture future) throws Exception {
|
||||
Map<String, WsClientHolder> metricToClientMap = wsClients.get(currentSubscriptionId);
|
||||
Map<String,WsClientHolder> metricToClientMap = wsClients.get(currentSubscriptionId);
|
||||
|
||||
if (metricToClientMap != null) {
|
||||
synchronized (metricToClientMap) {
|
||||
// close all clients for this subscriptionId
|
||||
LOG.debug("Channel closed, closing subscriptions for subscriptionId:{}",
|
||||
currentSubscriptionId);
|
||||
for (Map.Entry<String, WsClientHolder> entry : metricToClientMap.entrySet()) {
|
||||
LOG.debug("Channel closed, closing subscriptions for subscriptionId:{}", currentSubscriptionId);
|
||||
for (Map.Entry<String,WsClientHolder> entry : metricToClientMap.entrySet()) {
|
||||
entry.getValue().close(wsClientPool);
|
||||
}
|
||||
wsClients.remove(currentSubscriptionId);
|
||||
@@ -108,12 +108,10 @@ public class WsRelayHandler extends SimpleChannelInboundHandler<SubscriptionRequ
|
||||
if (origContext == null) {
|
||||
// send error because user never sent a CreateSubscription
|
||||
// and we have nowhere to send the results
|
||||
LOG.info("ChannelHandlerContext not found for subscriptionId:{} - createSubscription not called?",
|
||||
subscriptionId);
|
||||
sendErrorResponse(ctx, HttpResponseStatus.BAD_REQUEST,
|
||||
new IllegalArgumentException("Must call create first"));
|
||||
LOG.info("ChannelHandlerContext not found for subscriptionId:{} - createSubscription not called?", subscriptionId);
|
||||
sendErrorResponse(ctx, HttpResponseStatus.BAD_REQUEST, new IllegalArgumentException("Must call create first"));
|
||||
}
|
||||
Map<String, WsClientHolder> metricToClientMap = wsClients.get(subscriptionId);
|
||||
Map<String,WsClientHolder> metricToClientMap = wsClients.get(subscriptionId);
|
||||
if (metricToClientMap == null) {
|
||||
metricToClientMap = new HashMap<>();
|
||||
wsClients.put(subscriptionId, metricToClientMap);
|
||||
@@ -124,14 +122,13 @@ public class WsRelayHandler extends SimpleChannelInboundHandler<SubscriptionRequ
|
||||
if (hostClientHolder == null) {
|
||||
k = metricResolver.getHostPortKey(metric);
|
||||
client = wsClientPool.borrowObject(k);
|
||||
client.open(new WsClientHandler(origContext, add.getToken(),
|
||||
(balancerConfig.getWebsocket().getTimeout() / 2)));
|
||||
client.open(new WsClientHandler(origContext, add.getToken(), (balancerConfig.getWebsocket().getTimeout() / 2)));
|
||||
metricToClientMap.put(metric, new WsClientHolder(k, client));
|
||||
} else {
|
||||
client = hostClientHolder.getClient();
|
||||
}
|
||||
}
|
||||
Map<String, String> tags = null;
|
||||
Map<String,String> tags = null;
|
||||
|
||||
Long startTime = 0L;
|
||||
Long endTime = 0L;
|
||||
@@ -155,7 +152,7 @@ public class WsRelayHandler extends SimpleChannelInboundHandler<SubscriptionRequ
|
||||
RemoveSubscription remove = (RemoveSubscription) msg;
|
||||
metric = remove.getMetric();
|
||||
subscriptionId = remove.getSubscriptionId();
|
||||
Map<String, WsClientHolder> metricToClientMap = wsClients.get(subscriptionId);
|
||||
Map<String,WsClientHolder> metricToClientMap = wsClients.get(subscriptionId);
|
||||
if (metricToClientMap != null) {
|
||||
synchronized (metricToClientMap) {
|
||||
WsClientHolder hostClientHolder = metricToClientMap.get(metric);
|
||||
@@ -169,11 +166,11 @@ public class WsRelayHandler extends SimpleChannelInboundHandler<SubscriptionRequ
|
||||
} else if (msg instanceof CloseSubscription) {
|
||||
CloseSubscription close = (CloseSubscription) msg;
|
||||
subscriptionId = close.getSubscriptionId();
|
||||
Map<String, WsClientHolder> metricToClientMap = wsClients.get(subscriptionId);
|
||||
Map<String,WsClientHolder> metricToClientMap = wsClients.get(subscriptionId);
|
||||
if (metricToClientMap != null) {
|
||||
synchronized (metricToClientMap) {
|
||||
// close all clients for this subscriptionId
|
||||
for (Map.Entry<String, WsClientHolder> entry : metricToClientMap.entrySet()) {
|
||||
for (Map.Entry<String,WsClientHolder> entry : metricToClientMap.entrySet()) {
|
||||
entry.getValue().close(wsClientPool);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -29,8 +29,6 @@ import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
import java.util.concurrent.locks.ReentrantReadWriteLock;
|
||||
|
||||
import com.csvreader.CsvReader;
|
||||
import com.csvreader.CsvWriter;
|
||||
import org.apache.commons.lang3.SerializationUtils;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.curator.framework.CuratorFramework;
|
||||
@@ -57,6 +55,10 @@ import org.apache.hadoop.fs.Path;
|
||||
import org.apache.zookeeper.CreateMode;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import com.csvreader.CsvReader;
|
||||
import com.csvreader.CsvWriter;
|
||||
|
||||
import timely.ServerDetails;
|
||||
import timely.balancer.ArrivalRate;
|
||||
import timely.balancer.configuration.BalancerConfiguration;
|
||||
@@ -66,8 +68,8 @@ import timely.balancer.healthcheck.HealthChecker;
|
||||
public class BalancedMetricResolver implements MetricResolver {
|
||||
|
||||
private static final Logger LOG = LoggerFactory.getLogger(BalancedMetricResolver.class);
|
||||
private Map<String, TimelyBalancedHost> metricToHostMap = new TreeMap<>();
|
||||
private Map<String, ArrivalRate> metricMap = new HashMap<>();
|
||||
private Map<String,TimelyBalancedHost> metricToHostMap = new TreeMap<>();
|
||||
private Map<String,ArrivalRate> metricMap = new HashMap<>();
|
||||
private ReentrantReadWriteLock balancerLock = new ReentrantReadWriteLock();
|
||||
private List<TimelyBalancedHost> serverList = new ArrayList<>();
|
||||
private Random r = new Random();
|
||||
@@ -90,8 +92,7 @@ public class BalancedMetricResolver implements MetricResolver {
|
||||
HIGH_LOW, HIGH_AVG, AVG_LOW
|
||||
}
|
||||
|
||||
public BalancedMetricResolver(CuratorFramework curatorFramework, BalancerConfiguration balancerConfig,
|
||||
HealthChecker healthChecker) {
|
||||
public BalancedMetricResolver(CuratorFramework curatorFramework, BalancerConfiguration balancerConfig, HealthChecker healthChecker) {
|
||||
this.balancerConfig = balancerConfig;
|
||||
this.healthChecker = healthChecker;
|
||||
|
||||
@@ -99,15 +100,14 @@ public class BalancedMetricResolver implements MetricResolver {
|
||||
testIPRWLock(curatorFramework, assignmentsIPRWLock, ASSIGNMENTS_LOCK_PATH);
|
||||
startLeaderLatch(curatorFramework);
|
||||
startServiceListener(curatorFramework);
|
||||
assignmentsLastUpdatedInHdfs = new DistributedAtomicLong(curatorFramework, ASSIGNMENTS_LAST_UPDATED_PATH,
|
||||
new RetryForever(1000));
|
||||
assignmentsLastUpdatedInHdfs = new DistributedAtomicLong(curatorFramework, ASSIGNMENTS_LAST_UPDATED_PATH, new RetryForever(1000));
|
||||
|
||||
TreeCacheListener assignmentListener = new TreeCacheListener() {
|
||||
|
||||
@Override
|
||||
public void childEvent(CuratorFramework curatorFramework, TreeCacheEvent event) throws Exception {
|
||||
LOG.debug("Handling assignments event {}. assignmentsLastUpdatedInHdfs:{}", event.getType().toString(),
|
||||
new Date(assignmentsLastUpdatedInHdfs.get().postValue()));
|
||||
new Date(assignmentsLastUpdatedInHdfs.get().postValue()));
|
||||
if (event.getType().equals(TreeCacheEvent.Type.NODE_UPDATED)) {
|
||||
readAssignmentsFromHdfs(true);
|
||||
}
|
||||
@@ -168,12 +168,12 @@ public class BalancedMetricResolver implements MetricResolver {
|
||||
long lastLocalUpdate = assignmentsLastUpdatedLocal.get();
|
||||
long lastHdfsUpdate = assignmentsLastUpdatedInHdfs.get().postValue();
|
||||
if (lastLocalUpdate > lastHdfsUpdate) {
|
||||
LOG.debug("Leader writing assignments to hdfs lastLocalUpdate ({}) > lastHdfsUpdate ({})",
|
||||
new Date(lastLocalUpdate), new Date(lastHdfsUpdate));
|
||||
LOG.debug("Leader writing assignments to hdfs lastLocalUpdate ({}) > lastHdfsUpdate ({})", new Date(lastLocalUpdate),
|
||||
new Date(lastHdfsUpdate));
|
||||
writeAssignmentsToHdfs();
|
||||
} else {
|
||||
LOG.trace("Leader not writing assignments to hdfs lastLocalUpdate ({}) <= lastHdfsUpdate ({})",
|
||||
new Date(lastLocalUpdate), new Date(lastHdfsUpdate));
|
||||
LOG.trace("Leader not writing assignments to hdfs lastLocalUpdate ({}) <= lastHdfsUpdate ({})", new Date(lastLocalUpdate),
|
||||
new Date(lastHdfsUpdate));
|
||||
}
|
||||
} else {
|
||||
readAssignmentsFromHdfs(true);
|
||||
@@ -195,8 +195,7 @@ public class BalancedMetricResolver implements MetricResolver {
|
||||
byte[] currentNonCachedMetricsDistributedBytes = nonCachedMetricsIP.get().postValue();
|
||||
if (currentNonCachedMetricsDistributedBytes != null) {
|
||||
try {
|
||||
currentNonCachedMetricsDistributed = SerializationUtils
|
||||
.deserialize(currentNonCachedMetricsDistributedBytes);
|
||||
currentNonCachedMetricsDistributed = SerializationUtils.deserialize(currentNonCachedMetricsDistributedBytes);
|
||||
} catch (Exception e) {
|
||||
LOG.error(e.getMessage(), e);
|
||||
currentNonCachedMetricsDistributed = new TreeSet<>();
|
||||
@@ -255,8 +254,8 @@ public class BalancedMetricResolver implements MetricResolver {
|
||||
|
||||
private void startServiceListener(CuratorFramework curatorFramework) {
|
||||
try {
|
||||
ServiceDiscovery<ServerDetails> discovery = ServiceDiscoveryBuilder.builder(ServerDetails.class)
|
||||
.client(curatorFramework).basePath(SERVICE_DISCOVERY_PATH).build();
|
||||
ServiceDiscovery<ServerDetails> discovery = ServiceDiscoveryBuilder.builder(ServerDetails.class).client(curatorFramework)
|
||||
.basePath(SERVICE_DISCOVERY_PATH).build();
|
||||
discovery.start();
|
||||
Collection<ServiceInstance<ServerDetails>> instances = discovery.queryForInstances("timely-server");
|
||||
|
||||
@@ -264,10 +263,10 @@ public class BalancedMetricResolver implements MetricResolver {
|
||||
try {
|
||||
for (ServiceInstance<ServerDetails> si : instances) {
|
||||
ServerDetails pl = si.getPayload();
|
||||
TimelyBalancedHost tbh = TimelyBalancedHost.of(pl.getHost(), pl.getTcpPort(), pl.getHttpPort(),
|
||||
pl.getWsPort(), pl.getUdpPort(), new ArrivalRate(arrivalRateExecutor));
|
||||
LOG.info("adding service {} host:{} tcpPort:{} httpPort:{} wsPort:{} udpPort:{}", si.getId(),
|
||||
pl.getHost(), pl.getTcpPort(), pl.getHttpPort(), pl.getWsPort(), pl.getUdpPort());
|
||||
TimelyBalancedHost tbh = TimelyBalancedHost.of(pl.getHost(), pl.getTcpPort(), pl.getHttpPort(), pl.getWsPort(), pl.getUdpPort(),
|
||||
new ArrivalRate(arrivalRateExecutor));
|
||||
LOG.info("adding service {} host:{} tcpPort:{} httpPort:{} wsPort:{} udpPort:{}", si.getId(), pl.getHost(), pl.getTcpPort(),
|
||||
pl.getHttpPort(), pl.getWsPort(), pl.getUdpPort());
|
||||
tbh.setBalancerConfig(balancerConfig);
|
||||
serverList.add(tbh);
|
||||
}
|
||||
@@ -276,8 +275,7 @@ public class BalancedMetricResolver implements MetricResolver {
|
||||
balancerLock.writeLock().unlock();
|
||||
}
|
||||
|
||||
final ServiceCache<ServerDetails> serviceCache = discovery.serviceCacheBuilder().name("timely-server")
|
||||
.build();
|
||||
final ServiceCache<ServerDetails> serviceCache = discovery.serviceCacheBuilder().name("timely-server").build();
|
||||
ServiceCacheListener listener = new ServiceCacheListener() {
|
||||
|
||||
@Override
|
||||
@@ -289,9 +287,8 @@ public class BalancedMetricResolver implements MetricResolver {
|
||||
Set<TimelyBalancedHost> availableHosts = new HashSet<>();
|
||||
for (ServiceInstance<ServerDetails> si : instances) {
|
||||
ServerDetails pl = (ServerDetails) si.getPayload();
|
||||
TimelyBalancedHost tbh = TimelyBalancedHost.of(pl.getHost(), pl.getTcpPort(),
|
||||
pl.getHttpPort(), pl.getWsPort(), pl.getUdpPort(),
|
||||
new ArrivalRate(arrivalRateExecutor));
|
||||
TimelyBalancedHost tbh = TimelyBalancedHost.of(pl.getHost(), pl.getTcpPort(), pl.getHttpPort(), pl.getWsPort(), pl.getUdpPort(),
|
||||
new ArrivalRate(arrivalRateExecutor));
|
||||
tbh.setBalancerConfig(balancerConfig);
|
||||
availableHosts.add(tbh);
|
||||
}
|
||||
@@ -305,10 +302,9 @@ public class BalancedMetricResolver implements MetricResolver {
|
||||
availableHosts.remove(h);
|
||||
} else {
|
||||
itr.remove();
|
||||
LOG.info("removing service {}:{} host:{} tcpPort:{} httpPort:{} wsPort:{} udpPort:{}",
|
||||
h.getHost(), h.getTcpPort(), h.getHost(), h.getTcpPort(), h.getHttpPort(),
|
||||
h.getWsPort(), h.getUdpPort());
|
||||
for (Map.Entry<String, TimelyBalancedHost> e : metricToHostMap.entrySet()) {
|
||||
LOG.info("removing service {}:{} host:{} tcpPort:{} httpPort:{} wsPort:{} udpPort:{}", h.getHost(), h.getTcpPort(), h.getHost(),
|
||||
h.getTcpPort(), h.getHttpPort(), h.getWsPort(), h.getUdpPort());
|
||||
for (Map.Entry<String,TimelyBalancedHost> e : metricToHostMap.entrySet()) {
|
||||
if (e.getValue().equals(h)) {
|
||||
reassignMetrics.add(e.getKey());
|
||||
}
|
||||
@@ -319,9 +315,8 @@ public class BalancedMetricResolver implements MetricResolver {
|
||||
|
||||
// add new hosts that were not previously known
|
||||
for (TimelyBalancedHost h : availableHosts) {
|
||||
LOG.info("adding service {}:{} host:{} tcpPort:{} httpPort:{} wsPort:{} udpPort:{}",
|
||||
h.getHost(), h.getTcpPort(), h.getHost(), h.getTcpPort(), h.getHttpPort(),
|
||||
h.getWsPort(), h.getUdpPort());
|
||||
LOG.info("adding service {}:{} host:{} tcpPort:{} httpPort:{} wsPort:{} udpPort:{}", h.getHost(), h.getTcpPort(), h.getHost(),
|
||||
h.getTcpPort(), h.getHttpPort(), h.getWsPort(), h.getUdpPort());
|
||||
serverList.add(h);
|
||||
rebalanceNeeded = true;
|
||||
}
|
||||
@@ -364,12 +359,12 @@ public class BalancedMetricResolver implements MetricResolver {
|
||||
TimelyBalancedHost tbh = null;
|
||||
balancerLock.readLock().lock();
|
||||
try {
|
||||
Map<Double, TimelyBalancedHost> rateSortedHosts = new TreeMap<>();
|
||||
Map<Double,TimelyBalancedHost> rateSortedHosts = new TreeMap<>();
|
||||
for (TimelyBalancedHost s : serverList) {
|
||||
rateSortedHosts.put(s.getArrivalRate(), s);
|
||||
}
|
||||
|
||||
Iterator<Map.Entry<Double, TimelyBalancedHost>> itr = rateSortedHosts.entrySet().iterator();
|
||||
Iterator<Map.Entry<Double,TimelyBalancedHost>> itr = rateSortedHosts.entrySet().iterator();
|
||||
|
||||
while (itr.hasNext() && tbh == null) {
|
||||
TimelyBalancedHost currentTBH = itr.next().getValue();
|
||||
@@ -434,11 +429,11 @@ public class BalancedMetricResolver implements MetricResolver {
|
||||
return tbh;
|
||||
}
|
||||
|
||||
private TimelyBalancedHost chooseHost(Set<TimelyBalancedHost> potentialHosts,
|
||||
Map<TimelyBalancedHost, Double> calculatedRates, double referenceRate, BalanceType balanceType) {
|
||||
private TimelyBalancedHost chooseHost(Set<TimelyBalancedHost> potentialHosts, Map<TimelyBalancedHost,Double> calculatedRates, double referenceRate,
|
||||
BalanceType balanceType) {
|
||||
|
||||
TimelyBalancedHost tbh = null;
|
||||
Map<Long, TimelyBalancedHost> weightedList = new TreeMap<>();
|
||||
Map<Long,TimelyBalancedHost> weightedList = new TreeMap<>();
|
||||
long cumulativeWeight = 0;
|
||||
for (TimelyBalancedHost h : potentialHosts) {
|
||||
double currentDiff;
|
||||
@@ -453,7 +448,7 @@ public class BalancedMetricResolver implements MetricResolver {
|
||||
|
||||
if (cumulativeWeight > 0) {
|
||||
long randomWeight = r.nextLong() % cumulativeWeight;
|
||||
for (Map.Entry<Long, TimelyBalancedHost> e : weightedList.entrySet()) {
|
||||
for (Map.Entry<Long,TimelyBalancedHost> e : weightedList.entrySet()) {
|
||||
if (randomWeight <= e.getKey()) {
|
||||
tbh = e.getValue();
|
||||
break;
|
||||
@@ -470,7 +465,7 @@ public class BalancedMetricResolver implements MetricResolver {
|
||||
double controlBandPercentage = balancerConfig.getControlBandPercentage();
|
||||
// save current rates so that we can modify
|
||||
double totalArrivalRate = 0;
|
||||
Map<TimelyBalancedHost, Double> calculatedRates = new HashMap<>();
|
||||
Map<TimelyBalancedHost,Double> calculatedRates = new HashMap<>();
|
||||
for (TimelyBalancedHost h : serverList) {
|
||||
double tbhArrivalRate = h.getArrivalRate();
|
||||
calculatedRates.put(h, tbhArrivalRate);
|
||||
@@ -502,13 +497,11 @@ public class BalancedMetricResolver implements MetricResolver {
|
||||
LOG.debug("Host's arrival rates are within {} of the average", controlBandPercentage);
|
||||
} else if (!highHosts.isEmpty() && !lowHosts.isEmpty()) {
|
||||
LOG.debug("begin rebalancing {}", BalanceType.HIGH_LOW);
|
||||
numReassigned = rebalance(highHosts, lowHosts, calculatedRates, averageArrivalRate,
|
||||
BalanceType.HIGH_LOW);
|
||||
numReassigned = rebalance(highHosts, lowHosts, calculatedRates, averageArrivalRate, BalanceType.HIGH_LOW);
|
||||
LOG.debug("end rebalancing {} - reassigned {}", BalanceType.HIGH_LOW, numReassigned);
|
||||
} else if (lowHosts.isEmpty()) {
|
||||
LOG.debug("begin rebalancing {}", BalanceType.HIGH_AVG);
|
||||
numReassigned = rebalance(highHosts, avgHosts, calculatedRates, averageArrivalRate,
|
||||
BalanceType.HIGH_AVG);
|
||||
numReassigned = rebalance(highHosts, avgHosts, calculatedRates, averageArrivalRate, BalanceType.HIGH_AVG);
|
||||
LOG.debug("end rebalancing {} - reassigned {}", BalanceType.HIGH_AVG, numReassigned);
|
||||
} else {
|
||||
LOG.debug("begin rebalancing {}", BalanceType.AVG_LOW);
|
||||
@@ -519,14 +512,14 @@ public class BalancedMetricResolver implements MetricResolver {
|
||||
return numReassigned;
|
||||
}
|
||||
|
||||
public int rebalance(Set<TimelyBalancedHost> losingHosts, Set<TimelyBalancedHost> gainingHosts,
|
||||
Map<TimelyBalancedHost, Double> calculatedRates, double targetArrivalRate, BalanceType balanceType) {
|
||||
public int rebalance(Set<TimelyBalancedHost> losingHosts, Set<TimelyBalancedHost> gainingHosts, Map<TimelyBalancedHost,Double> calculatedRates,
|
||||
double targetArrivalRate, BalanceType balanceType) {
|
||||
|
||||
int numReassigned = 0;
|
||||
if (isLeader.get()) {
|
||||
balancerLock.writeLock().lock();
|
||||
try {
|
||||
Map<String, ArrivalRate> tempMetricMap = new HashMap<>();
|
||||
Map<String,ArrivalRate> tempMetricMap = new HashMap<>();
|
||||
tempMetricMap.putAll(metricMap);
|
||||
|
||||
Set<TimelyBalancedHost> focusedHosts;
|
||||
@@ -548,22 +541,22 @@ public class BalancedMetricResolver implements MetricResolver {
|
||||
}
|
||||
|
||||
// sort metrics by rate
|
||||
Map<Double, String> rateSortedMetrics = new TreeMap<>(Collections.reverseOrder());
|
||||
for (Map.Entry<String, ArrivalRate> e : tempMetricMap.entrySet()) {
|
||||
Map<Double,String> rateSortedMetrics = new TreeMap<>(Collections.reverseOrder());
|
||||
for (Map.Entry<String,ArrivalRate> e : tempMetricMap.entrySet()) {
|
||||
TimelyBalancedHost tbh = metricToHostMap.get(e.getKey());
|
||||
if (tbh != null && tbh.equals(h)) {
|
||||
rateSortedMetrics.put(e.getValue().getRate(), e.getKey());
|
||||
}
|
||||
}
|
||||
|
||||
LOG.trace("focusHost {}:{} desiredChange:{} rateSortedMetrics.size():{}", h.getHost(),
|
||||
h.getTcpPort(), desiredChange, rateSortedMetrics.size());
|
||||
LOG.trace("focusHost {}:{} desiredChange:{} rateSortedMetrics.size():{}", h.getHost(), h.getTcpPort(), desiredChange,
|
||||
rateSortedMetrics.size());
|
||||
|
||||
boolean doneWithHost = false;
|
||||
while (!doneWithHost) {
|
||||
Iterator<Map.Entry<Double, String>> itr = rateSortedMetrics.entrySet().iterator();
|
||||
Iterator<Map.Entry<Double,String>> itr = rateSortedMetrics.entrySet().iterator();
|
||||
while (!doneWithHost && itr.hasNext()) {
|
||||
Map.Entry<Double, String> e = null;
|
||||
Map.Entry<Double,String> e = null;
|
||||
// find largest metric that does not exceed desired change
|
||||
while (e == null && itr.hasNext()) {
|
||||
e = itr.next();
|
||||
@@ -578,8 +571,7 @@ public class BalancedMetricResolver implements MetricResolver {
|
||||
doneWithHost = true;
|
||||
} else {
|
||||
LOG.trace("Selected:{} rate:{}", e.getValue(), e.getKey());
|
||||
TimelyBalancedHost candidateHost = chooseHost(selectFromHosts, calculatedRates,
|
||||
calculatedRates.get(h), balanceType);
|
||||
TimelyBalancedHost candidateHost = chooseHost(selectFromHosts, calculatedRates, calculatedRates.get(h), balanceType);
|
||||
if (candidateHost == null) {
|
||||
LOG.trace("candidate host is null");
|
||||
doneWithHost = true;
|
||||
@@ -594,10 +586,8 @@ public class BalancedMetricResolver implements MetricResolver {
|
||||
// don't move this metric again this host or balance
|
||||
itr.remove();
|
||||
tempMetricMap.remove(metric);
|
||||
LOG.info(
|
||||
"rebalancing: reassigning metric:{} rate:{} from server {}:{} to {}:{} remaining delta {}",
|
||||
metric, metricRate, h.getHost(), h.getTcpPort(), candidateHost.getHost(),
|
||||
candidateHost.getTcpPort(), desiredChange);
|
||||
LOG.info("rebalancing: reassigning metric:{} rate:{} from server {}:{} to {}:{} remaining delta {}", metric, metricRate,
|
||||
h.getHost(), h.getTcpPort(), candidateHost.getHost(), candidateHost.getTcpPort(), desiredChange);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -646,8 +636,7 @@ public class BalancedMetricResolver implements MetricResolver {
|
||||
if (currentNonCachedMetricsDistributedBytes == null) {
|
||||
currentNonCachedMetricsIP = new TreeSet<>();
|
||||
} else {
|
||||
currentNonCachedMetricsIP = SerializationUtils
|
||||
.deserialize(currentNonCachedMetricsDistributedBytes);
|
||||
currentNonCachedMetricsIP = SerializationUtils.deserialize(currentNonCachedMetricsDistributedBytes);
|
||||
}
|
||||
if (currentNonCachedMetricsIP.containsAll(nonCachedMetricsUpdate)) {
|
||||
LOG.debug("nonCachedMetricsIP already contains {}", nonCachedMetricsUpdate);
|
||||
@@ -678,8 +667,7 @@ public class BalancedMetricResolver implements MetricResolver {
|
||||
} catch (Exception e1) {
|
||||
try {
|
||||
curatorFramework.delete().deletingChildrenIfNeeded().forPath(path);
|
||||
curatorFramework.create().creatingParentContainersIfNeeded().withMode(CreateMode.PERSISTENT)
|
||||
.forPath(path);
|
||||
curatorFramework.create().creatingParentContainersIfNeeded().withMode(CreateMode.PERSISTENT).forPath(path);
|
||||
} catch (Exception e2) {
|
||||
LOG.info(e2.getMessage(), e2);
|
||||
}
|
||||
@@ -775,8 +763,8 @@ public class BalancedMetricResolver implements MetricResolver {
|
||||
} else if (!tbh.isUp()) {
|
||||
TimelyBalancedHost oldTbh = tbh;
|
||||
tbh = getLeastUsedHost();
|
||||
LOG.debug("rebalancing from host that is down: reassigning metric {} from server {}:{} to {}:{}",
|
||||
metric, oldTbh.getHost(), oldTbh.getTcpPort(), tbh.getHost(), tbh.getTcpPort());
|
||||
LOG.debug("rebalancing from host that is down: reassigning metric {} from server {}:{} to {}:{}", metric, oldTbh.getHost(),
|
||||
oldTbh.getTcpPort(), tbh.getHost(), tbh.getTcpPort());
|
||||
if (!balancerLock.isWriteLockedByCurrentThread()) {
|
||||
balancerLock.readLock().unlock();
|
||||
balancerLock.writeLock().lock();
|
||||
@@ -899,20 +887,19 @@ public class BalancedMetricResolver implements MetricResolver {
|
||||
long lastHdfsUpdate = assignmentsLastUpdatedInHdfs.get().postValue();
|
||||
if (checkIfNecessary) {
|
||||
if (lastHdfsUpdate <= lastLocalUpdate) {
|
||||
LOG.debug("Not reading assignments from hdfs lastHdfsUpdate ({}) <= lastLocalUpdate ({})",
|
||||
new Date(lastHdfsUpdate), new Date(lastLocalUpdate));
|
||||
LOG.debug("Not reading assignments from hdfs lastHdfsUpdate ({}) <= lastLocalUpdate ({})", new Date(lastHdfsUpdate),
|
||||
new Date(lastLocalUpdate));
|
||||
return;
|
||||
}
|
||||
}
|
||||
LOG.debug("Reading assignments from hdfs lastHdfsUpdate ({}) > lastLocalUpdate ({})",
|
||||
new Date(lastHdfsUpdate), new Date(lastLocalUpdate));
|
||||
LOG.debug("Reading assignments from hdfs lastHdfsUpdate ({}) > lastLocalUpdate ({})", new Date(lastHdfsUpdate), new Date(lastLocalUpdate));
|
||||
} catch (Exception e) {
|
||||
LOG.error(e.getMessage(), e);
|
||||
}
|
||||
|
||||
// proceed with reading from HDFS
|
||||
|
||||
Map<String, TimelyBalancedHost> assignedMetricToHostMap = new TreeMap<>();
|
||||
Map<String,TimelyBalancedHost> assignedMetricToHostMap = new TreeMap<>();
|
||||
try {
|
||||
boolean acquired = false;
|
||||
while (!acquired) {
|
||||
@@ -949,23 +936,22 @@ public class BalancedMetricResolver implements MetricResolver {
|
||||
}
|
||||
|
||||
metricToHostMap.clear();
|
||||
for (Map.Entry<String, TimelyBalancedHost> e : assignedMetricToHostMap.entrySet()) {
|
||||
for (Map.Entry<String,TimelyBalancedHost> e : assignedMetricToHostMap.entrySet()) {
|
||||
if (StringUtils.isNotBlank(e.getKey()) && e.getValue() != null) {
|
||||
if (shouldCache(e.getKey())) {
|
||||
metricToHostMap.put(e.getKey(), e.getValue());
|
||||
}
|
||||
} else {
|
||||
Exception e1 = new IllegalStateException(
|
||||
"Bad assignment metric:" + e.getKey() + " host:" + e.getValue());
|
||||
Exception e1 = new IllegalStateException("Bad assignment metric:" + e.getKey() + " host:" + e.getValue());
|
||||
LOG.warn(e1.getMessage(), e1);
|
||||
}
|
||||
}
|
||||
|
||||
nonCachedMetricsLocalLock.readLock().lock();
|
||||
try {
|
||||
Iterator<Map.Entry<String, TimelyBalancedHost>> itr = metricToHostMap.entrySet().iterator();
|
||||
Iterator<Map.Entry<String,TimelyBalancedHost>> itr = metricToHostMap.entrySet().iterator();
|
||||
while (itr.hasNext()) {
|
||||
Map.Entry<String, TimelyBalancedHost> e = itr.next();
|
||||
Map.Entry<String,TimelyBalancedHost> e = itr.next();
|
||||
if (nonCachedMetrics.contains(e.getKey())) {
|
||||
itr.remove();
|
||||
}
|
||||
@@ -978,7 +964,7 @@ public class BalancedMetricResolver implements MetricResolver {
|
||||
|
||||
assignmentsLastUpdatedLocal.set(assignmentsLastUpdatedInHdfs.get().postValue());
|
||||
LOG.info("Read {} assignments from hdfs lastHdfsUpdate = lastLocalUpdate ({})", metricToHostMap.size(),
|
||||
new Date(assignmentsLastUpdatedLocal.get()));
|
||||
new Date(assignmentsLastUpdatedLocal.get()));
|
||||
} catch (FileNotFoundException e) {
|
||||
createAssignmentFile();
|
||||
metricToHostMap.clear();
|
||||
@@ -1019,14 +1005,13 @@ public class BalancedMetricResolver implements MetricResolver {
|
||||
writer.write("host");
|
||||
writer.write("tcpPort");
|
||||
writer.endRecord();
|
||||
for (Map.Entry<String, TimelyBalancedHost> e : metricToHostMap.entrySet()) {
|
||||
for (Map.Entry<String,TimelyBalancedHost> e : metricToHostMap.entrySet()) {
|
||||
if (!nonCachedMetrics.contains(e.getKey())) {
|
||||
writer.write(e.getKey());
|
||||
writer.write(e.getValue().getHost());
|
||||
writer.write(Integer.toString(e.getValue().getTcpPort()));
|
||||
writer.endRecord();
|
||||
LOG.trace("Saving assigment: {} to {}:{}", e.getKey(), e.getValue().getHost(),
|
||||
e.getValue().getTcpPort());
|
||||
LOG.trace("Saving assigment: {} to {}:{}", e.getKey(), e.getValue().getHost(), e.getValue().getTcpPort());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1036,8 +1021,8 @@ public class BalancedMetricResolver implements MetricResolver {
|
||||
if (!assignmentsLastUpdatedInHdfs.get().succeeded()) {
|
||||
assignmentsLastUpdatedInHdfs.forceSet(now);
|
||||
}
|
||||
LOG.debug("Wrote {} assignments to hdfs lastHdfsUpdate = lastLocalUpdate ({})",
|
||||
metricToHostMap.size(), new Date(assignmentsLastUpdatedLocal.get()));
|
||||
LOG.debug("Wrote {} assignments to hdfs lastHdfsUpdate = lastLocalUpdate ({})", metricToHostMap.size(),
|
||||
new Date(assignmentsLastUpdatedLocal.get()));
|
||||
// remove metric from metricMap (ArrivalRate) if not being cached
|
||||
metricMap.entrySet().removeIf(e -> !metricToHostMap.containsKey(e.getKey()));
|
||||
}
|
||||
|
||||
+2
-2
@@ -4,13 +4,13 @@
|
||||
<parent>
|
||||
<groupId>timely</groupId>
|
||||
<artifactId>timely</artifactId>
|
||||
<version>2.0.1</version>
|
||||
<version>2.0.2</version>
|
||||
</parent>
|
||||
<artifactId>timely-client</artifactId>
|
||||
<properties>
|
||||
<!-- PEP 440 compliant version gets substituted into src/main/python/template/setup.py -->
|
||||
<!-- use .dev0 instead of -SNAPSHOT -->
|
||||
<python.client.version>2.0.1</python.client.version>
|
||||
<python.client.version>2.0.2</python.client.version>
|
||||
</properties>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
|
||||
@@ -7,7 +7,7 @@ public class AddSubscription extends SubscriptionRequest {
|
||||
|
||||
private String operation = "add";
|
||||
private String metric = null;
|
||||
private Optional<Map<String, String>> tags = Optional.empty();
|
||||
private Optional<Map<String,String>> tags = Optional.empty();
|
||||
private Optional<Long> startTime = Optional.empty();
|
||||
private Optional<Long> endTime = Optional.empty();
|
||||
private Optional<Long> delayTime = Optional.empty();
|
||||
@@ -20,7 +20,7 @@ public class AddSubscription extends SubscriptionRequest {
|
||||
return metric;
|
||||
}
|
||||
|
||||
public Optional<Map<String, String>> getTags() {
|
||||
public Optional<Map<String,String>> getTags() {
|
||||
return tags;
|
||||
}
|
||||
|
||||
@@ -36,7 +36,7 @@ public class AddSubscription extends SubscriptionRequest {
|
||||
this.metric = metric;
|
||||
}
|
||||
|
||||
public void setTags(Optional<Map<String, String>> tags) {
|
||||
public void setTags(Optional<Map<String,String>> tags) {
|
||||
this.tags = tags;
|
||||
}
|
||||
|
||||
@@ -71,9 +71,8 @@ public class AddSubscription extends SubscriptionRequest {
|
||||
}
|
||||
if (obj instanceof AddSubscription) {
|
||||
AddSubscription other = (AddSubscription) obj;
|
||||
return (this.subscriptionId.equals(other.subscriptionId) && this.metric.equals(other.metric)
|
||||
&& this.tags.equals(other.tags) && this.startTime.equals(other.startTime)
|
||||
&& this.endTime.equals(other.endTime) && this.delayTime.equals(other.delayTime));
|
||||
return (this.subscriptionId.equals(other.subscriptionId) && this.metric.equals(other.metric) && this.tags.equals(other.tags)
|
||||
&& this.startTime.equals(other.startTime) && this.endTime.equals(other.endTime) && this.delayTime.equals(other.delayTime));
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -2,14 +2,16 @@ package timely.api.response;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonPropertyOrder;
|
||||
import org.apache.commons.lang3.builder.EqualsBuilder;
|
||||
import org.apache.commons.lang3.builder.HashCodeBuilder;
|
||||
import org.apache.commons.lang3.builder.ToStringBuilder;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonPropertyOrder;
|
||||
|
||||
import timely.model.Metric;
|
||||
import timely.model.Tag;
|
||||
|
||||
@JsonPropertyOrder({ "metric", "timestamp", "value", "tags", "subscriptionId", "complete" })
|
||||
@JsonPropertyOrder({"metric", "timestamp", "value", "tags", "subscriptionId", "complete"})
|
||||
public class MetricResponse {
|
||||
|
||||
private String metric;
|
||||
|
||||
@@ -15,8 +15,8 @@ import org.glassfish.tyrus.client.SslContextConfigurator;
|
||||
|
||||
public class HttpClient {
|
||||
|
||||
public static SSLContext getSSLContext(String trustStoreFile, String trustStoreType, String trustStorePass,
|
||||
String keyStoreFile, String keyStoreType, String keyStorePass) {
|
||||
public static SSLContext getSSLContext(String trustStoreFile, String trustStoreType, String trustStorePass, String keyStoreFile, String keyStoreType,
|
||||
String keyStorePass) {
|
||||
SslContextConfigurator scc = new SslContextConfigurator();
|
||||
scc.setTrustStoreFile(trustStoreFile);
|
||||
scc.setTrustStoreType(trustStoreType);
|
||||
@@ -31,12 +31,10 @@ public class HttpClient {
|
||||
return scc.createSSLContext();
|
||||
}
|
||||
|
||||
public static CloseableHttpClient get(String trustStoreFile, String trustStoreType, String trustStorePass,
|
||||
String keyStoreFile, String keyStoreType, String keyStorePass, CookieStore cookieStore,
|
||||
boolean hostVerificationEnabled) {
|
||||
public static CloseableHttpClient get(String trustStoreFile, String trustStoreType, String trustStorePass, String keyStoreFile, String keyStoreType,
|
||||
String keyStorePass, CookieStore cookieStore, boolean hostVerificationEnabled) {
|
||||
|
||||
SSLContext ssl = getSSLContext(trustStoreFile, trustStoreType, trustStorePass, keyStoreFile, keyStoreType,
|
||||
keyStorePass);
|
||||
SSLContext ssl = getSSLContext(trustStoreFile, trustStoreType, trustStorePass, keyStoreFile, keyStoreType, keyStorePass);
|
||||
return get(ssl, cookieStore, hostVerificationEnabled);
|
||||
}
|
||||
|
||||
@@ -44,8 +42,7 @@ public class HttpClient {
|
||||
return get(ssl, cookieStore, hostVerificationEnabled, false);
|
||||
}
|
||||
|
||||
public static CloseableHttpClient get(SSLContext ssl, CookieStore cookieStore, boolean hostVerificationEnabled,
|
||||
boolean clientAuth) {
|
||||
public static CloseableHttpClient get(SSLContext ssl, CookieStore cookieStore, boolean hostVerificationEnabled, boolean clientAuth) {
|
||||
HostnameVerifier hostnameVerifier;
|
||||
if (hostVerificationEnabled) {
|
||||
hostnameVerifier = SSLConnectionSocketFactory.getDefaultHostnameVerifier();
|
||||
@@ -55,16 +52,15 @@ public class HttpClient {
|
||||
RequestConfig defaultRequestConfig = RequestConfig.custom().setCookieSpec(CookieSpecs.STANDARD).build();
|
||||
HttpClientBuilder builder;
|
||||
if (cookieStore == null) {
|
||||
builder = HttpClients.custom().setSSLContext(ssl).disableCookieManagement()
|
||||
.setDefaultRequestConfig(defaultRequestConfig).setSSLHostnameVerifier(hostnameVerifier);
|
||||
builder = HttpClients.custom().setSSLContext(ssl).disableCookieManagement().setDefaultRequestConfig(defaultRequestConfig)
|
||||
.setSSLHostnameVerifier(hostnameVerifier);
|
||||
} else {
|
||||
builder = HttpClients.custom().setSSLContext(ssl).setDefaultCookieStore(cookieStore)
|
||||
.setDefaultRequestConfig(defaultRequestConfig).setSSLHostnameVerifier(hostnameVerifier);
|
||||
builder = HttpClients.custom().setSSLContext(ssl).setDefaultCookieStore(cookieStore).setDefaultRequestConfig(defaultRequestConfig)
|
||||
.setSSLHostnameVerifier(hostnameVerifier);
|
||||
}
|
||||
|
||||
if (clientAuth) {
|
||||
SSLConnectionSocketFactory sslCSF = new SSLConnectionSocketFactory(ssl,
|
||||
new String[] { "TLSv1.1", "TLSv1.2" }, null, hostnameVerifier);
|
||||
SSLConnectionSocketFactory sslCSF = new SSLConnectionSocketFactory(ssl, new String[] {"TLSv1.1", "TLSv1.2"}, null, hostnameVerifier);
|
||||
builder.setSSLSocketFactory(sslCSF);
|
||||
}
|
||||
return builder.build();
|
||||
|
||||
@@ -42,8 +42,7 @@ public class UdpClient implements AutoCloseable {
|
||||
this.sock.send(packet);
|
||||
}
|
||||
|
||||
public void flush() throws IOException {
|
||||
}
|
||||
public void flush() throws IOException {}
|
||||
|
||||
public void close() throws IOException {
|
||||
if (null != sock) {
|
||||
|
||||
@@ -33,6 +33,5 @@ public abstract class ClientHandler extends Endpoint {
|
||||
LOG.error("Error occurred on Websocket session" + session.getId(), error);
|
||||
}
|
||||
|
||||
public void beforeRequest(Map<String, List<String>> headers) {
|
||||
}
|
||||
public void beforeRequest(Map<String,List<String>> headers) {}
|
||||
}
|
||||
@@ -36,7 +36,7 @@ public class TimelyEndpointConfig implements ClientEndpointConfig {
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<String, Object> getUserProperties() {
|
||||
public Map<String,Object> getUserProperties() {
|
||||
return Collections.emptyMap();
|
||||
}
|
||||
|
||||
@@ -55,7 +55,7 @@ public class TimelyEndpointConfig implements ClientEndpointConfig {
|
||||
return new Configurator() {
|
||||
|
||||
@Override
|
||||
public void beforeRequest(Map<String, List<String>> headers) {
|
||||
public void beforeRequest(Map<String,List<String>> headers) {
|
||||
super.beforeRequest(headers);
|
||||
if (null != sessionCookie) {
|
||||
headers.put("Cookie", Collections.singletonList(sessionCookie));
|
||||
|
||||
@@ -10,7 +10,6 @@ import javax.net.ssl.SSLContext;
|
||||
import javax.websocket.DeploymentException;
|
||||
import javax.websocket.Session;
|
||||
|
||||
import com.google.common.base.Preconditions;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.http.HttpResponse;
|
||||
import org.apache.http.HttpStatus;
|
||||
@@ -28,6 +27,9 @@ import org.glassfish.tyrus.client.ClientProperties;
|
||||
import org.glassfish.tyrus.client.SslEngineConfigurator;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import com.google.common.base.Preconditions;
|
||||
|
||||
import timely.api.BasicAuthLogin;
|
||||
import timely.client.http.HttpClient;
|
||||
import timely.serialize.JsonSerializer;
|
||||
@@ -51,9 +53,8 @@ public class WebSocketClient implements AutoCloseable {
|
||||
protected Session session = null;
|
||||
protected volatile boolean closed = true;
|
||||
|
||||
protected WebSocketClient(SSLContext ssl, String timelyHostname, int timelyHttpsPort, int timelyWssPort,
|
||||
boolean clientAuth, boolean doLogin, String timelyUsername, String timelyPassword,
|
||||
boolean hostVerificationEnabled, int bufferSize) {
|
||||
protected WebSocketClient(SSLContext ssl, String timelyHostname, int timelyHttpsPort, int timelyWssPort, boolean clientAuth, boolean doLogin,
|
||||
String timelyUsername, String timelyPassword, boolean hostVerificationEnabled, int bufferSize) {
|
||||
this.ssl = ssl;
|
||||
this.timelyHostname = timelyHostname;
|
||||
this.timelyHttpsPort = timelyHttpsPort;
|
||||
@@ -71,19 +72,17 @@ public class WebSocketClient implements AutoCloseable {
|
||||
|
||||
if (doLogin && !clientAuth) {
|
||||
if ((StringUtils.isEmpty(timelyUsername) && !StringUtils.isEmpty(timelyPassword)
|
||||
|| (!StringUtils.isEmpty(timelyUsername) && StringUtils.isEmpty(timelyPassword)))) {
|
||||
|| (!StringUtils.isEmpty(timelyUsername) && StringUtils.isEmpty(timelyPassword)))) {
|
||||
throw new IllegalArgumentException("Both Timely username and password must be empty or non-empty");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
protected WebSocketClient(String timelyHostname, int timelyHttpsPort, int timelyWssPort, boolean clientAuth,
|
||||
boolean doLogin, String timelyUsername, String timelyPassword, String keyStoreFile, String keyStoreType,
|
||||
String keyStorePass, String trustStoreFile, String trustStoreType, String trustStorePass,
|
||||
boolean hostVerificationEnabled, int bufferSize) {
|
||||
this(HttpClient.getSSLContext(trustStoreFile, trustStoreType, trustStorePass, keyStoreFile, keyStoreType,
|
||||
keyStorePass), timelyHostname, timelyHttpsPort, timelyWssPort, clientAuth, doLogin, timelyUsername,
|
||||
timelyPassword, hostVerificationEnabled, bufferSize);
|
||||
protected WebSocketClient(String timelyHostname, int timelyHttpsPort, int timelyWssPort, boolean clientAuth, boolean doLogin, String timelyUsername,
|
||||
String timelyPassword, String keyStoreFile, String keyStoreType, String keyStorePass, String trustStoreFile, String trustStoreType,
|
||||
String trustStorePass, boolean hostVerificationEnabled, int bufferSize) {
|
||||
this(HttpClient.getSSLContext(trustStoreFile, trustStoreType, trustStorePass, keyStoreFile, keyStoreType, keyStorePass), timelyHostname,
|
||||
timelyHttpsPort, timelyWssPort, clientAuth, doLogin, timelyUsername, timelyPassword, hostVerificationEnabled, bufferSize);
|
||||
}
|
||||
|
||||
public void open(ClientHandler clientEndpoint) throws IOException, DeploymentException, URISyntaxException {
|
||||
@@ -115,8 +114,7 @@ public class WebSocketClient implements AutoCloseable {
|
||||
try {
|
||||
response = client.execute(request);
|
||||
if (response.getStatusLine().getStatusCode() != HttpStatus.SC_OK) {
|
||||
throw new HttpResponseException(response.getStatusLine().getStatusCode(),
|
||||
response.getStatusLine().getReasonPhrase());
|
||||
throw new HttpResponseException(response.getStatusLine().getStatusCode(), response.getStatusLine().getReasonPhrase());
|
||||
}
|
||||
for (Cookie c : cookieJar.getCookies()) {
|
||||
if (c.getName().equals("TSESSIONID")) {
|
||||
@@ -125,8 +123,7 @@ public class WebSocketClient implements AutoCloseable {
|
||||
}
|
||||
}
|
||||
if (null == sessionCookie) {
|
||||
throw new IllegalStateException(
|
||||
"Unable to find TSESSIONID cookie header in Timely login response");
|
||||
throw new IllegalStateException("Unable to find TSESSIONID cookie header in Timely login response");
|
||||
}
|
||||
} finally {
|
||||
HttpClientUtils.closeQuietly(response);
|
||||
@@ -142,8 +139,7 @@ public class WebSocketClient implements AutoCloseable {
|
||||
webSocketClient.getProperties().put(ClientProperties.SSL_ENGINE_CONFIGURATOR, sslEngine);
|
||||
webSocketClient.getProperties().put(ClientProperties.INCOMING_BUFFER_SIZE, bufferSize);
|
||||
String wssPath = "wss://" + timelyHostname + ":" + timelyWssPort + "/websocket";
|
||||
session = webSocketClient.connectToServer(clientEndpoint,
|
||||
new TimelyEndpointConfig(clientEndpoint, sessionCookie), new URI(wssPath));
|
||||
session = webSocketClient.connectToServer(clientEndpoint, new TimelyEndpointConfig(clientEndpoint, sessionCookie), new URI(wssPath));
|
||||
|
||||
final ByteBuffer pingData = ByteBuffer.allocate(0);
|
||||
webSocketClient.getScheduledExecutorService().scheduleAtFixedRate(() -> {
|
||||
|
||||
+10
-14
@@ -13,6 +13,7 @@ import javax.websocket.DeploymentException;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import timely.api.AddSubscription;
|
||||
import timely.api.CloseSubscription;
|
||||
import timely.api.CreateSubscription;
|
||||
@@ -26,22 +27,18 @@ public class WebSocketSubscriptionClient extends WebSocketClient {
|
||||
private static final Logger LOG = LoggerFactory.getLogger(WebSocketSubscriptionClient.class);
|
||||
private final String subscriptionId;
|
||||
|
||||
public WebSocketSubscriptionClient(SSLContext ssl, String timelyHostname, int timelyHttpsPort, int timelyWssPort,
|
||||
boolean clientAuth, boolean doLogin, String timelyUsername, String timelyPassword,
|
||||
boolean hostVerificationEnabled, int bufferSize) {
|
||||
super(ssl, timelyHostname, timelyHttpsPort, timelyWssPort, clientAuth, doLogin, timelyUsername, timelyPassword,
|
||||
hostVerificationEnabled, bufferSize);
|
||||
public WebSocketSubscriptionClient(SSLContext ssl, String timelyHostname, int timelyHttpsPort, int timelyWssPort, boolean clientAuth, boolean doLogin,
|
||||
String timelyUsername, String timelyPassword, boolean hostVerificationEnabled, int bufferSize) {
|
||||
super(ssl, timelyHostname, timelyHttpsPort, timelyWssPort, clientAuth, doLogin, timelyUsername, timelyPassword, hostVerificationEnabled, bufferSize);
|
||||
subscriptionId = UUID.randomUUID().toString();
|
||||
LOG.trace("Created WebSocketClient with subscriptionId {}", this.subscriptionId);
|
||||
}
|
||||
|
||||
public WebSocketSubscriptionClient(String timelyHostname, int timelyHttpsPort, int timelyWssPort,
|
||||
boolean clientAuth, boolean doLogin, String timelyUsername, String timelyPassword, String keyStoreFile,
|
||||
String keyStoreType, String keyStorePass, String trustStoreFile, String trustStoreType,
|
||||
String trustStorePass, boolean hostVerificationEnabled, int bufferSize) {
|
||||
super(timelyHostname, timelyHttpsPort, timelyWssPort, clientAuth, doLogin, timelyUsername, timelyPassword,
|
||||
keyStoreFile, keyStoreType, keyStorePass, trustStoreFile, trustStoreType, trustStorePass,
|
||||
hostVerificationEnabled, bufferSize);
|
||||
public WebSocketSubscriptionClient(String timelyHostname, int timelyHttpsPort, int timelyWssPort, boolean clientAuth, boolean doLogin,
|
||||
String timelyUsername, String timelyPassword, String keyStoreFile, String keyStoreType, String keyStorePass, String trustStoreFile,
|
||||
String trustStoreType, String trustStorePass, boolean hostVerificationEnabled, int bufferSize) {
|
||||
super(timelyHostname, timelyHttpsPort, timelyWssPort, clientAuth, doLogin, timelyUsername, timelyPassword, keyStoreFile, keyStoreType, keyStorePass,
|
||||
trustStoreFile, trustStoreType, trustStorePass, hostVerificationEnabled, bufferSize);
|
||||
subscriptionId = UUID.randomUUID().toString();
|
||||
LOG.trace("Created WebSocketClient with subscriptionId {}", this.subscriptionId);
|
||||
}
|
||||
@@ -69,8 +66,7 @@ public class WebSocketSubscriptionClient extends WebSocketClient {
|
||||
super.close();
|
||||
}
|
||||
|
||||
public Future<Void> addSubscription(String metric, Map<String, String> tags, long startTime, long endTime,
|
||||
long delayTime) throws IOException {
|
||||
public Future<Void> addSubscription(String metric, Map<String,String> tags, long startTime, long endTime, long delayTime) throws IOException {
|
||||
AddSubscription add = new AddSubscription();
|
||||
add.setSubscriptionId(subscriptionId);
|
||||
add.setMetric(metric);
|
||||
|
||||
@@ -10,18 +10,18 @@ import com.fasterxml.jackson.annotation.JsonInclude;
|
||||
import com.fasterxml.jackson.annotation.JsonPropertyOrder;
|
||||
import com.fasterxml.jackson.annotation.JsonRootName;
|
||||
import com.fasterxml.jackson.annotation.JsonUnwrapped;
|
||||
|
||||
import timely.model.parse.TagParser;
|
||||
|
||||
/**
|
||||
* A Metric consists of a metric name, tags, and a Value
|
||||
*
|
||||
* Json has been flattened to reduce overhead. Example:
|
||||
* {"name":"m1","timestamp":1,"measure":1.0,"tags":[{"k1":"v1"}]}
|
||||
* Json has been flattened to reduce overhead. Example: {"name":"m1","timestamp":1,"measure":1.0,"tags":[{"k1":"v1"}]}
|
||||
*
|
||||
*/
|
||||
@JsonRootName("metric")
|
||||
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||
@JsonPropertyOrder({ "name" })
|
||||
@JsonPropertyOrder({"name"})
|
||||
public class Metric implements Serializable {
|
||||
|
||||
private static final long serialVersionUID = 1L;
|
||||
@@ -134,8 +134,7 @@ public class Metric implements Serializable {
|
||||
}
|
||||
|
||||
/**
|
||||
* Simple Builder for Metric. Note: cannot re-use a builder after build is
|
||||
* called.
|
||||
* Simple Builder for Metric. Note: cannot re-use a builder after build is called.
|
||||
*/
|
||||
public static class Builder {
|
||||
|
||||
|
||||
@@ -57,17 +57,14 @@ public interface ObjectSizeOf {
|
||||
public static final short NUMBER_SIZE = 16;
|
||||
|
||||
/**
|
||||
* Get the size of an object. Note that we want something relatively fast that
|
||||
* gives us an order of magnitude here. The java Instrumentation agent mechanism
|
||||
* is a little too costly for general use here. This will look for the
|
||||
* ObjectSizeOf interface and if implemented on the object will use that.
|
||||
* Get the size of an object. Note that we want something relatively fast that gives us an order of magnitude here. The java Instrumentation agent
|
||||
* mechanism is a little too costly for general use here. This will look for the ObjectSizeOf interface and if implemented on the object will use that.
|
||||
* Otherwise it will do a simple navigation of the fields using reflection.
|
||||
*
|
||||
* @return an approximation of the object size
|
||||
*/
|
||||
public static long getObjectSize(Object o, boolean useSizeInBytesMethod, boolean debug) {
|
||||
long estSize = getObjectSize(o, new HashSet<ObjectInstance>(), new Stack<ObjectInstance>(),
|
||||
useSizeInBytesMethod, debug);
|
||||
long estSize = getObjectSize(o, new HashSet<ObjectInstance>(), new Stack<ObjectInstance>(), useSizeInBytesMethod, debug);
|
||||
return estSize;
|
||||
}
|
||||
|
||||
@@ -76,8 +73,7 @@ public interface ObjectSizeOf {
|
||||
return estSize;
|
||||
}
|
||||
|
||||
public static long getObjectSize(Object o, Set<ObjectInstance> visited, Stack<ObjectInstance> stack,
|
||||
boolean useSizeInBytesMethod, boolean debug) {
|
||||
public static long getObjectSize(Object o, Set<ObjectInstance> visited, Stack<ObjectInstance> stack, boolean useSizeInBytesMethod, boolean debug) {
|
||||
long totalSize = 0;
|
||||
stack.add(new ObjectInstance(o));
|
||||
while (!stack.isEmpty()) {
|
||||
@@ -96,8 +92,8 @@ public interface ObjectSizeOf {
|
||||
// the hard way...
|
||||
// do not include Class related objects or
|
||||
// reflection objects
|
||||
if ((o instanceof Class) || (o instanceof ClassLoader) || (o.getClass().getPackage() != null
|
||||
&& o.getClass().getPackage().getName().startsWith("java.lang.reflect"))) {
|
||||
if ((o instanceof Class) || (o instanceof ClassLoader)
|
||||
|| (o.getClass().getPackage() != null && o.getClass().getPackage().getName().startsWith("java.lang.reflect"))) {
|
||||
size = 0;
|
||||
} else if (o instanceof Number || o instanceof Boolean || o instanceof Character) {
|
||||
size = NUMBER_SIZE;
|
||||
|
||||
@@ -5,13 +5,15 @@ import java.util.Map;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import org.apache.commons.lang3.builder.CompareToBuilder;
|
||||
import org.apache.commons.lang3.builder.EqualsBuilder;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonAnyGetter;
|
||||
import com.fasterxml.jackson.annotation.JsonAnySetter;
|
||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
import com.fasterxml.jackson.annotation.JsonRootName;
|
||||
import com.google.common.base.Preconditions;
|
||||
import org.apache.commons.lang3.builder.CompareToBuilder;
|
||||
import org.apache.commons.lang3.builder.EqualsBuilder;
|
||||
|
||||
import timely.model.parse.TagParser;
|
||||
|
||||
/**
|
||||
@@ -27,8 +29,7 @@ public class Tag implements Comparable<Tag>, Serializable {
|
||||
private String key;
|
||||
private String value;
|
||||
|
||||
public Tag() {
|
||||
}
|
||||
public Tag() {}
|
||||
|
||||
public Tag(Tag other) {
|
||||
this.setKey(other.getKey());
|
||||
@@ -47,7 +48,7 @@ public class Tag implements Comparable<Tag>, Serializable {
|
||||
}
|
||||
|
||||
@JsonAnyGetter
|
||||
public Map<String, String> get() {
|
||||
public Map<String,String> get() {
|
||||
return Stream.of(this).collect(Collectors.toMap(Tag::getKey, Tag::getValue));
|
||||
}
|
||||
|
||||
|
||||
@@ -18,8 +18,7 @@ public class Value implements Comparator<Value>, Serializable {
|
||||
private Long timestamp;
|
||||
private Double measure;
|
||||
|
||||
public Value() {
|
||||
}
|
||||
public Value() {}
|
||||
|
||||
public Value(Value other) {
|
||||
this.setTimestamp(other.getTimestamp());
|
||||
@@ -74,11 +73,9 @@ public class Value implements Comparator<Value>, Serializable {
|
||||
if (null == o || getClass() != o.getClass())
|
||||
return false;
|
||||
|
||||
if (getTimestamp() != null ? !getTimestamp().equals(((Value) o).getTimestamp())
|
||||
: ((Value) o).getTimestamp() != null)
|
||||
if (getTimestamp() != null ? !getTimestamp().equals(((Value) o).getTimestamp()) : ((Value) o).getTimestamp() != null)
|
||||
return false;
|
||||
return !(getMeasure() != null ? !getMeasure().equals(((Value) o).getMeasure())
|
||||
: ((Value) o).getMeasure() != null);
|
||||
return !(getMeasure() != null ? !getMeasure().equals(((Value) o).getMeasure()) : ((Value) o).getMeasure() != null);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
||||
@@ -4,6 +4,7 @@ import java.util.List;
|
||||
|
||||
import com.google.common.base.Preconditions;
|
||||
import com.google.common.base.Splitter;
|
||||
|
||||
import timely.model.Metric;
|
||||
|
||||
public class MetricParser implements Parser<Metric> {
|
||||
|
||||
@@ -8,8 +8,7 @@ import java.util.Map;
|
||||
import timely.model.Tag;
|
||||
|
||||
/**
|
||||
* Parses tag1=val1,tag2=val2,tag3=val3 into a {@code List} of {@code Tag}
|
||||
* Combines {@code List} of {@code Tag} into {@code String} of
|
||||
* Parses tag1=val1,tag2=val2,tag3=val3 into a {@code List} of {@code Tag} Combines {@code List} of {@code Tag} into {@code String} of
|
||||
* tag1=val1,tag2=val2,tag3=val3.
|
||||
*/
|
||||
public class TagListParser implements Parser<List<Tag>>, Combiner<Collection<Tag>> {
|
||||
@@ -44,12 +43,11 @@ public class TagListParser implements Parser<List<Tag>>, Combiner<Collection<Tag
|
||||
}
|
||||
|
||||
/**
|
||||
* An alternative combiner for a map of string to string instead of a list of
|
||||
* tags
|
||||
* An alternative combiner for a map of string to string instead of a list of tags
|
||||
*
|
||||
* @return string representation
|
||||
*/
|
||||
public String combine(Map<String, String> tags) {
|
||||
public String combine(Map<String,String> tags) {
|
||||
StringBuilder builder = new StringBuilder();
|
||||
if (!tags.isEmpty()) {
|
||||
tags.forEach((k, v) -> builder.append(tagParser.combine(new Tag(k, v))).append(','));
|
||||
|
||||
@@ -4,11 +4,11 @@ import java.util.List;
|
||||
|
||||
import com.google.common.base.Joiner;
|
||||
import com.google.common.base.Splitter;
|
||||
|
||||
import timely.model.Tag;
|
||||
|
||||
/**
|
||||
* Parses tag1=val1 into a {@code Tag} Combines {@code Tag} into {@code String}
|
||||
* of tag1=val1
|
||||
* Parses tag1=val1 into a {@code Tag} Combines {@code Tag} into {@code String} of tag1=val1
|
||||
*/
|
||||
public class TagParser implements Parser<Tag>, Combiner<Tag> {
|
||||
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
package timely.api;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
|
||||
import timely.serialize.JsonSerializer;
|
||||
|
||||
public class SerializationTest {
|
||||
|
||||
@@ -6,9 +6,10 @@ import static org.junit.Assert.assertTrue;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import org.junit.Test;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
|
||||
/**
|
||||
*
|
||||
*
|
||||
@@ -30,8 +31,7 @@ public class MetricTest {
|
||||
Metric m4 = Metric.newBuilder().name("m4").tag("t2", "v2").tag("t1", "v1").value(1, 0.0).build();
|
||||
assertFalse(m1.equals(m4));
|
||||
|
||||
Metric m5 = Metric.newBuilder().name("m1").tag("t3", "v3").tag("t2", "v2").tag("t1", "v1").value(1, 0.0)
|
||||
.build();
|
||||
Metric m5 = Metric.newBuilder().name("m1").tag("t3", "v3").tag("t2", "v2").tag("t1", "v1").value(1, 0.0).build();
|
||||
assertFalse(m1.equals(m5));
|
||||
|
||||
Metric m6 = Metric.newBuilder().name("m1").tag("t2", "v2").tag("t1", "v1").value(2, 0.0).build();
|
||||
|
||||
@@ -5,6 +5,7 @@ import java.util.List;
|
||||
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
|
||||
import timely.model.Metric;
|
||||
import timely.model.Tag;
|
||||
|
||||
|
||||
@@ -7,6 +7,7 @@ import java.util.TreeMap;
|
||||
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
|
||||
import timely.model.Tag;
|
||||
|
||||
public class TagListParserTest {
|
||||
@@ -31,7 +32,7 @@ public class TagListParserTest {
|
||||
|
||||
@Test
|
||||
public void testListCombineMap() {
|
||||
Map<String, String> map = new TreeMap<>();
|
||||
Map<String,String> map = new TreeMap<>();
|
||||
map.put("tag1", "value1");
|
||||
map.put("tag2", "value2");
|
||||
String combined = new TagListParser().combine(map);
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>timely</groupId>
|
||||
<artifactId>timely</artifactId>
|
||||
<version>2.0.1</version>
|
||||
<version>2.0.2</version>
|
||||
</parent>
|
||||
<artifactId>collectd-nsq-plugin</artifactId>
|
||||
<dependencies>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>timely</groupId>
|
||||
<artifactId>timely</artifactId>
|
||||
<version>2.0.1</version>
|
||||
<version>2.0.2</version>
|
||||
</parent>
|
||||
<artifactId>collectd-plugin-parent</artifactId>
|
||||
<dependencies>
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>timely</groupId>
|
||||
<artifactId>timely</artifactId>
|
||||
<version>2.0.1</version>
|
||||
<version>2.0.2</version>
|
||||
</parent>
|
||||
<artifactId>collectd-timely-plugin</artifactId>
|
||||
<dependencies>
|
||||
|
||||
+1
-1
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>timely</groupId>
|
||||
<artifactId>timely</artifactId>
|
||||
<version>2.0.1</version>
|
||||
<version>2.0.2</version>
|
||||
</parent>
|
||||
<artifactId>timely-common</artifactId>
|
||||
<dependencies>
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
package timely.test;
|
||||
|
||||
public interface IntegrationTest {
|
||||
}
|
||||
public interface IntegrationTest {}
|
||||
+1
-1
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>timely</groupId>
|
||||
<artifactId>timely</artifactId>
|
||||
<version>2.0.1</version>
|
||||
<version>2.0.2</version>
|
||||
</parent>
|
||||
<artifactId>timely-app</artifactId>
|
||||
<packaging>pom</packaging>
|
||||
|
||||
Vendored
+2
-2
@@ -28,8 +28,8 @@
|
||||
}
|
||||
],
|
||||
"screenshots": [],
|
||||
"version": "2.0.1",
|
||||
"updated": "2024-01-26"
|
||||
"version": "2.0.2",
|
||||
"updated": "2024-01-31"
|
||||
},
|
||||
"dependencies": {
|
||||
"classnames": "2.2.6",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "timely-datasource",
|
||||
"version": "2.0.1",
|
||||
"version": "2.0.2",
|
||||
"description": "Timely",
|
||||
"backend": true,
|
||||
"alerting": true,
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<groupId>timely</groupId>
|
||||
<artifactId>timely</artifactId>
|
||||
<version>2.0.1</version>
|
||||
<version>2.0.2</version>
|
||||
<packaging>pom</packaging>
|
||||
<name>Timely</name>
|
||||
<description>A secure and scalable time series database based on Apache Accumulo</description>
|
||||
@@ -51,7 +51,7 @@
|
||||
<version.commons-pool2>2.6.1</version.commons-pool2>
|
||||
<version.commons-text>1.10.0</version.commons-text>
|
||||
<version.compression.gorilla>2.1.1</version.compression.gorilla>
|
||||
<version.curator>4.1.0</version.curator>
|
||||
<version.curator>5.5.0</version.curator>
|
||||
<version.datawave-in-memory-accumulo>2.1.1</version.datawave-in-memory-accumulo>
|
||||
<version.disruptor>3.4.2</version.disruptor>
|
||||
<version.easymock>4.0.2</version.easymock>
|
||||
@@ -605,7 +605,7 @@
|
||||
<compilerCompliance>${maven.compiler.source}</compilerCompliance>
|
||||
<compilerSource>${maven.compiler.source}</compilerSource>
|
||||
<compilerTargetPlatform>${maven.compiler.target}</compilerTargetPlatform>
|
||||
<configFile>${project.parent.basedir}/eclipse/Eclipse-Formatter.xml</configFile>
|
||||
<configFile>${project.parent.basedir}/src/main/resources/eclipse/Eclipse-Timely-Codestyle.xml</configFile>
|
||||
<lineEnding>LF</lineEnding>
|
||||
<skipCssFormatting>true</skipCssFormatting>
|
||||
<skipHtmlFormatting>true</skipHtmlFormatting>
|
||||
@@ -613,6 +613,13 @@
|
||||
<skipJsonFormatting>true</skipJsonFormatting>
|
||||
<skipXmlFormatting>true</skipXmlFormatting>
|
||||
</configuration>
|
||||
<executions>
|
||||
<execution>
|
||||
<goals>
|
||||
<goal>format</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>com.github.ekryd.sortpom</groupId>
|
||||
@@ -630,6 +637,33 @@
|
||||
<spaceBeforeCloseEmptyElement>true</spaceBeforeCloseEmptyElement>
|
||||
<verifyFail>Stop</verifyFail>
|
||||
</configuration>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>sort-pom</id>
|
||||
<goals>
|
||||
<goal>sort</goal>
|
||||
</goals>
|
||||
<phase>process-sources</phase>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>net.revelc.code</groupId>
|
||||
<artifactId>impsort-maven-plugin</artifactId>
|
||||
<version>${version.impsort}</version>
|
||||
<configuration>
|
||||
<groups>java.,javax.,*</groups>
|
||||
<removeUnused>true</removeUnused>
|
||||
</configuration>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>sort-imports</id>
|
||||
<goals>
|
||||
<goal>sort</goal>
|
||||
</goals>
|
||||
<phase>process-sources</phase>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
@@ -782,15 +816,6 @@
|
||||
<artifactId>rpm-maven-plugin</artifactId>
|
||||
<version>${version.maven-rpm}</version>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>net.revelc.code</groupId>
|
||||
<artifactId>impsort-maven-plugin</artifactId>
|
||||
<version>${version.impsort}</version>
|
||||
<configuration>
|
||||
<groups>java.,javax.,*</groups>
|
||||
<removeUnused>true</removeUnused>
|
||||
</configuration>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-resources-plugin</artifactId>
|
||||
@@ -891,6 +916,13 @@
|
||||
<plugin>
|
||||
<groupId>net.revelc.code</groupId>
|
||||
<artifactId>impsort-maven-plugin</artifactId>
|
||||
<configuration>
|
||||
<groups>java.,javax.,org.,com.</groups>
|
||||
<staticGroups>java,*</staticGroups>
|
||||
<excludes>
|
||||
<exclude>**/thrift/*.java</exclude>
|
||||
</excludes>
|
||||
</configuration>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>sort-imports</id>
|
||||
@@ -986,6 +1018,30 @@
|
||||
</plugins>
|
||||
</reporting>
|
||||
<profiles>
|
||||
<profile>
|
||||
<id>autoformat</id>
|
||||
<activation>
|
||||
<property>
|
||||
<name>!skipFormat</name>
|
||||
</property>
|
||||
</activation>
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>com.github.ekryd.sortpom</groupId>
|
||||
<artifactId>sortpom-maven-plugin</artifactId>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>net.revelc.code</groupId>
|
||||
<artifactId>impsort-maven-plugin</artifactId>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>net.revelc.code.formatter</groupId>
|
||||
<artifactId>formatter-maven-plugin</artifactId>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
</profile>
|
||||
<profile>
|
||||
<id>collectd</id>
|
||||
<modules>
|
||||
|
||||
+1
-1
@@ -4,7 +4,7 @@
|
||||
<parent>
|
||||
<groupId>timely</groupId>
|
||||
<artifactId>timely</artifactId>
|
||||
<version>2.0.1</version>
|
||||
<version>2.0.2</version>
|
||||
</parent>
|
||||
<artifactId>timely-server</artifactId>
|
||||
<dependencies>
|
||||
|
||||
@@ -16,6 +16,26 @@ import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.Future;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import org.apache.accumulo.core.client.AccumuloClient;
|
||||
import org.apache.accumulo.core.conf.ClientProperty;
|
||||
import org.apache.curator.RetryPolicy;
|
||||
import org.apache.curator.framework.CuratorFramework;
|
||||
import org.apache.curator.framework.CuratorFrameworkFactory;
|
||||
import org.apache.curator.framework.recipes.leader.LeaderLatch;
|
||||
import org.apache.curator.framework.recipes.leader.LeaderLatchListener;
|
||||
import org.apache.curator.retry.RetryForever;
|
||||
import org.apache.curator.x.discovery.ServiceDiscovery;
|
||||
import org.apache.curator.x.discovery.ServiceDiscoveryBuilder;
|
||||
import org.apache.curator.x.discovery.ServiceInstance;
|
||||
import org.apache.curator.x.discovery.ServiceInstanceBuilder;
|
||||
import org.apache.zookeeper.CreateMode;
|
||||
import org.apache.zookeeper.data.Stat;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.boot.WebApplicationType;
|
||||
import org.springframework.boot.builder.SpringApplicationBuilder;
|
||||
import org.springframework.context.ConfigurableApplicationContext;
|
||||
|
||||
import io.netty.bootstrap.Bootstrap;
|
||||
import io.netty.bootstrap.ServerBootstrap;
|
||||
import io.netty.buffer.PooledByteBufAllocator;
|
||||
@@ -60,25 +80,6 @@ import io.netty.handler.stream.ChunkedWriteHandler;
|
||||
import io.netty.handler.timeout.IdleStateHandler;
|
||||
import io.netty.util.NettyRuntime;
|
||||
import io.netty.util.internal.SystemPropertyUtil;
|
||||
import org.apache.accumulo.core.client.AccumuloClient;
|
||||
import org.apache.accumulo.core.conf.ClientProperty;
|
||||
import org.apache.curator.RetryPolicy;
|
||||
import org.apache.curator.framework.CuratorFramework;
|
||||
import org.apache.curator.framework.CuratorFrameworkFactory;
|
||||
import org.apache.curator.framework.recipes.leader.LeaderLatch;
|
||||
import org.apache.curator.framework.recipes.leader.LeaderLatchListener;
|
||||
import org.apache.curator.retry.RetryForever;
|
||||
import org.apache.curator.x.discovery.ServiceDiscovery;
|
||||
import org.apache.curator.x.discovery.ServiceDiscoveryBuilder;
|
||||
import org.apache.curator.x.discovery.ServiceInstance;
|
||||
import org.apache.curator.x.discovery.ServiceInstanceBuilder;
|
||||
import org.apache.zookeeper.CreateMode;
|
||||
import org.apache.zookeeper.data.Stat;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.boot.WebApplicationType;
|
||||
import org.springframework.boot.builder.SpringApplicationBuilder;
|
||||
import org.springframework.context.ConfigurableApplicationContext;
|
||||
import timely.auth.AuthCache;
|
||||
import timely.auth.JWTTokenHandler;
|
||||
import timely.auth.VisibilityCache;
|
||||
@@ -163,8 +164,7 @@ public class Server {
|
||||
private final int DEFAULT_EVENT_LOOP_THREADS;
|
||||
private LeaderLatch leaderLatch;
|
||||
|
||||
private String[] zkPaths = new String[] { SERVICE_DISCOVERY_PATH, LEADER_LATCH_PATH, NON_CACHED_METRICS,
|
||||
NON_CACHED_METRICS_LOCK_PATH };
|
||||
private String[] zkPaths = new String[] {SERVICE_DISCOVERY_PATH, LEADER_LATCH_PATH, NON_CACHED_METRICS, NON_CACHED_METRICS_LOCK_PATH};
|
||||
|
||||
private static boolean useEpoll() {
|
||||
|
||||
@@ -222,11 +222,11 @@ public class Server {
|
||||
|
||||
ServiceInstanceBuilder<ServerDetails> builder = ServiceInstance.builder();
|
||||
String serviceName = host + ":" + config.getServer().getTcpPort();
|
||||
ServiceInstance<ServerDetails> serviceInstance = builder.id(serviceName).name("timely-server")
|
||||
.address(config.getServer().getIp()).port(config.getServer().getTcpPort()).payload(payload).build();
|
||||
ServiceInstance<ServerDetails> serviceInstance = builder.id(serviceName).name("timely-server").address(config.getServer().getIp())
|
||||
.port(config.getServer().getTcpPort()).payload(payload).build();
|
||||
|
||||
ServiceDiscovery<ServerDetails> discovery = ServiceDiscoveryBuilder.builder(ServerDetails.class)
|
||||
.client(curatorFramework).basePath(SERVICE_DISCOVERY_PATH).build();
|
||||
ServiceDiscovery<ServerDetails> discovery = ServiceDiscoveryBuilder.builder(ServerDetails.class).client(curatorFramework)
|
||||
.basePath(SERVICE_DISCOVERY_PATH).build();
|
||||
discovery.start();
|
||||
discovery.registerService(serviceInstance);
|
||||
|
||||
@@ -431,8 +431,7 @@ public class Server {
|
||||
}
|
||||
|
||||
public Server(Configuration conf, AccumuloClient accumuloClient) {
|
||||
this(conf, accumuloClient, Math.max(1,
|
||||
SystemPropertyUtil.getInt("io.netty.eventLoopThreads", NettyRuntime.availableProcessors() * 2)));
|
||||
this(conf, accumuloClient, Math.max(1, SystemPropertyUtil.getInt("io.netty.eventLoopThreads", NettyRuntime.availableProcessors() * 2)));
|
||||
}
|
||||
|
||||
public Server(Configuration conf) {
|
||||
@@ -460,8 +459,7 @@ public class Server {
|
||||
try {
|
||||
Stat stat = curatorFramework.checkExists().forPath(s);
|
||||
if (stat == null) {
|
||||
curatorFramework.create().creatingParentContainersIfNeeded().withMode(CreateMode.PERSISTENT)
|
||||
.forPath(s);
|
||||
curatorFramework.create().creatingParentContainersIfNeeded().withMode(CreateMode.PERSISTENT).forPath(s);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
LOG.info(e.getMessage());
|
||||
@@ -495,13 +493,11 @@ public class Server {
|
||||
|
||||
RetryPolicy retryPolicy = new RetryForever(1000);
|
||||
int timeout = Long.valueOf(getTimeInMillis(config.getAccumulo().getZookeeperTimeout())).intValue();
|
||||
curatorFramework = CuratorFrameworkFactory.newClient(config.getAccumulo().getZookeepers(), timeout, 10000,
|
||||
retryPolicy);
|
||||
curatorFramework = CuratorFrameworkFactory.newClient(config.getAccumulo().getZookeepers(), timeout, 10000, retryPolicy);
|
||||
curatorFramework.start();
|
||||
ensureZkPaths(curatorFramework, zkPaths);
|
||||
|
||||
int nettyThreads = Math.max(1,
|
||||
SystemPropertyUtil.getInt("io.netty.eventLoopThreads", Runtime.getRuntime().availableProcessors() * 2));
|
||||
int nettyThreads = Math.max(1, SystemPropertyUtil.getInt("io.netty.eventLoopThreads", Runtime.getRuntime().availableProcessors() * 2));
|
||||
dataStore = DataStoreFactory.create(config, accumuloClient, nettyThreads);
|
||||
if (config.getCache().isEnabled()) {
|
||||
dataStoreCache = new DataStoreCache(curatorFramework, config);
|
||||
@@ -604,9 +600,8 @@ public class Server {
|
||||
}
|
||||
registerService(curatorFramework);
|
||||
shutdownHook();
|
||||
LOG.info(
|
||||
"Server started. Listening on {}:{} for TCP traffic, {}:{} for HTTP traffic, {}:{} for WebSocket traffic, and {}:{} for UDP traffic",
|
||||
tcpAddress, tcpPort, httpAddress, httpPort, wsAddress, wsPort, wsAddress, udpPort);
|
||||
LOG.info("Server started. Listening on {}:{} for TCP traffic, {}:{} for HTTP traffic, {}:{} for WebSocket traffic, and {}:{} for UDP traffic",
|
||||
tcpAddress, tcpPort, httpAddress, httpPort, wsAddress, wsPort, wsAddress, udpPort);
|
||||
}
|
||||
|
||||
protected SslContext createSSLContext(Configuration config) throws Exception {
|
||||
@@ -663,14 +658,12 @@ public class Server {
|
||||
CorsConfig cors = ccb.build();
|
||||
LOG.trace("Cors configuration: {}", cors);
|
||||
ch.pipeline().addLast("cors", new CorsHandler(cors));
|
||||
ch.pipeline().addLast("queryDecoder",
|
||||
new timely.netty.http.HttpRequestDecoder(config.getSecurity(), config.getHttp()));
|
||||
ch.pipeline().addLast("queryDecoder", new timely.netty.http.HttpRequestDecoder(config.getSecurity(), config.getHttp()));
|
||||
ch.pipeline().addLast("fileServer", new HttpStaticFileServerHandler()
|
||||
.setIgnoreSslHandshakeErrors(config.getSecurity().getServerSsl().isIgnoreSslHandshakeErrors()));
|
||||
.setIgnoreSslHandshakeErrors(config.getSecurity().getServerSsl().isIgnoreSslHandshakeErrors()));
|
||||
ch.pipeline().addLast("strict", new StrictTransportHandler(config));
|
||||
ch.pipeline().addLast("login", new X509LoginRequestHandler(config.getSecurity(), config.getHttp()));
|
||||
ch.pipeline().addLast("doLogin",
|
||||
new BasicAuthLoginRequestHandler(config.getSecurity(), config.getHttp()));
|
||||
ch.pipeline().addLast("doLogin", new BasicAuthLoginRequestHandler(config.getSecurity(), config.getHttp()));
|
||||
ch.pipeline().addLast("aggregators", new HttpAggregatorsRequestHandler());
|
||||
ch.pipeline().addLast("metrics", new HttpMetricsRequestHandler(config));
|
||||
ch.pipeline().addLast("query", new HttpQueryRequestHandler(dataStore));
|
||||
@@ -679,8 +672,8 @@ public class Server {
|
||||
ch.pipeline().addLast("version", new HttpVersionRequestHandler());
|
||||
ch.pipeline().addLast("cache", new HttpCacheRequestHandler(dataStoreCache));
|
||||
ch.pipeline().addLast("put", new HttpMetricPutHandler(dataStore));
|
||||
ch.pipeline().addLast("error", new TimelyExceptionHandler()
|
||||
.setIgnoreSslHandshakeErrors(config.getSecurity().getServerSsl().isIgnoreSslHandshakeErrors()));
|
||||
ch.pipeline().addLast("error",
|
||||
new TimelyExceptionHandler().setIgnoreSslHandshakeErrors(config.getSecurity().getServerSsl().isIgnoreSslHandshakeErrors()));
|
||||
|
||||
}
|
||||
};
|
||||
@@ -725,8 +718,7 @@ public class Server {
|
||||
ch.pipeline().addLast("aggregator", new HttpObjectAggregator(65536));
|
||||
ch.pipeline().addLast("sessionExtractor", new WebSocketFullRequestHandler());
|
||||
ch.pipeline().addLast("idle-handler", new IdleStateHandler(conf.getWebsocket().getTimeout(), 0, 0));
|
||||
ch.pipeline().addLast("ws-protocol",
|
||||
new WebSocketServerProtocolHandler(WS_PATH, null, true, 65536, false, true));
|
||||
ch.pipeline().addLast("ws-protocol", new WebSocketServerProtocolHandler(WS_PATH, null, true, 65536, false, true));
|
||||
ch.pipeline().addLast("wsDecoder", new WebSocketRequestDecoder(config.getSecurity()));
|
||||
ch.pipeline().addLast("aggregators", new WSAggregatorsRequestHandler());
|
||||
ch.pipeline().addLast("metrics", new WSMetricsRequestHandler(config));
|
||||
@@ -735,8 +727,7 @@ public class Server {
|
||||
ch.pipeline().addLast("suggest", new WSSuggestRequestHandler(dataStore));
|
||||
ch.pipeline().addLast("version", new WSVersionRequestHandler());
|
||||
ch.pipeline().addLast("put", new WSMetricPutHandler(dataStore));
|
||||
ch.pipeline().addLast("create",
|
||||
new WSCreateSubscriptionRequestHandler(dataStore, dataStoreCache, config));
|
||||
ch.pipeline().addLast("create", new WSCreateSubscriptionRequestHandler(dataStore, dataStoreCache, config));
|
||||
ch.pipeline().addLast("add", new WSAddSubscriptionRequestHandler());
|
||||
ch.pipeline().addLast("remove", new WSRemoveSubscriptionRequestHandler());
|
||||
ch.pipeline().addLast("close", new WSCloseSubscriptionRequestHandler());
|
||||
|
||||
@@ -15,6 +15,7 @@ import org.apache.accumulo.minicluster.MiniAccumuloConfig;
|
||||
import org.apache.accumulo.minicluster.ServerType;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import timely.configuration.Configuration;
|
||||
|
||||
public class StandaloneServer extends Server {
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package timely.adapter.accumulo;
|
||||
|
||||
import org.apache.accumulo.core.data.Key;
|
||||
|
||||
import timely.api.model.Meta;
|
||||
import timely.model.Metric;
|
||||
|
||||
|
||||
@@ -10,7 +10,6 @@ import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import com.google.common.base.Joiner;
|
||||
import org.apache.accumulo.core.client.lexicoder.LongLexicoder;
|
||||
import org.apache.accumulo.core.client.lexicoder.PairLexicoder;
|
||||
import org.apache.accumulo.core.client.lexicoder.StringLexicoder;
|
||||
@@ -21,6 +20,9 @@ import org.apache.accumulo.core.security.ColumnVisibility;
|
||||
import org.apache.accumulo.core.util.ComparablePair;
|
||||
import org.apache.accumulo.core.util.Pair;
|
||||
import org.apache.hadoop.io.Text;
|
||||
|
||||
import com.google.common.base.Joiner;
|
||||
|
||||
import timely.model.Metric;
|
||||
import timely.model.Tag;
|
||||
import timely.model.parse.TagListParser;
|
||||
@@ -31,11 +33,9 @@ import timely.model.parse.TagParser;
|
||||
*/
|
||||
public class MetricAdapter {
|
||||
|
||||
private static final PairLexicoder<String, Long> rowCoder = new PairLexicoder<>(new StringLexicoder(),
|
||||
new LongLexicoder());
|
||||
private static final PairLexicoder<String,Long> rowCoder = new PairLexicoder<>(new StringLexicoder(), new LongLexicoder());
|
||||
|
||||
private static final PairLexicoder<Long, String> colQualCoder = new PairLexicoder<>(new LongLexicoder(),
|
||||
new StringLexicoder());
|
||||
private static final PairLexicoder<Long,String> colQualCoder = new PairLexicoder<>(new LongLexicoder(), new StringLexicoder());
|
||||
|
||||
private static final TagParser tagParser = new TagParser();
|
||||
private static final TagListParser tagListParser = new TagListParser();
|
||||
@@ -68,9 +68,9 @@ public class MetricAdapter {
|
||||
return newTags;
|
||||
}
|
||||
|
||||
private static Map<String, String> escapeDelimiters(Map<String, String> tags) {
|
||||
Map<String, String> newTags = new LinkedHashMap<>();
|
||||
for (Map.Entry<String, String> t : tags.entrySet()) {
|
||||
private static Map<String,String> escapeDelimiters(Map<String,String> tags) {
|
||||
Map<String,String> newTags = new LinkedHashMap<>();
|
||||
for (Map.Entry<String,String> t : tags.entrySet()) {
|
||||
// escape all commas and equals in the tag key and value since they
|
||||
// are used as a delimiter
|
||||
String k = t.getKey();
|
||||
@@ -103,19 +103,19 @@ public class MetricAdapter {
|
||||
.collect(Collectors.joining(","));
|
||||
// @formatter:on
|
||||
byte[] cqBytes = encodeColQual(metric.getValue().getTimestamp(), cq);
|
||||
mutation.put(new Text(cf.getBytes(Charset.forName("UTF-8"))), new Text(cqBytes), extractVisibility(tags),
|
||||
metric.getValue().getTimestamp(), extractValue(metric));
|
||||
mutation.put(new Text(cf.getBytes(Charset.forName("UTF-8"))), new Text(cqBytes), extractVisibility(tags), metric.getValue().getTimestamp(),
|
||||
extractValue(metric));
|
||||
}
|
||||
return mutation;
|
||||
}
|
||||
|
||||
public static Key toKey(String metric, Map<String, String> tags, long timestamp) {
|
||||
public static Key toKey(String metric, Map<String,String> tags, long timestamp) {
|
||||
byte[] row = encodeRowKey(metric, timestamp);
|
||||
|
||||
tags = escapeDelimiters(tags);
|
||||
StringBuilder colQualSb = new StringBuilder();
|
||||
String cf = null;
|
||||
for (Map.Entry<String, String> entry : tags.entrySet()) {
|
||||
for (Map.Entry<String,String> entry : tags.entrySet()) {
|
||||
if (entry.getKey().equals(VISIBILITY_TAG))
|
||||
continue;
|
||||
|
||||
@@ -158,7 +158,7 @@ public class MetricAdapter {
|
||||
// @formatter:on
|
||||
}
|
||||
|
||||
public static ColumnVisibility extractVisibility(Map<String, String> tags) {
|
||||
public static ColumnVisibility extractVisibility(Map<String,String> tags) {
|
||||
if (tags.containsKey(VISIBILITY_TAG)) {
|
||||
return new ColumnVisibility(tags.get(VISIBILITY_TAG));
|
||||
} else {
|
||||
@@ -167,14 +167,14 @@ public class MetricAdapter {
|
||||
}
|
||||
|
||||
public static Metric parse(Key k, Value v, boolean includeVizTag) {
|
||||
ComparablePair<String, Long> row = rowCoder.decode(k.getRow().getBytes());
|
||||
ComparablePair<String,Long> row = rowCoder.decode(k.getRow().getBytes());
|
||||
// @formatter:off
|
||||
Metric.Builder builder = Metric.newBuilder()
|
||||
.name(row.getFirst())
|
||||
.value(k.getTimestamp(), ByteBuffer.wrap(v.get()).getDouble())
|
||||
.tag(tagParser.parse(k.getColumnFamily().toString()));
|
||||
// @formatter:on
|
||||
ComparablePair<Long, String> cq = colQualCoder.decode(k.getColumnQualifier().getBytes());
|
||||
ComparablePair<Long,String> cq = colQualCoder.decode(k.getColumnQualifier().getBytes());
|
||||
tagListParser.parse(cq.getSecond()).forEach(builder::tag);
|
||||
if (includeVizTag && k.getColumnVisibility().getLength() > 0) {
|
||||
tagListParser.parse("viz=" + k.getColumnVisibility().toString()).forEach(builder::tag);
|
||||
@@ -199,11 +199,11 @@ public class MetricAdapter {
|
||||
return colQualCoder.encode(new ComparablePair<>(timestamp, colQual));
|
||||
}
|
||||
|
||||
public static Pair<Long, String> decodeColQual(byte[] colQual) {
|
||||
public static Pair<Long,String> decodeColQual(byte[] colQual) {
|
||||
return colQualCoder.decode(colQual);
|
||||
}
|
||||
|
||||
public static Pair<String, Long> decodeRowKey(Key k) {
|
||||
public static Pair<String,Long> decodeRowKey(Key k) {
|
||||
return rowCoder.decode(k.getRow().getBytes());
|
||||
}
|
||||
}
|
||||
@@ -3,10 +3,11 @@ package timely.api.annotation;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import io.github.lukehutch.fastclasspathscanner.FastClasspathScanner;
|
||||
import io.github.lukehutch.fastclasspathscanner.scanner.ScanResult;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import io.github.lukehutch.fastclasspathscanner.FastClasspathScanner;
|
||||
import io.github.lukehutch.fastclasspathscanner.scanner.ScanResult;
|
||||
import timely.api.request.HttpGetRequest;
|
||||
import timely.api.request.HttpPostRequest;
|
||||
import timely.api.request.TcpRequest;
|
||||
@@ -26,8 +27,7 @@ public class AnnotationResolver {
|
||||
private static List<Class<?>> wsClasses = new ArrayList<>();
|
||||
private static List<Class<?>> udpClasses = new ArrayList<>();
|
||||
|
||||
private AnnotationResolver() {
|
||||
}
|
||||
private AnnotationResolver() {}
|
||||
|
||||
static {
|
||||
tcpClassNames = result.getNamesOfClassesWithAnnotation(Tcp.class);
|
||||
|
||||
@@ -95,8 +95,7 @@ public class Meta implements Comparable<Meta> {
|
||||
|
||||
public static Meta parse(Key k, Value v, String prefix) {
|
||||
if (k.getColumnQualifier().getLength() > 0) {
|
||||
return new Meta(k.getRow().toString().substring(prefix.length()), k.getColumnFamily().toString(),
|
||||
k.getColumnQualifier().toString());
|
||||
return new Meta(k.getRow().toString().substring(prefix.length()), k.getColumnFamily().toString(), k.getColumnQualifier().toString());
|
||||
} else if (k.getColumnFamily().getLength() > 0) {
|
||||
return new Meta(k.getRow().toString().substring(prefix.length()), k.getColumnFamily().toString(), null);
|
||||
|
||||
|
||||
@@ -4,6 +4,7 @@ import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
|
||||
import com.google.common.collect.HashMultimap;
|
||||
import com.google.common.collect.Multimap;
|
||||
|
||||
import timely.auth.TimelyUser;
|
||||
import timely.auth.util.HttpHeaderUtils;
|
||||
import timely.netty.http.auth.TimelyAuthenticationToken;
|
||||
@@ -11,12 +12,12 @@ import timely.netty.http.auth.TimelyAuthenticationToken;
|
||||
/**
|
||||
* Base class for requests that require auth
|
||||
*/
|
||||
@JsonIgnoreProperties({ "userName", "requestHeader" })
|
||||
@JsonIgnoreProperties({"userName", "requestHeader"})
|
||||
public class AuthenticatedRequest implements Request {
|
||||
|
||||
private String sessionId = null;
|
||||
private TimelyAuthenticationToken token = null;
|
||||
private Multimap<String, String> requestHeaders = HashMultimap.create();
|
||||
private Multimap<String,String> requestHeaders = HashMultimap.create();
|
||||
|
||||
public String getSessionId() {
|
||||
return sessionId;
|
||||
@@ -26,11 +27,11 @@ public class AuthenticatedRequest implements Request {
|
||||
this.sessionId = sessionId;
|
||||
}
|
||||
|
||||
public void addHeaders(Multimap<String, String> headers) {
|
||||
public void addHeaders(Multimap<String,String> headers) {
|
||||
requestHeaders.putAll(headers);
|
||||
}
|
||||
|
||||
public Multimap<String, String> getRequestHeaders() {
|
||||
public Multimap<String,String> getRequestHeaders() {
|
||||
return requestHeaders;
|
||||
}
|
||||
|
||||
|
||||
@@ -5,10 +5,12 @@ import java.nio.charset.StandardCharsets;
|
||||
import javax.xml.bind.annotation.XmlElement;
|
||||
import javax.xml.bind.annotation.XmlRootElement;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonUnwrapped;
|
||||
import io.netty.handler.codec.http.FullHttpRequest;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonUnwrapped;
|
||||
|
||||
import io.netty.handler.codec.http.FullHttpRequest;
|
||||
import timely.api.annotation.Http;
|
||||
import timely.api.annotation.Tcp;
|
||||
import timely.api.annotation.Udp;
|
||||
@@ -35,8 +37,7 @@ public class MetricRequest implements TcpRequest, HttpPostRequest, WebSocketRequ
|
||||
|
||||
private static final MetricParser metricParser = new MetricParser();
|
||||
|
||||
public MetricRequest() {
|
||||
}
|
||||
public MetricRequest() {}
|
||||
|
||||
public MetricRequest(Metric metric) {
|
||||
this.metric = metric;
|
||||
|
||||
@@ -3,5 +3,4 @@ package timely.api.request;
|
||||
import com.fasterxml.jackson.annotation.JsonTypeInfo;
|
||||
|
||||
@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "operation")
|
||||
public interface WebSocketRequest extends Request {
|
||||
}
|
||||
public interface WebSocketRequest extends Request {}
|
||||
@@ -9,7 +9,7 @@ import timely.api.annotation.WebSocket;
|
||||
public class AddSubscription extends SubscriptionRequest {
|
||||
|
||||
private String metric = null;
|
||||
private Optional<Map<String, String>> tags = Optional.empty();
|
||||
private Optional<Map<String,String>> tags = Optional.empty();
|
||||
private Optional<Long> startTime = Optional.empty();
|
||||
private Optional<Long> endTime = Optional.empty();
|
||||
private Optional<Long> delayTime = Optional.empty();
|
||||
@@ -22,11 +22,11 @@ public class AddSubscription extends SubscriptionRequest {
|
||||
this.metric = metric;
|
||||
}
|
||||
|
||||
public Optional<Map<String, String>> getTags() {
|
||||
public Optional<Map<String,String>> getTags() {
|
||||
return tags;
|
||||
}
|
||||
|
||||
public void setTags(Map<String, String> tags) {
|
||||
public void setTags(Map<String,String> tags) {
|
||||
this.tags = Optional.ofNullable(tags);
|
||||
}
|
||||
|
||||
|
||||
@@ -3,5 +3,4 @@ package timely.api.request.subscription;
|
||||
import timely.api.annotation.WebSocket;
|
||||
|
||||
@WebSocket(operation = "close")
|
||||
public class CloseSubscription extends SubscriptionRequest {
|
||||
}
|
||||
public class CloseSubscription extends SubscriptionRequest {}
|
||||
@@ -3,5 +3,4 @@ package timely.api.request.subscription;
|
||||
import timely.api.annotation.WebSocket;
|
||||
|
||||
@WebSocket(operation = "create")
|
||||
public class CreateSubscription extends SubscriptionRequest {
|
||||
}
|
||||
public class CreateSubscription extends SubscriptionRequest {}
|
||||
@@ -11,8 +11,7 @@ import timely.api.request.WebSocketRequest;
|
||||
|
||||
@Http(path = "/api/aggregators")
|
||||
@WebSocket(operation = "aggregators")
|
||||
public class AggregatorsRequest extends AuthenticatedRequest
|
||||
implements HttpGetRequest, HttpPostRequest, WebSocketRequest {
|
||||
public class AggregatorsRequest extends AuthenticatedRequest implements HttpGetRequest, HttpPostRequest, WebSocketRequest {
|
||||
|
||||
private FullHttpRequest httpRequest = null;
|
||||
|
||||
|
||||
@@ -6,13 +6,15 @@ import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import io.netty.handler.codec.http.FullHttpRequest;
|
||||
import io.netty.handler.codec.http.QueryStringDecoder;
|
||||
import org.apache.commons.lang3.builder.EqualsBuilder;
|
||||
import org.apache.commons.lang3.builder.HashCodeBuilder;
|
||||
import org.apache.commons.lang3.builder.ToStringBuilder;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
|
||||
import io.netty.handler.codec.http.FullHttpRequest;
|
||||
import io.netty.handler.codec.http.QueryStringDecoder;
|
||||
import timely.api.annotation.Http;
|
||||
import timely.api.annotation.WebSocket;
|
||||
import timely.api.request.AuthenticatedRequest;
|
||||
@@ -31,8 +33,7 @@ public class QueryRequest extends AuthenticatedRequest implements HttpGetRequest
|
||||
private long counterMax = 0;
|
||||
private long resetValue = 0;
|
||||
|
||||
public RateOption() {
|
||||
}
|
||||
public RateOption() {}
|
||||
|
||||
public RateOption(String options) {
|
||||
String[] parts = options.split(",");
|
||||
@@ -211,7 +212,7 @@ public class QueryRequest extends AuthenticatedRequest implements HttpGetRequest
|
||||
private boolean rate = false;
|
||||
private RateOption rateOptions = null;
|
||||
private Optional<String> downsample = Optional.empty();
|
||||
private Map<String, String> tags = new LinkedHashMap<>();
|
||||
private Map<String,String> tags = new LinkedHashMap<>();
|
||||
private Collection<Filter> filters = new ArrayList<>();
|
||||
private Collection<String> tsuids = new ArrayList<>();
|
||||
|
||||
@@ -255,11 +256,11 @@ public class QueryRequest extends AuthenticatedRequest implements HttpGetRequest
|
||||
this.downsample = downsample;
|
||||
}
|
||||
|
||||
public Map<String, String> getTags() {
|
||||
public Map<String,String> getTags() {
|
||||
return tags;
|
||||
}
|
||||
|
||||
public void setTags(Map<String, String> tags) {
|
||||
public void setTags(Map<String,String> tags) {
|
||||
this.tags = (tags == null) ? new LinkedHashMap<>() : tags;
|
||||
}
|
||||
|
||||
|
||||
@@ -3,13 +3,15 @@ package timely.api.request.timeseries;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import io.netty.handler.codec.http.FullHttpRequest;
|
||||
import io.netty.handler.codec.http.QueryStringDecoder;
|
||||
import org.apache.commons.lang3.builder.EqualsBuilder;
|
||||
import org.apache.commons.lang3.builder.HashCodeBuilder;
|
||||
import org.apache.commons.lang3.builder.ToStringBuilder;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
|
||||
import io.netty.handler.codec.http.FullHttpRequest;
|
||||
import io.netty.handler.codec.http.QueryStringDecoder;
|
||||
import timely.api.annotation.Http;
|
||||
import timely.api.annotation.WebSocket;
|
||||
import timely.api.request.AuthenticatedRequest;
|
||||
@@ -21,8 +23,7 @@ import timely.util.JsonUtil;
|
||||
|
||||
@Http(path = "/api/search/lookup")
|
||||
@WebSocket(operation = "lookup")
|
||||
public class SearchLookupRequest extends AuthenticatedRequest
|
||||
implements HttpGetRequest, HttpPostRequest, WebSocketRequest {
|
||||
public class SearchLookupRequest extends AuthenticatedRequest implements HttpGetRequest, HttpPostRequest, WebSocketRequest {
|
||||
|
||||
@JsonProperty("metric")
|
||||
private String query;
|
||||
|
||||
@@ -4,11 +4,13 @@ import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
import org.apache.commons.lang3.builder.ToStringBuilder;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
|
||||
import io.netty.handler.codec.http.FullHttpRequest;
|
||||
import io.netty.handler.codec.http.QueryStringDecoder;
|
||||
import org.apache.commons.lang3.builder.ToStringBuilder;
|
||||
import timely.api.annotation.Http;
|
||||
import timely.api.annotation.WebSocket;
|
||||
import timely.api.request.AuthenticatedRequest;
|
||||
|
||||
@@ -8,7 +8,7 @@ public class TimelyException extends Exception {
|
||||
private static final long serialVersionUID = 1L;
|
||||
private int code = 500;
|
||||
private String details = null;
|
||||
private Map<String, String> responseHeaders = new HashMap<>();
|
||||
private Map<String,String> responseHeaders = new HashMap<>();
|
||||
|
||||
public TimelyException(int code, String message, String details) {
|
||||
this(code, message, details, null);
|
||||
@@ -40,7 +40,7 @@ public class TimelyException extends Exception {
|
||||
this.responseHeaders.put(name, value);
|
||||
}
|
||||
|
||||
public Map<String, String> getResponseHeaders() {
|
||||
public Map<String,String> getResponseHeaders() {
|
||||
return this.responseHeaders;
|
||||
}
|
||||
}
|
||||
@@ -13,6 +13,7 @@ import com.fasterxml.jackson.databind.JsonSerializer;
|
||||
import com.fasterxml.jackson.databind.SerializerProvider;
|
||||
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
|
||||
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
|
||||
|
||||
import timely.api.response.timeseries.AggregatorsResponse.AggregatorsResponseDeserializer;
|
||||
import timely.api.response.timeseries.AggregatorsResponse.AggregatorsResponseSerializer;
|
||||
import timely.sample.Aggregator;
|
||||
@@ -30,8 +31,7 @@ public class AggregatorsResponse {
|
||||
public static class AggregatorsResponseSerializer extends JsonSerializer<AggregatorsResponse> {
|
||||
|
||||
@Override
|
||||
public void serialize(AggregatorsResponse value, JsonGenerator gen, SerializerProvider serializers)
|
||||
throws IOException, JsonProcessingException {
|
||||
public void serialize(AggregatorsResponse value, JsonGenerator gen, SerializerProvider serializers) throws IOException, JsonProcessingException {
|
||||
gen.writeStartArray();
|
||||
for (String a : value.getAggregators()) {
|
||||
gen.writeString(a);
|
||||
@@ -43,8 +43,7 @@ public class AggregatorsResponse {
|
||||
public static class AggregatorsResponseDeserializer extends JsonDeserializer<AggregatorsResponse> {
|
||||
|
||||
@Override
|
||||
public AggregatorsResponse deserialize(JsonParser p, DeserializationContext ctxt)
|
||||
throws IOException, JsonProcessingException {
|
||||
public AggregatorsResponse deserialize(JsonParser p, DeserializationContext ctxt) throws IOException, JsonProcessingException {
|
||||
return new AggregatorsResponse();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -10,11 +10,16 @@ import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.TreeSet;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.http.MediaType;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.fasterxml.jackson.databind.node.ArrayNode;
|
||||
import com.fasterxml.jackson.databind.node.ObjectNode;
|
||||
|
||||
import io.netty.buffer.Unpooled;
|
||||
import io.netty.handler.codec.http.DefaultFullHttpResponse;
|
||||
import io.netty.handler.codec.http.FullHttpResponse;
|
||||
@@ -22,9 +27,6 @@ import io.netty.handler.codec.http.HttpHeaderNames;
|
||||
import io.netty.handler.codec.http.HttpResponseStatus;
|
||||
import io.netty.handler.codec.http.HttpVersion;
|
||||
import io.netty.handler.codec.http.websocketx.TextWebSocketFrame;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.http.MediaType;
|
||||
import timely.adapter.accumulo.MetricAdapter;
|
||||
import timely.api.model.Meta;
|
||||
import timely.configuration.Configuration;
|
||||
@@ -127,8 +129,7 @@ public class MetricsResponse {
|
||||
} else {
|
||||
buf = this.generateHtml().toString().getBytes(StandardCharsets.UTF_8);
|
||||
}
|
||||
FullHttpResponse response = new DefaultFullHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.OK,
|
||||
Unpooled.copiedBuffer(buf));
|
||||
FullHttpResponse response = new DefaultFullHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.OK, Unpooled.copiedBuffer(buf));
|
||||
response.headers().set(HttpHeaderNames.CONTENT_TYPE, responseType);
|
||||
response.headers().set(HttpHeaderNames.CONTENT_LENGTH, response.content().readableBytes());
|
||||
return response;
|
||||
@@ -156,7 +157,7 @@ public class MetricsResponse {
|
||||
b.append(TR_END);
|
||||
String prevMetric = null;
|
||||
StringBuilder tags = new StringBuilder();
|
||||
Map<String, Long> tagMap = new HashMap<>();
|
||||
Map<String,Long> tagMap = new HashMap<>();
|
||||
for (Meta m : tree) {
|
||||
if (prevMetric != null && !m.getMetric().equals(prevMetric)) {
|
||||
tagMap.clear();
|
||||
@@ -193,7 +194,7 @@ public class MetricsResponse {
|
||||
protected String generateJson(final ObjectMapper mapper) throws JsonProcessingException {
|
||||
// map non-ignored metrics to their list of tags
|
||||
final MetaCache cache = MetaCacheFactory.getCache(conf);
|
||||
Map<String, List<JsonNode>> metricTagMap = new HashMap<>();
|
||||
Map<String,List<JsonNode>> metricTagMap = new HashMap<>();
|
||||
cache.forEach(m -> {
|
||||
if (!metricTagMap.containsKey(m.getMetric())) {
|
||||
metricTagMap.put(m.getMetric(), new ArrayList<>());
|
||||
|
||||
@@ -5,16 +5,17 @@ import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonPropertyOrder;
|
||||
import org.apache.commons.lang3.builder.ToStringBuilder;
|
||||
|
||||
@JsonPropertyOrder({ "metric", "tags", "aggregatedTags", "dps" })
|
||||
import com.fasterxml.jackson.annotation.JsonPropertyOrder;
|
||||
|
||||
@JsonPropertyOrder({"metric", "tags", "aggregatedTags", "dps"})
|
||||
public class QueryResponse {
|
||||
|
||||
private String metric;
|
||||
private Map<String, String> tags = new LinkedHashMap<>();
|
||||
private Map<String,String> tags = new LinkedHashMap<>();
|
||||
private List<String> aggregatedTags = new ArrayList<>();
|
||||
private Map<String, Object> dps = new LinkedHashMap<>();
|
||||
private Map<String,Object> dps = new LinkedHashMap<>();
|
||||
|
||||
public String getMetric() {
|
||||
return metric;
|
||||
@@ -24,11 +25,11 @@ public class QueryResponse {
|
||||
this.metric = metric;
|
||||
}
|
||||
|
||||
public Map<String, String> getTags() {
|
||||
public Map<String,String> getTags() {
|
||||
return tags;
|
||||
}
|
||||
|
||||
public void setTags(Map<String, String> tags) {
|
||||
public void setTags(Map<String,String> tags) {
|
||||
this.tags = tags;
|
||||
}
|
||||
|
||||
@@ -48,11 +49,11 @@ public class QueryResponse {
|
||||
this.aggregatedTags.add(tag);
|
||||
}
|
||||
|
||||
public Map<String, Object> getDps() {
|
||||
public Map<String,Object> getDps() {
|
||||
return dps;
|
||||
}
|
||||
|
||||
public void setDps(Map<String, Object> dps) {
|
||||
public void setDps(Map<String,Object> dps) {
|
||||
this.dps = dps;
|
||||
}
|
||||
|
||||
|
||||
@@ -5,25 +5,26 @@ import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonPropertyOrder;
|
||||
import org.apache.commons.lang3.builder.EqualsBuilder;
|
||||
import org.apache.commons.lang3.builder.HashCodeBuilder;
|
||||
|
||||
@JsonPropertyOrder({ "type", "metric", "tags", "limit", "time", "totalResults", "results" })
|
||||
import com.fasterxml.jackson.annotation.JsonPropertyOrder;
|
||||
|
||||
@JsonPropertyOrder({"type", "metric", "tags", "limit", "time", "totalResults", "results"})
|
||||
public class SearchLookupResponse {
|
||||
|
||||
@JsonPropertyOrder({ "tags", "metric", "tsuid" })
|
||||
@JsonPropertyOrder({"tags", "metric", "tsuid"})
|
||||
public static class Result {
|
||||
|
||||
private Map<String, String> tags = new LinkedHashMap<>();
|
||||
private Map<String,String> tags = new LinkedHashMap<>();
|
||||
private String metric;
|
||||
private String tsuid;
|
||||
|
||||
public Map<String, String> getTags() {
|
||||
public Map<String,String> getTags() {
|
||||
return tags;
|
||||
}
|
||||
|
||||
public void setTags(Map<String, String> tags) {
|
||||
public void setTags(Map<String,String> tags) {
|
||||
this.tags = tags;
|
||||
}
|
||||
|
||||
@@ -80,7 +81,7 @@ public class SearchLookupResponse {
|
||||
|
||||
private String type;
|
||||
private String metric;
|
||||
private Map<String, String> tags = new LinkedHashMap<>();
|
||||
private Map<String,String> tags = new LinkedHashMap<>();
|
||||
private int limit;
|
||||
private int time;
|
||||
private int totalResults;
|
||||
@@ -103,11 +104,11 @@ public class SearchLookupResponse {
|
||||
this.metric = metric;
|
||||
}
|
||||
|
||||
public Map<String, String> getTags() {
|
||||
public Map<String,String> getTags() {
|
||||
return tags;
|
||||
}
|
||||
|
||||
public void setTags(Map<String, String> tags) {
|
||||
public void setTags(Map<String,String> tags) {
|
||||
this.tags = tags;
|
||||
}
|
||||
|
||||
|
||||
@@ -14,6 +14,7 @@ import com.fasterxml.jackson.databind.JsonSerializer;
|
||||
import com.fasterxml.jackson.databind.SerializerProvider;
|
||||
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
|
||||
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
|
||||
|
||||
import timely.api.response.timeseries.SuggestResponse.SuggestDeserializer;
|
||||
import timely.api.response.timeseries.SuggestResponse.SuggestSerializer;
|
||||
|
||||
@@ -24,8 +25,7 @@ public class SuggestResponse {
|
||||
public static class SuggestSerializer extends JsonSerializer<SuggestResponse> {
|
||||
|
||||
@Override
|
||||
public void serialize(SuggestResponse value, JsonGenerator gen, SerializerProvider serializers)
|
||||
throws IOException, JsonProcessingException {
|
||||
public void serialize(SuggestResponse value, JsonGenerator gen, SerializerProvider serializers) throws IOException, JsonProcessingException {
|
||||
gen.writeStartArray();
|
||||
for (String s : value.getSuggestions()) {
|
||||
gen.writeString(s);
|
||||
@@ -37,8 +37,7 @@ public class SuggestResponse {
|
||||
public static class SuggestDeserializer extends JsonDeserializer<SuggestResponse> {
|
||||
|
||||
@Override
|
||||
public SuggestResponse deserialize(JsonParser p, DeserializationContext ctxt)
|
||||
throws IOException, JsonProcessingException {
|
||||
public SuggestResponse deserialize(JsonParser p, DeserializationContext ctxt) throws IOException, JsonProcessingException {
|
||||
SuggestResponse response = new SuggestResponse();
|
||||
if (p.getCurrentToken() == JsonToken.START_ARRAY) {
|
||||
while (p.nextToken() != JsonToken.END_ARRAY) {
|
||||
|
||||
@@ -6,12 +6,14 @@ import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import com.github.benmanes.caffeine.cache.Cache;
|
||||
import com.github.benmanes.caffeine.cache.Caffeine;
|
||||
import org.apache.accumulo.core.security.Authorizations;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import com.github.benmanes.caffeine.cache.Cache;
|
||||
import com.github.benmanes.caffeine.cache.Caffeine;
|
||||
|
||||
import timely.api.request.AuthenticatedRequest;
|
||||
import timely.api.request.AuthenticatedWebSocketRequest;
|
||||
import timely.auth.util.AuthorizationsUtil;
|
||||
@@ -21,7 +23,7 @@ import timely.netty.http.auth.TimelyAuthenticationToken;
|
||||
public class AuthCache {
|
||||
|
||||
private static final Logger LOG = LoggerFactory.getLogger(AuthCache.class);
|
||||
private static Cache<String, TimelyPrincipal> CACHE = null;
|
||||
private static Cache<String,TimelyPrincipal> CACHE = null;
|
||||
|
||||
private static int cacheExpirationMinutes = -1;
|
||||
private static int cacheRefreshMinutes = -1;
|
||||
@@ -42,12 +44,12 @@ public class AuthCache {
|
||||
cacheRefreshMinutes = security.getCacheRefreshMinutes();
|
||||
}
|
||||
|
||||
private static Cache<String, TimelyPrincipal> getCache() {
|
||||
private static Cache<String,TimelyPrincipal> getCache() {
|
||||
if (-1 == cacheExpirationMinutes) {
|
||||
throw new IllegalStateException("Cache session max age not configured.");
|
||||
}
|
||||
if (null == CACHE) {
|
||||
Caffeine<Object, Object> caffeine = Caffeine.newBuilder();
|
||||
Caffeine<Object,Object> caffeine = Caffeine.newBuilder();
|
||||
caffeine.expireAfterWrite(cacheExpirationMinutes, TimeUnit.MINUTES);
|
||||
if (cacheRefreshMinutes > 0) {
|
||||
caffeine.refreshAfterWrite(cacheRefreshMinutes, TimeUnit.MINUTES);
|
||||
@@ -65,18 +67,14 @@ public class AuthCache {
|
||||
String[] dnArray = key.split(" -> ");
|
||||
if (dnArray.length == 1) {
|
||||
SubjectIssuerDNPair pair = SubjectIssuerDNPair.parse(dnArray[0]);
|
||||
TimelyAuthenticationToken token = AuthenticationService.getAuthenticationToken(new Object(),
|
||||
pair.subjectDN(), pair.issuerDN());
|
||||
timelyPrincipal = AuthenticationService.authenticate(token, pair, token.getTimelyPrincipal().getName(),
|
||||
false);
|
||||
TimelyAuthenticationToken token = AuthenticationService.getAuthenticationToken(new Object(), pair.subjectDN(), pair.issuerDN());
|
||||
timelyPrincipal = AuthenticationService.authenticate(token, pair, token.getTimelyPrincipal().getName(), false);
|
||||
} else if (dnArray.length > 1) {
|
||||
Collection<TimelyUser> timelyUsers = new ArrayList<>();
|
||||
for (String s : dnArray) {
|
||||
SubjectIssuerDNPair pair = SubjectIssuerDNPair.parse(s);
|
||||
TimelyAuthenticationToken token = AuthenticationService.getAuthenticationToken(new Object(),
|
||||
pair.subjectDN(), pair.issuerDN());
|
||||
TimelyPrincipal p = AuthenticationService.authenticate(token, pair,
|
||||
token.getTimelyPrincipal().getName(), false);
|
||||
TimelyAuthenticationToken token = AuthenticationService.getAuthenticationToken(new Object(), pair.subjectDN(), pair.issuerDN());
|
||||
TimelyPrincipal p = AuthenticationService.authenticate(token, pair, token.getTimelyPrincipal().getName(), false);
|
||||
timelyUsers.addAll(p.getProxiedUsers());
|
||||
}
|
||||
timelyPrincipal = new TimelyPrincipal(timelyUsers);
|
||||
@@ -141,8 +139,7 @@ public class AuthCache {
|
||||
if (anonAccessAllowed) {
|
||||
auths = Collections.singletonList(Authorizations.EMPTY);
|
||||
} else {
|
||||
throw new IllegalArgumentException(
|
||||
"User must authenticate with a client certificate, OAuth token, or login credentials");
|
||||
throw new IllegalArgumentException("User must authenticate with a client certificate, OAuth token, or login credentials");
|
||||
}
|
||||
} else if (StringUtils.isNotBlank(sessionId)) {
|
||||
auths = getAuthorizations(sessionId);
|
||||
|
||||
@@ -5,10 +5,6 @@ import java.util.*;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import com.google.common.collect.Multimap;
|
||||
import io.netty.channel.ChannelHandlerContext;
|
||||
import io.netty.handler.codec.http.HttpResponseStatus;
|
||||
import io.netty.handler.ssl.SslHandler;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
@@ -17,6 +13,12 @@ import org.springframework.context.ApplicationContext;
|
||||
import org.springframework.context.support.ClassPathXmlApplicationContext;
|
||||
import org.springframework.security.authentication.AuthenticationManager;
|
||||
import org.springframework.security.core.Authentication;
|
||||
|
||||
import com.google.common.collect.Multimap;
|
||||
|
||||
import io.netty.channel.ChannelHandlerContext;
|
||||
import io.netty.handler.codec.http.HttpResponseStatus;
|
||||
import io.netty.handler.ssl.SslHandler;
|
||||
import timely.api.request.AuthenticatedRequest;
|
||||
import timely.api.response.TimelyException;
|
||||
import timely.auth.util.DnUtils;
|
||||
@@ -29,8 +31,8 @@ public class AuthenticationService {
|
||||
private static final Logger LOG = LoggerFactory.getLogger(AuthenticationService.class);
|
||||
private static ApplicationContext springContext;
|
||||
private static AuthenticationManager authManager;
|
||||
private static ConcurrentHashMap<String, Object> fetchingEntity = new ConcurrentHashMap<>();
|
||||
private static ConcurrentHashMap<String, Object> fetchingPrincipal = new ConcurrentHashMap<>();
|
||||
private static ConcurrentHashMap<String,Object> fetchingEntity = new ConcurrentHashMap<>();
|
||||
private static ConcurrentHashMap<String,Object> fetchingPrincipal = new ConcurrentHashMap<>();
|
||||
private static Collection<String> requiredRoles;
|
||||
private static Collection<String> requiredAuths;
|
||||
public static final String AUTH_HEADER = "Authorization";
|
||||
@@ -49,8 +51,7 @@ public class AuthenticationService {
|
||||
requiredAuths = uncheckedAuths;
|
||||
requiredAuths.removeIf(String::isEmpty);
|
||||
} catch (BeansException e) {
|
||||
throw new ServiceConfigurationError("Error setting up Authentication objects: " + e.getMessage(),
|
||||
e.getRootCause());
|
||||
throw new ServiceConfigurationError("Error setting up Authentication objects: " + e.getMessage(), e.getRootCause());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -66,8 +67,7 @@ public class AuthenticationService {
|
||||
return authenticate(authentication, pair, entity, true);
|
||||
}
|
||||
|
||||
public static TimelyPrincipal authenticate(Authentication authentication, SubjectIssuerDNPair pair, String entity,
|
||||
boolean useCache) {
|
||||
public static TimelyPrincipal authenticate(Authentication authentication, SubjectIssuerDNPair pair, String entity, boolean useCache) {
|
||||
|
||||
// only one thread should call the authenticationManager for a given entity
|
||||
TimelyPrincipal principal = null;
|
||||
@@ -115,12 +115,11 @@ public class AuthenticationService {
|
||||
String oauthToken = request.getRequestHeader(AUTH_HEADER);
|
||||
if (StringUtils.isBlank(sessionId) && StringUtils.isBlank(oauthToken) && clientCert == null) {
|
||||
throw new TimelyException(HttpResponseStatus.UNAUTHORIZED.code(), "User must authenticate",
|
||||
"User must authenticate with a client certificate, OAuth token, or login credentials");
|
||||
"User must authenticate with a client certificate, OAuth token, or login credentials");
|
||||
} else if (StringUtils.isNotBlank(sessionId)) {
|
||||
verifiedPrincipal = AuthCache.get(sessionId);
|
||||
if (verifiedPrincipal == null) {
|
||||
throw new TimelyException(HttpResponseStatus.UNAUTHORIZED.code(), "User must authenticate",
|
||||
"Unknown session id was submitted, log in again");
|
||||
throw new TimelyException(HttpResponseStatus.UNAUTHORIZED.code(), "User must authenticate", "Unknown session id was submitted, log in again");
|
||||
}
|
||||
} else {
|
||||
try {
|
||||
@@ -130,8 +129,7 @@ public class AuthenticationService {
|
||||
if (oauthToken != null) {
|
||||
// if the requestPrincipal came from an oauthToken, use that
|
||||
TimelyPrincipal cachedPrincipal = AuthCache.get(entity);
|
||||
if (cachedPrincipal == null
|
||||
|| (cachedPrincipal.getCreationTime() != requestPrincipal.getCreationTime())) {
|
||||
if (cachedPrincipal == null || (cachedPrincipal.getCreationTime() != requestPrincipal.getCreationTime())) {
|
||||
AuthCache.put(entity, requestPrincipal);
|
||||
}
|
||||
verifiedPrincipal = requestPrincipal;
|
||||
@@ -153,16 +151,14 @@ public class AuthenticationService {
|
||||
List<TimelyUser> authenticatedTimelyUsers = new ArrayList<>();
|
||||
for (TimelyUser user : requestPrincipal.getProxiedUsers()) {
|
||||
SubjectIssuerDNPair p = user.getDn();
|
||||
TimelyAuthenticationToken entityToken = getAuthenticationToken(clientCert,
|
||||
p.subjectDN(), p.issuerDN());
|
||||
TimelyPrincipal entityPrincipal = authenticate(entityToken, p,
|
||||
entityToken.getTimelyPrincipal().getName());
|
||||
TimelyAuthenticationToken entityToken = getAuthenticationToken(clientCert, p.subjectDN(), p.issuerDN());
|
||||
TimelyPrincipal entityPrincipal = authenticate(entityToken, p, entityToken.getTimelyPrincipal().getName());
|
||||
authenticatedTimelyUsers.addAll(entityPrincipal.getProxiedUsers());
|
||||
}
|
||||
verifiedPrincipal = new TimelyPrincipal(authenticatedTimelyUsers);
|
||||
AuthCache.put(verifiedPrincipal.getName(), verifiedPrincipal);
|
||||
LOG.debug("Authenticated user {} with authorizations {}",
|
||||
verifiedPrincipal.getName(), verifiedPrincipal.getAuthorizationsString());
|
||||
LOG.debug("Authenticated user {} with authorizations {}", verifiedPrincipal.getName(),
|
||||
verifiedPrincipal.getAuthorizationsString());
|
||||
} else {
|
||||
LOG.trace("Verified principal {} in AuthCache on 2nd attempt", entity);
|
||||
}
|
||||
@@ -211,8 +207,7 @@ public class AuthenticationService {
|
||||
if (!missingRoles.isEmpty() || !missingAuths.isEmpty()) {
|
||||
String message = "";
|
||||
if (!missingRoles.isEmpty() && !missingAuths.isEmpty()) {
|
||||
message = "User:" + primaryUser.getName() + " is missing role(s):" + missingRoles + " and auth(s):"
|
||||
+ missingAuths;
|
||||
message = "User:" + primaryUser.getName() + " is missing role(s):" + missingRoles + " and auth(s):" + missingAuths;
|
||||
|
||||
} else if (!missingRoles.isEmpty()) {
|
||||
message = "User:" + primaryUser.getName() + " is missing role(s):" + missingRoles;
|
||||
@@ -245,13 +240,13 @@ public class AuthenticationService {
|
||||
return new TimelyPrincipal(JWTTokenHandler.createUsersFromToken(token, PRINCIPALS_CLAIM));
|
||||
}
|
||||
|
||||
public static TimelyPrincipal createPrincipalFromHeaders(Multimap<String, String> httpHeaders) {
|
||||
public static TimelyPrincipal createPrincipalFromHeaders(Multimap<String,String> httpHeaders) {
|
||||
String header = HttpHeaderUtils.getSingleHeader(httpHeaders, AUTH_HEADER, true);
|
||||
String token = header.substring("Bearer".length()).trim();
|
||||
return createPrincipalFromToken(token);
|
||||
}
|
||||
|
||||
public static TimelyAuthenticationToken getAuthenticationToken(Multimap<String, String> headers) {
|
||||
public static TimelyAuthenticationToken getAuthenticationToken(Multimap<String,String> headers) {
|
||||
TimelyAuthenticationToken authenticationToken = null;
|
||||
try {
|
||||
TimelyPrincipal timelyPrincipal = createPrincipalFromHeaders(headers);
|
||||
@@ -262,20 +257,17 @@ public class AuthenticationService {
|
||||
return authenticationToken;
|
||||
}
|
||||
|
||||
public static TimelyAuthenticationToken getAuthenticationToken(X509Certificate clientCert,
|
||||
Multimap<String, String> headers) {
|
||||
public static TimelyAuthenticationToken getAuthenticationToken(X509Certificate clientCert, Multimap<String,String> headers) {
|
||||
TimelyAuthenticationToken authenticationToken = null;
|
||||
try {
|
||||
authenticationToken = new TimelyAuthenticationToken(clientCert.getSubjectDN().getName(), clientCert,
|
||||
headers);
|
||||
authenticationToken = new TimelyAuthenticationToken(clientCert.getSubjectDN().getName(), clientCert, headers);
|
||||
} catch (Exception e) {
|
||||
LOG.error(e.getMessage(), e);
|
||||
}
|
||||
return authenticationToken;
|
||||
}
|
||||
|
||||
public static TimelyAuthenticationToken getAuthenticationToken(Object clientCert, String subjectDn,
|
||||
String issuerDn) {
|
||||
public static TimelyAuthenticationToken getAuthenticationToken(Object clientCert, String subjectDn, String issuerDn) {
|
||||
TimelyAuthenticationToken authenticationToken = null;
|
||||
try {
|
||||
authenticationToken = new TimelyAuthenticationToken(subjectDn, issuerDn, clientCert);
|
||||
@@ -285,20 +277,16 @@ public class AuthenticationService {
|
||||
return authenticationToken;
|
||||
}
|
||||
|
||||
private static TimelyPrincipal getTimelyPrincipal(Authentication authentication,
|
||||
SubjectIssuerDNPair subjectIssuerDNPair) {
|
||||
private static TimelyPrincipal getTimelyPrincipal(Authentication authentication, SubjectIssuerDNPair subjectIssuerDNPair) {
|
||||
String subjectDn = subjectIssuerDNPair.subjectDN();
|
||||
TimelyUser.UserType userType = DnUtils.isServerDN(subjectDn) ? TimelyUser.UserType.SERVER
|
||||
: TimelyUser.UserType.USER;
|
||||
Collection<String> auths = authentication.getAuthorities().stream().map(a -> a.getAuthority())
|
||||
.collect(Collectors.toList());
|
||||
TimelyUser.UserType userType = DnUtils.isServerDN(subjectDn) ? TimelyUser.UserType.SERVER : TimelyUser.UserType.USER;
|
||||
Collection<String> auths = authentication.getAuthorities().stream().map(a -> a.getAuthority()).collect(Collectors.toList());
|
||||
Object o = authentication.getPrincipal();
|
||||
Collection<String> roles = null;
|
||||
if (o instanceof TimelyUserDetails) {
|
||||
roles = ((TimelyUserDetails) o).getRoles();
|
||||
}
|
||||
TimelyUser timelyUser = new TimelyUser(subjectIssuerDNPair, userType, auths, roles, null,
|
||||
System.currentTimeMillis());
|
||||
TimelyUser timelyUser = new TimelyUser(subjectIssuerDNPair, userType, auths, roles, null, System.currentTimeMillis());
|
||||
return new TimelyPrincipal(Arrays.asList(timelyUser));
|
||||
}
|
||||
}
|
||||
@@ -15,51 +15,33 @@ import org.apache.accumulo.core.security.Authorizations;
|
||||
public class AuthorizationsMinimizer {
|
||||
|
||||
/**
|
||||
* Given a collection of Accumulo {@link Authorizations} objects, this method
|
||||
* attempts to compute a "minimum" set that can be used for a multi-entity scan
|
||||
* of data from Accumulo. The idea is that there is a call chain with multiple
|
||||
* entities (e.g., a middleware broker acting on behalf of a GUI that is in turn
|
||||
* acting on behalf of a user). In order to ensure that only data visible to all
|
||||
* of the entities is returned, we must add a visibility filter for each entity
|
||||
* in the chain to ensure that each key/value pair returned by Accumulo can be
|
||||
* seen by all of the entities in the chain. Since we may be scanning a very
|
||||
* large number of keys, it would be ideal if we could reduce the number of
|
||||
* visibility filters to a small set if that can be done safely.
|
||||
* Given a collection of Accumulo {@link Authorizations} objects, this method attempts to compute a "minimum" set that can be used for a multi-entity scan
|
||||
* of data from Accumulo. The idea is that there is a call chain with multiple entities (e.g., a middleware broker acting on behalf of a GUI that is in turn
|
||||
* acting on behalf of a user). In order to ensure that only data visible to all of the entities is returned, we must add a visibility filter for each
|
||||
* entity in the chain to ensure that each key/value pair returned by Accumulo can be seen by all of the entities in the chain. Since we may be scanning a
|
||||
* very large number of keys, it would be ideal if we could reduce the number of visibility filters to a small set if that can be done safely.
|
||||
* <p>
|
||||
* The crux of the problem, and the reason we need multiple filters in the first
|
||||
* place (as opposed to just intersecting the authorizations for all all
|
||||
* entities and using that set), is the fact that Accumulo visibility
|
||||
* expressions may include disjunctions. Consider the following visibility
|
||||
* expression.
|
||||
* The crux of the problem, and the reason we need multiple filters in the first place (as opposed to just intersecting the authorizations for all all
|
||||
* entities and using that set), is the fact that Accumulo visibility expressions may include disjunctions. Consider the following visibility expression.
|
||||
* <p>
|
||||
* {@code (A & B & ORG1) | (A & C & ORG2)}
|
||||
* <p>
|
||||
* If the incoming user has the Accumulo authorizations A, B, and ORG1 and the
|
||||
* user is coming through a server with the Accumulo authorizations A, C, and
|
||||
* ORG2, then the server is allowed to see data protected with this expression
|
||||
* and the user is also allowed, so the data should be returned with those two
|
||||
* calling entities. If we simply computed an intersection of the authorizations
|
||||
* that set would be A. That set of authorizations does not match the visibility
|
||||
* expression and data that should be returned would not be.
|
||||
* If the incoming user has the Accumulo authorizations A, B, and ORG1 and the user is coming through a server with the Accumulo authorizations A, C, and
|
||||
* ORG2, then the server is allowed to see data protected with this expression and the user is also allowed, so the data should be returned with those two
|
||||
* calling entities. If we simply computed an intersection of the authorizations that set would be A. That set of authorizations does not match the
|
||||
* visibility expression and data that should be returned would not be.
|
||||
* <p>
|
||||
* Given that we must apply multiple visibility filters in order to properly
|
||||
* return data for a calling entity chain, it is desirable to minimize the
|
||||
* number of visibility filters used, if possible. If a set of authorizations is
|
||||
* a superset of any other set of authorizations, then it does not need to be
|
||||
* tested. As long as we are not changing any of the authorizations sets, we
|
||||
* know we will never allow any data to be returned that would otherwise not be.
|
||||
* By removing authorizations sets, we might only prevent the return of data
|
||||
* that should otherwise be returned. Consider a given set of authorizations
|
||||
* that is a subset of another. If the subset passes a visibility expression,
|
||||
* then the superset will too since it contains at least the same
|
||||
* authorizations, so there is no need to test against it. If the subset does
|
||||
* not pass, then the data is not visible to the chain anyway, so there is no
|
||||
* point in testing the superset.
|
||||
* Given that we must apply multiple visibility filters in order to properly return data for a calling entity chain, it is desirable to minimize the number
|
||||
* of visibility filters used, if possible. If a set of authorizations is a superset of any other set of authorizations, then it does not need to be tested.
|
||||
* As long as we are not changing any of the authorizations sets, we know we will never allow any data to be returned that would otherwise not be. By
|
||||
* removing authorizations sets, we might only prevent the return of data that should otherwise be returned. Consider a given set of authorizations that is
|
||||
* a subset of another. If the subset passes a visibility expression, then the superset will too since it contains at least the same authorizations, so
|
||||
* there is no need to test against it. If the subset does not pass, then the data is not visible to the chain anyway, so there is no point in testing the
|
||||
* superset.
|
||||
*
|
||||
* @param authorizations
|
||||
* the list of Authorizations for each entity in the call chain
|
||||
* @return a minimized set of Authorizations that allows visibility of exactly
|
||||
* the same data as {@code authorizations}
|
||||
* @return a minimized set of Authorizations that allows visibility of exactly the same data as {@code authorizations}
|
||||
*/
|
||||
public static Collection<Authorizations> minimize(Collection<Authorizations> authorizations) {
|
||||
if (authorizations.size() > 1) {
|
||||
@@ -67,16 +49,15 @@ public class AuthorizationsMinimizer {
|
||||
// individual authorizations).
|
||||
// Since we are adding to a LinkedHashSet, this will de-dupe any duplicate
|
||||
// authorization sets.
|
||||
final LinkedHashSet<Set<String>> allAuths = authorizations.stream().map(
|
||||
a -> a.getAuthorizations().stream().map(String::new).collect(Collectors.toCollection(HashSet::new)))
|
||||
.collect(Collectors.toCollection(LinkedHashSet::new));
|
||||
final LinkedHashSet<Set<String>> allAuths = authorizations.stream()
|
||||
.map(a -> a.getAuthorizations().stream().map(String::new).collect(Collectors.toCollection(HashSet::new)))
|
||||
.collect(Collectors.toCollection(LinkedHashSet::new));
|
||||
|
||||
// Go through the authorizations sets and remove any that are supersets of any
|
||||
// other.
|
||||
for (Iterator<Set<String>> it = allAuths.iterator(); it.hasNext(); /* empty */) {
|
||||
Set<String> currentSet = it.next();
|
||||
if (allAuths.stream().filter(a -> a != currentSet && a.size() <= currentSet.size())
|
||||
.anyMatch(currentSet::containsAll))
|
||||
if (allAuths.stream().filter(a -> a != currentSet && a.size() <= currentSet.size()).anyMatch(currentSet::containsAll))
|
||||
it.remove();
|
||||
}
|
||||
|
||||
@@ -84,8 +65,7 @@ public class AuthorizationsMinimizer {
|
||||
// set from
|
||||
// TreeSet<String> objects back into Authorizations objects.
|
||||
if (allAuths.size() < authorizations.size()) {
|
||||
authorizations = allAuths.stream().map(a -> new Authorizations(a.toArray(new String[0])))
|
||||
.collect(Collectors.toCollection(LinkedHashSet::new));
|
||||
authorizations = allAuths.stream().map(a -> new Authorizations(a.toArray(new String[0]))).collect(Collectors.toCollection(LinkedHashSet::new));
|
||||
}
|
||||
}
|
||||
return authorizations;
|
||||
|
||||
@@ -16,15 +16,13 @@ import org.apache.log4j.Logger;
|
||||
public class ConfigurableVisibilityFilter extends WrappingIterator implements OptionDescriber {
|
||||
|
||||
public static final String AUTHORIZATIONS_OPT = "authorizations";
|
||||
private SortedKeyValueIterator<Key, Value> delegate;
|
||||
private SortedKeyValueIterator<Key,Value> delegate;
|
||||
|
||||
private static final Logger log = Logger.getLogger(ConfigurableVisibilityFilter.class);
|
||||
|
||||
public ConfigurableVisibilityFilter() {
|
||||
}
|
||||
public ConfigurableVisibilityFilter() {}
|
||||
|
||||
public ConfigurableVisibilityFilter(SortedKeyValueIterator<Key, Value> iterator, Authorizations authorizations,
|
||||
byte[] defaultVisibility) {
|
||||
public ConfigurableVisibilityFilter(SortedKeyValueIterator<Key,Value> iterator, Authorizations authorizations, byte[] defaultVisibility) {
|
||||
setSource(iterator);
|
||||
}
|
||||
|
||||
@@ -34,8 +32,7 @@ public class ConfigurableVisibilityFilter extends WrappingIterator implements Op
|
||||
}
|
||||
|
||||
@Override
|
||||
public void init(SortedKeyValueIterator<Key, Value> source, Map<String, String> options, IteratorEnvironment env)
|
||||
throws IOException {
|
||||
public void init(SortedKeyValueIterator<Key,Value> source, Map<String,String> options, IteratorEnvironment env) throws IOException {
|
||||
Authorizations auths = Authorizations.EMPTY;
|
||||
if (options.containsKey(AUTHORIZATIONS_OPT))
|
||||
auths = new Authorizations(options.get(AUTHORIZATIONS_OPT).split(","));
|
||||
@@ -46,21 +43,20 @@ public class ConfigurableVisibilityFilter extends WrappingIterator implements Op
|
||||
}
|
||||
|
||||
@Override
|
||||
public SortedKeyValueIterator<Key, Value> deepCopy(IteratorEnvironment env) {
|
||||
public SortedKeyValueIterator<Key,Value> deepCopy(IteratorEnvironment env) {
|
||||
return new ConfigurableVisibilityFilter(this, env);
|
||||
}
|
||||
|
||||
@Override
|
||||
public IteratorOptions describeOptions() {
|
||||
IteratorOptions opts = new IteratorOptions(getClass().getSimpleName(),
|
||||
"Filters keys based to return only those whose visibility tests positive against the supplied authorizations",
|
||||
null, null);
|
||||
"Filters keys based to return only those whose visibility tests positive against the supplied authorizations", null, null);
|
||||
opts.addNamedOption(AUTHORIZATIONS_OPT, "Comma delimited list of scan authorizations");
|
||||
return opts;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean validateOptions(Map<String, String> options) {
|
||||
public boolean validateOptions(Map<String,String> options) {
|
||||
boolean valid = false;
|
||||
String auths = options.get(AUTHORIZATIONS_OPT);
|
||||
if (auths != null) {
|
||||
|
||||
@@ -11,13 +11,14 @@ import org.springframework.security.core.userdetails.AuthenticationUserDetailsSe
|
||||
import org.springframework.security.core.userdetails.UserDetails;
|
||||
import org.springframework.security.core.userdetails.UsernameNotFoundException;
|
||||
import org.springframework.security.web.authentication.preauth.PreAuthenticatedAuthenticationToken;
|
||||
|
||||
import timely.netty.http.auth.TimelyUserDetails;
|
||||
|
||||
public class FileUserDetailsService implements AuthenticationUserDetailsService<PreAuthenticatedAuthenticationToken> {
|
||||
|
||||
private HashMap<String, TimelyUser> users;
|
||||
private HashMap<String,TimelyUser> users;
|
||||
|
||||
public Map<String, TimelyUser> getUsers() {
|
||||
public Map<String,TimelyUser> getUsers() {
|
||||
return users;
|
||||
}
|
||||
|
||||
@@ -27,8 +28,7 @@ public class FileUserDetailsService implements AuthenticationUserDetailsService<
|
||||
}
|
||||
|
||||
@Override
|
||||
public UserDetails loadUserDetails(final PreAuthenticatedAuthenticationToken token)
|
||||
throws UsernameNotFoundException {
|
||||
public UserDetails loadUserDetails(final PreAuthenticatedAuthenticationToken token) throws UsernameNotFoundException {
|
||||
// Determine if the user is allowed to access the system, if not throw
|
||||
// UsernameNotFoundException
|
||||
String username = token.getName();
|
||||
|
||||
@@ -13,22 +13,23 @@ import java.util.List;
|
||||
import java.util.TreeSet;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.fasterxml.jackson.datatype.guava.GuavaModule;
|
||||
import io.jsonwebtoken.Claims;
|
||||
import io.jsonwebtoken.Jws;
|
||||
import io.jsonwebtoken.Jwts;
|
||||
import org.apache.accumulo.core.client.AccumuloClient;
|
||||
import org.apache.accumulo.core.security.Authorizations;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.util.ResourceUtils;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.fasterxml.jackson.datatype.guava.GuavaModule;
|
||||
|
||||
import io.jsonwebtoken.Claims;
|
||||
import io.jsonwebtoken.Jws;
|
||||
import io.jsonwebtoken.Jwts;
|
||||
import timely.configuration.JwtSsl;
|
||||
|
||||
/**
|
||||
* Converts between a String encoded JSON Web Token and a collection of
|
||||
* {@link TimelyUser}s.
|
||||
* Converts between a String encoded JSON Web Token and a collection of {@link TimelyUser}s.
|
||||
*/
|
||||
public class JWTTokenHandler {
|
||||
|
||||
@@ -48,8 +49,7 @@ public class JWTTokenHandler {
|
||||
JWTTokenHandler.signatureCheckKey = certificate.getPublicKey();
|
||||
} else {
|
||||
KeyStore keyStore = KeyStore.getInstance(type == null ? "JKS" : type);
|
||||
char[] keyPassword = ssl.getKeyStorePassword() == null ? null
|
||||
: ssl.getKeyStorePassword().toCharArray();
|
||||
char[] keyPassword = ssl.getKeyStorePassword() == null ? null : ssl.getKeyStorePassword().toCharArray();
|
||||
keyStore.load(ResourceUtils.getURL(ssl.getKeyStoreFile()).openStream(), keyPassword);
|
||||
String alias = keyStore.aliases().nextElement();
|
||||
Certificate certificate = keyStore.getCertificate(alias);
|
||||
@@ -64,10 +64,8 @@ public class JWTTokenHandler {
|
||||
|
||||
if (accumuloClient != null) {
|
||||
try {
|
||||
Authorizations currentAccumuloAuths = accumuloClient.securityOperations()
|
||||
.getUserAuthorizations(accumuloClient.whoami());
|
||||
currentAccumuloAuths.iterator()
|
||||
.forEachRemaining(a -> accumuloAuths.add(new String(a, Charset.forName("UTF-8"))));
|
||||
Authorizations currentAccumuloAuths = accumuloClient.securityOperations().getUserAuthorizations(accumuloClient.whoami());
|
||||
currentAccumuloAuths.iterator().forEachRemaining(a -> accumuloAuths.add(new String(a, Charset.forName("UTF-8"))));
|
||||
} catch (Exception e) {
|
||||
logger.error(e.getMessage(), e);
|
||||
}
|
||||
@@ -81,16 +79,15 @@ public class JWTTokenHandler {
|
||||
logger.trace("Resulting claims: {}", claims);
|
||||
List<?> principalsClaim = claims.get(claimName, List.class);
|
||||
if (principalsClaim == null || principalsClaim.isEmpty()) {
|
||||
throw new IllegalArgumentException(
|
||||
"JWT for " + claims.getSubject() + " does not contain any proxied principals.");
|
||||
throw new IllegalArgumentException("JWT for " + claims.getSubject() + " does not contain any proxied principals.");
|
||||
}
|
||||
// convert to TimelyUser and downgrade auths to contain only what the
|
||||
// accumuloUser has
|
||||
return principalsClaim.stream().map(obj -> objectMapper.convertValue(obj, TimelyUser.class)).map(u -> {
|
||||
Collection<String> intersectedAuths = new ArrayList<>(u.getAuths());
|
||||
intersectedAuths.removeIf(a -> !accumuloAuths.contains(a));
|
||||
return new TimelyUser(u.getDn(), u.getUserType(), u.getEmail(), intersectedAuths, u.getRoles(),
|
||||
u.getRoleToAuthMapping(), u.getCreationTime(), u.getExpirationTime());
|
||||
return new TimelyUser(u.getDn(), u.getUserType(), u.getEmail(), intersectedAuths, u.getRoles(), u.getRoleToAuthMapping(), u.getCreationTime(),
|
||||
u.getExpirationTime());
|
||||
|
||||
}).collect(Collectors.toList());
|
||||
}
|
||||
|
||||
@@ -5,7 +5,6 @@ import static timely.auth.AuthenticationService.PRINCIPALS_CLAIM;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
|
||||
import io.netty.handler.ssl.SslContext;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.http.client.reactive.ClientHttpConnector;
|
||||
@@ -16,6 +15,8 @@ import org.springframework.security.core.userdetails.UserDetails;
|
||||
import org.springframework.security.core.userdetails.UsernameNotFoundException;
|
||||
import org.springframework.security.web.authentication.preauth.PreAuthenticatedAuthenticationToken;
|
||||
import org.springframework.web.reactive.function.client.WebClient;
|
||||
|
||||
import io.netty.handler.ssl.SslContext;
|
||||
import reactor.netty.http.client.HttpClient;
|
||||
import timely.auth.util.SslHelper;
|
||||
import timely.configuration.AuthorizationProperties;
|
||||
@@ -34,8 +35,7 @@ public class RemoteUserDetailsService implements AuthenticationUserDetailsServic
|
||||
|
||||
}
|
||||
|
||||
public RemoteUserDetailsService(ClientSsl clientSsl, AuthorizationProperties authorizationProperties)
|
||||
throws Exception {
|
||||
public RemoteUserDetailsService(ClientSsl clientSsl, AuthorizationProperties authorizationProperties) throws Exception {
|
||||
this.clientSsl = clientSsl;
|
||||
this.authorizationProperties = authorizationProperties;
|
||||
initialize();
|
||||
@@ -46,8 +46,7 @@ public class RemoteUserDetailsService implements AuthenticationUserDetailsServic
|
||||
SslContext sslContext = SslHelper.getSslContext(clientSsl);
|
||||
HttpClient httpClient = HttpClient.create().secure(sslContextSpec -> sslContextSpec.sslContext(sslContext));
|
||||
ClientHttpConnector httpConnector = new ReactorClientHttpConnector(httpClient);
|
||||
this.webClient = webClientBuilder.clientConnector(httpConnector)
|
||||
.baseUrl(authorizationProperties.getAuthorizationUrl()).build();
|
||||
this.webClient = webClientBuilder.clientConnector(httpConnector).baseUrl(authorizationProperties.getAuthorizationUrl()).build();
|
||||
}
|
||||
|
||||
public void setClientSsl(ClientSsl clientSsl) {
|
||||
@@ -78,8 +77,7 @@ public class RemoteUserDetailsService implements AuthenticationUserDetailsServic
|
||||
.bodyToMono(String.class)
|
||||
.block();
|
||||
// @formatter:on
|
||||
log.trace("Authenticated {} via the authorization microservice with {} result", subjectDN,
|
||||
jwt == null ? "null" : "non-null");
|
||||
log.trace("Authenticated {} via the authorization microservice with {} result", subjectDN, jwt == null ? "null" : "non-null");
|
||||
if (jwt != null) {
|
||||
Collection<TimelyUser> timelyUsers = JWTTokenHandler.createUsersFromToken(jwt, PRINCIPALS_CLAIM);
|
||||
if (timelyUsers.iterator().hasNext()) {
|
||||
|
||||
@@ -5,7 +5,6 @@ import java.util.Iterator;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import com.google.common.collect.Lists;
|
||||
import org.apache.accumulo.core.client.IteratorSetting;
|
||||
import org.apache.accumulo.core.client.IteratorSetting.Column;
|
||||
import org.apache.accumulo.core.client.ScannerBase;
|
||||
@@ -17,9 +16,10 @@ import org.apache.accumulo.core.dataImpl.thrift.IterInfo;
|
||||
import org.apache.accumulo.core.security.Authorizations;
|
||||
import org.apache.hadoop.io.Text;
|
||||
|
||||
import com.google.common.collect.Lists;
|
||||
|
||||
/**
|
||||
* A simple wrapper around a {@link ScannerBase} that overrides the methods that
|
||||
* configure iterators.
|
||||
* A simple wrapper around a {@link ScannerBase} that overrides the methods that configure iterators.
|
||||
*
|
||||
* @see ScannerOptionsHelper
|
||||
*/
|
||||
@@ -37,21 +37,16 @@ public class ScannerBaseDelegate implements ScannerBase {
|
||||
@Override
|
||||
public void addScanIterator(IteratorSetting cfg) {
|
||||
if (cfg.getName().startsWith(SYSTEM_ITERATOR_NAME_PREFIX)) {
|
||||
throw new IllegalArgumentException(
|
||||
"Non-system iterators' names cannot start with " + SYSTEM_ITERATOR_NAME_PREFIX);
|
||||
throw new IllegalArgumentException("Non-system iterators' names cannot start with " + SYSTEM_ITERATOR_NAME_PREFIX);
|
||||
} else {
|
||||
delegate.addScanIterator(cfg);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds a "system" scan iterator. The iterator name is automatically prefixed
|
||||
* with {@link #SYSTEM_ITERATOR_NAME_PREFIX}. A "system" scan iterator can only
|
||||
* be modified or removed by calling
|
||||
* {@link #updateSystemScanIteratorOption(String, String, String)},
|
||||
* {@link #removeSystemScanIterator(String)}, or
|
||||
* {@link #clearSystemScanIterators()}. Updates the iterator configuration for
|
||||
* {@code iteratorName}. The iterator name is automatically prefixed with
|
||||
* Adds a "system" scan iterator. The iterator name is automatically prefixed with {@link #SYSTEM_ITERATOR_NAME_PREFIX}. A "system" scan iterator can only
|
||||
* be modified or removed by calling {@link #updateSystemScanIteratorOption(String, String, String)}, {@link #removeSystemScanIterator(String)}, or
|
||||
* {@link #clearSystemScanIterators()}. Updates the iterator configuration for {@code iteratorName}. The iterator name is automatically prefixed with
|
||||
* {@link #SYSTEM_ITERATOR_NAME_PREFIX}.
|
||||
*
|
||||
* @param cfg
|
||||
@@ -74,8 +69,7 @@ public class ScannerBaseDelegate implements ScannerBase {
|
||||
}
|
||||
|
||||
/**
|
||||
* Removes a "system" scan iterator. The iterator name is automatically prefixed
|
||||
* with {@link #SYSTEM_ITERATOR_NAME_PREFIX}.
|
||||
* Removes a "system" scan iterator. The iterator name is automatically prefixed with {@link #SYSTEM_ITERATOR_NAME_PREFIX}.
|
||||
*
|
||||
* @param iteratorName
|
||||
* the name of the system iterator to remove
|
||||
@@ -97,8 +91,7 @@ public class ScannerBaseDelegate implements ScannerBase {
|
||||
}
|
||||
|
||||
/**
|
||||
* Updates the iterator configuration for {@code iteratorName}. The iterator
|
||||
* name is automatically prefixed with {@link #SYSTEM_ITERATOR_NAME_PREFIX}.
|
||||
* Updates the iterator configuration for {@code iteratorName}. The iterator name is automatically prefixed with {@link #SYSTEM_ITERATOR_NAME_PREFIX}.
|
||||
*
|
||||
* @param iteratorName
|
||||
* the name of the system iterator to modify
|
||||
@@ -145,8 +138,7 @@ public class ScannerBaseDelegate implements ScannerBase {
|
||||
}
|
||||
}
|
||||
} else {
|
||||
throw new UnsupportedOperationException(
|
||||
"Cannot clear scan iterators on a non-ScannerOptions class! (" + delegate.getClass() + ")");
|
||||
throw new UnsupportedOperationException("Cannot clear scan iterators on a non-ScannerOptions class! (" + delegate.getClass() + ")");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -158,7 +150,7 @@ public class ScannerBaseDelegate implements ScannerBase {
|
||||
}
|
||||
|
||||
@Override
|
||||
public Iterator<Map.Entry<Key, Value>> iterator() {
|
||||
public Iterator<Map.Entry<Key,Value>> iterator() {
|
||||
return delegate.iterator();
|
||||
}
|
||||
|
||||
@@ -253,8 +245,7 @@ public class ScannerBaseDelegate implements ScannerBase {
|
||||
public Collection<IteratorSetting> getIterators() {
|
||||
Collection<IteratorSetting> settings = Lists.newArrayList();
|
||||
for (IterInfo iter : serverSideIteratorList) {
|
||||
IteratorSetting setting = new IteratorSetting(iter.getPriority(), iter.getIterName(),
|
||||
iter.getClassName());
|
||||
IteratorSetting setting = new IteratorSetting(iter.getPriority(), iter.getIterName(), iter.getClassName());
|
||||
setting.addOptions(serverSideIteratorOptions.get(iter.getIterName()));
|
||||
settings.add(setting);
|
||||
}
|
||||
|
||||
@@ -5,11 +5,11 @@ import java.io.Serializable;
|
||||
import com.fasterxml.jackson.annotation.JsonCreator;
|
||||
import com.fasterxml.jackson.annotation.JsonGetter;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
|
||||
import timely.auth.util.ProxiedEntityUtils;
|
||||
|
||||
/**
|
||||
* A simple pair containing a subject and (optional) issuer DN. The supplied DN
|
||||
* values are normalized into a lower-case form with the CN portion first.
|
||||
* A simple pair containing a subject and (optional) issuer DN. The supplied DN values are normalized into a lower-case form with the CN portion first.
|
||||
*/
|
||||
public class SubjectIssuerDNPair implements Serializable {
|
||||
|
||||
@@ -22,8 +22,7 @@ public class SubjectIssuerDNPair implements Serializable {
|
||||
}
|
||||
|
||||
@JsonCreator
|
||||
public static SubjectIssuerDNPair of(@JsonProperty("subjectDN") String subjectDN,
|
||||
@JsonProperty("issuerDN") String issuerDN) {
|
||||
public static SubjectIssuerDNPair of(@JsonProperty("subjectDN") String subjectDN, @JsonProperty("issuerDN") String issuerDN) {
|
||||
return new SubjectIssuerDNPair(subjectDN, issuerDN);
|
||||
}
|
||||
|
||||
|
||||
@@ -19,13 +19,11 @@ import javax.xml.bind.annotation.XmlType;
|
||||
import timely.auth.util.DnUtils;
|
||||
|
||||
/**
|
||||
* A {@link Principal} that represents a set of proxied {@link TimelyUser}s. For
|
||||
* example, this proxied user could represent a GUI server acting on behalf of a
|
||||
* user. The GUI server user represents the entity that made the call to us and
|
||||
* the other proxied user would be the actual end user.
|
||||
* A {@link Principal} that represents a set of proxied {@link TimelyUser}s. For example, this proxied user could represent a GUI server acting on behalf of a
|
||||
* user. The GUI server user represents the entity that made the call to us and the other proxied user would be the actual end user.
|
||||
*/
|
||||
@XmlRootElement
|
||||
@XmlType(factoryMethod = "anonymousPrincipal", propOrder = { "name", "proxiedUsers", "creationTime" })
|
||||
@XmlType(factoryMethod = "anonymousPrincipal", propOrder = {"name", "proxiedUsers", "creationTime"})
|
||||
@XmlAccessorType(XmlAccessType.NONE)
|
||||
public class TimelyPrincipal implements Principal, Serializable {
|
||||
|
||||
@@ -38,8 +36,7 @@ public class TimelyPrincipal implements Principal, Serializable {
|
||||
private final long creationTime;
|
||||
|
||||
/**
|
||||
* This constructor should not be used. It is here to allow JAX-B mapping and
|
||||
* CDI proxying of this class.
|
||||
* This constructor should not be used. It is here to allow JAX-B mapping and CDI proxying of this class.
|
||||
*/
|
||||
public TimelyPrincipal() {
|
||||
this(ANONYMOUS_USER);
|
||||
@@ -59,11 +56,9 @@ public class TimelyPrincipal implements Principal, Serializable {
|
||||
public TimelyPrincipal(Collection<TimelyUser> proxiedUsers, long creationTime) {
|
||||
this.creationTime = creationTime;
|
||||
TimelyUser first = proxiedUsers.stream().findFirst().orElse(null);
|
||||
this.primaryUser = proxiedUsers.stream().filter(u -> u.getUserType() == TimelyUser.UserType.USER).findFirst()
|
||||
.orElse(first);
|
||||
this.primaryUser = proxiedUsers.stream().filter(u -> u.getUserType() == TimelyUser.UserType.USER).findFirst().orElse(first);
|
||||
this.proxiedUsers.add(primaryUser);
|
||||
this.proxiedUsers
|
||||
.addAll(proxiedUsers.stream().filter(u -> !u.equals(primaryUser)).collect(Collectors.toList()));
|
||||
this.proxiedUsers.addAll(proxiedUsers.stream().filter(u -> !u.equals(primaryUser)).collect(Collectors.toList()));
|
||||
this.username = this.proxiedUsers.stream().map(TimelyUser::getName).collect(Collectors.joining(" -> "));
|
||||
}
|
||||
|
||||
@@ -76,8 +71,7 @@ public class TimelyPrincipal implements Principal, Serializable {
|
||||
}
|
||||
|
||||
public Collection<? extends Collection<String>> getAuthorizations() {
|
||||
return Collections
|
||||
.unmodifiableCollection(proxiedUsers.stream().map(TimelyUser::getAuths).collect(Collectors.toList()));
|
||||
return Collections.unmodifiableCollection(proxiedUsers.stream().map(TimelyUser::getAuths).collect(Collectors.toList()));
|
||||
}
|
||||
|
||||
public String[] getDNs() {
|
||||
@@ -145,7 +139,6 @@ public class TimelyPrincipal implements Principal, Serializable {
|
||||
}
|
||||
|
||||
public String getAuthorizationsString() {
|
||||
return proxiedUsers.stream().map(TimelyUser::getAuths).map(a -> a.toString())
|
||||
.collect(Collectors.joining(" -> ", "[", "]"));
|
||||
return proxiedUsers.stream().map(TimelyUser::getAuths).map(a -> a.toString()).collect(Collectors.joining(" -> ", "[", "]"));
|
||||
}
|
||||
}
|
||||
@@ -11,12 +11,12 @@ import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import com.google.common.collect.LinkedHashMultimap;
|
||||
import com.google.common.collect.Multimap;
|
||||
import com.google.common.collect.Multimaps;
|
||||
|
||||
import timely.auth.util.DnUtils;
|
||||
import timely.auth.util.ProxiedEntityUtils;
|
||||
|
||||
/**
|
||||
* A user of a TIMELY service. Typically, one or more of these users (a chain
|
||||
* where a user called an intermediate service which in turn called us) is
|
||||
* A user of a TIMELY service. Typically, one or more of these users (a chain where a user called an intermediate service which in turn called us) is
|
||||
* represented with a TimelyPrincipal.
|
||||
*/
|
||||
public class TimelyUser implements Serializable {
|
||||
@@ -27,8 +27,7 @@ public class TimelyUser implements Serializable {
|
||||
USER, SERVER
|
||||
}
|
||||
|
||||
public static final TimelyUser ANONYMOUS_USER = new TimelyUser(SubjectIssuerDNPair.of("ANONYMOUS"), UserType.USER,
|
||||
null, null, null, -1L);
|
||||
public static final TimelyUser ANONYMOUS_USER = new TimelyUser(SubjectIssuerDNPair.of("ANONYMOUS"), UserType.USER, null, null, null, -1L);
|
||||
private final String name;
|
||||
private final String commonName;
|
||||
private final String email;
|
||||
@@ -39,7 +38,7 @@ public class TimelyUser implements Serializable {
|
||||
private final Collection<String> unmodifiableAuths;
|
||||
private final Collection<String> roles;
|
||||
private final Collection<String> unmodifiableRoles;
|
||||
private final Multimap<String, String> roleToAuthMapping;
|
||||
private final Multimap<String,String> roleToAuthMapping;
|
||||
private final long creationTime;
|
||||
private final long expirationTime;
|
||||
|
||||
@@ -66,27 +65,25 @@ public class TimelyUser implements Serializable {
|
||||
* For testing
|
||||
*/
|
||||
public TimelyUser(String subjectDn, Collection<String> auths) {
|
||||
this(SubjectIssuerDNPair.of(subjectDn), UserType.USER, null, auths, null, null, System.currentTimeMillis(),
|
||||
-1L);
|
||||
this(SubjectIssuerDNPair.of(subjectDn), UserType.USER, null, auths, null, null, System.currentTimeMillis(), -1L);
|
||||
}
|
||||
|
||||
public TimelyUser(SubjectIssuerDNPair dn, UserType userType, Collection<String> auths, Collection<String> roles,
|
||||
Multimap<String, String> roleToAuthMapping, long creationTime) {
|
||||
public TimelyUser(SubjectIssuerDNPair dn, UserType userType, Collection<String> auths, Collection<String> roles, Multimap<String,String> roleToAuthMapping,
|
||||
long creationTime) {
|
||||
this(dn, userType, null, auths, roles, roleToAuthMapping, creationTime, -1L);
|
||||
}
|
||||
|
||||
public TimelyUser(SubjectIssuerDNPair dn, UserType userType, String email, Collection<String> auths,
|
||||
Collection<String> roles, Multimap<String, String> roleToAuthMapping, long creationTime) {
|
||||
public TimelyUser(SubjectIssuerDNPair dn, UserType userType, String email, Collection<String> auths, Collection<String> roles,
|
||||
Multimap<String,String> roleToAuthMapping, long creationTime) {
|
||||
this(dn, userType, email, auths, roles, roleToAuthMapping, creationTime, -1L);
|
||||
}
|
||||
|
||||
@JsonCreator
|
||||
public TimelyUser(@JsonProperty(value = "dn", required = true) SubjectIssuerDNPair dn,
|
||||
@JsonProperty(value = "userType", required = true) UserType userType, @JsonProperty("email") String email,
|
||||
@JsonProperty("auths") Collection<String> auths, @JsonProperty("roles") Collection<String> roles,
|
||||
@JsonProperty("roleToAuthMapping") Multimap<String, String> roleToAuthMapping,
|
||||
@JsonProperty(value = "creationTime", defaultValue = "-1L") long creationTime,
|
||||
@JsonProperty(value = "expirationTime", defaultValue = "-1L") long expirationTime) {
|
||||
public TimelyUser(@JsonProperty(value = "dn", required = true) SubjectIssuerDNPair dn, @JsonProperty(value = "userType", required = true) UserType userType,
|
||||
@JsonProperty("email") String email, @JsonProperty("auths") Collection<String> auths, @JsonProperty("roles") Collection<String> roles,
|
||||
@JsonProperty("roleToAuthMapping") Multimap<String,String> roleToAuthMapping,
|
||||
@JsonProperty(value = "creationTime", defaultValue = "-1L") long creationTime,
|
||||
@JsonProperty(value = "expirationTime", defaultValue = "-1L") long expirationTime) {
|
||||
this.name = dn.toString();
|
||||
this.commonName = DnUtils.getCommonName(dn.subjectDN());
|
||||
this.login = ProxiedEntityUtils.getShortName(dn.subjectDN());
|
||||
@@ -98,7 +95,7 @@ public class TimelyUser implements Serializable {
|
||||
this.roles = roles == null ? Collections.emptyList() : new LinkedHashSet<>(roles);
|
||||
this.unmodifiableRoles = Collections.unmodifiableCollection(this.roles);
|
||||
this.roleToAuthMapping = roleToAuthMapping == null ? LinkedHashMultimap.create()
|
||||
: Multimaps.unmodifiableMultimap(LinkedHashMultimap.create(roleToAuthMapping));
|
||||
: Multimaps.unmodifiableMultimap(LinkedHashMultimap.create(roleToAuthMapping));
|
||||
this.creationTime = creationTime;
|
||||
this.expirationTime = expirationTime;
|
||||
}
|
||||
@@ -136,7 +133,7 @@ public class TimelyUser implements Serializable {
|
||||
return unmodifiableRoles;
|
||||
}
|
||||
|
||||
public Multimap<String, String> getRoleToAuthMapping() {
|
||||
public Multimap<String,String> getRoleToAuthMapping() {
|
||||
return roleToAuthMapping;
|
||||
}
|
||||
|
||||
@@ -157,8 +154,7 @@ public class TimelyUser implements Serializable {
|
||||
|
||||
TimelyUser that = (TimelyUser) o;
|
||||
|
||||
return creationTime == that.creationTime && dn.equals(that.dn) && userType == that.userType
|
||||
&& auths.equals(that.auths) && roles.equals(that.roles);
|
||||
return creationTime == that.creationTime && dn.equals(that.dn) && userType == that.userType && auths.equals(that.auths) && roles.equals(that.roles);
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -173,7 +169,7 @@ public class TimelyUser implements Serializable {
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "TimelyUser{" + "name='" + getName() + "'" + ", userType=" + getUserType() + ", auths=" + getAuths()
|
||||
+ ", roles=" + getRoles() + ", creationTime=" + getCreationTime() + "}";
|
||||
return "TimelyUser{" + "name='" + getName() + "'" + ", userType=" + getUserType() + ", auths=" + getAuths() + ", roles=" + getRoles()
|
||||
+ ", creationTime=" + getCreationTime() + "}";
|
||||
}
|
||||
}
|
||||
@@ -2,22 +2,23 @@ package timely.auth;
|
||||
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import org.apache.accumulo.core.security.ColumnVisibility;
|
||||
|
||||
import com.github.benmanes.caffeine.cache.Cache;
|
||||
import com.github.benmanes.caffeine.cache.Caffeine;
|
||||
import org.apache.accumulo.core.security.ColumnVisibility;
|
||||
|
||||
import timely.configuration.Configuration;
|
||||
|
||||
public class VisibilityCache {
|
||||
|
||||
private static volatile Cache<String, ColumnVisibility> CACHE = null;
|
||||
private static volatile Cache<String,ColumnVisibility> CACHE = null;
|
||||
|
||||
public static synchronized void init(Configuration config) {
|
||||
if (CACHE == null) {
|
||||
long expireMinutes = config.getVisibilityCache().getExpirationMinutes();
|
||||
int initialCapacity = config.getVisibilityCache().getInitialCapacity();
|
||||
long maxCapacity = config.getVisibilityCache().getMaxCapacity();
|
||||
CACHE = Caffeine.newBuilder().expireAfterAccess(expireMinutes, TimeUnit.MINUTES)
|
||||
.initialCapacity(initialCapacity).maximumSize(maxCapacity).build();
|
||||
CACHE = Caffeine.newBuilder().expireAfterAccess(expireMinutes, TimeUnit.MINUTES).initialCapacity(initialCapacity).maximumSize(maxCapacity).build();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -9,10 +9,12 @@ import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.apache.accumulo.core.security.Authorizations;
|
||||
|
||||
import com.google.common.base.Splitter;
|
||||
import com.google.common.collect.Lists;
|
||||
import com.google.common.collect.Sets;
|
||||
import org.apache.accumulo.core.security.Authorizations;
|
||||
|
||||
import timely.auth.AuthorizationsMinimizer;
|
||||
|
||||
public class AuthorizationsUtil {
|
||||
@@ -31,13 +33,11 @@ public class AuthorizationsUtil {
|
||||
}
|
||||
|
||||
public static Authorizations toAuthorizations(Collection<String> auths) {
|
||||
return new Authorizations(auths.stream().map(String::trim).map(s -> s.getBytes(Charset.forName("UTF-8")))
|
||||
.collect(Collectors.toList()));
|
||||
return new Authorizations(auths.stream().map(String::trim).map(s -> s.getBytes(Charset.forName("UTF-8"))).collect(Collectors.toList()));
|
||||
}
|
||||
|
||||
public static List<String> splitAuths(String requestedAuths) {
|
||||
return Splitter.on(',').omitEmptyStrings().trimResults().splitToStream(requestedAuths)
|
||||
.collect(Collectors.toList());
|
||||
return Splitter.on(',').omitEmptyStrings().trimResults().splitToStream(requestedAuths).collect(Collectors.toList());
|
||||
}
|
||||
|
||||
public static Set<Authorizations> buildAuthorizations(Collection<? extends Collection<String>> userAuths) {
|
||||
|
||||
@@ -11,10 +11,11 @@ import java.util.List;
|
||||
import java.util.Properties;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import io.netty.handler.codec.http.QueryStringDecoder;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import io.netty.handler.codec.http.QueryStringDecoder;
|
||||
|
||||
public class DnUtils {
|
||||
|
||||
/** Config for injecting NPE OU identifiers */
|
||||
@@ -45,8 +46,7 @@ public class DnUtils {
|
||||
}
|
||||
}
|
||||
}
|
||||
String subjectDnPattern = System.getProperty(SUBJECT_DN_PATTERN_PROPERTY,
|
||||
PROPS.getProperty(SUBJECT_DN_PATTERN_PROPERTY));
|
||||
String subjectDnPattern = System.getProperty(SUBJECT_DN_PATTERN_PROPERTY, PROPS.getProperty(SUBJECT_DN_PATTERN_PROPERTY));
|
||||
try {
|
||||
if (null == subjectDnPattern || subjectDnPattern.isEmpty()) {
|
||||
throw new IllegalStateException(SUBJECT_DN_PATTERN_PROPERTY + " property value cannot be null");
|
||||
@@ -70,8 +70,7 @@ public class DnUtils {
|
||||
return ProxiedEntityUtils.buildProxiedDN(dns);
|
||||
}
|
||||
|
||||
public static Collection<String> buildNormalizedDNList(String subjectDN, String issuerDN, String proxiedSubjectDNs,
|
||||
String proxiedIssuerDNs) {
|
||||
public static Collection<String> buildNormalizedDNList(String subjectDN, String issuerDN, String proxiedSubjectDNs, String proxiedIssuerDNs) {
|
||||
subjectDN = normalizeDN(subjectDN);
|
||||
issuerDN = normalizeDN(issuerDN);
|
||||
List<String> dnList = new ArrayList<>();
|
||||
@@ -82,14 +81,12 @@ public class DnUtils {
|
||||
dnList.add(issuerDN.replaceAll("(?<!\\\\)([<>])", "\\\\$1"));
|
||||
if (proxiedSubjectDNs != null) {
|
||||
if (proxiedIssuerDNs == null)
|
||||
throw new IllegalArgumentException(
|
||||
"If proxied subject DNs are supplied, then issuer DNs must be supplied as well.");
|
||||
throw new IllegalArgumentException("If proxied subject DNs are supplied, then issuer DNs must be supplied as well.");
|
||||
String[] subjectDNarray = splitProxiedDNs(proxiedSubjectDNs, true);
|
||||
String[] issuerDNarray = splitProxiedDNs(proxiedIssuerDNs, true);
|
||||
if (subjectDNarray.length != issuerDNarray.length)
|
||||
throw new IllegalArgumentException(
|
||||
"Subject and issuer DN lists do not have the same number of entries: "
|
||||
+ Arrays.toString(subjectDNarray) + " vs " + Arrays.toString(issuerDNarray));
|
||||
throw new IllegalArgumentException("Subject and issuer DN lists do not have the same number of entries: " + Arrays.toString(subjectDNarray)
|
||||
+ " vs " + Arrays.toString(issuerDNarray));
|
||||
for (int i = 0; i < subjectDNarray.length; ++i) {
|
||||
subjectDNarray[i] = normalizeDN(subjectDNarray[i]);
|
||||
if (!subjects.contains(subjectDNarray[i])) {
|
||||
@@ -98,19 +95,16 @@ public class DnUtils {
|
||||
dnList.add(subjectDNarray[i]);
|
||||
dnList.add(issuerDNarray[i]);
|
||||
if (issuerDNarray[i].equalsIgnoreCase(subjectDNarray[i]))
|
||||
throw new IllegalArgumentException(
|
||||
"Subject DN " + issuerDNarray[i] + " was passed as an issuer DN.");
|
||||
throw new IllegalArgumentException("Subject DN " + issuerDNarray[i] + " was passed as an issuer DN.");
|
||||
if (SUBJECT_DN_PATTERN.matcher(issuerDNarray[i]).find())
|
||||
throw new IllegalArgumentException(
|
||||
"It appears that a subject DN (" + issuerDNarray[i] + ") was passed as an issuer DN.");
|
||||
throw new IllegalArgumentException("It appears that a subject DN (" + issuerDNarray[i] + ") was passed as an issuer DN.");
|
||||
}
|
||||
}
|
||||
}
|
||||
return dnList;
|
||||
}
|
||||
|
||||
public static String buildNormalizedProxyDN(String subjectDN, String issuerDN, String proxiedSubjectDNs,
|
||||
String proxiedIssuerDNs) {
|
||||
public static String buildNormalizedProxyDN(String subjectDN, String issuerDN, String proxiedSubjectDNs, String proxiedIssuerDNs) {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
for (String escapedDN : buildNormalizedDNList(subjectDN, issuerDN, proxiedSubjectDNs, proxiedIssuerDNs)) {
|
||||
if (sb.length() == 0)
|
||||
@@ -162,9 +156,8 @@ public class DnUtils {
|
||||
}
|
||||
|
||||
/**
|
||||
* Encapsulates the known/valid NPE OU list, and whatever else is needed for NPE
|
||||
* handling. Static inner-class, so that injection of the configured OUs is only
|
||||
* performed on-demand/if needed
|
||||
* Encapsulates the known/valid NPE OU list, and whatever else is needed for NPE handling. Static inner-class, so that injection of the configured OUs is
|
||||
* only performed on-demand/if needed
|
||||
*/
|
||||
public static class NpeUtils {
|
||||
|
||||
@@ -174,8 +167,7 @@ public class DnUtils {
|
||||
/** Parsed NPE OU identifiers */
|
||||
static final List<String> NPE_OU_LIST;
|
||||
|
||||
private NpeUtils() {
|
||||
}
|
||||
private NpeUtils() {}
|
||||
|
||||
static {
|
||||
List<String> npeOUs = new ArrayList<>();
|
||||
|
||||
@@ -4,14 +4,15 @@ import java.util.*;
|
||||
|
||||
import com.google.common.collect.HashMultimap;
|
||||
import com.google.common.collect.Multimap;
|
||||
|
||||
import io.netty.handler.codec.http.HttpHeaders;
|
||||
|
||||
public class HttpHeaderUtils {
|
||||
|
||||
public static Multimap<String, String> toMultimap(HttpHeaders headers) {
|
||||
Multimap<String, String> headerMultimap = HashMultimap.create();
|
||||
public static Multimap<String,String> toMultimap(HttpHeaders headers) {
|
||||
Multimap<String,String> headerMultimap = HashMultimap.create();
|
||||
if (headers != null) {
|
||||
for (Map.Entry<String, String> e : headers.entries()) {
|
||||
for (Map.Entry<String,String> e : headers.entries()) {
|
||||
// lower case for HTTP/2 compatibility; even for HTTP/1, these were always
|
||||
// case-insensitive
|
||||
headerMultimap.put(e.getKey().toLowerCase(), e.getValue());
|
||||
@@ -20,8 +21,7 @@ public class HttpHeaderUtils {
|
||||
return headerMultimap;
|
||||
}
|
||||
|
||||
public static String getSingleHeader(Multimap<String, String> headers, String headerName, boolean enforceOneValue)
|
||||
throws IllegalArgumentException {
|
||||
public static String getSingleHeader(Multimap<String,String> headers, String headerName, boolean enforceOneValue) throws IllegalArgumentException {
|
||||
if (headers == null) {
|
||||
return null;
|
||||
} else {
|
||||
|
||||
Loaded 100 of 254 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user