Compare commits

..
298 changed files with 13489 additions and 8135 deletions

No files matched your search

+1 -1
View File
@@ -4,7 +4,7 @@
<parent>
<groupId>timely</groupId>
<artifactId>timely</artifactId>
<version>2.1.1</version>
<version>2.1.0-SNAPSHOT</version>
</parent>
<artifactId>timely-analytics</artifactId>
<!--
+40
View File
@@ -0,0 +1,40 @@
#!/bin/bash
if [[ `uname` == "Darwin" ]]; then
THIS_SCRIPT=`python -c 'import os,sys; print os.path.realpath(sys.argv[1])' $0`
else
THIS_SCRIPT=`readlink -f $0`
fi
THIS_DIR="${THIS_SCRIPT%/*}"
BASE_DIR=$(cd $THIS_DIR/.. && pwd)
BIN_DIR=${BASE_DIR}/bin
CONF_DIR=${BASE_DIR}/conf
. ${BIN_DIR}/timely-balancer-env.sh
for i in $(seq 1 $NUM_PROFILES); do
if [ "$NUM_PROFILES" -eq "1" ]; then
FILENAME=timely-balancer.yml
else
FILENAME=timely-balancer-${i}.yml
fi
echo "creating configuration file ${CONF_DIR}/${FILENAME}"
cp ${CONF_DIR}/timely-balancer-template.yml ${CONF_DIR}/${FILENAME}
# if [ "$NUM_PROFILES" -eq "1" ]; then
# sed -i "s/\${INSTANCE}//g" ${CONF_DIR}/${FILENAME}
# else
# sed -i "s/\${INSTANCE}/${i}/g" ${CONF_DIR}/${FILENAME}
# fi
MULTIPLE=$(($i - 1))
TCP_PORT=$(($TCP_PORT_START + $MULTIPLE * $PORT_INCREMENT))
sed -i "s/\${TCP_PORT}/${TCP_PORT}/g" ${CONF_DIR}/${FILENAME}
HTTP_PORT=$(($HTTP_PORT_START + $MULTIPLE * $PORT_INCREMENT))
sed -i "s/\${HTTP_PORT}/${HTTP_PORT}/g" ${CONF_DIR}/${FILENAME}
WEBSOCKET_PORT=$(($WEBSOCKET_PORT_START + $MULTIPLE * $PORT_INCREMENT))
sed -i "s/\${WEBSOCKET_PORT}/${WEBSOCKET_PORT}/g" ${CONF_DIR}/${FILENAME}
UDP_PORT=$(($UDP_PORT_START + $MULTIPLE * $PORT_INCREMENT))
sed -i "s/\${UDP_PORT}/${UDP_PORT}/g" ${CONF_DIR}/${FILENAME}
done
+9 -4
View File
@@ -12,7 +12,12 @@ BIN_DIR=${BASE_DIR}/bin
. ${BIN_DIR}/timely-balancer-env.sh
for i in $(seq 1 $NUM_INSTANCES); do
echo ${BIN_DIR}/timely-balancer.sh $i
${BIN_DIR}/timely-balancer.sh $i
done
if [ "$NUM_PROFILES" -eq "1" ]; then
echo ${BIN_DIR}/timely-balancer.sh
${BIN_DIR}/timely-balancer.sh
else
for i in $(seq 1 $NUM_PROFILES); do
echo ${BIN_DIR}/timely-balancer.sh $i
${BIN_DIR}/timely-balancer.sh $i
done
fi
+1 -19
View File
@@ -1,21 +1,3 @@
#!/bin/bash
if [[ `uname` == "Darwin" ]]; then
THIS_SCRIPT=`python -c 'import os,sys; print os.path.realpath(sys.argv[1])' $0`
else
THIS_SCRIPT=`readlink -f $0`
fi
THIS_DIR="${THIS_SCRIPT%/*}"
BASE_DIR=$(cd $THIS_DIR/.. && pwd)
BIN_DIR=${BASE_DIR}/bin
. ${BIN_DIR}/timely-balancer-env.sh
for i in $(seq 1 $NUM_INSTANCES); do
PID=`jps -m | grep -E 'timely-balancer-.*-exec.jar' | grep "instance=${i}" | awk '{print $1}'`
if [ "$PID" != "" ]; then
echo "stopping timely-balancer instance ${i} [${PID}]"
kill -9 $PID
fi
done
pkill -9 -f 'timely-balancer-.*-exec.jar'
File diff suppressed because it is too large. Load diff
+11
View File
@@ -0,0 +1,11 @@
#!/bin/bash
export TRUSTSTORE_FILE=
export TRUSTSTORE_TYPE="jks";
export TRUSTSTORE_PASS=
export KEYSTORE_FILE=
export KEYSTORE_TYPE="pkcs12";
export KEYSTORE_PASS=
export NUM_QUERY_THREADS=4
export NUM_INGEST_THREADS=4
+21
View File
@@ -0,0 +1,21 @@
#!/bin/bash
if [[ `uname` == "Darwin" ]]; then
THIS_SCRIPT=`python -c 'import os,sys; print os.path.realpath(sys.argv[1])' $0`
else
THIS_SCRIPT=`readlink -f $0`
fi
THIS_DIR="${THIS_SCRIPT%/*}"
BASE_DIR=${THIS_DIR}/../..
TMP_DIR="${BASE_DIR}/tmp"
CONF_DIR="${BASE_DIR}/conf"
LIB_DIR="${BASE_DIR}/lib"
. ${THIS_DIR}/load-test-env.sh
export CLASSPATH="${CONF_DIR}:${THIS_DIR}:${LIB_DIR}/*"
JVM_ARGS="-Xmx1G -Xms1G"
echo "$JAVA_HOME/bin/java ${JVM_ARGS} timely.testing.MetricPutLoadTest $*"
$JAVA_HOME/bin/java ${JVM_ARGS} timely.testing.MetricPutLoadTest $*
+21
View File
@@ -0,0 +1,21 @@
#!/bin/bash
if [[ `uname` == "Darwin" ]]; then
THIS_SCRIPT=`python -c 'import os,sys; print os.path.realpath(sys.argv[1])' $0`
else
THIS_SCRIPT=`readlink -f $0`
fi
THIS_DIR="${THIS_SCRIPT%/*}"
BASE_DIR=${THIS_DIR}/../..
TMP_DIR="${BASE_DIR}/tmp"
CONF_DIR="${BASE_DIR}/conf"
LIB_DIR="${BASE_DIR}/lib"
. ${THIS_DIR}/load-test-env.sh
export CLASSPATH="${CONF_DIR}:${THIS_DIR}:${LIB_DIR}/*"
JVM_ARGS="-Xmx1G -Xms1G"
echo "$JAVA_HOME/bin/java ${JVM_ARGS} timely.testing.MetricQueryLoadTest $*"
$JAVA_HOME/bin/java ${JVM_ARGS} timely.testing.MetricQueryLoadTest $*
File renamed without changes.
+7 -10
View File
@@ -1,17 +1,16 @@
#!/bin/bash
NUM_INSTANCES=1
NUM_PROFILES=1
ACCUMULO_ZOOKEEPERS=localhost:2181
HADOOP_CONF_DIR=${HADOOP_CONF_DIR}
HADOOP_DEFAULT_FS=hdfs://localhost:8020
TCP_BASE_PORT=4242
HTTP_BASE_PORT=4243
WEBSOCKET_BASE_PORT=4244
UDP_BASE_PORT=4245
TCP_PORT_START=4242
HTTP_PORT_START=4243
WEBSOCKET_PORT_START=4244
UDP_PORT_START=4245
PORT_INCREMENT=100
HOST_IP=0.0.0.0
HOST=$(hostname -s)
HOST_VERIFICATION_ENABLED=true
HOST_FQDN=$(hostname -f)
GENERATE_KEYS=true
TRUST_STORE_TYPE=
@@ -23,8 +22,6 @@ KEY_STORE_PASSWORD=
AUTHORIZATION_URL=
AUTHORIZATION_CERTIFICATE_TYPE=
AUTHORIZATION_CERTIFICATE_FILE=
ANONYMOUS_HTTP=true
ANONYMOUS_WEBSOCKET=true
REQUIRED_ROLES=
REQUIRED_AUTHS=
+41 -39
View File
@@ -1,67 +1,69 @@
#!/bin/bash
if [[ $(uname) == "Darwin" ]]; then
THIS_SCRIPT=$(python -c 'import os,sys; print os.path.realpath(sys.argv[1])' "$0")
if [[ `uname` == "Darwin" ]]; then
THIS_SCRIPT=`python -c 'import os,sys; print os.path.realpath(sys.argv[1])' $0`
TCNATIVE_SUFFIX="jnilib"
else
THIS_SCRIPT=$(readlink -f "$0")
THIS_SCRIPT=`readlink -f $0`
TCNATIVE_SUFFIX="so"
fi
if [ -z "$1" ]; then
export INSTANCE="1"
else
export INSTANCE="$1"
fi
THIS_DIR="${THIS_SCRIPT%/*}"
PID=$(jps -m | grep -E 'timely-balancer-.*-exec.jar' | grep "instance=${INSTANCE}" | awk '{print $1}')
set -a
. ${THIS_DIR}/timely-balancer-env.sh
set +a
if [ -n "$1" ]; then
export INSTANCE="$1"
PROFILE_ARG="--spring.profiles.active=${INSTANCE}"
PID=`jps -m | grep -E 'timely-balancer-.*-exec' | grep "spring.profiles.active=${INSTANCE}" | awk '{print $1}'`
else
export INSTANCE="1"
PID=`jps -m | grep -E 'timely-balancer-.*-exec' | awk '{print $1}'`
fi
if [ "$PID" == "" ]; then
THIS_DIR="${THIS_SCRIPT%/*}"
BASE_DIR=$(cd "$THIS_DIR"/.. && pwd)
NATIVE_DIR="${THIS_DIR}/META-INF/native"
BASE_DIR=$(cd $THIS_DIR/.. && pwd)
TMP_DIR="${BASE_DIR}/tmp"
LIB_DIR="${BASE_DIR}/lib"
BIN_DIR="${BASE_DIR}/bin"
export CONF_DIR="${BASE_DIR}/conf"
NATIVE_DIR="${BIN_DIR}/META-INF/native"
set -a
. "${BIN_DIR}/timely-balancer-env.sh"
set +a
# use either a value from timely-balancer-env.sh or the default
export LOG_DIR="${TIMELY_LOG_DIR:-${BASE_DIR}/logs}"
LIB_DIR="${BASE_DIR}/lib"
export LOG_DIR="${BASE_DIR}/logs"
NUM_SERVER_THREADS=4
if [[ -e ${TMP_DIR} ]]; then
rm -rf "${TMP_DIR}"
rm -rf ${TMP_DIR}
fi
mkdir "${TMP_DIR}"
mkdir ${TMP_DIR}
if [[ -e ${NATIVE_DIR} ]]; then
rm -rf "${NATIVE_DIR}"
rm -rf ${NATIVE_DIR}
fi
mkdir -p "${NATIVE_DIR}"
mkdir -p ${NATIVE_DIR}
if [[ ! -e ${LOG_DIR} ]]; then
mkdir "${LOG_DIR}"
mkdir ${LOG_DIR}
fi
pushd "${BASE_DIR}/bin" || exit
"${JAVA_HOME}/bin/jar xf ${LIB_DIR}/netty-tcnative-boringssl-static*.jar META-INF/native/libnetty_tcnative_linux_x86_64.${TCNATIVE_SUFFIX}"
"${JAVA_HOME}/bin/jar xf ${LIB_DIR}/netty-all*.jar META-INF/native/libnetty_transport_native_epoll_x86_64.${TCNATIVE_SUFFIX}"
popd || exit
pushd ${BASE_DIR}/bin
$JAVA_HOME/bin/jar xf ${LIB_DIR}/netty-tcnative-boringssl-static*.jar META-INF/native/libnetty_tcnative_linux_x86_64.${TCNATIVE_SUFFIX}
$JAVA_HOME/bin/jar xf ${LIB_DIR}/netty-all*.jar META-INF/native/libnetty_transport_native_epoll_x86_64.${TCNATIVE_SUFFIX}
popd
JVM_ARGS="-Xmx2G -Xms2G -XX:NewSize=1G -XX:MaxNewSize=1G"
export CLASSPATH="${CONF_DIR}:${LIB_DIR}/*:${HADOOP_CONF_DIR}"
JVM_ARGS="-Dio.netty.eventLoopThreads=${NUM_SERVER_THREADS} -Dlog4j.configurationFile=${CONF_DIR}/log4j2.yml"
JVM_ARGS="${JVM_ARGS} -DLog4jContextSelector=org.apache.logging.log4j.core.async.AsyncLoggerContextSelector"
JVM_ARGS="${JVM_ARGS} -Djava.library.path=${NATIVE_DIR}"
JVM_ARGS="${JVM_ARGS} -Dlogging.config=${CONF_DIR}/log4j2.yml"
JVM_ARGS="${JVM_ARGS} -Dlog4j2.contextSelector=org.apache.logging.log4j.core.async.AsyncLoggerContextSelector"
JVM_ARGS="${JVM_ARGS} -XX:+UseG1GC -XX:+UseStringDeduplication"
JVM_ARGS="${JVM_ARGS} -Djava.net.preferIPv4Stack=true"
JVM_ARGS="${JVM_ARGS} -XX:+UseNUMA"
echo "${JAVA_HOME}/bin/java ${JVM_ARGS} -jar ${THIS_DIR}/timely-balancer-*-exec.jar --instance=${INSTANCE}"
nohup "${JAVA_HOME}"/bin/java ${JVM_ARGS} -jar "${THIS_DIR}"/timely-balancer-*-exec.jar --instance="${INSTANCE}" >> "${LOG_DIR}/timely-balancer${INSTANCE}.out" 2>&1 &
JVM_ARGS="${JVM_ARGS} -Xmx2G -Xms2G -XX:NewSize=1G -XX:MaxNewSize=1G"
JVM_ARGS="${JVM_ARGS} -XX:+UseCompressedOops -XX:+UseNUMA -Djava.net.preferIPv4Stack=true"
JVM_ARGS="${JVM_ARGS} -XX:+UseG1GC -XX:MaxGCPauseMillis=4000 -XX:ParallelGCThreads=20 -XX:+UseStringDeduplication"
echo "$JAVA_HOME/bin/java ${JVM_ARGS} -jar ${THIS_DIR}/timely-balancer-*-exec.jar ${PROFILE_ARG}"
nohup $JAVA_HOME/bin/java ${JVM_ARGS} -jar ${THIS_DIR}/timely-balancer-*-exec.jar ${PROFILE_ARG} >> ${LOG_DIR}/timely-balancer${INSTANCE}.out 2>&1 &
else
echo "timely-balancer --instance=${INSTANCE} already running with pid ${PID}"
echo "timely-balancer ${INSTANCE} already running with pid ${PID}"
fi
+8 -1
View File
@@ -9,9 +9,16 @@ Configuration:
- name: PID
value: "????"
- name: LOG_PATTERN
value: "%d{ISO8601} %-5p [%-40.40t] [%-40.40c{1.}] %m%n%wEx"
value: "%clr{%d{yyyy-MM-dd HH:mm:ss.SSS}}{faint} %clr{%5p} %clr{${sys:PID}}{magenta} %clr{---}{faint} %clr{[%15.15t]}{faint} %clr{%-40.40c{1.}}{cyan} %clr{:}{faint} %m%n%wEx"
Appenders:
Console:
name: Console
target: SYSTEM_OUT
follow: true
PatternLayout:
pattern: "${LOG_PATTERN}"
RollingFile:
- name: File
fileName: "${sys:logDir}/timely-balancer${env:INSTANCE}.log"
+9
View File
@@ -0,0 +1,9 @@
trust.store.type=${TRUST_STORE_TYPE}
trust.store.file=${TRUST_STORE_FILE}
trust.store.password=${TRUST_STORE_PASSWORD}
key.store.type=${KEY_STORE_TYPE}
key.store.file=${KEY_STORE_FILE}
key.store.password=${KEY_STORE_PASSWORD}
authorization.url=${AUTHORIZATION_URL}
authorization.certificate.type=${AUTHORIZATION_CERTIFICATE_TYPE}
authorization.certificate.file=${AUTHORIZATION_CERTIFICATE_FILE}
+142
View File
@@ -0,0 +1,142 @@
timely:
zookeeper:
servers: ${ACCUMULO_ZOOKEEPERS}
timeout: 30s
security:
session-max-age: 86400
cache-expiration-minutes: 60
cache-refresh-minutes: 5
allow-anonymous-http-access: true
allow-anonymous-ws-access: true
serverSsl:
key-store-type: ${KEY_STORE_TYPE}
key-store-file: ${KEY_STORE_FILE}
key-store-password: ${KEY_STORE_PASSWORD}
use-generated-keypair: ${GENERATE_KEYS}
ignore-ssl-handshake-errors: true
trust-store-type: ${TRUST_STORE_TYPE}
trust-store-file: ${TRUST_STORE_FILE}
trust-store-password: ${TRUST_STORE_PASSWORD}
clientSsl:
key-store-type: ${KEY_STORE_TYPE}
key-store-file: ${KEY_STORE_FILE}
key-store-password: ${KEY_STORE_PASSWORD}
trust-store-type: ${TRUST_STORE_TYPE}
trust-store-file: ${TRUST_STORE_FILE}
trust-store-password: ${TRUST_STORE_PASSWORD}
jwtSsl:
key-store-type: ${AUTHORIZATION_CERTIFICATE_TYPE}
key-store-file: ${AUTHORIZATION_CERTIFICATE_FILE}
key-store-password: ${AUTHORIZATION_CERTIFICATE_PASSWORD}
cache:
enabled: false
metric-age-off-hours:
default: 26
nonCachedMetrics:
balancer:
checkServerHealthInterval: 10000
serverFailuresBeforeDown: 6
serverSuccessesBeforeUp: 1
assignmentFile: /timely/balancer/assignments.csv
server:
ip: ${HOST_IP}
tcpPort: ${TCP_PORT}
numTcpPools: 1
tcpBufferSize: 100
tcpClientPool:
minIdlePerKey: 0
maxIdlePerKey: 40
maxTotalPerKey: 40
maxTotal: -1
lifo: true
fairness: false
maxWaitMillis: -1
minEvictableIdleTimeMillis: 1800000
evictorShutdownTimeoutMillis: 10000
softMinEvictableIdleTimeMillis: -1
numTestsPerEvictionRun: 3
timeBetweenEvictionRunsMillis: -1
blockWhenExhausted: true
jmxEnabled: true
testOnBorrow: false
testOnReturn: false
udpPort: ${UDP_PORT}
udpClientPool:
minIdlePerKey: 0
maxIdlePerKey: 40
maxTotalPerKey: 40
maxTotal: -1
lifo: true
fairness: false
maxWaitMillis: -1
minEvictableIdleTimeMillis: 1800000
evictorShutdownTimeoutMillis: 10000
softMinEvictableIdleTimeMillis: -1
numTestsPerEvictionRun: 3
timeBetweenEvictionRunsMillis: -1
blockWhenExhausted: true
jmxEnabled: true
testOnBorrow: false
testOnReturn: false
http:
host: ${HOST}
ip: ${HOST_IP}
port: ${HTTP_PORT}
redirect-path: /secure-me
strict-transport-max-age: 604800
cors:
allow-any-origin: false
allow-null-origin: false
allow-credentials: true
allowed-origins:
allowed-methods: DELETE,GET,HEAD,OPTIONS,PUT,POST
allowed-headers: content-type
httpClientPool:
minIdlePerKey: 0
maxIdlePerKey: 40
maxTotalPerKey: 40
maxTotal: -1
lifo: true
fairness: false
maxWaitMillis: -1
minEvictableIdleTimeMillis: 1800000
evictorShutdownTimeoutMillis: 10000
softMinEvictableIdleTimeMillis: -1
numTestsPerEvictionRun: 3
timeBetweenEvictionRunsMillis: -1
blockWhenExhausted: true
jmxEnabled: true
testOnBorrow: false
testOnReturn: false
websocket:
ip: ${HOST_IP}
port: ${WEBSOCKET_PORT}
timeout: 60
subscription-lag: 120
scanner-batch-size: 10000
flush-interval-seconds: 30
scanner-read-ahead: 1
subscription-batch-size: 5000
incomingBufferSize: 50000000
wsClientPool:
minIdlePerKey: 0
maxIdlePerKey: 40
maxTotalPerKey: 40
maxTotal: -1
lifo: true
fairness: false
maxWaitMillis: -1
minEvictableIdleTimeMillis: 1800000
evictorShutdownTimeoutMillis: 10000
softMinEvictableIdleTimeMillis: -1
numTestsPerEvictionRun: 3
timeBetweenEvictionRunsMillis: -1
blockWhenExhausted: true
jmxEnabled: true
testOnBorrow: false
testOnReturn: false
logging:
level:
ROOT: info
timey: debug
+97 -106
View File
@@ -1,5 +1,4 @@
timely:
instance: ${INSTANCE}
zookeeper:
servers: ${ACCUMULO_ZOOKEEPERS}
timeout: 30s
@@ -7,90 +6,82 @@ timely:
session-max-age: 86400
cache-expiration-minutes: 60
cache-refresh-minutes: 5
authorization-url: ${AUTHORIZATION_URL}
required-roles: ${REQUIRED_ROLES}
required-auths: ${REQUIRED_AUTHS}
allow-anonymous-http-access: ${ANONYMOUS_HTTP:true}
allow-anonymous-ws-access: ${ANONYMOUS_WEBSOCKET:true}
server-ssl:
key-store-type: ${KEY_STORE_TYPE}
key-store-file: ${KEY_STORE_FILE}
key-store-password: ${KEY_STORE_PASSWORD}
use-generated-keypair: ${GENERATE_KEYS}
allow-anonymous-http-access: true
allow-anonymous-ws-access: true
serverSsl:
key-store-type:
key-store-file:
key-store-password:
use-generated-keypair: true
ignore-ssl-handshake-errors: true
trust-store-type: ${TRUST_STORE_TYPE}
trust-store-file: ${TRUST_STORE_FILE}
trust-store-password: ${TRUST_STORE_PASSWORD}
client-ssl:
key-store-type: ${KEY_STORE_TYPE}
key-store-file: ${KEY_STORE_FILE}
key-store-password: ${KEY_STORE_PASSWORD}
trust-store-type: ${TRUST_STORE_TYPE}
trust-store-file: ${TRUST_STORE_FILE}
trust-store-password: ${TRUST_STORE_PASSWORD}
hostVerificationEnabled: ${HOST_VERIFICATION_ENABLED:true}
jwt-ssl:
key-store-type: ${AUTHORIZATION_CERTIFICATE_TYPE}
key-store-file: ${AUTHORIZATION_CERTIFICATE_FILE}
key-store-password: ${AUTHORIZATION_CERTIFICATE_PASSWORD}
trust-store-type:
trust-store-file:
trust-store-password:
clientSsl:
key-store-type:
key-store-file:
key-store-password:
trust-store-type:
trust-store-file:
trust-store-password:
jwtSsl:
key-store-type:
key-store-file:
key-store-password:
cache:
enabled: false
metric-age-off-hours:
default: 26
non-cached-metrics:
nonCachedMetrics:
balancer:
check-server-health-interval: 10000
server-failures-before-down: 6
server-successes-before-up: 1
default-fs: ${HADOOP_DEFAULT_FS}
fs-config-resources:
- ${HADOOP_CONF_DIR}/core-site.xml
- ${HADOOP_CONF_DIR}/hdfs-site.xml
assignment-file: hdfs:///timely/balancer/assignments.csv
checkServerHealthInterval: 10000
serverFailuresBeforeDown: 6
serverSuccessesBeforeUp: 1
assignmentFile: /timely/balancer/assignments.csv
server:
ip: ${HOST_IP}
tcp-base-port: ${TCP_BASE_PORT}
num-tcp-pools: 1
tcp-buffer-size: 100
tcp-client-pool:
min-idle-per-key: 0
max-idle-per-key: 40
max-total-per-key: 40
max-total: -1
ip: 0.0.0.0
tcpPort: 4242
numTcpPools: 1
tcpBufferSize: 100
tcpClientPool:
minIdlePerKey: 0
maxIdlePerKey: 40
maxTotalPerKey: 40
maxTotal: -1
lifo: true
fairness: false
max-wait-millis: -1
min-evictable-idle-time-millis: 1800000
evictor-shutdown-timeout-millis: 10000
soft-min-evictable-idle-time-millis: -1
num-tests-per-eviction-run: 3
time-between-eviction-runs-millis: -1
block-when-exhausted: true
jmx-enabled: true
test-on-borrow: false
test-on-return: false
udp-base-port: ${UDP_BASE_PORT}
udp-client-pool:
min-idle-per-key: 0
max-idle-per-key: 40
max-total-per-key: 40
max-total: -1
maxWaitMillis: -1
minEvictableIdleTimeMillis: 1800000
evictorShutdownTimeoutMillis: 10000
softMinEvictableIdleTimeMillis: -1
numTestsPerEvictionRun: 3
timeBetweenEvictionRunsMillis: -1
blockWhenExhausted: true
jmxEnabled: true
testOnBorrow: false
testOnReturn: false
udpPort: 4245
udpClientPool:
minIdlePerKey: 0
maxIdlePerKey: 40
maxTotalPerKey: 40
maxTotal: -1
lifo: true
fairness: false
max-wait-millis: -1
min-evictable-idle-time-millis: 1800000
evictor-shutdown-timeout-millis: 10000
soft-min-evictable-idle-time-millis: -1
num-tests-per-eviction-run: 3
time-between-eviction-runs-millis: -1
block-when-exhausted: true
jmx-enabled: true
test-on-borrow: false
test-on-return: false
maxWaitMillis: -1
minEvictableIdleTimeMillis: 1800000
evictorShutdownTimeoutMillis: 10000
softMinEvictableIdleTimeMillis: -1
numTestsPerEvictionRun: 3
timeBetweenEvictionRunsMillis: -1
blockWhenExhausted: true
jmxEnabled: true
testOnBorrow: false
testOnReturn: false
http:
host: ${HOST}
ip: ${HOST_IP}
base-port: ${HTTP_BASE_PORT}
host: localhost
ip: 0.0.0.0
port: 4243
redirect-path: /secure-me
strict-transport-max-age: 604800
cors:
@@ -101,51 +92,51 @@ timely:
allowed-methods: DELETE,GET,HEAD,OPTIONS,PUT,POST
allowed-headers: content-type
httpClientPool:
min-idle-per-key: 0
max-idle-per-key: 40
max-total-per-key: 40
max-total: -1
minIdlePerKey: 0
maxIdlePerKey: 40
maxTotalPerKey: 40
maxTotal: -1
lifo: true
fairness: false
max-wait-millis: -1
min-evictable-idle-time-millis: 1800000
evictor-shutdown-timeout-millis: 10000
soft-min-evictable-idle-time-millis: -1
num-tests-per-eviction-run: 3
time-between-eviction-runs-millis: -1
block-when-exhausted: true
jmx-enabled: true
test-on-borrow: false
test-on-return: false
maxWaitMillis: -1
minEvictableIdleTimeMillis: 1800000
evictorShutdownTimeoutMillis: 10000
softMinEvictableIdleTimeMillis: -1
numTestsPerEvictionRun: 3
timeBetweenEvictionRunsMillis: -1
blockWhenExhausted: true
jmxEnabled: true
testOnBorrow: false
testOnReturn: false
websocket:
ip: ${HOST_IP}
base-port: ${WEBSOCKET_BASE_PORT}
ip: 0.0.0.0
port: 5244
timeout: 60
subscription-lag: 120
scanner-batch-size: 10000
flush-interval-seconds: 30
scanner-read-ahead: 1
subscription-batch-size: 5000
incoming-buffer-size: 50000000
ws-client-pool:
min-idle-per-key: 0
max-idle-per-key: 40
max-total-per-key: 40
max-total: -1
incomingBufferSize: 50000000
wsClientPool:
minIdlePerKey: 0
maxIdlePerKey: 40
maxTotalPerKey: 40
maxTotal: -1
lifo: true
fairness: false
max-wait-millis: -1
min-evictable-idle-time-millis: 1800000
evictor-shutdown-timeout-millis: 10000
soft-min-evictable-idle-time-millis: -1
num-tests-per-eviction-run: 3
time-between-eviction-runs-millis: -1
block-when-exhausted: true
jmx-enabled: true
test-on-borrow: false
test-on-return: false
maxWaitMillis: -1
minEvictableIdleTimeMillis: 1800000
evictorShutdownTimeoutMillis: 10000
softMinEvictableIdleTimeMillis: -1
numTestsPerEvictionRun: 3
timeBetweenEvictionRunsMillis: -1
blockWhenExhausted: true
jmxEnabled: true
testOnBorrow: false
testOnReturn: false
logging:
level:
ROOT: info
timely: debug
timey: debug
+17 -16
View File
@@ -4,7 +4,7 @@
<parent>
<groupId>timely</groupId>
<artifactId>timely</artifactId>
<version>2.1.1</version>
<version>2.1.0-SNAPSHOT</version>
</parent>
<artifactId>timely-balancer</artifactId>
<description>Timely Balancer</description>
@@ -40,6 +40,10 @@
<groupId>commons-beanutils</groupId>
<artifactId>commons-beanutils</artifactId>
</dependency>
<dependency>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
</dependency>
<dependency>
<groupId>io.github.classgraph</groupId>
<artifactId>classgraph</artifactId>
@@ -56,6 +60,10 @@
<groupId>org.apache.accumulo</groupId>
<artifactId>accumulo-core</artifactId>
</dependency>
<dependency>
<groupId>org.apache.accumulo</groupId>
<artifactId>accumulo-minicluster</artifactId>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-collections4</artifactId>
@@ -92,6 +100,10 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-minicluster</artifactId>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-1.2-api</artifactId>
@@ -154,6 +166,10 @@
<groupId>timely</groupId>
<artifactId>timely-common</artifactId>
</dependency>
<dependency>
<groupId>timely</groupId>
<artifactId>timely-server</artifactId>
</dependency>
<dependency>
<groupId>com.lmax</groupId>
<artifactId>disruptor</artifactId>
@@ -164,11 +180,6 @@
<artifactId>netty-tcnative-boringssl-static</artifactId>
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>javax.activation</groupId>
<artifactId>activation</artifactId>
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
@@ -189,11 +200,6 @@
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-minicluster</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.jsoup</groupId>
<artifactId>jsoup</artifactId>
@@ -209,11 +215,6 @@
<artifactId>spring-boot-test-autoconfigure</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>timely</groupId>
<artifactId>timely-test</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
@@ -1,4 +1,4 @@
package timely.balancer;
package timely.balancer.component;
import static org.apache.accumulo.core.conf.ConfigurationTypeHelper.getTimeInMillis;
import static timely.Constants.NON_CACHED_METRICS;
@@ -9,24 +9,30 @@ import java.net.InetSocketAddress;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.List;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import javax.net.ssl.SSLContext;
import org.apache.commons.lang3.StringUtils;
import org.apache.curator.RetryPolicy;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.CuratorFrameworkFactory;
import org.apache.curator.retry.RetryForever;
import org.apache.hadoop.util.StringUtils;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.data.Stat;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
import org.springframework.context.ApplicationContext;
import org.springframework.stereotype.Component;
import io.netty.bootstrap.AbstractBootstrap;
import io.netty.bootstrap.Bootstrap;
@@ -58,9 +64,12 @@ import io.netty.handler.codec.http.HttpResponseEncoder;
import io.netty.handler.codec.http.HttpServerCodec;
import io.netty.handler.codec.http.websocketx.WebSocketServerProtocolHandler;
import io.netty.handler.logging.LoggingHandler;
import io.netty.handler.ssl.ClientAuth;
import io.netty.handler.ssl.OpenSslServerContext;
import io.netty.handler.ssl.OpenSslServerSessionContext;
import io.netty.handler.ssl.SslContext;
import io.netty.handler.ssl.SslContextBuilder;
import io.netty.handler.ssl.util.SelfSignedCertificate;
import io.netty.handler.stream.ChunkedWriteHandler;
import io.netty.handler.timeout.IdleStateHandler;
import io.netty.util.NettyRuntime;
@@ -78,9 +87,11 @@ import timely.balancer.netty.http.HttpRelayHandler;
import timely.balancer.netty.tcp.TcpRelayHandler;
import timely.balancer.netty.udp.UdpRelayHandler;
import timely.balancer.netty.ws.WsRelayHandler;
import timely.client.http.HttpClient;
import timely.common.component.AuthenticationService;
import timely.common.configuration.SecurityProperties;
import timely.common.configuration.SslClientProperties;
import timely.common.configuration.SslHelper;
import timely.common.configuration.SslServerProperties;
import timely.common.configuration.ZookeeperProperties;
import timely.netty.http.HttpStaticFileServerHandler;
@@ -96,6 +107,8 @@ import timely.netty.websocket.WebSocketFullRequestHandler;
import timely.netty.websocket.WebSocketRequestDecoder;
import timely.netty.websocket.subscription.WSTimelyExceptionHandler;
@Component
@ConditionalOnMissingBean(Balancer.class)
public class Balancer {
final public static String LEADER_LATCH_PATH = "/timely/balancer/leader";
@@ -107,6 +120,8 @@ public class Balancer {
private static final String OS_NAME = "os.name";
private static final String OS_VERSION = "os.version";
private static final String WS_PATH = "/websocket";
protected static final CountDownLatch LATCH = new CountDownLatch(1);
private BalancerProperties balancerProperties;
private SecurityProperties securityProperties;
private SslServerProperties sslServerProperties;
@@ -119,8 +134,7 @@ public class Balancer {
private AuthenticationService authenticationService;
private CuratorFramework curatorFramework;
private int quietPeriod;
private SslContext serverSslContext;
private SSLContext clientSSLContext;
private EventLoopGroup tcpWorkerGroup = null;
private EventLoopGroup tcpBossGroup = null;
private EventLoopGroup httpWorkerGroup = null;
@@ -143,26 +157,20 @@ public class Balancer {
final String osName = SystemPropertyUtil.get(OS_NAME).toLowerCase().trim();
final String osVersion = SystemPropertyUtil.get(OS_VERSION).toLowerCase();
// split at periods, keep the first two, join with a period
final String majMinVers = StringUtils.join(Arrays.stream(osVersion.split("\\.")).limit(2).collect(Collectors.toList()), ".");
final String majMinVers = StringUtils.join(".", Arrays.stream(osVersion.split("\\.")).limit(2).collect(Collectors.toList()));
if (osName.startsWith("linux")) {
try {
Runtime.Version currentVersion = Runtime.Version.parse(majMinVers);
Runtime.Version epollMinVersion = Runtime.Version.parse(EPOLL_MIN_VERSION);
return currentVersion.compareTo(epollMinVersion) > 0;
} catch (Exception e) {
log.error(e.getMessage(), e);
}
Runtime.Version currentVersion = Runtime.Version.parse(majMinVers);
Runtime.Version epollMinVersion = Runtime.Version.parse(EPOLL_MIN_VERSION);
return currentVersion.compareTo(epollMinVersion) > 0;
} else {
return false;
}
return false;
}
public Balancer(ApplicationContext applicationContext, SslContext serverSslContext, SSLContext clientSSLContext,
AuthenticationService authenticationService, MetricResolver metricResolver, BalancerProperties balancerProperties,
ZookeeperProperties zookeeperProperties, SecurityProperties securityProperties, SslServerProperties sslServerProperties,
SslClientProperties sslClientProperties, BalancerServerProperties balancerServerProperties, BalancerHttpProperties balancerHttpProperties,
BalancerWebsocketProperties balancerWebsocketProperties) {
this.serverSslContext = serverSslContext;
this.clientSSLContext = clientSSLContext;
public Balancer(ApplicationContext applicationContext, AuthenticationService authenticationService, MetricResolver metricResolver,
BalancerProperties balancerProperties, ZookeeperProperties zookeeperProperties, SecurityProperties securityProperties,
SslServerProperties sslServerProperties, SslClientProperties sslClientProperties, BalancerServerProperties balancerServerProperties,
BalancerHttpProperties balancerHttpProperties, BalancerWebsocketProperties balancerWebsocketProperties) {
this.quietPeriod = balancerServerProperties.getShutdownQuietPeriod();
this.applicationContext = applicationContext;
this.authenticationService = authenticationService;
@@ -178,7 +186,8 @@ public class Balancer {
DEFAULT_EVENT_LOOP_THREADS = Math.max(1, SystemPropertyUtil.getInt("io.netty.eventLoopThreads", NettyRuntime.availableProcessors() * 2));
}
public void start() {
@PostConstruct
public void startup() {
log.info("Starting {}", this.getClass().getSimpleName());
try {
// start curator framework
@@ -237,9 +246,9 @@ public class Balancer {
final int httpPort = balancerHttpProperties.getPort();
final String httpIp = balancerHttpProperties.getIp();
if (serverSslContext instanceof OpenSslServerContext) {
OpenSslServerContext openssl = (OpenSslServerContext) serverSslContext;
SslContext sslCtx = createSSLContext(sslServerProperties);
if (sslCtx instanceof OpenSslServerContext) {
OpenSslServerContext openssl = (OpenSslServerContext) sslCtx;
String application = "Timely_" + httpPort;
OpenSslServerSessionContext opensslCtx = openssl.sessionContext();
opensslCtx.setSessionCacheEnabled(true);
@@ -247,12 +256,13 @@ public class Balancer {
opensslCtx.setSessionIdContext(application.getBytes(StandardCharsets.UTF_8));
opensslCtx.setSessionTimeout(securityProperties.getSessionMaxAge());
}
SSLContext clientSSLContext = createSSLClientContext(sslClientProperties);
final ServerBootstrap httpServer = new ServerBootstrap();
httpServer.group(httpBossGroup, httpWorkerGroup);
httpServer.channel(channelClass);
httpServer.handler(new LoggingHandler());
HttpClientPool httpClientPool = new HttpClientPool(balancerHttpProperties, clientSSLContext, sslClientProperties);
httpServer.childHandler(setupHttpChannel(balancerHttpProperties, sslServerProperties, serverSslContext, metricResolver, httpClientPool));
HttpClientPool httpClientPool = new HttpClientPool(securityProperties, sslClientProperties, balancerHttpProperties, clientSSLContext);
httpServer.childHandler(setupHttpChannel(balancerHttpProperties, sslServerProperties, sslCtx, metricResolver, httpClientPool));
httpServer.option(ChannelOption.ALLOCATOR, PooledByteBufAllocator.DEFAULT);
httpServer.option(ChannelOption.SO_BACKLOG, 128);
httpServer.childOption(ChannelOption.SO_KEEPALIVE, true);
@@ -266,7 +276,7 @@ public class Balancer {
wsServer.channel(channelClass);
wsServer.handler(new LoggingHandler());
WsClientPool wsClientPool = new WsClientPool(balancerProperties, balancerWebsocketProperties, sslClientProperties, clientSSLContext);
wsServer.childHandler(setupWSChannel(balancerWebsocketProperties, securityProperties, serverSslContext, metricResolver, wsClientPool));
wsServer.childHandler(setupWSChannel(balancerWebsocketProperties, securityProperties, sslCtx, metricResolver, wsClientPool));
wsServer.option(ChannelOption.ALLOCATOR, PooledByteBufAllocator.DEFAULT);
wsServer.option(ChannelOption.SO_BACKLOG, 128);
wsServer.childOption(ChannelOption.SO_KEEPALIVE, true);
@@ -307,6 +317,7 @@ public class Balancer {
return channel;
}
@PreDestroy
public void shutdown() {
List<ChannelFuture> channelFutures = new ArrayList<>();
@@ -399,6 +410,31 @@ public class Balancer {
log.info("{} shut down.", this.getClass().getSimpleName());
}
protected SslContext createSSLContext(SslServerProperties sslServerProperties) throws Exception {
Boolean generate = sslServerProperties.isUseGeneratedKeypair();
SslContextBuilder ssl;
if (generate) {
log.warn("Using generated self signed server certificate");
Date begin = new Date();
Date end = new Date(begin.getTime() + 86400000);
SelfSignedCertificate ssc = new SelfSignedCertificate("localhost", begin, end);
ssl = SslContextBuilder.forServer(ssc.certificate(), ssc.privateKey());
} else {
ssl = SslHelper.getSslContextBuilder(sslServerProperties);
}
// Can't set to REQUIRE because the CORS pre-flight requests will fail.
ssl.clientAuth(ClientAuth.OPTIONAL);
return ssl.build();
}
protected SSLContext createSSLClientContext(SslClientProperties sslClientProperties) {
SslClientProperties ssl = sslClientProperties;
return HttpClient.getSSLContext(ssl.getTrustStoreFile(), ssl.getTrustStoreType(), ssl.getTrustStorePassword(), ssl.getKeyStoreFile(),
ssl.getKeyStoreType(), ssl.getKeyStorePassword());
}
private void ensureZkPaths(CuratorFramework curatorFramework, String[] paths) {
for (String s : paths) {
try {
@@ -1,4 +1,4 @@
package timely.balancer;
package timely.balancer.component;
import java.util.ArrayList;
import java.util.List;
@@ -6,8 +6,13 @@ import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
import org.springframework.stereotype.Component;
import timely.balancer.configuration.BalancerProperties;
import timely.balancer.configuration.BalancerServerProperties;
@@ -15,6 +20,8 @@ import timely.balancer.connection.TimelyBalancedHost;
import timely.balancer.connection.tcp.TcpClientPool;
import timely.balancer.healthcheck.TCPServerHealthCheck;
@Component
@ConditionalOnMissingBean(HealthChecker.class)
public class HealthChecker {
private static final Logger log = LoggerFactory.getLogger(HealthChecker.class);
@@ -29,7 +36,8 @@ public class HealthChecker {
this.balancerServerProperties = balancerServerProperties;
}
public void start() {
@PostConstruct
public void setup() {
TcpClientPool tcpClientPool = new TcpClientPool(balancerServerProperties);
this.tcpServerHealthCheck = new TCPServerHealthCheck(tcpClientPool);
this.executorService.scheduleAtFixedRate(() -> {
@@ -50,7 +58,8 @@ public class HealthChecker {
}, 0, balancerProperties.getCheckServerHealthInterval(), TimeUnit.MILLISECONDS);
}
public void shutdown() {
@PreDestroy
public void close() throws Exception {
this.executorService.shutdown();
try {
this.executorService.awaitTermination(5, TimeUnit.SECONDS);
@@ -1,15 +1,13 @@
package timely.balancer;
package timely.balancer.component;
import static org.apache.hadoop.fs.FileSystem.FS_DEFAULT_NAME_KEY;
import static timely.Constants.NON_CACHED_METRICS;
import static timely.Constants.NON_CACHED_METRICS_LOCK_PATH;
import static timely.Constants.SERVICE_DISCOVERY_PATH;
import static timely.balancer.Balancer.ASSIGNMENTS_LAST_UPDATED_PATH;
import static timely.balancer.Balancer.ASSIGNMENTS_LOCK_PATH;
import static timely.balancer.Balancer.LEADER_LATCH_PATH;
import static timely.balancer.component.Balancer.ASSIGNMENTS_LAST_UPDATED_PATH;
import static timely.balancer.component.Balancer.ASSIGNMENTS_LOCK_PATH;
import static timely.balancer.component.Balancer.LEADER_LATCH_PATH;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Collection;
@@ -31,6 +29,9 @@ import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import org.apache.commons.lang3.SerializationUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.curator.framework.CuratorFramework;
@@ -52,21 +53,25 @@ import org.apache.curator.x.discovery.details.ServiceCacheListener;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.zookeeper.CreateMode;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
import org.springframework.stereotype.Component;
import com.csvreader.CsvReader;
import com.csvreader.CsvWriter;
import timely.ServerDetails;
import timely.balancer.ArrivalRate;
import timely.balancer.configuration.BalancerProperties;
import timely.balancer.connection.TimelyBalancedHost;
import timely.common.configuration.CacheProperties;
@Component
@ConditionalOnMissingBean(MetricResolver.class)
public class MetricResolver {
private static final Logger log = LoggerFactory.getLogger(MetricResolver.class);
@@ -104,17 +109,13 @@ public class MetricResolver {
this.healthChecker = healthChecker;
}
public void start() {
@PostConstruct
public void setup() {
assignmentsIPRWLock = new InterProcessReadWriteLock(curatorFramework, ASSIGNMENTS_LOCK_PATH);
testIPRWLock(curatorFramework, assignmentsIPRWLock, ASSIGNMENTS_LOCK_PATH);
startLeaderLatch(curatorFramework);
startServiceListener(curatorFramework);
assignmentsLastUpdatedInHdfs = new DistributedAtomicLong(curatorFramework, ASSIGNMENTS_LAST_UPDATED_PATH, new RetryForever(1000));
try {
assignmentsLastUpdatedInHdfs.forceSet(getLastWrittenToHdfsTimestamp());
} catch (Exception e) {
log.error(e.getMessage(), e);
}
TreeCacheListener assignmentListener = (curframework, event) -> {
log.debug("Handling assignments event {}. assignmentsLastUpdatedInHdfs:{}", event.getType().toString(),
@@ -190,7 +191,8 @@ public class MetricResolver {
}, 10, 60, TimeUnit.SECONDS);
}
public void shutdown() {
@PreDestroy
public void close() throws Exception {
this.assignmentExecutor.shutdown();
try {
@@ -356,13 +358,8 @@ public class MetricResolver {
if (isLeader.get()) {
for (String s : reassignMetrics) {
TimelyBalancedHost h = getRoundRobinHost(null);
if (h == null) {
unassignMetric(s);
log.debug("Assigned server removed and none available. Unassigning {}", s);
} else {
assignMetric(s, h);
log.debug("Assigned server removed. Assigning {} to server {}:{}", s, h.getTcpPort());
}
assignMetric(s, h);
log.debug("Assigned server removed. Assigning {} to server {}:{}", s, h.getTcpPort());
}
}
@@ -790,19 +787,20 @@ public class MetricResolver {
tbh = metricToHostMap.get(metric);
if (tbh == null) {
tbh = getRoundRobinHost(null);
if (!balancerLock.isWriteLockedByCurrentThread()) {
balancerLock.readLock().unlock();
balancerLock.writeLock().lock();
}
assignMetric(metric, tbh);
} else if (!tbh.isUp()) {
TimelyBalancedHost oldTbh = tbh;
tbh = getLeastUsedHost();
if (tbh != null) {
log.debug("rebalancing from host that is down: reassigning metric {} from server {}:{} to {}:{}", metric, oldTbh.getHost(),
oldTbh.getTcpPort(), tbh.getHost(), tbh.getTcpPort());
log.debug("rebalancing from host that is down: reassigning metric {} from server {}:{} to {}:{}", metric, oldTbh.getHost(),
oldTbh.getTcpPort(), tbh.getHost(), tbh.getTcpPort());
if (!balancerLock.isWriteLockedByCurrentThread()) {
balancerLock.readLock().unlock();
balancerLock.writeLock().lock();
}
}
if (!balancerLock.isWriteLockedByCurrentThread()) {
balancerLock.readLock().unlock();
balancerLock.writeLock().lock();
}
if (tbh != null) {
assignMetric(metric, tbh);
}
}
@@ -901,45 +899,18 @@ public class MetricResolver {
return tbh;
}
protected FileSystem getFileSystem(Path path) throws IOException {
Configuration configuration = new Configuration();
if (StringUtils.isNotBlank(balancerProperties.getDefaultFs())) {
configuration.set(FS_DEFAULT_NAME_KEY, balancerProperties.getDefaultFs());
}
if (balancerProperties.getFsConfigResources() != null) {
for (String resource : balancerProperties.getFsConfigResources()) {
configuration.addResource(new Path(resource));
}
}
return FileSystem.get(path.toUri(), configuration);
}
protected void createAssignmentFile() {
try {
log.info("Creating assignment file: " + balancerProperties.getAssignmentFile());
Configuration configuration = new Configuration();
FileSystem fs = FileSystem.get(configuration);
Path assignmentFile = new Path(balancerProperties.getAssignmentFile());
FileSystem fs = getFileSystem(assignmentFile);
fs.create(assignmentFile);
} catch (Exception e) {
log.error(e.getMessage(), e);
}
}
protected long getLastWrittenToHdfsTimestamp() {
try {
Path assignmentFile = new Path(balancerProperties.getAssignmentFile());
FileSystem fs = getFileSystem(assignmentFile);
FileStatus fileStatus = fs.getFileStatus(assignmentFile);
return fileStatus.getModificationTime();
} catch (FileNotFoundException e) {
createAssignmentFile();
return System.currentTimeMillis();
} catch (Exception e) {
log.error(e.getMessage(), e);
return System.currentTimeMillis();
}
}
private void readAssignmentsFromHdfs(boolean checkIfNecessary) {
try {
@@ -966,8 +937,9 @@ public class MetricResolver {
acquired = assignmentsIPRWLock.readLock().acquire(60, TimeUnit.SECONDS);
}
balancerLock.writeLock().lock();
Configuration configuration = new Configuration();
FileSystem fs = FileSystem.get(configuration);
Path assignmentFile = new Path(balancerProperties.getAssignmentFile());
FileSystem fs = getFileSystem(assignmentFile);
FSDataInputStream iStream = fs.open(assignmentFile);
CsvReader reader = new CsvReader(iStream, ',', Charset.forName("UTF-8"));
reader.setUseTextQualifier(false);
@@ -989,9 +961,7 @@ public class MetricResolver {
} else {
log.trace("Found assigment: {} to {}:{}", metric, host, tcpPort);
}
if (tbh != null) {
assignedMetricToHostMap.put(metric, tbh);
}
assignedMetricToHostMap.put(metric, tbh);
}
}
@@ -1052,8 +1022,9 @@ public class MetricResolver {
nonCachedMetricsLocalLock.readLock().lock();
try {
if (!metricToHostMap.isEmpty()) {
Configuration configuration = new Configuration();
FileSystem fs = FileSystem.get(configuration);
Path assignmentFile = new Path(balancerProperties.getAssignmentFile());
FileSystem fs = getFileSystem(assignmentFile);
if (!fs.exists(assignmentFile.getParent())) {
fs.mkdirs(assignmentFile.getParent());
}
@@ -1065,7 +1036,7 @@ public class MetricResolver {
writer.write("tcpPort");
writer.endRecord();
for (Map.Entry<String,TimelyBalancedHost> e : metricToHostMap.entrySet()) {
if (e.getValue() != null && !nonCachedMetrics.contains(e.getKey())) {
if (!nonCachedMetrics.contains(e.getKey())) {
writer.write(e.getKey());
writer.write(e.getValue().getHost());
writer.write(Integer.toString(e.getValue().getTcpPort()));
@@ -1119,18 +1090,4 @@ public class MetricResolver {
}
}
}
protected void unassignMetric(String metric) {
if (isLeader.get()) {
if (StringUtils.isNotBlank(metric)) {
balancerLock.writeLock().lock();
try {
metricToHostMap.remove(metric);
assignmentsLastUpdatedLocal.set(System.currentTimeMillis());
} finally {
balancerLock.writeLock().unlock();
}
}
}
}
}
@@ -1,58 +1,10 @@
package timely.balancer.configuration;
import javax.net.ssl.SSLContext;
import org.apache.curator.framework.CuratorFramework;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.ApplicationContext;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import io.netty.handler.ssl.SslContext;
import timely.balancer.Balancer;
import timely.balancer.HealthChecker;
import timely.balancer.MetricResolver;
import timely.common.component.AuthenticationService;
import timely.common.configuration.CacheProperties;
import timely.common.configuration.SecurityProperties;
import timely.common.configuration.SslClientProperties;
import timely.common.configuration.SslServerProperties;
import timely.common.configuration.ZookeeperProperties;
@Configuration
@EnableConfigurationProperties({BalancerProperties.class, BalancerServerProperties.class, BalancerWebsocketProperties.class, BalancerHttpProperties.class})
public class BalancerConfiguration {
@Bean
@ConditionalOnMissingBean
public Balancer balancer(ApplicationContext applicationContext, @Qualifier("nettySslContext") SslContext serverSslContext,
@Qualifier("outboundJDKSslContext") SSLContext clientSSLContext, AuthenticationService authenticationService, MetricResolver metricResolver,
BalancerProperties balancerProperties, ZookeeperProperties zookeeperProperties, SecurityProperties securityProperties,
SslServerProperties sslServerProperties, SslClientProperties sslClientProperties, BalancerServerProperties balancerServerProperties,
BalancerHttpProperties balancerHttpProperties, BalancerWebsocketProperties balancerWebsocketProperties) {
Balancer balancer = new Balancer(applicationContext, serverSslContext, clientSSLContext, authenticationService, metricResolver, balancerProperties,
zookeeperProperties, securityProperties, sslServerProperties, sslClientProperties, balancerServerProperties, balancerHttpProperties,
balancerWebsocketProperties);
balancer.start();
return balancer;
}
@Bean
@ConditionalOnMissingBean
public HealthChecker healthChecker(BalancerProperties balancerProperties, BalancerServerProperties balancerServerProperties) {
HealthChecker healthChecker = new HealthChecker(balancerProperties, balancerServerProperties);
healthChecker.start();
return healthChecker;
}
@Bean
@ConditionalOnMissingBean
public MetricResolver metricResolver(CuratorFramework curatorFramework, BalancerProperties balancerProperties, CacheProperties cacheProperties,
HealthChecker healthChecker) {
MetricResolver metricResolver = new MetricResolver(curatorFramework, balancerProperties, cacheProperties, healthChecker);
metricResolver.start();
return metricResolver;
}
}
@@ -1,7 +1,5 @@
package timely.balancer.configuration;
import java.util.List;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;
import org.springframework.validation.annotation.Validated;
@@ -15,10 +13,6 @@ public class BalancerProperties {
private String assignmentFile;
private String defaultFs;
private List<String> fsConfigResources;
private Double controlBandPercentage = 0.05;
private long checkServerHealthInterval = 10000;
@@ -43,22 +37,6 @@ public class BalancerProperties {
this.assignmentFile = assignmentFile;
}
public String getDefaultFs() {
return defaultFs;
}
public void setDefaultFs(String defaultFs) {
this.defaultFs = defaultFs;
}
public List<String> getFsConfigResources() {
return fsConfigResources;
}
public void setFsConfigResources(List<String> fsConfigResources) {
this.fsConfigResources = fsConfigResources;
}
public void setControlBandPercentage(Double controlBandPercentage) {
this.controlBandPercentage = controlBandPercentage;
}
@@ -0,0 +1,6 @@
package timely.balancer.configuration;
import org.springframework.boot.autoconfigure.SpringBootApplication;
@SpringBootApplication(scanBasePackageClasses = {SpringBootstrap.class})
public class SpringBootstrap {}
@@ -8,15 +8,17 @@ import org.apache.commons.pool2.impl.DefaultPooledObject;
import org.apache.http.impl.client.CloseableHttpClient;
import timely.balancer.connection.TimelyBalancedHost;
import timely.client.http.HttpClient;
import timely.common.configuration.SecurityProperties;
import timely.common.configuration.SslClientProperties;
public class HttpClientFactory implements KeyedPooledObjectFactory<TimelyBalancedHost,CloseableHttpClient> {
private final SSLContext sslContext;
private final SecurityProperties security;
private SslClientProperties sslClientProperties;
public HttpClientFactory(SSLContext sslContext, SslClientProperties sslClientProperties) {
public HttpClientFactory(SecurityProperties security, SslClientProperties sslClientProperties, SSLContext sslContext) {
this.security = security;
this.sslContext = sslContext;
this.sslClientProperties = sslClientProperties;
}
@@ -24,8 +26,8 @@ public class HttpClientFactory implements KeyedPooledObjectFactory<TimelyBalance
@Override
public PooledObject<CloseableHttpClient> makeObject(TimelyBalancedHost k) throws Exception {
// disable cookie management because we are sharing connections
return new DefaultPooledObject<>(
HttpClient.get(this.sslContext, null, sslClientProperties.isHostVerificationEnabled(), sslClientProperties.isUseClientCert()));
return new DefaultPooledObject<>(timely.client.http.HttpClient.get(this.sslContext, null, sslClientProperties.isHostVerificationEnabled(),
sslClientProperties.isUseClientCert()));
}
@Override
@@ -7,11 +7,12 @@ import org.apache.http.impl.client.CloseableHttpClient;
import timely.balancer.configuration.BalancerHttpProperties;
import timely.balancer.connection.TimelyBalancedHost;
import timely.common.configuration.SecurityProperties;
import timely.common.configuration.SslClientProperties;
public class HttpClientPool extends GenericKeyedObjectPool<TimelyBalancedHost,CloseableHttpClient> {
public HttpClientPool(BalancerHttpProperties http, SSLContext sslContext, SslClientProperties sslClientProperties) {
super(new HttpClientFactory(sslContext, sslClientProperties), http.getHttpClientPool());
public HttpClientPool(SecurityProperties securityProperties, SslClientProperties sslClientProperties, BalancerHttpProperties http, SSLContext sslContext) {
super(new HttpClientFactory(securityProperties, sslClientProperties, sslContext), http.getHttpClientPool());
}
}
@@ -43,7 +43,7 @@ import timely.api.response.TimelyException;
import timely.auth.TimelyAuthenticationToken;
import timely.auth.util.HttpHeaderUtils;
import timely.auth.util.ProxiedEntityUtils;
import timely.balancer.MetricResolver;
import timely.balancer.component.MetricResolver;
import timely.balancer.connection.TimelyBalancedHost;
import timely.balancer.connection.http.HttpClientPool;
import timely.netty.http.TimelyHttpHandler;
@@ -14,7 +14,7 @@ import io.netty.channel.ChannelHandlerContext;
import io.netty.channel.SimpleChannelInboundHandler;
import timely.api.request.MetricRequest;
import timely.api.request.TcpRequest;
import timely.balancer.MetricResolver;
import timely.balancer.component.MetricResolver;
import timely.balancer.connection.TimelyBalancedHost;
import timely.balancer.connection.tcp.TcpClientPool;
import timely.client.tcp.TcpClient;
@@ -12,7 +12,7 @@ import io.netty.channel.ChannelHandlerContext;
import io.netty.channel.SimpleChannelInboundHandler;
import timely.api.request.MetricRequest;
import timely.api.request.UdpRequest;
import timely.balancer.MetricResolver;
import timely.balancer.component.MetricResolver;
import timely.balancer.connection.TimelyBalancedHost;
import timely.balancer.connection.udp.UdpClientPool;
import timely.client.udp.UdpClient;
@@ -28,7 +28,7 @@ import timely.api.request.websocket.RemoveSubscription;
import timely.api.request.websocket.SubscriptionRequest;
import timely.api.response.TimelyException;
import timely.auth.util.ProxiedEntityUtils;
import timely.balancer.MetricResolver;
import timely.balancer.component.MetricResolver;
import timely.balancer.configuration.BalancerWebsocketProperties;
import timely.balancer.connection.TimelyBalancedHost;
import timely.balancer.connection.ws.WsClientPool;
-11
View File
@@ -1,11 +0,0 @@
===============================================================================
_______ _ _ ____ _
|__ __(_) | | | _ \ | |
| | _ _ __ ___ ___| |_ _ | |_) | __ _| | __ _ _ __ ___ ___ _ __
| | | | '_ ` _ \ / _ \ | | | | | _ < / _` | |/ _` | '_ \ / __/ _ \ '__|
| | | | | | | | | __/ | |_| | | |_) | (_| | | (_| | | | | (_| __/ |
|_| |_|_| |_| |_|\___|_|\__, | |____/ \__,_|_|\__,_|_| |_|\___\___|_|
__/ |
|___/
===============================================================================
TimelyBalancer ${application.formatted-version}
+1 -1
View File
@@ -1,7 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<FindBugsFilter>
<Match>
<Class name="timely.balancer.Balancer" />
<Class name="timely.balancer.component.Balancer" />
<Method name="shutdown" />
<Bug pattern="UC_USELESS_OBJECT" />
</Match>
@@ -0,0 +1,33 @@
package timely.balancer.component;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.context.ApplicationContext;
import org.springframework.context.annotation.Primary;
import org.springframework.stereotype.Component;
import timely.balancer.configuration.BalancerHttpProperties;
import timely.balancer.configuration.BalancerProperties;
import timely.balancer.configuration.BalancerServerProperties;
import timely.balancer.configuration.BalancerWebsocketProperties;
import timely.common.component.AuthenticationService;
import timely.common.configuration.SecurityProperties;
import timely.common.configuration.SslClientProperties;
import timely.common.configuration.SslServerProperties;
import timely.common.configuration.ZookeeperProperties;
@Primary
@Component
public class TestBalancer extends Balancer {
private static final Logger log = LoggerFactory.getLogger(TestBalancer.class);
public TestBalancer(ApplicationContext applicationContext, AuthenticationService authenticationService, MetricResolver metricResolver,
BalancerProperties balancerProperties, ZookeeperProperties zookeeperProperties, SecurityProperties securityProperties,
SslServerProperties sslServerProperties, SslClientProperties sslClientProperties, BalancerServerProperties balancerServerProperties,
BalancerHttpProperties balancerHttpProperties, BalancerWebsocketProperties balancerWebsocketProperties) {
super(applicationContext, authenticationService, metricResolver, balancerProperties, zookeeperProperties, securityProperties, sslServerProperties,
sslClientProperties, balancerServerProperties, balancerHttpProperties, balancerWebsocketProperties);
DEFAULT_EVENT_LOOP_THREADS = 1;
log.info("Starting TestBalancer");
}
}
@@ -1,18 +1,32 @@
package timely.balancer.test;
package timely.balancer.component;
import java.util.List;
import timely.balancer.HealthChecker;
import org.springframework.context.annotation.Primary;
import org.springframework.stereotype.Component;
import timely.balancer.configuration.BalancerProperties;
import timely.balancer.configuration.BalancerServerProperties;
import timely.balancer.connection.TimelyBalancedHost;
@Primary
@Component
public class TestHealthChecker extends HealthChecker {
public TestHealthChecker(BalancerProperties balancerProperties, BalancerServerProperties balancerServerProperties) {
super(balancerProperties, balancerServerProperties);
}
@Override
public void setup() {
}
@Override
public void close() {
}
public List<TimelyBalancedHost> getTimelyHosts() {
return this.timelyHosts;
}
@@ -1,4 +1,4 @@
package timely.balancer.test;
package timely.balancer.component;
import java.util.List;
@@ -8,13 +8,15 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.context.annotation.Primary;
import org.springframework.stereotype.Component;
import timely.balancer.HealthChecker;
import timely.balancer.MetricResolver;
import timely.balancer.configuration.BalancerProperties;
import timely.balancer.connection.TimelyBalancedHost;
import timely.common.configuration.CacheProperties;
@Primary
@Component
public class TestMetricResolver extends MetricResolver {
private static final Logger log = LoggerFactory.getLogger(TestMetricResolver.class);
@@ -0,0 +1,20 @@
package timely.balancer.component;
import java.util.Collections;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.context.annotation.Primary;
import org.springframework.security.authentication.ProviderManager;
import org.springframework.security.web.authentication.preauth.PreAuthenticatedAuthenticationProvider;
import org.springframework.stereotype.Component;
@Primary
@Component
public class TestTimelyAuthenticationManager extends ProviderManager {
@Autowired
public TestTimelyAuthenticationManager(@Qualifier("file") PreAuthenticatedAuthenticationProvider file) {
super(Collections.singletonList(file));
}
}
@@ -1,56 +0,0 @@
package timely.balancer.configuration;
import javax.net.ssl.SSLContext;
import org.apache.curator.framework.CuratorFramework;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.context.ApplicationContext;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.Configuration;
import io.netty.handler.ssl.SslContext;
import timely.balancer.HealthChecker;
import timely.balancer.MetricResolver;
import timely.balancer.test.TestBalancer;
import timely.balancer.test.TestHealthChecker;
import timely.balancer.test.TestMetricResolver;
import timely.common.component.AuthenticationService;
import timely.common.configuration.CacheProperties;
import timely.common.configuration.SecurityProperties;
import timely.common.configuration.SslClientProperties;
import timely.common.configuration.SslServerProperties;
import timely.common.configuration.ZookeeperProperties;
@Configuration
@ComponentScan(basePackages = {"timely.balancer", "timely.common", "timely.test"})
public class TestBalancerConfiguration {
@Bean
public TestBalancer balancer(ApplicationContext applicationContext, @Qualifier("nettySslContext") SslContext serverSslContext,
@Qualifier("outboundJDKSslContext") SSLContext clientSSLContext, AuthenticationService authenticationService, MetricResolver metricResolver,
BalancerProperties balancerProperties, ZookeeperProperties zookeeperProperties, SecurityProperties securityProperties,
SslServerProperties sslServerProperties, SslClientProperties sslClientProperties, BalancerServerProperties balancerServerProperties,
BalancerHttpProperties balancerHttpProperties, BalancerWebsocketProperties balancerWebsocketProperties) {
TestBalancer balancer = new TestBalancer(applicationContext, serverSslContext, clientSSLContext, authenticationService, metricResolver,
balancerProperties, zookeeperProperties, securityProperties, sslServerProperties, sslClientProperties, balancerServerProperties,
balancerHttpProperties, balancerWebsocketProperties);
balancer.start();
return balancer;
}
@Bean
public TestHealthChecker healthChecker(BalancerProperties balancerProperties, BalancerServerProperties balancerServerProperties) {
TestHealthChecker healthChecker = new TestHealthChecker(balancerProperties, balancerServerProperties);
healthChecker.start();
return healthChecker;
}
@Bean
public TestMetricResolver metricResolver(CuratorFramework curatorFramework, BalancerProperties balancerProperties, CacheProperties cacheProperties,
HealthChecker healthChecker) {
TestMetricResolver metricResolver = new TestMetricResolver(curatorFramework, balancerProperties, cacheProperties, healthChecker);
metricResolver.start();
return metricResolver;
}
}
@@ -0,0 +1,21 @@
package timely.balancer.configuration;
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import timely.common.configuration.AccumuloProperties;
@Configuration
public class TestConfiguration {
@Bean
@ConditionalOnMissingBean
public AccumuloProperties accumuloProperties() {
AccumuloProperties accumuloProperties = new AccumuloProperties();
accumuloProperties.setInstanceName("default");
accumuloProperties.setUsername("root");
accumuloProperties.setPassword("secret");
return accumuloProperties;
}
}
@@ -1,4 +1,4 @@
package timely.test.configuration;
package timely.balancer.configuration;
import static org.apache.accumulo.core.conf.ConfigurationTypeHelper.getTimeInMillis;
@@ -9,7 +9,6 @@ import org.apache.curator.retry.RetryForever;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.DependsOn;
@@ -22,12 +21,11 @@ import timely.common.configuration.ZookeeperProperties;
* that need CuratorFramework autowired will wait.
*/
@Configuration
@EnableConfigurationProperties({ZookeeperProperties.class})
public class TestCuratorConfiguration {
private static final Logger log = LoggerFactory.getLogger(TestCuratorConfiguration.class);
@Bean(name = "CuratorFramework", destroyMethod = "close")
@Bean(destroyMethod = "close")
@ConditionalOnProperty(name = "timely.in-memory-accumulo.enabled", havingValue = "true")
@DependsOn("ZooKeeperServerMain")
public CuratorFramework inMemoryCuratorFramework(ZookeeperProperties zookeeperProperties) {
@@ -38,7 +36,7 @@ public class TestCuratorConfiguration {
return curatorFramework;
}
@Bean(name = "CuratorFramework", destroyMethod = "close")
@Bean(destroyMethod = "close")
@ConditionalOnProperty(name = "timely.mini-accumulo.enabled", havingValue = "true")
@DependsOn("MiniAccumuloCluster")
public CuratorFramework miniAccumuloCuratorFramework(ZookeeperProperties zookeeperProperties) {
@@ -1,4 +1,4 @@
package timely.test.configuration;
package timely.balancer.configuration;
import org.apache.accumulo.core.client.AccumuloClient;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
@@ -6,16 +6,18 @@ import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Scope;
import timely.balancer.test.BalancerTestRule;
import timely.common.configuration.MiniAccumuloProperties;
import timely.common.configuration.TimelyProperties;
import timely.test.TimelyTestRule;
@Configuration
@EnableConfigurationProperties({TimelyProperties.class})
@EnableConfigurationProperties({MiniAccumuloProperties.class, TimelyProperties.class})
public class TestWatcherConfiguration {
@Bean
@Scope("prototype")
public TimelyTestRule timelyTestWatcherAccumulo(AccumuloClient accumuloClient, TimelyProperties timelyProperties) throws Exception {
return new TimelyTestRule(timelyProperties, accumuloClient);
public BalancerTestRule timelyTestWatcherAccumulo(AccumuloClient accumuloClient, TimelyProperties timelyProperties) throws Exception {
return new BalancerTestRule(timelyProperties, accumuloClient);
}
}
@@ -0,0 +1,61 @@
package timely.balancer.test;
import org.apache.accumulo.core.client.AccumuloClient;
import org.junit.rules.TestWatcher;
import org.junit.runner.Description;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import timely.common.configuration.TimelyProperties;
import timely.util.TestUtils;
public class BalancerTestRule extends TestWatcher {
private static final Logger log = LoggerFactory.getLogger(BalancerTestRule.class);
private TimelyProperties timelyProperties;
private AccumuloClient accumuloClient;
public BalancerTestRule(TimelyProperties timelyProperties, AccumuloClient accumuloClient) {
this.timelyProperties = timelyProperties;
this.accumuloClient = accumuloClient;
}
@Override
protected void failed(Throwable e, Description description) {
log.error("Test {}:{} failed", description.getTestClass().getCanonicalName(), description.getMethodName());
log.error(e.getMessage(), e);
if (accumuloClient != null) {
if (accumuloClient.tableOperations().exists(timelyProperties.getMetricsTable())) {
separator();
log.info("Contents of {}", timelyProperties.getMetricsTable());
separator();
TestUtils.printTable(accumuloClient, timelyProperties.getMetricsTable(), true);
separator();
}
if (accumuloClient.tableOperations().exists(timelyProperties.getMetaTable())) {
log.info("Contents of {}", timelyProperties.getMetaTable());
separator();
TestUtils.printTable(accumuloClient, timelyProperties.getMetaTable(), false);
separator();
}
}
}
private void separator() {
log.info("-----------------------------------------------------------------------------------");
}
@Override
protected void starting(Description description) {
separator();
log.info("Starting test {}:{}", description.getTestClass().getCanonicalName(), description.getMethodName());
separator();
}
@Override
protected void finished(Description description) {
separator();
log.info("Finished test {}:{}", description.getTestClass().getCanonicalName(), description.getMethodName());
separator();
}
}
@@ -1,4 +1,4 @@
package timely.balancer.integration;
package timely.balancer.test;
import static timely.Constants.SERVICE_DISCOVERY_PATH;
@@ -27,11 +27,10 @@ import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.junit4.SpringRunner;
import timely.ServerDetails;
import timely.balancer.component.TestHealthChecker;
import timely.balancer.component.TestMetricResolver;
import timely.balancer.connection.TimelyBalancedHost;
import timely.balancer.test.TestHealthChecker;
import timely.balancer.test.TestMetricResolver;
import timely.test.IntegrationTest;
import timely.test.TimelyTestRule;
@Category(IntegrationTest.class)
@RunWith(SpringRunner.class)
@@ -42,7 +41,7 @@ public class MetricResolverIT {
@Autowired
@Rule
public TimelyTestRule testRule;
public BalancerTestRule balancerTestRule;
@Autowired
private TestMetricResolver metricResolver;
@@ -1,36 +0,0 @@
package timely.balancer.test;
import javax.net.ssl.SSLContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.context.ApplicationContext;
import io.netty.handler.ssl.SslContext;
import timely.balancer.Balancer;
import timely.balancer.MetricResolver;
import timely.balancer.configuration.BalancerHttpProperties;
import timely.balancer.configuration.BalancerProperties;
import timely.balancer.configuration.BalancerServerProperties;
import timely.balancer.configuration.BalancerWebsocketProperties;
import timely.common.component.AuthenticationService;
import timely.common.configuration.SecurityProperties;
import timely.common.configuration.SslClientProperties;
import timely.common.configuration.SslServerProperties;
import timely.common.configuration.ZookeeperProperties;
public class TestBalancer extends Balancer {
private static final Logger log = LoggerFactory.getLogger(TestBalancer.class);
public TestBalancer(ApplicationContext applicationContext, SslContext serverSslContext, SSLContext clientSSLContext,
AuthenticationService authenticationService, MetricResolver metricResolver, BalancerProperties balancerProperties,
ZookeeperProperties zookeeperProperties, SecurityProperties securityProperties, SslServerProperties sslServerProperties,
SslClientProperties sslClientProperties, BalancerServerProperties balancerServerProperties, BalancerHttpProperties balancerHttpProperties,
BalancerWebsocketProperties balancerWebsocketProperties) {
super(applicationContext, serverSslContext, clientSSLContext, authenticationService, metricResolver, balancerProperties, zookeeperProperties,
securityProperties, sslServerProperties, sslClientProperties, balancerServerProperties, balancerHttpProperties,
balancerWebsocketProperties);
DEFAULT_EVENT_LOOP_THREADS = 1;
log.info("Starting TestBalancer");
}
}
@@ -0,0 +1,48 @@
package timely.balancer.test;
public class TestServerInfo {
public String host;
public String ip;
public Integer tcpPort;
public Integer httpPort;
public Integer wsPort;
public Integer udpPort;
public TestServerInfo(String host, String ip, Integer tcpPort, Integer httpPort, Integer wsPort, Integer udpPort) {
this.host = host;
this.ip = ip;
this.tcpPort = tcpPort;
this.httpPort = httpPort;
this.wsPort = wsPort;
this.udpPort = udpPort;
}
static public TestServerInfo of(String host, String ip, Integer tcpPort, Integer httpPort, Integer wsPort, Integer udpPort) {
return new TestServerInfo(host, ip, tcpPort, httpPort, wsPort, udpPort);
}
public String getHost() {
return host;
}
public String getIp() {
return ip;
}
public Integer getTcpPort() {
return tcpPort;
}
public Integer getHttpPort() {
return httpPort;
}
public Integer getWsPort() {
return wsPort;
}
public Integer getUdpPort() {
return udpPort;
}
}
@@ -6,11 +6,6 @@ logging:
timely:
test: true
instance: 1
accumulo:
instance-name: default
username: root
password: secret
mini-accumulo:
enabled: true
rootAuths: A,B,C,D,E,F,G,H,I
@@ -23,87 +18,74 @@ timely:
cache-refresh-minutes: 5
allow-anonymous-http-access: true
allow-anonymous-ws-access: true
authorized-users:
- subject-dn: cn=example.com
authorizedUsers:
- subjectDn: cn=example.com
roles: G,H,I
auths: D,E,F
user-type: USER
- subject-dn: cn=test a. user, ou=example developers, o=example corp, c=us
issuer-dn: cn=example corp ca, o=example corp, c=us
userType: USER
- subjectDn: cn=test a. user, ou=example developers, o=example corp, c=us
issuerDn: cn=example corp ca, o=example corp, c=us
roles: G,H,I
auths: A,B,C
user-type: USER
server-ssl:
key-store-type: PKCS12
key-store-file: classpath:ssl/testServer.p12
key-store-password: ChangeIt
trust-store-type: JKS
trust-store-file: classpath:ssl/ca.jks
trust-store-password: ChangeIt
use-generated-keypair: false
ignore-ssl-handshake-errors: true
use-openssl: true
client-ssl:
key-store-type: PKCS12
key-store-file: classpath:ssl/testUser.p12
key-store-password: ChangeIt
trust-store-type: JKS
trust-store-file: classpath:ssl/ca.jks
trust-store-password: ChangeIt
userType: USER
serverSsl:
useGeneratedKeypair: true
ignoreSslHandshakeErrors: true
useOpenssl: true
cache:
enabled: false
metric-age-off-hours:
default: 26
non-cached-metrics:
nonCachedMetrics:
balancer:
check-server-health-interval: 10000
server-failures-before-down: 6
server-successes-before-up: 1
assignment-file: file://${java.io.tmpdir}/timely/balancer/assignments.csv
checkServerHealthInterval: 10000
serverFailuresBeforeDown: 6
serverSuccessesBeforeUp: 1
assignmentFile: file://${java.io.tmpdir}/timely/balancer/assignments.csv
server:
ip: 127.0.0.1
tcp-base-port: 4242
num-tcp-pools: 1
tcp-buffer-size: 100
tcp-client-pool:
min-idle-per-key: 0
max-idle-per-key: 40
max-total-per-key: 40
max-total: -1
tcpPort: 4242
numTcpPools: 1
tcpBufferSize: 100
tcpClientPool:
minIdlePerKey: 0
maxIdlePerKey: 40
maxTotalPerKey: 40
maxTotal: -1
lifo: true
fairness: false
max-wait-millis: -1
min-evictable-idle-time-millis: 1800000
evictor-shutdown-timeout-millis: 10000
soft-min-evictable-idle-time-millis: -1
num-tests-per-eviction-run: 3
time-between-eviction-runs-millis: -1
block-when-exhausted: true
jmx-enabled: true
test-on-borrow: false
test-on-return: false
udp-base-port: 4245
udp-client-pool:
min-idle-per-key: 0
max-idle-per-key: 40
max-total-per-key: 40
max-total: -1
maxWaitMillis: -1
minEvictableIdleTimeMillis: 1800000
evictorShutdownTimeoutMillis: 10000
softMinEvictableIdleTimeMillis: -1
numTestsPerEvictionRun: 3
timeBetweenEvictionRunsMillis: -1
blockWhenExhausted: true
jmxEnabled: true
testOnBorrow: false
testOnReturn: false
udpPort: 4245
udpClientPool:
minIdlePerKey: 0
maxIdlePerKey: 40
maxTotalPerKey: 40
maxTotal: -1
lifo: true
fairness: false
max-wait-millis: -1
min-evictable-idle-time-millis: 1800000
evictor-shutdown-timeout-millis: 10000
soft-min-evictable-idle-time-millis: -1
num-tests-per-eviction-run: 3
time-between-eviction-runs-millis: -1
block-when-exhausted: true
jmx-enabled: true
test-on-borrow: false
test-on-return: false
maxWaitMillis: -1
minEvictableIdleTimeMillis: 1800000
evictorShutdownTimeoutMillis: 10000
softMinEvictableIdleTimeMillis: -1
numTestsPerEvictionRun: 3
timeBetweenEvictionRunsMillis: -1
blockWhenExhausted: true
jmxEnabled: true
testOnBorrow: false
testOnReturn: false
http:
host: localhost
ip: 127.0.0.1
base-port: 4243
port: 4243
redirect-path: /secure-me
strict-transport-max-age: 604800
cors:
@@ -113,47 +95,47 @@ timely:
allowed-origins:
allowed-methods: DELETE,GET,HEAD,OPTIONS,PUT,POST
allowed-headers: content-type
http-client-pool:
min-idle-per-key: 0
max-idle-per-key: 40
max-total-per-key: 40
max-total: -1
httpClientPool:
minIdlePerKey: 0
maxIdlePerKey: 40
maxTotalPerKey: 40
maxTotal: -1
lifo: true
fairness: false
max-wait-millis: -1
min-evictable-idle-time-millis: 1800000
evictor-shutdown-timeout-millis: 10000
soft-min-evictable-idle-time-millis: -1
num-tests-per-eviction-run: 3
time-between-eviction-runs-millis: -1
block-when-exhausted: true
jmx-enabled: true
test-on-borrow: false
test-on-return: false
maxWaitMillis: -1
minEvictableIdleTimeMillis: 1800000
evictorShutdownTimeoutMillis: 10000
softMinEvictableIdleTimeMillis: -1
numTestsPerEvictionRun: 3
timeBetweenEvictionRunsMillis: -1
blockWhenExhausted: true
jmxEnabled: true
testOnBorrow: false
testOnReturn: false
websocket:
ip: 127.0.0.1
base-port: 5244
port: 5244
timeout: 60
subscription-lag: 120
scanner-batch-size: 10000
flush-interval-seconds: 30
scanner-read-ahead: 1
subscription-batch-size: 5000
incoming-buffer-size: 50000000
ws-client-pool:
min-idle-per-key: 0
max-idle-per-key: 40
max-total-per-key: 40
max-total: -1
incomingBufferSize: 50000000
wsClientPool:
minIdlePerKey: 0
maxIdlePerKey: 40
maxTotalPerKey: 40
maxTotal: -1
lifo: true
fairness: false
max-wait-millis: -1
min-evictable-idle-time-millis: 1800000
evictor-shutdown-timeout-millis: 10000
soft-min-evictable-idle-time-millis: -1
num-tests-per-eviction-run: 3
time-between-eviction-runs-millis: -1
block-when-exhausted: true
jmx-enabled: true
test-on-borrow: false
test-on-return: false
maxWaitMillis: -1
minEvictableIdleTimeMillis: 1800000
evictorShutdownTimeoutMillis: 10000
softMinEvictableIdleTimeMillis: -1
numTestsPerEvictionRun: 3
timeBetweenEvictionRunsMillis: -1
blockWhenExhausted: true
jmxEnabled: true
testOnBorrow: false
testOnReturn: false
-23
View File
@@ -1,23 +0,0 @@
Bag Attributes: <No Attributes>
subject=/C=US/O=Example Corp/CN=EXAMPLE CORP CA
issuer=/C=US/O=Example Corp/CN=EXAMPLE CORP CA
-----BEGIN CERTIFICATE-----
MIIDTjCCAjagAwIBAgIURUj8l+ms/i9jeabIBBi19KxEhAgwDQYJKoZIhvcNAQEL
BQAwPjELMAkGA1UEBhMCVVMxFTATBgNVBAoTDEV4YW1wbGUgQ29ycDEYMBYGA1UE
AxMPRVhBTVBMRSBDT1JQIENBMCAXDTIzMDIwMTIzMTUwMFoYDzIxMjMwMTA4MjMx
NTAwWjA+MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRXhhbXBsZSBDb3JwMRgwFgYD
VQQDEw9FWEFNUExFIENPUlAgQ0EwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK
AoIBAQDVBr2XF20SKmGXfvp2BUY44BF1fm3Qw83GkC66qIHgpgQxzJ8u44P34vV1
DYTj4XzrBpx23J0nCMEiAW8YW9MxZJPEokZafRNKFQAngVdgmAW9fHemHRQfaYwe
ySVz2UZ+z28DJ3C0QshvqXskrYstwLVs4Ss2uoX+XpeFbW9IVVg4aW+GX3LHPNTk
7Hji13Capp6BCiS61AGQAiMWNBW2aRgHVLOb7RDQug2Bka/dcJ44XxJDh26C6kV1
pNEZm0ToqWAiVfvyzionPec5im6NKX9kStZc+C/iikfx5vdmrvGIF5zxy6phdhJ7
AGd6cF6UBkkYm/SsIP3WTXpBek5LAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAP
BgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBSPLNlCmmp9PH+crzCx9TNp5BoCajAN
BgkqhkiG9w0BAQsFAAOCAQEAkjLnZKqLMjbu5Hy8YOVoKaEKahP+t/tNYZW/NtrC
xSxKmGLz5kVCkjpLgCykkcpu4h2wZW4MGZJ4nc87+tN22mJOg1WnqEpBzRsEPajk
7duVjdU2FDCarERj53hxF41l/u8BxvMGbFnrAijv5SzspeLi+VCGF3wu53Bi6UHi
7WO1dIhZLMHepqgtwbDDudxwlKbG/JWjJ7OspCNqOpktpY8BFUAQ42ctYZcOHy2w
kdZDcK5IhRztkBH2aSWOhudYh/CF7BOu2nSUdtR5NwhXnsStmSbud7ahw3W5Jc9/
IVAFXf2brkrfIzwrY5kmjnEPJ8mjUyLepwSlOlMTBrehzQ==
-----END CERTIFICATE-----
Binary file not shown.
@@ -1,51 +0,0 @@
Bag Attributes
localKeyID: 22 89 1C EA 4E FF 9A F6 11 1B E7 9B 5E 4F 59 B7 B1 0F AA FA
friendlyName: example-server
subject=/C=US/O=Example Corp/OU=Servers/CN=testserver.example.com
issuer=/C=US/O=Example Corp/CN=EXAMPLE CORP CA
-----BEGIN CERTIFICATE-----
MIIDwjCCAqqgAwIBAgIUe6yBgkY1m/oXuPXA9j/c/7+w16UwDQYJKoZIhvcNAQEL
BQAwPjELMAkGA1UEBhMCVVMxFTATBgNVBAoTDEV4YW1wbGUgQ29ycDEYMBYGA1UE
AxMPRVhBTVBMRSBDT1JQIENBMCAXDTIzMDIwMTIzMTUwMFoYDzIxMjMwMTA4MjMx
NTAwWjBXMQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRXhhbXBsZSBDb3JwMRAwDgYD
VQQLEwdTZXJ2ZXJzMR8wHQYDVQQDExZ0ZXN0c2VydmVyLmV4YW1wbGUuY29tMIIB
IjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAmb1pEWg9a6ZChQULazK75ppc
XScU5Ap29L70/5zcaOcbaBwqKhpKhdZCoZ1TD+Sdkk9kp7IlabCQjM2vVjknb6NY
msl3PbQFNva0EkH6s5cvpX4WhQDsWK77/4eztHT+0fpaG1YoKQn+Gy3VuW0CDyR6
GzGKifnI/aNLevgUBebbiiTPLEdGlo0jp3SXlZfoDZdbvnhBWI8Xc+aDekNlrXxW
X2TVnc02Fi914PECYJzkMpmxnBRPAcljU24WSqaUVgRoSbafkm4OXDw0xtvxnIrJ
s3iO0uZfRf7+OxAlqlvh4ZJ9/PufPMEKvmr6zea05KkFjBotKMip5Md43H3lQQID
AQABo4GcMIGZMA4GA1UdDwEB/wQEAwIFoDAdBgNVHSUEFjAUBggrBgEFBQcDAQYI
KwYBBQUHAwIwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQUDWwMrl9/uc1wplERWxas
bDROB4MwHwYDVR0jBBgwFoAUjyzZQppqfTx/nK8wsfUzaeQaAmowGgYDVR0RBBMw
EYIJbG9jYWxob3N0hwR/AAABMA0GCSqGSIb3DQEBCwUAA4IBAQBfG7StgQf3WNk/
EnaYuurwvaOOpcOV5NDcZ6tmgaXUrISPVDpyIWfsDGetNEXc+4gpefLQtGF2wuiW
odiLc4H9JwQ/yEkVYwzrFViPdZKVvxNU2EjiwO/e/3sl+SxxkTDy6o838uTXX+X5
Q8DabmL5L8ukcnqUHVOqW38oA1vRNBKiJX58JzOr8ezX8crZmqU9h8cNwG/V+ZXE
5U7+hSRzLm7gRwtixj8ZsTDUiGJxOdbyUKImPClt1Se9TizB2Q/xOPTdmYB1qiiB
+XVKkGwQPgIbHckLsZWhBor2u5cgdrBGW09Q+H88maagwq5sBHNMs33vkkgbUDpB
6WSEBJMi
-----END CERTIFICATE-----
Bag Attributes: <No Attributes>
subject=/C=US/O=Example Corp/CN=EXAMPLE CORP CA
issuer=/C=US/O=Example Corp/CN=EXAMPLE CORP CA
-----BEGIN CERTIFICATE-----
MIIDTjCCAjagAwIBAgIURUj8l+ms/i9jeabIBBi19KxEhAgwDQYJKoZIhvcNAQEL
BQAwPjELMAkGA1UEBhMCVVMxFTATBgNVBAoTDEV4YW1wbGUgQ29ycDEYMBYGA1UE
AxMPRVhBTVBMRSBDT1JQIENBMCAXDTIzMDIwMTIzMTUwMFoYDzIxMjMwMTA4MjMx
NTAwWjA+MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRXhhbXBsZSBDb3JwMRgwFgYD
VQQDEw9FWEFNUExFIENPUlAgQ0EwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK
AoIBAQDVBr2XF20SKmGXfvp2BUY44BF1fm3Qw83GkC66qIHgpgQxzJ8u44P34vV1
DYTj4XzrBpx23J0nCMEiAW8YW9MxZJPEokZafRNKFQAngVdgmAW9fHemHRQfaYwe
ySVz2UZ+z28DJ3C0QshvqXskrYstwLVs4Ss2uoX+XpeFbW9IVVg4aW+GX3LHPNTk
7Hji13Capp6BCiS61AGQAiMWNBW2aRgHVLOb7RDQug2Bka/dcJ44XxJDh26C6kV1
pNEZm0ToqWAiVfvyzionPec5im6NKX9kStZc+C/iikfx5vdmrvGIF5zxy6phdhJ7
AGd6cF6UBkkYm/SsIP3WTXpBek5LAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAP
BgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBSPLNlCmmp9PH+crzCx9TNp5BoCajAN
BgkqhkiG9w0BAQsFAAOCAQEAkjLnZKqLMjbu5Hy8YOVoKaEKahP+t/tNYZW/NtrC
xSxKmGLz5kVCkjpLgCykkcpu4h2wZW4MGZJ4nc87+tN22mJOg1WnqEpBzRsEPajk
7duVjdU2FDCarERj53hxF41l/u8BxvMGbFnrAijv5SzspeLi+VCGF3wu53Bi6UHi
7WO1dIhZLMHepqgtwbDDudxwlKbG/JWjJ7OspCNqOpktpY8BFUAQ42ctYZcOHy2w
kdZDcK5IhRztkBH2aSWOhudYh/CF7BOu2nSUdtR5NwhXnsStmSbud7ahw3W5Jc9/
IVAFXf2brkrfIzwrY5kmjnEPJ8mjUyLepwSlOlMTBrehzQ==
-----END CERTIFICATE-----
@@ -1,32 +0,0 @@
Bag Attributes
localKeyID: 22 89 1C EA 4E FF 9A F6 11 1B E7 9B 5E 4F 59 B7 B1 0F AA FA
friendlyName: example-server
Key Attributes: <No Attributes>
-----BEGIN PRIVATE KEY-----
MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCZvWkRaD1rpkKF
BQtrMrvmmlxdJxTkCnb0vvT/nNxo5xtoHCoqGkqF1kKhnVMP5J2ST2SnsiVpsJCM
za9WOSdvo1iayXc9tAU29rQSQfqzly+lfhaFAOxYrvv/h7O0dP7R+lobVigpCf4b
LdW5bQIPJHobMYqJ+cj9o0t6+BQF5tuKJM8sR0aWjSOndJeVl+gNl1u+eEFYjxdz
5oN6Q2WtfFZfZNWdzTYWL3Xg8QJgnOQymbGcFE8ByWNTbhZKppRWBGhJtp+Sbg5c
PDTG2/GcismzeI7S5l9F/v47ECWqW+Hhkn38+588wQq+avrN5rTkqQWMGi0oyKnk
x3jcfeVBAgMBAAECggEAZEHC7NkfhL4NjxV1Ftkk9AMhzJWZ6ARzArQsrSysB7T+
iGwliwNOKe1J0n74FdsubqVbrIheIYhFG61edEodT44kT/ylyBVHeUp0r5tbVBZ5
ueB56w5pEoQ6Ta8/H6VUL7alYEJVOj4L+clwINXXi+xrvKQYBlZgHheJN43j8NxS
9lCrVcm58rzX3i29S7ErBf7RqDabgD6j2m7rehnSq5Ps2wxwV/LXMl4Lxlwa0jz/
LV8szNMZViPOWAqOCy7NZbwCCaCfKVUAwmNGBkE3p4JVPeLNLzCBY9G0rHVmFJLK
yPnCB7PtCZeFTsU+8OHHWoxMFsaOO9Pp8FtB/r8PYQKBgQDI0OzEo3t/QKvbwAdz
9Eze0LQvstzZIidTWbYMTxH0g7DIt+3p11d61GKtfeuYUTPlLodufdv/BaP8G9sI
NzxisB3uH/787M9fwi1OzO7zMMZZCVa7Q1vZk6ta1qY3PekcYMPqjBJhxP1X3S7x
8+mrW5vTq/l8ZRVFeLvEkh6bpQKBgQDD/MCRJoq6nzd2DoTyT9m5qO+Jlztmu9+T
vqVOg7q0NO7U7dzyMbWgMvMFoEBidnyllU1rEj2YOo4OrI2v/JmSPUVS8VXsBkkG
UqInV4F/g0Wp3hzrkZFMqHLa6AydugmRNkOpgurRKOiwAL0Pgvk8+bOIbKMM7JBE
tZ6Zvi0gbQKBgD4S5r0505jYQsdgqm0bXQ+a77e20DGGLyXCm1Ves6wjcOf6gt4/
pFjhgccjH0or/6CeK1rqKLDB65ioe6IkgNP4oKfU2BsSkF+HRD4RduNRdl5bVqX4
AlJVJSilJSf6SS5ucuDzf1MdczpPso0z2k0PujFJ/YqILHUXBuOST0fJAoGAZAdX
nX3HZpKmsumcLzRUEq9HerkmRzvyU60FH/qZyyoND75Le42lfpsicCenVNnWqoh2
WASaTVztbk3hjwXkt7/v5vJoASwAD2sTefLulE9Cuf00U4MbXr3GyTWQYtw1z9ib
ViQMhy5S/KmT1Rvq65UibanM2mbPB1KPHTb0P00CgYEAgieJ1YXYYHPjABWJ7/S9
3Z/8eJY0kuUe1DRFaKOlfej8SOCFFIa90IW2mAnujh55H4N2lww4s/N+FIRQoQpA
1Hrch9tYRiXyouQc67dFlLxmCOVJIf31ZUt7I4HcPhsfr4fuCJXUPBs1jaDmwQsk
90VWrXaTwSjmP+yHMiPfaA0=
-----END PRIVATE KEY-----
Binary file not shown.
@@ -1,50 +0,0 @@
Bag Attributes
localKeyID: 57 F6 9F DF B2 94 61 47 B0 B0 2E 3A F7 79 98 C8 15 5C 64 FA
friendlyName: example-client
subject=/C=US/O=Example Corp/OU=Example Developers/CN=Test A. User
issuer=/C=US/O=Example Corp/CN=EXAMPLE CORP CA
-----BEGIN CERTIFICATE-----
MIIDmzCCAoOgAwIBAgIUFp4IuIaZ2XId7UNvzP6ZB9hChyUwDQYJKoZIhvcNAQEL
BQAwPjELMAkGA1UEBhMCVVMxFTATBgNVBAoTDEV4YW1wbGUgQ29ycDEYMBYGA1UE
AxMPRVhBTVBMRSBDT1JQIENBMCAXDTIzMDIwMTIzMTUwMFoYDzIxMjMwMTA4MjMx
NTAwWjBYMQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRXhhbXBsZSBDb3JwMRswGQYD
VQQLExJFeGFtcGxlIERldmVsb3BlcnMxFTATBgNVBAMTDFRlc3QgQS4gVXNlcjCC
ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMOMeN0ElMDxYKST2VD1wP6Z
jmy543Zv/QgRLTYizg3U9t3uZU0C3HqUJnK+c4+OJnRIyHuMXuJWEYVdairyAdYW
ZDF5sfiL9ikG9rZ2WxR2z7nsYLJcDGwKAt/Duzuoed6KSccEpiQc+BkoOTs9p87I
l4s7saX/anESMUY7YOomLSqS2zvQZh/zTjIdP1/9kG9PLzXPClJELuiC3pJCmr7Y
xdJIXecu6YKFpApy0y+rKcfYUVPML15gyRu9pIqH3U39y7mCkn+YP9Qc9GcZAoqR
iYdUEl1rFAgV/a3o0pmf1tFFRDOsoPSMUGNNQlaCjv1uZIbRm25nvvQlPfstFDEC
AwEAAaN1MHMwDgYDVR0PAQH/BAQDAgWgMBMGA1UdJQQMMAoGCCsGAQUFBwMCMAwG
A1UdEwEB/wQCMAAwHQYDVR0OBBYEFPAe3n59DcNwRcnpIwQY2kPawbCVMB8GA1Ud
IwQYMBaAFI8s2UKaan08f5yvMLH1M2nkGgJqMA0GCSqGSIb3DQEBCwUAA4IBAQBo
+zadU9DwR1rnHTBZF6CJXmiIdrQmArojH4a3NOHbNjFjRXm7dkd/tAy5LtgDJkww
yg+ZrOyOKHbFgk6IXRZ8nw2b1Ck9I9MDW1vxfjid9YLW2nhDaMcIeTXz3aIBO5i6
gMCV+xMcFaU/O57Ohia+ADUqrqu14nG9Moi171UWPbMr5N3gYn3Fs4VrTGYs4i0Z
5x2itm88WB3bWj+cd06zKQopU9b0Y76mQv9h+w4EHxq4uWA7GmIDDkvtI61MQo8+
twdX159Ms6o4lW43SRqVBHeMgSJbiLDkdkypYJvd9PvAjWgmQK/sD1SYKxfgYhtx
hqIvAjlGeDwdf+55EGjf
-----END CERTIFICATE-----
Bag Attributes: <No Attributes>
subject=/C=US/O=Example Corp/CN=EXAMPLE CORP CA
issuer=/C=US/O=Example Corp/CN=EXAMPLE CORP CA
-----BEGIN CERTIFICATE-----
MIIDTjCCAjagAwIBAgIURUj8l+ms/i9jeabIBBi19KxEhAgwDQYJKoZIhvcNAQEL
BQAwPjELMAkGA1UEBhMCVVMxFTATBgNVBAoTDEV4YW1wbGUgQ29ycDEYMBYGA1UE
AxMPRVhBTVBMRSBDT1JQIENBMCAXDTIzMDIwMTIzMTUwMFoYDzIxMjMwMTA4MjMx
NTAwWjA+MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRXhhbXBsZSBDb3JwMRgwFgYD
VQQDEw9FWEFNUExFIENPUlAgQ0EwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK
AoIBAQDVBr2XF20SKmGXfvp2BUY44BF1fm3Qw83GkC66qIHgpgQxzJ8u44P34vV1
DYTj4XzrBpx23J0nCMEiAW8YW9MxZJPEokZafRNKFQAngVdgmAW9fHemHRQfaYwe
ySVz2UZ+z28DJ3C0QshvqXskrYstwLVs4Ss2uoX+XpeFbW9IVVg4aW+GX3LHPNTk
7Hji13Capp6BCiS61AGQAiMWNBW2aRgHVLOb7RDQug2Bka/dcJ44XxJDh26C6kV1
pNEZm0ToqWAiVfvyzionPec5im6NKX9kStZc+C/iikfx5vdmrvGIF5zxy6phdhJ7
AGd6cF6UBkkYm/SsIP3WTXpBek5LAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAP
BgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBSPLNlCmmp9PH+crzCx9TNp5BoCajAN
BgkqhkiG9w0BAQsFAAOCAQEAkjLnZKqLMjbu5Hy8YOVoKaEKahP+t/tNYZW/NtrC
xSxKmGLz5kVCkjpLgCykkcpu4h2wZW4MGZJ4nc87+tN22mJOg1WnqEpBzRsEPajk
7duVjdU2FDCarERj53hxF41l/u8BxvMGbFnrAijv5SzspeLi+VCGF3wu53Bi6UHi
7WO1dIhZLMHepqgtwbDDudxwlKbG/JWjJ7OspCNqOpktpY8BFUAQ42ctYZcOHy2w
kdZDcK5IhRztkBH2aSWOhudYh/CF7BOu2nSUdtR5NwhXnsStmSbud7ahw3W5Jc9/
IVAFXf2brkrfIzwrY5kmjnEPJ8mjUyLepwSlOlMTBrehzQ==
-----END CERTIFICATE-----
@@ -1,32 +0,0 @@
Bag Attributes
localKeyID: 57 F6 9F DF B2 94 61 47 B0 B0 2E 3A F7 79 98 C8 15 5C 64 FA
friendlyName: example-client
Key Attributes: <No Attributes>
-----BEGIN PRIVATE KEY-----
MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDDjHjdBJTA8WCk
k9lQ9cD+mY5sueN2b/0IES02Is4N1Pbd7mVNAtx6lCZyvnOPjiZ0SMh7jF7iVhGF
XWoq8gHWFmQxebH4i/YpBva2dlsUds+57GCyXAxsCgLfw7s7qHneiknHBKYkHPgZ
KDk7PafOyJeLO7Gl/2pxEjFGO2DqJi0qkts70GYf804yHT9f/ZBvTy81zwpSRC7o
gt6SQpq+2MXSSF3nLumChaQKctMvqynH2FFTzC9eYMkbvaSKh91N/cu5gpJ/mD/U
HPRnGQKKkYmHVBJdaxQIFf2t6NKZn9bRRUQzrKD0jFBjTUJWgo79bmSG0ZtuZ770
JT37LRQxAgMBAAECggEBALFWsT1vLCAWRxi55Hkhhhbe12UO55O/V+969E+GHOw6
WG9IBaJjl+Oqe3vP8CkeNw15/CbfVCVa/KS/5Mpxkuy+fbrIrHfFJkZqxE8Q4uSa
IyRvk/nstG8+XQDlFuWxkmQZcXMFak6R3wWhjtWEwqwrChQA3Mi366YDY6BXtTpi
MuLJVWXqYVF2RirWX3a7eosR4MYBCJ1S8NnpI1w4oZwuew6iYiCJNkURctxnGKOe
hoop/711PEyIb/qHETBIlReY9scvZgJQZMgGe5mTGgFmAA8Bzo4e1/F2QrVIi7wy
REOSj4U+tP/QrYMSPk0cMpH5ft5+2mfh/o+D1ugm/AkCgYEA2qt3P4vlm1TMmJru
vMYc8iZjfe7UQTnD+J1EgrASBI0JP565zSZKBRIXnheumWhWS+2KR1MP8N6KAu4Q
xE/E7TKfLuE1CPzn7Rh/WZAxDqGNeQthk0qTbeBZ4/mHPKvt8V7c+N4vLjZ00uul
bKitlLpb49T+8FpSgRmk+nrBsesCgYEA5O6LmxH1+ip5NoMHoZVBhLbgJkxoXEzn
xHT7On8CjQ9B0T85NW9O38/a2gyIb5dIavcrh4dMvOKX8hdrVlal3chrXFRemyvy
VzpW7HZpMwthJb/QSim+GQ4u2Jsm+eXQAswq/yfowls9mp7K81McBgdfcdWgWuqc
Ttf3dBRS71MCgYBCPpu7ppgCfKgWuykEC0iNm0+cwROQRv21fX9Z0BlFQ6y/l0hW
e/kEnDyHakk8tD0gd2/it6kmoWRSCukNb9UKvsE2pIc6l9NzIm7CJUqXUh3Xw7jl
tDwBl2vkAp4TpUpKSyDKTBUi/OBbXYQZ7OfZHLPly4sIKXTBI5G1SCcRzQKBgE3Y
Y7ZmRBs+CBumMJkcVLwZJ10+jBoeuIzQLoF0oNxwfLP3aK+n2s02M7gDnP09Fw3w
YGfkdQ6hioNyvTgNnLcuqHsr4UOojO5mUxpzJVCLJQyeDei6Cv8gGIPlNjqxj6rz
3AcJfhdgyTDjoCVfdF8EcMsXlkB0GJSi0W0yYAUrAoGAVK5v0cbEEV+gxdQgh848
FwSQXx7i9jVOB2BUTLX2Q+gLUJibtFzLnh03sGCXtTa8sK+hLW4FCMWZa9Ev5ZjG
ylFyA96U97CNpS+2/ZW5+hvRZ/GuY/0yB8w26BeS38lI8RnmxSPi5cnexdl32NIX
5FkU9WJ4HKmi1n0304CkF+4=
-----END PRIVATE KEY-----
Binary file not shown.
+2 -2
View File
@@ -4,13 +4,13 @@
<parent>
<groupId>timely</groupId>
<artifactId>timely</artifactId>
<version>2.1.1</version>
<version>2.1.0-SNAPSHOT</version>
</parent>
<artifactId>timely-client</artifactId>
<properties>
<!-- PEP 440 compliant version gets substituted into src/main/python/template/setup.py -->
<!-- use .dev0 instead of -SNAPSHOT -->
<python.client.version>2.1.1</python.client.version>
<python.client.version>2.1.0.dev0</python.client.version>
</properties>
<dependencies>
<dependency>
+1 -1
View File
@@ -4,7 +4,7 @@
<parent>
<groupId>timely</groupId>
<artifactId>timely</artifactId>
<version>2.1.1</version>
<version>2.1.0-SNAPSHOT</version>
</parent>
<artifactId>collectd-nsq-plugin</artifactId>
<dependencies>
+1 -1
View File
@@ -4,7 +4,7 @@
<parent>
<groupId>timely</groupId>
<artifactId>timely</artifactId>
<version>2.1.1</version>
<version>2.1.0-SNAPSHOT</version>
</parent>
<artifactId>collectd-plugin-parent</artifactId>
<dependencies>
+1 -1
View File
@@ -4,7 +4,7 @@
<parent>
<groupId>timely</groupId>
<artifactId>timely</artifactId>
<version>2.1.1</version>
<version>2.1.0-SNAPSHOT</version>
</parent>
<artifactId>collectd-timely-plugin</artifactId>
<dependencies>
+5 -5
View File
@@ -4,7 +4,7 @@
<parent>
<groupId>timely</groupId>
<artifactId>timely</artifactId>
<version>2.1.1</version>
<version>2.1.0-SNAPSHOT</version>
</parent>
<artifactId>timely-common</artifactId>
<dependencies>
@@ -80,10 +80,6 @@
<groupId>io.projectreactor.netty</groupId>
<artifactId>reactor-netty</artifactId>
</dependency>
<dependency>
<groupId>javax.xml.bind</groupId>
<artifactId>jaxb-api</artifactId>
</dependency>
<dependency>
<groupId>org.apache.accumulo</groupId>
<artifactId>accumulo-core</artifactId>
@@ -98,6 +94,10 @@
<groupId>org.apache.accumulo</groupId>
<artifactId>accumulo-core</artifactId>
</dependency>
<dependency>
<groupId>org.apache.accumulo</groupId>
<artifactId>accumulo-minicluster</artifactId>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
@@ -34,7 +34,7 @@ import timely.common.configuration.SecurityProperties;
*/
public class JWTTokenHandler {
private static final Logger log = LoggerFactory.getLogger(JWTTokenHandler.class);
private static final Logger logger = LoggerFactory.getLogger(JWTTokenHandler.class);
private static Key signatureCheckKey;
private static ObjectMapper objectMapper;
private static Collection<String> accumuloAuths = new TreeSet<>();
@@ -68,17 +68,17 @@ public class JWTTokenHandler {
Authorizations currentAccumuloAuths = accumuloClient.securityOperations().getUserAuthorizations(accumuloClient.whoami());
currentAccumuloAuths.iterator().forEachRemaining(a -> accumuloAuths.add(new String(a, Charset.forName("UTF-8"))));
} catch (Exception e) {
log.error(e.getMessage(), e);
logger.error(e.getMessage(), e);
}
}
}
public static Collection<TimelyUser> createUsersFromToken(String token, String claimName) {
log.trace("Attempting to parse JWT {}", token);
logger.trace("Attempting to parse JWT {}", token);
JwtParser jwtParser = Jwts.parser().setSigningKey(signatureCheckKey).build();
Jws<Claims> claimsJws = jwtParser.parseClaimsJws(token);
Claims claims = claimsJws.getBody();
log.trace("Resulting claims: {}", claims);
logger.trace("Resulting claims: {}", claims);
List<?> principalsClaim = claims.get(claimName, List.class);
if (principalsClaim == null || principalsClaim.isEmpty()) {
throw new IllegalArgumentException("JWT for " + claims.getSubject() + " does not contain any proxied principals.");
@@ -1,13 +0,0 @@
package timely.auth;
import java.util.List;
import org.springframework.security.authentication.AuthenticationProvider;
import org.springframework.security.authentication.ProviderManager;
public class TimelyAuthenticationManager extends ProviderManager {
public TimelyAuthenticationManager(List<AuthenticationProvider> providers) {
super(providers);
}
}
@@ -16,7 +16,7 @@ import org.apache.accumulo.core.security.Authorizations;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.security.authentication.ProviderManager;
import org.springframework.security.authentication.AuthenticationManager;
import org.springframework.security.core.Authentication;
import org.springframework.stereotype.Component;
@@ -32,7 +32,6 @@ import timely.api.request.AuthenticatedWebSocketRequest;
import timely.api.response.TimelyException;
import timely.auth.JWTTokenHandler;
import timely.auth.SubjectIssuerDNPair;
import timely.auth.TimelyAuthenticationManager;
import timely.auth.TimelyAuthenticationToken;
import timely.auth.TimelyPrincipal;
import timely.auth.TimelyUser;
@@ -47,7 +46,7 @@ public class AuthenticationService {
public static final String AUTH_HEADER = "Authorization";
public static final String PRINCIPALS_CLAIM = "principals";
private ProviderManager authenticationManager;
private AuthenticationManager authenticationManager;
private ConcurrentHashMap<String,Object> fetchingEntity = new ConcurrentHashMap<>();
private ConcurrentHashMap<String,Object> fetchingPrincipal = new ConcurrentHashMap<>();
@@ -154,7 +153,7 @@ public class AuthenticationService {
}
verifiedPrincipal = new TimelyPrincipal(authenticatedTimelyUsers);
this.authCache.put(verifiedPrincipal.getName(), verifiedPrincipal);
log.debug("Authenticated user {} with authorizations {}", verifiedPrincipal.getName(),
log.debug("Authenticated user {} for request {} with authorizations {}", verifiedPrincipal.getName(), request.toString(),
verifiedPrincipal.getAuthorizationsString());
} else {
log.trace("Verified principal {} in AuthCache on 2nd attempt", entity);
@@ -0,0 +1,63 @@
package timely.common.component;
import javax.net.ssl.SSLContext;
import org.apache.http.client.HttpClient;
import org.apache.http.conn.ssl.NoopHostnameVerifier;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.client.HttpClients;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.boot.autoconfigure.condition.ConditionalOnBean;
import org.springframework.boot.web.client.RestTemplateCustomizer;
import org.springframework.core.annotation.Order;
import org.springframework.http.client.ClientHttpRequestFactory;
import org.springframework.http.client.HttpComponentsClientHttpRequestFactory;
import org.springframework.stereotype.Component;
import org.springframework.web.client.RestTemplate;
import timely.common.configuration.RestClientProperties;
/**
* Customizes Spring {@link RestTemplate} instances by using our configured SSL certificate to present a client certificate whenever asked by remote services.
*/
@Component
@Order(100) // execute this after standard customizers so we're sure to overwrite the client
// request factory
@ConditionalOnBean(name = "outboundJDKSslContext")
public class ClientCertRestTemplateCustomizer implements RestTemplateCustomizer {
private final SSLContext sslContext;
private final int maxConnectionsTotal;
private final int maxConnectionsPerRoute;
@Autowired
public ClientCertRestTemplateCustomizer(@Qualifier("outboundJDKSslContext") SSLContext sslContext, RestClientProperties restClientProperties) {
this.sslContext = sslContext;
this.maxConnectionsTotal = restClientProperties.getMaxConnectionsTotal();
this.maxConnectionsPerRoute = restClientProperties.getMaxConnectionsPerRoute();
}
@Override
public void customize(RestTemplate restTemplate) {
restTemplate.setRequestFactory(clientHttpRequestFactory());
}
protected ClientHttpRequestFactory clientHttpRequestFactory() {
HttpClient httpClient = customizeHttpClient(HttpClients.custom(), sslContext).build();
return new HttpComponentsClientHttpRequestFactory(httpClient);
}
protected HttpClientBuilder customizeHttpClient(HttpClientBuilder httpClientBuilder, SSLContext sslContext) {
if (sslContext != null) {
httpClientBuilder.setSSLContext(sslContext);
}
httpClientBuilder.setMaxConnTotal(maxConnectionsTotal);
httpClientBuilder.setMaxConnPerRoute(maxConnectionsPerRoute);
// TODO: We're allowing all hosts, since the cert presented by the service we're
// calling likely won't match its hostname (e.g., a docker host name)
// Instead, we could list the expected cert as a property (or use our server
// cert), and verify that the presented name matches.
return httpClientBuilder.setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE);
}
}
@@ -0,0 +1,36 @@
package timely.common.component;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.boot.autoconfigure.condition.ConditionalOnBean;
import org.springframework.boot.web.reactive.function.client.WebClientCustomizer;
import org.springframework.core.annotation.Order;
import org.springframework.http.client.reactive.ReactorClientHttpConnector;
import org.springframework.stereotype.Component;
import org.springframework.web.reactive.function.client.WebClient;
import io.netty.handler.ssl.SslContext;
import reactor.netty.http.client.HttpClient;
/**
* Customizes the Spring-provided {@link org.springframework.web.reactive.function.client.WebClient.Builder} in order to supply an {@link SslContext} that will
* provide a client certificate to the remote server if one is requested.
*/
@Component
@Order(100) // execute this after standard customizers so we're sure to overwrite the client
// connector
@ConditionalOnBean(name = "outboundNettySslContext")
public class ClientCertWebClientCustomizer implements WebClientCustomizer {
private final SslContext sslContext;
public ClientCertWebClientCustomizer(@Qualifier("outboundNettySslContext") SslContext sslContext) {
this.sslContext = sslContext;
}
@Override
public void customize(WebClient.Builder webClientBuilder) {
HttpClient httpClient = HttpClient.create().secure(sslContextSpec -> sslContextSpec.sslContext(sslContext));
ReactorClientHttpConnector clientHttpConnector = new ReactorClientHttpConnector(httpClient);
webClientBuilder.clientConnector(clientHttpConnector);
}
}
@@ -1,4 +1,4 @@
package timely.auth;
package timely.common.component;
import static timely.common.component.AuthenticationService.PRINCIPALS_CLAIM;
@@ -14,8 +14,14 @@ import org.springframework.security.core.AuthenticationException;
import org.springframework.security.core.userdetails.AuthenticationUserDetailsService;
import org.springframework.security.core.userdetails.UsernameNotFoundException;
import org.springframework.security.web.authentication.preauth.PreAuthenticatedAuthenticationToken;
import org.springframework.stereotype.Component;
import org.springframework.web.reactive.function.client.WebClient;
import timely.auth.JWTTokenHandler;
import timely.auth.SubjectIssuerDNPair;
import timely.auth.TimelyAuthenticationToken;
import timely.auth.TimelyPrincipal;
import timely.auth.TimelyUser;
import timely.auth.util.ProxiedEntityUtils;
import timely.common.configuration.SecurityProperties;
@@ -28,21 +34,22 @@ import timely.common.configuration.SecurityProperties;
* production, it is likely better for the JWT to be retrieved by a load balancer or other API gateway and inject the JWT header before calling a service.
* Therefore, this service is mostly useful for debugging and if there are problems with the suggested approach.
*/
@Component
public class RemoteAuthenticationProvider implements AuthenticationProvider {
private static final Logger log = LoggerFactory.getLogger(RemoteAuthenticationProvider.class);
private final Logger logger = LoggerFactory.getLogger(getClass());
private final WebClient webClient;
public static final String ENTITIES_HEADER = "X-ProxiedEntitiesChain";
public static final String ISSUERS_HEADER = "X-ProxiedIssuersChain";
public RemoteAuthenticationProvider(WebClient.Builder builder, SecurityProperties securityProperties) {
log.debug("Creating RemoteAuthenticationProvider");
this.webClient = builder.baseUrl(securityProperties.getAuthorizationUrl()).build();
this.webClient = builder.baseUrl(securityProperties.getAuthorizationUri()).build();
}
public TimelyPrincipal fetchTimelyPrincipal(TimelyAuthenticationToken token) throws UsernameNotFoundException {
log.debug("Authenticating {}", token.getTimelyPrincipal().getUsername());
logger.debug("Authenticating {} via the authorization microservice", token);
TimelyPrincipal principal = token.getTimelyPrincipal();
try {
// @formatter:off
@@ -53,7 +60,7 @@ public class RemoteAuthenticationProvider implements AuthenticationProvider {
.bodyToMono(String.class)
.block();
// @formatter:on
log.debug("Authenticated {} with {} result", principal.getUsername(), (jwt == null) ? "null" : "non-null");
logger.debug("Authentication {} on remote authentication service with {} result", principal.getUsername(), (jwt == null) ? "null" : "non-null");
if (jwt != null) {
Collection<TimelyUser> principals = JWTTokenHandler.createUsersFromToken(jwt, PRINCIPALS_CLAIM);
long createTime = principals.stream().map(TimelyUser::getCreationTime).min(Long::compareTo).orElse(System.currentTimeMillis());
@@ -62,7 +69,7 @@ public class RemoteAuthenticationProvider implements AuthenticationProvider {
throw new UsernameNotFoundException("No entities found for " + principal.getUsername());
}
} catch (Exception e) {
log.error("Failed performing lookup of {}: {}", principal.getUsername(), e);
logger.error("Failed performing lookup of {}: {}", principal.getUsername(), e);
throw new UsernameNotFoundException(e.getMessage(), e);
}
}
@@ -0,0 +1,18 @@
package timely.common.component;
import java.util.Arrays;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.security.authentication.ProviderManager;
import org.springframework.security.web.authentication.preauth.PreAuthenticatedAuthenticationProvider;
import org.springframework.stereotype.Component;
@Component
public class TimelyAuthenticationManager extends ProviderManager {
@Autowired
public TimelyAuthenticationManager(RemoteAuthenticationProvider remoteAuthenticationProvider,
PreAuthenticatedAuthenticationProvider fileAuthenticationProvider) {
super(Arrays.asList(remoteAuthenticationProvider, fileAuthenticationProvider));
}
}
@@ -2,7 +2,6 @@ package timely.common.configuration;
import org.apache.accumulo.core.client.Accumulo;
import org.apache.accumulo.core.client.AccumuloClient;
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.annotation.Bean;
@@ -10,7 +9,6 @@ import org.springframework.context.annotation.Configuration;
@Configuration
@EnableConfigurationProperties(AccumuloProperties.class)
@ConditionalOnExpression("T(org.apache.commons.lang3.StringUtils).isNotEmpty('${timely.accumulo.instance-name:}')")
public class AccumuloConfiguration {
@Bean
@@ -15,6 +15,7 @@ public class AccumuloProperties {
private String username;
@NotBlank
private String password;
private int accumuloClientPoolSize = 16;
@Valid
@NestedConfigurationProperty
private WriteProperties writeProperties = new WriteProperties();
@@ -53,4 +54,12 @@ public class AccumuloProperties {
public void setPassword(String password) {
this.password = password;
}
public void setAccumuloClientPoolSize(int accumuloClientPoolSize) {
this.accumuloClientPoolSize = accumuloClientPoolSize;
}
public int getAccumuloClientPoolSize() {
return accumuloClientPoolSize;
}
}
@@ -0,0 +1,46 @@
package timely.common.configuration;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.cloud.context.config.annotation.RefreshScope;
@RefreshScope
@ConfigurationProperties(prefix = "timely.security.authorization.remote")
public class AuthorizationProperties {
private boolean enabled = false;
private String authorizationUrl;
private String publicCertificateFile;
private String publicCertificateType;
public boolean isEnabled() {
return enabled;
}
public void setEnabled(boolean enabled) {
this.enabled = enabled;
}
public String getAuthorizationUrl() {
return authorizationUrl;
}
public void setAuthorizationUrl(String authorizationUrl) {
this.authorizationUrl = authorizationUrl;
}
public String getPublicCertificateFile() {
return publicCertificateFile;
}
public void setPublicCertificateFile(String publicCertificateFile) {
this.publicCertificateFile = publicCertificateFile;
}
public String getPublicCertificateType() {
return publicCertificateType;
}
public void setPublicCertificateType(String publicCertificateType) {
this.publicCertificateType = publicCertificateType;
}
}
@@ -11,6 +11,7 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Scope;
@Configuration
@ConditionalOnExpression("!T(org.springframework.util.StringUtils).isEmpty('${timely.zookeeper.servers:}')")
@@ -18,6 +19,7 @@ import org.springframework.context.annotation.Configuration;
public class CuratorConfiguration {
@Bean(destroyMethod = "close")
@Scope("singleton")
@ConditionalOnMissingBean
public CuratorFramework curatorFramework(ZookeeperProperties zookeeperProperties) {
RetryPolicy retryPolicy = new RetryForever(1000);
@@ -4,7 +4,6 @@ import javax.validation.Valid;
import javax.validation.constraints.NotBlank;
import javax.validation.constraints.NotNull;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.boot.context.properties.NestedConfigurationProperty;
import org.springframework.cloud.context.config.annotation.RefreshScope;
@@ -12,12 +11,11 @@ import org.springframework.cloud.context.config.annotation.RefreshScope;
@RefreshScope
@ConfigurationProperties(prefix = "timely.http")
public class HttpProperties {
@Autowired
TimelyProperties timelyProperties;
@NotBlank
private String ip;
@NotNull
private Integer basePort;
private Integer port;
@NotNull
private String host;
private String redirectPath = "/secure-me";
@@ -26,10 +24,6 @@ public class HttpProperties {
@NestedConfigurationProperty
private CorsProperties corsProperties = new CorsProperties();
private int getPortOffset() {
return (timelyProperties.getInstance() - 1) * timelyProperties.getPortIncrement();
}
public String getIp() {
return ip;
}
@@ -39,15 +33,11 @@ public class HttpProperties {
}
public int getPort() {
return basePort + getPortOffset();
return port;
}
public void setBasePort(int basePort) {
this.basePort = basePort;
}
public Integer getBasePort() {
return basePort;
public void setPort(int port) {
this.port = port;
}
public String getHost() {
@@ -1,4 +1,4 @@
package timely.test.configuration;
package timely.common.configuration;
import javax.inject.Singleton;
@@ -11,7 +11,6 @@ import org.springframework.context.annotation.Primary;
import datawave.accumulo.inmemory.InMemoryAccumuloClient;
import datawave.accumulo.inmemory.InMemoryInstance;
import timely.common.configuration.AccumuloProperties;
@Configuration
@ConditionalOnProperty(name = "timely.in-memory-accumulo.enabled", havingValue = "true")
@@ -1,4 +1,4 @@
package timely.test.configuration;
package timely.common.configuration;
import java.util.Arrays;
@@ -1,4 +1,4 @@
package timely.test.configuration;
package timely.common.configuration;
import java.io.File;
import java.io.IOException;
@@ -28,9 +28,6 @@ import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Primary;
import timely.common.configuration.AccumuloProperties;
import timely.common.configuration.ZookeeperProperties;
@Configuration
@ConditionalOnProperty(name = "timely.mini-accumulo.enabled", havingValue = "true")
@EnableConfigurationProperties({ZookeeperProperties.class, AccumuloProperties.class, MiniAccumuloProperties.class})
@@ -1,4 +1,4 @@
package timely.test.configuration;
package timely.common.configuration;
import java.util.Arrays;
@@ -1,73 +0,0 @@
package timely.common.configuration;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.http.client.reactive.ReactorClientHttpConnector;
import org.springframework.security.web.authentication.preauth.PreAuthenticatedAuthenticationProvider;
import org.springframework.web.reactive.function.client.WebClient;
import io.netty.handler.ssl.SslContext;
import reactor.netty.http.client.HttpClient;
import timely.auth.FileUserDetailsService;
import timely.auth.RemoteAuthenticationProvider;
import timely.auth.SubjectIssuerDNPair;
import timely.auth.TimelyAuthenticationManager;
import timely.auth.TimelyUser;
@Configuration
@EnableConfigurationProperties({SecurityProperties.class})
public class SecurityConfiguration {
@Bean
@ConditionalOnExpression("T(org.apache.commons.lang3.StringUtils).isNotEmpty('${timely.security.authorization-url:}')")
public TimelyAuthenticationManager remoteAuthenticationManager(RemoteAuthenticationProvider remoteAuthenticationProvider) {
return new TimelyAuthenticationManager(Arrays.asList(remoteAuthenticationProvider));
}
@Bean
@ConditionalOnExpression("T(org.apache.commons.lang3.StringUtils).isNotEmpty('${timely.security.authorization-url:}')")
public RemoteAuthenticationProvider remoteAuthenticationProvider(SecurityProperties securityProperties, WebClient.Builder builder,
@Qualifier("outboundNettySslContext") SslContext sslContext) {
HttpClient httpClient = HttpClient.create().secure(sslContextSpec -> sslContextSpec.sslContext(sslContext));
ReactorClientHttpConnector clientHttpConnector = new ReactorClientHttpConnector(httpClient);
builder.clientConnector(clientHttpConnector);
return new RemoteAuthenticationProvider(builder, securityProperties);
}
@Bean
@Qualifier("file")
@ConditionalOnExpression("T(org.apache.commons.lang3.StringUtils).isEmpty('${timely.security.authorization-url:}')")
public PreAuthenticatedAuthenticationProvider fileAuthenticationProvider(FileUserDetailsService fileUserDetailsService) {
PreAuthenticatedAuthenticationProvider authenticationProvider = new PreAuthenticatedAuthenticationProvider();
authenticationProvider.setPreAuthenticatedUserDetailsService(fileUserDetailsService);
return authenticationProvider;
}
@Bean
@ConditionalOnExpression("T(org.apache.commons.lang3.StringUtils).isEmpty('${timely.security.authorization-url:}')")
public TimelyAuthenticationManager fileAuthenticationManager(@Qualifier("file") PreAuthenticatedAuthenticationProvider fileAuthenticationProvider) {
return new TimelyAuthenticationManager(Arrays.asList(fileAuthenticationProvider));
}
@Bean
public FileUserDetailsService fileUserDetailsService(SecurityProperties securityProperties) {
List<TimelyUser> timelyUsers = new ArrayList<>();
List<AuthorizedUser> users = securityProperties.getAuthorizedUsers();
for (AuthorizedUser user : users) {
TimelyUser timelyUser = new TimelyUser(SubjectIssuerDNPair.of(user.getSubjectDn(), user.getIssuerDn()),
TimelyUser.UserType.valueOf(user.getUserType()), "", user.getAuthsCollection(), user.getRolesCollection(), null,
System.currentTimeMillis(), -1L);
timelyUsers.add(timelyUser);
}
FileUserDetailsService fileUserDetailsService = new FileUserDetailsService();
fileUserDetailsService.setUsers(timelyUsers);
return fileUserDetailsService;
}
}
@@ -1,14 +1,10 @@
package timely.common.configuration;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
import javax.validation.Valid;
import org.apache.commons.lang3.StringUtils;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.boot.context.properties.NestedConfigurationProperty;
import org.springframework.cloud.context.config.annotation.RefreshScope;
@@ -28,7 +24,7 @@ public class SecurityProperties {
@NestedConfigurationProperty
private SslServerProperties serverSsl = new SslServerProperties();
private String authorizationUrl;
private String authorizationUri = "https://authorization:8443/authorization/v1/authorize";
private List<String> requiredRoles;
private List<String> requiredAuths;
@@ -90,12 +86,12 @@ public class SecurityProperties {
return authorizedUsers;
}
public void setAuthorizationUrl(String authorizationUrl) {
this.authorizationUrl = authorizationUrl;
public void setAuthorizationUri(String authorizationUri) {
this.authorizationUri = authorizationUri;
}
public String getAuthorizationUrl() {
return authorizationUrl;
public String getAuthorizationUri() {
return authorizationUri;
}
public List<String> getRequiredAuths() {
@@ -106,13 +102,6 @@ public class SecurityProperties {
this.requiredAuths = requiredAuths;
}
public void setRequiredAuths(String requiredAuths) {
if (StringUtils.isNotBlank(requiredAuths)) {
this.requiredAuths = Arrays.stream(StringUtils.split(requiredAuths)).filter(Objects::nonNull).filter(s -> StringUtils.isNotBlank(s))
.map(String::trim).collect(Collectors.toList());
}
}
public List<String> getRequiredRoles() {
return requiredRoles;
}
@@ -120,11 +109,4 @@ public class SecurityProperties {
public void setRequiredRoles(List<String> requiredRoles) {
this.requiredRoles = requiredRoles;
}
public void setRequiredRoles(String requiredRoles) {
if (StringUtils.isNotBlank(requiredRoles)) {
this.requiredRoles = Arrays.stream(StringUtils.split(requiredRoles)).filter(Objects::nonNull).filter(s -> StringUtils.isNotBlank(s))
.map(String::trim).collect(Collectors.toList());
}
}
}
@@ -3,27 +3,21 @@ package timely.common.configuration;
import javax.validation.constraints.NotBlank;
import javax.validation.constraints.NotNull;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.cloud.context.config.annotation.RefreshScope;
@RefreshScope
@ConfigurationProperties(prefix = "timely.server")
public class ServerProperties {
@Autowired
TimelyProperties timelyProperties;
@NotBlank
private String ip;
@NotNull
private Integer tcpBasePort;
private Integer tcpPort;
@NotNull
private Integer udpBasePort;
private Integer udpPort;
private Integer shutdownQuietPeriod = 5;
private int getPortOffset() {
return (timelyProperties.getInstance() - 1) * timelyProperties.getPortIncrement();
}
@NotNull
public String getIp() {
return ip;
@@ -34,27 +28,19 @@ public class ServerProperties {
}
public int getTcpPort() {
return tcpBasePort + getPortOffset();
return tcpPort;
}
public void setTcpBasePort(int tcpPort) {
this.tcpBasePort = tcpPort;
}
public Integer getTcpBasePort() {
return tcpBasePort;
public void setTcpPort(int tcpPort) {
this.tcpPort = tcpPort;
}
public int getUdpPort() {
return udpBasePort + getPortOffset();
return udpPort;
}
public void setUdpBasePort(Integer udpBasePort) {
this.udpBasePort = udpBasePort;
}
public Integer getUdpBasePort() {
return udpBasePort;
public void setUdpPort(Integer udpPort) {
this.udpPort = udpPort;
}
/**
@@ -2,8 +2,6 @@ package timely.common.configuration;
import javax.net.ssl.SSLContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
@@ -17,16 +15,12 @@ import io.netty.handler.ssl.SslContext;
*/
@Configuration
@EnableConfigurationProperties({SslClientProperties.class})
@ConditionalOnExpression("!T(org.springframework.util.StringUtils).isEmpty('${timely.security.client-ssl.key-store-file:}')")
@ConditionalOnExpression("!T(org.springframework.util.StringUtils).isEmpty('${timely.security.clientSsl.keyStoreFile:}')")
public class SslClientConfig {
private static final Logger log = LoggerFactory.getLogger(SslClientConfig.class);
@Bean
@Qualifier("outboundJDKSslContext")
public SSLContext outboundJDKSslContext(SslClientProperties sslClientProperties) throws Exception {
log.debug("Creating outboundJDKSslContext using keyStoreFile:{} trustStoreFile:{}", sslClientProperties.getKeyStoreFile(),
sslClientProperties.getTrustStoreFile());
SSLContext sslContext = SSLContext.getInstance("TLS");
sslContext.init(SslHelper.getKeyManagerFactory(sslClientProperties).getKeyManagers(),
SslHelper.getTrustManagerFactory(sslClientProperties).getTrustManagers(), null);
@@ -36,8 +30,6 @@ public class SslClientConfig {
@Bean
@Qualifier("outboundNettySslContext")
public SslContext outboundNettySslContext(SslClientProperties sslClientProperties) throws Exception {
log.debug("Creating outboundNettySslContext using keyStoreFile:{} trustStoreFile:{}", sslClientProperties.getKeyStoreFile(),
sslClientProperties.getTrustStoreFile());
return SslHelper.getSslContext(sslClientProperties);
}
}
@@ -27,7 +27,7 @@ public class SslServerConfig {
private static final Logger log = LoggerFactory.getLogger(SslServerConfig.class);
@Bean
public SelfSignedCertificate selfSignedCertificate() throws Exception {
protected SelfSignedCertificate selfSignedCertificate() throws Exception {
Date begin = new Date();
Date end = new Date(begin.getTime() + TimeUnit.DAYS.toMillis(7));
return new SelfSignedCertificate("localhost", begin, end);
@@ -35,15 +35,13 @@ public class SslServerConfig {
@Bean
@Qualifier("nettySslContext")
public SslContext nettySslContext(SslServerProperties sslServerProperties, SelfSignedCertificate ssc) throws Exception {
protected SslContext nettySslContext(SslServerProperties sslServerProperties, SelfSignedCertificate ssc) throws Exception {
Boolean generate = sslServerProperties.isUseGeneratedKeypair();
SslContextBuilder sslContextBuilder;
if (generate) {
log.debug("Creating nettySslContext using generated self signed server certificate");
log.warn("Using generated self signed server certificate");
sslContextBuilder = SslContextBuilder.forServer(ssc.certificate(), ssc.privateKey());
} else {
log.debug("Creating nettySslContext using keyStoreFile:{} trustStoreFile:{}", sslServerProperties.getKeyStoreFile(),
sslServerProperties.getTrustStoreFile());
sslContextBuilder = SslHelper.getSslContextBuilder(sslServerProperties);
}
@@ -1,15 +1,7 @@
package timely.common.configuration;
import java.net.URL;
import javax.annotation.PostConstruct;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.cloud.context.config.annotation.RefreshScope;
import org.springframework.util.ResourceUtils;
import timely.validator.NotEmptyIfFieldSet;
@@ -19,8 +11,6 @@ import timely.validator.NotEmptyIfFieldSet;
@ConfigurationProperties(prefix = "timely.security.server-ssl")
public class SslServerProperties implements SslProperties {
private static final Logger log = LoggerFactory.getLogger(SslServerProperties.class);
private String[] ciphers = new String[] {"TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256", "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA", "TLS_RSA_WITH_AES_128_GCM_SHA256",
"TLS_RSA_WITH_AES_128_CBC_SHA", "SSL_RSA_WITH_3DES_EDE_CBC_SHA"};
private String[] enabledProtocols;
@@ -35,27 +25,6 @@ public class SslServerProperties implements SslProperties {
private boolean ignoreSslHandshakeErrors = false;
private boolean useOpenssl = true;
@PostConstruct
public void setTrustStoreSystemProperties() {
try {
if (StringUtils.isNotBlank(trustStoreFile)) {
// This step is to resolve any classpath: prefixes used in trustStoreFile
URL url = ResourceUtils.getURL(trustStoreFile);
if (url != null) {
System.setProperty("javax.net.ssl.trustStore", url.getFile());
if (StringUtils.isNotBlank(trustStoreType)) {
System.setProperty("javax.net.ssl.trustStoreType", trustStoreType);
}
if (StringUtils.isNotBlank(trustStorePassword)) {
System.setProperty("javax.net.ssl.trustStorePassword", trustStorePassword);
}
}
}
} catch (Exception e) {
log.error(e.getMessage(), e);
}
}
public String[] getCiphers() {
return ciphers;
}
@@ -4,8 +4,6 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import javax.validation.constraints.Min;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.cloud.context.config.annotation.RefreshScope;
import org.springframework.validation.annotation.Validated;
@@ -20,12 +18,10 @@ public class TimelyProperties {
private String metaTable = "timely.meta";
private HashMap<String,Integer> metricAgeOffDays = new HashMap<>();
private List<String> metricsReportIgnoredTags = new ArrayList<>();
@Min(1)
private int instance = 1;
private int portIncrement = 100;
private String instance = null;
private String defaultVisibility = null;
public boolean isTest() {
public boolean getTest() {
return test;
}
@@ -51,22 +47,14 @@ public class TimelyProperties {
return this;
}
public void setInstance(int instance) {
public void setInstance(String instance) {
this.instance = instance;
}
public int getInstance() {
public String getInstance() {
return instance;
}
public void setPortIncrement(int portIncrement) {
this.portIncrement = portIncrement;
}
public int getPortIncrement() {
return portIncrement;
}
public String getDefaultVisibility() {
return defaultVisibility;
}
@@ -0,0 +1,43 @@
package timely.common.configuration;
import java.util.ArrayList;
import java.util.List;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.security.web.authentication.preauth.PreAuthenticatedAuthenticationProvider;
import timely.auth.FileUserDetailsService;
import timely.auth.SubjectIssuerDNPair;
import timely.auth.TimelyUser;
@Configuration
@EnableConfigurationProperties({SecurityProperties.class})
public class UserDetailsConfiguration {
@Bean
public FileUserDetailsService fileUserDetailsService(SecurityProperties securityProperties) {
List<TimelyUser> timelyUsers = new ArrayList<>();
List<AuthorizedUser> users = securityProperties.getAuthorizedUsers();
for (AuthorizedUser user : users) {
TimelyUser timelyUser = new TimelyUser(SubjectIssuerDNPair.of(user.getSubjectDn(), user.getIssuerDn()),
TimelyUser.UserType.valueOf(user.getUserType()), "", user.getAuthsCollection(), user.getRolesCollection(), null,
System.currentTimeMillis(), -1L);
timelyUsers.add(timelyUser);
}
FileUserDetailsService fileUserDetailsService = new FileUserDetailsService();
fileUserDetailsService.setUsers(timelyUsers);
return fileUserDetailsService;
}
@Bean
@Qualifier("file")
PreAuthenticatedAuthenticationProvider fileAuthenticationProvider(FileUserDetailsService fileUserDetailsService) {
PreAuthenticatedAuthenticationProvider authenticationProvider = new PreAuthenticatedAuthenticationProvider();
authenticationProvider.setPreAuthenticatedUserDetailsService(fileUserDetailsService);
return authenticationProvider;
}
}
@@ -3,19 +3,17 @@ package timely.common.configuration;
import javax.validation.constraints.NotBlank;
import javax.validation.constraints.NotNull;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.cloud.context.config.annotation.RefreshScope;
@RefreshScope
@ConfigurationProperties(prefix = "timely.websocket")
public class WebsocketProperties {
@Autowired
TimelyProperties timelyProperties;
@NotBlank
private String ip;
@NotNull
private Integer basePort;
private Integer port;
private int timeout = 60;
private int subscriptionLag = 120;
private int scannerBatchSize = 5000;
@@ -23,10 +21,6 @@ public class WebsocketProperties {
private int scannerReadAhead = 1;
private int subscriptionBatchSize = 1000;
private int getPortOffset() {
return (timelyProperties.getInstance() - 1) * timelyProperties.getPortIncrement();
}
public String getIp() {
return ip;
}
@@ -36,15 +30,11 @@ public class WebsocketProperties {
}
public int getPort() {
return basePort + getPortOffset();
return port;
}
public Integer getBasePort() {
return basePort;
}
public void setBasePort(int basePort) {
this.basePort = basePort;
public void setPort(int port) {
this.port = port;
}
public int getTimeout() {
@@ -76,7 +76,7 @@ public class HttpRequestDecoder extends MessageToMessageDecoder<FullHttpRequest>
}
@Override
public void decode(ChannelHandlerContext ctx, FullHttpRequest msg, List<Object> out) throws Exception {
protected void decode(ChannelHandlerContext ctx, FullHttpRequest msg, List<Object> out) throws Exception {
log.trace(LOG_RECEIVED_REQUEST, msg);
@@ -22,10 +22,8 @@ public class MetricsBufferDecoder extends ByteToMessageDecoder {
if (in.readableBytes() < 8) {
return;
}
if (log.isTraceEnabled()) {
log.trace("Received {} bytes", in.readableBytes());
}
boolean isFlatBuffer;
log.debug("Received {} bytes", in.readableBytes());
boolean isFlatBuffer = false;
try {
isFlatBuffer = Metrics.MetricsBufferHasIdentifier(in.slice().nioBuffer());
} catch (Exception e) {
@@ -33,14 +31,10 @@ public class MetricsBufferDecoder extends ByteToMessageDecoder {
isFlatBuffer = false;
}
if (!isFlatBuffer) {
if (log.isTraceEnabled()) {
log.trace("Input is not a Flatbuffer");
}
log.debug("Input is not a Flatbuffer");
out.add(in.readBytes(in.readableBytes()));
} else {
if (log.isTraceEnabled()) {
log.trace("Input is a Flatbuffer");
}
log.debug("Input is a Flatbuffer");
ByteBuf copy = in.readBytes(in.readableBytes());
try {
Metrics metrics = Metrics.getRootAsMetrics(copy.nioBuffer());
+1 -1
View File
@@ -4,7 +4,7 @@
<parent>
<groupId>timely</groupId>
<artifactId>timely</artifactId>
<version>2.1.1</version>
<version>2.1.0-SNAPSHOT</version>
</parent>
<artifactId>timely-app</artifactId>
<packaging>pom</packaging>
+2 -2
View File
@@ -28,8 +28,8 @@
}
],
"screenshots": [],
"version": "2.1.1",
"updated": "2024-03-01"
"version": "2.1.0-SNAPSHOT",
"updated": "2024-01-30"
},
"dependencies": {
"classnames": "2.2.6",
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "timely-datasource",
"version": "2.1.1",
"version": "2.1.0-SNAPSHOT",
"description": "Timely",
"backend": true,
"alerting": true,
+73 -119
View File
@@ -3,7 +3,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>timely</groupId>
<artifactId>timely</artifactId>
<version>2.1.1</version>
<version>2.1.0-SNAPSHOT</version>
<packaging>pom</packaging>
<name>Timely</name>
<description>A secure and scalable time series database based on Apache Accumulo</description>
@@ -15,12 +15,11 @@
</licenses>
<modules>
<module>common</module>
<module>test</module>
<module>client</module>
<module>server</module>
<module>balancer</module>
<module>grafana</module>
<module>analytics</module>
<module>balancer</module>
</modules>
<scm>
<connection>scm:git:https://github.com/NationalSecurityAgency/timely.git</connection>
@@ -45,6 +44,7 @@
<spotbugs.excludes.file>${project.basedir}/src/main/spotbugs/excludes.xml</spotbugs.excludes.file>
<surefire.forkCount>1C</surefire.forkCount>
<version.accumulo>2.1.2</version.accumulo>
<version.buildnumber-maven>1.4</version.buildnumber-maven>
<version.caffeine>3.1.8</version.caffeine>
<version.classgraph>4.8.165</version.classgraph>
<version.collectd>5.7.1</version.collectd>
@@ -55,49 +55,46 @@
<version.commons-pool2>2.12.0</version.commons-pool2>
<version.commons-text>1.11.0</version.commons-text>
<version.compression.gorilla>2.1.1</version.compression.gorilla>
<version.curator>5.5.0</version.curator>
<version.curator>5.6.0</version.curator>
<version.datawave-in-memory-accumulo>3.0.1</version.datawave-in-memory-accumulo>
<version.disruptor>3.4.4</version.disruptor>
<version.docker.plugin>1.2.1</version.docker.plugin>
<version.easymock>4.0.2</version.easymock>
<version.eclipse-lifecycle>1.0.0</version.eclipse-lifecycle>
<version.fast-classpath-scanner>3.1.13</version.fast-classpath-scanner>
<version.flink>1.1.4</version.flink>
<version.google-flatbuffers>23.5.26</version.google-flatbuffers>
<version.guava>33.0.0-jre</version.guava>
<version.hadoop>3.3.6</version.hadoop>
<version.httpcomponents>4.5.14</version.httpcomponents>
<version.impsort>1.9.0</version.impsort>
<version.jackson>2.16.1</version.jackson>
<version.jacoco>0.8.11</version.jacoco>
<version.jakarta.websocket-api>1.1.2</version.jakarta.websocket-api>
<version.javacsv>2.0</version.javacsv>
<version.javassist>3.30.2-GA</version.javassist>
<version.jcip-annotations>1.0</version.jcip-annotations>
<version.jjwt>0.12.3</version.jjwt>
<version.jjwt>0.12.5</version.jjwt>
<version.jsoup>1.17.2</version.jsoup>
<version.junit>4.13.2</version.junit>
<version.log4j2>2.22.1</version.log4j2>
<version.maven-assembly-plugin>3.6.0</version.maven-assembly-plugin>
<version.maven-build-helper-plugin>3.5.0</version.maven-build-helper-plugin>
<version.maven-buildnumber-plugin>1.4</version.maven-buildnumber-plugin>
<version.maven-compiler-plugin>3.12.1</version.maven-compiler-plugin>
<version.maven-dependency-plugin>3.6.1</version.maven-dependency-plugin>
<version.maven-docker-plugin>1.2.1</version.maven-docker-plugin>
<version.maven-assembly>3.6.0</version.maven-assembly>
<version.maven-build-helper>3.5.0</version.maven-build-helper>
<version.maven-compiler>3.12.1</version.maven-compiler>
<version.maven-dependency>3.6.1</version.maven-dependency>
<version.maven-failsafe-plugin>3.2.5</version.maven-failsafe-plugin>
<version.maven-formatter-plugin>2.22.0</version.maven-formatter-plugin>
<version.maven-impsort-plugin>1.9.0</version.maven-impsort-plugin>
<version.maven-jar-plugin>3.3.0</version.maven-jar-plugin>
<version.maven-os-plugin>1.7.1</version.maven-os-plugin>
<version.maven-rpm-plugin>2.3.0</version.maven-rpm-plugin>
<version.maven-shade-plugin>3.5.1</version.maven-shade-plugin>
<version.maven-spotbugs-plugin>4.7.3.0</version.maven-spotbugs-plugin>
<version.maven-jar>3.3.0</version.maven-jar>
<version.maven-rpm>2.3.0</version.maven-rpm>
<version.maven-shade>3.5.1</version.maven-shade>
<version.maven-surefire-plugin>3.2.5</version.maven-surefire-plugin>
<version.micrometer>1.12.2</version.micrometer>
<version.netty>4.1.106.Final</version.netty>
<version.netty-tcnative>2.0.61.Final</version.netty-tcnative>
<version.os-maven>1.7.1</version.os-maven>
<version.powermock>1.6.4</version.powermock>
<version.reactor-netty>1.1.15</version.reactor-netty>
<version.slf4j>1.7.36</version.slf4j>
<version.snakeyaml>2.2</version.snakeyaml>
<version.spotbugs-maven>4.8.3.0</version.spotbugs-maven>
<version.spring-boot>2.7.18</version.spring-boot>
<version.spring-cloud>3.1.8</version.spring-cloud>
<version.spring-webflux>5.3.31</version.spring-webflux>
@@ -181,18 +178,22 @@
<groupId>commons-beanutils</groupId>
<artifactId>commons-beanutils</artifactId>
<version>1.9.4</version>
<exclusions>
<exclusion>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>commons-lang</groupId>
<artifactId>commons-lang</artifactId>
<version>${version.commons-lang}</version>
</dependency>
<dependency>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
<version>${version.commons-logging}</version>
</dependency>
<dependency>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
<version>${version.commons-logging}</version>
</dependency>
<dependency>
<groupId>fi.iki.yak</groupId>
<artifactId>compression-gorilla</artifactId>
@@ -255,21 +256,6 @@
<artifactId>jakarta.websocket-api</artifactId>
<version>${version.jakarta.websocket-api}</version>
</dependency>
<dependency>
<groupId>javax.activation</groupId>
<artifactId>activation</artifactId>
<version>1.1.1</version>
</dependency>
<dependency>
<groupId>javax.xml.bind</groupId>
<artifactId>jaxb-api</artifactId>
<version>2.3.1</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>${version.junit}</version>
</dependency>
<dependency>
<groupId>net.sourceforge.javacsv</groupId>
<artifactId>javacsv</artifactId>
@@ -280,10 +266,6 @@
<artifactId>accumulo-core</artifactId>
<version>${version.accumulo}</version>
<exclusions>
<exclusion>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
</exclusion>
<exclusion>
<groupId>io.netty</groupId>
<artifactId>netty</artifactId>
@@ -303,10 +285,6 @@
<artifactId>accumulo-minicluster</artifactId>
<version>${version.accumulo}</version>
<exclusions>
<exclusion>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
</exclusion>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
@@ -317,11 +295,6 @@
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.accumulo</groupId>
<artifactId>accumulo-tserver</artifactId>
<version>${version.accumulo}</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-collections4</artifactId>
@@ -390,10 +363,6 @@
<groupId>ch.qos.reload4j</groupId>
<artifactId>reload4j</artifactId>
</exclusion>
<exclusion>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
</exclusion>
<exclusion>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
@@ -475,10 +444,6 @@
<groupId>ch.qos.reload4j</groupId>
<artifactId>reload4j</artifactId>
</exclusion>
<exclusion>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
</exclusion>
<exclusion>
<groupId>jdk.tools</groupId>
<artifactId>jdk.tools</artifactId>
@@ -518,10 +483,6 @@
<groupId>ch.qos.reload4j</groupId>
<artifactId>reload4j</artifactId>
</exclusion>
<exclusion>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
</exclusion>
<exclusion>
<groupId>io.netty</groupId>
<artifactId>netty</artifactId>
@@ -564,12 +525,6 @@
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>${version.httpcomponents}</version>
<exclusions>
<exclusion>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<!-- This bridge allows log4j2 to work with code that is still using the log4j 1.2 classes/api -->
@@ -648,6 +603,11 @@
<artifactId>snakeyaml</artifactId>
<version>${version.snakeyaml}</version>
</dependency>
<dependency>
<groupId>timely</groupId>
<artifactId>collectd-plugin-parent</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>timely</groupId>
<artifactId>timely-balancer</artifactId>
@@ -665,7 +625,7 @@
</dependency>
<dependency>
<groupId>timely</groupId>
<artifactId>timely-test</artifactId>
<artifactId>timely-server</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
@@ -717,10 +677,34 @@
<build>
<pluginManagement>
<plugins>
<plugin>
<groupId>net.revelc.code.formatter</groupId>
<artifactId>formatter-maven-plugin</artifactId>
<version>2.22.0</version>
<configuration>
<compilerCompliance>${maven.compiler.source}</compilerCompliance>
<compilerSource>${maven.compiler.source}</compilerSource>
<compilerTargetPlatform>${maven.compiler.target}</compilerTargetPlatform>
<configFile>${project.parent.basedir}/src/main/resources/eclipse/Eclipse-Timely-Codestyle.xml</configFile>
<lineEnding>LF</lineEnding>
<skipCssFormatting>true</skipCssFormatting>
<skipHtmlFormatting>true</skipHtmlFormatting>
<skipJsFormatting>true</skipJsFormatting>
<skipJsonFormatting>true</skipJsonFormatting>
<skipXmlFormatting>true</skipXmlFormatting>
</configuration>
<executions>
<execution>
<goals>
<goal>format</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>com.spotify</groupId>
<artifactId>docker-maven-plugin</artifactId>
<version>${version.maven-docker-plugin}</version>
<version>${version.docker.plugin}</version>
<configuration>
<imageName>${docker.image.prefix}${project.artifactId}</imageName>
<forceTags>true</forceTags>
@@ -783,7 +767,7 @@
<plugin>
<groupId>net.revelc.code</groupId>
<artifactId>impsort-maven-plugin</artifactId>
<version>${version.maven-impsort-plugin}</version>
<version>${version.impsort}</version>
<configuration>
<groups>java.,javax.,*</groups>
<removeUnused>true</removeUnused>
@@ -798,39 +782,15 @@
</execution>
</executions>
</plugin>
<plugin>
<groupId>net.revelc.code.formatter</groupId>
<artifactId>formatter-maven-plugin</artifactId>
<version>${version.maven-formatter-plugin}</version>
<configuration>
<compilerCompliance>${maven.compiler.source}</compilerCompliance>
<compilerSource>${maven.compiler.source}</compilerSource>
<compilerTargetPlatform>${maven.compiler.target}</compilerTargetPlatform>
<configFile>${project.parent.basedir}/src/main/resources/eclipse/Eclipse-Timely-Codestyle.xml</configFile>
<lineEnding>LF</lineEnding>
<skipCssFormatting>true</skipCssFormatting>
<skipHtmlFormatting>true</skipHtmlFormatting>
<skipJsFormatting>true</skipJsFormatting>
<skipJsonFormatting>true</skipJsonFormatting>
<skipXmlFormatting>true</skipXmlFormatting>
</configuration>
<executions>
<execution>
<goals>
<goal>format</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<version>${version.maven-assembly-plugin}</version>
<version>${version.maven-assembly}</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>${version.maven-compiler-plugin}</version>
<version>${version.maven-compiler}</version>
<configuration>
<source>${maven.compiler.source}</source>
<target>${maven.compiler.target}</target>
@@ -839,7 +799,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<version>${version.maven-dependency-plugin}</version>
<version>${version.maven-dependency}</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
@@ -858,8 +818,6 @@
<!-- do not use the following slf4j binders -->
<exclude>ch.qos.reload4j:reload4j</exclude>
<exclude>org.slf4j:slf4j-reload4j</exclude>
<!-- commons-logging interferes with logger initialization -->
<exclude>commons-logging:commons-logging</exclude>
<!-- spring boot microservices do not use 1.x versions of log4j -->
<exclude>log4j:log4j</exclude>
<!-- spring boot microservices do not use 1.x versions of junit -->
@@ -900,7 +858,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<version>${version.maven-jar-plugin}</version>
<version>${version.maven-jar}</version>
<configuration>
<archive>
<manifest>
@@ -922,7 +880,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>${version.maven-shade-plugin}</version>
<version>${version.maven-shade}</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
@@ -944,12 +902,12 @@
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>build-helper-maven-plugin</artifactId>
<version>${version.maven-build-helper-plugin}</version>
<version>${version.maven-build-helper}</version>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>buildnumber-maven-plugin</artifactId>
<version>${version.maven-buildnumber-plugin}</version>
<version>${version.buildnumber-maven}</version>
<configuration>
<doCheck>false</doCheck>
<doUpdate>false</doUpdate>
@@ -958,7 +916,7 @@
<plugin>
<groupId>com.github.spotbugs</groupId>
<artifactId>spotbugs-maven-plugin</artifactId>
<version>${version.maven-spotbugs-plugin}</version>
<version>${version.spotbugs-maven}</version>
<configuration>
<effort>Max</effort>
<threshold>Medium</threshold>
@@ -973,7 +931,12 @@
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>rpm-maven-plugin</artifactId>
<version>${version.maven-rpm-plugin}</version>
<version>${version.maven-rpm}</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-resources-plugin</artifactId>
<version>3.1.0</version>
</plugin>
<plugin>
<groupId>org.springframework.boot</groupId>
@@ -1170,7 +1133,7 @@
<plugin>
<groupId>com.github.spotbugs</groupId>
<artifactId>spotbugs-maven-plugin</artifactId>
<version>${version.maven-spotbugs-plugin}</version>
<version>${version.spotbugs-maven}</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
@@ -1238,15 +1201,6 @@
<module>collectd-nsq-plugin</module>
<module>collectd-plugin-parent</module>
</modules>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>timely</groupId>
<artifactId>collectd-plugin-parent</artifactId>
<version>${project.version}</version>
</dependency>
</dependencies>
</dependencyManagement>
</profile>
<profile>
<id>add-spotbugs-excludes</id>
+40
View File
@@ -0,0 +1,40 @@
#!/bin/bash
if [[ `uname` == "Darwin" ]]; then
THIS_SCRIPT=`python -c 'import os,sys; print os.path.realpath(sys.argv[1])' $0`
else
THIS_SCRIPT=`readlink -f $0`
fi
THIS_DIR="${THIS_SCRIPT%/*}"
BASE_DIR=$(cd $THIS_DIR/.. && pwd)
BIN_DIR=${BASE_DIR}/bin
CONF_DIR=${BASE_DIR}/conf
. ${BIN_DIR}/timely-env.sh
for i in $(seq 1 $NUM_PROFILES); do
if [ "$NUM_PROFILES" -eq "1" ]; then
FILENAME=timely.yml
else
FILENAME=timely-${i}.yml
fi
echo "creating configuration file ${CONF_DIR}/${FILENAME}"
cp ${CONF_DIR}/timely-template.yml ${CONF_DIR}/${FILENAME}
# if [ "$NUM_PROFILES" -eq "1" ]; then
# sed -i "s/\${INSTANCE}//g" ${CONF_DIR}/${FILENAME}
# else
# sed -i "s/\${INSTANCE}/${i}/g" ${CONF_DIR}/${FILENAME}
# fi
MULTIPLE=$(($i - 1))
TCP_PORT=$(($TCP_PORT_START + $MULTIPLE * $PORT_INCREMENT))
sed -i "s/\${TCP_PORT}/${TCP_PORT}/g" ${CONF_DIR}/${FILENAME}
HTTP_PORT=$(($HTTP_PORT_START + $MULTIPLE * $PORT_INCREMENT))
sed -i "s/\${HTTP_PORT}/${HTTP_PORT}/g" ${CONF_DIR}/${FILENAME}
WEBSOCKET_PORT=$(($WEBSOCKET_PORT_START + $MULTIPLE * $PORT_INCREMENT))
sed -i "s/\${WEBSOCKET_PORT}/${WEBSOCKET_PORT}/g" ${CONF_DIR}/${FILENAME}
UDP_PORT=$(($UDP_PORT_START + $MULTIPLE * $PORT_INCREMENT))
sed -i "s/\${UDP_PORT}/${UDP_PORT}/g" ${CONF_DIR}/${FILENAME}
done
+21
View File
@@ -0,0 +1,21 @@
#!/bin/bash
if [[ `uname` == "Darwin" ]]; then
THIS_SCRIPT=`python -c 'import os,sys; print os.path.realpath(sys.argv[1])' $0`
else
THIS_SCRIPT=`readlink -f $0`
fi
THIS_DIR="${THIS_SCRIPT%/*}"
BASE_DIR=$(cd $THIS_DIR/.. && pwd)
CONF_DIR="${BASE_DIR}/conf"
LIB_DIR="${BASE_DIR}/lib"
export CLASSPATH="${CONF_DIR}:${LIB_DIR}/*"
JVM_ARGS="-Xmx256m -Xms128m"
JVM_ARGS="${JVM_ARGS} -Dlogging.config=${CONF_DIR}/log4j2-error-console.xml"
JVM_ARGS="${JVM_ARGS} -Dlog4j.configurationFile=${CONF_DIR}/log4j2-error-console.xml"
JVM_ARGS="${JVM_ARGS} -DLog4jContextSelector=org.apache.logging.log4j.core.async.AsyncLoggerContextSelector"
echo "$JAVA_HOME/bin/java ${JVM_ARGS} timely.util.split.GetMetricTableSplitPoints --spring.config.name=timely"
$JAVA_HOME/bin/java ${JVM_ARGS} timely.util.GetMetricTableSplitPoints --spring.config.name=timely
+21
View File
@@ -0,0 +1,21 @@
#!/bin/bash
if [[ `uname` == "Darwin" ]]; then
THIS_SCRIPT=`python -c 'import os,sys; print os.path.realpath(sys.argv[1])' $0`
else
THIS_SCRIPT=`readlink -f $0`
fi
THIS_DIR="${THIS_SCRIPT%/*}"
BASE_DIR=$(cd $THIS_DIR/.. && pwd)
CONF_DIR="${BASE_DIR}/conf"
LIB_DIR="${BASE_DIR}/lib"
export CLASSPATH="${CONF_DIR}:${LIB_DIR}/*"
JVM_ARGS="-Xmx1024m -Xms128m"
JVM_ARGS="${JVM_ARGS} -Dlogging.config=${CONF_DIR}/log4j2-error-console.xml"
JVM_ARGS="${JVM_ARGS} -Dlog4j.configurationFile=${CONF_DIR}/log4j2-error-console.xml"
JVM_ARGS="${JVM_ARGS} -DLog4jContextSelector=org.apache.logging.log4j.core.async.AsyncLoggerContextSelector"
echo "$JAVA_HOME/bin/java ${JVM_ARGS} timely.store.compaction.util.TabletMetadataConsole --spring.config.name=timely"
$JAVA_HOME/bin/java ${JVM_ARGS} timely.store.compaction.util.TabletMetadataConsole --spring.config.name=timely
+39
View File
@@ -0,0 +1,39 @@
#!/bin/bash
if [[ `uname` == "Darwin" ]]; then
THIS_SCRIPT=`python -c 'import os,sys; print os.path.realpath(sys.argv[1])' $0`
else
THIS_SCRIPT=`readlink -f $0`
fi
THIS_DIR="${THIS_SCRIPT%/*}"
BASE_DIR=$(cd $THIS_DIR/.. && pwd)
LIB_DIR="${BASE_DIR}/lib"
NUM_SERVER_THREADS=4
CONF_DIR="${BASE_DIR}/conf"
if [ -z ${TIMELY_HOST+x} ]; then
echo "TIMELY_HOST is unset, using 127.0.0.1"
TIMELY_HOST="127.0.0.1"
else
echo "TIMELY_HOST is set to '$TIMELY_HOST'"
fi
if [ -z ${TIMELY_PORT+x} ]; then
echo "TIMELY_PORT is unset, using 54321"
TIMELY_PORT=54321
else
echo "TIMELY_PORT is set to '$TIMELY_PORT'"
fi
CP=""
SEP=""
for j in ${LIB_DIR}/*.jar; do
CP="${CP}${SEP}${j}"
SEP=":"
done
JVM_ARGS="-Xmx128m -Xms128m -Dio.netty.eventLoopThreads=${NUM_SERVER_THREADS} -Dlog4j.configurationFile=${CONF_DIR}/log4j2-spring.xml"
echo "$JAVA_HOME/bin/java -classpath ${CP} ${JVM_ARGS} timely.util.ingest.InsertTestData ${TIMELY_HOST} ${TIMELY_PORT} --fast"
exec $JAVA_HOME/bin/java -classpath ${CP} ${JVM_ARGS} timely.util.ingest.InsertTestData ${TIMELY_HOST} ${TIMELY_PORT} --fast
+39
View File
@@ -0,0 +1,39 @@
#!/bin/bash
if [[ `uname` == "Darwin" ]]; then
THIS_SCRIPT=`python -c 'import os,sys; print os.path.realpath(sys.argv[1])' $0`
else
THIS_SCRIPT=`readlink -f $0`
fi
THIS_DIR="${THIS_SCRIPT%/*}"
BASE_DIR=$(cd $THIS_DIR/.. && pwd)
LIB_DIR="${BASE_DIR}/lib"
NUM_SERVER_THREADS=4
CONF_DIR="${BASE_DIR}/conf"
if [ -z ${TIMELY_HOST+x} ]; then
echo "TIMELY_HOST is unset, using 127.0.0.1"
TIMELY_HOST="127.0.0.1"
else
echo "TIMELY_HOST is set to '$TIMELY_HOST'"
fi
if [ -z ${TIMELY_PORT+x} ]; then
echo "TIMELY_PORT is unset, using 54321"
TIMELY_PORT=54321
else
echo "TIMELY_PORT is set to '$TIMELY_PORT'"
fi
CP=""
SEP=""
for j in ${LIB_DIR}/*.jar; do
CP="${CP}${SEP}${j}"
SEP=":"
done
JVM_ARGS="-Xmx128m -Xms128m -Dio.netty.eventLoopThreads=${NUM_SERVER_THREADS} -Dlog4j.configurationFile=${CONF_DIR}/log4j2-spring.xml"
echo "$JAVA_HOME/bin/java -classpath ${CP} ${JVM_ARGS} timely.util.ingest.InsertTestData ${TIMELY_HOST} ${TIMELY_PORT}"
exec $JAVA_HOME/bin/java -classpath ${CP} ${JVM_ARGS} timely.util.ingest.InsertTestData ${TIMELY_HOST} ${TIMELY_PORT}
+4 -4
View File
@@ -10,14 +10,14 @@ THIS_DIR="${THIS_SCRIPT%/*}"
BASE_DIR=$(cd $THIS_DIR/.. && pwd)
BIN_DIR=${BASE_DIR}/bin
. ${BIN_DIR}/timely-server-env.sh
. ${BIN_DIR}/timely-env.sh
if [ "$NUM_INSTANCES" -eq "1" ]; then
if [ "$NUM_PROFILES" -eq "1" ]; then
echo ${BIN_DIR}/timely-server.sh
${BIN_DIR}/timely-server.sh
else
for i in $(seq 1 $NUM_INSTANCES); do
for i in $(seq 1 $NUM_PROFILES); do
echo ${BIN_DIR}/timely-server.sh $i
${BIN_DIR}/timely-server.sh $i
done
fi
fi
+1 -19
View File
@@ -1,21 +1,3 @@
#!/bin/bash
if [[ `uname` == "Darwin" ]]; then
THIS_SCRIPT=`python -c 'import os,sys; print os.path.realpath(sys.argv[1])' $0`
else
THIS_SCRIPT=`readlink -f $0`
fi
THIS_DIR="${THIS_SCRIPT%/*}"
BASE_DIR=$(cd $THIS_DIR/.. && pwd)
BIN_DIR=${BASE_DIR}/bin
. ${BIN_DIR}/timely-server-env.sh
for i in $(seq 1 $NUM_INSTANCES); do
PID=`jps -m | grep -E 'timely-server-.*-exec.jar timelyserver' | grep "instance=${i}" | awk '{print $1}'`
if [ "$PID" != "" ]; then
echo "stopping timely-server instance ${i} [${PID}]"
kill -9 $PID
fi
done
pkill -9 -f 'timely-server-.*-exec.jar'
File diff suppressed because it is too large. Load diff
-25
View File
@@ -1,25 +0,0 @@
#!/bin/bash
if [[ `uname` == "Darwin" ]]; then
THIS_SCRIPT=`python -c 'import os,sys; print os.path.realpath(sys.argv[1])' $0`
else
THIS_SCRIPT=`readlink -f $0`
fi
THIS_DIR="${THIS_SCRIPT%/*}"
BASE_DIR=$(cd $THIS_DIR/../.. && pwd)
LIB_DIR="${BASE_DIR}/lib"
export BIN_DIR="${BASE_DIR}/bin"
export CONF_DIR="${BASE_DIR}/conf"
export LOG_DIR="${BASE_DIR}/logs"
set -a
. ${BIN_DIR}/timely-server-env.sh
set +a
JVM_ARGS="-Xmx1G -Xms1G"
JVM_ARGS="${JVM_ARGS} -Dlogging.config=${CONF_DIR}/log4j2-console.yml"
JVM_ARGS="${JVM_ARGS} -Dlog4j2.contextSelector=org.apache.logging.log4j.core.async.AsyncLoggerContextSelector"
echo "$JAVA_HOME/bin/java ${JVM_ARGS} -jar ${THIS_DIR}/timely-server*-exec.jar testingest --spring.profiles.active=timely,loadtest"
$JAVA_HOME/bin/java ${JVM_ARGS} -jar ${THIS_DIR}/timely-server-*-exec.jar testingest --spring.profiles.active=timely,loadtest
-25
View File
@@ -1,25 +0,0 @@
#!/bin/bash
if [[ `uname` == "Darwin" ]]; then
THIS_SCRIPT=`python -c 'import os,sys; print os.path.realpath(sys.argv[1])' $0`
else
THIS_SCRIPT=`readlink -f $0`
fi
THIS_DIR="${THIS_SCRIPT%/*}"
BASE_DIR=$(cd $THIS_DIR/../.. && pwd)
LIB_DIR="${BASE_DIR}/lib"
export BIN_DIR="${BASE_DIR}/bin"
export CONF_DIR="${BASE_DIR}/conf"
export LOG_DIR="${BASE_DIR}/logs"
set -a
. ${BIN_DIR}/timely-server-env.sh
set +a
JVM_ARGS="-Xmx1G -Xms1G"
JVM_ARGS="${JVM_ARGS} -Dlogging.config=${CONF_DIR}/log4j2-console.yml"
JVM_ARGS="${JVM_ARGS} -Dlog4j2.contextSelector=org.apache.logging.log4j.core.async.AsyncLoggerContextSelector"
echo "$JAVA_HOME/bin/java ${JVM_ARGS} -jar ${THIS_DIR}/timely-server*-exec.jar testquery --spring.profiles.active=timely,loadtest"
$JAVA_HOME/bin/java ${JVM_ARGS} -jar ${THIS_DIR}/timely-server-*-exec.jar testquery --spring.profiles.active=timely,loadtest
File diff suppressed because it is too large. Load diff
+11
View File
@@ -0,0 +1,11 @@
#!/bin/bash
export TRUSTSTORE_FILE=
export TRUSTSTORE_TYPE="jks";
export TRUSTSTORE_PASS=
export KEYSTORE_FILE=
export KEYSTORE_TYPE="pkcs12";
export KEYSTORE_PASS=
export NUM_QUERY_THREADS=4
export NUM_INGEST_THREADS=4
+21
View File
@@ -0,0 +1,21 @@
#!/bin/bash
if [[ `uname` == "Darwin" ]]; then
THIS_SCRIPT=`python -c 'import os,sys; print os.path.realpath(sys.argv[1])' $0`
else
THIS_SCRIPT=`readlink -f $0`
fi
THIS_DIR="${THIS_SCRIPT%/*}"
BASE_DIR=${THIS_DIR}/../..
TMP_DIR="${BASE_DIR}/tmp"
CONF_DIR="${BASE_DIR}/conf"
LIB_DIR="${BASE_DIR}/lib"
. ${THIS_DIR}/load-test-env.sh
export CLASSPATH="${CONF_DIR}:${THIS_DIR}:${LIB_DIR}/*"
JVM_ARGS="-Xmx1G -Xms1G"
echo "$JAVA_HOME/bin/java ${JVM_ARGS} timely.testing.MetricPutLoadTest $*"
$JAVA_HOME/bin/java ${JVM_ARGS} timely.testing.MetricPutLoadTest $*
Loaded 100 of 298 files, more files were not shown because too many files have changed in this diff. Show more