Compare commits

..
297 changed files with 8106 additions and 13350 deletions

No files matched your search

+1 -1
View File
@@ -4,7 +4,7 @@
<parent>
<groupId>timely</groupId>
<artifactId>timely</artifactId>
<version>2.1.0-SNAPSHOT</version>
<version>2.1.0</version>
</parent>
<artifactId>timely-analytics</artifactId>
<!--
@@ -1,40 +0,0 @@
#!/bin/bash
if [[ `uname` == "Darwin" ]]; then
THIS_SCRIPT=`python -c 'import os,sys; print os.path.realpath(sys.argv[1])' $0`
else
THIS_SCRIPT=`readlink -f $0`
fi
THIS_DIR="${THIS_SCRIPT%/*}"
BASE_DIR=$(cd $THIS_DIR/.. && pwd)
BIN_DIR=${BASE_DIR}/bin
CONF_DIR=${BASE_DIR}/conf
. ${BIN_DIR}/timely-balancer-env.sh
for i in $(seq 1 $NUM_PROFILES); do
if [ "$NUM_PROFILES" -eq "1" ]; then
FILENAME=timely-balancer.yml
else
FILENAME=timely-balancer-${i}.yml
fi
echo "creating configuration file ${CONF_DIR}/${FILENAME}"
cp ${CONF_DIR}/timely-balancer-template.yml ${CONF_DIR}/${FILENAME}
# if [ "$NUM_PROFILES" -eq "1" ]; then
# sed -i "s/\${INSTANCE}//g" ${CONF_DIR}/${FILENAME}
# else
# sed -i "s/\${INSTANCE}/${i}/g" ${CONF_DIR}/${FILENAME}
# fi
MULTIPLE=$(($i - 1))
TCP_PORT=$(($TCP_PORT_START + $MULTIPLE * $PORT_INCREMENT))
sed -i "s/\${TCP_PORT}/${TCP_PORT}/g" ${CONF_DIR}/${FILENAME}
HTTP_PORT=$(($HTTP_PORT_START + $MULTIPLE * $PORT_INCREMENT))
sed -i "s/\${HTTP_PORT}/${HTTP_PORT}/g" ${CONF_DIR}/${FILENAME}
WEBSOCKET_PORT=$(($WEBSOCKET_PORT_START + $MULTIPLE * $PORT_INCREMENT))
sed -i "s/\${WEBSOCKET_PORT}/${WEBSOCKET_PORT}/g" ${CONF_DIR}/${FILENAME}
UDP_PORT=$(($UDP_PORT_START + $MULTIPLE * $PORT_INCREMENT))
sed -i "s/\${UDP_PORT}/${UDP_PORT}/g" ${CONF_DIR}/${FILENAME}
done
+4 -9
View File
@@ -12,12 +12,7 @@ BIN_DIR=${BASE_DIR}/bin
. ${BIN_DIR}/timely-balancer-env.sh
if [ "$NUM_PROFILES" -eq "1" ]; then
echo ${BIN_DIR}/timely-balancer.sh
${BIN_DIR}/timely-balancer.sh
else
for i in $(seq 1 $NUM_PROFILES); do
echo ${BIN_DIR}/timely-balancer.sh $i
${BIN_DIR}/timely-balancer.sh $i
done
fi
for i in $(seq 1 $NUM_INSTANCES); do
echo ${BIN_DIR}/timely-balancer.sh $i
${BIN_DIR}/timely-balancer.sh $i
done
+19 -1
View File
@@ -1,3 +1,21 @@
#!/bin/bash
pkill -9 -f 'timely-balancer-.*-exec.jar'
if [[ `uname` == "Darwin" ]]; then
THIS_SCRIPT=`python -c 'import os,sys; print os.path.realpath(sys.argv[1])' $0`
else
THIS_SCRIPT=`readlink -f $0`
fi
THIS_DIR="${THIS_SCRIPT%/*}"
BASE_DIR=$(cd $THIS_DIR/.. && pwd)
BIN_DIR=${BASE_DIR}/bin
. ${BIN_DIR}/timely-balancer-env.sh
for i in $(seq 1 $NUM_INSTANCES); do
PID=`jps -m | grep -E 'timely-balancer-.*-exec.jar' | grep "instance=${i}" | awk '{print $1}'`
if [ "$PID" != "" ]; then
echo "stopping timely-balancer instance ${i} [${PID}]"
kill -9 $PID
fi
done
File diff suppressed because it is too large. Load diff
-11
View File
@@ -1,11 +0,0 @@
#!/bin/bash
export TRUSTSTORE_FILE=
export TRUSTSTORE_TYPE="jks";
export TRUSTSTORE_PASS=
export KEYSTORE_FILE=
export KEYSTORE_TYPE="pkcs12";
export KEYSTORE_PASS=
export NUM_QUERY_THREADS=4
export NUM_INGEST_THREADS=4
-21
View File
@@ -1,21 +0,0 @@
#!/bin/bash
if [[ `uname` == "Darwin" ]]; then
THIS_SCRIPT=`python -c 'import os,sys; print os.path.realpath(sys.argv[1])' $0`
else
THIS_SCRIPT=`readlink -f $0`
fi
THIS_DIR="${THIS_SCRIPT%/*}"
BASE_DIR=${THIS_DIR}/../..
TMP_DIR="${BASE_DIR}/tmp"
CONF_DIR="${BASE_DIR}/conf"
LIB_DIR="${BASE_DIR}/lib"
. ${THIS_DIR}/load-test-env.sh
export CLASSPATH="${CONF_DIR}:${THIS_DIR}:${LIB_DIR}/*"
JVM_ARGS="-Xmx1G -Xms1G"
echo "$JAVA_HOME/bin/java ${JVM_ARGS} timely.testing.MetricPutLoadTest $*"
$JAVA_HOME/bin/java ${JVM_ARGS} timely.testing.MetricPutLoadTest $*
-21
View File
@@ -1,21 +0,0 @@
#!/bin/bash
if [[ `uname` == "Darwin" ]]; then
THIS_SCRIPT=`python -c 'import os,sys; print os.path.realpath(sys.argv[1])' $0`
else
THIS_SCRIPT=`readlink -f $0`
fi
THIS_DIR="${THIS_SCRIPT%/*}"
BASE_DIR=${THIS_DIR}/../..
TMP_DIR="${BASE_DIR}/tmp"
CONF_DIR="${BASE_DIR}/conf"
LIB_DIR="${BASE_DIR}/lib"
. ${THIS_DIR}/load-test-env.sh
export CLASSPATH="${CONF_DIR}:${THIS_DIR}:${LIB_DIR}/*"
JVM_ARGS="-Xmx1G -Xms1G"
echo "$JAVA_HOME/bin/java ${JVM_ARGS} timely.testing.MetricQueryLoadTest $*"
$JAVA_HOME/bin/java ${JVM_ARGS} timely.testing.MetricQueryLoadTest $*
+10 -7
View File
@@ -1,16 +1,17 @@
#!/bin/bash
NUM_PROFILES=1
NUM_INSTANCES=1
ACCUMULO_ZOOKEEPERS=localhost:2181
TCP_PORT_START=4242
HTTP_PORT_START=4243
WEBSOCKET_PORT_START=4244
UDP_PORT_START=4245
PORT_INCREMENT=100
HADOOP_CONF_DIR=${HADOOP_CONF_DIR}
HADOOP_DEFAULT_FS=hdfs://localhost:8020
TCP_BASE_PORT=4242
HTTP_BASE_PORT=4243
WEBSOCKET_BASE_PORT=4244
UDP_BASE_PORT=4245
HOST_IP=0.0.0.0
HOST=$(hostname -s)
HOST_FQDN=$(hostname -f)
HOST_VERIFICATION_ENABLED=true
GENERATE_KEYS=true
TRUST_STORE_TYPE=
@@ -22,6 +23,8 @@ KEY_STORE_PASSWORD=
AUTHORIZATION_URL=
AUTHORIZATION_CERTIFICATE_TYPE=
AUTHORIZATION_CERTIFICATE_FILE=
ANONYMOUS_HTTP=true
ANONYMOUS_WEBSOCKET=true
REQUIRED_ROLES=
REQUIRED_AUTHS=
+37 -40
View File
@@ -1,69 +1,66 @@
#!/bin/bash
if [[ `uname` == "Darwin" ]]; then
THIS_SCRIPT=`python -c 'import os,sys; print os.path.realpath(sys.argv[1])' $0`
if [[ $(uname) == "Darwin" ]]; then
THIS_SCRIPT=$(python -c 'import os,sys; print os.path.realpath(sys.argv[1])' "$0")
TCNATIVE_SUFFIX="jnilib"
else
THIS_SCRIPT=`readlink -f $0`
THIS_SCRIPT=$(readlink -f "$0")
TCNATIVE_SUFFIX="so"
fi
THIS_DIR="${THIS_SCRIPT%/*}"
set -a
. ${THIS_DIR}/timely-balancer-env.sh
set +a
if [ -n "$1" ]; then
export INSTANCE="$1"
PROFILE_ARG="--spring.profiles.active=${INSTANCE}"
PID=`jps -m | grep -E 'timely-balancer-.*-exec' | grep "spring.profiles.active=${INSTANCE}" | awk '{print $1}'`
else
if [ -z "$1" ]; then
export INSTANCE="1"
PID=`jps -m | grep -E 'timely-balancer-.*-exec' | awk '{print $1}'`
else
export INSTANCE="$1"
fi
PID=$(jps -m | grep -E 'timely-balancer-.*-exec.jar' | grep "instance=${INSTANCE}" | awk '{print $1}')
if [ "$PID" == "" ]; then
NATIVE_DIR="${THIS_DIR}/META-INF/native"
BASE_DIR=$(cd $THIS_DIR/.. && pwd)
THIS_DIR="${THIS_SCRIPT%/*}"
BASE_DIR=$(cd "$THIS_DIR"/.. && pwd)
TMP_DIR="${BASE_DIR}/tmp"
export CONF_DIR="${BASE_DIR}/conf"
LIB_DIR="${BASE_DIR}/lib"
export LOG_DIR="${BASE_DIR}/logs"
NUM_SERVER_THREADS=4
BIN_DIR="${BASE_DIR}/bin"
export CONF_DIR="${BASE_DIR}/conf"
NATIVE_DIR="${BIN_DIR}/META-INF/native"
set -a
. "${BIN_DIR}/timely-balancer-env.sh"
set +a
# use either a value from timely-balancer-env.sh or the default
export LOG_DIR="${TIMELY_LOG_DIR:-${BASE_DIR}/logs}"
if [[ -e ${TMP_DIR} ]]; then
rm -rf ${TMP_DIR}
rm -rf "${TMP_DIR}"
fi
mkdir ${TMP_DIR}
mkdir "${TMP_DIR}"
if [[ -e ${NATIVE_DIR} ]]; then
rm -rf ${NATIVE_DIR}
rm -rf "${NATIVE_DIR}"
fi
mkdir -p ${NATIVE_DIR}
mkdir -p "${NATIVE_DIR}"
if [[ ! -e ${LOG_DIR} ]]; then
mkdir ${LOG_DIR}
mkdir "${LOG_DIR}"
fi
pushd ${BASE_DIR}/bin
$JAVA_HOME/bin/jar xf ${LIB_DIR}/netty-tcnative-boringssl-static*.jar META-INF/native/libnetty_tcnative_linux_x86_64.${TCNATIVE_SUFFIX}
$JAVA_HOME/bin/jar xf ${LIB_DIR}/netty-all*.jar META-INF/native/libnetty_transport_native_epoll_x86_64.${TCNATIVE_SUFFIX}
popd
pushd "${BASE_DIR}/bin" || exit
"${JAVA_HOME}/bin/jar xf ${LIB_DIR}/netty-tcnative-boringssl-static*.jar META-INF/native/libnetty_tcnative_linux_x86_64.${TCNATIVE_SUFFIX}"
"${JAVA_HOME}/bin/jar xf ${LIB_DIR}/netty-all*.jar META-INF/native/libnetty_transport_native_epoll_x86_64.${TCNATIVE_SUFFIX}"
popd || exit
export CLASSPATH="${CONF_DIR}:${LIB_DIR}/*:${HADOOP_CONF_DIR}"
JVM_ARGS="-Dio.netty.eventLoopThreads=${NUM_SERVER_THREADS} -Dlog4j.configurationFile=${CONF_DIR}/log4j2.yml"
JVM_ARGS="${JVM_ARGS} -DLog4jContextSelector=org.apache.logging.log4j.core.async.AsyncLoggerContextSelector"
JVM_ARGS="-Xmx2G -Xms2G -XX:NewSize=1G -XX:MaxNewSize=1G"
JVM_ARGS="${JVM_ARGS} -Djava.library.path=${NATIVE_DIR}"
JVM_ARGS="${JVM_ARGS} -Dlogging.config=${CONF_DIR}/log4j2.yml"
JVM_ARGS="${JVM_ARGS} -XX:+UseG1GC -XX:+UseStringDeduplication"
JVM_ARGS="${JVM_ARGS} -Djava.net.preferIPv4Stack=true"
JVM_ARGS="${JVM_ARGS} -XX:+UseNUMA"
JVM_ARGS="${JVM_ARGS} -Xmx2G -Xms2G -XX:NewSize=1G -XX:MaxNewSize=1G"
JVM_ARGS="${JVM_ARGS} -XX:+UseCompressedOops -XX:+UseNUMA -Djava.net.preferIPv4Stack=true"
JVM_ARGS="${JVM_ARGS} -XX:+UseG1GC -XX:MaxGCPauseMillis=4000 -XX:ParallelGCThreads=20 -XX:+UseStringDeduplication"
echo "$JAVA_HOME/bin/java ${JVM_ARGS} -jar ${THIS_DIR}/timely-balancer-*-exec.jar ${PROFILE_ARG}"
nohup $JAVA_HOME/bin/java ${JVM_ARGS} -jar ${THIS_DIR}/timely-balancer-*-exec.jar ${PROFILE_ARG} >> ${LOG_DIR}/timely-balancer${INSTANCE}.out 2>&1 &
echo "${JAVA_HOME}/bin/java ${JVM_ARGS} -jar ${THIS_DIR}/timely-balancer-*-exec.jar --instance=${INSTANCE}"
nohup "${JAVA_HOME}"/bin/java ${JVM_ARGS} -jar "${THIS_DIR}"/timely-balancer-*-exec.jar --instance="${INSTANCE}" >> "${LOG_DIR}/timely-balancer${INSTANCE}.out" 2>&1 &
else
echo "timely-balancer ${INSTANCE} already running with pid ${PID}"
echo "timely-balancer --instance=${INSTANCE} already running with pid ${PID}"
fi
-7
View File
@@ -12,13 +12,6 @@ Configuration:
value: "%clr{%d{yyyy-MM-dd HH:mm:ss.SSS}}{faint} %clr{%5p} %clr{${sys:PID}}{magenta} %clr{---}{faint} %clr{[%15.15t]}{faint} %clr{%-40.40c{1.}}{cyan} %clr{:}{faint} %m%n%wEx"
Appenders:
Console:
name: Console
target: SYSTEM_OUT
follow: true
PatternLayout:
pattern: "${LOG_PATTERN}"
RollingFile:
- name: File
fileName: "${sys:logDir}/timely-balancer${env:INSTANCE}.log"
-9
View File
@@ -1,9 +0,0 @@
trust.store.type=${TRUST_STORE_TYPE}
trust.store.file=${TRUST_STORE_FILE}
trust.store.password=${TRUST_STORE_PASSWORD}
key.store.type=${KEY_STORE_TYPE}
key.store.file=${KEY_STORE_FILE}
key.store.password=${KEY_STORE_PASSWORD}
authorization.url=${AUTHORIZATION_URL}
authorization.certificate.type=${AUTHORIZATION_CERTIFICATE_TYPE}
authorization.certificate.file=${AUTHORIZATION_CERTIFICATE_FILE}
-142
View File
@@ -1,142 +0,0 @@
timely:
zookeeper:
servers: ${ACCUMULO_ZOOKEEPERS}
timeout: 30s
security:
session-max-age: 86400
cache-expiration-minutes: 60
cache-refresh-minutes: 5
allow-anonymous-http-access: true
allow-anonymous-ws-access: true
serverSsl:
key-store-type: ${KEY_STORE_TYPE}
key-store-file: ${KEY_STORE_FILE}
key-store-password: ${KEY_STORE_PASSWORD}
use-generated-keypair: ${GENERATE_KEYS}
ignore-ssl-handshake-errors: true
trust-store-type: ${TRUST_STORE_TYPE}
trust-store-file: ${TRUST_STORE_FILE}
trust-store-password: ${TRUST_STORE_PASSWORD}
clientSsl:
key-store-type: ${KEY_STORE_TYPE}
key-store-file: ${KEY_STORE_FILE}
key-store-password: ${KEY_STORE_PASSWORD}
trust-store-type: ${TRUST_STORE_TYPE}
trust-store-file: ${TRUST_STORE_FILE}
trust-store-password: ${TRUST_STORE_PASSWORD}
jwtSsl:
key-store-type: ${AUTHORIZATION_CERTIFICATE_TYPE}
key-store-file: ${AUTHORIZATION_CERTIFICATE_FILE}
key-store-password: ${AUTHORIZATION_CERTIFICATE_PASSWORD}
cache:
enabled: false
metric-age-off-hours:
default: 26
nonCachedMetrics:
balancer:
checkServerHealthInterval: 10000
serverFailuresBeforeDown: 6
serverSuccessesBeforeUp: 1
assignmentFile: /timely/balancer/assignments.csv
server:
ip: ${HOST_IP}
tcpPort: ${TCP_PORT}
numTcpPools: 1
tcpBufferSize: 100
tcpClientPool:
minIdlePerKey: 0
maxIdlePerKey: 40
maxTotalPerKey: 40
maxTotal: -1
lifo: true
fairness: false
maxWaitMillis: -1
minEvictableIdleTimeMillis: 1800000
evictorShutdownTimeoutMillis: 10000
softMinEvictableIdleTimeMillis: -1
numTestsPerEvictionRun: 3
timeBetweenEvictionRunsMillis: -1
blockWhenExhausted: true
jmxEnabled: true
testOnBorrow: false
testOnReturn: false
udpPort: ${UDP_PORT}
udpClientPool:
minIdlePerKey: 0
maxIdlePerKey: 40
maxTotalPerKey: 40
maxTotal: -1
lifo: true
fairness: false
maxWaitMillis: -1
minEvictableIdleTimeMillis: 1800000
evictorShutdownTimeoutMillis: 10000
softMinEvictableIdleTimeMillis: -1
numTestsPerEvictionRun: 3
timeBetweenEvictionRunsMillis: -1
blockWhenExhausted: true
jmxEnabled: true
testOnBorrow: false
testOnReturn: false
http:
host: ${HOST}
ip: ${HOST_IP}
port: ${HTTP_PORT}
redirect-path: /secure-me
strict-transport-max-age: 604800
cors:
allow-any-origin: false
allow-null-origin: false
allow-credentials: true
allowed-origins:
allowed-methods: DELETE,GET,HEAD,OPTIONS,PUT,POST
allowed-headers: content-type
httpClientPool:
minIdlePerKey: 0
maxIdlePerKey: 40
maxTotalPerKey: 40
maxTotal: -1
lifo: true
fairness: false
maxWaitMillis: -1
minEvictableIdleTimeMillis: 1800000
evictorShutdownTimeoutMillis: 10000
softMinEvictableIdleTimeMillis: -1
numTestsPerEvictionRun: 3
timeBetweenEvictionRunsMillis: -1
blockWhenExhausted: true
jmxEnabled: true
testOnBorrow: false
testOnReturn: false
websocket:
ip: ${HOST_IP}
port: ${WEBSOCKET_PORT}
timeout: 60
subscription-lag: 120
scanner-batch-size: 10000
flush-interval-seconds: 30
scanner-read-ahead: 1
subscription-batch-size: 5000
incomingBufferSize: 50000000
wsClientPool:
minIdlePerKey: 0
maxIdlePerKey: 40
maxTotalPerKey: 40
maxTotal: -1
lifo: true
fairness: false
maxWaitMillis: -1
minEvictableIdleTimeMillis: 1800000
evictorShutdownTimeoutMillis: 10000
softMinEvictableIdleTimeMillis: -1
numTestsPerEvictionRun: 3
timeBetweenEvictionRunsMillis: -1
blockWhenExhausted: true
jmxEnabled: true
testOnBorrow: false
testOnReturn: false
logging:
level:
ROOT: info
timey: debug
+106 -97
View File
@@ -1,4 +1,5 @@
timely:
instance: ${INSTANCE}
zookeeper:
servers: ${ACCUMULO_ZOOKEEPERS}
timeout: 30s
@@ -6,82 +7,90 @@ timely:
session-max-age: 86400
cache-expiration-minutes: 60
cache-refresh-minutes: 5
allow-anonymous-http-access: true
allow-anonymous-ws-access: true
serverSsl:
key-store-type:
key-store-file:
key-store-password:
use-generated-keypair: true
authorization-url: ${AUTHORIZATION_URL}
required-roles: ${REQUIRED_ROLES}
required-auths: ${REQUIRED_AUTHS}
allow-anonymous-http-access: ${ANONYMOUS_HTTP:true}
allow-anonymous-ws-access: ${ANONYMOUS_WEBSOCKET:true}
server-ssl:
key-store-type: ${KEY_STORE_TYPE}
key-store-file: ${KEY_STORE_FILE}
key-store-password: ${KEY_STORE_PASSWORD}
use-generated-keypair: ${GENERATE_KEYS}
ignore-ssl-handshake-errors: true
trust-store-type:
trust-store-file:
trust-store-password:
clientSsl:
key-store-type:
key-store-file:
key-store-password:
trust-store-type:
trust-store-file:
trust-store-password:
jwtSsl:
key-store-type:
key-store-file:
key-store-password:
trust-store-type: ${TRUST_STORE_TYPE}
trust-store-file: ${TRUST_STORE_FILE}
trust-store-password: ${TRUST_STORE_PASSWORD}
client-ssl:
key-store-type: ${KEY_STORE_TYPE}
key-store-file: ${KEY_STORE_FILE}
key-store-password: ${KEY_STORE_PASSWORD}
trust-store-type: ${TRUST_STORE_TYPE}
trust-store-file: ${TRUST_STORE_FILE}
trust-store-password: ${TRUST_STORE_PASSWORD}
hostVerificationEnabled: ${HOST_VERIFICATION_ENABLED:true}
jwt-ssl:
key-store-type: ${AUTHORIZATION_CERTIFICATE_TYPE}
key-store-file: ${AUTHORIZATION_CERTIFICATE_FILE}
key-store-password: ${AUTHORIZATION_CERTIFICATE_PASSWORD}
cache:
enabled: false
metric-age-off-hours:
default: 26
nonCachedMetrics:
non-cached-metrics:
balancer:
checkServerHealthInterval: 10000
serverFailuresBeforeDown: 6
serverSuccessesBeforeUp: 1
assignmentFile: /timely/balancer/assignments.csv
check-server-health-interval: 10000
server-failures-before-down: 6
server-successes-before-up: 1
default-fs: ${HADOOP_DEFAULT_FS}
fs-config-resources:
- ${HADOOP_CONF_DIR}/core-site.xml
- ${HADOOP_CONF_DIR}/hdfs-site.xml
assignment-file: hdfs:///timely/balancer/assignments.csv
server:
ip: 0.0.0.0
tcpPort: 4242
numTcpPools: 1
tcpBufferSize: 100
tcpClientPool:
minIdlePerKey: 0
maxIdlePerKey: 40
maxTotalPerKey: 40
maxTotal: -1
ip: ${HOST_IP}
tcp-base-port: ${TCP_BASE_PORT}
num-tcp-pools: 1
tcp-buffer-size: 100
tcp-client-pool:
min-idle-per-key: 0
max-idle-per-key: 40
max-total-per-key: 40
max-total: -1
lifo: true
fairness: false
maxWaitMillis: -1
minEvictableIdleTimeMillis: 1800000
evictorShutdownTimeoutMillis: 10000
softMinEvictableIdleTimeMillis: -1
numTestsPerEvictionRun: 3
timeBetweenEvictionRunsMillis: -1
blockWhenExhausted: true
jmxEnabled: true
testOnBorrow: false
testOnReturn: false
udpPort: 4245
udpClientPool:
minIdlePerKey: 0
maxIdlePerKey: 40
maxTotalPerKey: 40
maxTotal: -1
max-wait-millis: -1
min-evictable-idle-time-millis: 1800000
evictor-shutdown-timeout-millis: 10000
soft-min-evictable-idle-time-millis: -1
num-tests-per-eviction-run: 3
time-between-eviction-runs-millis: -1
block-when-exhausted: true
jmx-enabled: true
test-on-borrow: false
test-on-return: false
udp-base-port: ${UDP_BASE_PORT}
udp-client-pool:
min-idle-per-key: 0
max-idle-per-key: 40
max-total-per-key: 40
max-total: -1
lifo: true
fairness: false
maxWaitMillis: -1
minEvictableIdleTimeMillis: 1800000
evictorShutdownTimeoutMillis: 10000
softMinEvictableIdleTimeMillis: -1
numTestsPerEvictionRun: 3
timeBetweenEvictionRunsMillis: -1
blockWhenExhausted: true
jmxEnabled: true
testOnBorrow: false
testOnReturn: false
max-wait-millis: -1
min-evictable-idle-time-millis: 1800000
evictor-shutdown-timeout-millis: 10000
soft-min-evictable-idle-time-millis: -1
num-tests-per-eviction-run: 3
time-between-eviction-runs-millis: -1
block-when-exhausted: true
jmx-enabled: true
test-on-borrow: false
test-on-return: false
http:
host: localhost
ip: 0.0.0.0
port: 4243
host: ${HOST}
ip: ${HOST_IP}
base-port: ${HTTP_BASE_PORT}
redirect-path: /secure-me
strict-transport-max-age: 604800
cors:
@@ -92,51 +101,51 @@ timely:
allowed-methods: DELETE,GET,HEAD,OPTIONS,PUT,POST
allowed-headers: content-type
httpClientPool:
minIdlePerKey: 0
maxIdlePerKey: 40
maxTotalPerKey: 40
maxTotal: -1
min-idle-per-key: 0
max-idle-per-key: 40
max-total-per-key: 40
max-total: -1
lifo: true
fairness: false
maxWaitMillis: -1
minEvictableIdleTimeMillis: 1800000
evictorShutdownTimeoutMillis: 10000
softMinEvictableIdleTimeMillis: -1
numTestsPerEvictionRun: 3
timeBetweenEvictionRunsMillis: -1
blockWhenExhausted: true
jmxEnabled: true
testOnBorrow: false
testOnReturn: false
max-wait-millis: -1
min-evictable-idle-time-millis: 1800000
evictor-shutdown-timeout-millis: 10000
soft-min-evictable-idle-time-millis: -1
num-tests-per-eviction-run: 3
time-between-eviction-runs-millis: -1
block-when-exhausted: true
jmx-enabled: true
test-on-borrow: false
test-on-return: false
websocket:
ip: 0.0.0.0
port: 5244
ip: ${HOST_IP}
base-port: ${WEBSOCKET_BASE_PORT}
timeout: 60
subscription-lag: 120
scanner-batch-size: 10000
flush-interval-seconds: 30
scanner-read-ahead: 1
subscription-batch-size: 5000
incomingBufferSize: 50000000
wsClientPool:
minIdlePerKey: 0
maxIdlePerKey: 40
maxTotalPerKey: 40
maxTotal: -1
incoming-buffer-size: 50000000
ws-client-pool:
min-idle-per-key: 0
max-idle-per-key: 40
max-total-per-key: 40
max-total: -1
lifo: true
fairness: false
maxWaitMillis: -1
minEvictableIdleTimeMillis: 1800000
evictorShutdownTimeoutMillis: 10000
softMinEvictableIdleTimeMillis: -1
numTestsPerEvictionRun: 3
timeBetweenEvictionRunsMillis: -1
blockWhenExhausted: true
jmxEnabled: true
testOnBorrow: false
testOnReturn: false
max-wait-millis: -1
min-evictable-idle-time-millis: 1800000
evictor-shutdown-timeout-millis: 10000
soft-min-evictable-idle-time-millis: -1
num-tests-per-eviction-run: 3
time-between-eviction-runs-millis: -1
block-when-exhausted: true
jmx-enabled: true
test-on-borrow: false
test-on-return: false
logging:
level:
ROOT: info
timey: debug
timely: debug
+16 -17
View File
@@ -4,7 +4,7 @@
<parent>
<groupId>timely</groupId>
<artifactId>timely</artifactId>
<version>2.1.0-SNAPSHOT</version>
<version>2.1.0</version>
</parent>
<artifactId>timely-balancer</artifactId>
<description>Timely Balancer</description>
@@ -40,10 +40,6 @@
<groupId>commons-beanutils</groupId>
<artifactId>commons-beanutils</artifactId>
</dependency>
<dependency>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
</dependency>
<dependency>
<groupId>io.github.classgraph</groupId>
<artifactId>classgraph</artifactId>
@@ -60,10 +56,6 @@
<groupId>org.apache.accumulo</groupId>
<artifactId>accumulo-core</artifactId>
</dependency>
<dependency>
<groupId>org.apache.accumulo</groupId>
<artifactId>accumulo-minicluster</artifactId>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-collections4</artifactId>
@@ -100,10 +92,6 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-minicluster</artifactId>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-1.2-api</artifactId>
@@ -166,10 +154,6 @@
<groupId>timely</groupId>
<artifactId>timely-common</artifactId>
</dependency>
<dependency>
<groupId>timely</groupId>
<artifactId>timely-server</artifactId>
</dependency>
<dependency>
<groupId>com.lmax</groupId>
<artifactId>disruptor</artifactId>
@@ -180,6 +164,11 @@
<artifactId>netty-tcnative-boringssl-static</artifactId>
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>javax.activation</groupId>
<artifactId>activation</artifactId>
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
@@ -200,6 +189,11 @@
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-minicluster</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.jsoup</groupId>
<artifactId>jsoup</artifactId>
@@ -215,6 +209,11 @@
<artifactId>spring-boot-test-autoconfigure</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>timely</groupId>
<artifactId>timely-test</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
@@ -1,4 +1,4 @@
package timely.balancer.component;
package timely.balancer;
import static org.apache.accumulo.core.conf.ConfigurationTypeHelper.getTimeInMillis;
import static timely.Constants.NON_CACHED_METRICS;
@@ -9,30 +9,24 @@ import java.net.InetSocketAddress;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.List;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import javax.net.ssl.SSLContext;
import org.apache.commons.lang3.StringUtils;
import org.apache.curator.RetryPolicy;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.CuratorFrameworkFactory;
import org.apache.curator.retry.RetryForever;
import org.apache.hadoop.util.StringUtils;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.data.Stat;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
import org.springframework.context.ApplicationContext;
import org.springframework.stereotype.Component;
import io.netty.bootstrap.AbstractBootstrap;
import io.netty.bootstrap.Bootstrap;
@@ -64,12 +58,9 @@ import io.netty.handler.codec.http.HttpResponseEncoder;
import io.netty.handler.codec.http.HttpServerCodec;
import io.netty.handler.codec.http.websocketx.WebSocketServerProtocolHandler;
import io.netty.handler.logging.LoggingHandler;
import io.netty.handler.ssl.ClientAuth;
import io.netty.handler.ssl.OpenSslServerContext;
import io.netty.handler.ssl.OpenSslServerSessionContext;
import io.netty.handler.ssl.SslContext;
import io.netty.handler.ssl.SslContextBuilder;
import io.netty.handler.ssl.util.SelfSignedCertificate;
import io.netty.handler.stream.ChunkedWriteHandler;
import io.netty.handler.timeout.IdleStateHandler;
import io.netty.util.NettyRuntime;
@@ -87,11 +78,9 @@ import timely.balancer.netty.http.HttpRelayHandler;
import timely.balancer.netty.tcp.TcpRelayHandler;
import timely.balancer.netty.udp.UdpRelayHandler;
import timely.balancer.netty.ws.WsRelayHandler;
import timely.client.http.HttpClient;
import timely.common.component.AuthenticationService;
import timely.common.configuration.SecurityProperties;
import timely.common.configuration.SslClientProperties;
import timely.common.configuration.SslHelper;
import timely.common.configuration.SslServerProperties;
import timely.common.configuration.ZookeeperProperties;
import timely.netty.http.HttpStaticFileServerHandler;
@@ -107,8 +96,6 @@ import timely.netty.websocket.WebSocketFullRequestHandler;
import timely.netty.websocket.WebSocketRequestDecoder;
import timely.netty.websocket.subscription.WSTimelyExceptionHandler;
@Component
@ConditionalOnMissingBean(Balancer.class)
public class Balancer {
final public static String LEADER_LATCH_PATH = "/timely/balancer/leader";
@@ -120,8 +107,6 @@ public class Balancer {
private static final String OS_NAME = "os.name";
private static final String OS_VERSION = "os.version";
private static final String WS_PATH = "/websocket";
protected static final CountDownLatch LATCH = new CountDownLatch(1);
private BalancerProperties balancerProperties;
private SecurityProperties securityProperties;
private SslServerProperties sslServerProperties;
@@ -134,7 +119,8 @@ public class Balancer {
private AuthenticationService authenticationService;
private CuratorFramework curatorFramework;
private int quietPeriod;
private SslContext serverSslContext;
private SSLContext clientSSLContext;
private EventLoopGroup tcpWorkerGroup = null;
private EventLoopGroup tcpBossGroup = null;
private EventLoopGroup httpWorkerGroup = null;
@@ -157,20 +143,26 @@ public class Balancer {
final String osName = SystemPropertyUtil.get(OS_NAME).toLowerCase().trim();
final String osVersion = SystemPropertyUtil.get(OS_VERSION).toLowerCase();
// split at periods, keep the first two, join with a period
final String majMinVers = StringUtils.join(".", Arrays.stream(osVersion.split("\\.")).limit(2).collect(Collectors.toList()));
final String majMinVers = StringUtils.join(Arrays.stream(osVersion.split("\\.")).limit(2).collect(Collectors.toList()), ".");
if (osName.startsWith("linux")) {
Runtime.Version currentVersion = Runtime.Version.parse(majMinVers);
Runtime.Version epollMinVersion = Runtime.Version.parse(EPOLL_MIN_VERSION);
return currentVersion.compareTo(epollMinVersion) > 0;
} else {
return false;
try {
Runtime.Version currentVersion = Runtime.Version.parse(majMinVers);
Runtime.Version epollMinVersion = Runtime.Version.parse(EPOLL_MIN_VERSION);
return currentVersion.compareTo(epollMinVersion) > 0;
} catch (Exception e) {
log.error(e.getMessage(), e);
}
}
return false;
}
public Balancer(ApplicationContext applicationContext, AuthenticationService authenticationService, MetricResolver metricResolver,
BalancerProperties balancerProperties, ZookeeperProperties zookeeperProperties, SecurityProperties securityProperties,
SslServerProperties sslServerProperties, SslClientProperties sslClientProperties, BalancerServerProperties balancerServerProperties,
BalancerHttpProperties balancerHttpProperties, BalancerWebsocketProperties balancerWebsocketProperties) {
public Balancer(ApplicationContext applicationContext, SslContext serverSslContext, SSLContext clientSSLContext,
AuthenticationService authenticationService, MetricResolver metricResolver, BalancerProperties balancerProperties,
ZookeeperProperties zookeeperProperties, SecurityProperties securityProperties, SslServerProperties sslServerProperties,
SslClientProperties sslClientProperties, BalancerServerProperties balancerServerProperties, BalancerHttpProperties balancerHttpProperties,
BalancerWebsocketProperties balancerWebsocketProperties) {
this.serverSslContext = serverSslContext;
this.clientSSLContext = clientSSLContext;
this.quietPeriod = balancerServerProperties.getShutdownQuietPeriod();
this.applicationContext = applicationContext;
this.authenticationService = authenticationService;
@@ -186,8 +178,7 @@ public class Balancer {
DEFAULT_EVENT_LOOP_THREADS = Math.max(1, SystemPropertyUtil.getInt("io.netty.eventLoopThreads", NettyRuntime.availableProcessors() * 2));
}
@PostConstruct
public void startup() {
public void start() {
log.info("Starting {}", this.getClass().getSimpleName());
try {
// start curator framework
@@ -246,9 +237,9 @@ public class Balancer {
final int httpPort = balancerHttpProperties.getPort();
final String httpIp = balancerHttpProperties.getIp();
SslContext sslCtx = createSSLContext(sslServerProperties);
if (sslCtx instanceof OpenSslServerContext) {
OpenSslServerContext openssl = (OpenSslServerContext) sslCtx;
if (serverSslContext instanceof OpenSslServerContext) {
OpenSslServerContext openssl = (OpenSslServerContext) serverSslContext;
String application = "Timely_" + httpPort;
OpenSslServerSessionContext opensslCtx = openssl.sessionContext();
opensslCtx.setSessionCacheEnabled(true);
@@ -256,13 +247,12 @@ public class Balancer {
opensslCtx.setSessionIdContext(application.getBytes(StandardCharsets.UTF_8));
opensslCtx.setSessionTimeout(securityProperties.getSessionMaxAge());
}
SSLContext clientSSLContext = createSSLClientContext(sslClientProperties);
final ServerBootstrap httpServer = new ServerBootstrap();
httpServer.group(httpBossGroup, httpWorkerGroup);
httpServer.channel(channelClass);
httpServer.handler(new LoggingHandler());
HttpClientPool httpClientPool = new HttpClientPool(securityProperties, sslClientProperties, balancerHttpProperties, clientSSLContext);
httpServer.childHandler(setupHttpChannel(balancerHttpProperties, sslServerProperties, sslCtx, metricResolver, httpClientPool));
HttpClientPool httpClientPool = new HttpClientPool(balancerHttpProperties, clientSSLContext, sslClientProperties);
httpServer.childHandler(setupHttpChannel(balancerHttpProperties, sslServerProperties, serverSslContext, metricResolver, httpClientPool));
httpServer.option(ChannelOption.ALLOCATOR, PooledByteBufAllocator.DEFAULT);
httpServer.option(ChannelOption.SO_BACKLOG, 128);
httpServer.childOption(ChannelOption.SO_KEEPALIVE, true);
@@ -276,7 +266,7 @@ public class Balancer {
wsServer.channel(channelClass);
wsServer.handler(new LoggingHandler());
WsClientPool wsClientPool = new WsClientPool(balancerProperties, balancerWebsocketProperties, sslClientProperties, clientSSLContext);
wsServer.childHandler(setupWSChannel(balancerWebsocketProperties, securityProperties, sslCtx, metricResolver, wsClientPool));
wsServer.childHandler(setupWSChannel(balancerWebsocketProperties, securityProperties, serverSslContext, metricResolver, wsClientPool));
wsServer.option(ChannelOption.ALLOCATOR, PooledByteBufAllocator.DEFAULT);
wsServer.option(ChannelOption.SO_BACKLOG, 128);
wsServer.childOption(ChannelOption.SO_KEEPALIVE, true);
@@ -317,7 +307,6 @@ public class Balancer {
return channel;
}
@PreDestroy
public void shutdown() {
List<ChannelFuture> channelFutures = new ArrayList<>();
@@ -410,31 +399,6 @@ public class Balancer {
log.info("{} shut down.", this.getClass().getSimpleName());
}
protected SslContext createSSLContext(SslServerProperties sslServerProperties) throws Exception {
Boolean generate = sslServerProperties.isUseGeneratedKeypair();
SslContextBuilder ssl;
if (generate) {
log.warn("Using generated self signed server certificate");
Date begin = new Date();
Date end = new Date(begin.getTime() + 86400000);
SelfSignedCertificate ssc = new SelfSignedCertificate("localhost", begin, end);
ssl = SslContextBuilder.forServer(ssc.certificate(), ssc.privateKey());
} else {
ssl = SslHelper.getSslContextBuilder(sslServerProperties);
}
// Can't set to REQUIRE because the CORS pre-flight requests will fail.
ssl.clientAuth(ClientAuth.OPTIONAL);
return ssl.build();
}
protected SSLContext createSSLClientContext(SslClientProperties sslClientProperties) {
SslClientProperties ssl = sslClientProperties;
return HttpClient.getSSLContext(ssl.getTrustStoreFile(), ssl.getTrustStoreType(), ssl.getTrustStorePassword(), ssl.getKeyStoreFile(),
ssl.getKeyStoreType(), ssl.getKeyStorePassword());
}
private void ensureZkPaths(CuratorFramework curatorFramework, String[] paths) {
for (String s : paths) {
try {
@@ -1,4 +1,4 @@
package timely.balancer.component;
package timely.balancer;
import java.util.ArrayList;
import java.util.List;
@@ -6,13 +6,8 @@ import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
import org.springframework.stereotype.Component;
import timely.balancer.configuration.BalancerProperties;
import timely.balancer.configuration.BalancerServerProperties;
@@ -20,8 +15,6 @@ import timely.balancer.connection.TimelyBalancedHost;
import timely.balancer.connection.tcp.TcpClientPool;
import timely.balancer.healthcheck.TCPServerHealthCheck;
@Component
@ConditionalOnMissingBean(HealthChecker.class)
public class HealthChecker {
private static final Logger log = LoggerFactory.getLogger(HealthChecker.class);
@@ -36,8 +29,7 @@ public class HealthChecker {
this.balancerServerProperties = balancerServerProperties;
}
@PostConstruct
public void setup() {
public void start() {
TcpClientPool tcpClientPool = new TcpClientPool(balancerServerProperties);
this.tcpServerHealthCheck = new TCPServerHealthCheck(tcpClientPool);
this.executorService.scheduleAtFixedRate(() -> {
@@ -58,8 +50,7 @@ public class HealthChecker {
}, 0, balancerProperties.getCheckServerHealthInterval(), TimeUnit.MILLISECONDS);
}
@PreDestroy
public void close() throws Exception {
public void shutdown() {
this.executorService.shutdown();
try {
this.executorService.awaitTermination(5, TimeUnit.SECONDS);
@@ -1,13 +1,15 @@
package timely.balancer.component;
package timely.balancer;
import static org.apache.hadoop.fs.FileSystem.FS_DEFAULT_NAME_KEY;
import static timely.Constants.NON_CACHED_METRICS;
import static timely.Constants.NON_CACHED_METRICS_LOCK_PATH;
import static timely.Constants.SERVICE_DISCOVERY_PATH;
import static timely.balancer.component.Balancer.ASSIGNMENTS_LAST_UPDATED_PATH;
import static timely.balancer.component.Balancer.ASSIGNMENTS_LOCK_PATH;
import static timely.balancer.component.Balancer.LEADER_LATCH_PATH;
import static timely.balancer.Balancer.ASSIGNMENTS_LAST_UPDATED_PATH;
import static timely.balancer.Balancer.ASSIGNMENTS_LOCK_PATH;
import static timely.balancer.Balancer.LEADER_LATCH_PATH;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Collection;
@@ -29,9 +31,6 @@ import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import org.apache.commons.lang3.SerializationUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.curator.framework.CuratorFramework;
@@ -53,25 +52,21 @@ import org.apache.curator.x.discovery.details.ServiceCacheListener;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.zookeeper.CreateMode;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
import org.springframework.stereotype.Component;
import com.csvreader.CsvReader;
import com.csvreader.CsvWriter;
import timely.ServerDetails;
import timely.balancer.ArrivalRate;
import timely.balancer.configuration.BalancerProperties;
import timely.balancer.connection.TimelyBalancedHost;
import timely.common.configuration.CacheProperties;
@Component
@ConditionalOnMissingBean(MetricResolver.class)
public class MetricResolver {
private static final Logger log = LoggerFactory.getLogger(MetricResolver.class);
@@ -109,13 +104,17 @@ public class MetricResolver {
this.healthChecker = healthChecker;
}
@PostConstruct
public void setup() {
public void start() {
assignmentsIPRWLock = new InterProcessReadWriteLock(curatorFramework, ASSIGNMENTS_LOCK_PATH);
testIPRWLock(curatorFramework, assignmentsIPRWLock, ASSIGNMENTS_LOCK_PATH);
startLeaderLatch(curatorFramework);
startServiceListener(curatorFramework);
assignmentsLastUpdatedInHdfs = new DistributedAtomicLong(curatorFramework, ASSIGNMENTS_LAST_UPDATED_PATH, new RetryForever(1000));
try {
assignmentsLastUpdatedInHdfs.forceSet(getLastWrittenToHdfsTimestamp());
} catch (Exception e) {
log.error(e.getMessage(), e);
}
TreeCacheListener assignmentListener = (curframework, event) -> {
log.debug("Handling assignments event {}. assignmentsLastUpdatedInHdfs:{}", event.getType().toString(),
@@ -191,8 +190,7 @@ public class MetricResolver {
}, 10, 60, TimeUnit.SECONDS);
}
@PreDestroy
public void close() throws Exception {
public void shutdown() {
this.assignmentExecutor.shutdown();
try {
@@ -358,8 +356,13 @@ public class MetricResolver {
if (isLeader.get()) {
for (String s : reassignMetrics) {
TimelyBalancedHost h = getRoundRobinHost(null);
assignMetric(s, h);
log.debug("Assigned server removed. Assigning {} to server {}:{}", s, h.getTcpPort());
if (h == null) {
unassignMetric(s);
log.debug("Assigned server removed and none available. Unassigning {}", s);
} else {
assignMetric(s, h);
log.debug("Assigned server removed. Assigning {} to server {}:{}", s, h.getTcpPort());
}
}
}
@@ -787,20 +790,19 @@ public class MetricResolver {
tbh = metricToHostMap.get(metric);
if (tbh == null) {
tbh = getRoundRobinHost(null);
if (!balancerLock.isWriteLockedByCurrentThread()) {
balancerLock.readLock().unlock();
balancerLock.writeLock().lock();
}
assignMetric(metric, tbh);
} else if (!tbh.isUp()) {
TimelyBalancedHost oldTbh = tbh;
tbh = getLeastUsedHost();
log.debug("rebalancing from host that is down: reassigning metric {} from server {}:{} to {}:{}", metric, oldTbh.getHost(),
oldTbh.getTcpPort(), tbh.getHost(), tbh.getTcpPort());
if (!balancerLock.isWriteLockedByCurrentThread()) {
balancerLock.readLock().unlock();
balancerLock.writeLock().lock();
if (tbh != null) {
log.debug("rebalancing from host that is down: reassigning metric {} from server {}:{} to {}:{}", metric, oldTbh.getHost(),
oldTbh.getTcpPort(), tbh.getHost(), tbh.getTcpPort());
}
}
if (!balancerLock.isWriteLockedByCurrentThread()) {
balancerLock.readLock().unlock();
balancerLock.writeLock().lock();
}
if (tbh != null) {
assignMetric(metric, tbh);
}
}
@@ -899,18 +901,45 @@ public class MetricResolver {
return tbh;
}
protected FileSystem getFileSystem(Path path) throws IOException {
Configuration configuration = new Configuration();
if (StringUtils.isNotBlank(balancerProperties.getDefaultFs())) {
configuration.set(FS_DEFAULT_NAME_KEY, balancerProperties.getDefaultFs());
}
if (balancerProperties.getFsConfigResources() != null) {
for (String resource : balancerProperties.getFsConfigResources()) {
configuration.addResource(new Path(resource));
}
}
return FileSystem.get(path.toUri(), configuration);
}
protected void createAssignmentFile() {
try {
log.info("Creating assignment file: " + balancerProperties.getAssignmentFile());
Configuration configuration = new Configuration();
FileSystem fs = FileSystem.get(configuration);
Path assignmentFile = new Path(balancerProperties.getAssignmentFile());
FileSystem fs = getFileSystem(assignmentFile);
fs.create(assignmentFile);
} catch (Exception e) {
log.error(e.getMessage(), e);
}
}
protected long getLastWrittenToHdfsTimestamp() {
try {
Path assignmentFile = new Path(balancerProperties.getAssignmentFile());
FileSystem fs = getFileSystem(assignmentFile);
FileStatus fileStatus = fs.getFileStatus(assignmentFile);
return fileStatus.getModificationTime();
} catch (FileNotFoundException e) {
createAssignmentFile();
return System.currentTimeMillis();
} catch (Exception e) {
log.error(e.getMessage(), e);
return System.currentTimeMillis();
}
}
private void readAssignmentsFromHdfs(boolean checkIfNecessary) {
try {
@@ -937,9 +966,8 @@ public class MetricResolver {
acquired = assignmentsIPRWLock.readLock().acquire(60, TimeUnit.SECONDS);
}
balancerLock.writeLock().lock();
Configuration configuration = new Configuration();
FileSystem fs = FileSystem.get(configuration);
Path assignmentFile = new Path(balancerProperties.getAssignmentFile());
FileSystem fs = getFileSystem(assignmentFile);
FSDataInputStream iStream = fs.open(assignmentFile);
CsvReader reader = new CsvReader(iStream, ',', Charset.forName("UTF-8"));
reader.setUseTextQualifier(false);
@@ -961,7 +989,9 @@ public class MetricResolver {
} else {
log.trace("Found assigment: {} to {}:{}", metric, host, tcpPort);
}
assignedMetricToHostMap.put(metric, tbh);
if (tbh != null) {
assignedMetricToHostMap.put(metric, tbh);
}
}
}
@@ -1022,9 +1052,8 @@ public class MetricResolver {
nonCachedMetricsLocalLock.readLock().lock();
try {
if (!metricToHostMap.isEmpty()) {
Configuration configuration = new Configuration();
FileSystem fs = FileSystem.get(configuration);
Path assignmentFile = new Path(balancerProperties.getAssignmentFile());
FileSystem fs = getFileSystem(assignmentFile);
if (!fs.exists(assignmentFile.getParent())) {
fs.mkdirs(assignmentFile.getParent());
}
@@ -1036,7 +1065,7 @@ public class MetricResolver {
writer.write("tcpPort");
writer.endRecord();
for (Map.Entry<String,TimelyBalancedHost> e : metricToHostMap.entrySet()) {
if (!nonCachedMetrics.contains(e.getKey())) {
if (e.getValue() != null && !nonCachedMetrics.contains(e.getKey())) {
writer.write(e.getKey());
writer.write(e.getValue().getHost());
writer.write(Integer.toString(e.getValue().getTcpPort()));
@@ -1090,4 +1119,18 @@ public class MetricResolver {
}
}
}
protected void unassignMetric(String metric) {
if (isLeader.get()) {
if (StringUtils.isNotBlank(metric)) {
balancerLock.writeLock().lock();
try {
metricToHostMap.remove(metric);
assignmentsLastUpdatedLocal.set(System.currentTimeMillis());
} finally {
balancerLock.writeLock().unlock();
}
}
}
}
}
@@ -1,10 +1,58 @@
package timely.balancer.configuration;
import javax.net.ssl.SSLContext;
import org.apache.curator.framework.CuratorFramework;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.ApplicationContext;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import io.netty.handler.ssl.SslContext;
import timely.balancer.Balancer;
import timely.balancer.HealthChecker;
import timely.balancer.MetricResolver;
import timely.common.component.AuthenticationService;
import timely.common.configuration.CacheProperties;
import timely.common.configuration.SecurityProperties;
import timely.common.configuration.SslClientProperties;
import timely.common.configuration.SslServerProperties;
import timely.common.configuration.ZookeeperProperties;
@Configuration
@EnableConfigurationProperties({BalancerProperties.class, BalancerServerProperties.class, BalancerWebsocketProperties.class, BalancerHttpProperties.class})
public class BalancerConfiguration {
@Bean
@ConditionalOnMissingBean
public Balancer balancer(ApplicationContext applicationContext, @Qualifier("nettySslContext") SslContext serverSslContext,
@Qualifier("outboundJDKSslContext") SSLContext clientSSLContext, AuthenticationService authenticationService, MetricResolver metricResolver,
BalancerProperties balancerProperties, ZookeeperProperties zookeeperProperties, SecurityProperties securityProperties,
SslServerProperties sslServerProperties, SslClientProperties sslClientProperties, BalancerServerProperties balancerServerProperties,
BalancerHttpProperties balancerHttpProperties, BalancerWebsocketProperties balancerWebsocketProperties) {
Balancer balancer = new Balancer(applicationContext, serverSslContext, clientSSLContext, authenticationService, metricResolver, balancerProperties,
zookeeperProperties, securityProperties, sslServerProperties, sslClientProperties, balancerServerProperties, balancerHttpProperties,
balancerWebsocketProperties);
balancer.start();
return balancer;
}
@Bean
@ConditionalOnMissingBean
public HealthChecker healthChecker(BalancerProperties balancerProperties, BalancerServerProperties balancerServerProperties) {
HealthChecker healthChecker = new HealthChecker(balancerProperties, balancerServerProperties);
healthChecker.start();
return healthChecker;
}
@Bean
@ConditionalOnMissingBean
public MetricResolver metricResolver(CuratorFramework curatorFramework, BalancerProperties balancerProperties, CacheProperties cacheProperties,
HealthChecker healthChecker) {
MetricResolver metricResolver = new MetricResolver(curatorFramework, balancerProperties, cacheProperties, healthChecker);
metricResolver.start();
return metricResolver;
}
}
@@ -1,5 +1,7 @@
package timely.balancer.configuration;
import java.util.List;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;
import org.springframework.validation.annotation.Validated;
@@ -13,6 +15,10 @@ public class BalancerProperties {
private String assignmentFile;
private String defaultFs;
private List<String> fsConfigResources;
private Double controlBandPercentage = 0.05;
private long checkServerHealthInterval = 10000;
@@ -37,6 +43,22 @@ public class BalancerProperties {
this.assignmentFile = assignmentFile;
}
public String getDefaultFs() {
return defaultFs;
}
public void setDefaultFs(String defaultFs) {
this.defaultFs = defaultFs;
}
public List<String> getFsConfigResources() {
return fsConfigResources;
}
public void setFsConfigResources(List<String> fsConfigResources) {
this.fsConfigResources = fsConfigResources;
}
public void setControlBandPercentage(Double controlBandPercentage) {
this.controlBandPercentage = controlBandPercentage;
}
@@ -1,6 +0,0 @@
package timely.balancer.configuration;
import org.springframework.boot.autoconfigure.SpringBootApplication;
@SpringBootApplication(scanBasePackageClasses = {SpringBootstrap.class})
public class SpringBootstrap {}
@@ -8,17 +8,15 @@ import org.apache.commons.pool2.impl.DefaultPooledObject;
import org.apache.http.impl.client.CloseableHttpClient;
import timely.balancer.connection.TimelyBalancedHost;
import timely.common.configuration.SecurityProperties;
import timely.client.http.HttpClient;
import timely.common.configuration.SslClientProperties;
public class HttpClientFactory implements KeyedPooledObjectFactory<TimelyBalancedHost,CloseableHttpClient> {
private final SSLContext sslContext;
private final SecurityProperties security;
private SslClientProperties sslClientProperties;
public HttpClientFactory(SecurityProperties security, SslClientProperties sslClientProperties, SSLContext sslContext) {
this.security = security;
public HttpClientFactory(SSLContext sslContext, SslClientProperties sslClientProperties) {
this.sslContext = sslContext;
this.sslClientProperties = sslClientProperties;
}
@@ -26,8 +24,8 @@ public class HttpClientFactory implements KeyedPooledObjectFactory<TimelyBalance
@Override
public PooledObject<CloseableHttpClient> makeObject(TimelyBalancedHost k) throws Exception {
// disable cookie management because we are sharing connections
return new DefaultPooledObject<>(timely.client.http.HttpClient.get(this.sslContext, null, sslClientProperties.isHostVerificationEnabled(),
sslClientProperties.isUseClientCert()));
return new DefaultPooledObject<>(
HttpClient.get(this.sslContext, null, sslClientProperties.isHostVerificationEnabled(), sslClientProperties.isUseClientCert()));
}
@Override
@@ -7,12 +7,11 @@ import org.apache.http.impl.client.CloseableHttpClient;
import timely.balancer.configuration.BalancerHttpProperties;
import timely.balancer.connection.TimelyBalancedHost;
import timely.common.configuration.SecurityProperties;
import timely.common.configuration.SslClientProperties;
public class HttpClientPool extends GenericKeyedObjectPool<TimelyBalancedHost,CloseableHttpClient> {
public HttpClientPool(SecurityProperties securityProperties, SslClientProperties sslClientProperties, BalancerHttpProperties http, SSLContext sslContext) {
super(new HttpClientFactory(securityProperties, sslClientProperties, sslContext), http.getHttpClientPool());
public HttpClientPool(BalancerHttpProperties http, SSLContext sslContext, SslClientProperties sslClientProperties) {
super(new HttpClientFactory(sslContext, sslClientProperties), http.getHttpClientPool());
}
}
@@ -43,7 +43,7 @@ import timely.api.response.TimelyException;
import timely.auth.TimelyAuthenticationToken;
import timely.auth.util.HttpHeaderUtils;
import timely.auth.util.ProxiedEntityUtils;
import timely.balancer.component.MetricResolver;
import timely.balancer.MetricResolver;
import timely.balancer.connection.TimelyBalancedHost;
import timely.balancer.connection.http.HttpClientPool;
import timely.netty.http.TimelyHttpHandler;
@@ -14,7 +14,7 @@ import io.netty.channel.ChannelHandlerContext;
import io.netty.channel.SimpleChannelInboundHandler;
import timely.api.request.MetricRequest;
import timely.api.request.TcpRequest;
import timely.balancer.component.MetricResolver;
import timely.balancer.MetricResolver;
import timely.balancer.connection.TimelyBalancedHost;
import timely.balancer.connection.tcp.TcpClientPool;
import timely.client.tcp.TcpClient;
@@ -12,7 +12,7 @@ import io.netty.channel.ChannelHandlerContext;
import io.netty.channel.SimpleChannelInboundHandler;
import timely.api.request.MetricRequest;
import timely.api.request.UdpRequest;
import timely.balancer.component.MetricResolver;
import timely.balancer.MetricResolver;
import timely.balancer.connection.TimelyBalancedHost;
import timely.balancer.connection.udp.UdpClientPool;
import timely.client.udp.UdpClient;
@@ -28,7 +28,7 @@ import timely.api.request.websocket.RemoveSubscription;
import timely.api.request.websocket.SubscriptionRequest;
import timely.api.response.TimelyException;
import timely.auth.util.ProxiedEntityUtils;
import timely.balancer.component.MetricResolver;
import timely.balancer.MetricResolver;
import timely.balancer.configuration.BalancerWebsocketProperties;
import timely.balancer.connection.TimelyBalancedHost;
import timely.balancer.connection.ws.WsClientPool;
+11
View File
@@ -0,0 +1,11 @@
===============================================================================
_______ _ _ ____ _
|__ __(_) | | | _ \ | |
| | _ _ __ ___ ___| |_ _ | |_) | __ _| | __ _ _ __ ___ ___ _ __
| | | | '_ ` _ \ / _ \ | | | | | _ < / _` | |/ _` | '_ \ / __/ _ \ '__|
| | | | | | | | | __/ | |_| | | |_) | (_| | | (_| | | | | (_| __/ |
|_| |_|_| |_| |_|\___|_|\__, | |____/ \__,_|_|\__,_|_| |_|\___\___|_|
__/ |
|___/
===============================================================================
TimelyBalancer ${application.formatted-version}
+1 -1
View File
@@ -1,7 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<FindBugsFilter>
<Match>
<Class name="timely.balancer.component.Balancer" />
<Class name="timely.balancer.Balancer" />
<Method name="shutdown" />
<Bug pattern="UC_USELESS_OBJECT" />
</Match>
@@ -1,33 +0,0 @@
package timely.balancer.component;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.context.ApplicationContext;
import org.springframework.context.annotation.Primary;
import org.springframework.stereotype.Component;
import timely.balancer.configuration.BalancerHttpProperties;
import timely.balancer.configuration.BalancerProperties;
import timely.balancer.configuration.BalancerServerProperties;
import timely.balancer.configuration.BalancerWebsocketProperties;
import timely.common.component.AuthenticationService;
import timely.common.configuration.SecurityProperties;
import timely.common.configuration.SslClientProperties;
import timely.common.configuration.SslServerProperties;
import timely.common.configuration.ZookeeperProperties;
@Primary
@Component
public class TestBalancer extends Balancer {
private static final Logger log = LoggerFactory.getLogger(TestBalancer.class);
public TestBalancer(ApplicationContext applicationContext, AuthenticationService authenticationService, MetricResolver metricResolver,
BalancerProperties balancerProperties, ZookeeperProperties zookeeperProperties, SecurityProperties securityProperties,
SslServerProperties sslServerProperties, SslClientProperties sslClientProperties, BalancerServerProperties balancerServerProperties,
BalancerHttpProperties balancerHttpProperties, BalancerWebsocketProperties balancerWebsocketProperties) {
super(applicationContext, authenticationService, metricResolver, balancerProperties, zookeeperProperties, securityProperties, sslServerProperties,
sslClientProperties, balancerServerProperties, balancerHttpProperties, balancerWebsocketProperties);
DEFAULT_EVENT_LOOP_THREADS = 1;
log.info("Starting TestBalancer");
}
}
@@ -1,20 +0,0 @@
package timely.balancer.component;
import java.util.Collections;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.context.annotation.Primary;
import org.springframework.security.authentication.ProviderManager;
import org.springframework.security.web.authentication.preauth.PreAuthenticatedAuthenticationProvider;
import org.springframework.stereotype.Component;
@Primary
@Component
public class TestTimelyAuthenticationManager extends ProviderManager {
@Autowired
public TestTimelyAuthenticationManager(@Qualifier("file") PreAuthenticatedAuthenticationProvider file) {
super(Collections.singletonList(file));
}
}
@@ -0,0 +1,56 @@
package timely.balancer.configuration;
import javax.net.ssl.SSLContext;
import org.apache.curator.framework.CuratorFramework;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.context.ApplicationContext;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.Configuration;
import io.netty.handler.ssl.SslContext;
import timely.balancer.HealthChecker;
import timely.balancer.MetricResolver;
import timely.balancer.test.TestBalancer;
import timely.balancer.test.TestHealthChecker;
import timely.balancer.test.TestMetricResolver;
import timely.common.component.AuthenticationService;
import timely.common.configuration.CacheProperties;
import timely.common.configuration.SecurityProperties;
import timely.common.configuration.SslClientProperties;
import timely.common.configuration.SslServerProperties;
import timely.common.configuration.ZookeeperProperties;
@Configuration
@ComponentScan(basePackages = {"timely.balancer", "timely.common", "timely.test"})
public class TestBalancerConfiguration {
@Bean
public TestBalancer balancer(ApplicationContext applicationContext, @Qualifier("nettySslContext") SslContext serverSslContext,
@Qualifier("outboundJDKSslContext") SSLContext clientSSLContext, AuthenticationService authenticationService, MetricResolver metricResolver,
BalancerProperties balancerProperties, ZookeeperProperties zookeeperProperties, SecurityProperties securityProperties,
SslServerProperties sslServerProperties, SslClientProperties sslClientProperties, BalancerServerProperties balancerServerProperties,
BalancerHttpProperties balancerHttpProperties, BalancerWebsocketProperties balancerWebsocketProperties) {
TestBalancer balancer = new TestBalancer(applicationContext, serverSslContext, clientSSLContext, authenticationService, metricResolver,
balancerProperties, zookeeperProperties, securityProperties, sslServerProperties, sslClientProperties, balancerServerProperties,
balancerHttpProperties, balancerWebsocketProperties);
balancer.start();
return balancer;
}
@Bean
public TestHealthChecker healthChecker(BalancerProperties balancerProperties, BalancerServerProperties balancerServerProperties) {
TestHealthChecker healthChecker = new TestHealthChecker(balancerProperties, balancerServerProperties);
healthChecker.start();
return healthChecker;
}
@Bean
public TestMetricResolver metricResolver(CuratorFramework curatorFramework, BalancerProperties balancerProperties, CacheProperties cacheProperties,
HealthChecker healthChecker) {
TestMetricResolver metricResolver = new TestMetricResolver(curatorFramework, balancerProperties, cacheProperties, healthChecker);
metricResolver.start();
return metricResolver;
}
}
@@ -1,21 +0,0 @@
package timely.balancer.configuration;
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import timely.common.configuration.AccumuloProperties;
@Configuration
public class TestConfiguration {
@Bean
@ConditionalOnMissingBean
public AccumuloProperties accumuloProperties() {
AccumuloProperties accumuloProperties = new AccumuloProperties();
accumuloProperties.setInstanceName("default");
accumuloProperties.setUsername("root");
accumuloProperties.setPassword("secret");
return accumuloProperties;
}
}
@@ -1,49 +0,0 @@
package timely.balancer.configuration;
import static org.apache.accumulo.core.conf.ConfigurationTypeHelper.getTimeInMillis;
import org.apache.curator.RetryPolicy;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.CuratorFrameworkFactory;
import org.apache.curator.retry.RetryForever;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.DependsOn;
import timely.common.configuration.ZookeeperProperties;
/*
* The purpose of this class is to wait for the creation of either ZooKeeperServerMain (in-memory)
* or MiniAccumuloCluster (mini accumulo) before creating a CuratorFramework so that the components
* that need CuratorFramework autowired will wait.
*/
@Configuration
public class TestCuratorConfiguration {
private static final Logger log = LoggerFactory.getLogger(TestCuratorConfiguration.class);
@Bean(destroyMethod = "close")
@ConditionalOnProperty(name = "timely.in-memory-accumulo.enabled", havingValue = "true")
@DependsOn("ZooKeeperServerMain")
public CuratorFramework inMemoryCuratorFramework(ZookeeperProperties zookeeperProperties) {
RetryPolicy retryPolicy = new RetryForever(1000);
int timeout = Long.valueOf(getTimeInMillis(zookeeperProperties.getTimeout())).intValue();
CuratorFramework curatorFramework = CuratorFrameworkFactory.newClient(zookeeperProperties.getServers(), timeout, 10000, retryPolicy);
curatorFramework.start();
return curatorFramework;
}
@Bean(destroyMethod = "close")
@ConditionalOnProperty(name = "timely.mini-accumulo.enabled", havingValue = "true")
@DependsOn("MiniAccumuloCluster")
public CuratorFramework miniAccumuloCuratorFramework(ZookeeperProperties zookeeperProperties) {
RetryPolicy retryPolicy = new RetryForever(1000);
int timeout = Long.valueOf(getTimeInMillis(zookeeperProperties.getTimeout())).intValue();
CuratorFramework curatorFramework = CuratorFrameworkFactory.newClient(zookeeperProperties.getServers(), timeout, 10000, retryPolicy);
curatorFramework.start();
return curatorFramework;
}
}
@@ -1,23 +0,0 @@
package timely.balancer.configuration;
import org.apache.accumulo.core.client.AccumuloClient;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Scope;
import timely.balancer.test.BalancerTestRule;
import timely.common.configuration.MiniAccumuloProperties;
import timely.common.configuration.TimelyProperties;
@Configuration
@EnableConfigurationProperties({MiniAccumuloProperties.class, TimelyProperties.class})
public class TestWatcherConfiguration {
@Bean
@Scope("prototype")
public BalancerTestRule timelyTestWatcherAccumulo(AccumuloClient accumuloClient, TimelyProperties timelyProperties) throws Exception {
return new BalancerTestRule(timelyProperties, accumuloClient);
}
}
@@ -1,4 +1,4 @@
package timely.balancer.test;
package timely.balancer.integration;
import static timely.Constants.SERVICE_DISCOVERY_PATH;
@@ -27,10 +27,11 @@ import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.junit4.SpringRunner;
import timely.ServerDetails;
import timely.balancer.component.TestHealthChecker;
import timely.balancer.component.TestMetricResolver;
import timely.balancer.connection.TimelyBalancedHost;
import timely.balancer.test.TestHealthChecker;
import timely.balancer.test.TestMetricResolver;
import timely.test.IntegrationTest;
import timely.test.TimelyTestRule;
@Category(IntegrationTest.class)
@RunWith(SpringRunner.class)
@@ -41,7 +42,7 @@ public class MetricResolverIT {
@Autowired
@Rule
public BalancerTestRule balancerTestRule;
public TimelyTestRule testRule;
@Autowired
private TestMetricResolver metricResolver;
@@ -1,61 +0,0 @@
package timely.balancer.test;
import org.apache.accumulo.core.client.AccumuloClient;
import org.junit.rules.TestWatcher;
import org.junit.runner.Description;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import timely.common.configuration.TimelyProperties;
import timely.util.TestUtils;
public class BalancerTestRule extends TestWatcher {
private static final Logger log = LoggerFactory.getLogger(BalancerTestRule.class);
private TimelyProperties timelyProperties;
private AccumuloClient accumuloClient;
public BalancerTestRule(TimelyProperties timelyProperties, AccumuloClient accumuloClient) {
this.timelyProperties = timelyProperties;
this.accumuloClient = accumuloClient;
}
@Override
protected void failed(Throwable e, Description description) {
log.error("Test {}:{} failed", description.getTestClass().getCanonicalName(), description.getMethodName());
log.error(e.getMessage(), e);
if (accumuloClient != null) {
if (accumuloClient.tableOperations().exists(timelyProperties.getMetricsTable())) {
separator();
log.info("Contents of {}", timelyProperties.getMetricsTable());
separator();
TestUtils.printTable(accumuloClient, timelyProperties.getMetricsTable(), true);
separator();
}
if (accumuloClient.tableOperations().exists(timelyProperties.getMetaTable())) {
log.info("Contents of {}", timelyProperties.getMetaTable());
separator();
TestUtils.printTable(accumuloClient, timelyProperties.getMetaTable(), false);
separator();
}
}
}
private void separator() {
log.info("-----------------------------------------------------------------------------------");
}
@Override
protected void starting(Description description) {
separator();
log.info("Starting test {}:{}", description.getTestClass().getCanonicalName(), description.getMethodName());
separator();
}
@Override
protected void finished(Description description) {
separator();
log.info("Finished test {}:{}", description.getTestClass().getCanonicalName(), description.getMethodName());
separator();
}
}
@@ -0,0 +1,36 @@
package timely.balancer.test;
import javax.net.ssl.SSLContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.context.ApplicationContext;
import io.netty.handler.ssl.SslContext;
import timely.balancer.Balancer;
import timely.balancer.MetricResolver;
import timely.balancer.configuration.BalancerHttpProperties;
import timely.balancer.configuration.BalancerProperties;
import timely.balancer.configuration.BalancerServerProperties;
import timely.balancer.configuration.BalancerWebsocketProperties;
import timely.common.component.AuthenticationService;
import timely.common.configuration.SecurityProperties;
import timely.common.configuration.SslClientProperties;
import timely.common.configuration.SslServerProperties;
import timely.common.configuration.ZookeeperProperties;
public class TestBalancer extends Balancer {
private static final Logger log = LoggerFactory.getLogger(TestBalancer.class);
public TestBalancer(ApplicationContext applicationContext, SslContext serverSslContext, SSLContext clientSSLContext,
AuthenticationService authenticationService, MetricResolver metricResolver, BalancerProperties balancerProperties,
ZookeeperProperties zookeeperProperties, SecurityProperties securityProperties, SslServerProperties sslServerProperties,
SslClientProperties sslClientProperties, BalancerServerProperties balancerServerProperties, BalancerHttpProperties balancerHttpProperties,
BalancerWebsocketProperties balancerWebsocketProperties) {
super(applicationContext, serverSslContext, clientSSLContext, authenticationService, metricResolver, balancerProperties, zookeeperProperties,
securityProperties, sslServerProperties, sslClientProperties, balancerServerProperties, balancerHttpProperties,
balancerWebsocketProperties);
DEFAULT_EVENT_LOOP_THREADS = 1;
log.info("Starting TestBalancer");
}
}
@@ -1,32 +1,18 @@
package timely.balancer.component;
package timely.balancer.test;
import java.util.List;
import org.springframework.context.annotation.Primary;
import org.springframework.stereotype.Component;
import timely.balancer.HealthChecker;
import timely.balancer.configuration.BalancerProperties;
import timely.balancer.configuration.BalancerServerProperties;
import timely.balancer.connection.TimelyBalancedHost;
@Primary
@Component
public class TestHealthChecker extends HealthChecker {
public TestHealthChecker(BalancerProperties balancerProperties, BalancerServerProperties balancerServerProperties) {
super(balancerProperties, balancerServerProperties);
}
@Override
public void setup() {
}
@Override
public void close() {
}
public List<TimelyBalancedHost> getTimelyHosts() {
return this.timelyHosts;
}
@@ -1,4 +1,4 @@
package timely.balancer.component;
package timely.balancer.test;
import java.util.List;
@@ -8,15 +8,13 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.context.annotation.Primary;
import org.springframework.stereotype.Component;
import timely.balancer.HealthChecker;
import timely.balancer.MetricResolver;
import timely.balancer.configuration.BalancerProperties;
import timely.balancer.connection.TimelyBalancedHost;
import timely.common.configuration.CacheProperties;
@Primary
@Component
public class TestMetricResolver extends MetricResolver {
private static final Logger log = LoggerFactory.getLogger(TestMetricResolver.class);
@@ -1,48 +0,0 @@
package timely.balancer.test;
public class TestServerInfo {
public String host;
public String ip;
public Integer tcpPort;
public Integer httpPort;
public Integer wsPort;
public Integer udpPort;
public TestServerInfo(String host, String ip, Integer tcpPort, Integer httpPort, Integer wsPort, Integer udpPort) {
this.host = host;
this.ip = ip;
this.tcpPort = tcpPort;
this.httpPort = httpPort;
this.wsPort = wsPort;
this.udpPort = udpPort;
}
static public TestServerInfo of(String host, String ip, Integer tcpPort, Integer httpPort, Integer wsPort, Integer udpPort) {
return new TestServerInfo(host, ip, tcpPort, httpPort, wsPort, udpPort);
}
public String getHost() {
return host;
}
public String getIp() {
return ip;
}
public Integer getTcpPort() {
return tcpPort;
}
public Integer getHttpPort() {
return httpPort;
}
public Integer getWsPort() {
return wsPort;
}
public Integer getUdpPort() {
return udpPort;
}
}
@@ -1,4 +1,4 @@
package timely.common.component;
package timely.balancer.test;
import java.util.Collections;
@@ -9,8 +9,8 @@ import org.springframework.security.authentication.ProviderManager;
import org.springframework.security.web.authentication.preauth.PreAuthenticatedAuthenticationProvider;
import org.springframework.stereotype.Component;
@Component
@Primary
@Component
public class TestTimelyAuthenticationManager extends ProviderManager {
@Autowired
@@ -6,6 +6,11 @@ logging:
timely:
test: true
instance: 1
accumulo:
instance-name: default
username: root
password: secret
mini-accumulo:
enabled: true
rootAuths: A,B,C,D,E,F,G,H,I
@@ -18,74 +23,87 @@ timely:
cache-refresh-minutes: 5
allow-anonymous-http-access: true
allow-anonymous-ws-access: true
authorizedUsers:
- subjectDn: cn=example.com
authorized-users:
- subject-dn: cn=example.com
roles: G,H,I
auths: D,E,F
userType: USER
- subjectDn: cn=test a. user, ou=example developers, o=example corp, c=us
issuerDn: cn=example corp ca, o=example corp, c=us
user-type: USER
- subject-dn: cn=test a. user, ou=example developers, o=example corp, c=us
issuer-dn: cn=example corp ca, o=example corp, c=us
roles: G,H,I
auths: A,B,C
userType: USER
serverSsl:
useGeneratedKeypair: true
ignoreSslHandshakeErrors: true
useOpenssl: true
user-type: USER
server-ssl:
key-store-type: PKCS12
key-store-file: classpath:ssl/testServer.p12
key-store-password: ChangeIt
trust-store-type: JKS
trust-store-file: classpath:ssl/ca.jks
trust-store-password: ChangeIt
use-generated-keypair: false
ignore-ssl-handshake-errors: true
use-openssl: true
client-ssl:
key-store-type: PKCS12
key-store-file: classpath:ssl/testUser.p12
key-store-password: ChangeIt
trust-store-type: JKS
trust-store-file: classpath:ssl/ca.jks
trust-store-password: ChangeIt
cache:
enabled: false
metric-age-off-hours:
default: 26
nonCachedMetrics:
non-cached-metrics:
balancer:
checkServerHealthInterval: 10000
serverFailuresBeforeDown: 6
serverSuccessesBeforeUp: 1
assignmentFile: file://${java.io.tmpdir}/timely/balancer/assignments.csv
check-server-health-interval: 10000
server-failures-before-down: 6
server-successes-before-up: 1
assignment-file: file://${java.io.tmpdir}/timely/balancer/assignments.csv
server:
ip: 127.0.0.1
tcpPort: 4242
numTcpPools: 1
tcpBufferSize: 100
tcpClientPool:
minIdlePerKey: 0
maxIdlePerKey: 40
maxTotalPerKey: 40
maxTotal: -1
tcp-base-port: 4242
num-tcp-pools: 1
tcp-buffer-size: 100
tcp-client-pool:
min-idle-per-key: 0
max-idle-per-key: 40
max-total-per-key: 40
max-total: -1
lifo: true
fairness: false
maxWaitMillis: -1
minEvictableIdleTimeMillis: 1800000
evictorShutdownTimeoutMillis: 10000
softMinEvictableIdleTimeMillis: -1
numTestsPerEvictionRun: 3
timeBetweenEvictionRunsMillis: -1
blockWhenExhausted: true
jmxEnabled: true
testOnBorrow: false
testOnReturn: false
udpPort: 4245
udpClientPool:
minIdlePerKey: 0
maxIdlePerKey: 40
maxTotalPerKey: 40
maxTotal: -1
max-wait-millis: -1
min-evictable-idle-time-millis: 1800000
evictor-shutdown-timeout-millis: 10000
soft-min-evictable-idle-time-millis: -1
num-tests-per-eviction-run: 3
time-between-eviction-runs-millis: -1
block-when-exhausted: true
jmx-enabled: true
test-on-borrow: false
test-on-return: false
udp-base-port: 4245
udp-client-pool:
min-idle-per-key: 0
max-idle-per-key: 40
max-total-per-key: 40
max-total: -1
lifo: true
fairness: false
maxWaitMillis: -1
minEvictableIdleTimeMillis: 1800000
evictorShutdownTimeoutMillis: 10000
softMinEvictableIdleTimeMillis: -1
numTestsPerEvictionRun: 3
timeBetweenEvictionRunsMillis: -1
blockWhenExhausted: true
jmxEnabled: true
testOnBorrow: false
testOnReturn: false
max-wait-millis: -1
min-evictable-idle-time-millis: 1800000
evictor-shutdown-timeout-millis: 10000
soft-min-evictable-idle-time-millis: -1
num-tests-per-eviction-run: 3
time-between-eviction-runs-millis: -1
block-when-exhausted: true
jmx-enabled: true
test-on-borrow: false
test-on-return: false
http:
host: localhost
ip: 127.0.0.1
port: 4243
base-port: 4243
redirect-path: /secure-me
strict-transport-max-age: 604800
cors:
@@ -95,47 +113,47 @@ timely:
allowed-origins:
allowed-methods: DELETE,GET,HEAD,OPTIONS,PUT,POST
allowed-headers: content-type
httpClientPool:
minIdlePerKey: 0
maxIdlePerKey: 40
maxTotalPerKey: 40
maxTotal: -1
http-client-pool:
min-idle-per-key: 0
max-idle-per-key: 40
max-total-per-key: 40
max-total: -1
lifo: true
fairness: false
maxWaitMillis: -1
minEvictableIdleTimeMillis: 1800000
evictorShutdownTimeoutMillis: 10000
softMinEvictableIdleTimeMillis: -1
numTestsPerEvictionRun: 3
timeBetweenEvictionRunsMillis: -1
blockWhenExhausted: true
jmxEnabled: true
testOnBorrow: false
testOnReturn: false
max-wait-millis: -1
min-evictable-idle-time-millis: 1800000
evictor-shutdown-timeout-millis: 10000
soft-min-evictable-idle-time-millis: -1
num-tests-per-eviction-run: 3
time-between-eviction-runs-millis: -1
block-when-exhausted: true
jmx-enabled: true
test-on-borrow: false
test-on-return: false
websocket:
ip: 127.0.0.1
port: 5244
base-port: 5244
timeout: 60
subscription-lag: 120
scanner-batch-size: 10000
flush-interval-seconds: 30
scanner-read-ahead: 1
subscription-batch-size: 5000
incomingBufferSize: 50000000
wsClientPool:
minIdlePerKey: 0
maxIdlePerKey: 40
maxTotalPerKey: 40
maxTotal: -1
incoming-buffer-size: 50000000
ws-client-pool:
min-idle-per-key: 0
max-idle-per-key: 40
max-total-per-key: 40
max-total: -1
lifo: true
fairness: false
maxWaitMillis: -1
minEvictableIdleTimeMillis: 1800000
evictorShutdownTimeoutMillis: 10000
softMinEvictableIdleTimeMillis: -1
numTestsPerEvictionRun: 3
timeBetweenEvictionRunsMillis: -1
blockWhenExhausted: true
jmxEnabled: true
testOnBorrow: false
testOnReturn: false
max-wait-millis: -1
min-evictable-idle-time-millis: 1800000
evictor-shutdown-timeout-millis: 10000
soft-min-evictable-idle-time-millis: -1
num-tests-per-eviction-run: 3
time-between-eviction-runs-millis: -1
block-when-exhausted: true
jmx-enabled: true
test-on-borrow: false
test-on-return: false
+23
View File
@@ -0,0 +1,23 @@
Bag Attributes: <No Attributes>
subject=/C=US/O=Example Corp/CN=EXAMPLE CORP CA
issuer=/C=US/O=Example Corp/CN=EXAMPLE CORP CA
-----BEGIN CERTIFICATE-----
MIIDTjCCAjagAwIBAgIURUj8l+ms/i9jeabIBBi19KxEhAgwDQYJKoZIhvcNAQEL
BQAwPjELMAkGA1UEBhMCVVMxFTATBgNVBAoTDEV4YW1wbGUgQ29ycDEYMBYGA1UE
AxMPRVhBTVBMRSBDT1JQIENBMCAXDTIzMDIwMTIzMTUwMFoYDzIxMjMwMTA4MjMx
NTAwWjA+MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRXhhbXBsZSBDb3JwMRgwFgYD
VQQDEw9FWEFNUExFIENPUlAgQ0EwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK
AoIBAQDVBr2XF20SKmGXfvp2BUY44BF1fm3Qw83GkC66qIHgpgQxzJ8u44P34vV1
DYTj4XzrBpx23J0nCMEiAW8YW9MxZJPEokZafRNKFQAngVdgmAW9fHemHRQfaYwe
ySVz2UZ+z28DJ3C0QshvqXskrYstwLVs4Ss2uoX+XpeFbW9IVVg4aW+GX3LHPNTk
7Hji13Capp6BCiS61AGQAiMWNBW2aRgHVLOb7RDQug2Bka/dcJ44XxJDh26C6kV1
pNEZm0ToqWAiVfvyzionPec5im6NKX9kStZc+C/iikfx5vdmrvGIF5zxy6phdhJ7
AGd6cF6UBkkYm/SsIP3WTXpBek5LAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAP
BgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBSPLNlCmmp9PH+crzCx9TNp5BoCajAN
BgkqhkiG9w0BAQsFAAOCAQEAkjLnZKqLMjbu5Hy8YOVoKaEKahP+t/tNYZW/NtrC
xSxKmGLz5kVCkjpLgCykkcpu4h2wZW4MGZJ4nc87+tN22mJOg1WnqEpBzRsEPajk
7duVjdU2FDCarERj53hxF41l/u8BxvMGbFnrAijv5SzspeLi+VCGF3wu53Bi6UHi
7WO1dIhZLMHepqgtwbDDudxwlKbG/JWjJ7OspCNqOpktpY8BFUAQ42ctYZcOHy2w
kdZDcK5IhRztkBH2aSWOhudYh/CF7BOu2nSUdtR5NwhXnsStmSbud7ahw3W5Jc9/
IVAFXf2brkrfIzwrY5kmjnEPJ8mjUyLepwSlOlMTBrehzQ==
-----END CERTIFICATE-----
Binary file not shown.
@@ -0,0 +1,51 @@
Bag Attributes
localKeyID: 22 89 1C EA 4E FF 9A F6 11 1B E7 9B 5E 4F 59 B7 B1 0F AA FA
friendlyName: example-server
subject=/C=US/O=Example Corp/OU=Servers/CN=testserver.example.com
issuer=/C=US/O=Example Corp/CN=EXAMPLE CORP CA
-----BEGIN CERTIFICATE-----
MIIDwjCCAqqgAwIBAgIUe6yBgkY1m/oXuPXA9j/c/7+w16UwDQYJKoZIhvcNAQEL
BQAwPjELMAkGA1UEBhMCVVMxFTATBgNVBAoTDEV4YW1wbGUgQ29ycDEYMBYGA1UE
AxMPRVhBTVBMRSBDT1JQIENBMCAXDTIzMDIwMTIzMTUwMFoYDzIxMjMwMTA4MjMx
NTAwWjBXMQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRXhhbXBsZSBDb3JwMRAwDgYD
VQQLEwdTZXJ2ZXJzMR8wHQYDVQQDExZ0ZXN0c2VydmVyLmV4YW1wbGUuY29tMIIB
IjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAmb1pEWg9a6ZChQULazK75ppc
XScU5Ap29L70/5zcaOcbaBwqKhpKhdZCoZ1TD+Sdkk9kp7IlabCQjM2vVjknb6NY
msl3PbQFNva0EkH6s5cvpX4WhQDsWK77/4eztHT+0fpaG1YoKQn+Gy3VuW0CDyR6
GzGKifnI/aNLevgUBebbiiTPLEdGlo0jp3SXlZfoDZdbvnhBWI8Xc+aDekNlrXxW
X2TVnc02Fi914PECYJzkMpmxnBRPAcljU24WSqaUVgRoSbafkm4OXDw0xtvxnIrJ
s3iO0uZfRf7+OxAlqlvh4ZJ9/PufPMEKvmr6zea05KkFjBotKMip5Md43H3lQQID
AQABo4GcMIGZMA4GA1UdDwEB/wQEAwIFoDAdBgNVHSUEFjAUBggrBgEFBQcDAQYI
KwYBBQUHAwIwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQUDWwMrl9/uc1wplERWxas
bDROB4MwHwYDVR0jBBgwFoAUjyzZQppqfTx/nK8wsfUzaeQaAmowGgYDVR0RBBMw
EYIJbG9jYWxob3N0hwR/AAABMA0GCSqGSIb3DQEBCwUAA4IBAQBfG7StgQf3WNk/
EnaYuurwvaOOpcOV5NDcZ6tmgaXUrISPVDpyIWfsDGetNEXc+4gpefLQtGF2wuiW
odiLc4H9JwQ/yEkVYwzrFViPdZKVvxNU2EjiwO/e/3sl+SxxkTDy6o838uTXX+X5
Q8DabmL5L8ukcnqUHVOqW38oA1vRNBKiJX58JzOr8ezX8crZmqU9h8cNwG/V+ZXE
5U7+hSRzLm7gRwtixj8ZsTDUiGJxOdbyUKImPClt1Se9TizB2Q/xOPTdmYB1qiiB
+XVKkGwQPgIbHckLsZWhBor2u5cgdrBGW09Q+H88maagwq5sBHNMs33vkkgbUDpB
6WSEBJMi
-----END CERTIFICATE-----
Bag Attributes: <No Attributes>
subject=/C=US/O=Example Corp/CN=EXAMPLE CORP CA
issuer=/C=US/O=Example Corp/CN=EXAMPLE CORP CA
-----BEGIN CERTIFICATE-----
MIIDTjCCAjagAwIBAgIURUj8l+ms/i9jeabIBBi19KxEhAgwDQYJKoZIhvcNAQEL
BQAwPjELMAkGA1UEBhMCVVMxFTATBgNVBAoTDEV4YW1wbGUgQ29ycDEYMBYGA1UE
AxMPRVhBTVBMRSBDT1JQIENBMCAXDTIzMDIwMTIzMTUwMFoYDzIxMjMwMTA4MjMx
NTAwWjA+MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRXhhbXBsZSBDb3JwMRgwFgYD
VQQDEw9FWEFNUExFIENPUlAgQ0EwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK
AoIBAQDVBr2XF20SKmGXfvp2BUY44BF1fm3Qw83GkC66qIHgpgQxzJ8u44P34vV1
DYTj4XzrBpx23J0nCMEiAW8YW9MxZJPEokZafRNKFQAngVdgmAW9fHemHRQfaYwe
ySVz2UZ+z28DJ3C0QshvqXskrYstwLVs4Ss2uoX+XpeFbW9IVVg4aW+GX3LHPNTk
7Hji13Capp6BCiS61AGQAiMWNBW2aRgHVLOb7RDQug2Bka/dcJ44XxJDh26C6kV1
pNEZm0ToqWAiVfvyzionPec5im6NKX9kStZc+C/iikfx5vdmrvGIF5zxy6phdhJ7
AGd6cF6UBkkYm/SsIP3WTXpBek5LAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAP
BgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBSPLNlCmmp9PH+crzCx9TNp5BoCajAN
BgkqhkiG9w0BAQsFAAOCAQEAkjLnZKqLMjbu5Hy8YOVoKaEKahP+t/tNYZW/NtrC
xSxKmGLz5kVCkjpLgCykkcpu4h2wZW4MGZJ4nc87+tN22mJOg1WnqEpBzRsEPajk
7duVjdU2FDCarERj53hxF41l/u8BxvMGbFnrAijv5SzspeLi+VCGF3wu53Bi6UHi
7WO1dIhZLMHepqgtwbDDudxwlKbG/JWjJ7OspCNqOpktpY8BFUAQ42ctYZcOHy2w
kdZDcK5IhRztkBH2aSWOhudYh/CF7BOu2nSUdtR5NwhXnsStmSbud7ahw3W5Jc9/
IVAFXf2brkrfIzwrY5kmjnEPJ8mjUyLepwSlOlMTBrehzQ==
-----END CERTIFICATE-----
@@ -0,0 +1,32 @@
Bag Attributes
localKeyID: 22 89 1C EA 4E FF 9A F6 11 1B E7 9B 5E 4F 59 B7 B1 0F AA FA
friendlyName: example-server
Key Attributes: <No Attributes>
-----BEGIN PRIVATE KEY-----
MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCZvWkRaD1rpkKF
BQtrMrvmmlxdJxTkCnb0vvT/nNxo5xtoHCoqGkqF1kKhnVMP5J2ST2SnsiVpsJCM
za9WOSdvo1iayXc9tAU29rQSQfqzly+lfhaFAOxYrvv/h7O0dP7R+lobVigpCf4b
LdW5bQIPJHobMYqJ+cj9o0t6+BQF5tuKJM8sR0aWjSOndJeVl+gNl1u+eEFYjxdz
5oN6Q2WtfFZfZNWdzTYWL3Xg8QJgnOQymbGcFE8ByWNTbhZKppRWBGhJtp+Sbg5c
PDTG2/GcismzeI7S5l9F/v47ECWqW+Hhkn38+588wQq+avrN5rTkqQWMGi0oyKnk
x3jcfeVBAgMBAAECggEAZEHC7NkfhL4NjxV1Ftkk9AMhzJWZ6ARzArQsrSysB7T+
iGwliwNOKe1J0n74FdsubqVbrIheIYhFG61edEodT44kT/ylyBVHeUp0r5tbVBZ5
ueB56w5pEoQ6Ta8/H6VUL7alYEJVOj4L+clwINXXi+xrvKQYBlZgHheJN43j8NxS
9lCrVcm58rzX3i29S7ErBf7RqDabgD6j2m7rehnSq5Ps2wxwV/LXMl4Lxlwa0jz/
LV8szNMZViPOWAqOCy7NZbwCCaCfKVUAwmNGBkE3p4JVPeLNLzCBY9G0rHVmFJLK
yPnCB7PtCZeFTsU+8OHHWoxMFsaOO9Pp8FtB/r8PYQKBgQDI0OzEo3t/QKvbwAdz
9Eze0LQvstzZIidTWbYMTxH0g7DIt+3p11d61GKtfeuYUTPlLodufdv/BaP8G9sI
NzxisB3uH/787M9fwi1OzO7zMMZZCVa7Q1vZk6ta1qY3PekcYMPqjBJhxP1X3S7x
8+mrW5vTq/l8ZRVFeLvEkh6bpQKBgQDD/MCRJoq6nzd2DoTyT9m5qO+Jlztmu9+T
vqVOg7q0NO7U7dzyMbWgMvMFoEBidnyllU1rEj2YOo4OrI2v/JmSPUVS8VXsBkkG
UqInV4F/g0Wp3hzrkZFMqHLa6AydugmRNkOpgurRKOiwAL0Pgvk8+bOIbKMM7JBE
tZ6Zvi0gbQKBgD4S5r0505jYQsdgqm0bXQ+a77e20DGGLyXCm1Ves6wjcOf6gt4/
pFjhgccjH0or/6CeK1rqKLDB65ioe6IkgNP4oKfU2BsSkF+HRD4RduNRdl5bVqX4
AlJVJSilJSf6SS5ucuDzf1MdczpPso0z2k0PujFJ/YqILHUXBuOST0fJAoGAZAdX
nX3HZpKmsumcLzRUEq9HerkmRzvyU60FH/qZyyoND75Le42lfpsicCenVNnWqoh2
WASaTVztbk3hjwXkt7/v5vJoASwAD2sTefLulE9Cuf00U4MbXr3GyTWQYtw1z9ib
ViQMhy5S/KmT1Rvq65UibanM2mbPB1KPHTb0P00CgYEAgieJ1YXYYHPjABWJ7/S9
3Z/8eJY0kuUe1DRFaKOlfej8SOCFFIa90IW2mAnujh55H4N2lww4s/N+FIRQoQpA
1Hrch9tYRiXyouQc67dFlLxmCOVJIf31ZUt7I4HcPhsfr4fuCJXUPBs1jaDmwQsk
90VWrXaTwSjmP+yHMiPfaA0=
-----END PRIVATE KEY-----
Binary file not shown.
@@ -0,0 +1,50 @@
Bag Attributes
localKeyID: 57 F6 9F DF B2 94 61 47 B0 B0 2E 3A F7 79 98 C8 15 5C 64 FA
friendlyName: example-client
subject=/C=US/O=Example Corp/OU=Example Developers/CN=Test A. User
issuer=/C=US/O=Example Corp/CN=EXAMPLE CORP CA
-----BEGIN CERTIFICATE-----
MIIDmzCCAoOgAwIBAgIUFp4IuIaZ2XId7UNvzP6ZB9hChyUwDQYJKoZIhvcNAQEL
BQAwPjELMAkGA1UEBhMCVVMxFTATBgNVBAoTDEV4YW1wbGUgQ29ycDEYMBYGA1UE
AxMPRVhBTVBMRSBDT1JQIENBMCAXDTIzMDIwMTIzMTUwMFoYDzIxMjMwMTA4MjMx
NTAwWjBYMQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRXhhbXBsZSBDb3JwMRswGQYD
VQQLExJFeGFtcGxlIERldmVsb3BlcnMxFTATBgNVBAMTDFRlc3QgQS4gVXNlcjCC
ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMOMeN0ElMDxYKST2VD1wP6Z
jmy543Zv/QgRLTYizg3U9t3uZU0C3HqUJnK+c4+OJnRIyHuMXuJWEYVdairyAdYW
ZDF5sfiL9ikG9rZ2WxR2z7nsYLJcDGwKAt/Duzuoed6KSccEpiQc+BkoOTs9p87I
l4s7saX/anESMUY7YOomLSqS2zvQZh/zTjIdP1/9kG9PLzXPClJELuiC3pJCmr7Y
xdJIXecu6YKFpApy0y+rKcfYUVPML15gyRu9pIqH3U39y7mCkn+YP9Qc9GcZAoqR
iYdUEl1rFAgV/a3o0pmf1tFFRDOsoPSMUGNNQlaCjv1uZIbRm25nvvQlPfstFDEC
AwEAAaN1MHMwDgYDVR0PAQH/BAQDAgWgMBMGA1UdJQQMMAoGCCsGAQUFBwMCMAwG
A1UdEwEB/wQCMAAwHQYDVR0OBBYEFPAe3n59DcNwRcnpIwQY2kPawbCVMB8GA1Ud
IwQYMBaAFI8s2UKaan08f5yvMLH1M2nkGgJqMA0GCSqGSIb3DQEBCwUAA4IBAQBo
+zadU9DwR1rnHTBZF6CJXmiIdrQmArojH4a3NOHbNjFjRXm7dkd/tAy5LtgDJkww
yg+ZrOyOKHbFgk6IXRZ8nw2b1Ck9I9MDW1vxfjid9YLW2nhDaMcIeTXz3aIBO5i6
gMCV+xMcFaU/O57Ohia+ADUqrqu14nG9Moi171UWPbMr5N3gYn3Fs4VrTGYs4i0Z
5x2itm88WB3bWj+cd06zKQopU9b0Y76mQv9h+w4EHxq4uWA7GmIDDkvtI61MQo8+
twdX159Ms6o4lW43SRqVBHeMgSJbiLDkdkypYJvd9PvAjWgmQK/sD1SYKxfgYhtx
hqIvAjlGeDwdf+55EGjf
-----END CERTIFICATE-----
Bag Attributes: <No Attributes>
subject=/C=US/O=Example Corp/CN=EXAMPLE CORP CA
issuer=/C=US/O=Example Corp/CN=EXAMPLE CORP CA
-----BEGIN CERTIFICATE-----
MIIDTjCCAjagAwIBAgIURUj8l+ms/i9jeabIBBi19KxEhAgwDQYJKoZIhvcNAQEL
BQAwPjELMAkGA1UEBhMCVVMxFTATBgNVBAoTDEV4YW1wbGUgQ29ycDEYMBYGA1UE
AxMPRVhBTVBMRSBDT1JQIENBMCAXDTIzMDIwMTIzMTUwMFoYDzIxMjMwMTA4MjMx
NTAwWjA+MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRXhhbXBsZSBDb3JwMRgwFgYD
VQQDEw9FWEFNUExFIENPUlAgQ0EwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK
AoIBAQDVBr2XF20SKmGXfvp2BUY44BF1fm3Qw83GkC66qIHgpgQxzJ8u44P34vV1
DYTj4XzrBpx23J0nCMEiAW8YW9MxZJPEokZafRNKFQAngVdgmAW9fHemHRQfaYwe
ySVz2UZ+z28DJ3C0QshvqXskrYstwLVs4Ss2uoX+XpeFbW9IVVg4aW+GX3LHPNTk
7Hji13Capp6BCiS61AGQAiMWNBW2aRgHVLOb7RDQug2Bka/dcJ44XxJDh26C6kV1
pNEZm0ToqWAiVfvyzionPec5im6NKX9kStZc+C/iikfx5vdmrvGIF5zxy6phdhJ7
AGd6cF6UBkkYm/SsIP3WTXpBek5LAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAP
BgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBSPLNlCmmp9PH+crzCx9TNp5BoCajAN
BgkqhkiG9w0BAQsFAAOCAQEAkjLnZKqLMjbu5Hy8YOVoKaEKahP+t/tNYZW/NtrC
xSxKmGLz5kVCkjpLgCykkcpu4h2wZW4MGZJ4nc87+tN22mJOg1WnqEpBzRsEPajk
7duVjdU2FDCarERj53hxF41l/u8BxvMGbFnrAijv5SzspeLi+VCGF3wu53Bi6UHi
7WO1dIhZLMHepqgtwbDDudxwlKbG/JWjJ7OspCNqOpktpY8BFUAQ42ctYZcOHy2w
kdZDcK5IhRztkBH2aSWOhudYh/CF7BOu2nSUdtR5NwhXnsStmSbud7ahw3W5Jc9/
IVAFXf2brkrfIzwrY5kmjnEPJ8mjUyLepwSlOlMTBrehzQ==
-----END CERTIFICATE-----
@@ -0,0 +1,32 @@
Bag Attributes
localKeyID: 57 F6 9F DF B2 94 61 47 B0 B0 2E 3A F7 79 98 C8 15 5C 64 FA
friendlyName: example-client
Key Attributes: <No Attributes>
-----BEGIN PRIVATE KEY-----
MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDDjHjdBJTA8WCk
k9lQ9cD+mY5sueN2b/0IES02Is4N1Pbd7mVNAtx6lCZyvnOPjiZ0SMh7jF7iVhGF
XWoq8gHWFmQxebH4i/YpBva2dlsUds+57GCyXAxsCgLfw7s7qHneiknHBKYkHPgZ
KDk7PafOyJeLO7Gl/2pxEjFGO2DqJi0qkts70GYf804yHT9f/ZBvTy81zwpSRC7o
gt6SQpq+2MXSSF3nLumChaQKctMvqynH2FFTzC9eYMkbvaSKh91N/cu5gpJ/mD/U
HPRnGQKKkYmHVBJdaxQIFf2t6NKZn9bRRUQzrKD0jFBjTUJWgo79bmSG0ZtuZ770
JT37LRQxAgMBAAECggEBALFWsT1vLCAWRxi55Hkhhhbe12UO55O/V+969E+GHOw6
WG9IBaJjl+Oqe3vP8CkeNw15/CbfVCVa/KS/5Mpxkuy+fbrIrHfFJkZqxE8Q4uSa
IyRvk/nstG8+XQDlFuWxkmQZcXMFak6R3wWhjtWEwqwrChQA3Mi366YDY6BXtTpi
MuLJVWXqYVF2RirWX3a7eosR4MYBCJ1S8NnpI1w4oZwuew6iYiCJNkURctxnGKOe
hoop/711PEyIb/qHETBIlReY9scvZgJQZMgGe5mTGgFmAA8Bzo4e1/F2QrVIi7wy
REOSj4U+tP/QrYMSPk0cMpH5ft5+2mfh/o+D1ugm/AkCgYEA2qt3P4vlm1TMmJru
vMYc8iZjfe7UQTnD+J1EgrASBI0JP565zSZKBRIXnheumWhWS+2KR1MP8N6KAu4Q
xE/E7TKfLuE1CPzn7Rh/WZAxDqGNeQthk0qTbeBZ4/mHPKvt8V7c+N4vLjZ00uul
bKitlLpb49T+8FpSgRmk+nrBsesCgYEA5O6LmxH1+ip5NoMHoZVBhLbgJkxoXEzn
xHT7On8CjQ9B0T85NW9O38/a2gyIb5dIavcrh4dMvOKX8hdrVlal3chrXFRemyvy
VzpW7HZpMwthJb/QSim+GQ4u2Jsm+eXQAswq/yfowls9mp7K81McBgdfcdWgWuqc
Ttf3dBRS71MCgYBCPpu7ppgCfKgWuykEC0iNm0+cwROQRv21fX9Z0BlFQ6y/l0hW
e/kEnDyHakk8tD0gd2/it6kmoWRSCukNb9UKvsE2pIc6l9NzIm7CJUqXUh3Xw7jl
tDwBl2vkAp4TpUpKSyDKTBUi/OBbXYQZ7OfZHLPly4sIKXTBI5G1SCcRzQKBgE3Y
Y7ZmRBs+CBumMJkcVLwZJ10+jBoeuIzQLoF0oNxwfLP3aK+n2s02M7gDnP09Fw3w
YGfkdQ6hioNyvTgNnLcuqHsr4UOojO5mUxpzJVCLJQyeDei6Cv8gGIPlNjqxj6rz
3AcJfhdgyTDjoCVfdF8EcMsXlkB0GJSi0W0yYAUrAoGAVK5v0cbEEV+gxdQgh848
FwSQXx7i9jVOB2BUTLX2Q+gLUJibtFzLnh03sGCXtTa8sK+hLW4FCMWZa9Ev5ZjG
ylFyA96U97CNpS+2/ZW5+hvRZ/GuY/0yB8w26BeS38lI8RnmxSPi5cnexdl32NIX
5FkU9WJ4HKmi1n0304CkF+4=
-----END PRIVATE KEY-----
Binary file not shown.
+2 -2
View File
@@ -4,13 +4,13 @@
<parent>
<groupId>timely</groupId>
<artifactId>timely</artifactId>
<version>2.1.0-SNAPSHOT</version>
<version>2.1.0</version>
</parent>
<artifactId>timely-client</artifactId>
<properties>
<!-- PEP 440 compliant version gets substituted into src/main/python/template/setup.py -->
<!-- use .dev0 instead of -SNAPSHOT -->
<python.client.version>2.1.0.dev0</python.client.version>
<python.client.version>2.1.0</python.client.version>
</properties>
<dependencies>
<dependency>
+1 -1
View File
@@ -4,7 +4,7 @@
<parent>
<groupId>timely</groupId>
<artifactId>timely</artifactId>
<version>2.1.0-SNAPSHOT</version>
<version>2.1.0</version>
</parent>
<artifactId>collectd-nsq-plugin</artifactId>
<dependencies>
+1 -1
View File
@@ -4,7 +4,7 @@
<parent>
<groupId>timely</groupId>
<artifactId>timely</artifactId>
<version>2.1.0-SNAPSHOT</version>
<version>2.1.0</version>
</parent>
<artifactId>collectd-plugin-parent</artifactId>
<dependencies>
+1 -1
View File
@@ -4,7 +4,7 @@
<parent>
<groupId>timely</groupId>
<artifactId>timely</artifactId>
<version>2.1.0-SNAPSHOT</version>
<version>2.1.0</version>
</parent>
<artifactId>collectd-timely-plugin</artifactId>
<dependencies>
+5 -5
View File
@@ -4,7 +4,7 @@
<parent>
<groupId>timely</groupId>
<artifactId>timely</artifactId>
<version>2.1.0-SNAPSHOT</version>
<version>2.1.0</version>
</parent>
<artifactId>timely-common</artifactId>
<dependencies>
@@ -80,6 +80,10 @@
<groupId>io.projectreactor.netty</groupId>
<artifactId>reactor-netty</artifactId>
</dependency>
<dependency>
<groupId>javax.xml.bind</groupId>
<artifactId>jaxb-api</artifactId>
</dependency>
<dependency>
<groupId>org.apache.accumulo</groupId>
<artifactId>accumulo-core</artifactId>
@@ -94,10 +98,6 @@
<groupId>org.apache.accumulo</groupId>
<artifactId>accumulo-core</artifactId>
</dependency>
<dependency>
<groupId>org.apache.accumulo</groupId>
<artifactId>accumulo-minicluster</artifactId>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
@@ -1,4 +1,4 @@
package timely.adapter.accumulo;
package timely.accumulo;
import org.apache.accumulo.core.data.Key;
@@ -1,4 +1,4 @@
package timely.adapter.accumulo;
package timely.accumulo;
import java.nio.ByteBuffer;
import java.nio.charset.Charset;
@@ -206,4 +206,9 @@ public class MetricAdapter {
public static Pair<String,Long> decodeRowKey(Key k) {
return rowCoder.decode(k.getRow().getBytes());
}
public static void main(String[] args) {
long l = 1707859146644L;
System.out.println(roundTimestampToLastHour(l));
}
}
@@ -34,7 +34,7 @@ import timely.common.configuration.SecurityProperties;
*/
public class JWTTokenHandler {
private static final Logger logger = LoggerFactory.getLogger(JWTTokenHandler.class);
private static final Logger log = LoggerFactory.getLogger(JWTTokenHandler.class);
private static Key signatureCheckKey;
private static ObjectMapper objectMapper;
private static Collection<String> accumuloAuths = new TreeSet<>();
@@ -68,17 +68,17 @@ public class JWTTokenHandler {
Authorizations currentAccumuloAuths = accumuloClient.securityOperations().getUserAuthorizations(accumuloClient.whoami());
currentAccumuloAuths.iterator().forEachRemaining(a -> accumuloAuths.add(new String(a, Charset.forName("UTF-8"))));
} catch (Exception e) {
logger.error(e.getMessage(), e);
log.error(e.getMessage(), e);
}
}
}
public static Collection<TimelyUser> createUsersFromToken(String token, String claimName) {
logger.trace("Attempting to parse JWT {}", token);
log.trace("Attempting to parse JWT {}", token);
JwtParser jwtParser = Jwts.parser().setSigningKey(signatureCheckKey).build();
Jws<Claims> claimsJws = jwtParser.parseClaimsJws(token);
Claims claims = claimsJws.getBody();
logger.trace("Resulting claims: {}", claims);
log.trace("Resulting claims: {}", claims);
List<?> principalsClaim = claims.get(claimName, List.class);
if (principalsClaim == null || principalsClaim.isEmpty()) {
throw new IllegalArgumentException("JWT for " + claims.getSubject() + " does not contain any proxied principals.");
@@ -1,4 +1,4 @@
package timely.common.component;
package timely.auth;
import static timely.common.component.AuthenticationService.PRINCIPALS_CLAIM;
@@ -14,14 +14,8 @@ import org.springframework.security.core.AuthenticationException;
import org.springframework.security.core.userdetails.AuthenticationUserDetailsService;
import org.springframework.security.core.userdetails.UsernameNotFoundException;
import org.springframework.security.web.authentication.preauth.PreAuthenticatedAuthenticationToken;
import org.springframework.stereotype.Component;
import org.springframework.web.reactive.function.client.WebClient;
import timely.auth.JWTTokenHandler;
import timely.auth.SubjectIssuerDNPair;
import timely.auth.TimelyAuthenticationToken;
import timely.auth.TimelyPrincipal;
import timely.auth.TimelyUser;
import timely.auth.util.ProxiedEntityUtils;
import timely.common.configuration.SecurityProperties;
@@ -34,22 +28,20 @@ import timely.common.configuration.SecurityProperties;
* production, it is likely better for the JWT to be retrieved by a load balancer or other API gateway and inject the JWT header before calling a service.
* Therefore, this service is mostly useful for debugging and if there are problems with the suggested approach.
*/
@Component
public class RemoteAuthenticationProvider implements AuthenticationProvider {
private final Logger logger = LoggerFactory.getLogger(getClass());
private static final Logger log = LoggerFactory.getLogger(RemoteAuthenticationProvider.class);
private final WebClient webClient;
public static final String ENTITIES_HEADER = "X-ProxiedEntitiesChain";
public static final String ISSUERS_HEADER = "X-ProxiedIssuersChain";
public RemoteAuthenticationProvider(WebClient.Builder builder, SecurityProperties securityProperties) {
this.webClient = builder.baseUrl(securityProperties.getAuthorizationUri()).build();
this.webClient = builder.baseUrl(securityProperties.getAuthorizationUrl()).build();
}
public TimelyPrincipal fetchTimelyPrincipal(TimelyAuthenticationToken token) throws UsernameNotFoundException {
logger.debug("Authenticating {} via the authorization microservice", token);
log.debug("Authenticating {}", token.getTimelyPrincipal().getUsername());
TimelyPrincipal principal = token.getTimelyPrincipal();
try {
// @formatter:off
@@ -60,7 +52,7 @@ public class RemoteAuthenticationProvider implements AuthenticationProvider {
.bodyToMono(String.class)
.block();
// @formatter:on
logger.debug("Authentication {} on remote authentication service with {} result", principal.getUsername(), (jwt == null) ? "null" : "non-null");
log.debug("Authenticated {} with {} result", principal.getUsername(), (jwt == null) ? "null" : "non-null");
if (jwt != null) {
Collection<TimelyUser> principals = JWTTokenHandler.createUsersFromToken(jwt, PRINCIPALS_CLAIM);
long createTime = principals.stream().map(TimelyUser::getCreationTime).min(Long::compareTo).orElse(System.currentTimeMillis());
@@ -69,7 +61,7 @@ public class RemoteAuthenticationProvider implements AuthenticationProvider {
throw new UsernameNotFoundException("No entities found for " + principal.getUsername());
}
} catch (Exception e) {
logger.error("Failed performing lookup of {}: {}", principal.getUsername(), e);
log.error("Failed performing lookup of {}: {}", principal.getUsername(), e);
throw new UsernameNotFoundException(e.getMessage(), e);
}
}
@@ -0,0 +1,13 @@
package timely.auth;
import java.util.List;
import org.springframework.security.authentication.AuthenticationProvider;
import org.springframework.security.authentication.ProviderManager;
public class TimelyAuthenticationManager extends ProviderManager {
public TimelyAuthenticationManager(List<AuthenticationProvider> providers) {
super(providers);
}
}
@@ -16,7 +16,7 @@ import org.apache.accumulo.core.security.Authorizations;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.security.authentication.AuthenticationManager;
import org.springframework.security.authentication.ProviderManager;
import org.springframework.security.core.Authentication;
import org.springframework.stereotype.Component;
@@ -32,6 +32,7 @@ import timely.api.request.AuthenticatedWebSocketRequest;
import timely.api.response.TimelyException;
import timely.auth.JWTTokenHandler;
import timely.auth.SubjectIssuerDNPair;
import timely.auth.TimelyAuthenticationManager;
import timely.auth.TimelyAuthenticationToken;
import timely.auth.TimelyPrincipal;
import timely.auth.TimelyUser;
@@ -46,7 +47,7 @@ public class AuthenticationService {
public static final String AUTH_HEADER = "Authorization";
public static final String PRINCIPALS_CLAIM = "principals";
private AuthenticationManager authenticationManager;
private ProviderManager authenticationManager;
private ConcurrentHashMap<String,Object> fetchingEntity = new ConcurrentHashMap<>();
private ConcurrentHashMap<String,Object> fetchingPrincipal = new ConcurrentHashMap<>();
@@ -153,7 +154,7 @@ public class AuthenticationService {
}
verifiedPrincipal = new TimelyPrincipal(authenticatedTimelyUsers);
this.authCache.put(verifiedPrincipal.getName(), verifiedPrincipal);
log.debug("Authenticated user {} for request {} with authorizations {}", verifiedPrincipal.getName(), request.toString(),
log.debug("Authenticated user {} with authorizations {}", verifiedPrincipal.getName(),
verifiedPrincipal.getAuthorizationsString());
} else {
log.trace("Verified principal {} in AuthCache on 2nd attempt", entity);
@@ -1,36 +0,0 @@
package timely.common.component;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.boot.autoconfigure.condition.ConditionalOnBean;
import org.springframework.boot.web.reactive.function.client.WebClientCustomizer;
import org.springframework.core.annotation.Order;
import org.springframework.http.client.reactive.ReactorClientHttpConnector;
import org.springframework.stereotype.Component;
import org.springframework.web.reactive.function.client.WebClient;
import io.netty.handler.ssl.SslContext;
import reactor.netty.http.client.HttpClient;
/**
* Customizes the Spring-provided {@link org.springframework.web.reactive.function.client.WebClient.Builder} in order to supply an {@link SslContext} that will
* provide a client certificate to the remote server if one is requested.
*/
@Component
@Order(100) // execute this after standard customizers so we're sure to overwrite the client
// connector
@ConditionalOnBean(name = "outboundNettySslContext")
public class ClientCertWebClientCustomizer implements WebClientCustomizer {
private final SslContext sslContext;
public ClientCertWebClientCustomizer(@Qualifier("outboundNettySslContext") SslContext sslContext) {
this.sslContext = sslContext;
}
@Override
public void customize(WebClient.Builder webClientBuilder) {
HttpClient httpClient = HttpClient.create().secure(sslContextSpec -> sslContextSpec.sslContext(sslContext));
ReactorClientHttpConnector clientHttpConnector = new ReactorClientHttpConnector(httpClient);
webClientBuilder.clientConnector(clientHttpConnector);
}
}
@@ -1,18 +0,0 @@
package timely.common.component;
import java.util.Arrays;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.security.authentication.ProviderManager;
import org.springframework.security.web.authentication.preauth.PreAuthenticatedAuthenticationProvider;
import org.springframework.stereotype.Component;
@Component
public class TimelyAuthenticationManager extends ProviderManager {
@Autowired
public TimelyAuthenticationManager(RemoteAuthenticationProvider remoteAuthenticationProvider,
PreAuthenticatedAuthenticationProvider fileAuthenticationProvider) {
super(Arrays.asList(remoteAuthenticationProvider, fileAuthenticationProvider));
}
}
@@ -2,6 +2,7 @@ package timely.common.configuration;
import org.apache.accumulo.core.client.Accumulo;
import org.apache.accumulo.core.client.AccumuloClient;
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.annotation.Bean;
@@ -9,6 +10,7 @@ import org.springframework.context.annotation.Configuration;
@Configuration
@EnableConfigurationProperties(AccumuloProperties.class)
@ConditionalOnExpression("T(org.apache.commons.lang3.StringUtils).isNotEmpty('${timely.accumulo.instance-name:}')")
public class AccumuloConfiguration {
@Bean
@@ -15,7 +15,6 @@ public class AccumuloProperties {
private String username;
@NotBlank
private String password;
private int accumuloClientPoolSize = 16;
@Valid
@NestedConfigurationProperty
private WriteProperties writeProperties = new WriteProperties();
@@ -54,12 +53,4 @@ public class AccumuloProperties {
public void setPassword(String password) {
this.password = password;
}
public void setAccumuloClientPoolSize(int accumuloClientPoolSize) {
this.accumuloClientPoolSize = accumuloClientPoolSize;
}
public int getAccumuloClientPoolSize() {
return accumuloClientPoolSize;
}
}
@@ -1,46 +0,0 @@
package timely.common.configuration;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.cloud.context.config.annotation.RefreshScope;
@RefreshScope
@ConfigurationProperties(prefix = "timely.security.authorization.remote")
public class AuthorizationProperties {
private boolean enabled = false;
private String authorizationUrl;
private String publicCertificateFile;
private String publicCertificateType;
public boolean isEnabled() {
return enabled;
}
public void setEnabled(boolean enabled) {
this.enabled = enabled;
}
public String getAuthorizationUrl() {
return authorizationUrl;
}
public void setAuthorizationUrl(String authorizationUrl) {
this.authorizationUrl = authorizationUrl;
}
public String getPublicCertificateFile() {
return publicCertificateFile;
}
public void setPublicCertificateFile(String publicCertificateFile) {
this.publicCertificateFile = publicCertificateFile;
}
public String getPublicCertificateType() {
return publicCertificateType;
}
public void setPublicCertificateType(String publicCertificateType) {
this.publicCertificateType = publicCertificateType;
}
}
@@ -0,0 +1,31 @@
package timely.common.configuration;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.boot.autoconfigure.condition.ConditionalOnBean;
import org.springframework.boot.web.reactive.function.client.WebClientCustomizer;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.annotation.Order;
import org.springframework.http.client.reactive.ReactorClientHttpConnector;
import io.netty.handler.ssl.SslContext;
import reactor.netty.http.client.HttpClient;
/**
* Customizes the Spring-provided {@link org.springframework.web.reactive.function.client.WebClient.Builder} with an {@link SslContext} that will provide a
* client certificate to the remote server if one is requested.
*/
@Configuration
@Order(100) // execute this after standard customizers to ensure that we overwrite the client connector
@ConditionalOnBean(name = "outboundNettySslContext")
public class ClientCertWebClientCustomizer {
@Bean
public WebClientCustomizer webClientCustomizer(@Qualifier("outboundNettySslContext") SslContext sslContext) {
return webClientBuilder -> {
HttpClient httpClient = HttpClient.create().secure(sslContextSpec -> sslContextSpec.sslContext(sslContext));
ReactorClientHttpConnector clientHttpConnector = new ReactorClientHttpConnector(httpClient);
webClientBuilder.clientConnector(clientHttpConnector);
};
}
}
@@ -11,7 +11,6 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Scope;
@Configuration
@ConditionalOnExpression("!T(org.springframework.util.StringUtils).isEmpty('${timely.zookeeper.servers:}')")
@@ -19,7 +18,6 @@ import org.springframework.context.annotation.Scope;
public class CuratorConfiguration {
@Bean(destroyMethod = "close")
@Scope("singleton")
@ConditionalOnMissingBean
public CuratorFramework curatorFramework(ZookeeperProperties zookeeperProperties) {
RetryPolicy retryPolicy = new RetryForever(1000);
@@ -4,6 +4,7 @@ import javax.validation.Valid;
import javax.validation.constraints.NotBlank;
import javax.validation.constraints.NotNull;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.boot.context.properties.NestedConfigurationProperty;
import org.springframework.cloud.context.config.annotation.RefreshScope;
@@ -11,11 +12,12 @@ import org.springframework.cloud.context.config.annotation.RefreshScope;
@RefreshScope
@ConfigurationProperties(prefix = "timely.http")
public class HttpProperties {
@Autowired
TimelyProperties timelyProperties;
@NotBlank
private String ip;
@NotNull
private Integer port;
private Integer basePort;
@NotNull
private String host;
private String redirectPath = "/secure-me";
@@ -24,6 +26,10 @@ public class HttpProperties {
@NestedConfigurationProperty
private CorsProperties corsProperties = new CorsProperties();
private int getPortOffset() {
return (timelyProperties.getInstance() - 1) * timelyProperties.getPortIncrement();
}
public String getIp() {
return ip;
}
@@ -33,11 +39,15 @@ public class HttpProperties {
}
public int getPort() {
return port;
return basePort + getPortOffset();
}
public void setPort(int port) {
this.port = port;
public void setBasePort(int basePort) {
this.basePort = basePort;
}
public Integer getBasePort() {
return basePort;
}
public String getHost() {
@@ -1,25 +1,56 @@
package timely.common.configuration;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.security.web.authentication.preauth.PreAuthenticatedAuthenticationProvider;
import org.springframework.web.reactive.function.client.WebClient;
import timely.auth.FileUserDetailsService;
import timely.auth.RemoteAuthenticationProvider;
import timely.auth.SubjectIssuerDNPair;
import timely.auth.TimelyAuthenticationManager;
import timely.auth.TimelyUser;
@Configuration
@EnableConfigurationProperties({SecurityProperties.class})
public class UserDetailsConfiguration {
public class SecurityConfiguration {
@Bean
@ConditionalOnExpression("T(org.apache.commons.lang3.StringUtils).isNotEmpty('${timely.security.authorization-url:}')")
public TimelyAuthenticationManager remoteAuthenticationManager(RemoteAuthenticationProvider remoteAuthenticationProvider) {
return new TimelyAuthenticationManager(Arrays.asList(remoteAuthenticationProvider));
}
@Bean
@ConditionalOnExpression("T(org.apache.commons.lang3.StringUtils).isNotEmpty('${timely.security.authorization-url:}')")
public RemoteAuthenticationProvider remoteAuthenticationProvider(SecurityProperties securityProperties, WebClient.Builder builder) {
return new RemoteAuthenticationProvider(builder, securityProperties);
}
@Bean
@Qualifier("file")
@ConditionalOnExpression("T(org.apache.commons.lang3.StringUtils).isEmpty('${timely.security.authorization-url:}')")
public PreAuthenticatedAuthenticationProvider fileAuthenticationProvider(FileUserDetailsService fileUserDetailsService) {
PreAuthenticatedAuthenticationProvider authenticationProvider = new PreAuthenticatedAuthenticationProvider();
authenticationProvider.setPreAuthenticatedUserDetailsService(fileUserDetailsService);
return authenticationProvider;
}
@Bean
@ConditionalOnExpression("T(org.apache.commons.lang3.StringUtils).isEmpty('${timely.security.authorization-url:}')")
public TimelyAuthenticationManager fileAuthenticationManager(@Qualifier("file") PreAuthenticatedAuthenticationProvider fileAuthenticationProvider) {
return new TimelyAuthenticationManager(Arrays.asList(fileAuthenticationProvider));
}
@Bean
public FileUserDetailsService fileUserDetailsService(SecurityProperties securityProperties) {
List<TimelyUser> timelyUsers = new ArrayList<>();
List<AuthorizedUser> users = securityProperties.getAuthorizedUsers();
for (AuthorizedUser user : users) {
@@ -32,12 +63,4 @@ public class UserDetailsConfiguration {
fileUserDetailsService.setUsers(timelyUsers);
return fileUserDetailsService;
}
@Bean
@Qualifier("file")
PreAuthenticatedAuthenticationProvider fileAuthenticationProvider(FileUserDetailsService fileUserDetailsService) {
PreAuthenticatedAuthenticationProvider authenticationProvider = new PreAuthenticatedAuthenticationProvider();
authenticationProvider.setPreAuthenticatedUserDetailsService(fileUserDetailsService);
return authenticationProvider;
}
}
@@ -1,10 +1,14 @@
package timely.common.configuration;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
import javax.validation.Valid;
import org.apache.commons.lang3.StringUtils;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.boot.context.properties.NestedConfigurationProperty;
import org.springframework.cloud.context.config.annotation.RefreshScope;
@@ -24,7 +28,7 @@ public class SecurityProperties {
@NestedConfigurationProperty
private SslServerProperties serverSsl = new SslServerProperties();
private String authorizationUri = "https://authorization:8443/authorization/v1/authorize";
private String authorizationUrl;
private List<String> requiredRoles;
private List<String> requiredAuths;
@@ -86,12 +90,12 @@ public class SecurityProperties {
return authorizedUsers;
}
public void setAuthorizationUri(String authorizationUri) {
this.authorizationUri = authorizationUri;
public void setAuthorizationUrl(String authorizationUrl) {
this.authorizationUrl = authorizationUrl;
}
public String getAuthorizationUri() {
return authorizationUri;
public String getAuthorizationUrl() {
return authorizationUrl;
}
public List<String> getRequiredAuths() {
@@ -102,6 +106,13 @@ public class SecurityProperties {
this.requiredAuths = requiredAuths;
}
public void setRequiredAuths(String requiredAuths) {
if (StringUtils.isNotBlank(requiredAuths)) {
this.requiredAuths = Arrays.stream(StringUtils.split(requiredAuths)).filter(Objects::nonNull).filter(s -> StringUtils.isNotBlank(s))
.map(String::trim).collect(Collectors.toList());
}
}
public List<String> getRequiredRoles() {
return requiredRoles;
}
@@ -109,4 +120,11 @@ public class SecurityProperties {
public void setRequiredRoles(List<String> requiredRoles) {
this.requiredRoles = requiredRoles;
}
public void setRequiredRoles(String requiredRoles) {
if (StringUtils.isNotBlank(requiredRoles)) {
this.requiredRoles = Arrays.stream(StringUtils.split(requiredRoles)).filter(Objects::nonNull).filter(s -> StringUtils.isNotBlank(s))
.map(String::trim).collect(Collectors.toList());
}
}
}
@@ -3,21 +3,27 @@ package timely.common.configuration;
import javax.validation.constraints.NotBlank;
import javax.validation.constraints.NotNull;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.cloud.context.config.annotation.RefreshScope;
@RefreshScope
@ConfigurationProperties(prefix = "timely.server")
public class ServerProperties {
@Autowired
TimelyProperties timelyProperties;
@NotBlank
private String ip;
@NotNull
private Integer tcpPort;
private Integer tcpBasePort;
@NotNull
private Integer udpPort;
private Integer udpBasePort;
private Integer shutdownQuietPeriod = 5;
private int getPortOffset() {
return (timelyProperties.getInstance() - 1) * timelyProperties.getPortIncrement();
}
@NotNull
public String getIp() {
return ip;
@@ -28,19 +34,27 @@ public class ServerProperties {
}
public int getTcpPort() {
return tcpPort;
return tcpBasePort + getPortOffset();
}
public void setTcpPort(int tcpPort) {
this.tcpPort = tcpPort;
public void setTcpBasePort(int tcpPort) {
this.tcpBasePort = tcpPort;
}
public Integer getTcpBasePort() {
return tcpBasePort;
}
public int getUdpPort() {
return udpPort;
return udpBasePort + getPortOffset();
}
public void setUdpPort(Integer udpPort) {
this.udpPort = udpPort;
public void setUdpBasePort(Integer udpBasePort) {
this.udpBasePort = udpBasePort;
}
public Integer getUdpBasePort() {
return udpBasePort;
}
/**
@@ -15,7 +15,7 @@ import io.netty.handler.ssl.SslContext;
*/
@Configuration
@EnableConfigurationProperties({SslClientProperties.class})
@ConditionalOnExpression("!T(org.springframework.util.StringUtils).isEmpty('${timely.security.clientSsl.keyStoreFile:}')")
@ConditionalOnExpression("!T(org.springframework.util.StringUtils).isEmpty('${timely.security.client-ssl.key-store-file:}')")
public class SslClientConfig {
@Bean
@@ -27,7 +27,7 @@ public class SslServerConfig {
private static final Logger log = LoggerFactory.getLogger(SslServerConfig.class);
@Bean
protected SelfSignedCertificate selfSignedCertificate() throws Exception {
public SelfSignedCertificate selfSignedCertificate() throws Exception {
Date begin = new Date();
Date end = new Date(begin.getTime() + TimeUnit.DAYS.toMillis(7));
return new SelfSignedCertificate("localhost", begin, end);
@@ -35,11 +35,11 @@ public class SslServerConfig {
@Bean
@Qualifier("nettySslContext")
protected SslContext nettySslContext(SslServerProperties sslServerProperties, SelfSignedCertificate ssc) throws Exception {
public SslContext nettySslContext(SslServerProperties sslServerProperties, SelfSignedCertificate ssc) throws Exception {
Boolean generate = sslServerProperties.isUseGeneratedKeypair();
SslContextBuilder sslContextBuilder;
if (generate) {
log.warn("Using generated self signed server certificate");
log.debug("Using generated self signed server certificate");
sslContextBuilder = SslContextBuilder.forServer(ssc.certificate(), ssc.privateKey());
} else {
sslContextBuilder = SslHelper.getSslContextBuilder(sslServerProperties);
@@ -1,7 +1,15 @@
package timely.common.configuration;
import java.net.URL;
import javax.annotation.PostConstruct;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.cloud.context.config.annotation.RefreshScope;
import org.springframework.util.ResourceUtils;
import timely.validator.NotEmptyIfFieldSet;
@@ -11,6 +19,8 @@ import timely.validator.NotEmptyIfFieldSet;
@ConfigurationProperties(prefix = "timely.security.server-ssl")
public class SslServerProperties implements SslProperties {
private static final Logger log = LoggerFactory.getLogger(SslServerProperties.class);
private String[] ciphers = new String[] {"TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256", "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA", "TLS_RSA_WITH_AES_128_GCM_SHA256",
"TLS_RSA_WITH_AES_128_CBC_SHA", "SSL_RSA_WITH_3DES_EDE_CBC_SHA"};
private String[] enabledProtocols;
@@ -25,6 +35,27 @@ public class SslServerProperties implements SslProperties {
private boolean ignoreSslHandshakeErrors = false;
private boolean useOpenssl = true;
@PostConstruct
public void setTrustStoreSystemProperties() {
try {
if (StringUtils.isNotBlank(trustStoreFile)) {
// This step is to resolve any classpath: prefixes used in trustStoreFile
URL url = ResourceUtils.getURL(trustStoreFile);
if (url != null) {
System.setProperty("javax.net.ssl.trustStore", url.getFile());
if (StringUtils.isNotBlank(trustStoreType)) {
System.setProperty("javax.net.ssl.trustStoreType", trustStoreType);
}
if (StringUtils.isNotBlank(trustStorePassword)) {
System.setProperty("javax.net.ssl.trustStorePassword", trustStorePassword);
}
}
}
} catch (Exception e) {
log.error(e.getMessage(), e);
}
}
public String[] getCiphers() {
return ciphers;
}
@@ -4,6 +4,8 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import javax.validation.constraints.Min;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.cloud.context.config.annotation.RefreshScope;
import org.springframework.validation.annotation.Validated;
@@ -18,10 +20,12 @@ public class TimelyProperties {
private String metaTable = "timely.meta";
private HashMap<String,Integer> metricAgeOffDays = new HashMap<>();
private List<String> metricsReportIgnoredTags = new ArrayList<>();
private String instance = null;
@Min(1)
private int instance = 1;
private int portIncrement = 100;
private String defaultVisibility = null;
public boolean getTest() {
public boolean isTest() {
return test;
}
@@ -47,14 +51,22 @@ public class TimelyProperties {
return this;
}
public void setInstance(String instance) {
public void setInstance(int instance) {
this.instance = instance;
}
public String getInstance() {
public int getInstance() {
return instance;
}
public void setPortIncrement(int portIncrement) {
this.portIncrement = portIncrement;
}
public int getPortIncrement() {
return portIncrement;
}
public String getDefaultVisibility() {
return defaultVisibility;
}
@@ -3,17 +3,19 @@ package timely.common.configuration;
import javax.validation.constraints.NotBlank;
import javax.validation.constraints.NotNull;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.cloud.context.config.annotation.RefreshScope;
@RefreshScope
@ConfigurationProperties(prefix = "timely.websocket")
public class WebsocketProperties {
@Autowired
TimelyProperties timelyProperties;
@NotBlank
private String ip;
@NotNull
private Integer port;
private Integer basePort;
private int timeout = 60;
private int subscriptionLag = 120;
private int scannerBatchSize = 5000;
@@ -21,6 +23,10 @@ public class WebsocketProperties {
private int scannerReadAhead = 1;
private int subscriptionBatchSize = 1000;
private int getPortOffset() {
return (timelyProperties.getInstance() - 1) * timelyProperties.getPortIncrement();
}
public String getIp() {
return ip;
}
@@ -30,11 +36,15 @@ public class WebsocketProperties {
}
public int getPort() {
return port;
return basePort + getPortOffset();
}
public void setPort(int port) {
this.port = port;
public Integer getBasePort() {
return basePort;
}
public void setBasePort(int basePort) {
this.basePort = basePort;
}
public int getTimeout() {
@@ -76,7 +76,7 @@ public class HttpRequestDecoder extends MessageToMessageDecoder<FullHttpRequest>
}
@Override
protected void decode(ChannelHandlerContext ctx, FullHttpRequest msg, List<Object> out) throws Exception {
public void decode(ChannelHandlerContext ctx, FullHttpRequest msg, List<Object> out) throws Exception {
log.trace(LOG_RECEIVED_REQUEST, msg);
@@ -22,8 +22,10 @@ public class MetricsBufferDecoder extends ByteToMessageDecoder {
if (in.readableBytes() < 8) {
return;
}
log.debug("Received {} bytes", in.readableBytes());
boolean isFlatBuffer = false;
if (log.isTraceEnabled()) {
log.trace("Received {} bytes", in.readableBytes());
}
boolean isFlatBuffer;
try {
isFlatBuffer = Metrics.MetricsBufferHasIdentifier(in.slice().nioBuffer());
} catch (Exception e) {
@@ -31,10 +33,14 @@ public class MetricsBufferDecoder extends ByteToMessageDecoder {
isFlatBuffer = false;
}
if (!isFlatBuffer) {
log.debug("Input is not a Flatbuffer");
if (log.isTraceEnabled()) {
log.trace("Input is not a Flatbuffer");
}
out.add(in.readBytes(in.readableBytes()));
} else {
log.debug("Input is a Flatbuffer");
if (log.isTraceEnabled()) {
log.trace("Input is a Flatbuffer");
}
ByteBuf copy = in.readBytes(in.readableBytes());
try {
Metrics metrics = Metrics.getRootAsMetrics(copy.nioBuffer());
@@ -1,4 +1,4 @@
package timely.api.model;
package timely.accumulo;
import java.nio.ByteBuffer;
import java.util.ArrayList;
@@ -15,11 +15,8 @@ import org.apache.accumulo.core.util.ComparablePair;
import org.junit.Assert;
import org.junit.Test;
import timely.adapter.accumulo.MetricAdapter;
import timely.api.response.MetricResponse;
import timely.model.Metric;
import timely.model.Tag;
import timely.util.JsonUtil;
public class MetricAdapterTest {
@@ -85,18 +82,6 @@ public class MetricAdapterTest {
Assert.assertArrayEquals(value, up.getValue());
}
@Test
public void testToMetricResponse() throws Exception {
String subscriptionId = "12345";
long ts = 1000L;
List<Tag> tags = new ArrayList<>();
tags.add(new Tag("tag1", "value1"));
Metric m = Metric.newBuilder().name("sys.cpu.user").value(ts, 2.0D).tags(tags).tag(MetricAdapter.VISIBILITY_TAG, "(a&b)|(c&d)").build();
String json = JsonUtil.getObjectMapper().writeValueAsString(MetricResponse.fromMetric(m, subscriptionId));
String expected = "{\"metric\":\"sys.cpu.user\",\"timestamp\":1000,\"value\":2.0,\"tags\":[{\"tag1\":\"value1\"},{\"viz\":\"(a&b)|(c&d)\"}],\"subscriptionId\":\"12345\",\"complete\":false}";
Assert.assertEquals(expected, json);
}
@Test
public void testParse() throws Exception {
PairLexicoder<String,Long> rowCoder = new PairLexicoder<>(new StringLexicoder(), new LongLexicoder());
@@ -119,7 +104,7 @@ public class MetricAdapterTest {
}
@Test
public void testParseWithEscapes() throws Exception {
public void testParseWithEscapes() {
PairLexicoder<String,Long> rowCoder = new PairLexicoder<>(new StringLexicoder(), new LongLexicoder());
byte[] row = rowCoder.encode(new ComparablePair<>("sys.cpu.user", 1000L));
byte[] value = new byte[Double.BYTES];
@@ -8,7 +8,7 @@ import org.junit.Test;
import io.netty.buffer.ByteBuf;
import io.netty.buffer.Unpooled;
import timely.adapter.accumulo.MetricAdapter;
import timely.accumulo.MetricAdapter;
import timely.api.request.MetricRequest;
import timely.api.request.VersionRequest;
import timely.model.Metric;
@@ -8,7 +8,7 @@ import org.junit.Test;
import io.netty.buffer.ByteBuf;
import io.netty.buffer.Unpooled;
import timely.adapter.accumulo.MetricAdapter;
import timely.accumulo.MetricAdapter;
import timely.api.request.MetricRequest;
import timely.model.Metric;
import timely.model.Tag;
+1 -1
View File
@@ -4,7 +4,7 @@
<parent>
<groupId>timely</groupId>
<artifactId>timely</artifactId>
<version>2.1.0-SNAPSHOT</version>
<version>2.1.0</version>
</parent>
<artifactId>timely-app</artifactId>
<packaging>pom</packaging>
+2 -2
View File
@@ -28,8 +28,8 @@
}
],
"screenshots": [],
"version": "2.1.0-SNAPSHOT",
"updated": "2024-01-30"
"version": "2.1.0",
"updated": "2024-02-26"
},
"dependencies": {
"classnames": "2.2.6",
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "timely-datasource",
"version": "2.1.0-SNAPSHOT",
"version": "2.1.0",
"description": "Timely",
"backend": true,
"alerting": true,
+119 -73
View File
@@ -3,7 +3,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>timely</groupId>
<artifactId>timely</artifactId>
<version>2.1.0-SNAPSHOT</version>
<version>2.1.0</version>
<packaging>pom</packaging>
<name>Timely</name>
<description>A secure and scalable time series database based on Apache Accumulo</description>
@@ -15,11 +15,12 @@
</licenses>
<modules>
<module>common</module>
<module>test</module>
<module>client</module>
<module>server</module>
<module>balancer</module>
<module>grafana</module>
<module>analytics</module>
<module>balancer</module>
</modules>
<scm>
<connection>scm:git:https://github.com/NationalSecurityAgency/timely.git</connection>
@@ -44,7 +45,6 @@
<spotbugs.excludes.file>${project.basedir}/src/main/spotbugs/excludes.xml</spotbugs.excludes.file>
<surefire.forkCount>1C</surefire.forkCount>
<version.accumulo>2.1.2</version.accumulo>
<version.buildnumber-maven>1.4</version.buildnumber-maven>
<version.caffeine>3.1.8</version.caffeine>
<version.classgraph>4.8.165</version.classgraph>
<version.collectd>5.7.1</version.collectd>
@@ -55,46 +55,49 @@
<version.commons-pool2>2.12.0</version.commons-pool2>
<version.commons-text>1.11.0</version.commons-text>
<version.compression.gorilla>2.1.1</version.compression.gorilla>
<version.curator>5.6.0</version.curator>
<version.curator>5.5.0</version.curator>
<version.datawave-in-memory-accumulo>3.0.1</version.datawave-in-memory-accumulo>
<version.disruptor>3.4.4</version.disruptor>
<version.docker.plugin>1.2.1</version.docker.plugin>
<version.easymock>4.0.2</version.easymock>
<version.eclipse-lifecycle>1.0.0</version.eclipse-lifecycle>
<version.fast-classpath-scanner>3.1.13</version.fast-classpath-scanner>
<version.flink>1.1.4</version.flink>
<version.google-flatbuffers>23.5.26</version.google-flatbuffers>
<version.guava>33.0.0-jre</version.guava>
<version.hadoop>3.3.6</version.hadoop>
<version.httpcomponents>4.5.14</version.httpcomponents>
<version.impsort>1.9.0</version.impsort>
<version.jackson>2.16.1</version.jackson>
<version.jacoco>0.8.11</version.jacoco>
<version.jakarta.websocket-api>1.1.2</version.jakarta.websocket-api>
<version.javacsv>2.0</version.javacsv>
<version.javassist>3.30.2-GA</version.javassist>
<version.jcip-annotations>1.0</version.jcip-annotations>
<version.jjwt>0.12.5</version.jjwt>
<version.jjwt>0.12.3</version.jjwt>
<version.jsoup>1.17.2</version.jsoup>
<version.junit>4.13.2</version.junit>
<version.log4j2>2.22.1</version.log4j2>
<version.maven-assembly>3.6.0</version.maven-assembly>
<version.maven-build-helper>3.5.0</version.maven-build-helper>
<version.maven-compiler>3.12.1</version.maven-compiler>
<version.maven-dependency>3.6.1</version.maven-dependency>
<version.maven-assembly-plugin>3.6.0</version.maven-assembly-plugin>
<version.maven-build-helper-plugin>3.5.0</version.maven-build-helper-plugin>
<version.maven-buildnumber-plugin>1.4</version.maven-buildnumber-plugin>
<version.maven-compiler-plugin>3.12.1</version.maven-compiler-plugin>
<version.maven-dependency-plugin>3.6.1</version.maven-dependency-plugin>
<version.maven-docker-plugin>1.2.1</version.maven-docker-plugin>
<version.maven-failsafe-plugin>3.2.5</version.maven-failsafe-plugin>
<version.maven-jar>3.3.0</version.maven-jar>
<version.maven-rpm>2.3.0</version.maven-rpm>
<version.maven-shade>3.5.1</version.maven-shade>
<version.maven-formatter-plugin>2.22.0</version.maven-formatter-plugin>
<version.maven-impsort-plugin>1.9.0</version.maven-impsort-plugin>
<version.maven-jar-plugin>3.3.0</version.maven-jar-plugin>
<version.maven-os-plugin>1.7.1</version.maven-os-plugin>
<version.maven-rpm-plugin>2.3.0</version.maven-rpm-plugin>
<version.maven-shade-plugin>3.5.1</version.maven-shade-plugin>
<version.maven-spotbugs-plugin>4.7.3.0</version.maven-spotbugs-plugin>
<version.maven-surefire-plugin>3.2.5</version.maven-surefire-plugin>
<version.micrometer>1.12.2</version.micrometer>
<version.netty>4.1.106.Final</version.netty>
<version.netty-tcnative>2.0.61.Final</version.netty-tcnative>
<version.os-maven>1.7.1</version.os-maven>
<version.powermock>1.6.4</version.powermock>
<version.reactor-netty>1.1.15</version.reactor-netty>
<version.slf4j>1.7.36</version.slf4j>
<version.snakeyaml>2.2</version.snakeyaml>
<version.spotbugs-maven>4.8.3.0</version.spotbugs-maven>
<version.spring-boot>2.7.18</version.spring-boot>
<version.spring-cloud>3.1.8</version.spring-cloud>
<version.spring-webflux>5.3.31</version.spring-webflux>
@@ -178,22 +181,18 @@
<groupId>commons-beanutils</groupId>
<artifactId>commons-beanutils</artifactId>
<version>1.9.4</version>
<exclusions>
<exclusion>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>commons-lang</groupId>
<artifactId>commons-lang</artifactId>
<version>${version.commons-lang}</version>
</dependency>
<dependency>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
<version>${version.commons-logging}</version>
</dependency>
<dependency>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
<version>${version.commons-logging}</version>
</dependency>
<dependency>
<groupId>fi.iki.yak</groupId>
<artifactId>compression-gorilla</artifactId>
@@ -256,6 +255,21 @@
<artifactId>jakarta.websocket-api</artifactId>
<version>${version.jakarta.websocket-api}</version>
</dependency>
<dependency>
<groupId>javax.activation</groupId>
<artifactId>activation</artifactId>
<version>1.1.1</version>
</dependency>
<dependency>
<groupId>javax.xml.bind</groupId>
<artifactId>jaxb-api</artifactId>
<version>2.3.1</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>${version.junit}</version>
</dependency>
<dependency>
<groupId>net.sourceforge.javacsv</groupId>
<artifactId>javacsv</artifactId>
@@ -266,6 +280,10 @@
<artifactId>accumulo-core</artifactId>
<version>${version.accumulo}</version>
<exclusions>
<exclusion>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
</exclusion>
<exclusion>
<groupId>io.netty</groupId>
<artifactId>netty</artifactId>
@@ -285,6 +303,10 @@
<artifactId>accumulo-minicluster</artifactId>
<version>${version.accumulo}</version>
<exclusions>
<exclusion>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
</exclusion>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
@@ -295,6 +317,11 @@
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.accumulo</groupId>
<artifactId>accumulo-tserver</artifactId>
<version>${version.accumulo}</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-collections4</artifactId>
@@ -363,6 +390,10 @@
<groupId>ch.qos.reload4j</groupId>
<artifactId>reload4j</artifactId>
</exclusion>
<exclusion>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
</exclusion>
<exclusion>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
@@ -444,6 +475,10 @@
<groupId>ch.qos.reload4j</groupId>
<artifactId>reload4j</artifactId>
</exclusion>
<exclusion>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
</exclusion>
<exclusion>
<groupId>jdk.tools</groupId>
<artifactId>jdk.tools</artifactId>
@@ -483,6 +518,10 @@
<groupId>ch.qos.reload4j</groupId>
<artifactId>reload4j</artifactId>
</exclusion>
<exclusion>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
</exclusion>
<exclusion>
<groupId>io.netty</groupId>
<artifactId>netty</artifactId>
@@ -525,6 +564,12 @@
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>${version.httpcomponents}</version>
<exclusions>
<exclusion>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<!-- This bridge allows log4j2 to work with code that is still using the log4j 1.2 classes/api -->
@@ -603,11 +648,6 @@
<artifactId>snakeyaml</artifactId>
<version>${version.snakeyaml}</version>
</dependency>
<dependency>
<groupId>timely</groupId>
<artifactId>collectd-plugin-parent</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>timely</groupId>
<artifactId>timely-balancer</artifactId>
@@ -625,7 +665,7 @@
</dependency>
<dependency>
<groupId>timely</groupId>
<artifactId>timely-server</artifactId>
<artifactId>timely-test</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
@@ -677,34 +717,10 @@
<build>
<pluginManagement>
<plugins>
<plugin>
<groupId>net.revelc.code.formatter</groupId>
<artifactId>formatter-maven-plugin</artifactId>
<version>2.22.0</version>
<configuration>
<compilerCompliance>${maven.compiler.source}</compilerCompliance>
<compilerSource>${maven.compiler.source}</compilerSource>
<compilerTargetPlatform>${maven.compiler.target}</compilerTargetPlatform>
<configFile>${project.parent.basedir}/src/main/resources/eclipse/Eclipse-Timely-Codestyle.xml</configFile>
<lineEnding>LF</lineEnding>
<skipCssFormatting>true</skipCssFormatting>
<skipHtmlFormatting>true</skipHtmlFormatting>
<skipJsFormatting>true</skipJsFormatting>
<skipJsonFormatting>true</skipJsonFormatting>
<skipXmlFormatting>true</skipXmlFormatting>
</configuration>
<executions>
<execution>
<goals>
<goal>format</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>com.spotify</groupId>
<artifactId>docker-maven-plugin</artifactId>
<version>${version.docker.plugin}</version>
<version>${version.maven-docker-plugin}</version>
<configuration>
<imageName>${docker.image.prefix}${project.artifactId}</imageName>
<forceTags>true</forceTags>
@@ -767,7 +783,7 @@
<plugin>
<groupId>net.revelc.code</groupId>
<artifactId>impsort-maven-plugin</artifactId>
<version>${version.impsort}</version>
<version>${version.maven-impsort-plugin}</version>
<configuration>
<groups>java.,javax.,*</groups>
<removeUnused>true</removeUnused>
@@ -782,15 +798,39 @@
</execution>
</executions>
</plugin>
<plugin>
<groupId>net.revelc.code.formatter</groupId>
<artifactId>formatter-maven-plugin</artifactId>
<version>${version.maven-formatter-plugin}</version>
<configuration>
<compilerCompliance>${maven.compiler.source}</compilerCompliance>
<compilerSource>${maven.compiler.source}</compilerSource>
<compilerTargetPlatform>${maven.compiler.target}</compilerTargetPlatform>
<configFile>${project.parent.basedir}/src/main/resources/eclipse/Eclipse-Timely-Codestyle.xml</configFile>
<lineEnding>LF</lineEnding>
<skipCssFormatting>true</skipCssFormatting>
<skipHtmlFormatting>true</skipHtmlFormatting>
<skipJsFormatting>true</skipJsFormatting>
<skipJsonFormatting>true</skipJsonFormatting>
<skipXmlFormatting>true</skipXmlFormatting>
</configuration>
<executions>
<execution>
<goals>
<goal>format</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<version>${version.maven-assembly}</version>
<version>${version.maven-assembly-plugin}</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>${version.maven-compiler}</version>
<version>${version.maven-compiler-plugin}</version>
<configuration>
<source>${maven.compiler.source}</source>
<target>${maven.compiler.target}</target>
@@ -799,7 +839,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<version>${version.maven-dependency}</version>
<version>${version.maven-dependency-plugin}</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
@@ -818,6 +858,8 @@
<!-- do not use the following slf4j binders -->
<exclude>ch.qos.reload4j:reload4j</exclude>
<exclude>org.slf4j:slf4j-reload4j</exclude>
<!-- commons-logging interferes with logger initialization -->
<exclude>commons-logging:commons-logging</exclude>
<!-- spring boot microservices do not use 1.x versions of log4j -->
<exclude>log4j:log4j</exclude>
<!-- spring boot microservices do not use 1.x versions of junit -->
@@ -858,7 +900,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<version>${version.maven-jar}</version>
<version>${version.maven-jar-plugin}</version>
<configuration>
<archive>
<manifest>
@@ -880,7 +922,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>${version.maven-shade}</version>
<version>${version.maven-shade-plugin}</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
@@ -902,12 +944,12 @@
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>build-helper-maven-plugin</artifactId>
<version>${version.maven-build-helper}</version>
<version>${version.maven-build-helper-plugin}</version>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>buildnumber-maven-plugin</artifactId>
<version>${version.buildnumber-maven}</version>
<version>${version.maven-buildnumber-plugin}</version>
<configuration>
<doCheck>false</doCheck>
<doUpdate>false</doUpdate>
@@ -916,7 +958,7 @@
<plugin>
<groupId>com.github.spotbugs</groupId>
<artifactId>spotbugs-maven-plugin</artifactId>
<version>${version.spotbugs-maven}</version>
<version>${version.maven-spotbugs-plugin}</version>
<configuration>
<effort>Max</effort>
<threshold>Medium</threshold>
@@ -931,12 +973,7 @@
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>rpm-maven-plugin</artifactId>
<version>${version.maven-rpm}</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-resources-plugin</artifactId>
<version>3.1.0</version>
<version>${version.maven-rpm-plugin}</version>
</plugin>
<plugin>
<groupId>org.springframework.boot</groupId>
@@ -1133,7 +1170,7 @@
<plugin>
<groupId>com.github.spotbugs</groupId>
<artifactId>spotbugs-maven-plugin</artifactId>
<version>${version.spotbugs-maven}</version>
<version>${version.maven-spotbugs-plugin}</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
@@ -1201,6 +1238,15 @@
<module>collectd-nsq-plugin</module>
<module>collectd-plugin-parent</module>
</modules>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>timely</groupId>
<artifactId>collectd-plugin-parent</artifactId>
<version>${project.version}</version>
</dependency>
</dependencies>
</dependencyManagement>
</profile>
<profile>
<id>add-spotbugs-excludes</id>
+1 -1
View File
@@ -11,7 +11,7 @@ BASE_DIR=$(cd $THIS_DIR/.. && pwd)
BIN_DIR=${BASE_DIR}/bin
CONF_DIR=${BASE_DIR}/conf
. ${BIN_DIR}/timely-env.sh
. ${BIN_DIR}/timely-server-env.sh
for i in $(seq 1 $NUM_PROFILES); do
if [ "$NUM_PROFILES" -eq "1" ]; then
-21
View File
@@ -1,21 +0,0 @@
#!/bin/bash
if [[ `uname` == "Darwin" ]]; then
THIS_SCRIPT=`python -c 'import os,sys; print os.path.realpath(sys.argv[1])' $0`
else
THIS_SCRIPT=`readlink -f $0`
fi
THIS_DIR="${THIS_SCRIPT%/*}"
BASE_DIR=$(cd $THIS_DIR/.. && pwd)
CONF_DIR="${BASE_DIR}/conf"
LIB_DIR="${BASE_DIR}/lib"
export CLASSPATH="${CONF_DIR}:${LIB_DIR}/*"
JVM_ARGS="-Xmx256m -Xms128m"
JVM_ARGS="${JVM_ARGS} -Dlogging.config=${CONF_DIR}/log4j2-error-console.xml"
JVM_ARGS="${JVM_ARGS} -Dlog4j.configurationFile=${CONF_DIR}/log4j2-error-console.xml"
JVM_ARGS="${JVM_ARGS} -DLog4jContextSelector=org.apache.logging.log4j.core.async.AsyncLoggerContextSelector"
echo "$JAVA_HOME/bin/java ${JVM_ARGS} timely.util.split.GetMetricTableSplitPoints --spring.config.name=timely"
$JAVA_HOME/bin/java ${JVM_ARGS} timely.util.GetMetricTableSplitPoints --spring.config.name=timely
-21
View File
@@ -1,21 +0,0 @@
#!/bin/bash
if [[ `uname` == "Darwin" ]]; then
THIS_SCRIPT=`python -c 'import os,sys; print os.path.realpath(sys.argv[1])' $0`
else
THIS_SCRIPT=`readlink -f $0`
fi
THIS_DIR="${THIS_SCRIPT%/*}"
BASE_DIR=$(cd $THIS_DIR/.. && pwd)
CONF_DIR="${BASE_DIR}/conf"
LIB_DIR="${BASE_DIR}/lib"
export CLASSPATH="${CONF_DIR}:${LIB_DIR}/*"
JVM_ARGS="-Xmx1024m -Xms128m"
JVM_ARGS="${JVM_ARGS} -Dlogging.config=${CONF_DIR}/log4j2-error-console.xml"
JVM_ARGS="${JVM_ARGS} -Dlog4j.configurationFile=${CONF_DIR}/log4j2-error-console.xml"
JVM_ARGS="${JVM_ARGS} -DLog4jContextSelector=org.apache.logging.log4j.core.async.AsyncLoggerContextSelector"
echo "$JAVA_HOME/bin/java ${JVM_ARGS} timely.store.compaction.util.TabletMetadataConsole --spring.config.name=timely"
$JAVA_HOME/bin/java ${JVM_ARGS} timely.store.compaction.util.TabletMetadataConsole --spring.config.name=timely
-39
View File
@@ -1,39 +0,0 @@
#!/bin/bash
if [[ `uname` == "Darwin" ]]; then
THIS_SCRIPT=`python -c 'import os,sys; print os.path.realpath(sys.argv[1])' $0`
else
THIS_SCRIPT=`readlink -f $0`
fi
THIS_DIR="${THIS_SCRIPT%/*}"
BASE_DIR=$(cd $THIS_DIR/.. && pwd)
LIB_DIR="${BASE_DIR}/lib"
NUM_SERVER_THREADS=4
CONF_DIR="${BASE_DIR}/conf"
if [ -z ${TIMELY_HOST+x} ]; then
echo "TIMELY_HOST is unset, using 127.0.0.1"
TIMELY_HOST="127.0.0.1"
else
echo "TIMELY_HOST is set to '$TIMELY_HOST'"
fi
if [ -z ${TIMELY_PORT+x} ]; then
echo "TIMELY_PORT is unset, using 54321"
TIMELY_PORT=54321
else
echo "TIMELY_PORT is set to '$TIMELY_PORT'"
fi
CP=""
SEP=""
for j in ${LIB_DIR}/*.jar; do
CP="${CP}${SEP}${j}"
SEP=":"
done
JVM_ARGS="-Xmx128m -Xms128m -Dio.netty.eventLoopThreads=${NUM_SERVER_THREADS} -Dlog4j.configurationFile=${CONF_DIR}/log4j2-spring.xml"
echo "$JAVA_HOME/bin/java -classpath ${CP} ${JVM_ARGS} timely.util.ingest.InsertTestData ${TIMELY_HOST} ${TIMELY_PORT} --fast"
exec $JAVA_HOME/bin/java -classpath ${CP} ${JVM_ARGS} timely.util.ingest.InsertTestData ${TIMELY_HOST} ${TIMELY_PORT} --fast
-39
View File
@@ -1,39 +0,0 @@
#!/bin/bash
if [[ `uname` == "Darwin" ]]; then
THIS_SCRIPT=`python -c 'import os,sys; print os.path.realpath(sys.argv[1])' $0`
else
THIS_SCRIPT=`readlink -f $0`
fi
THIS_DIR="${THIS_SCRIPT%/*}"
BASE_DIR=$(cd $THIS_DIR/.. && pwd)
LIB_DIR="${BASE_DIR}/lib"
NUM_SERVER_THREADS=4
CONF_DIR="${BASE_DIR}/conf"
if [ -z ${TIMELY_HOST+x} ]; then
echo "TIMELY_HOST is unset, using 127.0.0.1"
TIMELY_HOST="127.0.0.1"
else
echo "TIMELY_HOST is set to '$TIMELY_HOST'"
fi
if [ -z ${TIMELY_PORT+x} ]; then
echo "TIMELY_PORT is unset, using 54321"
TIMELY_PORT=54321
else
echo "TIMELY_PORT is set to '$TIMELY_PORT'"
fi
CP=""
SEP=""
for j in ${LIB_DIR}/*.jar; do
CP="${CP}${SEP}${j}"
SEP=":"
done
JVM_ARGS="-Xmx128m -Xms128m -Dio.netty.eventLoopThreads=${NUM_SERVER_THREADS} -Dlog4j.configurationFile=${CONF_DIR}/log4j2-spring.xml"
echo "$JAVA_HOME/bin/java -classpath ${CP} ${JVM_ARGS} timely.util.ingest.InsertTestData ${TIMELY_HOST} ${TIMELY_PORT}"
exec $JAVA_HOME/bin/java -classpath ${CP} ${JVM_ARGS} timely.util.ingest.InsertTestData ${TIMELY_HOST} ${TIMELY_PORT}
+4 -4
View File
@@ -10,14 +10,14 @@ THIS_DIR="${THIS_SCRIPT%/*}"
BASE_DIR=$(cd $THIS_DIR/.. && pwd)
BIN_DIR=${BASE_DIR}/bin
. ${BIN_DIR}/timely-env.sh
. ${BIN_DIR}/timely-server-env.sh
if [ "$NUM_PROFILES" -eq "1" ]; then
if [ "$NUM_INSTANCES" -eq "1" ]; then
echo ${BIN_DIR}/timely-server.sh
${BIN_DIR}/timely-server.sh
else
for i in $(seq 1 $NUM_PROFILES); do
for i in $(seq 1 $NUM_INSTANCES); do
echo ${BIN_DIR}/timely-server.sh $i
${BIN_DIR}/timely-server.sh $i
done
fi
fi
+19 -1
View File
@@ -1,3 +1,21 @@
#!/bin/bash
pkill -9 -f 'timely-server-.*-exec.jar'
if [[ `uname` == "Darwin" ]]; then
THIS_SCRIPT=`python -c 'import os,sys; print os.path.realpath(sys.argv[1])' $0`
else
THIS_SCRIPT=`readlink -f $0`
fi
THIS_DIR="${THIS_SCRIPT%/*}"
BASE_DIR=$(cd $THIS_DIR/.. && pwd)
BIN_DIR=${BASE_DIR}/bin
. ${BIN_DIR}/timely-server-env.sh
for i in $(seq 1 $NUM_INSTANCES); do
PID=`jps -m | grep -E 'timely-server-.*-exec.jar timelyserver' | grep "instance=${i}" | awk '{print $1}'`
if [ "$PID" != "" ]; then
echo "stopping timely-server instance ${i} [${PID}]"
kill -9 $PID
fi
done
File diff suppressed because it is too large. Load diff
+24
View File
@@ -0,0 +1,24 @@
#!/bin/bash
if [[ `uname` == "Darwin" ]]; then
THIS_SCRIPT=`python -c 'import os,sys; print os.path.realpath(sys.argv[1])' $0`
else
THIS_SCRIPT=`readlink -f $0`
fi
THIS_DIR="${THIS_SCRIPT%/*}"
BASE_DIR=$(cd $THIS_DIR/../.. && pwd)
LIB_DIR="${BASE_DIR}/lib"
export BIN_DIR="${BASE_DIR}/bin"
export CONF_DIR="${BASE_DIR}/conf"
export LOG_DIR="${BASE_DIR}/logs"
set -a
. ${BIN_DIR}/timely-server-env.sh
set +a
JVM_ARGS="-Xmx1G -Xms1G"
JVM_ARGS="${JVM_ARGS} -Dlogging.config=${CONF_DIR}/log4j2-console.yml"
echo "$JAVA_HOME/bin/java ${JVM_ARGS} -jar ${THIS_DIR}/timely-server*-exec.jar testingest --spring.profiles.active=timely,loadtest"
$JAVA_HOME/bin/java ${JVM_ARGS} -jar ${THIS_DIR}/timely-server-*-exec.jar testingest --spring.profiles.active=timely,loadtest
+24
View File
@@ -0,0 +1,24 @@
#!/bin/bash
if [[ `uname` == "Darwin" ]]; then
THIS_SCRIPT=`python -c 'import os,sys; print os.path.realpath(sys.argv[1])' $0`
else
THIS_SCRIPT=`readlink -f $0`
fi
THIS_DIR="${THIS_SCRIPT%/*}"
BASE_DIR=$(cd $THIS_DIR/../.. && pwd)
LIB_DIR="${BASE_DIR}/lib"
export BIN_DIR="${BASE_DIR}/bin"
export CONF_DIR="${BASE_DIR}/conf"
export LOG_DIR="${BASE_DIR}/logs"
set -a
. ${BIN_DIR}/timely-server-env.sh
set +a
JVM_ARGS="-Xmx1G -Xms1G"
JVM_ARGS="${JVM_ARGS} -Dlogging.config=${CONF_DIR}/log4j2-console.yml"
echo "$JAVA_HOME/bin/java ${JVM_ARGS} -jar ${THIS_DIR}/timely-server*-exec.jar testquery --spring.profiles.active=timely,loadtest"
$JAVA_HOME/bin/java ${JVM_ARGS} -jar ${THIS_DIR}/timely-server-*-exec.jar testquery --spring.profiles.active=timely,loadtest
File renamed without changes.
File diff suppressed because it is too large. Load diff
Loaded 100 of 297 files, more files were not shown because too many files have changed in this diff. Show more