ifixes #219 - create scripts to help with running and testing multiple instances

server/bin/create-timely-profiles.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+
+if [[ `uname` == "Darwin" ]]; then
+    THIS_SCRIPT=`python -c 'import os,sys; print os.path.realpath(sys.argv[1])' $0`
+else
+    THIS_SCRIPT=`readlink -f $0`
+fi
+
+THIS_DIR="${THIS_SCRIPT%/*}"
+BASE_DIR=$(cd $THIS_DIR/.. && pwd)
+BIN_DIR=${BASE_DIR}/bin
+CONF_DIR=${BASE_DIR}/conf
+
+. ${BIN_DIR}/timely-env.sh
+
+HOST_IP=$(hostname -I | awk '{print $1}')
+HOST=$(hostname -s)
+HOST_FQDN=$(hostname -f)
+
+for i in $(seq 1 $NUM_PROFILES); do
+    if [ "$NUM_PROFILES" -eq "1" ]; then
+        FILENAME=timely.yml
+    else
+        FILENAME=timely-${i}.yml
+    fi
+
+    echo "creating configuration file ${CONF_DIR}/${FILENAME}"
+    cp ${CONF_DIR}/timely-template.yml ${CONF_DIR}/${FILENAME}
+    sed -i "s/\${HOST}/${HOST}/g" ${CONF_DIR}/${FILENAME}
+    sed -i "s/\${HOST_FQDN}/${HOST_FQDN}/g" ${CONF_DIR}/${FILENAME}
+    sed -i "s/\${HOST_IP}/${HOST_IP}/g" ${CONF_DIR}/${FILENAME}
+    sed -i "s/\${ZOOKEEPERS}/${ZOOKEEPERS}/g" ${CONF_DIR}/${FILENAME}
+    sed -i "s/\${AGE_OFF_DAYS}/${AGE_OFF_DAYS}/g" ${CONF_DIR}/${FILENAME}
+    MULTIPLE=$(($i - 1))
+    TCP_PORT=$(($TCP_PORT_START + $MULTIPLE * $PORT_INCREMENT))
+    sed -i "s/\${TCP_PORT}/${TCP_PORT}/g" ${CONF_DIR}/${FILENAME}
+    HTTP_PORT=$(($HTTP_PORT_START + $MULTIPLE * $PORT_INCREMENT))
+    sed -i "s/\${HTTP_PORT}/${HTTP_PORT}/g" ${CONF_DIR}/${FILENAME}
+    WEBSOCKET_PORT=$(($WEBSOCKET_PORT_START + $MULTIPLE * $PORT_INCREMENT))
+    sed -i "s/\${WEBSOCKET_PORT}/${WEBSOCKET_PORT}/g" ${CONF_DIR}/${FILENAME}
+    UDP_PORT=$(($UDP_PORT_START + $MULTIPLE * $PORT_INCREMENT))
+    sed -i "s/\${UDP_PORT}/${UDP_PORT}/g" ${CONF_DIR}/${FILENAME}
+    sed -i "s/\${ACCUMULO_USER}/${ACCUMULO_USER}/g" ${CONF_DIR}/${FILENAME}
+    sed -i "s/\${ACCUMULO_PASSWORD}/${ACCUMULO_PASSWORD}/g" ${CONF_DIR}/${FILENAME}
+    sed -i "s/\${INSTANCE}/${i}/g" ${CONF_DIR}/${FILENAME}
+done
+

server/bin/get-metric-split-points.sh

@@ -7,7 +7,7 @@ else
 fi
 
 THIS_DIR="${THIS_SCRIPT%/*}"
-BASE_DIR=${THIS_DIR}/..
+BASE_DIR=$(cd $THIS_DIR/.. && pwd)
 CONF_DIR="${BASE_DIR}/conf"
 LIB_DIR="${BASE_DIR}/lib"
 

server/bin/get-tablet-summary.sh

@@ -7,7 +7,7 @@ else
 fi
 
 THIS_DIR="${THIS_SCRIPT%/*}"
-BASE_DIR=${THIS_DIR}/..
+BASE_DIR=$(cd $THIS_DIR/.. && pwd)
 CONF_DIR="${BASE_DIR}/conf"
 LIB_DIR="${BASE_DIR}/lib"
 

server/bin/insert-test-data-fast.sh

@@ -7,7 +7,7 @@ else
 fi
 
 THIS_DIR="${THIS_SCRIPT%/*}"
-BASE_DIR=${THIS_DIR}/..
+BASE_DIR=$(cd $THIS_DIR/.. && pwd)
 LIB_DIR="${BASE_DIR}/lib"
 NUM_SERVER_THREADS=4
 CONF_DIR="${BASE_DIR}/conf"

server/bin/insert-test-data.sh

@@ -7,7 +7,7 @@ else
 fi
 
 THIS_DIR="${THIS_SCRIPT%/*}"
-BASE_DIR=${THIS_DIR}/..
+BASE_DIR=$(cd $THIS_DIR/.. && pwd)
 LIB_DIR="${BASE_DIR}/lib"
 NUM_SERVER_THREADS=4
 CONF_DIR="${BASE_DIR}/conf"

server/bin/start-all.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+if [[ `uname` == "Darwin" ]]; then
+    THIS_SCRIPT=`python -c 'import os,sys; print os.path.realpath(sys.argv[1])' $0`
+else
+    THIS_SCRIPT=`readlink -f $0`
+fi
+
+THIS_DIR="${THIS_SCRIPT%/*}"
+BASE_DIR=$(cd $THIS_DIR/.. && pwd)
+BIN_DIR=${BASE_DIR}/bin
+
+. ${BIN_DIR}/timely-env.sh
+
+if [ "$NUM_PROFILES" -eq "1" ]; then
+    echo ${BIN_DIR}/timely-server.sh
+    ${BIN_DIR}/timely-server.sh
+else
+    for i in $(seq 1 $NUM_PROFILES); do
+        echo ${BIN_DIR}/timely-server.sh $i
+        ${BIN_DIR}/timely-server.sh $i
+    done
+fi

server/bin/stop-all.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+pkill -9 -f timely.Server

server/bin/timely-env.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+NUM_PROFILES=1
+ZOOKEEPERS=localhost:2181
+TCP_PORT_START=4242
+HTTP_PORT_START=4243
+WEBSOCKET_PORT_START=4244
+UDP_PORT_START=4245
+PORT_INCREMENT=100
+AGE_OFF_DAYS=7
+ACCUMULO_USER=root
+ACCUMULO_PASSWORD=secret

server/bin/timely-server.sh

@@ -9,33 +9,62 @@ else
 fi
 
 THIS_DIR="${THIS_SCRIPT%/*}"
-NATIVE_DIR="${THIS_DIR}/META-INF/native"
-BASE_DIR=${THIS_DIR}/..
-TMP_DIR="${BASE_DIR}/tmp"
-CONF_DIR="${BASE_DIR}/conf"
-LIB_DIR="${BASE_DIR}/lib"
-NUM_SERVER_THREADS=4
 
-if [[ -e ${TMP_DIR} ]]; then
-  rm -rf ${TMP_DIR}
+if [ -n "$1" ]; then
+  export PROFILE_NUM=$1
+  PROFILE_ARG="--spring.profiles.active=${PROFILE_NUM}"
+  PID=`jps -m | grep Server | grep "spring.profiles.active=${PROFILE_NUM}" | awk '{print $1}'`
+else
+  export PROFILE_NUM=""
+  PID=`jps -m | grep Server | awk '{print $1}'`
 fi
-mkdir ${TMP_DIR}
 
-if [[ -e ${NATIVE_DIR} ]]; then
-  rm -rf ${NATIVE_DIR}
+if [ "$PID" == "" ]; then
+
+    NATIVE_DIR="${THIS_DIR}/META-INF/native"
+    BASE_DIR=$(cd $THIS_DIR/.. && pwd)
+    TMP_DIR="${BASE_DIR}/tmp"
+    CONF_DIR="${BASE_DIR}/conf"
+    LIB_DIR="${BASE_DIR}/lib"
+    export LOG_DIR="${BASE_DIR}/logs"
+    NUM_SERVER_THREADS=4
+
+    if [[ -e ${TMP_DIR} ]]; then
+      rm -rf ${TMP_DIR}
+    fi
+    mkdir ${TMP_DIR}
+
+    if [[ -e ${NATIVE_DIR} ]]; then
+      rm -rf ${NATIVE_DIR}
+    fi
+    mkdir -p ${NATIVE_DIR}
+
+    if [[ ! -e ${LOG_DIR} ]]; then
+      mkdir ${LOG_DIR}
+    fi
+
+    pushd ${BASE_DIR}/bin
+    $JAVA_HOME/bin/jar xf ${LIB_DIR}/netty-tcnative-boringssl-static*.jar META-INF/native/libnetty_tcnative_linux_x86_64.${TCNATIVE_SUFFIX}
+    $JAVA_HOME/bin/jar xf ${LIB_DIR}/netty-all*.jar META-INF/native/libnetty_transport_native_epoll_x86_64.${TCNATIVE_SUFFIX}
+    popd
+
+    export CLASSPATH="${CONF_DIR}:${LIB_DIR}/*:${HADOOP_CONF_DIR}"
+    JVM_ARGS="-Dio.netty.eventLoopThreads=${NUM_SERVER_THREADS} -Dlog4j.configurationFile=${CONF_DIR}/log4j2-spring.xml"
+    JVM_ARGS="${JVM_ARGS} -DLog4jContextSelector=org.apache.logging.log4j.core.async.AsyncLoggerContextSelector"
+    JVM_ARGS="${JVM_ARGS} -Djava.library.path=${NATIVE_DIR}"
+
+    JVM_ARGS="${JVM_ARGS} -Xmx4G -Xms4G -XX:NewSize=1G -XX:MaxNewSize=1G -XX:+UnlockDiagnosticVMOptions"
+    JVM_ARGS="${JVM_ARGS} -XX:+UseConcMarkSweepGC -XX:+CMSScavengeBeforeRemark -XX:+ScavengeBeforeFullGC -XX:ReservedCodeCacheSize=384M -XX:+UseCodeCacheFlushing -XX:+UseParNewGC"
+    JVM_ARGS="${JVM_ARGS} -XX:SurvivorRatio=8 -XX:ParallelGCThreads=16 -XX:ConcGCThreads=8 -XX:+UseCondCardMark  -XX:ParGCCardsPerStrideChunk=32768"
+    JVM_ARGS="${JVM_ARGS} -XX:+CMSClassUnloadingEnabled -XX:+CMSParallelRemarkEnabled -XX:CMSInitiatingOccupancyFraction=30 -XX:+UseCMSInitiatingOccupancyOnly"
+    JVM_ARGS="${JVM_ARGS} -XX:+UseCompressedOops -XX:+UseNUMA -Djava.net.preferIPv4Stack=true"
+
+    echo "$JAVA_HOME/bin/java ${JVM_ARGS} timely.Server --spring.config.name=timely ${PROFILE_ARG}"
+    nohup $JAVA_HOME/bin/java ${JVM_ARGS} timely.Server --spring.config.name=timely ${PROFILE_ARG} >> ${LOG_DIR}/timely-server${PROFILE_NUM}-startup.log 2>&1 &
+else
+    if [ "$PROFILE_NUM" == ""]; then
+        echo "timely-server already running with pid ${PID}"
+    else
+        echo "timely-server ${PROFILE_NUM} already running with pid ${PID}"
+    fi
 fi
-mkdir -p ${NATIVE_DIR}
-
-pushd ${BASE_DIR}/bin
-$JAVA_HOME/bin/jar xf ${LIB_DIR}/netty-tcnative-boringssl-static*.jar META-INF/native/libnetty_tcnative_linux_x86_64.${TCNATIVE_SUFFIX}
-$JAVA_HOME/bin/jar xf ${LIB_DIR}/netty-all*.jar META-INF/native/libnetty_transport_native_epoll_x86_64.${TCNATIVE_SUFFIX}
-popd
-
-export CLASSPATH="${CONF_DIR}:${LIB_DIR}/*:${HADOOP_CONF_DIR}"
-JVM_ARGS="-Xmx256m -Xms256m -Dio.netty.eventLoopThreads=${NUM_SERVER_THREADS} -Dlog4j.configurationFile=${CONF_DIR}/log4j2-spring.xml"
-JVM_ARGS="${JVM_ARGS} -DLog4jContextSelector=org.apache.logging.log4j.core.async.AsyncLoggerContextSelector"
-JVM_ARGS="${JVM_ARGS} -Djava.library.path=${NATIVE_DIR}"
-
-echo "$JAVA_HOME/bin/java ${JVM_ARGS} timely.Server --spring.config.name=timely"
-$JAVA_HOME/bin/java ${JVM_ARGS} timely.Server --spring.config.name=timely
-

server/bin/timely-standalone.sh

@@ -10,7 +10,7 @@ fi
 
 THIS_DIR="${THIS_SCRIPT%/*}"
 NATIVE_DIR="${THIS_DIR}/META-INF/native"
-BASE_DIR=${THIS_DIR}/..
+BASE_DIR=$(cd $THIS_DIR/.. && pwd)
 TMP_DIR="${BASE_DIR}/tmp"
 CONF_DIR="${BASE_DIR}/conf"
 LIB_DIR="${BASE_DIR}/lib"

server/conf/log4j2-spring.xml

@@ -5,33 +5,19 @@
     <Property name="LOG_PATTERN">%clr{%d{yyyy-MM-dd HH:mm:ss.SSS}}{faint} %clr{%5p} %clr{${sys:PID}}{magenta} %clr{---}{faint} %clr{[%15.15t]}{faint} %clr{%-40.40c{1.}}{cyan} %clr{:}{faint} %m%n%wEx</Property>
   </Properties>
   <Appenders>
-    <RollingRandomAccessFile name="Rolling" fileName="logs/timely-server.log"
-    						 filePattern="logs/$${date:yyyy-MM}/timely-server-%d{MM-dd-yyyy}-%i.log.gz">
+    <RollingRandomAccessFile name="Rolling" fileName="${env:LOG_DIR}/timely-server${env:PROFILE_NUM}.log"
+    						 filePattern="${env:LOG_DIR}/$${date:yyyy-MM}/timely-server${env:PROFILE_NUM}-%d{MM-dd-yyyy}-%i.log.gz">
       <PatternLayout pattern="${LOG_PATTERN}"/>
       <Policies>
       	<SizeBasedTriggeringPolicy size="1024 MB" />
       </Policies>
       <DefaultRolloverStrategy max="10" />
     </RollingRandomAccessFile>
-    <RollingRandomAccessFile name="Subscription" fileName="logs/timely-subscriptions.log"
-    						 filePattern="logs/$${date:yyyy-MM}/timely-subscriptions-%d{MM-dd-yyyy}-%i.log.gz">
-      <PatternLayout pattern="${LOG_PATTERN}"/>
-      <Policies>
-      	<SizeBasedTriggeringPolicy size="1024 MB" />
-      </Policies>
-      <DefaultRolloverStrategy max="10" />
-    </RollingRandomAccessFile>
-    <Console name="Console" target="SYSTEM_OUT">
-      <PatternLayout pattern="${LOG_PATTERN}"/>
-    </Console>
   </Appenders>
   <Loggers>
-    <Root level="debug" includeLocation="false">
-      <AppenderRef ref="Console" level="warn" />
+    <Root level="warn" includeLocation="false">
       <AppenderRef ref="Rolling" level="debug" />
     </Root>
-    <Logger name="timely" additivity="false">
-      <AppenderRef ref="Console" level="debug" />
-    </Logger>
+    <Logger name="timely" level="debug"/>
   </Loggers>
 </Configuration>

server/conf/timely-template.yml

@@ -0,0 +1,74 @@
+timely:
+  instance: ${INSTANCE}
+  metrics-table: timely.metrics
+  meta-table: timely.meta
+  metric-age-off-days:
+    default: ${AGE_OFF_DAYS}
+  cache:
+    enabled: true
+    age-off-hours:
+      default: 26
+  metrics-report-ignored-tags:
+  defaultVisibility:
+  accumulo:
+    instance-name: accumulo
+    zookeepers: ${ZOOKEEPERS}
+    username: ${ACCUMULO_USER}
+    password: ${ACCUMULO_PASSWORD}
+    write:
+      latency: 5s
+      threads: 3
+      bufferSize: 52428800
+    scan:
+      threads: 4
+      max-downsample-memory: 10000000
+  security:
+    session-max-age: 86400
+    cache-expiration-minutes: 60
+    cache-refresh-minutes: 5
+    allow-anonymous-http-access: true
+    allow-anonymous-ws-access: true
+    server-ssl:
+      certificate-file:
+      key-file:
+      key-password:
+      use-generated-keypair: true
+      trust-store-file:
+      use-openssl: true
+      #use-ciphers:
+    jwtCheckKeyStore:
+    jwtCheckKeyType:
+    jwtCheckKeyPassword:
+  server:
+    ip: ${HOST_IP}
+    tcp-port: ${TCP_PORT}
+    udp-port: ${UDP_PORT}
+  http:
+    host: ${HOST}
+    ip: ${HOST_IP}
+    port: ${HTTP_PORT}
+    redirect-path: /secure-me
+    strict-transport-max-age: 608400
+    cors:
+      allow-any-origin: false
+      allow-null-origin: false
+      allow-credentials: true
+      allowed-origins:
+      allowed-methods: DELETE,GET,HEAD,OPTIONS,PUT,POST
+      allowed-headers: content-type
+  websocket:
+    ip: ${HOST_IP}
+    port: ${WEBSOCKET_PORT}
+    timeout: 60
+    subscription-lag: 120
+    scanner-batch-size: 10000
+    flush-interval-seconds: 30
+    scanner-read-ahead: 1
+    subscription-batch-size: 5000
+  meta-cache:
+    expiration-minutes: 1440
+    cache-refresh-minutes: 60
+    max-tag-values: 100
+  visibility-cache:
+    expiration-minutes: 60
+