mirror of
https://github.com/NationalSecurityAgency/datawave.git
synced 2026-11-10 11:41:46 -10:00
19f1d2e17c
* updates to build datawave quickstart with a given repository location. * updates to build datawave quickstart with a given repository location. * MR suggestions * Fix github actions build * Updating tests * Added in args to be able to sed docker build environment variables: * Fixes to get maven-based dependency downloads working for quickstart * debug prints * more testing * more testing * more testing * potential fix * potential fix * potential fix * potential fix * another test * another test * another test * fixed? * added instructions for downloading tarballs from maven to the docker readme * Delete docker/rabbitmq-query-config directory * Update env.sh --------- Co-authored-by: Whitney O'Meara <[email protected]> Co-authored-by: Whitney O'Meara <[email protected]>
101 lines
3.0 KiB
Bash
Executable File
101 lines
3.0 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
source ${SCRIPT_DIR}/common/common.sh
|
|
|
|
PAUSE='false'
|
|
POOL="${POOL:-pool1}"
|
|
MAX_PAGES=100
|
|
QUERY_TYPE='query'
|
|
|
|
# QUERY PARAMETERS
|
|
#QUERY_LOGIC='EventQuery'
|
|
#BEGIN='19660908 000000.000'
|
|
#END='20161002 235959.999'
|
|
#COLUMN_VISIBILITY='PUBLIC'
|
|
#QUERY='GENRES:[Action to Western]'
|
|
#QUERY_SYNTAX='LUCENE'
|
|
#AUTHS='PUBLIC,PRIVATE,BAR,FOO'
|
|
#QUERY_NAME='Developer Test Query'
|
|
#PAGE_SIZE='10'
|
|
|
|
runQuery() {
|
|
createTempPem
|
|
|
|
FOLDER="${QUERY_TYPE}_$(date +%Y%m%d_%I%M%S.%N)"
|
|
|
|
mkdir $FOLDER
|
|
cd $FOLDER
|
|
|
|
SYSTEM_FROM=$(hostname)
|
|
|
|
echo "$(date): Creating query"
|
|
echo "$(date): Creating query" > querySummary.txt
|
|
|
|
curl -s -D headers_0.txt -k -E ${TMP_PEM} \
|
|
-H "Accept: application/xml" \
|
|
-H "Pool: $POOL" \
|
|
--data-urlencode "begin=${BEGIN}" \
|
|
--data-urlencode "end=${END}" \
|
|
--data-urlencode "columnVisibility=${COLUMN_VISIBILITY}" \
|
|
--data-urlencode "query=${QUERY}" \
|
|
--data-urlencode "query.syntax=${QUERY_SYNTAX}" \
|
|
--data-urlencode "auths=${AUTHS}" \
|
|
--data-urlencode "systemFrom=${SYSTEM_FROM}" \
|
|
--data-urlencode "queryName=${QUERY_NAME}" \
|
|
--data-urlencode "pagesize=${PAGE_SIZE}" \
|
|
${DATAWAVE_ENDPOINT}/${QUERY_LOGIC}/create -o createResponse.xml -w '%{http_code}\n' >> querySummary.txt
|
|
|
|
i=1
|
|
|
|
QUERY_ID=$(get_query_id < createResponse.xml)
|
|
|
|
TOTAL_EVENTS=0
|
|
TOTAL_PAGES=0
|
|
|
|
while [ $i -gt 0 ] && [ $i -lt $MAX_PAGES ]; do
|
|
if [ "$PAUSE" == "true" ]; then
|
|
echo "press any key to continue"
|
|
read -n 1
|
|
fi
|
|
|
|
echo "$(date): Requesting page $i for $QUERY_ID"
|
|
echo "$(date): Requesting page $i for $QUERY_ID" >> querySummary.txt
|
|
curl -s -D headers_$i.txt -b headers_0.txt -q -k -E ${TMP_PEM} \
|
|
-H "Accept: application/xml" \
|
|
-H "Pool: $POOL" \
|
|
${DATAWAVE_ENDPOINT}/$QUERY_ID/next -o nextResponse_$i.xml -w '%{http_code}\n' >> querySummary.txt
|
|
|
|
CONTINUE=`grep 'HTTP/.* 200' headers_$i.txt`
|
|
|
|
if [ -z "$CONTINUE" ]; then
|
|
i=-1
|
|
else
|
|
NUM_EVENTS=$(get_num_events < nextResponse_$i.xml)
|
|
TOTAL_EVENTS=$((TOTAL_EVENTS + NUM_EVENTS))
|
|
TOTAL_PAGES=$((TOTAL_PAGES + 1))
|
|
echo "$(date): Page $i contained $NUM_EVENTS events"
|
|
echo "$(date): Page $i contained $NUM_EVENTS events" >> querySummary.txt
|
|
|
|
((i++))
|
|
fi
|
|
done
|
|
|
|
echo "$(date): Returned $TOTAL_PAGES pages"
|
|
echo "$(date): Returned $TOTAL_PAGES pages" >> querySummary.txt
|
|
|
|
echo "$(date): Returned $TOTAL_EVENTS events"
|
|
echo "$(date): Returned $TOTAL_EVENTS events" >> querySummary.txt
|
|
|
|
echo "$(date): Closing $QUERY_ID"
|
|
echo "$(date): Closing $QUERY_ID" >> querySummary.txt
|
|
# close the query
|
|
curl -s -D close_headers.txt -q -k -X POST -E ${TMP_PEM} \
|
|
-H "Accept: application/xml" \
|
|
-H "Pool: $POOL" \
|
|
${DATAWAVE_ENDPOINT}/$QUERY_ID/close -o closeResponse.xml -w '%{http_code}\n' >> querySummary.txt
|
|
|
|
cd ../
|
|
|
|
logMetrics
|
|
}
|