bats: no need for openssl, netcat; allow to prevent tests from running with a lock (#3016)

* bats: lock/unlock instance data; replace openssl w/ cfssl; update dep list

* concat pem without openssl :-/

* unused/unreachable code

* lint

* redundant {} (shellharden)

test/README.md

@@ -61,8 +61,6 @@ architectures.
  - `curl`
  - `daemonize`
  - `jq`
- - `nc`
- - `openssl`
  - `python3`
 
 ## Running all tests
@@ -241,6 +239,11 @@ according to the specific needs of the group of tests in the file.
    crowdsec instance. Crowdsec must not be running while this operation is
    performed.
 
+ - instance-data lock/unlock
+
+When playing around with a local crowdsec installation, you can run "instance-data lock"
+to prevent the bats suite from running, so it won't overwrite your configuration or data.
+
  - `instance-crowdsec [ start | stop ]`
 
    Runs (or stops) crowdsec as a background process. PID and lockfiles are

test/bats/11_bouncers_tls.bats

@@ -35,13 +35,14 @@ setup_file() {
     # Generate revoked client certs
     for cert_name in "revoked_1" "revoked_2"; do
         cfssl gencert -ca "${tmpdir}/inter.pem" -ca-key "${tmpdir}/inter-key.pem" -config "${CFDIR}/profiles.json" -profile=client "${CFDIR}/bouncer.json" 2>/dev/null | cfssljson --bare "${tmpdir}/${cert_name}"
-        serial="$(openssl x509 -noout -serial -in "${tmpdir}/${cert_name}.pem" | cut -d '=' -f2)"
-        echo "ibase=16; ${serial}" | bc >"${tmpdir}/serials_${cert_name}.txt"
+        cfssl certinfo -cert "${tmpdir}/${cert_name}.pem" | jq -r '.serial_number' > "${tmpdir}/serials_${cert_name}.txt"
     done
 
     # Generate separate CRL blocks and concatenate them
     for cert_name in "revoked_1" "revoked_2"; do
-        cfssl gencrl "${tmpdir}/serials_${cert_name}.txt" "${tmpdir}/ca.pem" "${tmpdir}/ca-key.pem" | base64 -d | openssl crl -inform DER -out "${tmpdir}/crl_${cert_name}.pem"
+        echo '-----BEGIN X509 CRL-----' > "${tmpdir}/crl_${cert_name}.pem"
+        cfssl gencrl "${tmpdir}/serials_${cert_name}.txt" "${tmpdir}/ca.pem" "${tmpdir}/ca-key.pem" >> "${tmpdir}/crl_${cert_name}.pem"
+        echo '-----END X509 CRL-----' >> "${tmpdir}/crl_${cert_name}.pem"
     done
     cat "${tmpdir}/crl_revoked_1.pem" "${tmpdir}/crl_revoked_2.pem" >"${tmpdir}/crl.pem"
 

test/bats/30_machines_tls.bats

@@ -38,13 +38,14 @@ setup_file() {
     # Generate revoked client cert
     for cert_name in "revoked_1" "revoked_2"; do
         cfssl gencert -ca "${tmpdir}/inter.pem" -ca-key "${tmpdir}/inter-key.pem" -config "${CFDIR}/profiles.json" -profile=client "${CFDIR}/agent.json" 2>/dev/null | cfssljson --bare "${tmpdir}/${cert_name}"
-        serial="$(openssl x509 -noout -serial -in "${tmpdir}/${cert_name}.pem" | cut -d '=' -f2)"
-        echo "ibase=16; ${serial}" | bc >"${tmpdir}/serials_${cert_name}.txt"
+        cfssl certinfo -cert "${tmpdir}/${cert_name}.pem" | jq -r '.serial_number' > "${tmpdir}/serials_${cert_name}.txt"
     done
 
     # Generate separate CRL blocks and concatenate them
     for cert_name in "revoked_1" "revoked_2"; do
-        cfssl gencrl "${tmpdir}/serials_${cert_name}.txt" "${tmpdir}/ca.pem" "${tmpdir}/ca-key.pem" | base64 -d | openssl crl -inform DER -out "${tmpdir}/crl_${cert_name}.pem"
+        echo '-----BEGIN X509 CRL-----' > "${tmpdir}/crl_${cert_name}.pem"
+        cfssl gencrl "${tmpdir}/serials_${cert_name}.txt" "${tmpdir}/ca.pem" "${tmpdir}/ca-key.pem" >> "${tmpdir}/crl_${cert_name}.pem"
+        echo '-----END X509 CRL-----' >> "${tmpdir}/crl_${cert_name}.pem"
     done
     cat "${tmpdir}/crl_revoked_1.pem" "${tmpdir}/crl_revoked_2.pem" >"${tmpdir}/crl.pem"
 

test/bin/mock-http.py

@@ -6,6 +6,7 @@ import sys
 
 from http.server import HTTPServer, BaseHTTPRequestHandler
 
+
 class RequestHandler(BaseHTTPRequestHandler):
     def do_POST(self):
         request_path = self.path
@@ -18,7 +19,7 @@ class RequestHandler(BaseHTTPRequestHandler):
         }
         print(json.dumps(log))
         self.send_response(200)
-        self.send_header('Content-type','application/json')
+        self.send_header('Content-type', 'application/json')
         self.end_headers()
         self.wfile.write(json.dumps({}).encode())
         self.wfile.flush()
@@ -27,6 +28,7 @@ class RequestHandler(BaseHTTPRequestHandler):
     def log_message(self, format, *args):
         return
 
+
 def main(argv):
     try:
         port = int(argv[1])
@@ -42,6 +44,6 @@ def main(argv):
     return 0
 
 
-if __name__ == "__main__" :
+if __name__ == "__main__":
     logging.basicConfig(level=logging.INFO)
     sys.exit(main(sys.argv))

test/bin/wait-for

@@ -39,7 +39,7 @@ async def monitor(cmd, args, want_out, want_err, timeout):
 
     status = None
 
-    async def read_stream(p, stream, outstream, pattern):
+    async def read_stream(stream, outstream, pattern):
         nonlocal status
         if stream is None:
             return
@@ -84,8 +84,8 @@ async def monitor(cmd, args, want_out, want_err, timeout):
         await asyncio.wait_for(
             asyncio.wait([
                 asyncio.create_task(process.wait()),
-                asyncio.create_task(read_stream(process, process.stdout, sys.stdout, out_regex)),
-                asyncio.create_task(read_stream(process, process.stderr, sys.stderr, err_regex))
+                asyncio.create_task(read_stream(process.stdout, sys.stdout, out_regex)),
+                asyncio.create_task(read_stream(process.stderr, sys.stderr, err_regex))
             ]), timeout)
         if status is None:
             status = process.returncode

test/bin/wait-for-port

@@ -54,10 +54,6 @@ def main(argv):
         if not args.quiet:
             write_error(ex)
         sys.exit(1)
-    else:
-        sys.exit(0)
-
-    sys.exit(1)
 
 
 if __name__ == "__main__":

test/disable-capi

@@ -5,4 +5,4 @@ THIS_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
 # shellcheck disable=SC1091
 . "${THIS_DIR}/.environment.sh"
 
-yq e 'del(.api.server.online_client)' -i "${CONFIG_YAML}"
+yq e 'del(.api.server.online_client)' -i "$CONFIG_YAML"

test/enable-capi

@@ -5,7 +5,7 @@ THIS_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
 # shellcheck disable=SC1091
 . "${THIS_DIR}/.environment.sh"
 
-online_api_credentials="$(dirname "${CONFIG_YAML}")/online_api_credentials.yaml"
+online_api_credentials="$(dirname "$CONFIG_YAML")/online_api_credentials.yaml"
 export online_api_credentials
 
-yq e '.api.server.online_client.credentials_path=strenv(online_api_credentials)' -i "${CONFIG_YAML}"
+yq e '.api.server.online_client.credentials_path=strenv(online_api_credentials)' -i "$CONFIG_YAML"

test/instance-crowdsec

@@ -2,15 +2,15 @@
 
 #shellcheck disable=SC1007
 THIS_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
-cd "${THIS_DIR}" || exit 1
+cd "$THIS_DIR" || exit 1
 # shellcheck disable=SC1091
 . ./.environment.sh
 
 backend_script="./lib/init/crowdsec-${INIT_BACKEND}"
 
-if [[ ! -x "${backend_script}" ]]; then
+if [[ ! -x "$backend_script" ]]; then
     echo "unknown init system '${INIT_BACKEND}'" >&2
     exit 1
 fi
 
-exec "${backend_script}" "$@"
+exec "$backend_script" "$@"

test/instance-data

@@ -2,15 +2,15 @@
 
 #shellcheck disable=SC1007
 THIS_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
-cd "${THIS_DIR}" || exit 1
+cd "$THIS_DIR" || exit 1
 # shellcheck disable=SC1091
 . ./.environment.sh
 
 backend_script="./lib/config/config-${CONFIG_BACKEND}"
 
-if [[ ! -x "${backend_script}" ]]; then
+if [[ ! -x "$backend_script" ]]; then
     echo "unknown config backend '${CONFIG_BACKEND}'" >&2
     exit 1
 fi
 
-exec "${backend_script}" "$@"
+exec "$backend_script" "$@"

test/instance-db

@@ -2,7 +2,7 @@
 
 #shellcheck disable=SC1007
 THIS_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
-cd "${THIS_DIR}" || exit 1
+cd "$THIS_DIR" || exit 1
 # shellcheck disable=SC1091
 . ./.environment.sh
 
@@ -10,9 +10,9 @@ cd "${THIS_DIR}" || exit 1
 
 backend_script="./lib/db/instance-${DB_BACKEND}"
 
-if [[ ! -x "${backend_script}" ]]; then
+if [[ ! -x "$backend_script" ]]; then
     echo "unknown database '${DB_BACKEND}'" >&2
     exit 1
 fi
 
-exec "${backend_script}" "$@"
+exec "$backend_script" "$@"

test/instance-mock-http

@@ -13,7 +13,7 @@ about() {
 
 #shellcheck disable=SC1007
 THIS_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
-cd "${THIS_DIR}"
+cd "$THIS_DIR"
 # shellcheck disable=SC1091
 . ./.environment.sh
 
@@ -31,7 +31,7 @@ DAEMON_PID=${PID_DIR}/mock-http.pid
 start_instance() {
     [[ $# -lt 1 ]] && about
     daemonize \
-        -p "${DAEMON_PID}" \
+        -p "$DAEMON_PID" \
         -e "${LOG_DIR}/mock-http.err" \
         -o "${LOG_DIR}/mock-http.out" \
         /usr/bin/env python3 -u "${THIS_DIR}/bin/mock-http.py" "$1"
@@ -40,10 +40,10 @@ start_instance() {
 }
 
 stop_instance() {
-    if [[ -f "${DAEMON_PID}" ]]; then
+    if [[ -f "$DAEMON_PID" ]]; then
         # terminate with extreme prejudice, all the application data will be thrown away anyway
-        kill -9 "$(cat "${DAEMON_PID}")" > /dev/null 2>&1
-        rm -f -- "${DAEMON_PID}"
+        kill -9 "$(cat "$DAEMON_PID")" > /dev/null 2>&1
+        rm -f -- "$DAEMON_PID"
     fi
 }
 

test/lib/config/config-local

@@ -9,7 +9,7 @@ die() {
 }
 
 about() {
-    die "usage: ${script_name} [make | load | clean]"
+    die "usage: ${script_name} [make | load | lock | unlock | clean]"
 }
 
 #shellcheck disable=SC1007
@@ -134,7 +134,16 @@ make_init_data() {
     remove_init_data
 }
 
+lock_init_data() {
+    touch "${LOCAL_INIT_DIR}/.lock"
+}
+
+unlock_init_data() {
+    rm -f "${LOCAL_INIT_DIR}/.lock"
+}
+
 load_init_data() {
+    [[ -f "${LOCAL_INIT_DIR}/.lock" ]] && die "init data is locked"
     ./bin/assert-crowdsec-not-running || die "Cannot load fixture data."
 
     if [[ ! -f "${LOCAL_INIT_DIR}/init-config-data.tar" ]]; then
@@ -164,6 +173,12 @@ case "$1" in
     load)
         load_init_data
         ;;
+    lock)
+        lock_init_data
+        ;;
+    unlock)
+        unlock_init_data
+        ;;
     clean)
         remove_init_data
         ;;

test/run-tests

@@ -16,14 +16,16 @@ TEST_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
 
 echo "Running tests..."
 echo "DB_BACKEND: ${DB_BACKEND}"
-if [[ -z "${TEST_COVERAGE}" ]]; then
+if [[ -z "$TEST_COVERAGE" ]]; then
     echo "Coverage report: no"
 else
     echo "Coverage report: yes"
 fi
 
+[[ -f "$LOCAL_INIT_DIR/.lock" ]] && die "init data is locked: are you doing some manual test? if so, please finish what you are doing, run 'instance-data unlock' and retry"
+
 dump_backend="$(cat "${LOCAL_INIT_DIR}/.backend")"
-if [[ "${DB_BACKEND}" != "${dump_backend}" ]]; then
+if [[ "$DB_BACKEND" != "$dump_backend" ]]; then
     die "Can't run with backend '${DB_BACKEND}' because the test data was build with '${dump_backend}'"
 fi