romm/docker/init_scripts/init

#!/usr/bin/env bash

set -o errexit           # treat errors as fatal
set -o nounset           # treat unset variables as an error
set -o pipefail          # treat errors in pipes as fatal
shopt -s inherit_errexit # inherit errexit

LOGLEVEL="${LOGLEVEL:="INFO"}"

# make it possible to disable the inotify watcher process
ENABLE_RESCAN_ON_FILESYSTEM_CHANGE="${ENABLE_RESCAN_ON_FILESYSTEM_CHANGE:="false"}"
ENABLE_SCHEDULED_CLEANUP_ORPHANED_RESOURCES="${ENABLE_SCHEDULED_CLEANUP_ORPHANED_RESOURCES:="false"}"
ENABLE_SCHEDULED_RESCAN="${ENABLE_SCHEDULED_RESCAN:="false"}"
ENABLE_SCHEDULED_UPDATE_LAUNCHBOX_METADATA="${ENABLE_SCHEDULED_UPDATE_LAUNCHBOX_METADATA:="false"}"
ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB="${ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB:="false"}"
ENABLE_SYNC_FOLDER_WATCHER="${ENABLE_SYNC_FOLDER_WATCHER:="false"}"
ENABLE_SYNC_PUSH_PULL="${ENABLE_SYNC_PUSH_PULL:="false"}"
sync_base_path="${ROMM_BASE_PATH:-/romm}/sync"
# The values the backend's safe_str_to_bool reads as true, matched lowercased.
TRUTHY_PATTERN='^(1|true|yes|on)$'

# if REDIS_HOST is set, we assume that an external redis is used
REDIS_HOST="${REDIS_HOST:=""}"

# snapshot policy of the internal valkey, as "<seconds> <changes>" pairs. Saving
# hourly keeps a graceful shutdown lossless while a hard crash can only lose an
# hour of sessions and cached metadata. Set to an empty value to never snapshot.
REDIS_SAVE_POLICY="${REDIS_SAVE_POLICY="3600 1"}"

# logger colors
RED='\033[0;31m'
LIGHTMAGENTA='\033[0;95m'
GREEN='\033[0;32m'
YELLOW='\033[0;33m'
BLUE='\033[0;34m'
CYAN='\033[0;36m'
RESET='\033[0;00m'

print_banner() {
	local version
	version=$(python3 -c "exec(open('__version__.py').read()); print(__version__)")
	info_log "               _____                 __  __ "
	info_log '              |  __ \               |  \/  |'
	info_log '              | |__) |___  _ __ ___ | \  / |'
	info_log "              |  _  // _ \\| '_ \` _ \\| |\\/| |"
	info_log '              | | \ \ (_) | | | | | | |  | |'
	info_log '              |_|  \_\___/|_| |_| |_|_|  |_|'
	info_log ""
	info_log "The beautiful, powerful, self-hosted Rom manager and player"
	info_log ""
	info_log "Version: ${version}"
	info_log ""
}

debug_log() {
	# print debug log output if enabled
	if [[ ${LOGLEVEL} == "DEBUG" ]]; then
		echo -e "${LIGHTMAGENTA}DEBUG:    ${BLUE}[RomM]${LIGHTMAGENTA}[init]${CYAN}[$(date +"%Y-%m-%d %T")]${RESET}" "${@}" || true
	fi
}

info_log() {
	echo -e "${GREEN}INFO:     ${BLUE}[RomM]${LIGHTMAGENTA}[init]${CYAN}[$(date +"%Y-%m-%d %T")]${RESET}" "${@}" || true
}

warn_log() {
	echo -e "${YELLOW}WARNING:  ${BLUE}[RomM]${LIGHTMAGENTA}[init]${CYAN}[$(date +"%Y-%m-%d %T")]${RESET}" "${@}" || true
}

error_log() {
	echo -e "${RED}ERROR:    ${BLUE}[RomM]${LIGHTMAGENTA}[init]${CYAN}[$(date +"%Y-%m-%d %T")]${RESET}" "${@}" || true
	exit 1
}

# Populate a caller-provided array with the opentelemetry-instrument wrapper
# argv tokens for service "$2", or leave it empty if OTEL is disabled or the
# wrapper binary is missing. Use "${arr[@]}" to exec directly, or
# "${arr[*]@Q}" to embed as a shell-quoted prefix string.
otel_prefix() {
	local -n out_arr="$1"
	if [[ ${OTEL_SDK_DISABLED:-false} == "true" ]]; then return 0; fi
	if ! command -v opentelemetry-instrument >/dev/null 2>&1; then
		warn_log "opentelemetry-instrument not found, starting $2 without OpenTelemetry instrumentation"
		return 0
	fi
	# shellcheck disable=SC2034  # nameref binds out_arr to caller variable
	out_arr=(opentelemetry-instrument --service_name "${OTEL_SERVICE_NAME_PREFIX-}$2")
}

# Commands to run initial startup tasks
run_startup() {
	local -a wrap=()
	otel_prefix wrap startup
	if ! PYTHONPATH="/backend:${PYTHONPATH-}" "${wrap[@]}" python3 /backend/startup.py; then
		error_log "Startup script failed, exiting"
	fi
}

wait_for_gunicorn() {
	local host="127.0.0.1"
	local port="${DEV_PORT:-5000}"
	debug_log "Waiting for gunicorn on ${host}:${port}..."
	local wait_seconds=${WEB_SERVER_GUNICORN_WAIT_SECONDS:=30}
	local retries=$((wait_seconds * 2))

	# Temporarily disable errexit while polling the TCP port.
	set +o errexit
	while ((retries > 0)); do
		if (echo >/dev/tcp/"${host}"/"${port}") 2>/dev/null; then
			set -o errexit
			debug_log "Gunicorn is ready and accepting connections"
			return 0
		fi
		sleep 0.5
		((retries--))
	done
	set -o errexit

	warn_log "Gunicorn did not become ready after waiting ${wait_seconds}s!"
}

# Runs our main process and creates a corresponding PID file.
# WEB_SERVER_KEEPALIVE must outlive nginx's upstream idle timeout (60s), or the
# pooled connection is reaped before nginx can reuse it.
start_bin_gunicorn() {
	# Wire LOGLEVEL into Gunicorn's logging config so it respects the env var.
	local gunicorn_level="${LOGLEVEL^^}"
	local gunicorn_log_config="/tmp/gunicorn/logging.conf"
	# Copy the config to a writable runtime path so this works on read-only root filesystems.
	mkdir -p /tmp/gunicorn
	cp /etc/gunicorn/logging.conf "${gunicorn_log_config}"
	sed -i '/\[logger_gunicorn\]/,/^\[/ s/^level=.*/level='"${gunicorn_level}"'/' "${gunicorn_log_config}"

	# commands to start our main application and store its PID to check for crashes
	info_log "Starting backend"

	export PYTHONUNBUFFERED=1
	export PYTHONDONTWRITEBYTECODE=1

	local -a wrap=()
	otel_prefix wrap api
	"${wrap[@]}" gunicorn \
		--bind=0.0.0.0:"${DEV_PORT:-5000}" \
		--pid=/tmp/gunicorn.pid \
		--no-control-socket \
		--forwarded-allow-ips="${FORWARDED_ALLOW_IPS:-127.0.0.1,::1,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16,100.64.0.0/10,fc00::/7}" \
		--worker-class uvicorn_worker.UvicornWorker \
		--workers "${WEB_SERVER_CONCURRENCY:-4}" \
		--timeout "${WEB_SERVER_TIMEOUT:-300}" \
		--keep-alive "${WEB_SERVER_KEEPALIVE:-65}" \
		--max-requests "${WEB_SERVER_MAX_REQUESTS:-1000}" \
		--max-requests-jitter "${WEB_SERVER_MAX_REQUESTS_JITTER:-100}" \
		--worker-connections "${WEB_SERVER_WORKER_CONNECTIONS:-1000}" \
		--error-logfile - \
		--log-config "${gunicorn_log_config}" \
		main:app &
}

# Commands to start nginx (handling PID creation internally)
start_bin_nginx() {
	wait_for_gunicorn

	info_log "Starting nginx"
	if [[ ${EUID} -ne 0 ]]; then
		nginx
	else
		# if container runs as root, drop permissions
		nginx -g 'user romm;'
	fi

	: "${ROMM_BASE_URL:=http://0.0.0.0:${ROMM_PORT:-8080}}"
	info_log "🚀 RomM is now available at ${ROMM_BASE_URL}"
}

# Commands to start valkey-server (handling PID creation internally)
start_bin_valkey-server() {
	info_log "Starting internal valkey"

	# Most of the keyspace is bulky, rebuildable metadata (Switch TitleDB,
	# LaunchBox), so every snapshot rewrites hundreds of MB of dump.rdb. Valkey's
	# stock policy does that every few minutes even on an idle instance, which
	# adds up to gigabytes of disk writes per hour.
	local -a save_flags=(--save "")
	if [[ -n ${REDIS_SAVE_POLICY} ]]; then
		# shellcheck disable=SC2206 # each "<seconds> <changes>" token is its own argument
		save_flags=(--save ${REDIS_SAVE_POLICY})
	fi

	if [[ -f /usr/local/etc/valkey/valkey.conf ]]; then
		if [[ ${LOGLEVEL} == "DEBUG" ]]; then
			valkey-server /usr/local/etc/valkey/valkey.conf &
		else
			valkey-server /usr/local/etc/valkey/valkey.conf >/dev/null 2>&1 &
		fi
	else
		if [[ ${LOGLEVEL} == "DEBUG" ]]; then
			valkey-server --dir /redis-data "${save_flags[@]}" &
		else
			valkey-server --dir /redis-data "${save_flags[@]}" >/dev/null 2>&1 &
		fi
	fi

	VALKEY_PID=$!
	echo "${VALKEY_PID}" >/tmp/valkey-server.pid

	local host="127.0.0.1"
	local port="6379"
	local max_retries=120
	local retry=0

	debug_log "Waiting for internal valkey to be ready..."

	# Temporarily disable errexit for this part of the script
	set +o errexit

	while ((retry < max_retries)); do
		# Attempt to check if valkey TCP port is open
		if (echo >/dev/tcp/"${host}"/"${port}") 2>/dev/null; then
			debug_log "Internal valkey is ready and accepting connections"
			set -o errexit # Re-enable errexit after success
			return 0
		fi

		sleep 0.5
		((retry++))
	done

	error_log "Internal valkey did not become ready after $((max_retries * 500))ms"
}

# The URL carries the password, so it goes through RQ_REDIS_URL rather than
# --url, which would put it on a world-readable command line. REDIS_SSL is a
# boolean to the app, so "false" and "0" mean plaintext.
build_redis_url() {
	local scheme="redis" userinfo="" ssl="${REDIS_SSL-}"

	if [[ ${ssl,,} =~ ${TRUTHY_PATTERN} ]]; then
		scheme="rediss"
	fi

	if [[ -n ${REDIS_PASSWORD-} ]]; then
		userinfo="${REDIS_USERNAME-}:${REDIS_PASSWORD}@"
	elif [[ -n ${REDIS_USERNAME-} ]]; then
		userinfo="${REDIS_USERNAME}@"
	fi

	echo "${scheme}://${userinfo}${REDIS_HOST:-127.0.0.1}:${REDIS_PORT:-6379}/${REDIS_DB:-0}"
}

# Commands to start the RQ cron scheduler, which runs the periodic tasks listed
# in the cron config. Delayed jobs are released by the worker itself.
start_bin_rq_cron() {
	info_log "Starting RQ cron scheduler"

	local redis_url
	redis_url="$(build_redis_url)"

	PYTHONPATH="/backend:${PYTHONPATH-}" \
		RQ_REDIS_URL="${redis_url}" \
		rq cron \
		--path /backend \
		--connection-class handler.redis_handler.RomMRedis \
		--logging-level "${LOGLEVEL}" \
		tasks.cron_config &
	echo "$!" >/tmp/rq_cron.pid
}

# Commands to start RQ worker
start_bin_rq_worker() {
	info_log "Starting RQ worker"
	start_rq_worker rq_worker RomMWorker high default low
}

# Scans get a worker of their own, see SCAN_QUEUE_NAME.
start_bin_rq_scan_worker() {
	info_log "Starting RQ scan worker"
	start_rq_worker rq_scan_worker RomMWorker scans
}

# Streaming teardowns get a worker of their own, see STREAMING_QUEUE_NAME. It
# preloads the reaper, which runs every minute when streaming is on.
start_bin_rq_streaming_worker() {
	info_log "Starting RQ streaming worker"
	start_rq_worker rq_streaming_worker PreloadingWorker streaming
}

# Set PYTHONPATH so RQ can find the tasks module.
# Use a worker class that drops the noisy per-sweep "cleaning registries for
# queue" log line. The maintenance interval keeps its default (~10 min) so
# orphaned STARTED jobs and stale workers are still pruned promptly.
# --with-scheduler releases delayed jobs, which is how the watcher's rescans
# wait out their delay.
start_rq_worker() {
	local name="$1"
	local worker_class="$2"
	shift 2

	local redis_url
	redis_url="$(build_redis_url)"

	PYTHONPATH="/backend:${PYTHONPATH-}" \
		RQ_REDIS_URL="${redis_url}" \
		rq worker \
		--path /backend \
		--worker-class "handler.rq_worker.${worker_class}" \
		--connection-class handler.redis_handler.RomMRedis \
		--results-ttl "${TASK_RESULT_TTL:-86400}" \
		--logging_level "${LOGLEVEL}" \
		--with-scheduler \
		"$@" &
	# RQ writes --pid only after importing the worker class, too late for the watchdog.
	echo "$!" >"/tmp/${name}.pid"
}

start_bin_watcher() {
	info_log "Starting watcher"
	local -a wrap=()
	otel_prefix wrap watcher
	watchfiles \
		--target-type command \
		"${wrap[*]@Q} python3 watcher.py" \
		/romm/library &
	WATCHER_PID=$!
	echo "${WATCHER_PID}" >/tmp/watcher.pid
}

start_bin_sync_watcher() {
	info_log "Starting sync folder watcher"
	local -a wrap=()
	otel_prefix wrap sync_watcher
	watchfiles \
		--target-type command \
		"${wrap[*]@Q} python3 sync_watcher.py" \
		"${sync_base_path}" &
	SYNC_WATCHER_PID=$!
	echo "${SYNC_WATCHER_PID}" >/tmp/sync_watcher.pid
}

watchdog_process_pid() {
	PROCESS=$1
	if [[ -f "/tmp/${PROCESS}.pid" ]]; then
		# Check if the pid we last wrote to our state file is actually active
		PID=$(cat "/tmp/${PROCESS}.pid") || true
		if [[ ! -d "/proc/${PID}" ]]; then
			warn_log "${PROCESS} exited unexpectedly, restarting it"
			start_bin_"${PROCESS}"
		fi
	else
		# Start process if we dont have a corresponding PID file
		start_bin_"${PROCESS}"
	fi
}

stop_process_pid() {
	PROCESS=$1
	if [[ -f "/tmp/${PROCESS}.pid" ]]; then
		PID=$(cat "/tmp/${PROCESS}.pid") || true
		if [[ -d "/proc/${PID}" ]]; then
			info_log "Stopping ${PROCESS}"
			kill "${PID}" || true
			# wait for process exit
			while [[ -e "/proc/${PID}" ]]; do sleep 0.1; done
		fi
	fi
}

shutdown() {
	# shutdown in reverse order
	stop_process_pid rq_worker
	stop_process_pid rq_scan_worker
	stop_process_pid rq_streaming_worker
	stop_process_pid rq_cron
	stop_process_pid sync_watcher
	stop_process_pid watcher
	stop_process_pid nginx
	stop_process_pid gunicorn
	stop_process_pid valkey-server
}

# switch to backend directory
cd /backend || { error_log "/backend directory doesn't seem to exist"; }

print_banner

# setup trap handler
exited=0
trap 'exited=1 && shutdown' SIGINT SIGTERM EXIT

# clear any leftover PID files
rm /tmp/*.pid -f

# Disable OpenTelemetry if no OTEL_ prefixed environment variables are set
if ! printenv | grep -q '^OTEL_'; then
	info_log "No OpenTelemetry environment variables found, disabling OpenTelemetry SDK"
	export OTEL_SDK_DISABLED=true
fi

# Set ROMM_AUTH_SECRET_KEY if not already set
if [[ -z ${ROMM_AUTH_SECRET_KEY-} ]]; then
	ROMM_AUTH_SECRET_KEY=$(python3 -c "import secrets; print(secrets.token_hex(32))")
	warn_log "ROMM_AUTH_SECRET_KEY not set, generating a random secret key!"
	export ROMM_AUTH_SECRET_KEY
fi

# Start Valkey server if REDIS_HOST is not set (which would mean user is using an external Redis/Valkey)
if [[ -z ${REDIS_HOST} ]]; then
	watchdog_process_pid valkey-server
else
	info_log "REDIS_HOST is set, not starting internal valkey-server"
fi

# Run needed database migrations once at startup
info_log "Running database migrations"
if alembic upgrade head; then
	info_log "Database migrations succeeded"
else
	error_log "Failed to run database migrations"
fi

# Startup process requires database and cache to be already available
run_startup

# The image's /romm is only writable by UID 1000, so the sync folder needs its own mount.
if [[ ${ENABLE_SYNC_FOLDER_WATCHER,,} =~ ${TRUTHY_PATTERN} ]]; then
	if ! mkdir -p "${sync_base_path}" 2>/dev/null || [[ ! -w ${sync_base_path} ]]; then
		warn_log "${sync_base_path} is not writable, so the sync folder watcher and sync folder scan are disabled. Mount a host folder there to use them."
		ENABLE_SYNC_FOLDER_WATCHER="false"
	fi
fi

# Push-pull only reads its SSH keys, so a read-only mount is enough.
sync_ssh_keys_path="${SYNC_SSH_KEYS_PATH:-${sync_base_path}/keys}"
if [[ ${ENABLE_SYNC_PUSH_PULL,,} =~ ${TRUTHY_PATTERN} ]] && ! mkdir -p "${sync_ssh_keys_path}" 2>/dev/null; then
	warn_log "${sync_ssh_keys_path} doesn't exist and can't be created, so push-pull sync is disabled. Mount your SSH keys there, or set SYNC_SSH_KEYS_PATH."
	ENABLE_SYNC_PUSH_PULL="false"
fi

# main loop
while ! ((exited)); do
	watchdog_process_pid gunicorn

	# always run the cron scheduler
	watchdog_process_pid rq_cron

	watchdog_process_pid rq_worker

	watchdog_process_pid rq_scan_worker

	watchdog_process_pid rq_streaming_worker

	# only start the watcher if enabled
	if [[ ${ENABLE_RESCAN_ON_FILESYSTEM_CHANGE,,} =~ ${TRUTHY_PATTERN} ]]; then
		watchdog_process_pid watcher
	fi

	# only start the sync folder watcher if enabled
	if [[ ${ENABLE_SYNC_FOLDER_WATCHER,,} =~ ${TRUTHY_PATTERN} ]]; then
		watchdog_process_pid sync_watcher
	fi

	watchdog_process_pid nginx

	# check for died processes every 5 seconds
	sleep 5
done