dify/dev/start-worker

#!/bin/bash

set -x

# Help function
show_help() {
  echo "Usage: $0 [OPTIONS]"
  echo ""
  echo "Options:"
  echo "  -q, --queues QUEUES    Comma-separated list of queues to process"
  echo "  -c, --concurrency NUM  Number of worker processes (default: 1)"
  echo "  -P, --pool POOL        Pool implementation (default: gevent)"
  echo "  --loglevel LEVEL       Log level (default: INFO)"
  echo "  -e, --env-file FILE    Path to an env file to source before starting"
  echo "  -h, --help             Show this help message"
  echo ""
  echo "Examples:"
  echo "  $0 --queues dataset,workflow"
  echo "  $0 --queues workflow_professional,workflow_team --concurrency 4"
  echo "  $0 --queues dataset --concurrency 2 --pool prefork"
  echo ""
  echo "Available queues:"
  echo "  dataset                - RAG indexing and document processing"
  echo "  dataset_summary        - LLM-heavy summary index generation (isolated from indexing)"
  echo "  workflow               - Workflow triggers (community edition)"
  echo "  workflow_professional  - Professional tier workflows (cloud edition)"
  echo "  workflow_team         - Team tier workflows (cloud edition)"
  echo "  workflow_sandbox      - Sandbox tier workflows (cloud edition)"
  echo "  schedule_poller       - Schedule polling tasks"
  echo "  schedule_executor     - Schedule execution tasks"
  echo "  mail                  - Email notifications"
  echo "  ops_trace             - Operations tracing"
  echo "  app_deletion          - Application cleanup"
  echo "  plugin                - Plugin operations"
  echo "  workflow_storage      - Workflow storage tasks"
  echo "  conversation          - Conversation tasks"
  echo "  priority_pipeline     - High priority pipeline tasks"
  echo "  pipeline              - Standard pipeline tasks"
  echo "  triggered_workflow_dispatcher - Trigger dispatcher tasks"
  echo "  trigger_refresh_executor - Trigger refresh tasks"
  echo "  retention               - Retention tasks"
  echo "  knowledge_fs_lifecycle  - KnowledgeFS provisioning and deletion lifecycle"
}

# Parse command line arguments
QUEUES=""
CONCURRENCY=1
POOL="gevent"
LOGLEVEL="INFO"

ENV_FILE=""

while [[ $# -gt 0 ]]; do
  case $1 in
    -q|--queues)
      QUEUES="$2"
      shift 2
      ;;
    -c|--concurrency)
      CONCURRENCY="$2"
      shift 2
      ;;
    -P|--pool)
      POOL="$2"
      shift 2
      ;;
    --loglevel)
      LOGLEVEL="$2"
      shift 2
      ;;
    -e|--env-file)
      ENV_FILE="$2"
      shift 2
      ;;
    -h|--help)
      show_help
      exit 0
      ;;
    *)
      echo "Unknown option: $1"
      show_help
      exit 1
      ;;
  esac
done

SCRIPT_DIR="$(dirname "$(realpath "$0")")"
cd "$SCRIPT_DIR/../api"

if [[ -n "${ENV_FILE}" ]]; then
  if [[ ! -f "${ENV_FILE}" ]]; then
    echo "Env file ${ENV_FILE} not found"
    exit 1
  fi

  echo "Loading environment variables from ${ENV_FILE}"
  # Export everything sourced from the env file
  set -a
  source "${ENV_FILE}"
  set +a
fi

# If no queues are specified, use product-edition defaults
if [[ -z "${QUEUES}" ]]; then
  # Configure queues based on product edition
  if [[ "${DEPLOYMENT_EDITION:-COMMUNITY}" == "CLOUD" ]]; then
    # Cloud edition: separate queues for dataset and trigger tasks
    QUEUES="dataset,dataset_summary,priority_dataset,priority_pipeline,pipeline,mail,ops_trace,app_deletion,plugin,workflow_storage,conversation,workflow_professional,workflow_team,workflow_sandbox,schedule_poller,schedule_executor,triggered_workflow_dispatcher,trigger_refresh_executor,retention,workflow_based_app_execution,knowledge_fs_lifecycle"
  else
    # Self-hosted editions: dataset and workflow have separate queues
    QUEUES="dataset,dataset_summary,priority_dataset,priority_pipeline,pipeline,mail,ops_trace,app_deletion,plugin,workflow_storage,conversation,workflow,schedule_poller,schedule_executor,triggered_workflow_dispatcher,trigger_refresh_executor,retention,workflow_based_app_execution,knowledge_fs_lifecycle"
  fi

  echo "No queues specified, using edition-based defaults: ${QUEUES}"
else
  echo "Using specified queues: ${QUEUES}"
fi

echo "Starting Celery worker with:"
echo "  Queues: ${QUEUES}"
echo "  Concurrency: ${CONCURRENCY}"
echo "  Pool: ${POOL}"
echo "  Log Level: ${LOGLEVEL}"

uv run \
  celery -A app.celery worker \
  -P ${POOL} -c ${CONCURRENCY} --loglevel ${LOGLEVEL} -Q ${QUEUES}