infiniflow/ragflow
1
0
Fork 0
mirror of https://github.com/infiniflow/ragflow.git synced 2026-10-12 02:16:39 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
108e2bb3a3c8c1ef06fa301d9c8a315e62ec9bb1
ragflow/build.sh
T

1241 lines
55 KiB
Bash
Executable File
Raw Blame History

#!/bin/bash
set -e
# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m' # No Color
# Get script directory
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$SCRIPT_DIR"
# Build directories
CPP_DIR="$PROJECT_ROOT/internal/binding/cpp"
BUILD_DIR="$CPP_DIR/cmake-build-release"
RAGFLOW_SERVER_BINARY="$PROJECT_ROOT/bin/ragflow_server"
RAGFLOW_CLI_BINARY="$PROJECT_ROOT/bin/ragflow-cli"
# Strip symbols from Go binaries (set via --strip / -s)
STRIP_SYMBOLS=""
# Native static library settings. These are the user-cache paths (~/ragflow-native-libs/).
# If /opt/ragflow-native-libs/ exists (pre-seeded in CI runner image), it takes priority
# and skips the network (download_deps.py) fallback.
SYSTEM_DEPS="/opt/ragflow-native-libs"
# office_oxide native library settings — static linking
OFFICE_OXIDE_PREFIX="${HOME}/ragflow-native-libs/office_oxide"
OFFICE_OXIDE_VERSION="0.1.12"
# pdfium native library settings — static linking (kognitos/pdfium-static)
PDFIUM_STATIC_PREFIX="${HOME}/ragflow-native-libs/pdfium-static"
PDFIUM_STATIC_VERSION="7809"
# pdf_oxide native library settings — static linking (go-ffi tarball)
PDF_OXIDE_PREFIX="${HOME}/ragflow-native-libs/pdf_oxide"
PDF_OXIDE_VERSION="0.3.73"
# onnxruntime native library settings — static linking for the in-process
# (Go) DeepDoc backend. libonnxruntime*.a is linked into the server binary
# (--undefined=OrtGetApiBase + --dynamic-list, no --whole-archive); OrtGetApiBase
# is then resolved via dlopen(NULL) (the process-global symbol table, not the
# executable's own path), so no libonnxruntime.so is needed at runtime.
# Downloaded by ragflow_deps/download_deps.py into onnxruntime/static_lib.
ONNXRUNTIME_STATIC_PREFIX="${HOME}/ragflow-native-libs/onnxruntime/static_lib"
# Copy a dependency from the system pre-seed directory to the user cache.
# Returns 0 if the dep was copied or already exists in cache, 1 otherwise.
_seed_from_system() {
local dep_name="$1" # e.g. "pdfium-static", "pdf_oxide", "office_oxide"
local dep_dir="${HOME}/ragflow-native-libs/${dep_name}"
local sys_dir="${SYSTEM_DEPS}/${dep_name}"
echo "check if dep ${dep_name} exists in ${dep_dir} or ${sys_dir}"
if [ -d "$dep_dir" ]; then
echo " ${dep_name} → ${dep_dir} (user cache)"
return 0 # already cached
fi
if [ -d "$sys_dir" ]; then
echo " ${dep_name} → ${sys_dir} (system)"
mkdir -p "$(dirname "$dep_dir")"
cp -r "$sys_dir" "$dep_dir"
return 0
fi
echo " ${dep_name} not found in system or user cache"
return 1
}
# ── cl100k_base BPE table ─────────────────────────────────────────────
# internal/tokenizer loads this table from disk only (no network) and PANICS in
# NumTokensFromString / TrimContentToTokenLimit when it is absent: a deployment
# that forgot the file must not silently zero every token count (see
# internal/tokenizer/bpe_loader.go and failfast_test.go). CI provisions it in a
# workflow step before running the Go tests; build.sh does the same so the
# documented local test commands are self-contained. Order mirrors CI: system
# pre-seed first, then the network. A failure here is deliberately NOT fatal —
# the loader stays the final gate, and its panic message names the file.
CL100K_TABLE_URL="https://openaipublic.blob.core.windows.net/encodings/cl100k_base.tiktoken"
CL100K_TABLE_RELPATH="ragflow_deps/cl100k_base.tiktoken"
SYSTEM_DEPS_TOKENIZER="/opt/ragflow_deps"
# Provision the cl100k_base BPE table into ragflow_deps/ (no-op if present).
# Returns 0 when the table is in place, 1 when it could not be provisioned.
_ensure_cl100k_table() {
local dest="${PROJECT_ROOT}/${CL100K_TABLE_RELPATH}"
local sys_copy="${SYSTEM_DEPS_TOKENIZER}/cl100k_base.tiktoken"
echo "check if the cl100k_base BPE table exists at ${dest}"
if [ -s "$dest" ]; then
echo " cl100k_base.tiktoken → ${dest} (already present)"
return 0
fi
if [ -s "$sys_copy" ]; then
mkdir -p "$(dirname "$dest")"
cp "$sys_copy" "$dest"
echo " cl100k_base.tiktoken → ${dest} (system)"
return 0
fi
# Download to a temp file and move it into place only after a successful
# transfer: an interrupted download would otherwise leave a truncated table
# behind, which the loader rejects on digest.
local tmp="${dest}.download"
mkdir -p "$(dirname "$dest")"
if command -v curl >/dev/null 2>&1; then
if curl -fsSL -o "$tmp" "$CL100K_TABLE_URL"; then
mv "$tmp" "$dest"
echo " cl100k_base.tiktoken → ${dest} (downloaded)"
return 0
fi
elif command -v wget >/dev/null 2>&1; then
if wget -q -O "$tmp" "$CL100K_TABLE_URL"; then
mv "$tmp" "$dest"
echo " cl100k_base.tiktoken → ${dest} (downloaded)"
return 0
fi
fi
rm -f "$tmp"
echo -e " ${YELLOW}cl100k_base BPE table NOT provisioned${NC} (offline, or the network is blocked)"
echo " internal/tokenizer panics on a missing table by design, so every test that"
echo " counts tokens will fail loudly. Recover with any of:"
echo " - re-run this command (a transient network error usually clears);"
echo " - curl -fsSL -o ${CL100K_TABLE_RELPATH} ${CL100K_TABLE_URL}"
echo " - or point TIKTOKEN_CACHE_DIR at a directory holding the table."
return 1
}
echo -e "${GREEN}=== RAGFlow Go Server Build Script ===${NC}"
# Function to print section headers
print_section() {
echo -e "\n${YELLOW}>>> $1${NC}"
}
# Detect the package-install command for pcre2 development files.
# Outputs the command on stdout; empty string if no supported manager is found.
detect_pcre2_install_cmd() {
if [ "$(uname)" = "Darwin" ]; then
echo "brew install pcre2"
elif command -v apt-get >/dev/null 2>&1; then
echo "sudo apt-get install -y libpcre2-dev"
elif command -v zypper >/dev/null 2>&1; then
echo "sudo zypper install -y pcre2-devel"
elif command -v dnf >/dev/null 2>&1; then
echo "sudo dnf install -y pcre2-devel"
elif command -v pacman >/dev/null 2>&1; then
echo "sudo pacman -S --noconfirm pcre2"
else
echo ""
fi
}
# Check whether libpcre2-8 is available (static or shared).
check_pcre2() {
# Prefer pkg-config when available — works across distros.
if command -v pkg-config >/dev/null 2>&1 && pkg-config --exists libpcre2-8; then
return 0
fi
# Fall back to known library paths:
# Debian/Ubuntu -> /usr/lib/x86_64-linux-gnu
# openSUSE/RHEL -> /usr/lib64
# generic Linux -> /usr/lib, /usr/local/lib
# macOS Homebrew -> /opt/homebrew/lib (Apple Silicon), /usr/local/lib (Intel)
for p in \
/usr/lib/x86_64-linux-gnu/libpcre2-8.a \
/usr/lib/x86_64-linux-gnu/libpcre2-8.so \
/usr/lib64/libpcre2-8.a \
/usr/lib64/libpcre2-8.so \
/usr/lib/libpcre2-8.a \
/usr/lib/libpcre2-8.so \
/usr/local/lib/libpcre2-8.a \
/usr/local/lib/libpcre2-8.so \
/usr/local/lib/libpcre2-8.dylib \
/opt/homebrew/lib/libpcre2-8.a \
/opt/homebrew/lib/libpcre2-8.dylib; do
[ -f "$p" ] && return 0
done
return 1
}
# Check dependencies
check_cpp_deps() {
print_section "Checking c++ dependencies"
command -v cmake >/dev/null 2>&1 || { echo -e "${RED}Error: cmake is required but not installed.${NC}"; exit 1; }
command -v clang++ >/dev/null 2>&1 || { echo -e "${RED}Error: clang++ is required but not installed.${NC}"; exit 1; }
if check_pcre2; then
echo "✓ pcre2 library found"
else
install_cmd="$(detect_pcre2_install_cmd)"
echo -e "${YELLOW}Warning: libpcre2-8 not found. You may need to install it:${NC}"
if [ -n "$install_cmd" ]; then
echo " $install_cmd"
else
echo " (No supported package manager detected — install pcre2 development files manually)"
fi
fi
echo "✓ Required tools are available"
}
check_go_deps() {
print_section "Checking go dependencies"
command -v go >/dev/null 2>&1 || { echo -e "${RED}Error: go is required but not installed.${NC}"; exit 1; }
echo "✓ Required tools are available"
check_ort_version_consistency
}
# Fail fast when the ONNX Runtime native version is declared inconsistently.
# The in-process (Go) DeepDoc backend statically links libonnxruntime.a built
# from ONE exact ORT release; a mismatch links a wrong/missing .a and only fails
# at runtime (dlopen(NULL) can't find OrtGetApiBase). The version is pinned in
# three Go-side locations that must all agree.
check_ort_version_consistency() {
print_section "Checking ONNX Runtime version consistency"
local env_go deps dockerfile
env_go="$(grep -m1 -E 'DeepDocORTVersion[[:space:]]*=[[:space:]]*"' "${PROJECT_ROOT}/internal/common/environments.go" | sed -E 's/.*"([^"]+)".*/\1/')"
deps="$(grep -m1 -E '^ORT_VERSION[[:space:]]*=[[:space:]]*"' "${PROJECT_ROOT}/ragflow_deps/download_deps.py" | sed -E 's/.*"([^"]+)".*/\1/')"
dockerfile="$(grep -m1 -E 'ARG[[:space:]]+ORT_VERSION=' "${PROJECT_ROOT}/Dockerfile" | sed -E 's/.*ORT_VERSION=([0-9][^"[:space:]]*).*/\1/')"
if [ -z "$env_go" ] || [ -z "$deps" ] || [ -z "$dockerfile" ]; then
echo -e "${RED}Error: could not parse the ONNX Runtime version from one of the pinned locations${NC}" >&2
exit 1
fi
if [ "$env_go" != "$deps" ] || [ "$env_go" != "$dockerfile" ]; then
echo -e "${RED}Error: ONNX Runtime version is inconsistent — fix before building:${NC}" >&2
printf ' %-10s %s\n' "$env_go" "internal/common/environments.go:DeepDocORTVersion"
printf ' %-10s %s\n' "$deps" "ragflow_deps/download_deps.py:ORT_VERSION"
printf ' %-10s %s\n' "$dockerfile" "dockerfile:ARG ORT_VERSION"
exit 1
fi
echo -e "${GREEN}✓ ONNX Runtime native version consistent: ${env_go}${NC}"
}
# Check the ONNX Runtime static archive the Go DeepDoc backend links.
#
# This mirrors check_office_oxide_deps: it verifies the on-disk archive
# actually matches the released artifact, so a stale/incompatible .a — one
# re-issued under the SAME release tag and asset name, or an older download —
# fails fast HERE with an actionable message, instead of producing a binary
# that links fine but crashes at runtime when the shared-initializer feature
# (SessionGetInitializer*, added by infiniflow/ragflow-build) runs.
#
# The onnxruntime_go binding reaches ORT purely through the OrtApi
# function-pointer table (onnxruntime_go/shared_initializer.go ->
# onnxruntime_wrapper.c -> ort_api->SessionGetInitializer*), so a stale .a
# that lacks those custom slots links successfully and only fails at runtime.
# The check below catches that by comparing the local release ZIP against the
# published {asset}.sha256 sidecar (same source of truth the download scripts
# use). Devs who ran a download script have the zip under ragflow_deps/; CI
# seeds from /opt and has no zip, where the bake is authoritative.
check_onnxruntime_deps() {
local ort_version
ort_version="$(grep -m1 -E 'DeepDocORTVersion[[:space:]]*=[[:space:]]*"' \
"${PROJECT_ROOT}/internal/common/environments.go" \
| sed -E 's/.*"([^"]+)".*/\1/')"
if [ -z "$ort_version" ]; then
echo -e "${RED}Error: cannot parse DeepDocORTVersion from internal/common/environments.go${NC}" >&2
return 1
fi
# Locate the ORT static archives under the version dir. Populate the
# globals ORT_A_FILES (space-separated .a paths) and ORT_A_SHA256 (sha256
# of the primary libonnxruntime.a) for the caller's version-stamped
# cache path.
ORT_A_FILES=""
local primary=""
local f
while IFS= read -r f; do
case "$(basename "$f")" in
*cuda*|*tensorrt*|*coreml*|*dml*|*migraphx*) continue ;;
esac
case "$f" in
*/onnxruntime-linux-x64-static_lib-"${ort_version}"*/lib/*.a)
ORT_A_FILES="$ORT_A_FILES $f"
[ -z "$primary" ] && primary="$f"
case "$(basename "$f")" in
libonnxruntime.a) primary="$f" ;;
esac
;;
esac
done < <(find "$ONNXRUNTIME_STATIC_PREFIX" -type f -name '*.a' 2>/dev/null)
if [ -z "$ORT_A_FILES" ]; then
local avail
avail="$(find "$ONNXRUNTIME_STATIC_PREFIX" -maxdepth 1 -type d \
-name 'onnxruntime-linux-x64-static_lib-*' -exec basename {} \; 2>/dev/null | tr '\n' ' ')"
echo -e "${RED}Error: no ONNX Runtime ${ort_version} static lib under $ONNXRUNTIME_STATIC_PREFIX${NC}" >&2
echo " available: ${avail:-<none>}" >&2
echo " DeepDocORTVersion=${ort_version}; fetch it with:" >&2
echo " uv run python3 ragflow_deps/download_deps.py" >&2
echo " or pre-seed /opt/ragflow-native-libs/onnxruntime (CI image)." >&2
return 1
fi
# Pinned sha256 of the published onnxruntime-v${ort_version} archive.
#
# Upstream does not re-issue this archive, so we pin the digest here and
# verify the LOCAL copy against it — no network access at build time. This
# replaces the previous check that fetched the .sha256 sidecar over the
# network on every build, which hung the build when the machine was offline
# or GitHub was unreachable (curl had no connect timeout).
#
# A mismatch means the local copy is corrupt or stale; we only WARN (not
# fail) so a one-off re-issue cannot break the build. The downloader
# ragflow_deps/download_deps.py still verifies against
# the published sidecar at download time, so a re-issued archive is picked
# up on the next download.
local asset="onnxruntime-v${ort_version}-linux-x86_64.zip"
local zip_path="${PROJECT_ROOT}/ragflow_deps/${asset}"
local expected="439308c93822d26cf04341cab3e77c595bbe88a12b54043210d14a0ca5574bd1"
if [ -f "$zip_path" ]; then
local actual
actual="$(sha256sum "$zip_path" | awk '{print $1}')"
if [ "$actual" != "$expected" ]; then
echo -e "${YELLOW}Warning: ONNX Runtime archive ${asset} sha256 mismatch${NC}" >&2
echo " expected sha256: $expected" >&2
echo " local sha256: $actual" >&2
echo " The local copy may be corrupt or stale. Refresh it with:" >&2
echo " rm -f ${zip_path}" >&2
echo " uv run python3 ragflow_deps/download_deps.py" >&2
else
echo " ✓ ${asset} sha256 matches pinned digest"
fi
else
echo " (no local ${asset}; skipping sha256 check — CI seed assumed authoritative)"
fi
# Content hash of the primary archive for the version-stamped cache path.
ORT_A_SHA256="$(sha256sum "$primary" | awk '{print $1}')"
echo "✓ onnxruntime v${ort_version} static lib verified under $ONNXRUNTIME_STATIC_PREFIX"
}
# Check office_oxide native library
check_office_oxide_deps() {
print_section "Checking office_oxide native library"
_seed_from_system "office_oxide" || true
local lib_file="liboffice_oxide.a"
local lib_path="${OFFICE_OXIDE_PREFIX}/lib/${lib_file}"
local header_path="${OFFICE_OXIDE_PREFIX}/include/office_oxide_c/office_oxide.h"
if [ ! -f "$lib_path" ] || [ ! -f "$header_path" ]; then
echo -e "${RED}Error: office_oxide native library not found${NC}"
echo " Expected: ${lib_path}"
echo " Run: uv run python3 ragflow_deps/download_deps.py"
echo " Or manually download: https://github.com/yfedoseev/office_oxide/releases/download/v${OFFICE_OXIDE_VERSION}/native-linux-x86_64.tar.gz"
exit 1
fi
# Verify the on-disk lib matches the pinned version. A stale older lib
# (e.g. v0.1.7) silently reintroduces the PPT97 content-loss bug
# (github.com/yfedoseev/office_oxide#85, fixed in v0.1.8): PlainText()
# on a legacy .ppt returns stale metadata instead of slide text.
if ! strings "$lib_path" 2>/dev/null | grep -Fxq "$OFFICE_OXIDE_VERSION"; then
local found_version
found_version=$(strings "$lib_path" 2>/dev/null | grep -E "^0\.[0-9]+\.[0-9]+$" | head -1)
echo -e "${YELLOW}office_oxide native lib version mismatch (required v${OFFICE_OXIDE_VERSION}; found: ${found_version:-unknown}); refreshing from download${NC}"
rm -rf "${OFFICE_OXIDE_PREFIX}" "${PROJECT_ROOT}/office_oxide-linux-x86_64.tar.gz"
if ! (cd "${PROJECT_ROOT}" && uv run python3 ragflow_deps/download_deps.py); then
echo -e "${RED}Error: office_oxide native lib version mismatch${NC}"
echo " Required: v${OFFICE_OXIDE_VERSION}; found: ${found_version:-unknown}"
echo " A stale lib silently loses PPT97 (.ppt) slide content. Refresh:"
echo " rm -rf ~/ragflow-native-libs/office_oxide ragflow_deps/office_oxide-linux-x86_64.tar.gz"
echo " uv run python3 ragflow_deps/download_deps.py"
exit 1
fi
lib_path="${OFFICE_OXIDE_PREFIX}/lib/${lib_file}"
header_path="${OFFICE_OXIDE_PREFIX}/include/office_oxide_c/office_oxide.h"
if [ ! -f "$lib_path" ] || [ ! -f "$header_path" ]; then
echo -e "${RED}Error: office_oxide native library not found after refresh${NC}"
exit 1
fi
if ! strings "$lib_path" 2>/dev/null | grep -Fxq "$OFFICE_OXIDE_VERSION"; then
found_version=$(strings "$lib_path" 2>/dev/null | grep -E "^0\.[0-9]+\.[0-9]+$" | head -1)
echo -e "${RED}Error: office_oxide native lib version mismatch after refresh${NC}"
echo " Required: v${OFFICE_OXIDE_VERSION}; found: ${found_version:-unknown}"
exit 1
fi
fi
echo "✓ office_oxide v${OFFICE_OXIDE_VERSION} native library found at ${OFFICE_OXIDE_PREFIX}"
return 0
}
# Check pdfium static library.
check_pdfium_deps() {
_seed_from_system "pdfium-static" || true
local lib_path="${PDFIUM_STATIC_PREFIX}/lib/libpdfium.a"
if [ -f "$lib_path" ]; then
echo " pdfium (static) → ${PDFIUM_STATIC_PREFIX}"
return 0
fi
echo " pdfium (static) not found"
echo " Expected: ${lib_path}"
echo " Run: uv run python3 ragflow_deps/download_deps.py"
echo " Or: curl -fsSL https://github.com/kognitos/pdfium-static/releases/download/chromium%2F${PDFIUM_STATIC_VERSION}/pdfium-linux-x64-static.tgz | tar xz -C ${PDFIUM_STATIC_PREFIX}"
return 1
}
# Check pdf_oxide static library.
# pdf_oxide_validate_version <pool_text> <required_version>
#
# Validates the pdf_oxide version embedded in a lib's string constant pool
# against the pinned <required_version>. The marker "pdf_oxide <version>" is
# merged into Rust's constant pool and has no fixed length, so a suffixed build
# (e.g. 0.3.73.1) and a clean 0.3.73 whose next pooled constant begins with a
# digit are byte-identical and cannot be told apart.
#
# Capture exactly three dotted segments and reject any trailing [0-9.] as
# ambiguous, rather than silently truncating it into a (wrong) match.
#
# Returns:
# 0 exact match (prints the found version)
# 1 version missing or mismatched (prints the found version, or empty)
# 2 ambiguous: the marker is followed by extra digits/dots in the pool
pdf_oxide_validate_version() {
local pool_text="$1" required="$2"
local found esc
found=$(printf '%s\n' "$pool_text" \
| grep -oE "pdf_oxide [0-9]+\.[0-9]+\.[0-9]+" | head -1 | cut -d' ' -f2)
if [ -z "$found" ]; then
return 1
fi
# A digit or dot immediately after the captured "pdf_oxide X.Y.Z" marker
# means a suffixed build (0.3.73.1) or a constant concatenated on to the
# version. Treat it as ambiguous rather than as a wrong version.
# Anchor to the exact captured version (dots escaped) so the third segment
# cannot backtrack and swallow the trailing digit, which would otherwise
# flag a clean 0.3.73<letter> marker as ambiguous.
esc="${found//./\\.}"
if printf '%s\n' "$pool_text" | grep -qE "pdf_oxide ${esc}[0-9.]"; then
return 2
fi
if [ "$found" != "$required" ]; then
printf '%s\n' "$found"
return 1
fi
printf '%s\n' "$found"
return 0
}
check_pdf_oxide_deps() {
_seed_from_system "pdf_oxide" || true
# Map platform to tarball-internal subdirectory.
local platform_subdir
case "$(uname -s)" in
Linux)
case "$(uname -m)" in
x86_64) platform_subdir="linux_amd64" ;;
aarch64|arm64) platform_subdir="linux_arm64" ;;
*) echo " pdf_oxide (static) → unsupported arch"; return 1 ;;
esac
;;
Darwin)
case "$(uname -m)" in
x86_64) platform_subdir="darwin_amd64" ;;
arm64) platform_subdir="darwin_arm64" ;;
*) echo " pdf_oxide (static) → unsupported arch"; return 1 ;;
esac
;;
*) echo " pdf_oxide (static) → unsupported OS"; return 1 ;;
esac
local lib_path="${PDF_OXIDE_PREFIX}/lib/${platform_subdir}/libpdf_oxide.a"
if [ -f "$lib_path" ]; then
# Verify the on-disk lib matches the pinned version. _seed_from_system
# accepts an existing user-cache directory without inspecting it, so a
# lib left over from an earlier pin is reused silently and the upgrade
# becomes a no-op.
#
# The version marker "pdf_oxide <version>" is merged into Rust's string
# constant pool (unlike office_oxide's standalone "0.1.9" line), so we
# capture exactly three dotted segments and reject any trailing [0-9.]
# as ambiguous — a suffixed build (0.3.73.1) or a constant that is
# byte-identical to one must not be silently accepted. The "pdf_oxide "
# prefix keeps bare version numbers of vendored dependencies out of the
# match.
local pool_text found_version rc
pool_text=$(strings "$lib_path" 2>/dev/null)
found_version=$(pdf_oxide_validate_version "$pool_text" "$PDF_OXIDE_VERSION"); rc=$?
case "$rc" in
0)
echo " pdf_oxide (static) → ${PDF_OXIDE_PREFIX}"
return 0
;;
2)
echo -e "${RED}Error: pdf_oxide native lib version ambiguous${NC}"
echo " The version marker is followed by extra digits/dots in the"
echo " string pool; this is usually a suffixed build (e.g. ${PDF_OXIDE_VERSION}.1)"
echo " or a concatenated constant byte-identical to one. Refresh the lib"
echo " to a clean pin."
echo " Required: v${PDF_OXIDE_VERSION}"
return 1
;;
*)
echo -e "${RED}Error: pdf_oxide native lib version mismatch${NC}"
echo " Required: v${PDF_OXIDE_VERSION}; found: ${found_version:-unknown}"
echo " A stale lib silently reverts PDF parsing fixes. Refresh:"
echo " rm -rf ${PDF_OXIDE_PREFIX} ragflow_deps/pdf_oxide-go-ffi-linux-amd64.tar.gz"
echo " uv run python3 ragflow_deps/download_deps.py"
return 1
;;
esac
fi
echo " pdf_oxide (static) not found"
echo " Expected: ${lib_path}"
echo " Run: uv run python3 ragflow_deps/download_deps.py"
echo " Or: curl -fsSL https://github.com/yfedoseev/pdf_oxide/releases/download/v${PDF_OXIDE_VERSION}/pdf_oxide-go-ffi-linux-amd64.tar.gz | tar xz -C ${PDF_OXIDE_PREFIX}"
return 1
}
# Build C++ static library
build_cpp() {
print_section "Building C++ static library"
mkdir -p "$BUILD_DIR"
cd "$BUILD_DIR"
echo "Running cmake..."
cmake .. -DCMAKE_BUILD_TYPE=Release
echo "Building librag_tokenizer_c_api.a..."
local jobs
jobs="$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 1)"
make rag_tokenizer_c_api -j"$jobs"
if [ ! -f "$BUILD_DIR/librag_tokenizer_c_api.a" ]; then
echo -e "${RED}Error: Failed to build C++ static library${NC}"
exit 1
fi
# Rename the tokenizer's bundled re2 symbols into a private namespace so they
# never collide with onnxruntime's copy of re2 at final link time.
#
# Why this is needed: onnxruntime.a and librag_tokenizer_c_api.a both embed a
# copy of re2, but they are built against different toolchains/libstdc++ and
# are ABI-incompatible. Symbol-localization approaches (--exclude-libs, linker
# version scripts) cannot fix it: both re2 copies must live in the same global
# symbol table, and the linker resolves the tokenizer's re2 references to
# whichever copy was archived first (here, onnxruntime's), regardless of which
# copy is localized. SIGSEGV in re2::DFA::InlinedSearchLoop is the symptom.
#
# Renaming the tokenizer's re2 symbols into a private prefix gives each copy
# its own namespace, so the tokenizer calls its own ABI-compatible re2 and
# onnxruntime keeps calling its own. Verified to eliminate the crash.
local tok_a="$BUILD_DIR/librag_tokenizer_c_api.a"
local rename_map="$BUILD_DIR/re2_rename.map"
if command -v objcopy >/dev/null 2>&1; then
nm "$tok_a" \
| awk '$2 ~ /^[TDBRWtdbrwiIVv]$/ && $3 ~ /^_ZN3re2|_ZNK3re2|_ZTVN3re2|_ZTIN3re2|_ZTSN3re2/ { print $3" ragtokre2_"$3 }' \
| sort -u > "$rename_map"
if [ -s "$rename_map" ]; then
objcopy --redefine-syms="$rename_map" "$tok_a"
echo -e "${GREEN}✓ Renamed $(wc -l < "$rename_map") tokenizer re2 symbols into private namespace (ragtokre2_)${NC}"
fi
else
echo -e "${YELLOW}Warning: objcopy not found, skipping re2 symbol rename (re2 collision with onnxruntime may cause SIGSEGV)${NC}"
fi
echo -e "${GREEN}✓ C++ static library built successfully${NC}"
}
# Build C++ test executable
build_cpp_test() {
print_section "Building C++ test executable"
if [ ! -d "$BUILD_DIR" ]; then
echo "Build directory not found, running cmake first..."
mkdir -p "$BUILD_DIR"
cd "$BUILD_DIR"
cmake .. -DCMAKE_BUILD_TYPE=Release
else
cd "$BUILD_DIR"
fi
echo "Building rag_analyzer_c_test..."
local jobs
jobs="$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 1)"
make rag_analyzer_c_test -j"$jobs"
if [ ! -f "$BUILD_DIR/rag_analyzer_c_test" ]; then
echo -e "${RED}Error: Failed to build rag_analyzer_c_test${NC}"
exit 1
fi
echo -e "${GREEN}✓ C++ test executable built successfully: $BUILD_DIR/rag_analyzer_c_test${NC}"
}
# Build Go server
build_go() {
print_section "Building RAGFlow go"
cd "$PROJECT_ROOT"
# Check if C++ library exists
if [ ! -f "$BUILD_DIR/librag_tokenizer_c_api.a" ]; then
echo -e "${RED}Error: C++ static library not found. Run with --cpp first.${NC}"
exit 1
fi
if check_pcre2; then
echo "✓ pcre2 library found"
else
install_cmd="$(detect_pcre2_install_cmd)"
if [ -z "$install_cmd" ]; then
echo -e "${RED}Error: libpcre2-8 not found and no supported package manager detected.${NC}"
echo "Please install pcre2 development files manually."
exit 1
fi
if [ "$(uname)" = "Darwin" ]; then
echo -e "${RED}Error: libpcre2-8 not found. Install with: $install_cmd${NC}"
exit 1
fi
echo -e "${YELLOW}Warning: libpcre2-8 not found. Installing with: $install_cmd${NC}"
eval "$install_cmd"
fi
setup_cgo_env
# The in-process (Go) DeepDoc backend is statically linked against ONNX
# Runtime (--undefined=OrtGetApiBase + -Wl,--dynamic-list, see
# setup_cgo_env). The forked onnxruntime_go binding resolves OrtGetApiBase
# only at RUNTIME via dlopen(NULL), so a missing ORT archive still lets
# `go build` SUCCEED and yields a server binary that FAILS AT STARTUP with a
# fatal "no in-process DeepDoc backend serving". That is exactly the
# breakage colleagues hit when ORT was not present at build time and
# setup_cgo_env silently skipped it. Fail the build HERE instead of
# deferring the breakage to runtime: a server binary without ORT is unusable,
# not a degraded one.
if ! printf '%s' "$CGO_LDFLAGS" | grep -q 'libonnxruntime'; then
echo -e "${RED}Error: ONNX Runtime static libraries are not linked.${NC}" >&2
echo " The in-process DeepDoc backend requires libonnxruntime.a to be" >&2
echo " statically linked into ragflow_server (--undefined=OrtGetApiBase +" >&2
echo " -Wl,--dynamic-list). Without it the binary compiles but dies" >&2
echo " at startup with a fatal 'no in-process DeepDoc backend serving'." >&2
echo " Fetch the static libs with:" >&2
echo " uv run python3 ragflow_deps/download_deps.py" >&2
echo " or pre-seed them at /opt/ragflow-native-libs/onnxruntime (CI image)." >&2
echo " This production binary must statically include ORT — there is no" >&2
echo " ORT-free build path. ORT ends up unlinked only via one of:" >&2
echo " - the ORT static_lib dir was never seeded: run" >&2
echo " 'uv run python3 ragflow_deps/download_deps.py', or pre-seed" >&2
echo " /opt/ragflow-native-libs/onnxruntime as the CI image does;" >&2
echo " - setup_cgo_env did not add libonnxruntime to CGO_LDFLAGS." >&2
return 1
fi
local strip_flags=()
[ -n "$STRIP_SYMBOLS" ] && strip_flags=(-ldflags="-s -w")
echo "Building RAGFlow binary: $RAGFLOW_CLI_BINARY, $RAGFLOW_SERVER_BINARY"
set -x
GOPROXY=${GOPROXY:-https://goproxy.cn,https://proxy.golang.org,direct} CGO_ENABLED=1 \
go build -tags cgo,static,sonic "${strip_flags[@]}" -o "$RAGFLOW_CLI_BINARY" cmd/ragflow-cli.go
GOPROXY=${GOPROXY:-https://goproxy.cn,https://proxy.golang.org,direct} CGO_ENABLED=1 \
CGO_CFLAGS="$CGO_CFLAGS" CGO_LDFLAGS="$CGO_LDFLAGS" \
go build -tags cgo,static,sonic "${strip_flags[@]}" -o "$RAGFLOW_SERVER_BINARY" \
cmd/ragflow_server.go cmd/deepdoc_server_ee.go
set +x
if [ ! -f "$RAGFLOW_SERVER_BINARY" ]; then
echo -e "${RED}Error: Failed to build RAGFlow main binary${NC}"
exit 1
fi
echo -e "${GREEN}✓ Go ragflow-cli built successfully: $RAGFLOW_CLI_BINARY${NC}"
echo -e "${GREEN}✓ Go ragflow_server built successfully: $RAGFLOW_SERVER_BINARY${NC}"
}
# Configure CGO flags for native libraries (office_oxide, pdfium, pdf_oxide).
# All three are statically linked — no LD_LIBRARY_PATH or -Wl,-rpath needed.
setup_cgo_env() {
# ── office_oxide ──────────────────────────────────────────────────
check_office_oxide_deps
# Go's build cache keys CGO_LDFLAGS as a string — it does NOT hash the
# file content of referenced .a archives. So swapping the .a in-place
# (same path) does NOT invalidate the cache, and `go build` silently
# reuses a stale binary linked against the old .a.
#
# Create a version-stamped symlink directory so the flag string
# includes the actual linked version. When the .a is upgraded, the
# path changes → Go cache key changes → automatic relink.
local office_oxide_lib_dir="${OFFICE_OXIDE_PREFIX}/lib"
local versioned_lib_dir="${office_oxide_lib_dir}/v${OFFICE_OXIDE_VERSION}"
mkdir -p "$versioned_lib_dir"
ln -sf "${office_oxide_lib_dir}/liboffice_oxide.a" \
"${versioned_lib_dir}/liboffice_oxide.a"
export CGO_CFLAGS="-I${OFFICE_OXIDE_PREFIX}/include/office_oxide_c${CGO_CFLAGS:+ $CGO_CFLAGS}"
export CGO_LDFLAGS="${versioned_lib_dir}/liboffice_oxide.a"
# ── pdfium ────────────────────────────────────────────────────────
check_pdfium_deps || return 1
export CGO_LDFLAGS="$CGO_LDFLAGS ${PDFIUM_STATIC_PREFIX}/lib/libpdfium.a"
# Linux: Chromium-built objects use Clang's .eh_frame format which GNU ld
# cannot merge. Use lld (LLVM linker) which handles them correctly.
# --allow-multiple-definition: pdf_oxide and office_oxide are both Rust
# staticlibs that embed the Rust runtime; linking them together produces
# duplicate rust_eh_personality / compiler-rt builtins. Those duplicates are
# ABI-IDENTICAL, so merging them is benign.
if [ "$(uname -s)" = "Linux" ]; then
if ! command -v ld.lld >/dev/null 2>&1; then
echo -e "${RED}Error: ld.lld not found. Install with: sudo apt install lld-20 && sudo ln -s /usr/bin/ld.lld-20 /usr/bin/ld.lld${NC}"
echo " lld is required to static-link Chromium-built pdfium (.eh_frame format)"
return 1
fi
export CGO_LDFLAGS="$CGO_LDFLAGS \
${PDFIUM_STATIC_PREFIX}/lib/libc++.a \
${PDFIUM_STATIC_PREFIX}/lib/libc++abi.a \
-fuse-ld=lld -Wl,--allow-multiple-definition"
# The re2 regex-library collision between onnxruntime.a and
# librag_tokenizer_c_api.a is fixed at the .a level in build_cpp():
# the tokenizer's bundled re2 symbols are renamed into a private
# namespace (ragtokre2_) so the two re2 copies never share a symbol.
# See build_cpp() for details.
fi
# ── pdf_oxide ─────────────────────────────────────────────────────
check_pdf_oxide_deps || return 1
# The go-ffi tarball places the .a under lib/<platform_subdir>/.
local pdf_oxide_subdir
case "$(uname -s)" in
Linux)
case "$(uname -m)" in
x86_64) pdf_oxide_subdir="linux_amd64" ;;
aarch64|arm64) pdf_oxide_subdir="linux_arm64" ;;
*) echo "pdf_oxide: unsupported arch"; return 1 ;;
esac
;;
Darwin)
case "$(uname -m)" in
x86_64) pdf_oxide_subdir="darwin_amd64" ;;
arm64) pdf_oxide_subdir="darwin_arm64" ;;
*) echo "pdf_oxide: unsupported arch"; return 1 ;;
esac
;;
esac
# Version-stamp the archive path so an in-place .a upgrade invalidates
# Go's build cache. See the office_oxide block above for why the raw path
# is not enough: the cache key hashes CGO_LDFLAGS as a string, not the
# contents of the archives it names.
local pdf_oxide_lib_dir="${PDF_OXIDE_PREFIX}/lib/${pdf_oxide_subdir}"
local pdf_oxide_versioned_dir="${pdf_oxide_lib_dir}/v${PDF_OXIDE_VERSION}"
mkdir -p "$pdf_oxide_versioned_dir"
ln -sf "${pdf_oxide_lib_dir}/libpdf_oxide.a" \
"${pdf_oxide_versioned_dir}/libpdf_oxide.a"
export CGO_LDFLAGS="$CGO_LDFLAGS ${pdf_oxide_versioned_dir}/libpdf_oxide.a"
# ── onnxruntime (static, resolved via dlopen(NULL)) ────────────────
# macOS native builds of the in-process DeepDoc backend are not supported:
# ONNX Runtime is statically linked with GNU ld flags
# (-Wl,--undefined=OrtGetApiBase + -Wl,--dynamic-list) and resolved at
# runtime via dlopen(NULL); Apple's ld64 does not understand these flags.
# Build on Linux or cross-compile there.
case "$(uname -s)" in
Darwin)
echo "Error: macOS native build of the in-process DeepDoc backend is not supported." >&2
echo " ONNX Runtime is linked with GNU ld flags (-Wl,--undefined=OrtGetApiBase + -Wl,--dynamic-list)" >&2
echo " and resolved via dlopen(NULL); Apple's ld64 does not support them. Build on Linux." >&2
return 1
;;
esac
# Statically link libonnxruntime*.a into the binary. The org Go binding
# (onnxruntime_go, github.com/infiniflow/onnxruntime_go) resolves OrtGetApiBase
# with dlopen(NULL) + dlsym(handle, "OrtGetApiBase"), so the ONLY symbol that
# must be visible in the process-global table is OrtGetApiBase.
#
# We therefore do NOT use --whole-archive: that flag force-pulls every .o in
# the archive (including unregistered "dead" kernels we never call), which
# is why every ORT size-trimming build flag had near-zero effect before.
# Instead we link the archives normally and let GNU ld's archive-level GC
# drop any kernel/EP object that nothing references. The onnxruntime_go
# binding reaches ORT purely through the OrtApi function-pointer table
# returned by OrtGetApiBase, and a minimal/reduced-ops build registers only
# the operators the deepdoc models actually use, so the reachable closure is
# small. --dynamic-list exports just OrtGetApiBase (the only symbol dlopen
# needs) and leaves Go's runtime symbols untouched, unlike a "local: *"
# version script which breaks PIE absolute relocations. No libonnxruntime.so
# is required or supported at runtime; there is no dynamic .so fallback.
#
# Seed the static ORT archives from the system pre-bake (/opt, laid down
# by the CI runner image) into the user cache before the link check
# below. Mirrors the existing _seed_from_system calls for the other
# native libs so CI never downloads ORT at build/test time.
_seed_from_system "onnxruntime" || true
check_onnxruntime_deps || return 1
# Version-stamp the archive paths so an in-place .a upgrade invalidates
# Go's build cache. Go's cache keys CGO_LDFLAGS as a string and does NOT
# hash the referenced .a contents, so swapping the .a in place (same path,
# e.g. a re-issue under the same tag/asset name) silently reuses a stale
# linked binary. Stamp with the primary archive's sha256 so the flag string
# changes when the content changes → automatic relink.
local ort_version
ort_version="$(grep -m1 -E 'DeepDocORTVersion[[:space:]]*=[[:space:]]*"' \
"${PROJECT_ROOT}/internal/common/environments.go" \
| sed -E 's/.*"([^"]+)".*/\1/')"
local ort_versioned_dir="${ONNXRUNTIME_STATIC_PREFIX}/v${ort_version}-${ORT_A_SHA256:0:16}"
mkdir -p "$ort_versioned_dir"
local ort_a_versioned=""
local f
for f in $ORT_A_FILES; do
ln -sf "$f" "$ort_versioned_dir/$(basename "$f")"
ort_a_versioned="$ort_a_versioned $ort_versioned_dir/$(basename "$f")"
done
# Export exactly one symbol (OrtGetApiBase) for the binding's
# dlopen(NULL)+dlsym lookup via --dynamic-list (NOT --version-script
# with "local: *", which hides Go's runtime type symbols and breaks
# the PIE absolute relocations). No --whole-archive, so GNU ld's
# archive-level GC drops any ORT kernel/EP object nothing references.
#
# The dynamic list is written to a STABLE, project-scoped path (not
# mktemp) so CGO_LDFLAGS is reproducible across builds and survives
# across invocations; its content never changes, so overwriting is
# safe and nothing leaks in /tmp. .cache/ is gitignored.
local ort_dynamic_list="${PROJECT_ROOT}/.cache/ort_dynamic_list.txt"
mkdir -p "$(dirname "$ort_dynamic_list")"
printf '{\n OrtGetApiBase;\n};\n' > "$ort_dynamic_list"
# --undefined=OrtGetApiBase force-pulls the archive member that
# defines OrtGetApiBase (the Go binding reaches ORT only via
# dlsym("OrtGetApiBase"), so nothing references it at link time and
# it would otherwise be GC'd). From there the minimal build's CPU-EP
# registration call chain pulls in the operators the models use.
# The explicit space between $ort_dynamic_list and the versioned .a
# list keeps the two as separate linker arguments regardless of the
# list's leading space.
export CGO_LDFLAGS="$CGO_LDFLAGS -Wl,--undefined=OrtGetApiBase -Wl,--dynamic-list=$ort_dynamic_list $ort_a_versioned -lstdc++"
echo " onnxruntime (static) → $ONNXRUNTIME_STATIC_PREFIX"
# The re2 regex-library collision between onnxruntime.a and
# librag_tokenizer_c_api.a is fixed at the .a level in build_cpp():
# the tokenizer's bundled re2 symbols are renamed into a private
# namespace (ragtokre2_) so the two re2 copies never share a symbol
# name. --dynamic-list (above) exports OrtGetApiBase into the process
# dynamic symbol table, which is what the binding's dlopen(NULL)+dlsym
# lookup needs at runtime (no --export-dynamic required).
# ── platform-specific system libraries ────────────────────────────
case "$(uname -s)" in
Linux)
export CGO_LDFLAGS="$CGO_LDFLAGS -lm -lpthread -ldl -lrt -lgcc_s -lutil -lc"
;;
Darwin)
export CGO_LDFLAGS="$CGO_LDFLAGS \
-framework CoreGraphics -framework CoreFoundation \
-framework Security -framework SystemConfiguration \
-liconv -lresolv -lc++"
;;
esac
echo "CGO_CFLAGS: $CGO_CFLAGS"
echo "CGO_LDFLAGS: $CGO_LDFLAGS"
}
# Run Go unit tests with the same CGO env as `build_go`. Any extra args are
# forwarded to `go test`, e.g. `./build.sh --test -run TestFoo ./internal/admin/...`.
run_go_tests() {
print_section "Running Go tests"
# The tokenizer panics instead of counting 0 when the cl100k table is
# missing, so provision it before the tests, as CI does. Non-fatal: without
# it only the token-counting tests fail, and they say why.
_ensure_cl100k_table || true
cd "$PROJECT_ROOT"
setup_cgo_env
if [ "$#" -eq 0 ]; then
set -- ./...
fi
GOPROXY=${GOPROXY:-https://goproxy.cn,https://proxy.golang.org,direct} CGO_ENABLED=1 \
CGO_CFLAGS="$CGO_CFLAGS" CGO_LDFLAGS="$CGO_LDFLAGS" \
go test -tags cgo,static -count=1 "$@"
run_native_tests
}
# Run the unit tests of the native package (DeepDoc det/DLA/TSR/OCR-rec Go
# ports), now a regular package under the MAIN module gated by the `cgo` build
# tag (same isolation as office_oxide/pdfium). It used to be a nested Go module
# (own go.mod) that the root `./...` never descended into; after the merge it is
# covered by the normal module tests. The build is pure-Go geometry plus the
# cgo onnxruntime binding, so it runs whenever CGO_ENABLED=1 (the same gate the
# rest of the native C libs use). Model-backed integration tests are handled by
# run_native_integration_tests.
run_native_tests() {
print_section "Running native unit tests"
( cd "$PROJECT_ROOT" && \
GOPROXY=${GOPROXY:-https://goproxy.cn,https://proxy.golang.org,direct} \
CGO_ENABLED=1 \
go test -tags cgo,static -count=1 ./internal/deepdoc/native/... )
}
# Run the model-backed integration tests of the native package and the
# native_analyzer package (the DocAnalyzer the PDF parser consumes). These
# require MODEL_DIR at runtime; ONNX Runtime is statically linked and resolved
# via dlopen(NULL), so no ORT_LIB is needed. The tests self-skip when MODEL_DIR
# is unset or the binary lacks static ORT (see native_integration_test.go
# skipIfNoModels / native_analyzer_test.go analyzerWithModels). Run under the
# `cgo integration` tier.
#
# Why -race is split instead of applied to the whole suite:
# - The DLA/TSR/OCR-rec/Det golden-comparison tests are single-threaded and
# deterministic. -race multiplies their (model-heavy) heap ~10x for ZERO
# race-detection value, and that is what used to OOM-kill the runner
# (SIGTERM) under `cgo integration`. So they run WITHOUT -race.
# - Only the concurrency-correctness tests (TestInferenceConcurrency* in the
# native package, and native_analyzer's TestAnalyzerConcurrentBatchAndSingle)
# actually benefit from -race: they hammer the shared session pools from many
# goroutines, and the detector catches data races on the pools / per-session
# in-out tensors / cross-call batch state. Those run WITH -race.
# ORT's C internals are not instrumented, but all our shared mutable state lives
# in Go, which the detector covers. CGO_ENABLED=1 is required for both the build
# and the race runtime.
run_native_integration_tests() {
# Optional isolation: NATIVE_TEST_RUN forwards a -run filter and
# NATIVE_TEST_V adds -v so a single native test can be exercised in
# isolation (e.g. `NATIVE_TEST_RUN='TestNativeLoadsOrtModels$' NATIVE_TEST_V=1`).
local native_run_filter=()
if [ -n "${NATIVE_TEST_RUN:-}" ]; then
native_run_filter+=(-run "${NATIVE_TEST_RUN}")
fi
if [ -n "${NATIVE_TEST_V:-}" ]; then
native_run_filter+=(-v)
fi
print_section "Running native integration tests (golden/comparison, no race)"
( cd "$PROJECT_ROOT" && \
GOPROXY=${GOPROXY:-https://goproxy.cn,https://proxy.golang.org,direct} \
CGO_ENABLED=1 \
go test -tags "cgo static integration fetch_testdata" -count=1 "${native_run_filter[@]}" ./internal/deepdoc/native/... )
print_section "Running native integration concurrency tests (race detector on)"
( cd "$PROJECT_ROOT" && \
GOPROXY=${GOPROXY:-https://goproxy.cn,https://proxy.golang.org,direct} \
CGO_ENABLED=1 \
go test -tags "cgo static integration fetch_testdata" -race -count=1 \
-run 'TestInferenceConcurrency' ./internal/deepdoc/native/... )
print_section "Running native_analyzer race tests (race detector on)"
( cd "$PROJECT_ROOT" && \
GOPROXY=${GOPROXY:-https://goproxy.cn,https://proxy.golang.org,direct} \
CGO_ENABLED=1 \
go test -tags "cgo static integration fetch_testdata" -race -count=1 \
./internal/deepdoc/parser/pdf/inference/native_analyzer/... )
}
# Run Go tests gated behind a build tag (or space-separated tag list), e.g.
# `./build.sh --test-integration -run TestFoo ./internal/engine/...`.
# See "Go Test Tiers" in AGENTS.md for the tier definitions.
run_go_tests_tagged() {
local tags="$1"; shift
print_section "Running Go tests (tags: ${tags})"
# Same cl100k table requirement as the unit tier (integration/e2e runs count
# tokens too); non-fatal, see _ensure_cl100k_table.
_ensure_cl100k_table || true
cd "$PROJECT_ROOT"
setup_cgo_env
if [ "$#" -eq 0 ]; then
set -- ./...
fi
GOPROXY=${GOPROXY:-https://goproxy.cn,https://proxy.golang.org,direct} CGO_ENABLED=1 \
CGO_CFLAGS="$CGO_CFLAGS" CGO_LDFLAGS="$CGO_LDFLAGS" \
go test -tags "${tags} static" -count=1 "$@"
}
# Clean build artifacts
clean() {
print_section "Cleaning build artifacts"
rm -rf "$BUILD_DIR"
rm -f "$RAGFLOW_SERVER_BINARY"
rm -f "$RAGFLOW_CLI_BINARY"
echo -e "${GREEN}✓ Build artifacts cleaned${NC}"
}
# Run the server
run() {
if [ ! -f "$RAGFLOW_SERVER_BINARY" ]; then
echo -e "${RED}Error: $RAGFLOW_SERVER_BINARY not found. Build first with --all or --go${NC}"
exit 1
fi
cd "$PROJECT_ROOT"
# admin_server must be running before ragflow_server, otherwise ragflow_server's
# heartbeats to admin will error out (see internal/development.md).
print_section "Starting admin server (background)"
"$RAGFLOW_SERVER_BINARY" --admin &
ADMIN_PID=$!
# One trap for both background services: a second `trap ... EXIT INT TERM`
# would replace this one rather than add to it, leaving admin_server holding
# port 9381 after the foreground server exits. INGESTOR_PID is cleared first
# so a value inherited from the environment cannot be signalled during the
# window before the ingestor starts.
INGESTOR_PID=""
trap 'kill "$ADMIN_PID" ${INGESTOR_PID:+"$INGESTOR_PID"} 2>/dev/null || true' EXIT INT TERM
# Give admin_server a moment to bind its listening port (9381) before
# ragflow_server starts sending heartbeats to it.
sleep 1
print_section "Starting ingestor (background)"
"$RAGFLOW_SERVER_BINARY" --ingestor &
INGESTOR_PID=$!
sleep 1
print_section "Starting RAGFlow server (foreground)"
"$RAGFLOW_SERVER_BINARY" --api
}
# Show help
show_help() {
# Quoted delimiter so backticks, `$var`, and `\$` in the help text are
# printed literally instead of being interpreted as command substitution.
cat << 'EOF'
Usage: $0 [OPTIONS]
Build script for RAGFlow Go server with C++ bindings.
OPTIONS:
--all, -a Build everything (C++ library + Go server) [default]
--cpp, -c Build only C++ static library
--cpp-test Build C++ test executable (builds the C++ library if needed)
--go, -g Build only Go server (requires C++ library to be built)
--test, -t Run Go unit tests (no build tag). Sets up the CGO env and
native static libs (office_oxide/pdfium/pdf_oxide) needed to
build (same contract as the Go tier table in AGENTS.md).
Extra args are forwarded to `go test`, e.g.
`$0 --test -run TestFoo ./internal/admin/...`
--test-integration Run Go tests tagged 'integration' (need real services,
e.g. MySQL/MinIO/ES/Infinity/LLM). e.g.
`$0 --test-integration ./internal/engine/...`
--test-integration-go Run ONLY the generic 'integration' tier (MySQL/MinIO/
Redis/NATS/Infinity/ES) without the model-backed native
DeepDoc tests. Use this for the CI integration job, which
brings up the service stack separately from the native
backend job. e.g. `$0 --test-integration-go ./internal/storage/...`
--test-e2e Run Go tests tagged 'e2e' (full-pipeline, heavy).
--test-manual Run Go tests tagged 'manual' (very slow; local opt-in
ONLY, never run in CI).
--test-all Run 'integration' + 'e2e' tests (excludes 'manual').
--test-native Run the in-process (Go) DeepDoc backend tests tagged
'cgo integration' (needs libonnxruntime + the
InfiniFlow/deepdoc model snapshot; self-skip otherwise).
e.g. `$0 --test-native`
--clean, -C Clean all build artifacts
--check-ort-version Verify the ONNX Runtime native version is declared
consistently across all sources (exit 1 on mismatch).
--run, -r Build and run the server
--strip, -s Strip debug symbols from Go binaries (-ldflags="-s -w")
(disabled by default, useful for smaller production binaries)
--help, -h Show this help message
EXAMPLES:
$0 # Build everything
$0 --cpp # Build only C++ library
$0 --go # Build only Go server
$0 --cpp-test # Build C++ test executable
$0 --test # Run all Go tests (unit tier, no build tag)
$0 --test -run TestFoo ./internal/admin/... # Targeted Go tests
$0 --test-integration ./internal/engine/... # integration + native tiers
$0 --test-integration-go ./internal/storage/... # generic integration tier only
$0 --test-e2e # e2e tier
$0 --test-manual # manual tier (very slow)
$0 --test-all # integration + e2e (no manual)
$0 --run # Build and run
$0 --clean # Clean build artifacts
DEPENDENCIES:
- cmake >= 4.0
- go >= 1.26.4
- clang++ with C++20 support
- office_oxide native library (download with: uv run python3 ragflow_deps/download_deps.py)
- lld (Linux only): sudo apt install lld-20 && sudo ln -s /usr/bin/ld.lld-20 /usr/bin/ld.lld
- pcre2 development files
- Debian/Ubuntu: libpcre2-dev
- openSUSE/RHEL/Fedora: pcre2-devel
- macOS (Homebrew): pcre2
EOF
}
# Main function
main() {
# Parse --strip / -s before other arguments
local args=()
for arg in "$@"; do
case "$arg" in
--strip|-s) STRIP_SYMBOLS="1" ;;
*) args+=("$arg") ;;
esac
done
case "${args[0]:-}" in
--cpp|-c)
check_cpp_deps
build_cpp
;;
--cpp-test)
check_cpp_deps
build_cpp_test
;;
--go|-g)
check_go_deps
build_go
;;
--test|-t)
check_go_deps
if [ "${args[1]:-}" = "--" ]; then
run_go_tests "${args[@]:2}"
else
run_go_tests "${args[@]:1}"
fi
;;
--test-integration)
check_go_deps
if [ "${args[1]:-}" = "--" ]; then
run_go_tests_tagged integration "${args[@]:2}"
else
run_go_tests_tagged integration "${args[@]:1}"
fi
run_native_integration_tests
;;
--test-integration-go)
# Runs only the generic `integration` test tier (MySQL/MinIO/Redis/
# NATS/Infinity/ES/etc.) WITHOUT the model-backed native DeepDoc
# tests. The native tests need MODEL_DIR + the InfiniFlow/deepdoc
# snapshot and are covered separately by `--test-native` /
# `ragflow_native_backend` in CI; running them here would also force
# the `fetch_testdata` tag's init-time network fetch of testdata,
# which is undesirable in a generic CI run. Use this target for the
# CI integration job. e.g. `$0 --test-integration-go ./internal/storage/...`
check_go_deps
if [ "${args[1]:-}" = "--" ]; then
run_go_tests_tagged integration "${args[@]:2}"
else
run_go_tests_tagged integration "${args[@]:1}"
fi
;;
--test-e2e)
check_go_deps
if [ "${args[1]:-}" = "--" ]; then
run_go_tests_tagged e2e "${args[@]:2}"
else
run_go_tests_tagged e2e "${args[@]:1}"
fi
;;
--test-manual)
check_go_deps
if [ "${args[1]:-}" = "--" ]; then
run_go_tests_tagged manual "${args[@]:2}"
else
run_go_tests_tagged manual "${args[@]:1}"
fi
;;
--test-all)
check_go_deps
if [ "${args[1]:-}" = "--" ]; then
run_go_tests_tagged "integration e2e" "${args[@]:2}"
else
run_go_tests_tagged "integration e2e" "${args[@]:1}"
fi
;;
--test-native)
check_go_deps
if [ "${args[1]:-}" = "--" ]; then
pkgs=("${args[@]:2}")
else
pkgs=("${args[@]:1}")
fi
if [ "${#pkgs[@]}" -eq 0 ]; then
pkgs=(./internal/deepdoc/parser/pdf/inference/native_analyzer/...)
fi
# The in-process (Go) DeepDoc backend needs the .ort weights. Default
# MODEL_DIR to the canonical repo model dir (internal/rag/res/deepdoc) so a
# local run needs no MODEL_DIR export after `download_deps.py`.
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
export MODEL_DIR="${MODEL_DIR:-$REPO_ROOT/internal/rag/res/deepdoc}"
run_go_tests_tagged "cgo integration" "${pkgs[@]}"
run_native_integration_tests
;;
--clean|-C)
clean
;;
--check-ort-version)
check_ort_version_consistency
;;
--run|-r)
check_cpp_deps
check_go_deps
build_cpp
build_go
run
;;
--help|-h)
show_help
;;
--all|-a|"")
check_cpp_deps
check_go_deps
build_cpp
build_go
echo -e "\n${GREEN}=== Build completed successfully! ===${NC}"
echo "Binary: $RAGFLOW_SERVER_BINARY, $RAGFLOW_CLI_BINARY"
;;
*)
echo -e "${RED}Unknown option: ${args[0]}${NC}"
show_help
exit 1
;;
esac
}
# Only run the build when executed directly. When sourced (e.g. by tests),
# skip main so the version-gate helpers can be unit-tested in isolation.
if [ "${BASH_SOURCE[0]}" = "$0" ]; then
main "$@"
fi
Reference in New Issue View Git Blame Copy Permalink