diff --git a/.dockerignore b/.dockerignore index 1b4707c99e..941513916c 100644 --- a/.dockerignore +++ b/.dockerignore @@ -35,14 +35,12 @@ target/ # ── ragflow_deps build context (built as a separate image, mounted ── # ── from infiniflow/ragflow_deps:latest by the main Dockerfile) ── # Excluding the directory contents keeps the main build context small -# regardless of which deps files download_deps.py currently fetches. +# regardless of which resources are prepared for the dependency image. # The deps image is built from this directory with: # cd ragflow_deps && docker build -f Dockerfile -t infiniflow/ragflow_deps . -# The two ORT version-pin files are re-included: build.sh's -# check_ort_version_consistency (run by Dockerfile_go's `--go` build) greps -# them. Re-including requires NOT excluding the directory itself. +# The Go downloader is re-included because build.sh reads its ORT version pin. +# Re-including requires NOT excluding the directory itself. ragflow_deps/* -!ragflow_deps/download_go_deps.py !ragflow_deps/download_deps.py # ── IDE and editor config ────────────────────────────────────────────────── diff --git a/.github/workflows/sep-tests.yml b/.github/workflows/sep-tests.yml index 5645114c5a..dd168e6033 100644 --- a/.github/workflows/sep-tests.yml +++ b/.github/workflows/sep-tests.yml @@ -923,7 +923,7 @@ jobs: run: | set -euo pipefail uv sync --python 3.13 --group test --frozen - uv run pytest ragflow_deps/test_download_go_deps.py ragflow_deps/test_tokenizer_assets.py -q --durations=100 + uv run pytest ragflow_deps/test_download_deps.py ragflow_deps/test_tokenizer_assets.py -q --durations=100 - name: Build ragflow:nightly run: | diff --git a/AGENTS.md b/AGENTS.md index a0016b30af..3911b37149 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -53,7 +53,7 @@ Use this file as the local operating guide for the current codebase. Prefer the - `web/`: frontend application. - `docker/`: local and production compose files. - `sdk/python/`: legacy Python client SDK pending deletion. -- `ragflow_deps/download_go_deps.py`: current Python helper for Go native libraries and model files; replace it before deleting it. `build.sh` currently reads its ONNX Runtime version pin. +- `ragflow_deps/download_deps.py`: Python helper for Go native libraries, DeepDoc models, and tokenizer assets. Replace it with non-Python preparation before removing Python dependencies. - `test/`: contains Python tests, including tests that import the legacy Python implementation directly. Migrate needed coverage to Go tests or frontend tests, then remove the Python tests; keep only tests for supported code. ## Go-Specific Rules @@ -129,7 +129,7 @@ bash build.sh --test-manual # local opt-in only bash build.sh --test-all # integration + e2e; excludes manual ``` -Native libraries and Go DeepDoc models currently come from `ragflow_deps/download_go_deps.py`; see `README_zh.md` for its isolated setup. The Go tokenizer also needs `ragflow_deps/cl100k_base.tiktoken`, which the current Python dependency pipeline provides. These are build/runtime resource preparation steps, not a Python server requirement. +Native libraries, Go DeepDoc models, tokenizer assets, and `ragflow_deps/cl100k_base.tiktoken` come from `ragflow_deps/download_deps.py`; see `README_zh.md` for its isolated setup. The downloader installs stagehand into the SDK cache using the version in `go.mod`. Build the Go resource image with `cd ragflow_deps && docker build -f Dockerfile -t infiniflow/ragflow_deps:latest .` after downloading the resources. ### Frontend ```bash diff --git a/Dockerfile b/Dockerfile index c3e7633f25..55518459ab 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,7 +30,6 @@ RUN --mount=type=bind,from=infiniflow/ragflow_deps:latest,source=/huggingface.co --exclude='layout.manual.ort' \ --exclude='layout.paper.ort' \ -cf - \ - /huggingface.co/InfiniFlow/text_concat_xgb_v1.0 \ /huggingface.co/InfiniFlow/deepdoc \ | tar -xf - --strip-components=3 -C /ragflow/internal/rag/res/deepdoc && \ #ln -s layout.onnx /ragflow/internal/rag/res/deepdoc/layout.laws.onnx && \ @@ -186,9 +185,9 @@ COPY internal internal COPY cmd cmd COPY build.sh ./ # build.sh's check_ort_version_consistency (run via `./build.sh --go`) greps the -# ORT version pins from these files; without them the --go build fails with +# ORT version pin from this file; without it the --go build fails with # "could not parse the ONNX Runtime version from one of the pinned locations". -COPY ragflow_deps/download_go_deps.py ragflow_deps/download_deps.py ./ragflow_deps/ +COPY ragflow_deps/download_deps.py ./ragflow_deps/ COPY Dockerfile ./ # ONNX Runtime static archives: build.sh's _seed_from_system looks for the ORT diff --git a/README.md b/README.md index b53c059436..694cd3465f 100644 --- a/README.md +++ b/README.md @@ -255,12 +255,12 @@ Install the Go version specified in `go.mod` (currently Go 1.27), Clang 20, LLD cd ragflow ``` -2. Prepare native libraries and model files with the [Go dependency download script](./ragflow_deps/download_go_deps.py), then build the Go services: +2. Prepare native libraries, model files, and tokenizer assets with the [Go dependency download script](./ragflow_deps/download_deps.py), then build the Go services: ```bash python3 -m venv /tmp/ragflow-go-download-venv /tmp/ragflow-go-download-venv/bin/python -m pip install requests huggingface-hub - /tmp/ragflow-go-download-venv/bin/python ragflow_deps/download_go_deps.py + /tmp/ragflow-go-download-venv/bin/python ragflow_deps/download_deps.py bash build.sh --all ``` diff --git a/README_ar.md b/README_ar.md index cfa7b18e6c..4155e49b1f 100644 --- a/README_ar.md +++ b/README_ar.md @@ -281,7 +281,7 @@ ```bash python3 -m venv /tmp/ragflow-go-download-venv /tmp/ragflow-go-download-venv/bin/python -m pip install requests huggingface-hub - /tmp/ragflow-go-download-venv/bin/python ragflow_deps/download_go_deps.py + /tmp/ragflow-go-download-venv/bin/python ragflow_deps/download_deps.py bash build.sh --all ``` diff --git a/README_fr.md b/README_fr.md index d9707706be..f8f6b2fae1 100644 --- a/README_fr.md +++ b/README_fr.md @@ -269,7 +269,7 @@ Pour changer de moteur de documents, modifier la configuration, redémarrer les ```bash python3 -m venv /tmp/ragflow-go-download-venv /tmp/ragflow-go-download-venv/bin/python -m pip install requests huggingface-hub - /tmp/ragflow-go-download-venv/bin/python ragflow_deps/download_go_deps.py + /tmp/ragflow-go-download-venv/bin/python ragflow_deps/download_deps.py bash build.sh --all ``` diff --git a/README_id.md b/README_id.md index 8baf7f0442..de3d5c93e5 100644 --- a/README_id.md +++ b/README_id.md @@ -277,7 +277,7 @@ Untuk mengganti mesin dokumen, mengubah konfigurasi, memulai ulang layanan, sert ```bash python3 -m venv /tmp/ragflow-go-download-venv /tmp/ragflow-go-download-venv/bin/python -m pip install requests huggingface-hub - /tmp/ragflow-go-download-venv/bin/python ragflow_deps/download_go_deps.py + /tmp/ragflow-go-download-venv/bin/python ragflow_deps/download_deps.py bash build.sh --all ``` diff --git a/README_ja.md b/README_ja.md index bb2d434ede..2e8b71808e 100644 --- a/README_ja.md +++ b/README_ja.md @@ -259,7 +259,7 @@ Go版のDockerデプロイでは `docker/.env` と `docker/docker-compose.yml` ```bash python3 -m venv /tmp/ragflow-go-download-venv /tmp/ragflow-go-download-venv/bin/python -m pip install requests huggingface-hub - /tmp/ragflow-go-download-venv/bin/python ragflow_deps/download_go_deps.py + /tmp/ragflow-go-download-venv/bin/python ragflow_deps/download_deps.py bash build.sh --all ``` diff --git a/README_ko.md b/README_ko.md index 68b7044c03..122403ba4c 100644 --- a/README_ko.md +++ b/README_ko.md @@ -259,7 +259,7 @@ Go Docker 배포는 `docker/.env`와 `docker/docker-compose.yml`을 사용하며 ```bash python3 -m venv /tmp/ragflow-go-download-venv /tmp/ragflow-go-download-venv/bin/python -m pip install requests huggingface-hub - /tmp/ragflow-go-download-venv/bin/python ragflow_deps/download_go_deps.py + /tmp/ragflow-go-download-venv/bin/python ragflow_deps/download_deps.py bash build.sh --all ``` diff --git a/README_pt_br.md b/README_pt_br.md index 49bec0c864..ad017d5ae3 100644 --- a/README_pt_br.md +++ b/README_pt_br.md @@ -265,7 +265,7 @@ Para trocar o mecanismo de documentos, alterar configurações, reiniciar servi ```bash python3 -m venv /tmp/ragflow-go-download-venv /tmp/ragflow-go-download-venv/bin/python -m pip install requests huggingface-hub - /tmp/ragflow-go-download-venv/bin/python ragflow_deps/download_go_deps.py + /tmp/ragflow-go-download-venv/bin/python ragflow_deps/download_deps.py bash build.sh --all ``` diff --git a/README_ru.md b/README_ru.md index d09cc90d22..fd3b7371e3 100644 --- a/README_ru.md +++ b/README_ru.md @@ -275,7 +275,7 @@ ```bash python3 -m venv /tmp/ragflow-go-download-venv /tmp/ragflow-go-download-venv/bin/python -m pip install requests huggingface-hub - /tmp/ragflow-go-download-venv/bin/python ragflow_deps/download_go_deps.py + /tmp/ragflow-go-download-venv/bin/python ragflow_deps/download_deps.py bash build.sh --all ``` diff --git a/README_tr.md b/README_tr.md index 0395c24b43..bbb20b8fd4 100644 --- a/README_tr.md +++ b/README_tr.md @@ -277,7 +277,7 @@ Belge motorunu değiştirme, yapılandırma değişikliklerinden sonra hizmetler ```bash python3 -m venv /tmp/ragflow-go-download-venv /tmp/ragflow-go-download-venv/bin/python -m pip install requests huggingface-hub - /tmp/ragflow-go-download-venv/bin/python ragflow_deps/download_go_deps.py + /tmp/ragflow-go-download-venv/bin/python ragflow_deps/download_deps.py bash build.sh --all ``` diff --git a/README_tzh.md b/README_tzh.md index ade350faf6..b6e2ff903b 100644 --- a/README_tzh.md +++ b/README_tzh.md @@ -272,12 +272,12 @@ Go 版 Docker 部署使用 `docker/.env` 和 `docker/docker-compose.yml`，以 K git clone https://github.com/infiniflow/ragflow.git cd ragflow ``` -2. 使用 [Go 依賴下載腳本](./ragflow_deps/download_go_deps.py)準備原生程式庫和模型檔案，再編譯 Go 服務： +2. 使用 [Go 依賴下載腳本](./ragflow_deps/download_deps.py)準備原生程式庫、模型檔案和 tokenizer 資源，再編譯 Go 服務： ```bash python3 -m venv /tmp/ragflow-go-download-venv /tmp/ragflow-go-download-venv/bin/python -m pip install requests huggingface-hub - /tmp/ragflow-go-download-venv/bin/python ragflow_deps/download_go_deps.py + /tmp/ragflow-go-download-venv/bin/python ragflow_deps/download_deps.py bash build.sh --all ``` diff --git a/README_zh.md b/README_zh.md index 65a18287da..abc0bf0ba9 100644 --- a/README_zh.md +++ b/README_zh.md @@ -279,12 +279,12 @@ Go 版 Docker 部署使用 `docker/.env` 和 `docker/docker-compose.yml`，使 cd ragflow ``` -2. 按 [Go 依赖下载脚本](./ragflow_deps/download_go_deps.py)准备原生库和模型文件，再编译 Go 服务： +2. 按 [Go 依赖下载脚本](./ragflow_deps/download_deps.py)准备原生库、模型文件和 tokenizer 资源，再编译 Go 服务： ```bash python3 -m venv /tmp/ragflow-go-download-venv /tmp/ragflow-go-download-venv/bin/python -m pip install requests huggingface-hub - /tmp/ragflow-go-download-venv/bin/python ragflow_deps/download_go_deps.py + /tmp/ragflow-go-download-venv/bin/python ragflow_deps/download_deps.py bash build.sh --all ``` diff --git a/build.sh b/build.sh index e3a89964a4..1dd09bc8cd 100755 --- a/build.sh +++ b/build.sh @@ -42,8 +42,7 @@ PDF_OXIDE_VERSION="0.3.73" # (--undefined=OrtGetApiBase + --dynamic-list, no --whole-archive); OrtGetApiBase # is then resolved via dlopen(NULL) (the process-global symbol table, not the # executable's own path), so no libonnxruntime.so is needed at runtime. -# Downloaded by ragflow_deps/download_go_deps.py (and -# ragflow_deps/download_deps.py) into onnxruntime/static_lib. +# Downloaded by ragflow_deps/download_deps.py into onnxruntime/static_lib. ONNXRUNTIME_STATIC_PREFIX="${HOME}/ragflow-native-libs/onnxruntime/static_lib" # Copy a dependency from the system pre-seed directory to the user cache. @@ -219,26 +218,24 @@ check_go_deps() { # The in-process (Go) DeepDoc backend statically links libonnxruntime.a built # from ONE exact ORT release; a mismatch links a wrong/missing .a and only fails # at runtime (dlopen(NULL) can't find OrtGetApiBase). The version is pinned in -# four Go-side locations that must all agree. +# three Go-side locations that must all agree. check_ort_version_consistency() { print_section "Checking ONNX Runtime version consistency" - local env_go d1 d2 dockerfile + local env_go deps dockerfile env_go="$(grep -m1 -E 'DeepDocORTVersion[[:space:]]*=[[:space:]]*"' "${PROJECT_ROOT}/internal/common/environments.go" | sed -E 's/.*"([^"]+)".*/\1/')" - d1="$(grep -m1 -E '^ORT_VERSION[[:space:]]*=[[:space:]]*"' "${PROJECT_ROOT}/ragflow_deps/download_go_deps.py" | sed -E 's/.*"([^"]+)".*/\1/')" - d2="$(grep -m1 -E '^ORT_VERSION[[:space:]]*=[[:space:]]*"' "${PROJECT_ROOT}/ragflow_deps/download_deps.py" | sed -E 's/.*"([^"]+)".*/\1/')" + deps="$(grep -m1 -E '^ORT_VERSION[[:space:]]*=[[:space:]]*"' "${PROJECT_ROOT}/ragflow_deps/download_deps.py" | sed -E 's/.*"([^"]+)".*/\1/')" dockerfile="$(grep -m1 -E 'ARG[[:space:]]+ORT_VERSION=' "${PROJECT_ROOT}/Dockerfile" | sed -E 's/.*ORT_VERSION=([0-9][^"[:space:]]*).*/\1/')" - if [ -z "$env_go" ] || [ -z "$d1" ] || [ -z "$d2" ] || [ -z "$dockerfile" ]; then + if [ -z "$env_go" ] || [ -z "$deps" ] || [ -z "$dockerfile" ]; then echo -e "${RED}Error: could not parse the ONNX Runtime version from one of the pinned locations${NC}" >&2 exit 1 fi - if [ "$env_go" != "$d1" ] || [ "$env_go" != "$d2" ] || [ "$env_go" != "$dockerfile" ]; then + if [ "$env_go" != "$deps" ] || [ "$env_go" != "$dockerfile" ]; then echo -e "${RED}Error: ONNX Runtime version is inconsistent — fix before building:${NC}" >&2 printf ' %-10s %s\n' "$env_go" "internal/common/environments.go:DeepDocORTVersion" - printf ' %-10s %s\n' "$d1" "ragflow_deps/download_go_deps.py:ORT_VERSION" - printf ' %-10s %s\n' "$d2" "ragflow_deps/download_deps.py:ORT_VERSION" + printf ' %-10s %s\n' "$deps" "ragflow_deps/download_deps.py:ORT_VERSION" printf ' %-10s %s\n' "$dockerfile" "dockerfile:ARG ORT_VERSION" exit 1 fi @@ -302,7 +299,7 @@ check_onnxruntime_deps() { echo -e "${RED}Error: no ONNX Runtime ${ort_version} static lib under $ONNXRUNTIME_STATIC_PREFIX${NC}" >&2 echo " available: ${avail:-}" >&2 echo " DeepDocORTVersion=${ort_version}; fetch it with:" >&2 - echo " uv run python3 ragflow_deps/download_go_deps.py" >&2 + echo " uv run python3 ragflow_deps/download_deps.py" >&2 echo " or pre-seed /opt/ragflow-native-libs/onnxruntime (CI image)." >&2 return 1 fi @@ -316,8 +313,8 @@ check_onnxruntime_deps() { # or GitHub was unreachable (curl had no connect timeout). # # A mismatch means the local copy is corrupt or stale; we only WARN (not - # fail) so a one-off re-issue cannot break the build. The download scripts - # (ragflow_deps/download_go_deps.py, download_deps.py) still verify against + # fail) so a one-off re-issue cannot break the build. The downloader + # ragflow_deps/download_deps.py still verifies against # the published sidecar at download time, so a re-issued archive is picked # up on the next download. local asset="onnxruntime-v${ort_version}-linux-x86_64.zip" @@ -332,7 +329,7 @@ check_onnxruntime_deps() { echo " local sha256: $actual" >&2 echo " The local copy may be corrupt or stale. Refresh it with:" >&2 echo " rm -f ${zip_path}" >&2 - echo " uv run python3 ragflow_deps/download_go_deps.py # or ragflow_deps/download_deps.py" >&2 + echo " uv run python3 ragflow_deps/download_deps.py" >&2 else echo " ✓ ${asset} sha256 matches pinned digest" fi @@ -357,7 +354,7 @@ check_office_oxide_deps() { if [ ! -f "$lib_path" ] || [ ! -f "$header_path" ]; then echo -e "${RED}Error: office_oxide native library not found${NC}" echo " Expected: ${lib_path}" - echo " Run: uv run python3 ragflow_deps/download_go_deps.py" + echo " Run: uv run python3 ragflow_deps/download_deps.py" echo " Or manually download: https://github.com/yfedoseev/office_oxide/releases/download/v${OFFICE_OXIDE_VERSION}/native-linux-x86_64.tar.gz" exit 1 fi @@ -371,12 +368,12 @@ check_office_oxide_deps() { found_version=$(strings "$lib_path" 2>/dev/null | grep -E "^0\.[0-9]+\.[0-9]+$" | head -1) echo -e "${YELLOW}office_oxide native lib version mismatch (required v${OFFICE_OXIDE_VERSION}; found: ${found_version:-unknown}); refreshing from download${NC}" rm -rf "${OFFICE_OXIDE_PREFIX}" "${PROJECT_ROOT}/office_oxide-linux-x86_64.tar.gz" - if ! (cd "${PROJECT_ROOT}" && uv run python3 ragflow_deps/download_go_deps.py); then + if ! (cd "${PROJECT_ROOT}" && uv run python3 ragflow_deps/download_deps.py); then echo -e "${RED}Error: office_oxide native lib version mismatch${NC}" echo " Required: v${OFFICE_OXIDE_VERSION}; found: ${found_version:-unknown}" echo " A stale lib silently loses PPT97 (.ppt) slide content. Refresh:" echo " rm -rf ~/ragflow-native-libs/office_oxide ragflow_deps/office_oxide-linux-x86_64.tar.gz" - echo " uv run python3 ragflow_deps/download_go_deps.py" + echo " uv run python3 ragflow_deps/download_deps.py" exit 1 fi lib_path="${OFFICE_OXIDE_PREFIX}/lib/${lib_file}" @@ -409,7 +406,7 @@ check_pdfium_deps() { echo " pdfium (static) not found" echo " Expected: ${lib_path}" - echo " Run: uv run python3 ragflow_deps/download_go_deps.py" + echo " Run: uv run python3 ragflow_deps/download_deps.py" echo " Or: curl -fsSL https://github.com/kognitos/pdfium-static/releases/download/chromium%2F${PDFIUM_STATIC_VERSION}/pdfium-linux-x64-static.tgz | tar xz -C ${PDFIUM_STATIC_PREFIX}" return 1 } @@ -515,7 +512,7 @@ check_pdf_oxide_deps() { echo " Required: v${PDF_OXIDE_VERSION}; found: ${found_version:-unknown}" echo " A stale lib silently reverts PDF parsing fixes. Refresh:" echo " rm -rf ${PDF_OXIDE_PREFIX} ragflow_deps/pdf_oxide-go-ffi-linux-amd64.tar.gz" - echo " uv run python3 ragflow_deps/download_go_deps.py" + echo " uv run python3 ragflow_deps/download_deps.py" return 1 ;; esac @@ -523,7 +520,7 @@ check_pdf_oxide_deps() { echo " pdf_oxide (static) not found" echo " Expected: ${lib_path}" - echo " Run: uv run python3 ragflow_deps/download_go_deps.py" + echo " Run: uv run python3 ragflow_deps/download_deps.py" echo " Or: curl -fsSL https://github.com/yfedoseev/pdf_oxide/releases/download/v${PDF_OXIDE_VERSION}/pdf_oxide-go-ffi-linux-amd64.tar.gz | tar xz -C ${PDF_OXIDE_PREFIX}" return 1 } @@ -653,12 +650,12 @@ build_go() { echo " -Wl,--dynamic-list). Without it the binary compiles but dies" >&2 echo " at startup with a fatal 'no in-process DeepDoc backend serving'." >&2 echo " Fetch the static libs with:" >&2 - echo " uv run python3 ragflow_deps/download_go_deps.py" >&2 + echo " uv run python3 ragflow_deps/download_deps.py" >&2 echo " or pre-seed them at /opt/ragflow-native-libs/onnxruntime (CI image)." >&2 echo " This production binary must statically include ORT — there is no" >&2 echo " ORT-free build path. ORT ends up unlinked only via one of:" >&2 echo " - the ORT static_lib dir was never seeded: run" >&2 - echo " 'uv run python3 ragflow_deps/download_go_deps.py', or pre-seed" >&2 + echo " 'uv run python3 ragflow_deps/download_deps.py', or pre-seed" >&2 echo " /opt/ragflow-native-libs/onnxruntime as the CI image does;" >&2 echo " - setup_cgo_env did not add libonnxruntime to CGO_LDFLAGS." >&2 return 1 @@ -1096,7 +1093,7 @@ DEPENDENCIES: - cmake >= 4.0 - go >= 1.26.4 - clang++ with C++20 support - - office_oxide native library (download with: uv run python3 ragflow_deps/download_go_deps.py) + - office_oxide native library (download with: uv run python3 ragflow_deps/download_deps.py) - lld (Linux only): sudo apt install lld-20 && sudo ln -s /usr/bin/ld.lld-20 /usr/bin/ld.lld - pcre2 development files - Debian/Ubuntu: libpcre2-dev @@ -1198,7 +1195,7 @@ main() { fi # The in-process (Go) DeepDoc backend needs the .ort weights. Default # MODEL_DIR to the canonical repo model dir (internal/rag/res/deepdoc) so a - # local run needs no MODEL_DIR export after `download_go_deps.py`. + # local run needs no MODEL_DIR export after `download_deps.py`. REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" export MODEL_DIR="${MODEL_DIR:-$REPO_ROOT/internal/rag/res/deepdoc}" run_go_tests_tagged "cgo integration" "${pkgs[@]}" diff --git a/cmd/ragflow_server.go b/cmd/ragflow_server.go index 2b66a6d5d8..305268d7f6 100644 --- a/cmd/ragflow_server.go +++ b/cmd/ragflow_server.go @@ -1674,10 +1674,14 @@ func logTokenizerCounters() { if len(unavailable) > 0 { common.Warn("embedding tokenizers unavailable; models that declare them cannot be ingested until the asset is restored", zap.Strings("unavailable", unavailable), - zap.String("hint", "run `uv run ragflow_deps/download_go_deps.py`, or set "+common.EnvModelAssetsDir+" to a directory holding them")) + zap.String("hint", "run `uv run ragflow_deps/download_deps.py`, or set "+common.EnvModelAssetsDir+" to a directory holding them")) } } +// resolveDeepDocModelDir picks the model directory: the explicit DEEPDOC_MODEL_DIR +// env, else the RAGFlow default model dir (internal/rag/res/deepdoc), +// else the assets fetched by ragflow_deps/download_deps.py. The first +// candidate that actually contains the required weights wins. func resolveDeepDocModelDir() string { if v := strings.TrimSpace(common.GetEnv(common.EnvDeepDocModelDir)); v != "" { return v diff --git a/docker/launch_backend_service.sh b/docker/launch_backend_service.sh index 8fa2de7038..eba2d5f96e 100755 --- a/docker/launch_backend_service.sh +++ b/docker/launch_backend_service.sh @@ -72,9 +72,7 @@ STOP=false PIDS=() # Set the path to the NLTK data directory -# download_deps.py downloads NLTK data into ragflow_deps/nltk_data (see -# ragflow_deps/download_deps.py); point NLTK_DATA there directly instead of a -# stale top-level ./nltk_data so this matches what's actually populated. +# Use the repository's NLTK data directory when it is provisioned. export NLTK_DATA="$(pwd)/ragflow_deps/nltk_data" # Function to handle termination signals diff --git a/docs/develop/launch_ragflow_from_source.md b/docs/develop/launch_ragflow_from_source.md index d22e488772..dc1752c09a 100644 --- a/docs/develop/launch_ragflow_from_source.md +++ b/docs/develop/launch_ragflow_from_source.md @@ -27,7 +27,7 @@ All long-running RAGFlow processes started below use `bin/ragflow_server`. - Go 1.27 or later, as declared in `go.mod` (check `go version`). - CMake 4.0 or later, Clang 20, LLD 20, and PCRE2 development headers. - Node.js 18.20.4 or later and npm for the frontend. -- Python 3.10 or later only for `ragflow_deps/download_go_deps.py`, which downloads the native libraries and model resources required by the Go build. +- Python 3.10 or later only for `ragflow_deps/download_deps.py`, which downloads the native libraries and model resources required by the Go build. See the [Docker installation guide](https://docs.docker.com/engine/install/) if Docker is not installed. Use the Go version declared in `go.mod` and the compiler versions listed above when preparing the build environment. @@ -59,10 +59,12 @@ Download the native libraries and Go DeepDoc model weights with a small, isolate ```bash python3 -m venv /tmp/ragflow-go-download-venv /tmp/ragflow-go-download-venv/bin/python -m pip install requests huggingface-hub -/tmp/ragflow-go-download-venv/bin/python ragflow_deps/download_go_deps.py +/tmp/ragflow-go-download-venv/bin/python ragflow_deps/download_deps.py ``` -The downloader fetches the static libraries needed by `build.sh`, plus `det.ort`, `layout.ort`, `tsr.ort`, `rec.ort`, and `ocr.res` into `internal/rag/res/deepdoc/`. These files are required by the in-process Go DeepDoc backend. Keep the server's working directory at the repository root so it can find them automatically; if you launch it elsewhere, set `DEEPDOC_MODEL_DIR` to the absolute path of `rag/res/deepdoc`. +The downloader fetches the static libraries needed by `build.sh`, plus `det.ort`, `layout.ort`, `tsr.ort`, `rec.ort`, and `ocr.res` into `internal/rag/res/deepdoc/`. These files are required by the in-process Go DeepDoc backend. Keep the server's working directory at the repository root so it can find them automatically; if you launch it elsewhere, set `DEEPDOC_MODEL_DIR` to the absolute path of `internal/rag/res/deepdoc`. + +It also downloads `ragflow_deps/cl100k_base.tiktoken` and the embedding tokenizer assets, and installs the stagehand driver into `${XDG_CACHE_HOME:-$HOME/.cache}/stagehand/lib/go_/`. Native archives support Linux x86_64. The isolated download environment can be removed after the resources have been prepared. @@ -72,7 +74,7 @@ Build the C++ bindings and Go binaries: bash build.sh --all ``` -For a smaller production binary, use `bash build.sh --strip --all` instead. The build script also tries to place `ragflow_deps/cl100k_base.tiktoken` in the repository. If it reports that the BPE table could not be provisioned, download it before starting the server: +For a smaller production binary, use `bash build.sh --strip --all` instead. The dependency downloader prepares `ragflow_deps/cl100k_base.tiktoken`. If the table is missing, download it before starting the server: ```bash curl -fsSL -o ragflow_deps/cl100k_base.tiktoken https://openaipublic.blob.core.windows.net/encodings/cl100k_base.tiktoken @@ -88,6 +90,16 @@ It should print API usage information and may exit with status 1. This command o The Go/C++ build requires LLD 20. Confirm that `ld.lld --version` reports LLD 20 before building; merely installing `lld-20` is insufficient when an older `ld.lld` remains the system default. If the binary builds successfully but exits or crashes before reaching Go `main`, verify the selected linker and rebuild with LLD 20. +To prepare the resource image used by the Go Dockerfile, run the downloader above, then build from `ragflow_deps/`: + +```bash +cd ragflow_deps +docker build -f Dockerfile -t infiniflow/ragflow_deps:latest . +cd .. +``` + +The downloader prepares the five Go DeepDoc files under `ragflow_deps/huggingface.co/InfiniFlow/deepdoc/` for the resource image, as well as `internal/rag/res/deepdoc/` for local use. The Go Dockerfile copies these models into the runtime model directory. Analyzer dictionaries and fonts are installed separately by the Go Dockerfile. + ## 2. Start supporting services With the default `conf/service_conf.yaml`, the host-run Go processes need Elasticsearch, MySQL, MinIO, NATS, Kvrocks, and ClickHouse. Start these services explicitly from the repository root: diff --git a/internal/agent/component/io/pdf_writer.go b/internal/agent/component/io/pdf_writer.go index 9505f4e649..f5dc750fca 100644 --- a/internal/agent/component/io/pdf_writer.go +++ b/internal/agent/component/io/pdf_writer.go @@ -59,7 +59,7 @@ var defaultPDFLatinFontPaths = []string{ } var defaultPDFCJKFontPaths = []string{ - // Use the language-specific Noto CJK Variable TTF shipped by Dockerfile_go. + // Use the language-specific Noto CJK Variable TTF shipped in the runtime image. // gopdf uses its default variation instance; it does not expose variable // font axes, so this is used as the regular CJK face. "/usr/local/share/fonts/truetype/noto/NotoSansCJKsc-VF.ttf", diff --git a/internal/common/environments.go b/internal/common/environments.go index 91b6583fe0..f3ca53cc40 100644 --- a/internal/common/environments.go +++ b/internal/common/environments.go @@ -292,11 +292,9 @@ const ( // do set-membership checks); keep it stable so logs and diffs stay readable. // // External consumers that re-list these names must stay in sync: -// - ragflow_deps/download_go_deps.py re-lists them as DEEPDOC_MODEL_FILES +// - ragflow_deps/download_deps.py re-lists them as DEEPDOC_MODEL_FILES // (it fetches the files one by one, so it MUST be edited by hand when this // slice changes); -// - ragflow_deps/download_deps.py snapshots the whole InfiniFlow/deepdoc repo -// (so .ort lands in the model dir automatically — no FILES edit needed). var DeepDocModelFiles = []string{ "det.ort", "layout.ort", @@ -320,9 +318,9 @@ func HasModelFiles(dir string) bool { // DeepDocORTVersion is the onnxruntime native release the in-process (Go) // DeepDoc backend is built and tested against (e.g. "1.29.0"). It is ONE OF -// FOUR raw version declarations that must stay equal (the other three are -// ORT_VERSION in ragflow_deps/download_go_deps.py and ragflow_deps/download_deps.py, -// and ARG ORT_VERSION in Dockerfile_go) — NOT a single source of truth. The +// THREE raw version declarations that must stay equal (the other two are +// ORT_VERSION in ragflow_deps/download_deps.py and ARG ORT_VERSION in +// Dockerfile) — NOT a single source of truth. The // download URL and extracted dir name are built from those ORT_VERSION // constants, not from this one. The Go binding // (github.com/infiniflow/onnxruntime_go, the org mirror of yalue/onnxruntime_go) @@ -332,12 +330,11 @@ func HasModelFiles(dir string) bool { // the same minor line. ONNX Runtime is linked statically (libonnxruntime.a), // so there is no .so / SONAME at runtime. // -// To bump ORT, the four Go-side native pins above must change together +// To bump ORT, the three Go-side native pins above must change together // (drift breaks the static link or the runtime OrtGetApiBase lookup): // - DeepDocORTVersion (here, Go) -// - ORT_VERSION in ragflow_deps/download_go_deps.py // - ORT_VERSION in ragflow_deps/download_deps.py -// - ARG ORT_VERSION in Dockerfile_go +// - ARG ORT_VERSION in Dockerfile // // Separately, keep these on the same ORT minor line but version them // independently of the Go native lib (see pyproject.toml / development.md): diff --git a/internal/deepdoc/native/run.sh b/internal/deepdoc/native/run.sh index 8dd6d7b06e..d8bcffc808 100755 --- a/internal/deepdoc/native/run.sh +++ b/internal/deepdoc/native/run.sh @@ -15,7 +15,7 @@ cd "$SCRIPT_DIR" # dlopen(NULL); no ORT_LIB is needed. # # Default MODEL_DIR to the canonical repo model dir (internal/rag/res/deepdoc) so running -# `download_go_deps.py` is enough — no MODEL_DIR export is required for local +# `download_deps.py` is enough — no MODEL_DIR export is required for local # Go DeepDoc runs. Override with `MODEL_DIR=... bash run.sh` if needed. REPO_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)" MODEL_DIR="${MODEL_DIR:-$REPO_ROOT/internal/rag/res/deepdoc}" diff --git a/internal/development.md b/internal/development.md index f40b1b7d37..3bec130058 100644 --- a/internal/development.md +++ b/internal/development.md @@ -39,15 +39,28 @@ go version ### 1.4 Install dependent library ```shell sudo apt install libpcre2-dev -python3 ragflow_deps/download_go_deps.py +uv run ragflow_deps/download_deps.py ``` +Install `uv` before running the downloader; it installs the script's declared Python dependencies. + +This downloader provisions Go native libraries, `.ort` weights, model tokenizer assets, and `cl100k_base.tiktoken`. It installs the local stagehand driver into the SDK's cache, using the SDK version from `go.mod`. Native archives support Linux x86_64. + +Build the Go resource image from `ragflow_deps/` after preparing these files: + +```shell +cd ragflow_deps +docker build -f Dockerfile -t infiniflow/ragflow_deps:latest . +cd .. +``` + +The image contains Go DeepDoc weights, tokenizer assets, the BPE table, and stagehand binaries. DeepDoc weights are prepared under `ragflow_deps/huggingface.co/InfiniFlow/deepdoc/` for packaging and under `internal/rag/res/deepdoc/` for local use. > **Note**: If you use IDEs like GoLand to run/debug directly (via Run/Debug buttons), or run `go build` / `go run` from command line, set these CGO environment variables: > > ```bash -> RAGFLOW_DEPS="${HOME}/ragflow-native-libs" # created by download_go_deps.py + download_deps.py +> RAGFLOW_DEPS="${HOME}/ragflow-native-libs" # created by download_deps.py > PLATFORM="linux_amd64" # or darwin_amd64, linux_arm64, darwin_arm64 -> # NOTE: the ONNX Runtime static lib fetched by download_go_deps.py is +> # NOTE: the ONNX Runtime static lib fetched by download_deps.py is > # linux-x64 ONLY (onnxruntime-linux-x64-static_lib-*). On darwin_* / non-amd64 > # PLATFORM values the production DeepDoc backend cannot be linked, so those > # PLATFORM examples cover office_oxide/pdfium/pdf_oxide only — ORT is a @@ -97,7 +110,7 @@ python3 ragflow_deps/download_go_deps.py > Since `build.sh` (`build_go`) now **fails fast** when ORT is absent from > `CGO_LDFLAGS`, this breakage surfaces at build time instead of at runtime. If > you see `Error: ONNX Runtime static libraries are not linked`, run -> `uv run python3 ragflow_deps/download_go_deps.py` (or pre-seed +> `uv run ragflow_deps/download_deps.py` (or pre-seed > `/opt/ragflow-native-libs/onnxruntime` as the CI runner image does). There is > no ORT-free production build path — if ORT is absent the binary fails at > startup, so the remedy is always to seed the static lib above, never to build @@ -107,8 +120,8 @@ python3 ragflow_deps/download_go_deps.py > that must stay in sync. Bumping it in one spot and not the others fails the > build with `Error: ONNX Runtime version is inconsistent`: > - `internal/common/environments.go` — `DeepDocORTVersion` -> - `Dockerfile_go` — `ARG ORT_VERSION` -> - `ragflow_deps/download_go_deps.py` and `ragflow_deps/download_deps.py` — `ORT_VERSION` +> - `Dockerfile` — `ARG ORT_VERSION` +> - `ragflow_deps/download_deps.py` — `ORT_VERSION` > > `build.sh` runs this consistency check automatically before the Go build > (through `check_go_deps`) and fails fast on any mismatch. Run it on demand @@ -120,13 +133,12 @@ python3 ragflow_deps/download_go_deps.py > **Note**: `build.sh` also guards ONNX Runtime **archive integrity** and the > **link cache**, because a silently-stale `.a` is easy to miss: > - `check_onnxruntime_deps` compares the local release ZIP under `ragflow_deps/` -> (`onnxruntime-v-linux-x86_64.zip`) against the published -> `onnxruntime-v-linux-x86_64.zip.sha256` sidecar — the same source of -> truth the download scripts use. If they differ (the archive was re-issued -> under the same tag/asset name, or an older download is present) the build -> fails fast with `Error: ONNX Runtime archive ... is stale`, printing the -> expected/actual sha256 and the exact refresh command -> (`rm -f ` + `uv run python3 ragflow_deps/download_go_deps.py`). CI seeds +> (`onnxruntime-v-linux-x86_64.zip`) against the SHA-256 digest pinned in +> `build.sh`, without a network request. The downloader separately checks the +> published `onnxruntime-v-linux-x86_64.zip.sha256` sidecar. A local ZIP +> mismatch only emits a warning with the expected/actual SHA-256 and the +> refresh command (`rm -f ` + `uv run ragflow_deps/download_deps.py`); +> the build continues checking and linking the extracted static libraries. CI seeds > ORT from `/opt` and has no local zip, so the check is skipped there (the bake > is authoritative). This matters because the `onnxruntime_go` binding reaches > ORT only through the OrtApi function-pointer table @@ -139,7 +151,7 @@ python3 ragflow_deps/download_go_deps.py > would otherwise silently reuse a stale linked binary. Stamping the path with > the archive's sha256 changes the flag string whenever the content changes → > automatic relink. If you ever see a stale-`.a` crash after an ORT re-issue, -> re-run `uv run python3 ragflow_deps/download_go_deps.py` (or `download_deps.py`) +> re-run `uv run ragflow_deps/download_deps.py` > so the stamp moves; a plain `go clean -cache` also forces it. ### 1.5 Build RAGFlow @@ -174,9 +186,10 @@ supersedes the other, so do not delete one to "clean up". | Format | `.ort` | `.onnx` | | Files | `det.ort`, `layout.ort`, `tsr.ort`, `rec.ort`, `ocr.res` | `det.onnx`, `layout.onnx`, `tsr.onnx`, `rec.onnx`, `ocr.res` | -`download_go_deps.py` (§1.4) fetches the five required files — four `.ort` plus -`ocr.res` — into `internal/rag/res/deepdoc/`; `download_deps.py` snapshots the whole -`InfiniFlow/deepdoc` repo and therefore carries both formats. +`download_deps.py` (§1.4) fetches the five required files — four `.ort` plus +`ocr.res` — into `internal/rag/res/deepdoc/`. Python `.onnx` weights require separate provisioning. +The resource image packages these files under +`/huggingface.co/InfiniFlow/deepdoc/` for the Go runtime image. Auto-discovery is **relative to the server process's working directory**: `resolveDeepDocModelDir()` (`cmd/ragflow_server.go`) probes @@ -191,14 +204,14 @@ missing from the model directory. > **Note**: A `internal/rag/res/deepdoc/` populated before the `.ort` switch holds only > `.onnx` and will NOT serve the Go backend, even though the directory looks -> fully populated. Re-run `download_go_deps.py` after updating. +> fully populated. Re-run `download_deps.py` after updating. - **Confirm it is serving** — the server logs, at startup: `in-process DeepDoc backend registered (production backend)` If you instead see a fatal `no in-process DeepDoc backend serving`, it has two possible causes: ORT was not linked into the binary, or the model directory is missing one of the five required files listed above. Check the - weights first, then re-run `uv run python3 ragflow_deps/download_go_deps.py` and rebuild + weights first, then re-run `uv run ragflow_deps/download_deps.py` and rebuild (§1.4 explains the ORT link failure; `build.sh` fails fast with `Error: ONNX Runtime static libraries are not linked` before that happens). diff --git a/internal/ingestion/component/chunker/tool-py/capture_golden.py b/internal/ingestion/component/chunker/tool-py/capture_golden.py index aedb2e4a1b..2b3cbe8182 100644 --- a/internal/ingestion/component/chunker/tool-py/capture_golden.py +++ b/internal/ingestion/component/chunker/tool-py/capture_golden.py @@ -136,7 +136,7 @@ def _assert_tokenizer_alive() -> None: from rag.nlp import num_tokens_from_string probe = "RAGFlow chunker parity tokenizer liveness probe." - hint = "Fix the encoding first: uv run python3 ragflow_deps/download_deps.py (provides ragflow_deps/cl100k_base.tiktoken)." + hint = "Fix the encoding first: bash build.sh --go provisions ragflow_deps/cl100k_base.tiktoken." # Two shapes to catch. common/token_utils.py builds the encoder on first use and # lets that failure propagate, so an unavailable BPE table raises here; a failure # to encode an otherwise valid string is still reported as 0. diff --git a/internal/ingestion/task/embedder.go b/internal/ingestion/task/embedder.go index 1fc65c9070..2707130975 100644 --- a/internal/ingestion/task/embedder.go +++ b/internal/ingestion/task/embedder.go @@ -96,7 +96,7 @@ func (e *embedder) limiter() tokenizer.Limiter { } if id != "" && !tokenizer.CounterExact(id) { e.limiterErr = fmt.Errorf( - "embedding tokenizer %q is declared for %s but its asset is unavailable (check that ragflow_deps/huggingface.co is present; run `uv run ragflow_deps/download_go_deps.py`): refusing to count with the calibrated estimate", + "embedding tokenizer %q is declared for %s but its asset is unavailable (check that ragflow_deps/huggingface.co is present; run `uv run ragflow_deps/download_deps.py`): refusing to count with the calibrated estimate", id, e.quotaKey()) } e.limiterVal = tokenizer.LimiterFor(id, string(e.quotaKey()), tokenizer.DefaultCalibration()) diff --git a/internal/service/nlp/wordnet_helpers_test.go b/internal/service/nlp/wordnet_helpers_test.go index 9e17f6e377..71a5208b54 100644 --- a/internal/service/nlp/wordnet_helpers_test.go +++ b/internal/service/nlp/wordnet_helpers_test.go @@ -21,14 +21,13 @@ import ( ) // requireWordNetData skips the test when the WordNet dictionary files -// are not present at testWordNetDir. The data ships outside the repo -// (downloaded by Python `download_deps.py`), so CI hosts without it -// must skip rather than fail. The check looks for the lemma-pos offset +// are not present at testWordNetDir. The data ships outside the repo, so +// hosts without it must skip rather than fail. The check looks for the lemma-pos offset // file the loader opens first. func requireWordNetData(t testing.TB) { t.Helper() probe := filepath.Join(testWordNetDir, "index.noun") if _, err := os.Stat(probe); err != nil { - t.Skipf("WordNet data not present at %s; skipping (run download_deps.py to populate)", testWordNetDir) + t.Skipf("WordNet data not present at %s", testWordNetDir) } } diff --git a/internal/tokenizer/asset_dir_test.go b/internal/tokenizer/asset_dir_test.go index bcdc6c3e1f..e95e020161 100644 --- a/internal/tokenizer/asset_dir_test.go +++ b/internal/tokenizer/asset_dir_test.go @@ -34,7 +34,7 @@ import ( const assetDirChildEnv = "RAGFLOW_TOKENIZER_ASSET_DIR_CHILD" -// realSPMAsset is the asset shipped by ragflow_deps/download_go_deps.py, relative to the +// realSPMAsset is the asset shipped by ragflow_deps/download_deps.py, relative to the // repository root. var realSPMAsset = filepath.Join("ragflow_deps", "huggingface.co", "BAAI", "bge-m3", "sentencepiece.bpe.model") @@ -44,7 +44,7 @@ func TestModelAssetsDirIsHonoured(t *testing.T) { } source := filepath.Join("..", "..", realSPMAsset) if _, err := os.Stat(source); err != nil { - t.Skipf("the SPM asset is not present here (%v); run ragflow_deps/download_go_deps.py", err) + t.Skipf("the SPM asset is not present here (%v); run ragflow_deps/download_deps.py", err) } dir := t.TempDir() diff --git a/internal/tokenizer/bpe_loader.go b/internal/tokenizer/bpe_loader.go index 17eae01eae..3a4dcc82bd 100644 --- a/internal/tokenizer/bpe_loader.go +++ b/internal/tokenizer/bpe_loader.go @@ -56,7 +56,7 @@ func init() { // (223921b76ee99bde995b7ff738513eef100fb51d18c93597a113bcffe865b2a7 for // cl100k_base); that value identifies the path, while the value below verifies // the bytes we actually load. Compute it from the table shipped by -// ragflow_deps/download_go_deps.py: `sha1sum cl100k_base.tiktoken`. +// ragflow_deps/download_deps.py: `sha1sum cl100k_base.tiktoken`. var expectedBpeHashes = map[string]string{ "https://openaipublic.blob.core.windows.net/encodings/cl100k_base.tiktoken": "6494e42d5aad2bbb441ea9793af9e7db335c8d9c", } diff --git a/internal/tokenizer/bpe_loader_test.go b/internal/tokenizer/bpe_loader_test.go index cb00102155..3c3ca84867 100644 --- a/internal/tokenizer/bpe_loader_test.go +++ b/internal/tokenizer/bpe_loader_test.go @@ -117,8 +117,8 @@ func TestLocalBpeLoader_ReadsSha1FileFromWorkingDirectory(t *testing.T) { } } -// A developer checkout that has run download_deps.py but never started the -// Python side only has the file under its download name. +// A developer checkout provisioned with build.sh can have the file under +// its download name. func TestLocalBpeLoader_ReadsBundledVocabFromAncestor(t *testing.T) { dir := isolate(t) writeBpeTable(t, filepath.Join(dir, "ragflow_deps", "cl100k_base.tiktoken"), 10) diff --git a/internal/tokenizer/counter_oracle_test.go b/internal/tokenizer/counter_oracle_test.go index 4ad6d4c019..2e9b22927e 100644 --- a/internal/tokenizer/counter_oracle_test.go +++ b/internal/tokenizer/counter_oracle_test.go @@ -118,7 +118,7 @@ func checkCounterAgainstOracle(t *testing.T, fx oracleFixture) { // asset it needs is not on disk. for _, status := range CounterStatuses() { if status.ID == fx.Counter { - t.Fatalf("fixture %q needs counter %q, which is declared but unavailable: its asset is missing from ragflow_deps/huggingface.co (run `uv run ragflow_deps/download_go_deps.py`)", fx.Model, fx.Counter) + t.Fatalf("fixture %q needs counter %q, which is declared but unavailable: its asset is missing from ragflow_deps/huggingface.co (run `uv run ragflow_deps/download_deps.py`)", fx.Model, fx.Counter) } } t.Fatalf("fixture %q names counter %q, which this build does not implement", fx.Model, fx.Counter) diff --git a/internal/tokenizer/embedding_token_limits.md b/internal/tokenizer/embedding_token_limits.md index 9a0ec6c4d9..a9ef2f098e 100644 --- a/internal/tokenizer/embedding_token_limits.md +++ b/internal/tokenizer/embedding_token_limits.md @@ -180,7 +180,7 @@ that declares an unavailable counter fails instead of being calibrated silently. | id | family / models | asset (SHA-1 pinned) | implementation | |---|---|---|---| -| `cl100k_base` | OpenAI `text-embedding-3-*`, `ada-002` | `ragflow_deps/cl100k_base.tiktoken` (pre-existing) | tiktoken-go + offline loader (`bpe_loader.go`) | +| `cl100k_base` | OpenAI `text-embedding-3-*`, `ada-002` | `ragflow_deps/cl100k_base.tiktoken` (`download_deps.py`) | tiktoken-go + offline loader (`bpe_loader.go`) | | `xlmr-spm` | `bge-m3`, `multilingual-e5-*`, `m3e`, `gte-multilingual`, `jina-embeddings-v3` | `…/BAAI/bge-m3/sentencepiece.bpe.model` `7e88c49f…` | `spm.go`: Unigram Viterbi, byte fallback, NFKC + whitespace + charmap-lite | | `bert-wordpiece` | `bge-{large,base,small}-en-v1.5`, `bge-en-icl`, `e5-{base,large}-v2`, `gte-{base,large}`, `jina-embeddings-v2-*` | `…/BAAI/bge-large-en-v1.5/vocab.txt` `c3b41053…` | `wordpiece.go`: BertNormalizer + BertPreTokenizer + `##` longest-match | | `qwen-bpe` | `Qwen3-Embedding-*` | `…/Qwen/Qwen3-Embedding-0.6B/tokenizer.json` `e6592f4d…` | `bpe.go`: hand-written Split scanner, GPT-2 byte alphabet, BPE | @@ -347,11 +347,11 @@ an unrelated `vocab.txt` there silently replaces the verified artifact; and audi 1. **Identify the architecture** from the model's own artifact (`tokenizer.json`, or `vocab.txt` / `sentencepiece.bpe.model`). If the vocabulary is byte-identical to a family fixture, only step 4 is needed; if it differs, the model needs its own fixture. -2. **Add the artifact** to `ragflow_deps/download_go_deps.py`'s `TOKENIZER_ASSETS` list - +2. **Add the artifact** to `ragflow_deps/download_deps.py`'s `TOKENIZER_ASSETS` list - individual files, not `snapshot_download` of a multi-GB repo - pin its SHA-1 in the loader map so a mismatch fails at load instead of counting with the wrong table, and add - it to the copy loop in `Dockerfile` / `Dockerfile_base` / `Dockerfile_go` so every runtime image ships it - (the `ragflow_deps` image is built from this script's output, so it needs a refresh too - + it to the copy loop in `Dockerfile` / `Dockerfile_base` so every runtime image ships it + (the published `ragflow_deps` image also needs the asset, so refresh it - see §Assets and the oracle for the whole chain). 3. **Add the fixture line** to `ORACLES` in `scripts/gen_tokenizer_oracle.py`, generate it, and run `TestCountersMatchOracle`. Passing means every sample agrees, or sits inside the @@ -388,8 +388,7 @@ property tier green, `gofmt` clean. ## Assets and the oracle -Assets are fetched, never committed: `ragflow_deps/download_go_deps.py` (the Go-side -downloader - `download_deps.py` is upstream and stays untouched) has a `TOKENIZER_ASSETS` +Assets are fetched, never committed: `ragflow_deps/download_deps.py` has a `TOKENIZER_ASSETS` list that downloads **individual files** (not `snapshot_download` of a repo that also carries multi-GB weights) into `ragflow_deps/huggingface.co//`, which `.gitignore` already excludes (`huggingface.co/`). Loaders read them from disk with a @@ -397,13 +396,12 @@ SHA-1 pin, walk up from the working and executable directories like `bpe_loader. perform no network I/O. A missing asset fails loudly at every layer that can report one: the runtime image does not build without it (the copy loop below exits non-zero), an embedder whose model declares a tokenizer it cannot load refuses to count rather than substituting the -calibrated estimate, and the cl100k table - the one asset that still comes from -`download_deps.py`, which fetches it from the OpenAI blob - has always been a startup panic. +calibrated estimate, and the cl100k table fetched from the OpenAI blob has always been a startup panic when missing. -**The whole chain, because a gap in it is invisible**: `download_go_deps.py` writes -`ragflow_deps/huggingface.co/…`; the `ragflow_deps` image is built from that directory -(`ragflow_deps/Dockerfile` copies `huggingface.co` to `/huggingface.co`); the root -`Dockerfile`, `Dockerfile_base` and `Dockerfile_go` bind-mount that image and copy the four *runtime* +**The whole chain, because a gap in it is invisible**: `download_deps.py` writes +`ragflow_deps/huggingface.co/…`; the published `ragflow_deps` image contains those +files under `/huggingface.co`; the root +`Dockerfile` and `Dockerfile_base` bind-mount that image and copy the four *runtime* tokenizer assets into `/ragflow/ragflow_deps/huggingface.co/…`, which is where the counters look (the working directory is `/ragflow`). The `tokenizer.json` files that exist only as cross-check oracles are deliberately **not** shipped. Two consequences @@ -435,11 +433,14 @@ an operator can point at whichever tree they have: | a `huggingface.co`-like root | `//` | | a flat directory | `/` | -The variable is deliberately **not** embedding-specific: DeepDoc's weights use the same -layout (`huggingface.co/InfiniFlow/deepdoc`) and `resolveDeepDocModelDir` honours it too, -so one mount can serve every downloaded model asset. `ragflow_deps/download_go_deps.py` and -the three runtime Dockerfiles keep writing the same tree, so nothing changes for a default -deployment. The layout and the precedence are pinned by tests: +The variable is deliberately **not** embedding-specific: `resolveDeepDocModelDir` +also checks `/huggingface.co/InfiniFlow/deepdoc` when `MODEL_ASSETS_DIR` +points at ``. The downloader prepares the five Go DeepDoc files in that tree +under `ragflow_deps/`, as well as in `internal/rag/res/deepdoc/` for local use. +One mounted `ragflow_deps/` directory can therefore serve both asset types. +`ragflow_deps/Dockerfile` packages the tree under `/huggingface.co/` when built +from `ragflow_deps/`. +The layout and the precedence are pinned by tests: `internal/common/model_assets_test.go` for the candidate list, and `internal/tokenizer/asset_dir_test.go` for the end-to-end case (a child process with an unrelated working directory loads the SPM counter from the configured tree). @@ -449,7 +450,7 @@ Two things make the state of the assets observable: - each counter reports the file it loaded (`SourcePath`), so logs and failures can name it; - the ingestor logs an **availability report once at startup** - `embedding tokenizer counters {available: [...]}` plus, when something is missing, a - warning naming the unavailable ones and pointing at `ragflow_deps/download_go_deps.py` / + warning naming the unavailable ones and pointing at `ragflow_deps/download_deps.py` / `MODEL_ASSETS_DIR`. The report is diagnostics rather than the guard: startup is not fatal (untagged models count with the calibrated estimate by design, and they keep working), but ingesting a model that declares a missing asset fails outright - so this report is what @@ -671,8 +672,8 @@ means "what the model does": against the model is not tagged in the catalog; it falls back to the calibrated path. Guessing a tag is the unsafe direction. 2. **Nothing is hand-typed.** Fixtures and expected counts come from the model's - tokenizer; assets come from the downloaders (`download_go_deps.py` for the tokenizer - files, `download_deps.py` for the cl100k table) with a SHA-1 pin. + tokenizer; model tokenizer assets and the cl100k table come from `download_deps.py` + with a SHA-1 pin. 3. **Equality, not resemblance** (see above), and the dangerous direction (under-count) is the one the tests are shaped around. 4. **Every approximation is named** in §Known approximations together with the diff --git a/internal/tokenizer/embedlimit.go b/internal/tokenizer/embedlimit.go index b6990df1ba..18b56075d0 100644 --- a/internal/tokenizer/embedlimit.go +++ b/internal/tokenizer/embedlimit.go @@ -663,7 +663,7 @@ func RefuseUnavailableCounter(tokenizerID, calibrationKey string) error { return nil } return fmt.Errorf( - "model tokenizer %q is declared for %s but its asset is unavailable (check that ragflow_deps/huggingface.co is present; run `uv run ragflow_deps/download_go_deps.py`): refusing to count with the calibrated estimate", + "model tokenizer %q is declared for %s but its asset is unavailable (check that ragflow_deps/huggingface.co is present; run `uv run ragflow_deps/download_deps.py`): refusing to count with the calibrated estimate", tokenizerID, calibrationKey) } diff --git a/internal/tokenizer/spm.go b/internal/tokenizer/spm.go index 03540bcfc5..2e88499f8d 100644 --- a/internal/tokenizer/spm.go +++ b/internal/tokenizer/spm.go @@ -850,7 +850,7 @@ func readTokenizerAsset(names []string, pinKey string, pins map[string]string) ( } return candidate, raw, nil } - return "", nil, fmt.Errorf("no tokenizer asset found; run `uv run ragflow_deps/download_go_deps.py` or set %s to a directory holding them; tried: %s", + return "", nil, fmt.Errorf("no tokenizer asset found; run `uv run ragflow_deps/download_deps.py` or set %s to a directory holding them; tried: %s", common.EnvModelAssetsDir, strings.Join(tried, ", ")) } diff --git a/ragflow_deps/Dockerfile b/ragflow_deps/Dockerfile index 236b38eb71..ade16bfd9b 100644 --- a/ragflow_deps/Dockerfile +++ b/ragflow_deps/Dockerfile @@ -1,10 +1,7 @@ -# This builds an image that contains the resources needed by Dockerfile -# +# Build from ragflow_deps/ after running download_deps.py: +# docker build -f Dockerfile -t infiniflow/ragflow_deps:latest . FROM scratch -# Copy resources downloaded via download_deps.py -COPY chromedriver-linux64-121-0-6167-85 chrome-linux64-121-0-6167-85 cl100k_base.tiktoken libssl1.1_1.1.1f-1ubuntu2_amd64.deb libssl1.1_1.1.1f-1ubuntu2_arm64.deb tika-server-standard-3.3.0.jar tika-server-standard-3.3.0.jar.md5 libssl*.deb uv-x86_64-unknown-linux-gnu.tar.gz uv-aarch64-unknown-linux-gnu.tar.gz stagehand-server-v3-linux-x64 stagehand-server-v3-linux-arm64 / - -COPY nltk_data /nltk_data +COPY cl100k_base.tiktoken stagehand-server-v3-linux-x64 stagehand-server-v3-linux-arm64 / COPY huggingface.co /huggingface.co diff --git a/ragflow_deps/download_deps.py b/ragflow_deps/download_deps.py index fdb8ad2536..0a7f3970ab 100755 --- a/ragflow_deps/download_deps.py +++ b/ragflow_deps/download_deps.py @@ -4,63 +4,71 @@ # /// script # requires-python = ">=3.10" # dependencies = [ -# "nltk", -# "huggingface-hub" +# "huggingface-hub", +# "requests", # ] # /// -# This script downloads every artifact that the `infiniflow/ragflow_deps` -# Docker image bakes in. Run it from anywhere — the `__main__` block -# chdir's into this file's own directory, so all outputs land under -# `ragflow_deps/` regardless of the caller's CWD. +# This script prepares Go dependencies on Linux x86_64: the native static +# libraries (pdfium / pdf_oxide / office_oxide / onnxruntime) for `build.sh`, +# and the Go DeepDoc `.ort` weights (det/layout/tsr/rec.ort + ocr.res) so a Go +# dev can run the in-process backend locally. It also downloads the cl100k BPE +# table and installs the local stagehand driver in the Go SDK's cache. +# Archives and tokenizer assets land under ragflow_deps/; DeepDoc weights land +# under internal/rag/res/deepdoc/, regardless of the caller's working directory. # -# Build-context relationship: `ragflow_deps/Dockerfile` is built with -# `ragflow_deps/` as its build context, so the files written here MUST -# sit at the top of `ragflow_deps/`. The Dockerfile's COPY lines assume -# top-level paths (`huggingface.co`, `nltk_data`, `cl100k_base.tiktoken`, -# `*.deb`, `*.jar`, `*.tar.gz`, `stagehand-server-v3-linux-`). +# Downloaded archives and tokenizer assets land under `ragflow_deps/`. # # Typical workflow: # -# uv run python3 ragflow_deps/download_deps.py # download -# cd ragflow_deps -# docker build -f Dockerfile -t infiniflow/ragflow_deps . +# uv run ragflow_deps/download_deps.py +# Go DeepDoc weights: in addition to the native libs, this script downloads the +# five Go model files (internal/common.DeepDocModelFiles) from InfiniFlow/deepdoc +# straight into the repo's canonical model directory `internal/rag/res/deepdoc/` (one level +# up from this script). The Go server auto-discovers that directory via +# resolveDeepDocModelDir(), and build.sh --test-native / internal/deepdoc/native/run.sh +# default MODEL_DIR there too — so after running this script NO MODEL_DIR env needs +# to be set: # -# The main `Dockerfile` (built from the project root) pulls this image -# via `--mount=type=bind,from=infiniflow/ragflow_deps:latest,...` and -# is unaffected by where these files live locally. +# bash build.sh --test-native # or: cd internal/deepdoc/native && bash run.sh +# +# Embedding tokenizer assets: the Go token counters in internal/tokenizer load a +# tokenizer file per family (XLM-R SentencePiece, BERT WordPiece, two byte-level BPE +# families) from `huggingface.co//` at the top of ragflow_deps/ - the path +# the loaders search and the one ragflow_deps/Dockerfile ships. This script fetches +# them too. Models that declare a missing tokenizer cannot be ingested +# (see internal/tokenizer/embedding_token_limits.md). import argparse import hashlib import os +import platform +import re import shutil import sys -import urllib.request +import tarfile +import tempfile +import zipfile -# NLTK >=3.10 refuses proxied downloads (SSRF guard) unless opted in; the -# runners sit behind a proxy, so allow proxied fetches before importing nltk. -os.environ.setdefault("NLTK_ALLOW_PROXIED_URLOPEN", "1") +import requests -import nltk -from huggingface_hub import snapshot_download - -# mirrors internal/common.DeepDocORTVersion (Go in-process backend). ONE OF -# FOUR places (with that Go constant, ORT_VERSION in ragflow_deps/download_go_deps.py, -# and ARG ORT_VERSION in Dockerfile_go) that must carry the same ONNX Runtime -# native release for the statically-linked Go DeepDoc backend. This file's -# download URL and extracted dir name are derived from ORT_VERSION here, but -# there is no single source of truth — keep all four equal. build.sh -# --check-ort-version greps this file (and the other three) to fail fast on -# drift. (The Python pip onnxruntime== pin in pyproject.toml is versioned -# independently and is not part of this check.) +# Mirrors internal/common.DeepDocORTVersion (Go in-process backend). ONE OF +# THREE places (with that Go constant and ARG ORT_VERSION in Dockerfile) +# that must carry the same ONNX Runtime +# native release for the statically-linked Go DeepDoc backend. There is no +# single source of truth — keep all three equal. build.sh --check-ort-version +# greps this file (and the other two) to fail fast on drift. (The Python pip +# onnxruntime== pin in pyproject.toml is versioned independently and is not +# part of this check.) # # Source of the native static archives: infiniflow/ragflow-build (our own # ORT-only minimal build), NOT the third-party csukuangfj/onnxruntime-libs # account. The release tag is `onnxruntime-v{ORT_VERSION}` and the asset is # `onnxruntime-v{ORT_VERSION}-linux-x86_64.zip`. The archive is occasionally -# re-issued under this SAME tag/asset name with patched content; download_deps.py -# detects that via a `{asset}.sha256` sidecar and re-downloads/re-extracts, so a -# stale local copy never silently lingers. +# re-issued under this SAME tag/asset name with patched content; this script +# detects that via a `{asset}.sha256` +# sidecar and re-download/re-extract, so a stale local copy never silently +# lingers. ORT_VERSION = "1.29.0" @@ -88,18 +96,115 @@ def _sha256_of(path): return h.hexdigest() +# Mirrors internal/common.DeepDocModelFiles (Go in-process DeepDoc backend). +# These are the ONLY weights the Go backend loads; the full InfiniFlow/deepdoc +# repo also ships .onnx (Python-only), which this Go-only script deliberately +# skips to keep the download lean. +DEEPDOC_REPO = "InfiniFlow/deepdoc" +DEEPDOC_MODEL_FILES = ["det.ort", "layout.ort", "tsr.ort", "rec.ort", "ocr.res"] + +# Embedding tokenizer assets for the Go counters (internal/tokenizer). Each entry is +# (repo, file, kind): +# +# "runtime" - the counters load it in production, so the runtime images must ship it +# (the copy loop in Dockerfile / Dockerfile_base) and the loader must pin +# its SHA-1; +# "oracle" - only scripts/gen_tokenizer_oracle.py needs it, to regenerate the test +# fixtures, so it is deliberately NOT shipped to the image. +# +# ragflow_deps/test_tokenizer_assets.py checks these download, packaging, and pin lists. +# +# Fetched per file, not by snapshot: these repos also carry multi-GB weights we do not +# want. Missing tokenizer assets fail dependency preparation. +TOKENIZER_ASSETS = [ + # XLM-R SentencePiece (Unigram) - the BAAI bge / multilingual-e5 / m3e / + # gte-multilingual / jina-v3 family shares this vocabulary. + ("BAAI/bge-m3", "sentencepiece.bpe.model", "runtime"), + # BERT WordPiece vocab for the bge-*-en / e5-* / gte-base family. + ("BAAI/bge-large-en-v1.5", "vocab.txt", "runtime"), + # Byte-level BPE families (Qwen3-Embedding, Mistral/Llama embeddings). + ("Qwen/Qwen3-Embedding-0.6B", "tokenizer.json", "runtime"), + ("intfloat/e5-mistral-7b-instruct", "tokenizer.json", "runtime"), + # Cross-check oracles only: the HF tokenizer.json files the same families convert to. + ("BAAI/bge-m3", "tokenizer.json", "oracle"), + ("BAAI/bge-large-en-v1.5", "tokenizer.json", "oracle"), +] + + +def prune_stale_onnxruntime(static_lib_dir, version): + """Remove ONNX Runtime version dirs under static_lib that do NOT match + `version`. Without this, a version bump leaves the stale dir next to + the new one and build.sh's `find ... -name '*.a'` links BOTH (duplicate + symbols / wrong version, silently).""" + if not os.path.isdir(static_lib_dir): + return + expected = _ort_normalized_dir(version) + for name in os.listdir(static_lib_dir): + if not name.startswith("onnxruntime-linux-x64-static_lib-"): + continue + if name == expected: + continue + stale = os.path.join(static_lib_dir, name) + print(f" Removing stale ONNX Runtime dir: {stale}") + shutil.rmtree(stale) + + +def has_static_archives(directory): + """True when `directory` holds at least one static archive (.a).""" + return any(f.endswith(".a") for _, _, files in os.walk(directory) for f in files) + + +def extract_onnxruntime(static_lib_dir, archive_path, version): + """Ensure the ONNX Runtime static archives for `version` sit under + `static_lib_dir`. Returns True when that version is available afterwards + (extracted now or already present), False when the archive is missing. + + The infiniflow/ragflow-build release zip carries a top-level dir named + onnxruntime-v{version}-linux-x86_64, so a present + `static_lib_dir` is NOT evidence that THIS version is extracted: after a + version bump the stale dir is pruned and the new one must be extracted. + """ + if not os.path.isfile(archive_path): + print(f" Skipping extraction: {os.path.basename(archive_path)} not found") + return False + prune_stale_onnxruntime(static_lib_dir, version) + version_dir = os.path.join(static_lib_dir, _ort_normalized_dir(version)) + if os.path.isdir(version_dir) and has_static_archives(version_dir): + print(f" ✓ onnxruntime/static_lib ({version}) already extracted to {version_dir}") + return True + os.makedirs(static_lib_dir, exist_ok=True) + print(f" Extracting {os.path.basename(archive_path)} → {static_lib_dir}") + with zipfile.ZipFile(archive_path) as zf: + zf.extractall(static_lib_dir) + # The infiniflow/ragflow-build release zip carries a top-level dir named + # onnxruntime-v{version}-linux-x86_64, but build.sh's glob and the stale + # checks above all expect onnxruntime-linux-x64-static_lib-{version}-glibc2_28. + # Rename it so every consumer shares one name convention (driven by + # ORT_VERSION). + extracted = os.path.join(static_lib_dir, _ort_extracted_dir(version)) + normalized = os.path.join(static_lib_dir, _ort_normalized_dir(version)) + if os.path.isdir(extracted) and extracted != normalized: + if os.path.exists(normalized): + shutil.rmtree(normalized) + print(f" Renaming {os.path.basename(extracted)} → {os.path.basename(normalized)}") + os.rename(extracted, normalized) + return True + + +def _go_module_version(module): + repo_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + with open(os.path.join(repo_root, "go.mod")) as file: + version = re.search(rf"{re.escape(module)}\s+v([^\s]+)", file.read()) + if version is None: + raise RuntimeError(f"{module} version not found in go.mod") + return version.group(1) + + def get_urls(use_china_mirrors=False) -> list[str | list[str]]: + office_version = _go_module_version("github.com/yfedoseev/office_oxide/go") if use_china_mirrors: return [ - "http://mirrors.tuna.tsinghua.edu.cn/ubuntu/pool/main/o/openssl/libssl1.1_1.1.1f-1ubuntu2_amd64.deb", - "http://mirrors.tuna.tsinghua.edu.cn/ubuntu-ports/pool/main/o/openssl/libssl1.1_1.1.1f-1ubuntu2_arm64.deb", - "https://repo.huaweicloud.com/repository/maven/org/apache/tika/tika-server-standard/3.3.0/tika-server-standard-3.3.0.jar", - "https://repo.huaweicloud.com/repository/maven/org/apache/tika/tika-server-standard/3.3.0/tika-server-standard-3.3.0.jar.md5", "https://openaipublic.blob.core.windows.net/encodings/cl100k_base.tiktoken", - ["https://registry.npmmirror.com/-/binary/chrome-for-testing/121.0.6167.85/linux64/chrome-linux64.zip", "chrome-linux64-121-0-6167-85"], - ["https://registry.npmmirror.com/-/binary/chrome-for-testing/121.0.6167.85/linux64/chromedriver-linux64.zip", "chromedriver-linux64-121-0-6167-85"], - "https://github.com/astral-sh/uv/releases/download/0.9.16/uv-x86_64-unknown-linux-gnu.tar.gz", - "https://github.com/astral-sh/uv/releases/download/0.9.16/uv-aarch64-unknown-linux-gnu.tar.gz", # stagehand-server-v3 Node.js SEA binaries (used by Browser # component in local mode). # @@ -114,42 +219,22 @@ def get_urls(use_china_mirrors=False) -> list[str | list[str]]: # latest server release. There is no version # correspondence to maintain; "both on v3.x" is the # compatibility contract. - "https://github.com/browserbase/stagehand/releases/download/stagehand-server-v3/v3.7.2/stagehand-server-v3-linux-x64", - "https://github.com/browserbase/stagehand/releases/download/stagehand-server-v3/v3.7.2/stagehand-server-v3-linux-arm64", - # Native static libraries for Go build (pdfium, pdf_oxide, office_oxide) - # Used by build.sh's check_*_deps functions — pre-downloaded to avoid - # network access during CI. - ["https://github.com/kognitos/pdfium-static/releases/download/chromium%2F7809/pdfium-linux-x64-static.tgz", "pdfium-linux-x64-static.tgz"], - ["https://github.com/yfedoseev/pdf_oxide/releases/download/v0.3.73/pdf_oxide-go-ffi-linux-amd64.tar.gz", "pdf_oxide-go-ffi-linux-amd64.tar.gz"], - ["https://github.com/yfedoseev/office_oxide/releases/download/v0.1.12/native-linux-x86_64.tar.gz", "office_oxide-linux-x86_64.tar.gz"], - # ONNX Runtime static archives for the Go in-process (DeepDoc) - # backend. Statically linked into the server binary (see build.sh: - # ONNXRUNTIME_STATIC_PREFIX — no --whole-archive, so unreferenced - # kernels are dropped; only OrtGetApiBase is exported, via - # --dynamic-list), so no libonnxruntime.so is needed at runtime — - # OrtGetApiBase is resolved via dlopen(NULL) (the process-global - # symbol table, not the executable's own path). Our own - # infiniflow/ragflow-build ORT-only minimal build - # (onnxruntime-v{ORT_VERSION}) is CPU-only and glibc2_28-based, - # matching ORT_VERSION's C-API line (ABI-compatible with - # onnxruntime_go) and the onnxruntime the Python goldens were - # generated with. + "https://gh-proxy.com/https://github.com/browserbase/stagehand/releases/download/stagehand-server-v3/v3.7.2/stagehand-server-v3-linux-x64", + "https://gh-proxy.com/https://github.com/browserbase/stagehand/releases/download/stagehand-server-v3/v3.7.2/stagehand-server-v3-linux-arm64", + # Native static libraries for Go build (pdfium, pdf_oxide, + # office_oxide, onnxruntime). Used by build.sh's check_*_deps + # functions — pre-downloaded to avoid network access during CI. + ["https://gh-proxy.com/https://github.com/kognitos/pdfium-static/releases/download/chromium%2F7809/pdfium-linux-x64-static.tgz", "pdfium-linux-x64-static.tgz"], + ["https://gh-proxy.com/https://github.com/yfedoseev/pdf_oxide/releases/download/v0.3.73/pdf_oxide-go-ffi-linux-amd64.tar.gz", "pdf_oxide-go-ffi-linux-amd64.tar.gz"], + [f"https://gh-proxy.com/https://github.com/yfedoseev/office_oxide/releases/download/v{office_version}/native-linux-x86_64.tar.gz", "office_oxide-linux-x86_64.tar.gz"], [ - f"https://github.com/infiniflow/ragflow-build/releases/download/onnxruntime-v{ORT_VERSION}/{_ort_asset_name(ORT_VERSION)}", + f"https://gh-proxy.com/https://github.com/infiniflow/ragflow-build/releases/download/onnxruntime-v{ORT_VERSION}/{_ort_asset_name(ORT_VERSION)}", _ort_asset_name(ORT_VERSION), ], ] else: return [ - "http://archive.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.1f-1ubuntu2_amd64.deb", - "http://ports.ubuntu.com/pool/main/o/openssl/libssl1.1_1.1.1f-1ubuntu2_arm64.deb", - "https://repo1.maven.org/maven2/org/apache/tika/tika-server-standard/3.3.0/tika-server-standard-3.3.0.jar", - "https://repo1.maven.org/maven2/org/apache/tika/tika-server-standard/3.3.0/tika-server-standard-3.3.0.jar.md5", "https://openaipublic.blob.core.windows.net/encodings/cl100k_base.tiktoken", - ["https://storage.googleapis.com/chrome-for-testing-public/121.0.6167.85/linux64/chrome-linux64.zip", "chrome-linux64-121-0-6167-85"], - ["https://storage.googleapis.com/chrome-for-testing-public/121.0.6167.85/linux64/chromedriver-linux64.zip", "chromedriver-linux64-121-0-6167-85"], - "https://github.com/astral-sh/uv/releases/download/0.9.16/uv-x86_64-unknown-linux-gnu.tar.gz", - "https://github.com/astral-sh/uv/releases/download/0.9.16/uv-aarch64-unknown-linux-gnu.tar.gz", # stagehand-server-v3 Node.js SEA binaries (used by Browser # component in local mode). # @@ -166,24 +251,12 @@ def get_urls(use_china_mirrors=False) -> list[str | list[str]]: # compatibility contract. "https://github.com/browserbase/stagehand/releases/download/stagehand-server-v3/v3.7.2/stagehand-server-v3-linux-x64", "https://github.com/browserbase/stagehand/releases/download/stagehand-server-v3/v3.7.2/stagehand-server-v3-linux-arm64", - # Native static libraries for Go build (pdfium, pdf_oxide, office_oxide) - # Used by build.sh's check_*_deps functions — pre-downloaded to avoid - # network access during CI. + # Native static libraries for Go build (pdfium, pdf_oxide, + # office_oxide, onnxruntime). Used by build.sh's check_*_deps + # functions — pre-downloaded to avoid network access during CI. ["https://github.com/kognitos/pdfium-static/releases/download/chromium%2F7809/pdfium-linux-x64-static.tgz", "pdfium-linux-x64-static.tgz"], ["https://github.com/yfedoseev/pdf_oxide/releases/download/v0.3.73/pdf_oxide-go-ffi-linux-amd64.tar.gz", "pdf_oxide-go-ffi-linux-amd64.tar.gz"], - ["https://github.com/yfedoseev/office_oxide/releases/download/v0.1.12/native-linux-x86_64.tar.gz", "office_oxide-linux-x86_64.tar.gz"], - # ONNX Runtime static archives for the Go in-process (DeepDoc) - # backend. Statically linked into the server binary (see build.sh: - # ONNXRUNTIME_STATIC_PREFIX — no --whole-archive, so unreferenced - # kernels are dropped; only OrtGetApiBase is exported, via - # --dynamic-list), so no libonnxruntime.so is needed at runtime — - # OrtGetApiBase is resolved via dlopen(NULL) (the process-global - # symbol table, not the executable's own path). Our own - # infiniflow/ragflow-build ORT-only minimal build - # (onnxruntime-v{ORT_VERSION}) is CPU-only and glibc2_28-based, - # matching ORT_VERSION's C-API line (ABI-compatible with - # onnxruntime_go) and the onnxruntime the Python goldens were - # generated with. + [f"https://github.com/yfedoseev/office_oxide/releases/download/v{office_version}/native-linux-x86_64.tar.gz", "office_oxide-linux-x86_64.tar.gz"], [ f"https://github.com/infiniflow/ragflow-build/releases/download/onnxruntime-v{ORT_VERSION}/{_ort_asset_name(ORT_VERSION)}", _ort_asset_name(ORT_VERSION), @@ -191,23 +264,238 @@ def get_urls(use_china_mirrors=False) -> list[str | list[str]]: ] -repos = [ - "InfiniFlow/text_concat_xgb_v1.0", - "InfiniFlow/deepdoc", -] +def download_with_progress(url, filename): + filename = os.fspath(filename) + temporary = None + try: + with requests.get(url, stream=True, timeout=(15, 60)) as response: + response.raise_for_status() + total_size = int(response.headers.get("content-length", 0)) + with tempfile.NamedTemporaryFile(dir=os.path.dirname(os.path.abspath(filename)), delete=False) as file: + temporary = file.name + downloaded = 0 + for data in response.iter_content(1 << 20): + file.write(data) + downloaded += len(data) + if total_size > 0: + progress = (downloaded / total_size) * 100 + sys.stdout.write(f"\rProgress: {progress:.1f}% ({downloaded}/{total_size} bytes)") + sys.stdout.flush() + if downloaded == 0: + raise requests.RequestException(f"Empty dependency download: {url}") + os.replace(temporary, filename) + finally: + if temporary is not None and os.path.exists(temporary): + os.unlink(temporary) + print() -def download_model(repository_id): - local_directory = os.path.abspath(os.path.join("huggingface.co", repository_id)) - os.makedirs(local_directory, exist_ok=True) - snapshot_download(repo_id=repository_id, local_dir=local_directory) +def _extract_native_archive(archive_path, target, required_files, version=None): + def complete(directory): + for name in required_files: + path = os.path.join(directory, name) + if not os.path.isfile(path) or os.path.getsize(path) == 0: + return False + if version is not None and name.endswith(".a"): + with open(path, "rb") as file: + if b"\x00" + version.encode() + b"\x00" not in file.read(): + return False + return True + + if complete(target): + print(f" ✓ {os.path.basename(target)} already extracted to {target}") + return + os.makedirs(os.path.dirname(target), exist_ok=True) + with tempfile.TemporaryDirectory(dir=os.path.dirname(target)) as staging: + with tarfile.open(archive_path) as archive: + members = archive.getmembers() + for member in members: + if os.path.isabs(member.name) or ".." in member.name.split("/") or not (member.isfile() or member.isdir()): + raise tarfile.TarError(f"Unsafe native archive member: {member.name}") + archive.extractall(staging, members=members) + if not complete(staging): + raise RuntimeError(f"{archive_path} is missing required native files or version {version}") + if os.path.isdir(target): + shutil.rmtree(target) + os.replace(staging, target) + print(f" ✓ extracted {os.path.basename(archive_path)} to {target}") + + +def _install_stagehand(): + if platform.system() != "Linux": + raise RuntimeError("The downloaded native dependencies support Linux only") + architecture = {"x86_64": "x64", "aarch64": "arm64"}.get(platform.machine()) + if architecture is None: + raise RuntimeError(f"Unsupported stagehand architecture: {platform.machine()}") + version = _go_module_version("github.com/browserbase/stagehand-go/v3") + cache_root = os.environ.get("XDG_CACHE_HOME") or os.path.expanduser("~/.cache") + target_dir = os.path.join(cache_root, "stagehand", "lib", f"go_{version}") + os.makedirs(target_dir, exist_ok=True) + filename = f"stagehand-server-v3-linux-{architecture}" + source = os.path.join(os.path.dirname(os.path.abspath(__file__)), filename) + target = os.path.join(target_dir, filename) + with tempfile.TemporaryDirectory(dir=target_dir) as staging: + prepared = os.path.join(staging, filename) + shutil.copyfile(source, prepared) + os.chmod(prepared, 0o755) + os.replace(prepared, target) + print(f" ✓ stagehand installed to {target}") + + +def _valid_download(filename): + if not os.path.isfile(filename) or os.path.getsize(filename) == 0: + return False + if filename.endswith((".tar.gz", ".tgz")): + if not tarfile.is_tarfile(filename): + return False + if os.path.basename(filename) == "office_oxide-linux-x86_64.tar.gz": + version = _go_module_version("github.com/yfedoseev/office_oxide/go") + with tarfile.open(filename) as archive: + try: + member = next((member for member in archive.getmembers() if os.path.normpath(member.name) == "lib/liboffice_oxide.a"), None) + if member is None: + return False + library = archive.extractfile(member) + if library is None: + return False + with library: + return b"\x00" + version.encode() + b"\x00" in library.read() + except (KeyError, tarfile.TarError, EOFError): + return False + return True + if filename.endswith(".zip"): + return zipfile.is_zipfile(filename) + if os.path.basename(filename).startswith("stagehand-server-"): + with open(filename, "rb") as file: + return file.read(4) == b"\x7fELF" + return True + + +def download_go_models(use_china_mirrors=False): + """Download the Go DeepDoc `.ort` weights so a Go dev can run the in-process + backend with no further setup. + + The files are written into the repo's canonical model directory + `internal/rag/res/deepdoc/` (relative to the repo root, one level up from this + script). The Go server auto-discovers that directory via + resolveDeepDocModelDir(), and build.sh --test-native / run.sh default + MODEL_DIR there too — so after this script runs, no MODEL_DIR env needs to + be set. + + Uses hf_hub_download per-file (not snapshot_download) to fetch only the + five Go model files; the `.onnx` siblings are Python-only and skipped. + On China mirrors, route through hf-mirror.com via HF_ENDPOINT. + """ + if use_china_mirrors: + os.environ["HF_ENDPOINT"] = "https://hf-mirror.com" + # Imported lazily so the module stays importable (and its unit tests stay + # huggingface-free) without the huggingface_hub dependency installed. + from huggingface_hub import hf_hub_download + + # Canonical local-dev model dir the Go backend auto-discovers. + repo_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + target_dir = os.path.join(repo_root, "internal", "rag", "res", "deepdoc") + os.makedirs(target_dir, exist_ok=True) + + missing = [] + for fname in DEEPDOC_MODEL_FILES: + dest = os.path.join(target_dir, fname) + if os.path.isfile(dest) and os.path.getsize(dest) > 0: + print(f" ✓ {fname} already present") + continue + print(f"Downloading deepdoc model {fname}...") + try: + hf_hub_download(repo_id=DEEPDOC_REPO, filename=fname, local_dir=target_dir) + if not os.path.isfile(dest) or os.path.getsize(dest) == 0: + raise RuntimeError(f"Empty or missing model: {dest}") + except Exception as e: # noqa: BLE001 - collected and surfaced below + missing.append((fname, e)) + + if missing: + for fname, e in missing: + print(f" ERROR: failed to download {fname}: {e}", file=sys.stderr) + print( + "\n" + "The Go in-process DeepDoc backend loads ONLY the .ort weights listed in\n" + "internal/common.DeepDocModelFiles. They are fetched from " + f"{DEEPDOC_REPO} into:\n" + f" {target_dir}\n" + "Without them the backend cannot serve — the server exits with a fatal\n" + '"no in-process DeepDoc backend serving". To recover:\n' + " - re-run this script (a transient HF/network error usually clears);\n" + " - behind the GFW, re-run with --china-mirrors (routes via hf-mirror.com);\n" + " - or copy the missing files into that directory by hand.", + file=sys.stderr, + ) + sys.exit(1) + + # Keep the dependency image's existing huggingface.co build-context layout. + image_dir = os.path.join(repo_root, "ragflow_deps", "huggingface.co", DEEPDOC_REPO) + os.makedirs(image_dir, exist_ok=True) + for fname in DEEPDOC_MODEL_FILES: + shutil.copyfile(os.path.join(target_dir, fname), os.path.join(image_dir, fname)) + + print(f" ✓ Go DeepDoc models ready under {target_dir}") + print(" No MODEL_DIR env needed: the Go backend auto-discovers this directory.") + + +def download_tokenizer_assets(use_china_mirrors=False): + """Download the embedding tokenizer assets the Go counters load. + + Written to `ragflow_deps/huggingface.co//`: the path internal/tokenizer + searches and the one ragflow_deps/Dockerfile builds its image from. Fetched per file + (these repos also carry multi-GB weights we do not want), routed via hf-mirror.com + when --china-mirrors is set. + + A missing asset is fatal: models declaring its tokenizer refuse ingestion. + Runtime images also reject missing files during their copy step. + """ + if use_china_mirrors: + os.environ["HF_ENDPOINT"] = "https://hf-mirror.com" + # Imported lazily so the module stays importable (and its unit tests stay + # huggingface-free) without the huggingface_hub dependency installed. + from huggingface_hub import hf_hub_download + + base = os.path.join(os.path.dirname(os.path.abspath(__file__)), "huggingface.co") + missing = [] + for repo_id, filename, _kind in TOKENIZER_ASSETS: + target_dir = os.path.join(base, repo_id) + target_file = os.path.join(target_dir, filename) + if os.path.isfile(target_file) and os.path.getsize(target_file) > 0: + print(f" ✓ {repo_id}/{filename} already present") + continue + os.makedirs(target_dir, exist_ok=True) + print(f"Downloading tokenizer asset {repo_id}/{filename}...") + try: + hf_hub_download(repo_id=repo_id, filename=filename, local_dir=target_dir) + if not os.path.isfile(target_file) or os.path.getsize(target_file) == 0: + raise RuntimeError(f"Empty or missing tokenizer asset: {target_file}") + except Exception as e: # noqa: BLE001 - collected and surfaced below + missing.append((repo_id, filename, e)) + + if missing: + for repo_id, filename, e in missing: + print(f" ERROR: could not fetch {repo_id}/{filename}: {e}", file=sys.stderr) + print( + "\n" + "The embedding tokenizer counters in internal/tokenizer load these files from\n" + f" {base}\n" + "Without one, models that declare its tokenizer cannot be ingested.\n" + "The runtime images refuse to\n" + "build on the same condition (their copy loops exit 1), so this script fails\n" + "instead of handing a broken tree to the image build. To recover:\n" + " - re-run this script (a transient HF/network error usually clears);\n" + " - behind the GFW, re-run with --china-mirrors (routes via hf-mirror.com);\n" + " - or point MODEL_ASSETS_DIR at a directory holding the same layout.\n", + file=sys.stderr, + ) + sys.exit(1) + + print(" ✓ embedding tokenizer assets ready") if __name__ == "__main__": - # Anchor CWD to this file's directory so all relative outputs - # (huggingface.co/, nltk_data/, *.deb, *.jar, *.tar.gz, etc.) land - # at the top of ragflow_deps/ regardless of where the user invokes - # the script from. This is the build context for `ragflow_deps/Dockerfile`. + # Anchor archives, the BPE table, and tokenizer assets to ragflow_deps/. os.chdir(os.path.dirname(os.path.abspath(__file__))) parser = argparse.ArgumentParser(description="Download dependencies with optional China mirror support") @@ -216,12 +504,6 @@ if __name__ == "__main__": urls = get_urls(args.china_mirrors) - # Some mirrors (e.g. archive.ubuntu.com) reject the default urllib - # User-Agent with HTTP 403, so install an opener with a browser-like UA. - opener = urllib.request.build_opener() - opener.addheaders = [("User-Agent", "Mozilla/5.0")] - urllib.request.install_opener(opener) - for url in urls: download_url = url[0] if isinstance(url, list) else url filename = url[1] if isinstance(url, list) else url.split("/")[-1] @@ -232,20 +514,20 @@ if __name__ == "__main__": # exports SessionGetInitializer*). A pure existence check would then # keep a colleague's stale local copy and fail to link onnxruntime_go. # Verify against the published .sha256 sidecar so a re-issued archive - # is always re-downloaded and re-extracted. Every other archive keeps - # the legacy existence-based skip. + # is always re-downloaded and re-extracted. is_ort = filename == _ort_asset_name(ORT_VERSION) expected_sha = None if is_ort: sidecar_url = download_url + ".sha256" try: - with urllib.request.urlopen(sidecar_url) as resp: - expected_sha = resp.read().decode().split()[0] + resp = requests.get(sidecar_url, timeout=30) + resp.raise_for_status() + expected_sha = resp.text.split()[0] except Exception as exc: # noqa: BLE001 - best-effort; fall back to legacy print(f" WARNING: could not fetch {sidecar_url} ({exc}); skipping checksum for {filename}") needs_download = True - if os.path.exists(filename): + if _valid_download(filename): if expected_sha is not None: actual = _sha256_of(filename) if actual == expected_sha: @@ -257,13 +539,17 @@ if __name__ == "__main__": needs_download = False if needs_download: - urllib.request.urlretrieve(download_url, filename) + download_with_progress(download_url, filename) if expected_sha is not None: actual = _sha256_of(filename) if actual != expected_sha: + os.unlink(filename) print(f" ERROR: {filename} checksum mismatch after download (got {actual}, expected {expected_sha})", file=sys.stderr) sys.exit(1) print(f" ✓ {filename} checksum verified ({actual})") + if not _valid_download(filename): + os.unlink(filename) + raise RuntimeError(f"Invalid downloaded dependency: {filename}") # Force re-extract below: drop any previously extracted version dir # so the same-named re-issued archive actually refreshes the .a files. if is_ort: @@ -276,77 +562,43 @@ if __name__ == "__main__": # Extract native static libraries to ~/ragflow-native-libs for Go build. # Ensures build.sh can find them without network access. native_deps_dir = os.path.expanduser("~/ragflow-native-libs") + office_version = _go_module_version("github.com/yfedoseev/office_oxide/go") extractions = [ - ("pdfium-linux-x64-static.tgz", "pdfium-static"), - ("pdf_oxide-go-ffi-linux-amd64.tar.gz", "pdf_oxide"), - ("office_oxide-linux-x86_64.tar.gz", "office_oxide"), - (_ort_asset_name(ORT_VERSION), os.path.join("onnxruntime", "static_lib")), + ("pdfium-linux-x64-static.tgz", "pdfium-static", ["lib/libpdfium.a", "lib/libc++.a", "lib/libc++abi.a", "include/fpdfview.h"], None), + ("pdf_oxide-go-ffi-linux-amd64.tar.gz", "pdf_oxide", ["lib/linux_amd64/libpdf_oxide.a", "include/pdf_oxide.h"], None), + ("office_oxide-linux-x86_64.tar.gz", "office_oxide", ["lib/liboffice_oxide.a", "include/office_oxide_c/office_oxide.h"], office_version), ] - import tarfile - import zipfile - def _prune_stale_onnxruntime(static_lib_dir, version): - """Remove ONNX Runtime version dirs under static_lib that do NOT match - `version`. Without this, a version bump leaves the stale dir next to - the new one and build.sh's `find ... -name '*.a'` links BOTH (duplicate - symbols / wrong version, silently).""" - if not os.path.isdir(static_lib_dir): - return - expected = _ort_normalized_dir(version) - for name in os.listdir(static_lib_dir): - if not name.startswith("onnxruntime-linux-x64-static_lib-"): - continue - if name == expected: - continue - stale = os.path.join(static_lib_dir, name) - print(f" Removing stale ONNX Runtime dir: {stale}") - shutil.rmtree(stale) - - for archive, subdir in extractions: + for archive, subdir, required_files, version in extractions: archive_path = os.path.join(os.getcwd(), archive) if not os.path.isfile(archive_path): print(f" Skipping extraction: {archive} not found") continue target = os.path.join(native_deps_dir, subdir) + try: + _extract_native_archive(archive_path, target, required_files, version) + except (tarfile.TarError, EOFError, RuntimeError): + os.unlink(archive_path) + raise - # The infiniflow/ragflow-build release zip carries a top-level dir - # named onnxruntime-v{ORT_VERSION}-linux-x86_64. A plain - # "any .a present?" skip would keep a STALE version in place after a - # bump: the new zip downloads, but extraction is skipped because the - # old .a is still under static_lib, so the bump silently does nothing. - # Prune stale version dirs and only skip when the matching version is - # already extracted. - if subdir == os.path.join("onnxruntime", "static_lib"): - _prune_stale_onnxruntime(target, ORT_VERSION) - version_dir = os.path.join(target, _ort_normalized_dir(ORT_VERSION)) - if os.path.isdir(version_dir) and any(f.endswith(".a") for _, _, files in os.walk(version_dir) for f in files): - print(f" ✓ {subdir} ({ORT_VERSION}) already extracted to {version_dir}") - continue + _install_stagehand() - if os.path.isdir(target) and any(f.endswith(".a") for _, _, files in os.walk(target) for f in files): - print(f" ✓ {subdir} already extracted to {target}") - continue - os.makedirs(target, exist_ok=True) - print(f" Extracting {archive} → {target}") - if archive_path.endswith(".zip"): - with zipfile.ZipFile(archive_path) as zf: - zf.extractall(target) - # The infiniflow/ragflow-build release zip carries a top-level dir - # named onnxruntime-v{version}-linux-x86_64, but build.sh's glob and - # the stale checks above all expect - # onnxruntime-linux-x64-static_lib-{version}-glibc2_28. Rename it so - # every consumer shares one name convention (driven by ORT_VERSION). - if subdir == os.path.join("onnxruntime", "static_lib"): - extracted = os.path.join(target, _ort_extracted_dir(ORT_VERSION)) - normalized = os.path.join(target, _ort_normalized_dir(ORT_VERSION)) - if os.path.isdir(extracted) and extracted != normalized: - if os.path.exists(normalized): - shutil.rmtree(normalized) - print(f" Renaming {os.path.basename(extracted)} → {os.path.basename(normalized)}") - os.rename(extracted, normalized) - else: - with tarfile.open(archive_path) as tf: - tf.extractall(target) + if not extract_onnxruntime( + os.path.join(native_deps_dir, "onnxruntime", "static_lib"), + os.path.join(os.getcwd(), _ort_asset_name(ORT_VERSION)), + ORT_VERSION, + ): + # The archive was not downloaded or failed to extract, so no .a landed. + # Fail loud instead of exiting 0: build.sh's ORT guard would otherwise + # reject the build later with a less actionable message, and a missing + # .a left here is exactly the "silent green" this PR is meant to prevent. + print( + f" ERROR: ONNX Runtime static archives for {ORT_VERSION} were not " + f"extracted to {os.path.join(native_deps_dir, 'onnxruntime', 'static_lib')}. " + f"Check the download above; build.sh will refuse to link without them.", + file=sys.stderr, + ) + sys.exit(1) # ONNX Runtime is statically linked into the server binary, so there is no # runtime .so to surface. Log where build.sh (ONNXRUNTIME_STATIC_PREFIX) will @@ -358,38 +610,13 @@ if __name__ == "__main__": if ort_a_files: print(f" ✓ onnxruntime static archives ready: {len(ort_a_files)} .a under {ort_static_dir}") else: - print(f" Skipping onnxruntime static check: no .a found under {ort_static_dir}") - - local_dir = os.path.abspath("nltk_data") - # NLTK >=3.8.2 gates `wordnet` behind `omw-1.4`; both must be provisioned - # or tokenization-backed paths raise LookupError at runtime. - for data in ["omw-1.4", "wordnet", "punkt", "punkt_tab"]: - print(f"Downloading nltk {data}...") - nltk.download(data, download_dir=local_dir) - - for repo_id in repos: - print(f"Downloading huggingface repo {repo_id}...") - download_model(repo_id) - - # Guard: the Go in-process DeepDoc backend loads the .ort weights from the - # InfiniFlow/deepdoc snapshot pulled above. snapshot_download fetches the - # whole repo, so these must be present; fail loudly if a future repo layout - # drops them, so the Go backend can never silently ship without its models. - # (internal/common.DeepDocModelFiles is the authoritative list.) - deepdoc_local = os.path.abspath(os.path.join("huggingface.co", "InfiniFlow", "deepdoc")) - go_model_files = ["det.ort", "layout.ort", "tsr.ort", "rec.ort", "ocr.res"] - if not os.path.isdir(deepdoc_local): - print( - f" ERROR: {deepdoc_local} does not exist; the InfiniFlow/deepdoc snapshot did not materialize.", - file=sys.stderr, - ) + print(f" ERROR: ONNX Runtime .a files still missing under {ort_static_dir} after extraction; build.sh will refuse to link.", file=sys.stderr) sys.exit(1) - missing_models = [f for f in go_model_files if not os.path.isfile(os.path.join(deepdoc_local, f))] - if missing_models: - for f in missing_models: - print( - f" ERROR: expected Go model file {f} missing from {deepdoc_local}; the InfiniFlow/deepdoc snapshot no longer ships .ort weights.", - file=sys.stderr, - ) - sys.exit(1) - print(f" ✓ Go .ort model files present under {deepdoc_local}") + + # Download the Go DeepDoc `.ort` weights so this script is a one-stop for Go + # dev: native libs (above) + model files (below) from a single invocation. + download_go_models(args.china_mirrors) + + # Same one-stop idea for the token counters: their tokenizer files, in the layout + # the loaders search and ragflow_deps/Dockerfile ships. + download_tokenizer_assets(args.china_mirrors) diff --git a/ragflow_deps/download_go_deps.py b/ragflow_deps/download_go_deps.py deleted file mode 100755 index a23b27b0bf..0000000000 --- a/ragflow_deps/download_go_deps.py +++ /dev/null @@ -1,535 +0,0 @@ -#!/usr/bin/env python3 - -# PEP 723 metadata -# /// script -# requires-python = ">=3.10" -# dependencies = [ -# "huggingface-hub", -# "requests", -# ] -# /// - -# This script downloads every artifact the Go build needs: the native static -# libraries (pdfium / pdf_oxide / office_oxide / onnxruntime) for `build.sh`, -# and the Go DeepDoc `.ort` weights (det/layout/tsr/rec.ort + ocr.res) so a Go -# dev can run the in-process backend locally without separately running -# `download_deps.py`. Run it from anywhere — the `__main__` block chdir's into -# this file's own directory, so all outputs land under `ragflow_deps/` -# regardless of the caller's CWD. -# -# Build-context relationship: `ragflow_deps/Dockerfile` is built with -# `ragflow_deps/` as its build context, so the files written here MUST -# sit at the top of `ragflow_deps/`. The Dockerfile's COPY lines assume -# top-level paths (`huggingface.co`, `nltk_data`, `cl100k_base.tiktoken`, -# `*.deb`, `*.jar`, `*.tar.gz`, `stagehand-server-v3-linux-`). -# -# Typical workflow: -# -# uv run python3 ragflow_deps/download_go_deps.py # download -# cd ragflow_deps -# docker build -f Dockerfile -t infiniflow/ragflow_deps . -# -# The main `Dockerfile` (built from the project root) pulls this image -# via `--mount=type=bind,from=infiniflow/ragflow_deps:latest,...` and -# is unaffected by where these files live locally. -# -# Go DeepDoc weights: in addition to the native libs, this script downloads the -# five Go model files (internal/common.DeepDocModelFiles) from InfiniFlow/deepdoc -# straight into the repo's canonical model directory `internal/rag/res/deepdoc/` (one level -# up from this script). The Go server auto-discovers that directory via -# resolveDeepDocModelDir(), and build.sh --test-native / internal/deepdoc/native/run.sh -# default MODEL_DIR there too — so after running this script NO MODEL_DIR env needs -# to be set: -# -# bash build.sh --test-native # or: cd internal/deepdoc/native && bash run.sh -# -# Embedding tokenizer assets: the Go token counters in internal/tokenizer load a -# tokenizer file per family (XLM-R SentencePiece, BERT WordPiece, two byte-level BPE -# families) from `huggingface.co//` at the top of ragflow_deps/ - the path -# the loaders search and the one ragflow_deps/Dockerfile ships. This script fetches -# them too, so one invocation gives a Go checkout everything it counts with. Without -# them every model that declares a tokenizer silently counts with the calibrated -# cl100k estimate instead (see internal/tokenizer/embedding_token_limits.md). - -import argparse -import hashlib -import os -import shutil -import sys -import zipfile - -import requests - -# Mirrors internal/common.DeepDocORTVersion (Go in-process backend). ONE OF -# FOUR places (with that Go constant, ORT_VERSION in ragflow_deps/download_deps.py, -# and ARG ORT_VERSION in Dockerfile_go) that must carry the same ONNX Runtime -# native release for the statically-linked Go DeepDoc backend. There is no -# single source of truth — keep all four equal. build.sh --check-ort-version -# greps this file (and the other three) to fail fast on drift. (The Python pip -# onnxruntime== pin in pyproject.toml is versioned independently and is not -# part of this check.) -# -# Source of the native static archives: infiniflow/ragflow-build (our own -# ORT-only minimal build), NOT the third-party csukuangfj/onnxruntime-libs -# account. The release tag is `onnxruntime-v{ORT_VERSION}` and the asset is -# `onnxruntime-v{ORT_VERSION}-linux-x86_64.zip`. The archive is occasionally -# re-issued under this SAME tag/asset name with patched content; both download -# scripts (this one and download_deps.py) detect that via a `{asset}.sha256` -# sidecar and re-download/re-extract, so a stale local copy never silently -# lingers. -ORT_VERSION = "1.29.0" - - -def _ort_asset_name(version): - """Release asset filename under infiniflow/ragflow-build tag onnxruntime-v{version}.""" - return f"onnxruntime-v{version}-linux-x86_64.zip" - - -def _ort_extracted_dir(version): - """Top-level directory name INSIDE the release zip (what extractall creates).""" - return f"onnxruntime-v{version}-linux-x86_64" - - -def _ort_normalized_dir(version): - """Directory name build.sh's `find ... -name '*.a'` glob expects under static_lib.""" - return f"onnxruntime-linux-x64-static_lib-{version}-glibc2_28" - - -def _sha256_of(path): - """sha256 of a file, streamed in chunks so large archives don't blow memory.""" - h = hashlib.sha256() - with open(path, "rb") as f: - for chunk in iter(lambda: f.read(1 << 20), b""): - h.update(chunk) - return h.hexdigest() - - -# Mirrors internal/common.DeepDocModelFiles (Go in-process DeepDoc backend). -# These are the ONLY weights the Go backend loads; the full InfiniFlow/deepdoc -# repo also ships .onnx (Python-only), which this Go-only script deliberately -# skips to keep the download lean. -DEEPDOC_REPO = "InfiniFlow/deepdoc" -DEEPDOC_MODEL_FILES = ["det.ort", "layout.ort", "tsr.ort", "rec.ort", "ocr.res"] - -# Embedding tokenizer assets for the Go counters (internal/tokenizer). Each entry is -# (repo, file, kind): -# -# "runtime" - the counters load it in production, so the runtime images must ship it -# (the copy loop in Dockerfile / Dockerfile_base) and the loader must pin -# its SHA-1; -# "oracle" - only scripts/gen_tokenizer_oracle.py needs it, to regenerate the test -# fixtures, so it is deliberately NOT shipped to the image. -# -# ragflow_deps/test_tokenizer_assets.py holds those three places together, because a -# drift here is invisible at runtime: the counter simply becomes unavailable and ingest -# counts with the calibrated fallback. -# -# Fetched per file, not by snapshot: these repos also carry multi-GB weights we do not -# want. A missing asset is NOT fatal (unlike the DeepDoc weights above, without which -# the server cannot run at all). -TOKENIZER_ASSETS = [ - # XLM-R SentencePiece (Unigram) - the BAAI bge / multilingual-e5 / m3e / - # gte-multilingual / jina-v3 family shares this vocabulary. - ("BAAI/bge-m3", "sentencepiece.bpe.model", "runtime"), - # BERT WordPiece vocab for the bge-*-en / e5-* / gte-base family. - ("BAAI/bge-large-en-v1.5", "vocab.txt", "runtime"), - # Byte-level BPE families (Qwen3-Embedding, Mistral/Llama embeddings). - ("Qwen/Qwen3-Embedding-0.6B", "tokenizer.json", "runtime"), - ("intfloat/e5-mistral-7b-instruct", "tokenizer.json", "runtime"), - # Cross-check oracles only: the HF tokenizer.json files the same families convert to. - ("BAAI/bge-m3", "tokenizer.json", "oracle"), - ("BAAI/bge-large-en-v1.5", "tokenizer.json", "oracle"), -] - - -def prune_stale_onnxruntime(static_lib_dir, version): - """Remove ONNX Runtime version dirs under static_lib that do NOT match - `version`. Without this, a version bump leaves the stale dir next to - the new one and build.sh's `find ... -name '*.a'` links BOTH (duplicate - symbols / wrong version, silently).""" - if not os.path.isdir(static_lib_dir): - return - expected = _ort_normalized_dir(version) - for name in os.listdir(static_lib_dir): - if not name.startswith("onnxruntime-linux-x64-static_lib-"): - continue - if name == expected: - continue - stale = os.path.join(static_lib_dir, name) - print(f" Removing stale ONNX Runtime dir: {stale}") - shutil.rmtree(stale) - - -def has_static_archives(directory): - """True when `directory` holds at least one static archive (.a).""" - return any(f.endswith(".a") for _, _, files in os.walk(directory) for f in files) - - -def extract_onnxruntime(static_lib_dir, archive_path, version): - """Ensure the ONNX Runtime static archives for `version` sit under - `static_lib_dir`. Returns True when that version is available afterwards - (extracted now or already present), False when the archive is missing. - - The infiniflow/ragflow-build release zip carries a top-level dir named - onnxruntime-v{version}-linux-x86_64, so a present - `static_lib_dir` is NOT evidence that THIS version is extracted: after a - version bump the stale dir is pruned and the new one must be extracted. - """ - if not os.path.isfile(archive_path): - print(f" Skipping extraction: {os.path.basename(archive_path)} not found") - return False - prune_stale_onnxruntime(static_lib_dir, version) - version_dir = os.path.join(static_lib_dir, _ort_normalized_dir(version)) - if os.path.isdir(version_dir) and has_static_archives(version_dir): - print(f" ✓ onnxruntime/static_lib ({version}) already extracted to {version_dir}") - return True - os.makedirs(static_lib_dir, exist_ok=True) - print(f" Extracting {os.path.basename(archive_path)} → {static_lib_dir}") - with zipfile.ZipFile(archive_path) as zf: - zf.extractall(static_lib_dir) - # The infiniflow/ragflow-build release zip carries a top-level dir named - # onnxruntime-v{version}-linux-x86_64, but build.sh's glob and the stale - # checks above all expect onnxruntime-linux-x64-static_lib-{version}-glibc2_28. - # Rename it so every consumer shares one name convention (driven by - # ORT_VERSION). - extracted = os.path.join(static_lib_dir, _ort_extracted_dir(version)) - normalized = os.path.join(static_lib_dir, _ort_normalized_dir(version)) - if os.path.isdir(extracted) and extracted != normalized: - if os.path.exists(normalized): - shutil.rmtree(normalized) - print(f" Renaming {os.path.basename(extracted)} → {os.path.basename(normalized)}") - os.rename(extracted, normalized) - return True - - -def get_urls(use_china_mirrors=False) -> list[str | list[str]]: - if use_china_mirrors: - return [ - # stagehand-server-v3 Node.js SEA binaries (used by Browser - # component in local mode). - # - # The stagehand-go Go module (pinned in go.mod) and the - # stagehand-server binary (this release) are LOOSELY - # MATCHED — both stay on the v3.x line and remain - # protocol-compatible. The two version numbers do NOT - # track each other: the Go SDK is at v3.21.0 while the - # current latest server release is v3.7.2. - # - # On every go.mod bump, refresh this URL to the current - # latest server release. There is no version - # correspondence to maintain; "both on v3.x" is the - # compatibility contract. - "https://gh-proxy.com/https://github.com/browserbase/stagehand/releases/download/stagehand-server-v3/v3.7.2/stagehand-server-v3-linux-x64", - "https://gh-proxy.com/https://github.com/browserbase/stagehand/releases/download/stagehand-server-v3/v3.7.2/stagehand-server-v3-linux-arm64", - # Native static libraries for Go build (pdfium, pdf_oxide, - # office_oxide, onnxruntime). Used by build.sh's check_*_deps - # functions — pre-downloaded to avoid network access during CI. - ["https://gh-proxy.com/https://github.com/kognitos/pdfium-static/releases/download/chromium%2F7809/pdfium-linux-x64-static.tgz", "pdfium-linux-x64-static.tgz"], - ["https://gh-proxy.com/https://github.com/yfedoseev/pdf_oxide/releases/download/v0.3.73/pdf_oxide-go-ffi-linux-amd64.tar.gz", "pdf_oxide-go-ffi-linux-amd64.tar.gz"], - ["https://gh-proxy.com/https://github.com/yfedoseev/office_oxide/releases/download/v0.1.12/native-linux-x86_64.tar.gz", "office_oxide-linux-x86_64.tar.gz"], - [ - f"https://gh-proxy.com/https://github.com/infiniflow/ragflow-build/releases/download/onnxruntime-v{ORT_VERSION}/{_ort_asset_name(ORT_VERSION)}", - _ort_asset_name(ORT_VERSION), - ], - ] - else: - return [ - # stagehand-server-v3 Node.js SEA binaries (used by Browser - # component in local mode). - # - # The stagehand-go Go module (pinned in go.mod) and the - # stagehand-server binary (this release) are LOOSELY - # MATCHED — both stay on the v3.x line and remain - # protocol-compatible. The two version numbers do NOT - # track each other: the Go SDK is at v3.21.0 while the - # current latest server release is v3.7.2. - # - # On every go.mod bump, refresh this URL to the current - # latest server release. There is no version - # correspondence to maintain; "both on v3.x" is the - # compatibility contract. - "https://github.com/browserbase/stagehand/releases/download/stagehand-server-v3/v3.7.2/stagehand-server-v3-linux-x64", - "https://github.com/browserbase/stagehand/releases/download/stagehand-server-v3/v3.7.2/stagehand-server-v3-linux-arm64", - # Native static libraries for Go build (pdfium, pdf_oxide, - # office_oxide, onnxruntime). Used by build.sh's check_*_deps - # functions — pre-downloaded to avoid network access during CI. - ["https://github.com/kognitos/pdfium-static/releases/download/chromium%2F7809/pdfium-linux-x64-static.tgz", "pdfium-linux-x64-static.tgz"], - ["https://github.com/yfedoseev/pdf_oxide/releases/download/v0.3.73/pdf_oxide-go-ffi-linux-amd64.tar.gz", "pdf_oxide-go-ffi-linux-amd64.tar.gz"], - ["https://github.com/yfedoseev/office_oxide/releases/download/v0.1.12/native-linux-x86_64.tar.gz", "office_oxide-linux-x86_64.tar.gz"], - [ - f"https://github.com/infiniflow/ragflow-build/releases/download/onnxruntime-v{ORT_VERSION}/{_ort_asset_name(ORT_VERSION)}", - _ort_asset_name(ORT_VERSION), - ], - ] - - -def download_with_progress(url, filename): - response = requests.get(url, stream=True) - total_size = int(response.headers.get("content-length", 0)) - block_size = 1024 - - with open(filename, "wb") as file: - downloaded = 0 - for data in response.iter_content(block_size): - file.write(data) - downloaded += len(data) - - if total_size > 0: - progress = (downloaded / total_size) * 100 - sys.stdout.write(f"\rProgress: {progress:.1f}% ({downloaded}/{total_size} bytes)") - sys.stdout.flush() - - print() - - -def download_go_models(use_china_mirrors=False): - """Download the Go DeepDoc `.ort` weights so a Go dev can run the in-process - backend with no further setup. - - The files are written into the repo's canonical model directory - `internal/rag/res/deepdoc/` (relative to the repo root, one level up from this - script). The Go server auto-discovers that directory via - resolveDeepDocModelDir(), and build.sh --test-native / run.sh default - MODEL_DIR there too — so after this script runs, no MODEL_DIR env needs to - be set. - - Uses hf_hub_download per-file (not snapshot_download) to fetch only the - five Go model files; the `.onnx` siblings are Python-only and skipped. - On China mirrors, route through hf-mirror.com via HF_ENDPOINT. - """ - if use_china_mirrors: - os.environ["HF_ENDPOINT"] = "https://hf-mirror.com" - # Imported lazily so the module stays importable (and its unit tests stay - # huggingface-free) without the huggingface_hub dependency installed. - from huggingface_hub import hf_hub_download - - # Canonical local-dev model dir the Go backend auto-discovers. - repo_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - target_dir = os.path.join(repo_root, "internal", "rag", "res", "deepdoc") - os.makedirs(target_dir, exist_ok=True) - - missing = [] - for fname in DEEPDOC_MODEL_FILES: - dest = os.path.join(target_dir, fname) - if os.path.isfile(dest): - print(f" ✓ {fname} already present") - continue - print(f"Downloading deepdoc model {fname}...") - try: - hf_hub_download(repo_id=DEEPDOC_REPO, filename=fname, local_dir=target_dir) - except Exception as e: # noqa: BLE001 - collected and surfaced below - missing.append((fname, e)) - - if missing: - for fname, e in missing: - print(f" ERROR: failed to download {fname}: {e}", file=sys.stderr) - print( - "\n" - "The Go in-process DeepDoc backend loads ONLY the .ort weights listed in\n" - "internal/common.DeepDocModelFiles. They are fetched from " - f"{DEEPDOC_REPO} into:\n" - f" {target_dir}\n" - "Without them the backend cannot serve — the server exits with a fatal\n" - '"no in-process DeepDoc backend serving". To recover:\n' - " - re-run this script (a transient HF/network error usually clears);\n" - " - behind the GFW, re-run with --china-mirrors (routes via hf-mirror.com);\n" - " - or run `uv run python3 ragflow_deps/download_deps.py`, which snapshots\n" - f" all of {DEEPDOC_REPO} (it also provides the Python-side .onnx);\n" - " - or copy the missing files into that directory by hand.", - file=sys.stderr, - ) - sys.exit(1) - - print(f" ✓ Go DeepDoc models ready under {target_dir}") - print(" No MODEL_DIR env needed: the Go backend auto-discovers this directory.") - - -def download_tokenizer_assets(use_china_mirrors=False): - """Download the embedding tokenizer assets the Go counters load. - - Written to `ragflow_deps/huggingface.co//`: the path internal/tokenizer - searches and the one ragflow_deps/Dockerfile builds its image from. Fetched per file - (these repos also carry multi-GB weights we do not want), routed via hf-mirror.com - when --china-mirrors is set. - - A missing asset is fatal, like every other asset this script provisions. The counters - would otherwise fall back to the calibrated cl100k estimate, which under-counts some - tokenizers - and an under-count is what makes a provider answer 400. The runtime - images refuse to build on exactly the same condition (the Dockerfile copy loops exit - 1), so provisioning has to fail here rather than hand them a broken tree. - """ - if use_china_mirrors: - os.environ["HF_ENDPOINT"] = "https://hf-mirror.com" - # Imported lazily so the module stays importable (and its unit tests stay - # huggingface-free) without the huggingface_hub dependency installed. - from huggingface_hub import hf_hub_download - - base = os.path.join(os.path.dirname(os.path.abspath(__file__)), "huggingface.co") - missing = [] - for repo_id, filename, _kind in TOKENIZER_ASSETS: - target_dir = os.path.join(base, repo_id) - if os.path.isfile(os.path.join(target_dir, filename)): - print(f" ✓ {repo_id}/{filename} already present") - continue - os.makedirs(target_dir, exist_ok=True) - print(f"Downloading tokenizer asset {repo_id}/{filename}...") - try: - hf_hub_download(repo_id=repo_id, filename=filename, local_dir=target_dir) - except Exception as e: # noqa: BLE001 - collected and surfaced below - missing.append((repo_id, filename, e)) - - if missing: - for repo_id, filename, e in missing: - print(f" ERROR: could not fetch {repo_id}/{filename}: {e}", file=sys.stderr) - print( - "\n" - "The embedding tokenizer counters in internal/tokenizer load these files from\n" - f" {base}\n" - "Without one, the models that declare its counter fall back to the calibrated\n" - "cl100k estimate: safe, but less precise - it under-counts some tokenizers, and\n" - "an under-count is a request the provider rejects. The runtime images refuse to\n" - "build on the same condition (their copy loops exit 1), so this script fails\n" - "instead of handing a broken tree to the image build. To recover:\n" - " - re-run this script (a transient HF/network error usually clears);\n" - " - behind the GFW, re-run with --china-mirrors (routes via hf-mirror.com);\n" - " - or point MODEL_ASSETS_DIR at a directory holding the same layout.\n", - file=sys.stderr, - ) - sys.exit(1) - - print(" ✓ embedding tokenizer assets ready") - - -if __name__ == "__main__": - # Anchor CWD to this file's directory so all relative outputs - # (huggingface.co/, nltk_data/, *.deb, *.jar, *.tar.gz, etc.) land - # at the top of ragflow_deps/ regardless of where the user invokes - # the script from. This is the build context for `ragflow_deps/Dockerfile`. - os.chdir(os.path.dirname(os.path.abspath(__file__))) - - parser = argparse.ArgumentParser(description="Download dependencies with optional China mirror support") - parser.add_argument("--china-mirrors", action="store_true", help="Use China-accessible mirrors for downloads") - args = parser.parse_args() - - urls = get_urls(args.china_mirrors) - - # Some mirrors (e.g. archive.ubuntu.com) reject the default urllib - # User-Agent with HTTP 403, so install an opener with a browser-like UA. - # opener = urllib.request.build_opener() - # opener.addheaders = [("User-Agent", "Mozilla/5.0")] - # urllib.request.install_opener(opener) - - for url in urls: - download_url = url[0] if isinstance(url, list) else url - filename = url[1] if isinstance(url, list) else url.split("/")[-1] - print(f"Downloading {filename} from {download_url}...") - - # The ONNX Runtime archive is re-issued under the SAME release tag and - # asset name whenever its content changes (e.g. the patched build that - # exports SessionGetInitializer*). A pure existence check would then - # keep a colleague's stale local copy and fail to link onnxruntime_go. - # Verify against the published .sha256 sidecar so a re-issued archive - # is always re-downloaded and re-extracted. Every other archive keeps - # the legacy existence-based skip. - is_ort = filename == _ort_asset_name(ORT_VERSION) - expected_sha = None - if is_ort: - sidecar_url = download_url + ".sha256" - try: - resp = requests.get(sidecar_url, timeout=30) - resp.raise_for_status() - expected_sha = resp.text.split()[0] - except Exception as exc: # noqa: BLE001 - best-effort; fall back to legacy - print(f" WARNING: could not fetch {sidecar_url} ({exc}); skipping checksum for {filename}") - - needs_download = True - if os.path.exists(filename): - if expected_sha is not None: - actual = _sha256_of(filename) - if actual == expected_sha: - print(f" ✓ {filename} checksum matches released {expected_sha}; skipping download") - needs_download = False - else: - print(f" {filename} checksum mismatch (local {actual} != released {expected_sha}); re-downloading") - else: - needs_download = False - - if needs_download: - download_with_progress(download_url, filename) - if expected_sha is not None: - actual = _sha256_of(filename) - if actual != expected_sha: - print(f" ERROR: {filename} checksum mismatch after download (got {actual}, expected {expected_sha})", file=sys.stderr) - sys.exit(1) - print(f" ✓ {filename} checksum verified ({actual})") - # Force re-extract below: drop any previously extracted version dir - # so the same-named re-issued archive actually refreshes the .a files. - if is_ort: - native_libs = os.path.expanduser("~/ragflow-native-libs") - version_dir = os.path.join(native_libs, "onnxruntime", "static_lib", _ort_normalized_dir(ORT_VERSION)) - if os.path.isdir(version_dir): - print(f" Removing stale extracted ONNX Runtime dir: {version_dir}") - shutil.rmtree(version_dir) - - # Extract native static libraries to ~/ragflow-native-libs for Go build. - # Ensures build.sh can find them without network access. - native_deps_dir = os.path.expanduser("~/ragflow-native-libs") - import tarfile - - extractions = [ - ("pdfium-linux-x64-static.tgz", "pdfium-static"), - ("pdf_oxide-go-ffi-linux-amd64.tar.gz", "pdf_oxide"), - ("office_oxide-linux-x86_64.tar.gz", "office_oxide"), - ] - - for archive, subdir in extractions: - archive_path = os.path.join(os.getcwd(), archive) - if not os.path.isfile(archive_path): - print(f" Skipping extraction: {archive} not found") - continue - target = os.path.join(native_deps_dir, subdir) - if os.path.isdir(target): - print(f" ✓ {subdir} already extracted to {target}") - continue - os.makedirs(target, exist_ok=True) - print(f" Extracting {archive} → {target}") - with tarfile.open(archive_path) as tf: - tf.extractall(target) - - if not extract_onnxruntime( - os.path.join(native_deps_dir, "onnxruntime", "static_lib"), - os.path.join(os.getcwd(), _ort_asset_name(ORT_VERSION)), - ORT_VERSION, - ): - # The archive was not downloaded or failed to extract, so no .a landed. - # Fail loud instead of exiting 0: build.sh's ORT guard would otherwise - # reject the build later with a less actionable message, and a missing - # .a left here is exactly the "silent green" this PR is meant to prevent. - print( - f" ERROR: ONNX Runtime static archives for {ORT_VERSION} were not " - f"extracted to {os.path.join(native_deps_dir, 'onnxruntime', 'static_lib')}. " - f"Check the download above; build.sh will refuse to link without them.", - file=sys.stderr, - ) - sys.exit(1) - - # ONNX Runtime is statically linked into the server binary, so there is no - # runtime .so to surface. Log where build.sh (ONNXRUNTIME_STATIC_PREFIX) will - # find the archives — the .a files live under - # ~/ragflow-native-libs/onnxruntime/static_lib. The in-process backend - # resolves OrtGetApiBase via dlopen(NULL); there is no dynamic .so fallback. - ort_static_dir = os.path.join(native_deps_dir, "onnxruntime", "static_lib") - ort_a_files = [os.path.join(root, f) for root, _, files in os.walk(ort_static_dir) for f in files if f.endswith(".a")] - if ort_a_files: - print(f" ✓ onnxruntime static archives ready: {len(ort_a_files)} .a under {ort_static_dir}") - else: - print(f" ERROR: ONNX Runtime .a files still missing under {ort_static_dir} after extraction; build.sh will refuse to link.", file=sys.stderr) - sys.exit(1) - - # Download the Go DeepDoc `.ort` weights so this script is a one-stop for Go - # dev: native libs (above) + model files (below) from a single invocation. - download_go_models(args.china_mirrors) - - # Same one-stop idea for the token counters: their tokenizer files, in the layout - # the loaders search and ragflow_deps/Dockerfile ships. - download_tokenizer_assets(args.china_mirrors) diff --git a/ragflow_deps/test_download_deps.py b/ragflow_deps/test_download_deps.py new file mode 100644 index 0000000000..6eb44c0e23 --- /dev/null +++ b/ragflow_deps/test_download_deps.py @@ -0,0 +1,253 @@ +# Tests for ragflow_deps/download_deps.py ONNX Runtime extraction. +# +# build.sh's build_go() fails fast when libonnxruntime.a is not linked, so these +# tests must guarantee the archive is really landed on disk — above all after an +# ORT version bump, where the zip's version-stamped top-level dir changes but +# static_lib/ still exists from the previous run. +# +# The release zip from infiniflow/ragflow-build carries a top-level dir named +# onnxruntime-v{version}-linux-x86_64; extract_onnxruntime() must rename it to +# the build.sh-expected onnxruntime-linux-x64-static_lib-{version}-glibc2_28 so +# every consumer shares one name convention. These tests pin that rename. + +import http.server +import io +import os +import runpy +import shutil +import sys +import tarfile +import threading +import types +import zipfile +from pathlib import Path + +import download_deps as deps +import pytest +import requests +from download_deps import ( + _ort_asset_name, + _ort_extracted_dir, + _ort_normalized_dir, + extract_onnxruntime, + has_static_archives, +) + + +def make_ort_zip(path, version): + """Build a zip shaped like the infiniflow/ragflow-build ORT release: a + top-level dir named onnxruntime-v{version}-linux-x86_64 holding + lib/libonnxruntime.a. extract_onnxruntime() must rename it to the + build.sh-expected onnxruntime-linux-x64-static_lib-{version}-glibc2_28.""" + member = f"{_ort_extracted_dir(version)}/lib/libonnxruntime.a" + with zipfile.ZipFile(path, "w") as zf: + zf.writestr(member, b"!\nort-payload") + return path + + +def test_extracts_on_first_run(tmp_path): + # static_lib/ does not exist yet: the plain first-run path. + static_lib = tmp_path / "onnxruntime" / "static_lib" + version = "1.29.0" + archive = make_ort_zip(tmp_path / _ort_asset_name(version), version) + + assert extract_onnxruntime(str(static_lib), str(archive), version) is True + + extracted = static_lib / _ort_extracted_dir(version) + assert not extracted.exists(), "release zip top-level dir must be renamed away" + version_dir = static_lib / _ort_normalized_dir(version) + assert version_dir.is_dir() + assert has_static_archives(str(version_dir)) + + +def test_extracts_after_version_bump_when_static_lib_exists(tmp_path): + """Regression: a static_lib/ left over from a previous version must not + suppress extraction of the bumped version. Pruning the stale dir leaves + static_lib/ in place; if extraction is then skipped no .a is ever landed and + build.sh's ORT guard rejects the build.""" + static_lib = tmp_path / "onnxruntime" / "static_lib" + static_lib.mkdir(parents=True) + stale = static_lib / _ort_normalized_dir("1.28.0") + (stale / "lib").mkdir(parents=True) + (stale / "lib" / "libonnxruntime.a").write_bytes(b"old-ort") + + version = "1.29.0" + archive = make_ort_zip(tmp_path / _ort_asset_name(version), version) + + assert extract_onnxruntime(str(static_lib), str(archive), version) is True + + assert not stale.exists(), "stale ORT version should be pruned" + extracted = static_lib / _ort_extracted_dir(version) + assert not extracted.exists(), "release zip top-level dir must be renamed away" + version_dir = static_lib / _ort_normalized_dir(version) + assert version_dir.is_dir(), "bumped ORT version was silently not extracted" + assert has_static_archives(str(version_dir)), "bumped ORT version landed no .a" + + +def test_skips_when_archive_missing(tmp_path): + static_lib = tmp_path / "onnxruntime" / "static_lib" + assert extract_onnxruntime(str(static_lib), str(tmp_path / "missing.zip"), "1.29.0") is False + + +def test_idempotent_when_matching_version_already_present(tmp_path): + static_lib = tmp_path / "onnxruntime" / "static_lib" + version = "1.29.0" + archive = make_ort_zip(tmp_path / _ort_asset_name(version), version) + + assert extract_onnxruntime(str(static_lib), str(archive), version) is True + before = sorted(os.listdir(static_lib)) + assert extract_onnxruntime(str(static_lib), str(archive), version) is True + assert sorted(os.listdir(static_lib)) == before + + +@pytest.fixture +def download_server(): + class Handler(http.server.BaseHTTPRequestHandler): + def do_GET(self): + if self.path == "/missing": + self.send_response(404) + self.end_headers() + self.wfile.write(b"not an artifact") + else: + self.send_response(200) + self.send_header("Content-Length", "100" if self.path == "/interrupted" else "7") + self.end_headers() + self.wfile.write(b"payload") + + def log_message(self, *args): + pass + + server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=server.serve_forever) + thread.start() + try: + yield f"http://127.0.0.1:{server.server_port}" + finally: + server.shutdown() + thread.join() + server.server_close() + + +@pytest.mark.parametrize("endpoint", ["missing", "interrupted"]) +def test_failed_download_preserves_previous_file(tmp_path, download_server, endpoint): + dest = tmp_path / "artifact" + dest.write_bytes(b"previous valid artifact") + with pytest.raises(requests.RequestException): + deps.download_with_progress(f"{download_server}/{endpoint}", dest) + assert dest.read_bytes() == b"previous valid artifact" + assert list(tmp_path.iterdir()) == [dest] + + +def test_successful_download_replaces_previous_file(tmp_path, download_server): + dest = tmp_path / "artifact" + dest.write_bytes(b"old") + deps.download_with_progress(f"{download_server}/complete", dest) + assert dest.read_bytes() == b"payload" + assert list(tmp_path.iterdir()) == [dest] + + +def test_dependency_preparation_recovers_empty_native_directory(tmp_path, monkeypatch): + root = Path(__file__).resolve().parents[1] + context = tmp_path / "ragflow_deps" + context.mkdir() + script = context / "download_deps.py" + shutil.copy(root / "ragflow_deps/download_deps.py", script) + (tmp_path / "go.mod").write_text("""module fixture + +require ( + github.com/yfedoseev/office_oxide/go v0.1.12 + github.com/browserbase/stagehand-go/v3 v3.22.0 +) +""") + native = tmp_path / "native" + required = { + "pdfium-linux-x64-static.tgz": ("pdfium-static", ["lib/libpdfium.a", "lib/libc++.a", "lib/libc++abi.a", "include/fpdfview.h"]), + "pdf_oxide-go-ffi-linux-amd64.tar.gz": ("pdf_oxide", ["lib/linux_amd64/libpdf_oxide.a", "include/pdf_oxide.h"]), + "office_oxide-linux-x86_64.tar.gz": ("office_oxide", ["lib/liboffice_oxide.a", "include/office_oxide_c/office_oxide.h"]), + } + for archive_name, (directory, files) in required.items(): + (native / directory).mkdir(parents=True) + with tarfile.open(context / archive_name, "w:gz") as archive: + for name in files: + item = tarfile.TarInfo("./" + name) + payload = b"payload\x000.1.12\x00" if name.endswith("liboffice_oxide.a") else b"payload" + item.size = len(payload) + archive.addfile(item, io.BytesIO(payload)) + (native / "office_oxide/lib").mkdir() + (native / "office_oxide/include/office_oxide_c").mkdir(parents=True) + (native / "office_oxide/lib/liboffice_oxide.a").write_bytes(b"old\x000.1.9\x00") + (native / "office_oxide/include/office_oxide_c/office_oxide.h").write_bytes(b"old") + make_ort_zip(context / _ort_asset_name("1.29.0"), "1.29.0") + for arch in ["x64", "arm64"]: + (context / f"stagehand-server-v3-linux-{arch}").write_bytes(b"\x7fELFpayload") + (context / "cl100k_base.tiktoken").write_bytes(b"configured table") + + def offline(*args, **kwargs): + raise requests.ConnectionError("offline sidecar") + + def hf_download(*, repo_id, filename, local_dir): + Path(local_dir).mkdir(parents=True, exist_ok=True) + (Path(local_dir) / filename).write_bytes(b"model") + + monkeypatch.setitem(sys.modules, "huggingface_hub", types.SimpleNamespace(hf_hub_download=hf_download)) + monkeypatch.setattr(requests, "get", offline) + original_expanduser = os.path.expanduser + monkeypatch.setattr(os.path, "expanduser", lambda path: str(native) if path == "~/ragflow-native-libs" else original_expanduser(path)) + monkeypatch.setenv("XDG_CACHE_HOME", str(tmp_path / "cache")) + monkeypatch.setattr(sys, "argv", [str(script)]) + monkeypatch.chdir(tmp_path) + runpy.run_path(str(script), run_name="__main__") + for directory, files in required.values(): + for name in files: + expected = b"payload\x000.1.12\x00" if name.endswith("liboffice_oxide.a") else b"payload" + assert (native / directory / name).read_bytes() == expected + stagehand = tmp_path / "cache/stagehand/lib/go_3.22.0/stagehand-server-v3-linux-x64" + assert stagehand.read_bytes() == b"\x7fELFpayload" + assert os.access(stagehand, os.X_OK) + for filename in ["det.ort", "layout.ort", "tsr.ort", "rec.ort", "ocr.res"]: + assert (context / "huggingface.co/InfiniFlow/deepdoc" / filename).read_bytes() == b"model" + + +def test_incomplete_native_archive_preserves_existing_directory(tmp_path): + target = tmp_path / "native" + target.mkdir() + (target / "existing").write_bytes(b"keep") + archive = tmp_path / "incomplete.tar.gz" + with tarfile.open(archive, "w:gz"): + pass + with pytest.raises(RuntimeError, match="missing required native file"): + deps._extract_native_archive(archive, target, ["lib/library.a"]) + assert (target / "existing").read_bytes() == b"keep" + + +@pytest.mark.parametrize("kind", ["traversal", "absolute", "symlink", "hardlink", "fifo", "device"]) +def test_native_archive_rejects_unsafe_members(tmp_path, kind): + target = tmp_path / "native" + target.mkdir() + (target / "existing").write_bytes(b"keep") + outside = tmp_path / "outside" + outside.write_bytes(b"untouched") + archive_path = tmp_path / "unsafe.tar.gz" + with tarfile.open(archive_path, "w:gz") as archive: + library = tarfile.TarInfo("./lib/library.a") + library.size = len(b"payload") + archive.addfile(library, io.BytesIO(b"payload")) + member = tarfile.TarInfo("unexpected") + if kind == "traversal": + member.name = "../outside" + elif kind == "absolute": + member.name = str(outside) + else: + member.type = {"symlink": tarfile.SYMTYPE, "hardlink": tarfile.LNKTYPE, "fifo": tarfile.FIFOTYPE, "device": tarfile.CHRTYPE}[kind] + member.linkname = str(outside) + payload = b"unsafe" if member.isfile() else b"" + member.size = len(payload) + archive.addfile(member, io.BytesIO(payload)) + + with pytest.raises(tarfile.TarError, match="Unsafe native archive member"): + deps._extract_native_archive(archive_path, str(target), ["lib/library.a"]) + + assert outside.read_bytes() == b"untouched" + assert (target / "existing").read_bytes() == b"keep" + assert not (target / "lib/library.a").exists() + assert sorted(path.name for path in tmp_path.iterdir()) == ["native", "outside", "unsafe.tar.gz"] diff --git a/ragflow_deps/test_download_go_deps.py b/ragflow_deps/test_download_go_deps.py deleted file mode 100644 index 78c834ad1b..0000000000 --- a/ragflow_deps/test_download_go_deps.py +++ /dev/null @@ -1,88 +0,0 @@ -# Tests for ragflow_deps/download_go_deps.py ONNX Runtime extraction. -# -# build.sh's build_go() fails fast when libonnxruntime.a is not linked, so these -# tests must guarantee the archive is really landed on disk — above all after an -# ORT version bump, where the zip's version-stamped top-level dir changes but -# static_lib/ still exists from the previous run. -# -# The release zip from infiniflow/ragflow-build carries a top-level dir named -# onnxruntime-v{version}-linux-x86_64; extract_onnxruntime() must rename it to -# the build.sh-expected onnxruntime-linux-x64-static_lib-{version}-glibc2_28 so -# every consumer shares one name convention. These tests pin that rename. - -import os -import zipfile - -from download_go_deps import ( - _ort_asset_name, - _ort_extracted_dir, - _ort_normalized_dir, - extract_onnxruntime, - has_static_archives, -) - - -def make_ort_zip(path, version): - """Build a zip shaped like the infiniflow/ragflow-build ORT release: a - top-level dir named onnxruntime-v{version}-linux-x86_64 holding - lib/libonnxruntime.a. extract_onnxruntime() must rename it to the - build.sh-expected onnxruntime-linux-x64-static_lib-{version}-glibc2_28.""" - member = f"{_ort_extracted_dir(version)}/lib/libonnxruntime.a" - with zipfile.ZipFile(path, "w") as zf: - zf.writestr(member, b"!\nort-payload") - return path - - -def test_extracts_on_first_run(tmp_path): - # static_lib/ does not exist yet: the plain first-run path. - static_lib = tmp_path / "onnxruntime" / "static_lib" - version = "1.29.0" - archive = make_ort_zip(tmp_path / _ort_asset_name(version), version) - - assert extract_onnxruntime(str(static_lib), str(archive), version) is True - - extracted = static_lib / _ort_extracted_dir(version) - assert not extracted.exists(), "release zip top-level dir must be renamed away" - version_dir = static_lib / _ort_normalized_dir(version) - assert version_dir.is_dir() - assert has_static_archives(str(version_dir)) - - -def test_extracts_after_version_bump_when_static_lib_exists(tmp_path): - """Regression: a static_lib/ left over from a previous version must not - suppress extraction of the bumped version. Pruning the stale dir leaves - static_lib/ in place; if extraction is then skipped no .a is ever landed and - build.sh's ORT guard rejects the build.""" - static_lib = tmp_path / "onnxruntime" / "static_lib" - static_lib.mkdir(parents=True) - stale = static_lib / _ort_normalized_dir("1.28.0") - (stale / "lib").mkdir(parents=True) - (stale / "lib" / "libonnxruntime.a").write_bytes(b"old-ort") - - version = "1.29.0" - archive = make_ort_zip(tmp_path / _ort_asset_name(version), version) - - assert extract_onnxruntime(str(static_lib), str(archive), version) is True - - assert not stale.exists(), "stale ORT version should be pruned" - extracted = static_lib / _ort_extracted_dir(version) - assert not extracted.exists(), "release zip top-level dir must be renamed away" - version_dir = static_lib / _ort_normalized_dir(version) - assert version_dir.is_dir(), "bumped ORT version was silently not extracted" - assert has_static_archives(str(version_dir)), "bumped ORT version landed no .a" - - -def test_skips_when_archive_missing(tmp_path): - static_lib = tmp_path / "onnxruntime" / "static_lib" - assert extract_onnxruntime(str(static_lib), str(tmp_path / "missing.zip"), "1.29.0") is False - - -def test_idempotent_when_matching_version_already_present(tmp_path): - static_lib = tmp_path / "onnxruntime" / "static_lib" - version = "1.29.0" - archive = make_ort_zip(tmp_path / _ort_asset_name(version), version) - - assert extract_onnxruntime(str(static_lib), str(archive), version) is True - before = sorted(os.listdir(static_lib)) - assert extract_onnxruntime(str(static_lib), str(archive), version) is True - assert sorted(os.listdir(static_lib)) == before diff --git a/ragflow_deps/test_tokenizer_assets.py b/ragflow_deps/test_tokenizer_assets.py index efdf75018d..cc3ed9c214 100644 --- a/ragflow_deps/test_tokenizer_assets.py +++ b/ragflow_deps/test_tokenizer_assets.py @@ -3,14 +3,10 @@ # The Go counters in internal/tokenizer read their vocabulary from # ragflow_deps/huggingface.co//. Three places have to agree: # -# 1. ragflow_deps/download_go_deps.py - fetches the file, and says whether it is -# "runtime" (the counters load it) or "oracle" -# (only scripts/gen_tokenizer_oracle.py does). -# It is the Go-side downloader: everything the -# Go side reads comes from here, and -# ragflow_deps/download_deps.py (upstream) is -# deliberately left alone; -# 2. Dockerfile / Dockerfile_base / Dockerfile_go +# 1. ragflow_deps/download_deps.py - fetches the file, and says whether it is +# "runtime" (the counters load it) or "oracle" +# (only scripts/gen_tokenizer_oracle.py does); +# 2. Dockerfile / Dockerfile_base # - copy the runtime ones into the image at the # path the counters search; # 3. internal/tokenizer/*.go - pin the SHA-1 of the runtime ones. @@ -32,8 +28,7 @@ ROOT = HERE.parent RUNTIME_IMAGES = ("Dockerfile_base", "Dockerfile") GO_SOURCES = ("spm.go", "wordpiece.go", "bpe.go") KINDS = ("runtime", "oracle") -DOWNLOADER = "download_go_deps.py" -UPSTREAM_DOWNLOADER = "download_deps.py" +DOWNLOADER = "download_deps.py" def load_tokenizer_assets(): @@ -88,16 +83,3 @@ def test_oracle_only_assets_are_not_shipped(): text = image_text(image) for repo, filename, _kind in assets: assert f"{repo}/{filename}" not in text, f"{image} ships {repo}/{filename}, which only the fixture generator needs" - - -def test_assets_are_fetched_by_the_go_downloader_only(): - """The Go-side assets belong to the Go-side downloader. - - ragflow_deps/download_deps.py is an upstream file (it snapshots the Python side's - dependencies); adding the Go counters' tokenizer files there split the list across - two scripts and made the upstream file carry Go concerns. Keeping the fetch in - download_go_deps.py is what makes a Go checkout self-sufficient. - """ - upstream = (HERE / UPSTREAM_DOWNLOADER).read_text(encoding="utf-8") - for repo, filename, _kind in load_tokenizer_assets(): - assert filename not in upstream, f"{UPSTREAM_DOWNLOADER} mentions {repo}/{filename}; the embedding tokenizer assets are fetched by {DOWNLOADER} only" diff --git a/scripts/gen_tokenizer_oracle.py b/scripts/gen_tokenizer_oracle.py index 564551a0c9..9dd793cf66 100755 --- a/scripts/gen_tokenizer_oracle.py +++ b/scripts/gen_tokenizer_oracle.py @@ -180,7 +180,7 @@ ORACLES = { # the served behaviour comes from tokenizer.json above. SPM_MODEL = "huggingface.co/BAAI/bge-m3/sentencepiece.bpe.model" -# The cl100k table as shipped in ragflow_deps (the OpenAI blob download_deps.py fetches). +# The cl100k table as provisioned by download_deps.py or build.sh in ragflow_deps. # The Go counter loads this file; tiktoken reads its own cached copy of the same blob, so # the check in main() is what keeps the check-out honest about what it ships. CL100K_TABLE = "cl100k_base.tiktoken" @@ -337,7 +337,7 @@ def main() -> int: else: tokenizer_path = os.path.join(args.deps, rel) if not os.path.isfile(tokenizer_path): - print(f"missing oracle tokenizer: {tokenizer_path}\nrun `uv run ragflow_deps/download_go_deps.py`", file=sys.stderr) + print(f"missing oracle tokenizer: {tokenizer_path}\nrun `uv run ragflow_deps/download_deps.py`", file=sys.stderr) return 1 tokenizer = Tokenizer.from_file(tokenizer_path) vocab = tokenizer.get_vocab() @@ -379,7 +379,7 @@ def main() -> int: # --spm-crosscheck was asked for, so the asset it reads has to be there: # skipping quietly would turn "the second opinion agrees" into a claim # nobody made. Assets are not optional in this script - they are checked. - print(f"missing sentencepiece model: {model_file}\nrun `uv run ragflow_deps/download_go_deps.py`", file=sys.stderr) + print(f"missing sentencepiece model: {model_file}\nrun `uv run ragflow_deps/download_deps.py`", file=sys.stderr) return 1 processor = spm.SentencePieceProcessor(model_file=model_file) divergent = 0