diff --git a/.dockerignore b/.dockerignore index 1ed7cfa457..1b4707c99e 100644 --- a/.dockerignore +++ b/.dockerignore @@ -34,11 +34,16 @@ target/ # ── ragflow_deps build context (built as a separate image, mounted ── # ── from infiniflow/ragflow_deps:latest by the main Dockerfile) ── -# Excluding the entire directory keeps the main build context small +# Excluding the directory contents keeps the main build context small # regardless of which deps files download_deps.py currently fetches. # The deps image is built from this directory with: # cd ragflow_deps && docker build -f Dockerfile -t infiniflow/ragflow_deps . -ragflow_deps/ +# The two ORT version-pin files are re-included: build.sh's +# check_ort_version_consistency (run by Dockerfile_go's `--go` build) greps +# them. Re-including requires NOT excluding the directory itself. +ragflow_deps/* +!ragflow_deps/download_go_deps.py +!ragflow_deps/download_deps.py # ── IDE and editor config ────────────────────────────────────────────────── .idea/ diff --git a/Dockerfile_go b/Dockerfile_go index 785b050492..6632a1907e 100644 --- a/Dockerfile_go +++ b/Dockerfile_go @@ -159,6 +159,11 @@ RUN --mount=type=cache,id=ragflow_gomod,target=/root/.cache/gomod \ COPY internal internal COPY cmd cmd COPY build.sh ./ +# build.sh's check_ort_version_consistency (run via `./build.sh --go`) greps the +# ORT version pins from these files; without them the --go build fails with +# "could not parse the ONNX Runtime version from one of the pinned locations". +COPY ragflow_deps/download_go_deps.py ragflow_deps/download_deps.py ./ragflow_deps/ +COPY Dockerfile_go ./ # ONNX Runtime static archives: build.sh's _seed_from_system looks for the ORT # static libs under ONNXRUNTIME_STATIC_PREFIX (default ~/ragflow-native-libs/onnxruntime). @@ -169,10 +174,19 @@ COPY build.sh ./ # silently skipped and the binary fails at startup with # "no in-process DeepDoc backend serving" (dlopen(NULL)/dlsym can't find OrtGetApiBase). # ${HOME} (not hard-coded /root) keeps the path consistent with build.sh regardless of HOME. -ARG ORT_VERSION=1.23.2 +ARG ORT_VERSION=1.29.0 RUN set -eux; \ - mkdir -p "${HOME}/ragflow-native-libs/onnxruntime"; \ - cp -rn /opt/ragflow-native-libs/onnxruntime/* "${HOME}/ragflow-native-libs/onnxruntime/"; \ + mkdir -p "${HOME}/ragflow-native-libs/onnxruntime/static_lib"; \ + _src="/opt/ragflow-native-libs/onnxruntime/static_lib"; \ + _v="$(ls -d "${_src}"/onnxruntime-linux-x64-static_lib-"${ORT_VERSION}"-* 2>/dev/null | head -1)"; \ + if [ -z "${_v}" ]; then \ + _v="$(ls -d "${_src}"/onnxruntime-linux-x64-static_lib-* 2>/dev/null | head -1)"; \ + fi; \ + if [ -n "${_v}" ]; then \ + cp -r "${_v}" "${HOME}/ragflow-native-libs/onnxruntime/static_lib/"; \ + else \ + echo "Warning: no pre-baked onnxruntime static_lib in runner image; build.sh will attempt download" >&2; \ + fi; \ find "${HOME}/ragflow-native-libs/onnxruntime/static_lib" -name '*.a' | head -5 RUN git config --global safe.directory "*" && \ diff --git a/build.sh b/build.sh index ed91bcf540..45da44de77 100755 --- a/build.sh +++ b/build.sh @@ -150,6 +150,38 @@ check_go_deps() { command -v go >/dev/null 2>&1 || { echo -e "${RED}Error: go is required but not installed.${NC}"; exit 1; } echo "✓ Required tools are available" + check_ort_version_consistency +} + +# Fail fast when the ONNX Runtime native version is declared inconsistently. +# The in-process (Go) DeepDoc backend statically links libonnxruntime.a built +# from ONE exact ORT release; a mismatch links a wrong/missing .a and only fails +# at runtime (dlopen(NULL) can't find OrtGetApiBase). The version is pinned in +# four Go-side locations that must all agree. +check_ort_version_consistency() { + print_section "Checking ONNX Runtime version consistency" + + local env_go d1 d2 dockerfile + env_go="$(grep -m1 -E 'DeepDocORTVersion[[:space:]]*=[[:space:]]*"' "${PROJECT_ROOT}/internal/common/environments.go" | sed -E 's/.*"([^"]+)".*/\1/')" + d1="$(grep -m1 -E '^ORT_VERSION[[:space:]]*=[[:space:]]*"' "${PROJECT_ROOT}/ragflow_deps/download_go_deps.py" | sed -E 's/.*"([^"]+)".*/\1/')" + d2="$(grep -m1 -E '^ORT_VERSION[[:space:]]*=[[:space:]]*"' "${PROJECT_ROOT}/ragflow_deps/download_deps.py" | sed -E 's/.*"([^"]+)".*/\1/')" + dockerfile="$(grep -m1 -E 'ARG[[:space:]]+ORT_VERSION=' "${PROJECT_ROOT}/Dockerfile_go" | sed -E 's/.*ORT_VERSION=([0-9][^"[:space:]]*).*/\1/')" + + if [ -z "$env_go" ] || [ -z "$d1" ] || [ -z "$d2" ] || [ -z "$dockerfile" ]; then + echo -e "${RED}Error: could not parse the ONNX Runtime version from one of the pinned locations${NC}" >&2 + exit 1 + fi + + if [ "$env_go" != "$d1" ] || [ "$env_go" != "$d2" ] || [ "$env_go" != "$dockerfile" ]; then + echo -e "${RED}Error: ONNX Runtime version is inconsistent — fix before building:${NC}" >&2 + printf ' %-10s %s\n' "$env_go" "internal/common/environments.go:DeepDocORTVersion" + printf ' %-10s %s\n' "$d1" "ragflow_deps/download_go_deps.py:ORT_VERSION" + printf ' %-10s %s\n' "$d2" "ragflow_deps/download_deps.py:ORT_VERSION" + printf ' %-10s %s\n' "$dockerfile" "Dockerfile_go:ARG ORT_VERSION" + exit 1 + fi + + echo -e "${GREEN}✓ ONNX Runtime native version consistent: ${env_go}${NC}" } # Check office_oxide native library @@ -544,32 +576,34 @@ setup_cgo_env() { if [ -d "$ONNXRUNTIME_STATIC_PREFIX" ]; then # Collect every .a, but skip GPU-only providers we never build # against (would pull in CUDA/cuDNN/TensorRT which we don't ship). + # + # Select the ORT static lib that matches the Go deepdoc backend's + # required version (DeepDocORTVersion in internal/common/environments.go). + # Go and Python build/link against independent ORT versions, so the + # static_lib prefix legitimately holds more than one + # onnxruntime-linux-x64-static_lib-* dir at once (e.g. the Python-side + # 1.23.x next to the Go-side 1.29.0). We pick the dir that matches + # DeepDocORTVersion rather than failing when a second version dir is + # present. This avoids silently linking the wrong version while still + # keeping the bake self-documenting. + local ort_version + ort_version="$(grep -m1 -E 'DeepDocORTVersion[[:space:]]*=[[:space:]]*"' \ + "${PROJECT_ROOT}/internal/common/environments.go" \ + | sed -E 's/.*"([^"]+)".*/\1/')" + if [ -z "$ort_version" ]; then + echo " Error: cannot parse DeepDocORTVersion from internal/common/environments.go" >&2 + return 1 + fi local ort_a="" - local seen_version_dir="" while IFS= read -r f; do case "$(basename "$f")" in *cuda*|*tensorrt*|*coreml*|*dml*|*migraphx*) continue ;; esac - # Guard against coexisting stale version dirs: if .a files span - # more than one onnxruntime-linux-x64-static_lib-* dir, fail fast - # instead of silently linking two ORT versions (duplicate symbols - # / wrong version). Re-run `download_deps.py` to prune stale dirs - # after a version bump, or remove the old dir by hand. case "$f" in - */onnxruntime-linux-x64-static_lib-*/lib/*.a) - local vdir="${f#*/onnxruntime-linux-x64-static_lib-}" - vdir="${vdir%%/*}" - if [ -z "$seen_version_dir" ]; then - seen_version_dir="$vdir" - elif [ "$seen_version_dir" != "$vdir" ]; then - echo " Error: multiple ONNX Runtime versions found under $ONNXRUNTIME_STATIC_PREFIX" >&2 - echo " $seen_version_dir AND $vdir" >&2 - echo " Remove the stale version dir (or re-run download_deps.py to prune it)." >&2 - return 1 - fi - ;; + # Only collect .a from the dir matching the required version. + */onnxruntime-linux-x64-static_lib-"${ort_version}"*/lib/*.a) + ort_a="$ort_a $f" ;; esac - ort_a="$ort_a $f" done < <(find "$ONNXRUNTIME_STATIC_PREFIX" -type f -name '*.a' 2>/dev/null) if [ -n "$ort_a" ]; then @@ -604,7 +638,13 @@ setup_cgo_env() { # dynamic symbol table, which is what the binding's dlopen(NULL)+dlsym # lookup needs at runtime (no --export-dynamic required). else - echo " onnxruntime static_lib dir has no .a files; the in-process DeepDoc backend cannot link ORT" >&2 + local avail + avail="$(find "$ONNXRUNTIME_STATIC_PREFIX" -maxdepth 1 -type d \ + -name 'onnxruntime-linux-x64-static_lib-*' -exec basename {} \; 2>/dev/null | tr '\n' ' ')" + echo " Error: no ONNX Runtime ${ort_version} static lib under $ONNXRUNTIME_STATIC_PREFIX" >&2 + echo " available: ${avail:-}" >&2 + echo " DeepDocORTVersion=${ort_version}; bake/download the matching ORT (or update DeepDocORTVersion)." >&2 + return 1 fi else echo " onnxruntime static_lib not found ($ONNXRUNTIME_STATIC_PREFIX); the in-process DeepDoc backend cannot link ORT" >&2 @@ -809,6 +849,8 @@ OPTIONS: InfiniFlow/deepdoc model snapshot; self-skip otherwise). e.g. `$0 --test-native` --clean, -C Clean all build artifacts + --check-ort-version Verify the ONNX Runtime native version is declared + consistently across all sources (exit 1 on mismatch). --run, -r Build and run the server --strip, -s Strip debug symbols from Go binaries (-ldflags="-s -w") (disabled by default, useful for smaller production binaries) @@ -927,6 +969,9 @@ main() { --clean|-C) clean ;; + --check-ort-version) + check_ort_version_consistency + ;; --run|-r) check_cpp_deps check_go_deps diff --git a/internal/common/environments.go b/internal/common/environments.go index 3b4ed59c97..9740817bbb 100644 --- a/internal/common/environments.go +++ b/internal/common/environments.go @@ -267,11 +267,12 @@ func HasModelFiles(dir string) bool { } // DeepDocORTVersion is the onnxruntime native release the in-process (Go) -// DeepDoc backend is built and tested against (e.g. "1.23.2"). It is ONE OF -// THREE raw version declarations that must stay equal (the other two are -// ORT_VERSION in ragflow_deps/download_go_deps.py and ragflow_deps/download_deps.py) -// — NOT a single source of truth. The download URL and extracted dir name are -// built from those ORT_VERSION constants, not from this one. The Go binding +// DeepDoc backend is built and tested against (e.g. "1.29.0"). It is ONE OF +// FOUR raw version declarations that must stay equal (the other three are +// ORT_VERSION in ragflow_deps/download_go_deps.py and ragflow_deps/download_deps.py, +// and ARG ORT_VERSION in Dockerfile_go) — NOT a single source of truth. The +// download URL and extracted dir name are built from those ORT_VERSION +// constants, not from this one. The Go binding // (github.com/infiniflow/onnxruntime_go, the org mirror of yalue/onnxruntime_go) // and the pip onnxruntime== pin must // track this MINOR version: the binding uses its own release numbering @@ -279,11 +280,15 @@ func HasModelFiles(dir string) bool { // the same minor line. ONNX Runtime is linked statically (libonnxruntime.a), // so there is no .so / SONAME at runtime. // -// To bump ORT, ALL of the following must change together (drift breaks the -// static link or the runtime OrtGetApiBase lookup): -// - DeepDocORTVersion (here, Go) AND ORT_VERSION in BOTH -// ragflow_deps/download_go_deps.py and ragflow_deps/download_deps.py; -// - the onnxruntime== pin in pyproject.toml and the onnxruntime / -// onnxruntime-gpu pins in .github/workflows/deepdoc-drift.yml; +// To bump ORT, the four Go-side native pins above must change together +// (drift breaks the static link or the runtime OrtGetApiBase lookup): +// - DeepDocORTVersion (here, Go) +// - ORT_VERSION in ragflow_deps/download_go_deps.py +// - ORT_VERSION in ragflow_deps/download_deps.py +// - ARG ORT_VERSION in Dockerfile_go +// +// Separately, keep these on the same ORT minor line but version them +// independently of the Go native lib (see pyproject.toml / development.md): +// - the onnxruntime== / onnxruntime-gpu== pins in pyproject.toml (Python side) // - the onnxruntime_go binding minor in go.mod. -const DeepDocORTVersion = "1.23.2" +const DeepDocORTVersion = "1.29.0" diff --git a/internal/development.md b/internal/development.md index 37a39164dc..12c59c8ca9 100644 --- a/internal/development.md +++ b/internal/development.md @@ -103,6 +103,19 @@ python3 ragflow_deps/download_go_deps.py > startup, so the remedy is always to seed the static lib above, never to build > without it. +> **Note**: The ONNX Runtime native version is pinned in several Go-side places +> that must stay in sync. Bumping it in one spot and not the others fails the +> build with `Error: ONNX Runtime version is inconsistent`: +> - `internal/common/environments.go` — `DeepDocORTVersion` +> - `Dockerfile_go` — `ARG ORT_VERSION` +> - `ragflow_deps/download_go_deps.py` and `ragflow_deps/download_deps.py` — `ORT_VERSION` +> +> `build.sh` runs this consistency check automatically before the Go build +> (through `check_go_deps`) and fails fast on any mismatch. Run it on demand +> with `./build.sh --check-ort-version`. To upgrade ORT, edit every entry above +> to the same version, then run the check. The Python pip `onnxruntime==` pin in +> `pyproject.toml` is versioned independently and is intentionally not part of +> this check. ### 1.5 Build RAGFlow diff --git a/pyproject.toml b/pyproject.toml index 6e328ff646..84a857e6d0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -75,8 +75,8 @@ dependencies = [ "mypy-boto3-s3==1.40.26", "Office365-REST-Python-Client==2.6.2", "ollama>=0.5.0", - "onnxruntime==1.23.2; sys_platform == 'darwin' or platform_machine != 'x86_64'", # must match internal/common.DeepDocORTVersion (native ORT minor) - "onnxruntime-gpu==1.23.2; sys_platform != 'darwin' and platform_machine == 'x86_64'", # must match internal/common.DeepDocORTVersion (native ORT minor) + "onnxruntime==1.23.2; sys_platform == 'darwin' or platform_machine != 'x86_64'", # Python-side only; versioned independently of the Go native ORT static lib (internal/common.DeepDocORTVersion) + "onnxruntime-gpu==1.23.2; sys_platform != 'darwin' and platform_machine == 'x86_64'", # Python-side only; versioned independently of the Go native ORT static lib (internal/common.DeepDocORTVersion) "opencv-python==4.10.0.84", "opencv-python-headless==4.10.0.84", "opendal>=0.45.0,<0.46.0", diff --git a/ragflow_deps/download_deps.py b/ragflow_deps/download_deps.py index adcaf1a938..f7c7bb6052 100644 --- a/ragflow_deps/download_deps.py +++ b/ragflow_deps/download_deps.py @@ -43,12 +43,36 @@ os.environ.setdefault("NLTK_ALLOW_PROXIED_URLOPEN", "1") import nltk from huggingface_hub import snapshot_download -# mirrors internal/common.DeepDocORTVersion (Go in-process backend). Single -# source for the onnxruntime native release: the download URL, .tgz name, -# extracted dir name, and SONAME below are all derived from it. The pip -# onnxruntime== pin (pyproject.toml) and the onnxruntime_go binding minor -# (go.mod) must stay on the same minor line. -ORT_VERSION = "1.23.2" +# mirrors internal/common.DeepDocORTVersion (Go in-process backend). ONE OF +# FOUR places (with that Go constant, ORT_VERSION in ragflow_deps/download_go_deps.py, +# and ARG ORT_VERSION in Dockerfile_go) that must carry the same ONNX Runtime +# native release for the statically-linked Go DeepDoc backend. This file's +# download URL and extracted dir name are derived from ORT_VERSION here, but +# there is no single source of truth — keep all four equal. build.sh +# --check-ort-version greps this file (and the other three) to fail fast on +# drift. (The Python pip onnxruntime== pin in pyproject.toml is versioned +# independently and is not part of this check.) +# +# Source of the native static archives: infiniflow/ragflow-build (our own +# ORT-only minimal build), NOT the third-party csukuangfj/onnxruntime-libs +# account. The release tag is `onnxruntime-v{ORT_VERSION}` and the asset is +# `onnxruntime-v{ORT_VERSION}-linux-x86_64.zip`. +ORT_VERSION = "1.29.0" + + +def _ort_asset_name(version): + """Release asset filename under infiniflow/ragflow-build tag onnxruntime-v{version}.""" + return f"onnxruntime-v{version}-linux-x86_64.zip" + + +def _ort_extracted_dir(version): + """Top-level directory name INSIDE the release zip (what extractall creates).""" + return f"onnxruntime-v{version}-linux-x86_64" + + +def _ort_normalized_dir(version): + """Directory name build.sh's `find ... -name '*.a'` glob expects under static_lib.""" + return f"onnxruntime-linux-x64-static_lib-{version}-glibc2_28" def get_urls(use_china_mirrors=False) -> list[str | list[str]]: @@ -91,14 +115,15 @@ def get_urls(use_china_mirrors=False) -> list[str | list[str]]: # kernels are dropped; only OrtGetApiBase is exported, via # --dynamic-list), so no libonnxruntime.so is needed at runtime — # OrtGetApiBase is resolved via dlopen(NULL) (the process-global - # symbol table, not the executable's own path). csukuangfj's - # static_lib build is - # CPU-only and glibc2_28-based, matching ORT_VERSION's C-API line - # (ABI-compatible with onnxruntime_go) and the onnxruntime the - # Python goldens were generated with. + # symbol table, not the executable's own path). Our own + # infiniflow/ragflow-build ORT-only minimal build + # (onnxruntime-v{ORT_VERSION}) is CPU-only and glibc2_28-based, + # matching ORT_VERSION's C-API line (ABI-compatible with + # onnxruntime_go) and the onnxruntime the Python goldens were + # generated with. [ - f"https://github.com/csukuangfj/onnxruntime-libs/releases/download/v{ORT_VERSION}/onnxruntime-linux-x64-static_lib-{ORT_VERSION}-glibc2_28.zip", - f"onnxruntime-linux-x64-static_lib-{ORT_VERSION}-glibc2_28.zip", + f"https://github.com/infiniflow/ragflow-build/releases/download/onnxruntime-v{ORT_VERSION}/{_ort_asset_name(ORT_VERSION)}", + _ort_asset_name(ORT_VERSION), ], ] else: @@ -140,14 +165,15 @@ def get_urls(use_china_mirrors=False) -> list[str | list[str]]: # kernels are dropped; only OrtGetApiBase is exported, via # --dynamic-list), so no libonnxruntime.so is needed at runtime — # OrtGetApiBase is resolved via dlopen(NULL) (the process-global - # symbol table, not the executable's own path). csukuangfj's - # static_lib build is - # CPU-only and glibc2_28-based, matching ORT_VERSION's C-API line - # (ABI-compatible with onnxruntime_go) and the onnxruntime the - # Python goldens were generated with. + # symbol table, not the executable's own path). Our own + # infiniflow/ragflow-build ORT-only minimal build + # (onnxruntime-v{ORT_VERSION}) is CPU-only and glibc2_28-based, + # matching ORT_VERSION's C-API line (ABI-compatible with + # onnxruntime_go) and the onnxruntime the Python goldens were + # generated with. [ - f"https://github.com/csukuangfj/onnxruntime-libs/releases/download/v{ORT_VERSION}/onnxruntime-linux-x64-static_lib-{ORT_VERSION}-glibc2_28.zip", - f"onnxruntime-linux-x64-static_lib-{ORT_VERSION}-glibc2_28.zip", + f"https://github.com/infiniflow/ragflow-build/releases/download/onnxruntime-v{ORT_VERSION}/{_ort_asset_name(ORT_VERSION)}", + _ort_asset_name(ORT_VERSION), ], ] @@ -197,7 +223,7 @@ if __name__ == "__main__": ("pdfium-linux-x64-static.tgz", "pdfium-static"), ("pdf_oxide-go-ffi-linux-amd64.tar.gz", "pdf_oxide"), ("office_oxide-linux-x86_64.tar.gz", "office_oxide"), - (f"onnxruntime-linux-x64-static_lib-{ORT_VERSION}-glibc2_28.zip", os.path.join("onnxruntime", "static_lib")), + (_ort_asset_name(ORT_VERSION), os.path.join("onnxruntime", "static_lib")), ] import tarfile import zipfile @@ -209,7 +235,7 @@ if __name__ == "__main__": symbols / wrong version, silently).""" if not os.path.isdir(static_lib_dir): return - expected = f"onnxruntime-linux-x64-static_lib-{version}-glibc2_28" + expected = _ort_normalized_dir(version) for name in os.listdir(static_lib_dir): if not name.startswith("onnxruntime-linux-x64-static_lib-"): continue @@ -226,8 +252,8 @@ if __name__ == "__main__": continue target = os.path.join(native_deps_dir, subdir) - # ONNX Runtime ships a version-stamped top-level dir inside the zip - # (onnxruntime-linux-x64-static_lib--glibc2_28/). A plain + # The infiniflow/ragflow-build release zip carries a top-level dir + # named onnxruntime-v{ORT_VERSION}-linux-x86_64. A plain # "any .a present?" skip would keep a STALE version in place after a # bump: the new zip downloads, but extraction is skipped because the # old .a is still under static_lib, so the bump silently does nothing. @@ -235,7 +261,7 @@ if __name__ == "__main__": # already extracted. if subdir == os.path.join("onnxruntime", "static_lib"): _prune_stale_onnxruntime(target, ORT_VERSION) - version_dir = os.path.join(target, f"onnxruntime-linux-x64-static_lib-{ORT_VERSION}-glibc2_28") + version_dir = os.path.join(target, _ort_normalized_dir(ORT_VERSION)) if os.path.isdir(version_dir) and any(f.endswith(".a") for _, _, files in os.walk(version_dir) for f in files): print(f" ✓ {subdir} ({ORT_VERSION}) already extracted to {version_dir}") continue @@ -248,6 +274,19 @@ if __name__ == "__main__": if archive_path.endswith(".zip"): with zipfile.ZipFile(archive_path) as zf: zf.extractall(target) + # The infiniflow/ragflow-build release zip carries a top-level dir + # named onnxruntime-v{version}-linux-x86_64, but build.sh's glob and + # the stale checks above all expect + # onnxruntime-linux-x64-static_lib-{version}-glibc2_28. Rename it so + # every consumer shares one name convention (driven by ORT_VERSION). + if subdir == os.path.join("onnxruntime", "static_lib"): + extracted = os.path.join(target, _ort_extracted_dir(ORT_VERSION)) + normalized = os.path.join(target, _ort_normalized_dir(ORT_VERSION)) + if os.path.isdir(extracted) and extracted != normalized: + if os.path.exists(normalized): + shutil.rmtree(normalized) + print(f" Renaming {os.path.basename(extracted)} → {os.path.basename(normalized)}") + os.rename(extracted, normalized) else: with tarfile.open(archive_path) as tf: tf.extractall(target) diff --git a/ragflow_deps/download_go_deps.py b/ragflow_deps/download_go_deps.py index 36f83744d4..92a56f07eb 100644 --- a/ragflow_deps/download_go_deps.py +++ b/ragflow_deps/download_go_deps.py @@ -53,13 +53,35 @@ import zipfile import requests # Mirrors internal/common.DeepDocORTVersion (Go in-process backend). ONE OF -# THREE places (with that Go constant and the ORT_VERSION in -# ragflow_deps/download_deps.py) that must carry the same ONNX Runtime native -# release for the statically-linked Go DeepDoc backend. There is no single -# source of truth — keep all three equal. The pip onnxruntime== pin -# (pyproject.toml) and the onnxruntime_go binding minor (go.mod) must stay on -# the same minor line. -ORT_VERSION = "1.23.2" +# FOUR places (with that Go constant, ORT_VERSION in ragflow_deps/download_deps.py, +# and ARG ORT_VERSION in Dockerfile_go) that must carry the same ONNX Runtime +# native release for the statically-linked Go DeepDoc backend. There is no +# single source of truth — keep all four equal. build.sh --check-ort-version +# greps this file (and the other three) to fail fast on drift. (The Python pip +# onnxruntime== pin in pyproject.toml is versioned independently and is not +# part of this check.) +# +# Source of the native static archives: infiniflow/ragflow-build (our own +# ORT-only minimal build), NOT the third-party csukuangfj/onnxruntime-libs +# account. The release tag is `onnxruntime-v{ORT_VERSION}` and the asset is +# `onnxruntime-v{ORT_VERSION}-linux-x86_64.zip`. +ORT_VERSION = "1.29.0" + + +def _ort_asset_name(version): + """Release asset filename under infiniflow/ragflow-build tag onnxruntime-v{version}.""" + return f"onnxruntime-v{version}-linux-x86_64.zip" + + +def _ort_extracted_dir(version): + """Top-level directory name INSIDE the release zip (what extractall creates).""" + return f"onnxruntime-v{version}-linux-x86_64" + + +def _ort_normalized_dir(version): + """Directory name build.sh's `find ... -name '*.a'` glob expects under static_lib.""" + return f"onnxruntime-linux-x64-static_lib-{version}-glibc2_28" + # Mirrors internal/common.DeepDocModelFiles (Go in-process DeepDoc backend). # These are the ONLY weights the Go backend loads; the full InfiniFlow/deepdoc @@ -76,7 +98,7 @@ def prune_stale_onnxruntime(static_lib_dir, version): symbols / wrong version, silently).""" if not os.path.isdir(static_lib_dir): return - expected = f"onnxruntime-linux-x64-static_lib-{version}-glibc2_28" + expected = _ort_normalized_dir(version) for name in os.listdir(static_lib_dir): if not name.startswith("onnxruntime-linux-x64-static_lib-"): continue @@ -97,8 +119,8 @@ def extract_onnxruntime(static_lib_dir, archive_path, version): `static_lib_dir`. Returns True when that version is available afterwards (extracted now or already present), False when the archive is missing. - ORT ships a version-stamped top-level dir inside the zip - (onnxruntime-linux-x64-static_lib--glibc2_28/), so a present + The infiniflow/ragflow-build release zip carries a top-level dir named + onnxruntime-v{version}-linux-x86_64, so a present `static_lib_dir` is NOT evidence that THIS version is extracted: after a version bump the stale dir is pruned and the new one must be extracted. """ @@ -106,7 +128,7 @@ def extract_onnxruntime(static_lib_dir, archive_path, version): print(f" Skipping extraction: {os.path.basename(archive_path)} not found") return False prune_stale_onnxruntime(static_lib_dir, version) - version_dir = os.path.join(static_lib_dir, f"onnxruntime-linux-x64-static_lib-{version}-glibc2_28") + version_dir = os.path.join(static_lib_dir, _ort_normalized_dir(version)) if os.path.isdir(version_dir) and has_static_archives(version_dir): print(f" ✓ onnxruntime/static_lib ({version}) already extracted to {version_dir}") return True @@ -114,6 +136,18 @@ def extract_onnxruntime(static_lib_dir, archive_path, version): print(f" Extracting {os.path.basename(archive_path)} → {static_lib_dir}") with zipfile.ZipFile(archive_path) as zf: zf.extractall(static_lib_dir) + # The infiniflow/ragflow-build release zip carries a top-level dir named + # onnxruntime-v{version}-linux-x86_64, but build.sh's glob and the stale + # checks above all expect onnxruntime-linux-x64-static_lib-{version}-glibc2_28. + # Rename it so every consumer shares one name convention (driven by + # ORT_VERSION). + extracted = os.path.join(static_lib_dir, _ort_extracted_dir(version)) + normalized = os.path.join(static_lib_dir, _ort_normalized_dir(version)) + if os.path.isdir(extracted) and extracted != normalized: + if os.path.exists(normalized): + shutil.rmtree(normalized) + print(f" Renaming {os.path.basename(extracted)} → {os.path.basename(normalized)}") + os.rename(extracted, normalized) return True @@ -143,8 +177,8 @@ def get_urls(use_china_mirrors=False) -> list[str | list[str]]: ["https://gh-proxy.com/https://github.com/yfedoseev/pdf_oxide/releases/download/v0.3.73/pdf_oxide-go-ffi-linux-amd64.tar.gz", "pdf_oxide-go-ffi-linux-amd64.tar.gz"], ["https://gh-proxy.com/https://github.com/yfedoseev/office_oxide/releases/download/v0.1.9/native-linux-x86_64.tar.gz", "office_oxide-linux-x86_64.tar.gz"], [ - f"https://gh-proxy.com/https://github.com/csukuangfj/onnxruntime-libs/releases/download/v{ORT_VERSION}/onnxruntime-linux-x64-static_lib-{ORT_VERSION}-glibc2_28.zip", - f"onnxruntime-linux-x64-static_lib-{ORT_VERSION}-glibc2_28.zip", + f"https://gh-proxy.com/https://github.com/infiniflow/ragflow-build/releases/download/onnxruntime-v{ORT_VERSION}/{_ort_asset_name(ORT_VERSION)}", + _ort_asset_name(ORT_VERSION), ], ] else: @@ -172,8 +206,8 @@ def get_urls(use_china_mirrors=False) -> list[str | list[str]]: ["https://github.com/yfedoseev/pdf_oxide/releases/download/v0.3.73/pdf_oxide-go-ffi-linux-amd64.tar.gz", "pdf_oxide-go-ffi-linux-amd64.tar.gz"], ["https://github.com/yfedoseev/office_oxide/releases/download/v0.1.9/native-linux-x86_64.tar.gz", "office_oxide-linux-x86_64.tar.gz"], [ - f"https://github.com/csukuangfj/onnxruntime-libs/releases/download/v{ORT_VERSION}/onnxruntime-linux-x64-static_lib-{ORT_VERSION}-glibc2_28.zip", - f"onnxruntime-linux-x64-static_lib-{ORT_VERSION}-glibc2_28.zip", + f"https://github.com/infiniflow/ragflow-build/releases/download/onnxruntime-v{ORT_VERSION}/{_ort_asset_name(ORT_VERSION)}", + _ort_asset_name(ORT_VERSION), ], ] @@ -312,7 +346,7 @@ if __name__ == "__main__": if not extract_onnxruntime( os.path.join(native_deps_dir, "onnxruntime", "static_lib"), - os.path.join(os.getcwd(), f"onnxruntime-linux-x64-static_lib-{ORT_VERSION}-glibc2_28.zip"), + os.path.join(os.getcwd(), _ort_asset_name(ORT_VERSION)), ORT_VERSION, ): # The archive was not downloaded or failed to extract, so no .a landed. diff --git a/ragflow_deps/test_download_go_deps.py b/ragflow_deps/test_download_go_deps.py index 6660797236..78c834ad1b 100644 --- a/ragflow_deps/test_download_go_deps.py +++ b/ragflow_deps/test_download_go_deps.py @@ -4,17 +4,30 @@ # tests must guarantee the archive is really landed on disk — above all after an # ORT version bump, where the zip's version-stamped top-level dir changes but # static_lib/ still exists from the previous run. +# +# The release zip from infiniflow/ragflow-build carries a top-level dir named +# onnxruntime-v{version}-linux-x86_64; extract_onnxruntime() must rename it to +# the build.sh-expected onnxruntime-linux-x64-static_lib-{version}-glibc2_28 so +# every consumer shares one name convention. These tests pin that rename. import os import zipfile -from download_go_deps import extract_onnxruntime, has_static_archives +from download_go_deps import ( + _ort_asset_name, + _ort_extracted_dir, + _ort_normalized_dir, + extract_onnxruntime, + has_static_archives, +) def make_ort_zip(path, version): - """Build a zip shaped like the upstream ORT release: a version-stamped - top-level dir holding lib/libonnxruntime.a.""" - member = f"onnxruntime-linux-x64-static_lib-{version}-glibc2_28/lib/libonnxruntime.a" + """Build a zip shaped like the infiniflow/ragflow-build ORT release: a + top-level dir named onnxruntime-v{version}-linux-x86_64 holding + lib/libonnxruntime.a. extract_onnxruntime() must rename it to the + build.sh-expected onnxruntime-linux-x64-static_lib-{version}-glibc2_28.""" + member = f"{_ort_extracted_dir(version)}/lib/libonnxruntime.a" with zipfile.ZipFile(path, "w") as zf: zf.writestr(member, b"!\nort-payload") return path @@ -23,12 +36,14 @@ def make_ort_zip(path, version): def test_extracts_on_first_run(tmp_path): # static_lib/ does not exist yet: the plain first-run path. static_lib = tmp_path / "onnxruntime" / "static_lib" - version = "1.23.2" - archive = make_ort_zip(tmp_path / f"onnxruntime-linux-x64-static_lib-{version}-glibc2_28.zip", version) + version = "1.29.0" + archive = make_ort_zip(tmp_path / _ort_asset_name(version), version) assert extract_onnxruntime(str(static_lib), str(archive), version) is True - version_dir = static_lib / f"onnxruntime-linux-x64-static_lib-{version}-glibc2_28" + extracted = static_lib / _ort_extracted_dir(version) + assert not extracted.exists(), "release zip top-level dir must be renamed away" + version_dir = static_lib / _ort_normalized_dir(version) assert version_dir.is_dir() assert has_static_archives(str(version_dir)) @@ -40,30 +55,32 @@ def test_extracts_after_version_bump_when_static_lib_exists(tmp_path): build.sh's ORT guard rejects the build.""" static_lib = tmp_path / "onnxruntime" / "static_lib" static_lib.mkdir(parents=True) - stale = static_lib / "onnxruntime-linux-x64-static_lib-1.23.1-glibc2_28" + stale = static_lib / _ort_normalized_dir("1.28.0") (stale / "lib").mkdir(parents=True) (stale / "lib" / "libonnxruntime.a").write_bytes(b"old-ort") - version = "1.23.2" - archive = make_ort_zip(tmp_path / f"onnxruntime-linux-x64-static_lib-{version}-glibc2_28.zip", version) + version = "1.29.0" + archive = make_ort_zip(tmp_path / _ort_asset_name(version), version) assert extract_onnxruntime(str(static_lib), str(archive), version) is True assert not stale.exists(), "stale ORT version should be pruned" - version_dir = static_lib / f"onnxruntime-linux-x64-static_lib-{version}-glibc2_28" + extracted = static_lib / _ort_extracted_dir(version) + assert not extracted.exists(), "release zip top-level dir must be renamed away" + version_dir = static_lib / _ort_normalized_dir(version) assert version_dir.is_dir(), "bumped ORT version was silently not extracted" assert has_static_archives(str(version_dir)), "bumped ORT version landed no .a" def test_skips_when_archive_missing(tmp_path): static_lib = tmp_path / "onnxruntime" / "static_lib" - assert extract_onnxruntime(str(static_lib), str(tmp_path / "missing.zip"), "1.23.2") is False + assert extract_onnxruntime(str(static_lib), str(tmp_path / "missing.zip"), "1.29.0") is False def test_idempotent_when_matching_version_already_present(tmp_path): static_lib = tmp_path / "onnxruntime" / "static_lib" - version = "1.23.2" - archive = make_ort_zip(tmp_path / f"onnxruntime-linux-x64-static_lib-{version}-glibc2_28.zip", version) + version = "1.29.0" + archive = make_ort_zip(tmp_path / _ort_asset_name(version), version) assert extract_onnxruntime(str(static_lib), str(archive), version) is True before = sorted(os.listdir(static_lib))