mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-10-12 02:16:39 +00:00
Unify ONNX Runtime native to 1.29.0 from infiniflow/ragflow-build (#19446)
Switch Go DeepDoc's static ONNX Runtime to our own infiniflow/ragflow-build release (v1.29.0) instead of third-party. Keeps fused kernels; normalizes dir naming; added ORT version consistency check. --------- Co-authored-by: maoyifeng <[email protected]>
This commit is contained in:
xugangqiang
+7
-2
@@ -34,11 +34,16 @@ target/
|
||||
|
||||
# ── ragflow_deps build context (built as a separate image, mounted ──
|
||||
# ── from infiniflow/ragflow_deps:latest by the main Dockerfile) ──
|
||||
# Excluding the entire directory keeps the main build context small
|
||||
# Excluding the directory contents keeps the main build context small
|
||||
# regardless of which deps files download_deps.py currently fetches.
|
||||
# The deps image is built from this directory with:
|
||||
# cd ragflow_deps && docker build -f Dockerfile -t infiniflow/ragflow_deps .
|
||||
ragflow_deps/
|
||||
# The two ORT version-pin files are re-included: build.sh's
|
||||
# check_ort_version_consistency (run by Dockerfile_go's `--go` build) greps
|
||||
# them. Re-including requires NOT excluding the directory itself.
|
||||
ragflow_deps/*
|
||||
!ragflow_deps/download_go_deps.py
|
||||
!ragflow_deps/download_deps.py
|
||||
|
||||
# ── IDE and editor config ──────────────────────────────────────────────────
|
||||
.idea/
|
||||
|
||||
+17
-3
@@ -159,6 +159,11 @@ RUN --mount=type=cache,id=ragflow_gomod,target=/root/.cache/gomod \
|
||||
COPY internal internal
|
||||
COPY cmd cmd
|
||||
COPY build.sh ./
|
||||
# build.sh's check_ort_version_consistency (run via `./build.sh --go`) greps the
|
||||
# ORT version pins from these files; without them the --go build fails with
|
||||
# "could not parse the ONNX Runtime version from one of the pinned locations".
|
||||
COPY ragflow_deps/download_go_deps.py ragflow_deps/download_deps.py ./ragflow_deps/
|
||||
COPY Dockerfile_go ./
|
||||
|
||||
# ONNX Runtime static archives: build.sh's _seed_from_system looks for the ORT
|
||||
# static libs under ONNXRUNTIME_STATIC_PREFIX (default ~/ragflow-native-libs/onnxruntime).
|
||||
@@ -169,10 +174,19 @@ COPY build.sh ./
|
||||
# silently skipped and the binary fails at startup with
|
||||
# "no in-process DeepDoc backend serving" (dlopen(NULL)/dlsym can't find OrtGetApiBase).
|
||||
# ${HOME} (not hard-coded /root) keeps the path consistent with build.sh regardless of HOME.
|
||||
ARG ORT_VERSION=1.23.2
|
||||
ARG ORT_VERSION=1.29.0
|
||||
RUN set -eux; \
|
||||
mkdir -p "${HOME}/ragflow-native-libs/onnxruntime"; \
|
||||
cp -rn /opt/ragflow-native-libs/onnxruntime/* "${HOME}/ragflow-native-libs/onnxruntime/"; \
|
||||
mkdir -p "${HOME}/ragflow-native-libs/onnxruntime/static_lib"; \
|
||||
_src="/opt/ragflow-native-libs/onnxruntime/static_lib"; \
|
||||
_v="$(ls -d "${_src}"/onnxruntime-linux-x64-static_lib-"${ORT_VERSION}"-* 2>/dev/null | head -1)"; \
|
||||
if [ -z "${_v}" ]; then \
|
||||
_v="$(ls -d "${_src}"/onnxruntime-linux-x64-static_lib-* 2>/dev/null | head -1)"; \
|
||||
fi; \
|
||||
if [ -n "${_v}" ]; then \
|
||||
cp -r "${_v}" "${HOME}/ragflow-native-libs/onnxruntime/static_lib/"; \
|
||||
else \
|
||||
echo "Warning: no pre-baked onnxruntime static_lib in runner image; build.sh will attempt download" >&2; \
|
||||
fi; \
|
||||
find "${HOME}/ragflow-native-libs/onnxruntime/static_lib" -name '*.a' | head -5
|
||||
|
||||
RUN git config --global safe.directory "*" && \
|
||||
|
||||
@@ -150,6 +150,38 @@ check_go_deps() {
|
||||
command -v go >/dev/null 2>&1 || { echo -e "${RED}Error: go is required but not installed.${NC}"; exit 1; }
|
||||
|
||||
echo "✓ Required tools are available"
|
||||
check_ort_version_consistency
|
||||
}
|
||||
|
||||
# Fail fast when the ONNX Runtime native version is declared inconsistently.
|
||||
# The in-process (Go) DeepDoc backend statically links libonnxruntime.a built
|
||||
# from ONE exact ORT release; a mismatch links a wrong/missing .a and only fails
|
||||
# at runtime (dlopen(NULL) can't find OrtGetApiBase). The version is pinned in
|
||||
# four Go-side locations that must all agree.
|
||||
check_ort_version_consistency() {
|
||||
print_section "Checking ONNX Runtime version consistency"
|
||||
|
||||
local env_go d1 d2 dockerfile
|
||||
env_go="$(grep -m1 -E 'DeepDocORTVersion[[:space:]]*=[[:space:]]*"' "${PROJECT_ROOT}/internal/common/environments.go" | sed -E 's/.*"([^"]+)".*/\1/')"
|
||||
d1="$(grep -m1 -E '^ORT_VERSION[[:space:]]*=[[:space:]]*"' "${PROJECT_ROOT}/ragflow_deps/download_go_deps.py" | sed -E 's/.*"([^"]+)".*/\1/')"
|
||||
d2="$(grep -m1 -E '^ORT_VERSION[[:space:]]*=[[:space:]]*"' "${PROJECT_ROOT}/ragflow_deps/download_deps.py" | sed -E 's/.*"([^"]+)".*/\1/')"
|
||||
dockerfile="$(grep -m1 -E 'ARG[[:space:]]+ORT_VERSION=' "${PROJECT_ROOT}/Dockerfile_go" | sed -E 's/.*ORT_VERSION=([0-9][^"[:space:]]*).*/\1/')"
|
||||
|
||||
if [ -z "$env_go" ] || [ -z "$d1" ] || [ -z "$d2" ] || [ -z "$dockerfile" ]; then
|
||||
echo -e "${RED}Error: could not parse the ONNX Runtime version from one of the pinned locations${NC}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ "$env_go" != "$d1" ] || [ "$env_go" != "$d2" ] || [ "$env_go" != "$dockerfile" ]; then
|
||||
echo -e "${RED}Error: ONNX Runtime version is inconsistent — fix before building:${NC}" >&2
|
||||
printf ' %-10s %s\n' "$env_go" "internal/common/environments.go:DeepDocORTVersion"
|
||||
printf ' %-10s %s\n' "$d1" "ragflow_deps/download_go_deps.py:ORT_VERSION"
|
||||
printf ' %-10s %s\n' "$d2" "ragflow_deps/download_deps.py:ORT_VERSION"
|
||||
printf ' %-10s %s\n' "$dockerfile" "Dockerfile_go:ARG ORT_VERSION"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo -e "${GREEN}✓ ONNX Runtime native version consistent: ${env_go}${NC}"
|
||||
}
|
||||
|
||||
# Check office_oxide native library
|
||||
@@ -544,32 +576,34 @@ setup_cgo_env() {
|
||||
if [ -d "$ONNXRUNTIME_STATIC_PREFIX" ]; then
|
||||
# Collect every .a, but skip GPU-only providers we never build
|
||||
# against (would pull in CUDA/cuDNN/TensorRT which we don't ship).
|
||||
#
|
||||
# Select the ORT static lib that matches the Go deepdoc backend's
|
||||
# required version (DeepDocORTVersion in internal/common/environments.go).
|
||||
# Go and Python build/link against independent ORT versions, so the
|
||||
# static_lib prefix legitimately holds more than one
|
||||
# onnxruntime-linux-x64-static_lib-* dir at once (e.g. the Python-side
|
||||
# 1.23.x next to the Go-side 1.29.0). We pick the dir that matches
|
||||
# DeepDocORTVersion rather than failing when a second version dir is
|
||||
# present. This avoids silently linking the wrong version while still
|
||||
# keeping the bake self-documenting.
|
||||
local ort_version
|
||||
ort_version="$(grep -m1 -E 'DeepDocORTVersion[[:space:]]*=[[:space:]]*"' \
|
||||
"${PROJECT_ROOT}/internal/common/environments.go" \
|
||||
| sed -E 's/.*"([^"]+)".*/\1/')"
|
||||
if [ -z "$ort_version" ]; then
|
||||
echo " Error: cannot parse DeepDocORTVersion from internal/common/environments.go" >&2
|
||||
return 1
|
||||
fi
|
||||
local ort_a=""
|
||||
local seen_version_dir=""
|
||||
while IFS= read -r f; do
|
||||
case "$(basename "$f")" in
|
||||
*cuda*|*tensorrt*|*coreml*|*dml*|*migraphx*) continue ;;
|
||||
esac
|
||||
# Guard against coexisting stale version dirs: if .a files span
|
||||
# more than one onnxruntime-linux-x64-static_lib-* dir, fail fast
|
||||
# instead of silently linking two ORT versions (duplicate symbols
|
||||
# / wrong version). Re-run `download_deps.py` to prune stale dirs
|
||||
# after a version bump, or remove the old dir by hand.
|
||||
case "$f" in
|
||||
*/onnxruntime-linux-x64-static_lib-*/lib/*.a)
|
||||
local vdir="${f#*/onnxruntime-linux-x64-static_lib-}"
|
||||
vdir="${vdir%%/*}"
|
||||
if [ -z "$seen_version_dir" ]; then
|
||||
seen_version_dir="$vdir"
|
||||
elif [ "$seen_version_dir" != "$vdir" ]; then
|
||||
echo " Error: multiple ONNX Runtime versions found under $ONNXRUNTIME_STATIC_PREFIX" >&2
|
||||
echo " $seen_version_dir AND $vdir" >&2
|
||||
echo " Remove the stale version dir (or re-run download_deps.py to prune it)." >&2
|
||||
return 1
|
||||
fi
|
||||
;;
|
||||
# Only collect .a from the dir matching the required version.
|
||||
*/onnxruntime-linux-x64-static_lib-"${ort_version}"*/lib/*.a)
|
||||
ort_a="$ort_a $f" ;;
|
||||
esac
|
||||
ort_a="$ort_a $f"
|
||||
done < <(find "$ONNXRUNTIME_STATIC_PREFIX" -type f -name '*.a' 2>/dev/null)
|
||||
|
||||
if [ -n "$ort_a" ]; then
|
||||
@@ -604,7 +638,13 @@ setup_cgo_env() {
|
||||
# dynamic symbol table, which is what the binding's dlopen(NULL)+dlsym
|
||||
# lookup needs at runtime (no --export-dynamic required).
|
||||
else
|
||||
echo " onnxruntime static_lib dir has no .a files; the in-process DeepDoc backend cannot link ORT" >&2
|
||||
local avail
|
||||
avail="$(find "$ONNXRUNTIME_STATIC_PREFIX" -maxdepth 1 -type d \
|
||||
-name 'onnxruntime-linux-x64-static_lib-*' -exec basename {} \; 2>/dev/null | tr '\n' ' ')"
|
||||
echo " Error: no ONNX Runtime ${ort_version} static lib under $ONNXRUNTIME_STATIC_PREFIX" >&2
|
||||
echo " available: ${avail:-<none>}" >&2
|
||||
echo " DeepDocORTVersion=${ort_version}; bake/download the matching ORT (or update DeepDocORTVersion)." >&2
|
||||
return 1
|
||||
fi
|
||||
else
|
||||
echo " onnxruntime static_lib not found ($ONNXRUNTIME_STATIC_PREFIX); the in-process DeepDoc backend cannot link ORT" >&2
|
||||
@@ -809,6 +849,8 @@ OPTIONS:
|
||||
InfiniFlow/deepdoc model snapshot; self-skip otherwise).
|
||||
e.g. `$0 --test-native`
|
||||
--clean, -C Clean all build artifacts
|
||||
--check-ort-version Verify the ONNX Runtime native version is declared
|
||||
consistently across all sources (exit 1 on mismatch).
|
||||
--run, -r Build and run the server
|
||||
--strip, -s Strip debug symbols from Go binaries (-ldflags="-s -w")
|
||||
(disabled by default, useful for smaller production binaries)
|
||||
@@ -927,6 +969,9 @@ main() {
|
||||
--clean|-C)
|
||||
clean
|
||||
;;
|
||||
--check-ort-version)
|
||||
check_ort_version_consistency
|
||||
;;
|
||||
--run|-r)
|
||||
check_cpp_deps
|
||||
check_go_deps
|
||||
|
||||
@@ -267,11 +267,12 @@ func HasModelFiles(dir string) bool {
|
||||
}
|
||||
|
||||
// DeepDocORTVersion is the onnxruntime native release the in-process (Go)
|
||||
// DeepDoc backend is built and tested against (e.g. "1.23.2"). It is ONE OF
|
||||
// THREE raw version declarations that must stay equal (the other two are
|
||||
// ORT_VERSION in ragflow_deps/download_go_deps.py and ragflow_deps/download_deps.py)
|
||||
// — NOT a single source of truth. The download URL and extracted dir name are
|
||||
// built from those ORT_VERSION constants, not from this one. The Go binding
|
||||
// DeepDoc backend is built and tested against (e.g. "1.29.0"). It is ONE OF
|
||||
// FOUR raw version declarations that must stay equal (the other three are
|
||||
// ORT_VERSION in ragflow_deps/download_go_deps.py and ragflow_deps/download_deps.py,
|
||||
// and ARG ORT_VERSION in Dockerfile_go) — NOT a single source of truth. The
|
||||
// download URL and extracted dir name are built from those ORT_VERSION
|
||||
// constants, not from this one. The Go binding
|
||||
// (github.com/infiniflow/onnxruntime_go, the org mirror of yalue/onnxruntime_go)
|
||||
// and the pip onnxruntime== pin must
|
||||
// track this MINOR version: the binding uses its own release numbering
|
||||
@@ -279,11 +280,15 @@ func HasModelFiles(dir string) bool {
|
||||
// the same minor line. ONNX Runtime is linked statically (libonnxruntime.a),
|
||||
// so there is no .so / SONAME at runtime.
|
||||
//
|
||||
// To bump ORT, ALL of the following must change together (drift breaks the
|
||||
// static link or the runtime OrtGetApiBase lookup):
|
||||
// - DeepDocORTVersion (here, Go) AND ORT_VERSION in BOTH
|
||||
// ragflow_deps/download_go_deps.py and ragflow_deps/download_deps.py;
|
||||
// - the onnxruntime== pin in pyproject.toml and the onnxruntime /
|
||||
// onnxruntime-gpu pins in .github/workflows/deepdoc-drift.yml;
|
||||
// To bump ORT, the four Go-side native pins above must change together
|
||||
// (drift breaks the static link or the runtime OrtGetApiBase lookup):
|
||||
// - DeepDocORTVersion (here, Go)
|
||||
// - ORT_VERSION in ragflow_deps/download_go_deps.py
|
||||
// - ORT_VERSION in ragflow_deps/download_deps.py
|
||||
// - ARG ORT_VERSION in Dockerfile_go
|
||||
//
|
||||
// Separately, keep these on the same ORT minor line but version them
|
||||
// independently of the Go native lib (see pyproject.toml / development.md):
|
||||
// - the onnxruntime== / onnxruntime-gpu== pins in pyproject.toml (Python side)
|
||||
// - the onnxruntime_go binding minor in go.mod.
|
||||
const DeepDocORTVersion = "1.23.2"
|
||||
const DeepDocORTVersion = "1.29.0"
|
||||
|
||||
@@ -103,6 +103,19 @@ python3 ragflow_deps/download_go_deps.py
|
||||
> startup, so the remedy is always to seed the static lib above, never to build
|
||||
> without it.
|
||||
|
||||
> **Note**: The ONNX Runtime native version is pinned in several Go-side places
|
||||
> that must stay in sync. Bumping it in one spot and not the others fails the
|
||||
> build with `Error: ONNX Runtime version is inconsistent`:
|
||||
> - `internal/common/environments.go` — `DeepDocORTVersion`
|
||||
> - `Dockerfile_go` — `ARG ORT_VERSION`
|
||||
> - `ragflow_deps/download_go_deps.py` and `ragflow_deps/download_deps.py` — `ORT_VERSION`
|
||||
>
|
||||
> `build.sh` runs this consistency check automatically before the Go build
|
||||
> (through `check_go_deps`) and fails fast on any mismatch. Run it on demand
|
||||
> with `./build.sh --check-ort-version`. To upgrade ORT, edit every entry above
|
||||
> to the same version, then run the check. The Python pip `onnxruntime==` pin in
|
||||
> `pyproject.toml` is versioned independently and is intentionally not part of
|
||||
> this check.
|
||||
|
||||
### 1.5 Build RAGFlow
|
||||
|
||||
|
||||
+2
-2
@@ -75,8 +75,8 @@ dependencies = [
|
||||
"mypy-boto3-s3==1.40.26",
|
||||
"Office365-REST-Python-Client==2.6.2",
|
||||
"ollama>=0.5.0",
|
||||
"onnxruntime==1.23.2; sys_platform == 'darwin' or platform_machine != 'x86_64'", # must match internal/common.DeepDocORTVersion (native ORT minor)
|
||||
"onnxruntime-gpu==1.23.2; sys_platform != 'darwin' and platform_machine == 'x86_64'", # must match internal/common.DeepDocORTVersion (native ORT minor)
|
||||
"onnxruntime==1.23.2; sys_platform == 'darwin' or platform_machine != 'x86_64'", # Python-side only; versioned independently of the Go native ORT static lib (internal/common.DeepDocORTVersion)
|
||||
"onnxruntime-gpu==1.23.2; sys_platform != 'darwin' and platform_machine == 'x86_64'", # Python-side only; versioned independently of the Go native ORT static lib (internal/common.DeepDocORTVersion)
|
||||
"opencv-python==4.10.0.84",
|
||||
"opencv-python-headless==4.10.0.84",
|
||||
"opendal>=0.45.0,<0.46.0",
|
||||
|
||||
@@ -43,12 +43,36 @@ os.environ.setdefault("NLTK_ALLOW_PROXIED_URLOPEN", "1")
|
||||
import nltk
|
||||
from huggingface_hub import snapshot_download
|
||||
|
||||
# mirrors internal/common.DeepDocORTVersion (Go in-process backend). Single
|
||||
# source for the onnxruntime native release: the download URL, .tgz name,
|
||||
# extracted dir name, and SONAME below are all derived from it. The pip
|
||||
# onnxruntime== pin (pyproject.toml) and the onnxruntime_go binding minor
|
||||
# (go.mod) must stay on the same minor line.
|
||||
ORT_VERSION = "1.23.2"
|
||||
# mirrors internal/common.DeepDocORTVersion (Go in-process backend). ONE OF
|
||||
# FOUR places (with that Go constant, ORT_VERSION in ragflow_deps/download_go_deps.py,
|
||||
# and ARG ORT_VERSION in Dockerfile_go) that must carry the same ONNX Runtime
|
||||
# native release for the statically-linked Go DeepDoc backend. This file's
|
||||
# download URL and extracted dir name are derived from ORT_VERSION here, but
|
||||
# there is no single source of truth — keep all four equal. build.sh
|
||||
# --check-ort-version greps this file (and the other three) to fail fast on
|
||||
# drift. (The Python pip onnxruntime== pin in pyproject.toml is versioned
|
||||
# independently and is not part of this check.)
|
||||
#
|
||||
# Source of the native static archives: infiniflow/ragflow-build (our own
|
||||
# ORT-only minimal build), NOT the third-party csukuangfj/onnxruntime-libs
|
||||
# account. The release tag is `onnxruntime-v{ORT_VERSION}` and the asset is
|
||||
# `onnxruntime-v{ORT_VERSION}-linux-x86_64.zip`.
|
||||
ORT_VERSION = "1.29.0"
|
||||
|
||||
|
||||
def _ort_asset_name(version):
|
||||
"""Release asset filename under infiniflow/ragflow-build tag onnxruntime-v{version}."""
|
||||
return f"onnxruntime-v{version}-linux-x86_64.zip"
|
||||
|
||||
|
||||
def _ort_extracted_dir(version):
|
||||
"""Top-level directory name INSIDE the release zip (what extractall creates)."""
|
||||
return f"onnxruntime-v{version}-linux-x86_64"
|
||||
|
||||
|
||||
def _ort_normalized_dir(version):
|
||||
"""Directory name build.sh's `find ... -name '*.a'` glob expects under static_lib."""
|
||||
return f"onnxruntime-linux-x64-static_lib-{version}-glibc2_28"
|
||||
|
||||
|
||||
def get_urls(use_china_mirrors=False) -> list[str | list[str]]:
|
||||
@@ -91,14 +115,15 @@ def get_urls(use_china_mirrors=False) -> list[str | list[str]]:
|
||||
# kernels are dropped; only OrtGetApiBase is exported, via
|
||||
# --dynamic-list), so no libonnxruntime.so is needed at runtime —
|
||||
# OrtGetApiBase is resolved via dlopen(NULL) (the process-global
|
||||
# symbol table, not the executable's own path). csukuangfj's
|
||||
# static_lib build is
|
||||
# CPU-only and glibc2_28-based, matching ORT_VERSION's C-API line
|
||||
# (ABI-compatible with onnxruntime_go) and the onnxruntime the
|
||||
# Python goldens were generated with.
|
||||
# symbol table, not the executable's own path). Our own
|
||||
# infiniflow/ragflow-build ORT-only minimal build
|
||||
# (onnxruntime-v{ORT_VERSION}) is CPU-only and glibc2_28-based,
|
||||
# matching ORT_VERSION's C-API line (ABI-compatible with
|
||||
# onnxruntime_go) and the onnxruntime the Python goldens were
|
||||
# generated with.
|
||||
[
|
||||
f"https://github.com/csukuangfj/onnxruntime-libs/releases/download/v{ORT_VERSION}/onnxruntime-linux-x64-static_lib-{ORT_VERSION}-glibc2_28.zip",
|
||||
f"onnxruntime-linux-x64-static_lib-{ORT_VERSION}-glibc2_28.zip",
|
||||
f"https://github.com/infiniflow/ragflow-build/releases/download/onnxruntime-v{ORT_VERSION}/{_ort_asset_name(ORT_VERSION)}",
|
||||
_ort_asset_name(ORT_VERSION),
|
||||
],
|
||||
]
|
||||
else:
|
||||
@@ -140,14 +165,15 @@ def get_urls(use_china_mirrors=False) -> list[str | list[str]]:
|
||||
# kernels are dropped; only OrtGetApiBase is exported, via
|
||||
# --dynamic-list), so no libonnxruntime.so is needed at runtime —
|
||||
# OrtGetApiBase is resolved via dlopen(NULL) (the process-global
|
||||
# symbol table, not the executable's own path). csukuangfj's
|
||||
# static_lib build is
|
||||
# CPU-only and glibc2_28-based, matching ORT_VERSION's C-API line
|
||||
# (ABI-compatible with onnxruntime_go) and the onnxruntime the
|
||||
# Python goldens were generated with.
|
||||
# symbol table, not the executable's own path). Our own
|
||||
# infiniflow/ragflow-build ORT-only minimal build
|
||||
# (onnxruntime-v{ORT_VERSION}) is CPU-only and glibc2_28-based,
|
||||
# matching ORT_VERSION's C-API line (ABI-compatible with
|
||||
# onnxruntime_go) and the onnxruntime the Python goldens were
|
||||
# generated with.
|
||||
[
|
||||
f"https://github.com/csukuangfj/onnxruntime-libs/releases/download/v{ORT_VERSION}/onnxruntime-linux-x64-static_lib-{ORT_VERSION}-glibc2_28.zip",
|
||||
f"onnxruntime-linux-x64-static_lib-{ORT_VERSION}-glibc2_28.zip",
|
||||
f"https://github.com/infiniflow/ragflow-build/releases/download/onnxruntime-v{ORT_VERSION}/{_ort_asset_name(ORT_VERSION)}",
|
||||
_ort_asset_name(ORT_VERSION),
|
||||
],
|
||||
]
|
||||
|
||||
@@ -197,7 +223,7 @@ if __name__ == "__main__":
|
||||
("pdfium-linux-x64-static.tgz", "pdfium-static"),
|
||||
("pdf_oxide-go-ffi-linux-amd64.tar.gz", "pdf_oxide"),
|
||||
("office_oxide-linux-x86_64.tar.gz", "office_oxide"),
|
||||
(f"onnxruntime-linux-x64-static_lib-{ORT_VERSION}-glibc2_28.zip", os.path.join("onnxruntime", "static_lib")),
|
||||
(_ort_asset_name(ORT_VERSION), os.path.join("onnxruntime", "static_lib")),
|
||||
]
|
||||
import tarfile
|
||||
import zipfile
|
||||
@@ -209,7 +235,7 @@ if __name__ == "__main__":
|
||||
symbols / wrong version, silently)."""
|
||||
if not os.path.isdir(static_lib_dir):
|
||||
return
|
||||
expected = f"onnxruntime-linux-x64-static_lib-{version}-glibc2_28"
|
||||
expected = _ort_normalized_dir(version)
|
||||
for name in os.listdir(static_lib_dir):
|
||||
if not name.startswith("onnxruntime-linux-x64-static_lib-"):
|
||||
continue
|
||||
@@ -226,8 +252,8 @@ if __name__ == "__main__":
|
||||
continue
|
||||
target = os.path.join(native_deps_dir, subdir)
|
||||
|
||||
# ONNX Runtime ships a version-stamped top-level dir inside the zip
|
||||
# (onnxruntime-linux-x64-static_lib-<ORT_VERSION>-glibc2_28/). A plain
|
||||
# The infiniflow/ragflow-build release zip carries a top-level dir
|
||||
# named onnxruntime-v{ORT_VERSION}-linux-x86_64. A plain
|
||||
# "any .a present?" skip would keep a STALE version in place after a
|
||||
# bump: the new zip downloads, but extraction is skipped because the
|
||||
# old .a is still under static_lib, so the bump silently does nothing.
|
||||
@@ -235,7 +261,7 @@ if __name__ == "__main__":
|
||||
# already extracted.
|
||||
if subdir == os.path.join("onnxruntime", "static_lib"):
|
||||
_prune_stale_onnxruntime(target, ORT_VERSION)
|
||||
version_dir = os.path.join(target, f"onnxruntime-linux-x64-static_lib-{ORT_VERSION}-glibc2_28")
|
||||
version_dir = os.path.join(target, _ort_normalized_dir(ORT_VERSION))
|
||||
if os.path.isdir(version_dir) and any(f.endswith(".a") for _, _, files in os.walk(version_dir) for f in files):
|
||||
print(f" ✓ {subdir} ({ORT_VERSION}) already extracted to {version_dir}")
|
||||
continue
|
||||
@@ -248,6 +274,19 @@ if __name__ == "__main__":
|
||||
if archive_path.endswith(".zip"):
|
||||
with zipfile.ZipFile(archive_path) as zf:
|
||||
zf.extractall(target)
|
||||
# The infiniflow/ragflow-build release zip carries a top-level dir
|
||||
# named onnxruntime-v{version}-linux-x86_64, but build.sh's glob and
|
||||
# the stale checks above all expect
|
||||
# onnxruntime-linux-x64-static_lib-{version}-glibc2_28. Rename it so
|
||||
# every consumer shares one name convention (driven by ORT_VERSION).
|
||||
if subdir == os.path.join("onnxruntime", "static_lib"):
|
||||
extracted = os.path.join(target, _ort_extracted_dir(ORT_VERSION))
|
||||
normalized = os.path.join(target, _ort_normalized_dir(ORT_VERSION))
|
||||
if os.path.isdir(extracted) and extracted != normalized:
|
||||
if os.path.exists(normalized):
|
||||
shutil.rmtree(normalized)
|
||||
print(f" Renaming {os.path.basename(extracted)} → {os.path.basename(normalized)}")
|
||||
os.rename(extracted, normalized)
|
||||
else:
|
||||
with tarfile.open(archive_path) as tf:
|
||||
tf.extractall(target)
|
||||
|
||||
@@ -53,13 +53,35 @@ import zipfile
|
||||
import requests
|
||||
|
||||
# Mirrors internal/common.DeepDocORTVersion (Go in-process backend). ONE OF
|
||||
# THREE places (with that Go constant and the ORT_VERSION in
|
||||
# ragflow_deps/download_deps.py) that must carry the same ONNX Runtime native
|
||||
# release for the statically-linked Go DeepDoc backend. There is no single
|
||||
# source of truth — keep all three equal. The pip onnxruntime== pin
|
||||
# (pyproject.toml) and the onnxruntime_go binding minor (go.mod) must stay on
|
||||
# the same minor line.
|
||||
ORT_VERSION = "1.23.2"
|
||||
# FOUR places (with that Go constant, ORT_VERSION in ragflow_deps/download_deps.py,
|
||||
# and ARG ORT_VERSION in Dockerfile_go) that must carry the same ONNX Runtime
|
||||
# native release for the statically-linked Go DeepDoc backend. There is no
|
||||
# single source of truth — keep all four equal. build.sh --check-ort-version
|
||||
# greps this file (and the other three) to fail fast on drift. (The Python pip
|
||||
# onnxruntime== pin in pyproject.toml is versioned independently and is not
|
||||
# part of this check.)
|
||||
#
|
||||
# Source of the native static archives: infiniflow/ragflow-build (our own
|
||||
# ORT-only minimal build), NOT the third-party csukuangfj/onnxruntime-libs
|
||||
# account. The release tag is `onnxruntime-v{ORT_VERSION}` and the asset is
|
||||
# `onnxruntime-v{ORT_VERSION}-linux-x86_64.zip`.
|
||||
ORT_VERSION = "1.29.0"
|
||||
|
||||
|
||||
def _ort_asset_name(version):
|
||||
"""Release asset filename under infiniflow/ragflow-build tag onnxruntime-v{version}."""
|
||||
return f"onnxruntime-v{version}-linux-x86_64.zip"
|
||||
|
||||
|
||||
def _ort_extracted_dir(version):
|
||||
"""Top-level directory name INSIDE the release zip (what extractall creates)."""
|
||||
return f"onnxruntime-v{version}-linux-x86_64"
|
||||
|
||||
|
||||
def _ort_normalized_dir(version):
|
||||
"""Directory name build.sh's `find ... -name '*.a'` glob expects under static_lib."""
|
||||
return f"onnxruntime-linux-x64-static_lib-{version}-glibc2_28"
|
||||
|
||||
|
||||
# Mirrors internal/common.DeepDocModelFiles (Go in-process DeepDoc backend).
|
||||
# These are the ONLY weights the Go backend loads; the full InfiniFlow/deepdoc
|
||||
@@ -76,7 +98,7 @@ def prune_stale_onnxruntime(static_lib_dir, version):
|
||||
symbols / wrong version, silently)."""
|
||||
if not os.path.isdir(static_lib_dir):
|
||||
return
|
||||
expected = f"onnxruntime-linux-x64-static_lib-{version}-glibc2_28"
|
||||
expected = _ort_normalized_dir(version)
|
||||
for name in os.listdir(static_lib_dir):
|
||||
if not name.startswith("onnxruntime-linux-x64-static_lib-"):
|
||||
continue
|
||||
@@ -97,8 +119,8 @@ def extract_onnxruntime(static_lib_dir, archive_path, version):
|
||||
`static_lib_dir`. Returns True when that version is available afterwards
|
||||
(extracted now or already present), False when the archive is missing.
|
||||
|
||||
ORT ships a version-stamped top-level dir inside the zip
|
||||
(onnxruntime-linux-x64-static_lib-<version>-glibc2_28/), so a present
|
||||
The infiniflow/ragflow-build release zip carries a top-level dir named
|
||||
onnxruntime-v{version}-linux-x86_64, so a present
|
||||
`static_lib_dir` is NOT evidence that THIS version is extracted: after a
|
||||
version bump the stale dir is pruned and the new one must be extracted.
|
||||
"""
|
||||
@@ -106,7 +128,7 @@ def extract_onnxruntime(static_lib_dir, archive_path, version):
|
||||
print(f" Skipping extraction: {os.path.basename(archive_path)} not found")
|
||||
return False
|
||||
prune_stale_onnxruntime(static_lib_dir, version)
|
||||
version_dir = os.path.join(static_lib_dir, f"onnxruntime-linux-x64-static_lib-{version}-glibc2_28")
|
||||
version_dir = os.path.join(static_lib_dir, _ort_normalized_dir(version))
|
||||
if os.path.isdir(version_dir) and has_static_archives(version_dir):
|
||||
print(f" ✓ onnxruntime/static_lib ({version}) already extracted to {version_dir}")
|
||||
return True
|
||||
@@ -114,6 +136,18 @@ def extract_onnxruntime(static_lib_dir, archive_path, version):
|
||||
print(f" Extracting {os.path.basename(archive_path)} → {static_lib_dir}")
|
||||
with zipfile.ZipFile(archive_path) as zf:
|
||||
zf.extractall(static_lib_dir)
|
||||
# The infiniflow/ragflow-build release zip carries a top-level dir named
|
||||
# onnxruntime-v{version}-linux-x86_64, but build.sh's glob and the stale
|
||||
# checks above all expect onnxruntime-linux-x64-static_lib-{version}-glibc2_28.
|
||||
# Rename it so every consumer shares one name convention (driven by
|
||||
# ORT_VERSION).
|
||||
extracted = os.path.join(static_lib_dir, _ort_extracted_dir(version))
|
||||
normalized = os.path.join(static_lib_dir, _ort_normalized_dir(version))
|
||||
if os.path.isdir(extracted) and extracted != normalized:
|
||||
if os.path.exists(normalized):
|
||||
shutil.rmtree(normalized)
|
||||
print(f" Renaming {os.path.basename(extracted)} → {os.path.basename(normalized)}")
|
||||
os.rename(extracted, normalized)
|
||||
return True
|
||||
|
||||
|
||||
@@ -143,8 +177,8 @@ def get_urls(use_china_mirrors=False) -> list[str | list[str]]:
|
||||
["https://gh-proxy.com/https://github.com/yfedoseev/pdf_oxide/releases/download/v0.3.73/pdf_oxide-go-ffi-linux-amd64.tar.gz", "pdf_oxide-go-ffi-linux-amd64.tar.gz"],
|
||||
["https://gh-proxy.com/https://github.com/yfedoseev/office_oxide/releases/download/v0.1.9/native-linux-x86_64.tar.gz", "office_oxide-linux-x86_64.tar.gz"],
|
||||
[
|
||||
f"https://gh-proxy.com/https://github.com/csukuangfj/onnxruntime-libs/releases/download/v{ORT_VERSION}/onnxruntime-linux-x64-static_lib-{ORT_VERSION}-glibc2_28.zip",
|
||||
f"onnxruntime-linux-x64-static_lib-{ORT_VERSION}-glibc2_28.zip",
|
||||
f"https://gh-proxy.com/https://github.com/infiniflow/ragflow-build/releases/download/onnxruntime-v{ORT_VERSION}/{_ort_asset_name(ORT_VERSION)}",
|
||||
_ort_asset_name(ORT_VERSION),
|
||||
],
|
||||
]
|
||||
else:
|
||||
@@ -172,8 +206,8 @@ def get_urls(use_china_mirrors=False) -> list[str | list[str]]:
|
||||
["https://github.com/yfedoseev/pdf_oxide/releases/download/v0.3.73/pdf_oxide-go-ffi-linux-amd64.tar.gz", "pdf_oxide-go-ffi-linux-amd64.tar.gz"],
|
||||
["https://github.com/yfedoseev/office_oxide/releases/download/v0.1.9/native-linux-x86_64.tar.gz", "office_oxide-linux-x86_64.tar.gz"],
|
||||
[
|
||||
f"https://github.com/csukuangfj/onnxruntime-libs/releases/download/v{ORT_VERSION}/onnxruntime-linux-x64-static_lib-{ORT_VERSION}-glibc2_28.zip",
|
||||
f"onnxruntime-linux-x64-static_lib-{ORT_VERSION}-glibc2_28.zip",
|
||||
f"https://github.com/infiniflow/ragflow-build/releases/download/onnxruntime-v{ORT_VERSION}/{_ort_asset_name(ORT_VERSION)}",
|
||||
_ort_asset_name(ORT_VERSION),
|
||||
],
|
||||
]
|
||||
|
||||
@@ -312,7 +346,7 @@ if __name__ == "__main__":
|
||||
|
||||
if not extract_onnxruntime(
|
||||
os.path.join(native_deps_dir, "onnxruntime", "static_lib"),
|
||||
os.path.join(os.getcwd(), f"onnxruntime-linux-x64-static_lib-{ORT_VERSION}-glibc2_28.zip"),
|
||||
os.path.join(os.getcwd(), _ort_asset_name(ORT_VERSION)),
|
||||
ORT_VERSION,
|
||||
):
|
||||
# The archive was not downloaded or failed to extract, so no .a landed.
|
||||
|
||||
@@ -4,17 +4,30 @@
|
||||
# tests must guarantee the archive is really landed on disk — above all after an
|
||||
# ORT version bump, where the zip's version-stamped top-level dir changes but
|
||||
# static_lib/ still exists from the previous run.
|
||||
#
|
||||
# The release zip from infiniflow/ragflow-build carries a top-level dir named
|
||||
# onnxruntime-v{version}-linux-x86_64; extract_onnxruntime() must rename it to
|
||||
# the build.sh-expected onnxruntime-linux-x64-static_lib-{version}-glibc2_28 so
|
||||
# every consumer shares one name convention. These tests pin that rename.
|
||||
|
||||
import os
|
||||
import zipfile
|
||||
|
||||
from download_go_deps import extract_onnxruntime, has_static_archives
|
||||
from download_go_deps import (
|
||||
_ort_asset_name,
|
||||
_ort_extracted_dir,
|
||||
_ort_normalized_dir,
|
||||
extract_onnxruntime,
|
||||
has_static_archives,
|
||||
)
|
||||
|
||||
|
||||
def make_ort_zip(path, version):
|
||||
"""Build a zip shaped like the upstream ORT release: a version-stamped
|
||||
top-level dir holding lib/libonnxruntime.a."""
|
||||
member = f"onnxruntime-linux-x64-static_lib-{version}-glibc2_28/lib/libonnxruntime.a"
|
||||
"""Build a zip shaped like the infiniflow/ragflow-build ORT release: a
|
||||
top-level dir named onnxruntime-v{version}-linux-x86_64 holding
|
||||
lib/libonnxruntime.a. extract_onnxruntime() must rename it to the
|
||||
build.sh-expected onnxruntime-linux-x64-static_lib-{version}-glibc2_28."""
|
||||
member = f"{_ort_extracted_dir(version)}/lib/libonnxruntime.a"
|
||||
with zipfile.ZipFile(path, "w") as zf:
|
||||
zf.writestr(member, b"!<arch>\nort-payload")
|
||||
return path
|
||||
@@ -23,12 +36,14 @@ def make_ort_zip(path, version):
|
||||
def test_extracts_on_first_run(tmp_path):
|
||||
# static_lib/ does not exist yet: the plain first-run path.
|
||||
static_lib = tmp_path / "onnxruntime" / "static_lib"
|
||||
version = "1.23.2"
|
||||
archive = make_ort_zip(tmp_path / f"onnxruntime-linux-x64-static_lib-{version}-glibc2_28.zip", version)
|
||||
version = "1.29.0"
|
||||
archive = make_ort_zip(tmp_path / _ort_asset_name(version), version)
|
||||
|
||||
assert extract_onnxruntime(str(static_lib), str(archive), version) is True
|
||||
|
||||
version_dir = static_lib / f"onnxruntime-linux-x64-static_lib-{version}-glibc2_28"
|
||||
extracted = static_lib / _ort_extracted_dir(version)
|
||||
assert not extracted.exists(), "release zip top-level dir must be renamed away"
|
||||
version_dir = static_lib / _ort_normalized_dir(version)
|
||||
assert version_dir.is_dir()
|
||||
assert has_static_archives(str(version_dir))
|
||||
|
||||
@@ -40,30 +55,32 @@ def test_extracts_after_version_bump_when_static_lib_exists(tmp_path):
|
||||
build.sh's ORT guard rejects the build."""
|
||||
static_lib = tmp_path / "onnxruntime" / "static_lib"
|
||||
static_lib.mkdir(parents=True)
|
||||
stale = static_lib / "onnxruntime-linux-x64-static_lib-1.23.1-glibc2_28"
|
||||
stale = static_lib / _ort_normalized_dir("1.28.0")
|
||||
(stale / "lib").mkdir(parents=True)
|
||||
(stale / "lib" / "libonnxruntime.a").write_bytes(b"old-ort")
|
||||
|
||||
version = "1.23.2"
|
||||
archive = make_ort_zip(tmp_path / f"onnxruntime-linux-x64-static_lib-{version}-glibc2_28.zip", version)
|
||||
version = "1.29.0"
|
||||
archive = make_ort_zip(tmp_path / _ort_asset_name(version), version)
|
||||
|
||||
assert extract_onnxruntime(str(static_lib), str(archive), version) is True
|
||||
|
||||
assert not stale.exists(), "stale ORT version should be pruned"
|
||||
version_dir = static_lib / f"onnxruntime-linux-x64-static_lib-{version}-glibc2_28"
|
||||
extracted = static_lib / _ort_extracted_dir(version)
|
||||
assert not extracted.exists(), "release zip top-level dir must be renamed away"
|
||||
version_dir = static_lib / _ort_normalized_dir(version)
|
||||
assert version_dir.is_dir(), "bumped ORT version was silently not extracted"
|
||||
assert has_static_archives(str(version_dir)), "bumped ORT version landed no .a"
|
||||
|
||||
|
||||
def test_skips_when_archive_missing(tmp_path):
|
||||
static_lib = tmp_path / "onnxruntime" / "static_lib"
|
||||
assert extract_onnxruntime(str(static_lib), str(tmp_path / "missing.zip"), "1.23.2") is False
|
||||
assert extract_onnxruntime(str(static_lib), str(tmp_path / "missing.zip"), "1.29.0") is False
|
||||
|
||||
|
||||
def test_idempotent_when_matching_version_already_present(tmp_path):
|
||||
static_lib = tmp_path / "onnxruntime" / "static_lib"
|
||||
version = "1.23.2"
|
||||
archive = make_ort_zip(tmp_path / f"onnxruntime-linux-x64-static_lib-{version}-glibc2_28.zip", version)
|
||||
version = "1.29.0"
|
||||
archive = make_ort_zip(tmp_path / _ort_asset_name(version), version)
|
||||
|
||||
assert extract_onnxruntime(str(static_lib), str(archive), version) is True
|
||||
before = sorted(os.listdir(static_lib))
|
||||
|
||||
Reference in New Issue
Block a user