diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 17f2e543ef..c79c0012a0 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -272,6 +272,19 @@ jobs: fetch-depth: 0 fetch-tags: true + - name: Build server binary (with native libs / ORT) + # The production Dockerfile does `COPY bin bin`, so the binary must be + # built (and ORT statically linked from the runner image at + # /opt/ragflow-native-libs) before the docker build. This makes ORT's + # presence in the released image an explicit, visible precondition + # instead of relying on a pre-populated bin/ from elsewhere. + env: + CGO_ENABLED: '1' + run: | + set -euo pipefail + ./build.sh --cpp + ./build.sh --go + - name: Build and push image run: | sudo docker login --username infiniflow --password-stdin <<< ${{ secrets.DOCKERHUB_TOKEN }} diff --git a/.github/workflows/sep-tests.yml b/.github/workflows/sep-tests.yml index fbe1a74c9e..372273f5e7 100644 --- a/.github/workflows/sep-tests.yml +++ b/.github/workflows/sep-tests.yml @@ -382,6 +382,7 @@ jobs: ./build.sh --test -- $PKGS fi + - name: Build ragflow:nightly run: | set -euo pipefail @@ -1001,6 +1002,7 @@ jobs: ./build.sh --test -- $PKGS fi + - name: Build ragflow:nightly run: | set -euo pipefail @@ -1488,3 +1490,85 @@ jobs: sudo docker network rm "${guard}" >/dev/null 2>&1 || true done fi + + ragflow_native_backend: + name: ragflow_native_backend + # Runs AFTER both per-engine go jobs (which contain the doc-engine-specific + # server build + Python RESTFUL correctness runs). The DeepDoc native + # backend is doc-engine-independent, so running it ONCE here — instead of + # duplicated (and memory-competing) inside each go job — removes the + # redundant second -race pass and the concurrent -race OOM risk. Running it + # last also keeps a flaky -race OOM from ever obscuring the functional + # verdict produced by the go jobs. + needs: [ragflow_tests_infinity, ragflow_tests_elasticsearch] + if: ${{ github.event_name != 'pull_request' && github.event_name != 'pull_request_target' || (github.event.pull_request.draft == false && contains(github.event.pull_request.labels.*.name, 'ci') && (github.event.action != 'labeled' || github.event.label.name == 'ci')) }} + runs-on: [ "self-hosted", "ragflow-test" ] + steps: + - name: Ensure workspace ownership + run: | + echo "Workflow triggered by ${{ github.event_name }}" + echo "chown -R ${USER} ${GITHUB_WORKSPACE}" && sudo chown -R ${USER} ${GITHUB_WORKSPACE} + + - name: Check out code + uses: actions/checkout@v6 + with: + ref: ${{ (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && format('refs/pull/{0}/merge', github.event.pull_request.number) || github.sha }} + fetch-depth: 0 + fetch-tags: true + allow-unsafe-pr-checkout: true + + - name: Build C++ static library (ragflow native backend) + # Produces librag_tokenizer_c_api.a with the re2 symbols renamed into + # the ragtokre2_ namespace (so it no longer collides with onnxruntime's + # bundled re2 copy at link time). Only the .a is needed for the native + # integration tests; the per-engine go jobs build the full server. + run: | + set -euo pipefail + BUILDER_CONTAINER=ragflow_build_${GITHUB_RUN_ID}_native_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ') + cleanup_builder() { + if [[ -n "${BUILDER_CONTAINER:-}" ]]; then + sudo docker rm -f -v "${BUILDER_CONTAINER}" >/dev/null 2>&1 || true + fi + } + trap cleanup_builder EXIT + + TZ=${TZ:-$(readlink -f /etc/localtime | awk -F '/zoneinfo/' '{print $2}')} + sudo docker run --privileged -d --name "${BUILDER_CONTAINER}" \ + -e TZ="${TZ}" \ + -e UV_INDEX=https://mirrors.aliyun.com/pypi/simple \ + -v "${PWD}:/ragflow" \ + -v "${PWD}/internal/binding/cpp/resource:/usr/share/infinity/resource" \ + infiniflow/infinity_builder:ubuntu22_clang20 + sudo docker exec "${BUILDER_CONTAINER}" bash -c 'git config --global safe.directory "*" && cd /ragflow && ./build.sh --cpp' + + - name: Cache external DeepDoc testdata + # The native backend tests fetch pinned test assets from + # xugangqiang/ragflow-testdata (see scripts/fetch_deepdoc_testdata.sh). + # Cache them across runs, keyed by the pinned ref recorded in + # internal/deepdoc/native/testdata.ref so a ref bump invalidates the cache. + uses: actions/cache@v4 + with: + path: ${{ github.workspace }}/.cache/ragflow-testdata + key: deepdoc-testdata-${{ hashFiles('internal/deepdoc/native/testdata.ref') }} + + - name: Run in-process DeepDoc backend integration tests (correctness + race) + # Doc-engine-independent: the same native backend (det/layout/tsr/rec + # ONNX models + onnxruntime) is exercised regardless of DOC_ENGINE, so + # it is run ONCE here instead of duplicated inside each go job. + # + # Inside build.sh, the golden/comparison tests run WITHOUT -race (no + # race-detection value, ~10x memory tax) and only the concurrency + # correctness tests run WITH -race (see run_native_integration_tests). + # + # XDG_CACHE_HOME is pinned to a workspace path so the test fetch script + # and this job's actions/cache step agree on a single, deterministic + # location (the self-hosted runner's $HOME is not stable across steps). + env: + MODEL_DIR: /opt/ragflow-deepdoc-models + CGO_ENABLED: '1' + DEEPDOC_NATIVE_REQUIRED: '1' + XDG_CACHE_HOME: ${{ github.workspace }}/.cache + run: | + set -euo pipefail + ./build.sh --test-native + diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index cdc9c1ac66..c7ca472306 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1098,3 +1098,84 @@ jobs: if [[ -n ${PORT_RESERVATION:-} ]]; then rm -f "${PORT_RESERVATION}" fi + + ragflow_native_backend: + name: ragflow_native_backend + # Runs AFTER both per-engine go jobs (which contain the doc-engine-specific + # server build + Python RESTFUL correctness runs). The DeepDoc native + # backend is doc-engine-independent, so running it ONCE here — instead of + # duplicated (and memory-competing) inside each go job — removes the + # redundant second -race pass and the concurrent -race OOM risk. Running it + # last also keeps a flaky -race OOM from ever obscuring the functional + # verdict produced by the go jobs. + needs: [ragflow_tests_infinity, ragflow_tests_elasticsearch] + if: ${{ github.event_name != 'pull_request' || (github.event.pull_request.draft == false && contains(github.event.pull_request.labels.*.name, 'ci') && (github.event.action != 'labeled' || github.event.label.name == 'ci')) }} + runs-on: [ "self-hosted", "ragflow-test" ] + steps: + - name: Ensure workspace ownership + run: | + echo "Workflow triggered by ${{ github.event_name }}" + echo "chown -R ${USER} ${GITHUB_WORKSPACE}" && sudo chown -R ${USER} ${GITHUB_WORKSPACE} + + - name: Check out code + uses: actions/checkout@v6 + with: + ref: ${{ (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && format('refs/pull/{0}/merge', github.event.pull_request.number) || github.sha }} + fetch-depth: 0 + fetch-tags: true + + - name: Build C++ static library (ragflow native backend) + # Produces librag_tokenizer_c_api.a with the re2 symbols renamed into + # the ragtokre2_ namespace (so it no longer collides with onnxruntime's + # bundled re2 copy at link time). Only the .a is needed for the native + # integration tests; the per-engine go jobs build the full server. + run: | + set -euo pipefail + BUILDER_CONTAINER=ragflow_build_${GITHUB_RUN_ID}_native_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ') + cleanup_builder() { + if [[ -n "${BUILDER_CONTAINER:-}" ]]; then + sudo docker rm -f -v "${BUILDER_CONTAINER}" >/dev/null 2>&1 || true + fi + } + trap cleanup_builder EXIT + + TZ=${TZ:-$(readlink -f /etc/localtime | awk -F '/zoneinfo/' '{print $2}')} + sudo docker run --privileged -d --name "${BUILDER_CONTAINER}" \ + -e TZ="${TZ}" \ + -e UV_INDEX=https://mirrors.aliyun.com/pypi/simple \ + -v "${PWD}:/ragflow" \ + -v "${PWD}/internal/binding/cpp/resource:/usr/share/infinity/resource" \ + infiniflow/infinity_builder:ubuntu22_clang20 + sudo docker exec "${BUILDER_CONTAINER}" bash -c 'git config --global safe.directory "*" && cd /ragflow && ./build.sh --cpp' + + - name: Cache external DeepDoc testdata + # The native backend tests fetch pinned test assets from + # xugangqiang/ragflow-testdata (see scripts/fetch_deepdoc_testdata.sh). + # Cache them across runs, keyed by the pinned ref recorded in + # internal/deepdoc/native/testdata.ref so a ref bump invalidates the cache. + uses: actions/cache@v4 + with: + path: ${{ github.workspace }}/.cache/ragflow-testdata + key: deepdoc-testdata-${{ hashFiles('internal/deepdoc/native/testdata.ref') }} + + - name: Run in-process DeepDoc backend integration tests (correctness + race) + # Doc-engine-independent: the same native backend (det/layout/tsr/rec + # ONNX models + onnxruntime) is exercised regardless of DOC_ENGINE, so + # it is run ONCE here instead of duplicated inside each go job. + # + # Inside build.sh, the golden/comparison tests run WITHOUT -race (no + # race-detection value, ~10x memory tax) and only the concurrency + # correctness tests run WITH -race (see run_native_integration_tests). + # + # XDG_CACHE_HOME is pinned to a workspace path so the test fetch script + # and this job's actions/cache step agree on a single, deterministic + # location (the self-hosted runner's $HOME is not stable across steps). + env: + MODEL_DIR: /opt/ragflow-deepdoc-models + CGO_ENABLED: '1' + DEEPDOC_NATIVE_REQUIRED: '1' + XDG_CACHE_HOME: ${{ github.workspace }}/.cache + run: | + set -euo pipefail + ./build.sh --test-native + diff --git a/.gitignore b/.gitignore index 521b9b9fa6..77c6acb579 100644 --- a/.gitignore +++ b/.gitignore @@ -267,3 +267,10 @@ internal/deepdoc/parser/docx/tool/ internal/ingestion/task/tool/generate_dataflow_golden.py internal/ingestion/task/tool/README.md internal/cpp/cmake-build-release + +# downloaded native static-lib artifacts (fetched by ragflow_deps/download_deps.py) +ragflow_deps/*.zip + +# external DeepDoc testdata, fetched on demand via scripts/fetch_deepdoc_testdata.sh +# (symlinked into place from ~/.cache/ragflow-testdata; never committed) +internal/deepdoc/native/testdata diff --git a/build.sh b/build.sh index 18a873d6b6..7183fd2056 100755 --- a/build.sh +++ b/build.sh @@ -37,6 +37,13 @@ PDFIUM_STATIC_VERSION="7809" PDF_OXIDE_PREFIX="${HOME}/ragflow-native-libs/pdf_oxide" PDF_OXIDE_VERSION="0.3.67" +# onnxruntime native library settings — static linking for the in-process +# (Go) DeepDoc backend. libonnxruntime*.a is linked into the server binary +# (--whole-archive + --export-dynamic); OrtGetApiBase is then resolved via +# dlopen(self), so no libonnxruntime.so is needed at runtime. Downloaded by +# ragflow_deps/download_deps.py into onnxruntime/static_lib. +ONNXRUNTIME_STATIC_PREFIX="${HOME}/ragflow-native-libs/onnxruntime/static_lib" + # Copy a dependency from the system pre-seed directory to the user cache. # Returns 0 if the dep was copied or already exists in cache, 1 otherwise. _seed_from_system() { @@ -253,6 +260,34 @@ build_cpp() { exit 1 fi + # Rename the tokenizer's bundled re2 symbols into a private namespace so they + # never collide with onnxruntime's copy of re2 at final link time. + # + # Why this is needed: onnxruntime.a and librag_tokenizer_c_api.a both embed a + # copy of re2, but they are built against different toolchains/libstdc++ and + # are ABI-incompatible. Symbol-localization approaches (--exclude-libs, linker + # version scripts) cannot fix it: both re2 copies must live in the same global + # symbol table, and the linker resolves the tokenizer's re2 references to + # whichever copy was archived first (here, onnxruntime's), regardless of which + # copy is localized. SIGSEGV in re2::DFA::InlinedSearchLoop is the symptom. + # + # Renaming the tokenizer's re2 symbols into a private prefix gives each copy + # its own namespace, so the tokenizer calls its own ABI-compatible re2 and + # onnxruntime keeps calling its own. Verified to eliminate the crash. + local tok_a="$BUILD_DIR/librag_tokenizer_c_api.a" + local rename_map="$BUILD_DIR/re2_rename.map" + if command -v objcopy >/dev/null 2>&1; then + nm "$tok_a" \ + | awk '$2 ~ /^[TDBRWtdbrwiIVv]$/ && $3 ~ /^_ZN3re2|_ZNK3re2|_ZTVN3re2|_ZTIN3re2|_ZTSN3re2/ { print $3" ragtokre2_"$3 }' \ + | sort -u > "$rename_map" + if [ -s "$rename_map" ]; then + objcopy --redefine-syms="$rename_map" "$tok_a" + echo -e "${GREEN}✓ Renamed $(wc -l < "$rename_map") tokenizer re2 symbols into private namespace (ragtokre2_)${NC}" + fi + else + echo -e "${YELLOW}Warning: objcopy not found, skipping re2 symbol rename (re2 collision with onnxruntime may cause SIGSEGV)${NC}" + fi + echo -e "${GREEN}✓ C++ static library built successfully${NC}" } @@ -322,7 +357,8 @@ build_go() { GOPROXY=${GOPROXY:-https://goproxy.cn,https://proxy.golang.org,direct} CGO_ENABLED=1 \ CGO_CFLAGS="$CGO_CFLAGS" CGO_LDFLAGS="$CGO_LDFLAGS" \ - go build "${strip_flags[@]}" -o "$RAGFLOW_SERVER_BINARY" cmd/ragflow_server.go + go build -tags cgo "${strip_flags[@]}" -o "$RAGFLOW_SERVER_BINARY" \ + cmd/ragflow_server.go if [ ! -f "$RAGFLOW_SERVER_BINARY" ]; then @@ -364,7 +400,8 @@ setup_cgo_env() { # cannot merge. Use lld (LLVM linker) which handles them correctly. # --allow-multiple-definition: pdf_oxide and office_oxide are both Rust # staticlibs that embed the Rust runtime; linking them together produces - # duplicate rust_eh_personality symbols. + # duplicate rust_eh_personality / compiler-rt builtins. Those duplicates are + # ABI-IDENTICAL, so merging them is benign. if [ "$(uname -s)" = "Linux" ]; then if ! command -v ld.lld >/dev/null 2>&1; then echo -e "${RED}Error: ld.lld not found. Install with: sudo apt install lld-20 && sudo ln -s /usr/bin/ld.lld-20 /usr/bin/ld.lld${NC}" @@ -375,6 +412,11 @@ setup_cgo_env() { ${PDFIUM_STATIC_PREFIX}/lib/libc++.a \ ${PDFIUM_STATIC_PREFIX}/lib/libc++abi.a \ -fuse-ld=lld -Wl,--allow-multiple-definition" + # The re2 regex-library collision between onnxruntime.a and + # librag_tokenizer_c_api.a is fixed at the .a level in build_cpp(): + # the tokenizer's bundled re2 symbols are renamed into a private + # namespace (ragtokre2_) so the two re2 copies never share a symbol. + # See build_cpp() for details. fi # ── pdf_oxide ───────────────────────────────────────────────────── @@ -399,6 +441,80 @@ setup_cgo_env() { esac export CGO_LDFLAGS="$CGO_LDFLAGS ${PDF_OXIDE_PREFIX}/lib/${pdf_oxide_subdir}/libpdf_oxide.a" + # ── onnxruntime (static, resolved via dlopen(NULL)) ──────────────── + # macOS native builds of the in-process DeepDoc backend are not supported: + # ONNX Runtime is statically linked with GNU ld flags (--whole-archive / + # --export-dynamic) and resolved at runtime via dlopen(NULL); Apple's ld64 + # does not understand these flags. Build on Linux or cross-compile there. + case "$(uname -s)" in + Darwin) + echo "Error: macOS native build of the in-process DeepDoc backend is not supported." >&2 + echo " ONNX Runtime is linked with GNU ld flags (--whole-archive / --export-dynamic)" >&2 + echo " and resolved via dlopen(NULL); Apple's ld64 does not support them. Build on Linux." >&2 + return 1 + ;; + esac + # Statically link libonnxruntime*.a into the binary. The forked Go binding + # (onnxruntime_go, github.com/xugangqiang/onnxruntime_go) resolves + # OrtGetApiBase with dlopen(NULL), so the symbols must (a) be pulled in + # wholesale with --whole-archive (ORT registers its execution providers + # lazily at runtime, beyond what a normal link would keep) and (b) be + # exported with --export-dynamic so the process-global symbol table + # dlopen finds them. No libonnxruntime.so is required or supported at + # runtime; there is no dynamic .so fallback. + # + # Seed the static ORT archives from the system pre-bake (/opt, laid down + # by the CI runner image) into the user cache before the link check + # below. Mirrors the existing _seed_from_system calls for the other + # native libs so CI never downloads ORT at build/test time. + _seed_from_system "onnxruntime" || true + if [ -d "$ONNXRUNTIME_STATIC_PREFIX" ]; then + # Collect every .a, but skip GPU-only providers we never build + # against (would pull in CUDA/cuDNN/TensorRT which we don't ship). + local ort_a="" + local seen_version_dir="" + while IFS= read -r f; do + case "$(basename "$f")" in + *cuda*|*tensorrt*|*coreml*|*dml*|*migraphx*) continue ;; + esac + # Guard against coexisting stale version dirs: if .a files span + # more than one onnxruntime-linux-x64-static_lib-* dir, fail fast + # instead of silently linking two ORT versions (duplicate symbols + # / wrong version). Re-run `download_deps.py` to prune stale dirs + # after a version bump, or remove the old dir by hand. + case "$f" in + */onnxruntime-linux-x64-static_lib-*/lib/*.a) + local vdir="${f#*/onnxruntime-linux-x64-static_lib-}" + vdir="${vdir%%/*}" + if [ -z "$seen_version_dir" ]; then + seen_version_dir="$vdir" + elif [ "$seen_version_dir" != "$vdir" ]; then + echo " Error: multiple ONNX Runtime versions found under $ONNXRUNTIME_STATIC_PREFIX" >&2 + echo " $seen_version_dir AND $vdir" >&2 + echo " Remove the stale version dir (or re-run download_deps.py to prune it)." >&2 + return 1 + fi + ;; + esac + ort_a="$ort_a $f" + done < <(find "$ONNXRUNTIME_STATIC_PREFIX" -type f -name '*.a' 2>/dev/null) + + if [ -n "$ort_a" ]; then + export CGO_LDFLAGS="$CGO_LDFLAGS -Wl,--export-dynamic -Wl,--whole-archive$ort_a -Wl,--no-whole-archive -lstdc++" + echo " onnxruntime (static) → $ONNXRUNTIME_STATIC_PREFIX" + # The re2 regex-library collision between onnxruntime.a and + # librag_tokenizer_c_api.a is fixed at the .a level in build_cpp(): + # the tokenizer's bundled re2 symbols are renamed into a private + # namespace (ragtokre2_) so the two re2 copies never share a symbol + # name. --export-dynamic is required because OrtGetApiBase is + # resolved via dlopen(NULL) at runtime (see the block comment above). + else + echo " onnxruntime static_lib dir has no .a files; the in-process DeepDoc backend cannot link ORT" >&2 + fi + else + echo " onnxruntime static_lib not found ($ONNXRUNTIME_STATIC_PREFIX); the in-process DeepDoc backend cannot link ORT" >&2 + fi + # ── platform-specific system libraries ──────────────────────────── case "$(uname -s)" in Linux) @@ -430,6 +546,67 @@ run_go_tests() { GOPROXY=${GOPROXY:-https://goproxy.cn,https://proxy.golang.org,direct} CGO_ENABLED=1 \ CGO_CFLAGS="$CGO_CFLAGS" CGO_LDFLAGS="$CGO_LDFLAGS" \ go test -count=1 "$@" + + run_native_tests +} + +# Run the unit tests of the native package (DeepDoc det/DLA/TSR/OCR-rec Go +# ports), now a regular package under the MAIN module gated by the `cgo` build +# tag (same isolation as office_oxide/pdfium). It used to be a nested Go module +# (own go.mod) that the root `./...` never descended into; after the merge it is +# covered by the normal module tests. The build is pure-Go geometry plus the +# cgo onnxruntime binding, so it runs whenever CGO_ENABLED=1 (the same gate the +# rest of the native C libs use). Model-backed integration tests are handled by +# run_native_integration_tests. +run_native_tests() { + print_section "Running native unit tests" + ( cd "$PROJECT_ROOT" && \ + GOPROXY=${GOPROXY:-https://goproxy.cn,https://proxy.golang.org,direct} \ + CGO_ENABLED=1 \ + go test -tags cgo -count=1 ./internal/deepdoc/native/... ) +} + +# Run the model-backed integration tests of the native package and the +# native_analyzer package (the DocAnalyzer the PDF parser consumes). These +# require MODEL_DIR at runtime; ONNX Runtime is statically linked and resolved +# via dlopen(NULL), so no ORT_LIB is needed. The tests self-skip when MODEL_DIR +# is unset or the binary lacks static ORT (see native_integration_test.go +# skipIfNoModels / native_analyzer_test.go analyzerWithModels). Run under the +# `cgo integration` tier. +# +# Why -race is split instead of applied to the whole suite: +# - The DLA/TSR/OCR-rec/Det golden-comparison tests are single-threaded and +# deterministic. -race multiplies their (model-heavy) heap ~10x for ZERO +# race-detection value, and that is what used to OOM-kill the runner +# (SIGTERM) under `cgo integration`. So they run WITHOUT -race. +# - Only the concurrency-correctness tests (TestInferenceConcurrency* in the +# native package, and native_analyzer's TestAnalyzerConcurrentBatchAndSingle) +# actually benefit from -race: they hammer the shared session pools from many +# goroutines, and the detector catches data races on the pools / per-session +# in-out tensors / cross-call batch state. Those run WITH -race. +# ORT's C internals are not instrumented, but all our shared mutable state lives +# in Go, which the detector covers. CGO_ENABLED=1 is required for both the build +# and the race runtime. +run_native_integration_tests() { + print_section "Running native integration tests (golden/comparison, no race)" + ( cd "$PROJECT_ROOT" && \ + GOPROXY=${GOPROXY:-https://goproxy.cn,https://proxy.golang.org,direct} \ + CGO_ENABLED=1 \ + go test -tags "cgo integration fetch_testdata" -count=1 ./internal/deepdoc/native/... ) + + print_section "Running native integration concurrency tests (race detector on)" + ( cd "$PROJECT_ROOT" && \ + GOPROXY=${GOPROXY:-https://goproxy.cn,https://proxy.golang.org,direct} \ + CGO_ENABLED=1 \ + go test -tags "cgo integration fetch_testdata" -race -count=1 \ + -run 'TestInferenceConcurrency' ./internal/deepdoc/native/... ) + + print_section "Running native_analyzer race tests (race detector on)" + ( cd "$PROJECT_ROOT" && \ + GOPROXY=${GOPROXY:-https://goproxy.cn,https://proxy.golang.org,direct} \ + CGO_ENABLED=1 \ + go test -tags "cgo integration fetch_testdata" -race -count=1 \ + ./internal/deepdoc/parser/pdf/inference/native_analyzer/... ) } # Run Go tests gated behind a build tag (or space-separated tag list), e.g. @@ -522,6 +699,10 @@ OPTIONS: --test-manual Run Go tests tagged 'manual' (very slow; local opt-in ONLY, never run in CI). --test-all Run 'integration' + 'e2e' tests (excludes 'manual'). + --test-native Run the in-process (Go) DeepDoc backend tests tagged + 'cgo integration' (needs libonnxruntime + the + InfiniFlow/deepdoc model snapshot; self-skip otherwise). + e.g. `$0 --test-native` --clean, -C Clean all build artifacts --run, -r Build and run the server --strip, -s Strip debug symbols from Go binaries (-ldflags="-s -w") @@ -594,6 +775,7 @@ main() { else run_go_tests_tagged integration "${args[@]:1}" fi + run_native_integration_tests ;; --test-e2e) check_go_deps @@ -619,6 +801,19 @@ main() { run_go_tests_tagged "integration e2e" "${args[@]:1}" fi ;; + --test-native) + check_go_deps + if [ "${args[1]:-}" = "--" ]; then + pkgs=("${args[@]:2}") + else + pkgs=("${args[@]:1}") + fi + if [ "${#pkgs[@]}" -eq 0 ]; then + pkgs=(./internal/deepdoc/parser/pdf/inference/native_analyzer/...) + fi + run_go_tests_tagged "cgo integration" "${pkgs[@]}" + run_native_integration_tests + ;; --clean|-C) clean ;; diff --git a/cmd/ragflow_server.go b/cmd/ragflow_server.go index aeb75bb0b1..01e316c5e1 100644 --- a/cmd/ragflow_server.go +++ b/cmd/ragflow_server.go @@ -23,6 +23,7 @@ import ( "net/http" "os" "os/signal" + "path/filepath" "ragflow/internal/admin" "ragflow/internal/agent/audio" "ragflow/internal/agent/canvas" @@ -57,10 +58,12 @@ import ( "ragflow/internal/agent/component" "ragflow/internal/common" "ragflow/internal/dao" + "ragflow/internal/deepdoc/parser/pdf/inference/native_analyzer" "ragflow/internal/engine" "ragflow/internal/engine/redis" _ "ragflow/internal/ingestion/wire" "ragflow/internal/server" + "ragflow/internal/servermode" "ragflow/internal/utility" ) @@ -157,6 +160,11 @@ func parseArgs() (*serverArgs, error) { return args, nil } +// registerNativeDeepDoc wires the in-process (Go) DeepDoc backend as the local +// fallback used when no external DeepDoc HTTP service is configured. It is +// compiled into the server built with -tags cgo, which statically links the +// ONNX Runtime backend (libonnxruntime.a); the unit-test tier builds without +// cgo and stays free of the onnxruntime dependency. func printHelp(args *serverArgs) { switch { case args.mode == nil: @@ -275,6 +283,19 @@ func main() { os.Exit(1) } + // Register the in-process (Go) DeepDoc backend for the modes that run the + // document parsing pipeline. This server is built with -tags cgo and links + // ONNX Runtime statically (libonnxruntime.a, resolved via dlopen(NULL)), + // so the in-process backend is the production backend with no external + // DeepDoc HTTP service. Fail fast if it cannot serve — but only for modes + // that actually need it: "api" parses in-process (dataflow debug and other + // routes) and "ingestor" runs the ingestion pipeline. "admin"/"syncer" + // never instantiate the analyzer, so they must not fail-fast when ORT and + // models are absent on their node. + if servermode.NeedsDeepDoc(*arguments.mode) { + registerNativeDeepDoc() + } + globalConfig := server.GetConfig() // override default port if provided @@ -1054,3 +1075,77 @@ func configureTTSSynthesizer(modelProviderService *service.ModelProviderService) audio.SetModelProviderSynthesizer(audio.NewTTSDispatchFunc(modelProviderService)) common.Info("agent: TTS model-provider dispatch installed (audio.Synthesize → ModelProviderService.AudioSpeech)") } + +// registerNativeDeepDoc wires the in-process (Go) DeepDoc backend as the local +// inference backend. The server is built with -tags cgo and links ONNX Runtime +// statically (libonnxruntime.a, resolved at runtime via dlopen(NULL) from the +// running binary — see the onnxruntime_go fork), so there is no external +// DeepDoc HTTP service and no dynamic .so deployment. +// +// Fail-fast contract (P0): the in-process backend must be available at startup +// (ORT + models present). There is NO silent degradation to an empty analyzer: +// if the backend is not serving, the server aborts. +func registerNativeDeepDoc() { + modelDir := resolveDeepDocModelDir() + dropScore := resolveDeepDocDropScore() + + if err := infnative.Register(modelDir, dropScore); err != nil { + common.Warn("in-process DeepDoc backend unavailable", + zap.String("reason", err.Error())) + } + + // The in-process (Go) DeepDoc backend is the ONLY production backend. Fail + // fast rather than silently parsing without layout/table/OCR if the local + // backend cannot serve (ORT + models must be present when built with -tags + // cgo). + if !infnative.Serving() { + common.Fatal("no in-process DeepDoc backend serving: provide the local ORT "+ + "runtime + models and build with -tags cgo", + zap.String("model_dir", modelDir), + zap.String("ort_lib", "static (libonnxruntime.a via dlopen(NULL))")) + } + common.Info("in-process DeepDoc backend registered (production backend)", + zap.String("model_dir", modelDir)) +} + +// resolveDeepDocModelDir picks the model directory: the explicit DEEPDOC_MODEL_DIR +// env, else the RAGFlow default (rag/res/deepdoc, mirroring deepdoc_server.py), +// else the snapshot fetched by ragflow_deps/download_deps.py. The first +// candidate that actually contains the required weights wins. +func resolveDeepDocModelDir() string { + if v := strings.TrimSpace(common.GetEnv(common.EnvDeepDocModelDir)); v != "" { + return v + } + wd, _ := os.Getwd() + candidates := []string{ + filepath.Join(wd, "rag", "res", "deepdoc"), + filepath.Join(wd, "huggingface.co", "InfiniFlow", "deepdoc"), + } + for _, c := range candidates { + if dirHasModels(c) { + return c + } + } + // None verified; return the canonical default so any error message points + // at the conventional location. + return filepath.Join(wd, "rag", "res", "deepdoc") +} + +// resolveDeepDocDropScore returns the explicit DEEPDOC_DROP_SCORE env, else the +// in-process backend's default (infnative.DefaultDropScore, which mirrors +// the Python inference service's Recognizer.drop_score). +func resolveDeepDocDropScore() float64 { + if v := strings.TrimSpace(common.GetEnv(common.EnvDeepDocDropScore)); v != "" { + if f, err := strconv.ParseFloat(v, 64); err == nil { + return f + } + common.Warn("invalid DEEPDOC_DROP_SCORE, using default", + zap.String("value", v), zap.Float64("default", infnative.DefaultDropScore)) + } + return infnative.DefaultDropScore +} + +// dirHasModels reports whether dir contains every required model file. +func dirHasModels(dir string) bool { + return common.HasModelFiles(dir) +} diff --git a/deepdoc_go_alignment_report.md b/deepdoc_go_alignment_report.md new file mode 100644 index 0000000000..409b916744 --- /dev/null +++ b/deepdoc_go_alignment_report.md @@ -0,0 +1,1139 @@ +# DeepDoc Go In-Process Backend vs Python Inference Service + +> Status: completed. +> Scope: `internal/deepdoc/native`, `internal/deepdoc/parser/pdf`, +> `internal/deepdoc/parser/pdf/inference/native_analyzer`. + +## Contents + +- [1. Background](#1-background) +- [2. Python inference service dependency inventory](#2-python-inference-service-dependency-inventory) +- [3. Go-side migration](#3-go-side-migration) +- [4. Caller perspective: are the two semantically consistent?](#4-caller-perspective-are-the-two-semantically-consistent) +- [5. Real OCR batch + removal of the external Python service dependency](#5-real-ocr-batch--removal-of-the-external-python-service-dependency) +- [6. Implementation notes / pitfalls](#6-implementation-notes--pitfalls) +- [7. Open items](#7-open-items) +- [Key file index](#key-file-index) +- [The native verification harness (design record)](#the-native-verification-harness-design-record) +- [Equivalence proof](#equivalence-proof) + +## 1. Background + +RAGFlow PDF parsing relies on DeepDoc inference (OCR text detection / layout +analysis DLA / table structure TSR / text recognition OCR). The original +deployment depended on a standalone **Python inference service** +(`deepdoc_server`). This project reimplements an **in-process Go backend**: + +- Pure Go + ONNX Runtime reimplements the same models and the same + post-processing; +- it is packaged into the single server binary (`bin/ragflow_server`) — no + separate Python service to build or deploy; +- both implementations satisfy the same `DocAnalyzer` interface, so callers are + unaware of which one is active. + +## 2. Python inference service dependency inventory + +### Runtime inference (used directly by `deepdoc/vision`) + +| Package | Purpose | +|---|---| +| `onnxruntime>=1.20.0` | runs the det / layout / tsr / rec ONNX models | +| `opencv-python-headless` (cv2) | image decode, resize, findContours, minAreaRect+boxPoints, fillPoly, approxPolyDP, NMS | +| `numpy` | tensor / array math | +| `pillow` (PIL) | server-side image decode / encode | +| `pyclipper>=1.4.0` | det box unclip (polygon dilation) | +| `shapely` | only `Polygon.area/length` for unclip | +| `six` | py2/3 compat shim | + +### Service layer (HTTP) + +`litserve>=0.2.17`, `python-multipart` + +### Rasterization (outside the inference boundary) + +`pdfplumber` (PDF → image @216 DPI) + +### Model download + +`huggingface_hub` (`snapshot_download`) + +## 3. Go-side migration + +| Python dependency | Go migration | Status / residual | +|---|---|---| +| `onnxruntime` | `onnxruntime_go` (cgo; `InitORT` resolves ORT via `dlopen(NULL)` from the binary; bounded pool in `session.go`/`session_pool.go`) | same `.onnx` models, sha256-locked | +| `cv2.resize (INTER_LINEAR)` | pure-Go `bilinearResize` | float vs fixed-point → **~3px detection floor** (accepted) | +| `cv2.findContours (RETR_LIST)` | hand-written Moore-neighbour (Suzuki-Abe) `findContours` (`det.go`) | residual = box geometry drift (1:1 box set, mean IoU 0.969) | +| `cv2.minAreaRect + boxPoints` | pure-Go rotated-rect `minAreaRect` (`geometry.go`) | aligned | +| `cv2.fillPoly` | pure-Go scanline `fillPoly` (`box_score_fast`) | bit-level identical to cv2 | +| `cv2.approxPolyDP / arcLength` | not ported | only used in Python `poly` path; quad path does not use it | +| `cv2` NMS | `nms.go` (only DLA-table 0.45 / TSR 0.2; det has no NMS) | aligned | +| `cv2 imdecode` / PIL | Go `image` `Decode` (auto format detect + size/pixel caps) | decode contributes ~0 | +| `pyclipper` (unclip, JT_ROUND) | `clipper_offset.go` (Clipper1 port) | 0px-aligned with pyclipper | +| `shapely Polygon.area/length` | computed directly inside `clipper_offset.go` | no dependency needed | +| `numpy` | Go `[]float32` | — | +| `pdfplumber` (PDF→raster @216 DPI) | production uses Go `pdfium.RenderPage` @216 DPI + `FPDF_LCD_TEXT` | measured DLA ≤0.03px | +| `litserve` / `python-multipart` / `six` (HTTP service) | not migrated | replaced by in-process `NativeAnalyzer`; no external HTTP client path | +| `huggingface_hub` (model download) | not migrated | `ragflow_deps/download_deps.py` fetches the same snapshot, sha256-locked | + +**Summary:** every inference dependency is migrated — `onnxruntime`→`onnxruntime_go`, +`cv2`→pure-Go geometry reimplementation (resize / findContours / minAreaRect / +fillPoly / NMS), `pyclipper`→`clipper_offset.go`, `shapely`/`PIL`/`numpy`→native +Go. The only things **not** migrated are three categories: the HTTP service layer +(replaced by in-process), model download (Python script retained), and +`pdfplumber` (replaced by `pdfium`, measured-aligned). + +## 4. Caller perspective: are the two semantically consistent? + +**Conclusion: semantically aligned; bounded numeric differences that do not +affect upper layers.** + +Callers consume four `DocAnalyzer` outputs: + +| Output | Semantic consistency | Measured evidence | +|---|---|---| +| **Text content** (OCRRecognize) | consistent | on the 35-PDF baseline only 2/35 differ, and only by confidence 0.008 — the text itself is identical | +| **Layout** (DLA) | consistent | 171/174 (98.3%) match <1px; end-to-end raster-aligned text pages ≤0.03px | +| **Table structure** (TSR) | consistent | 85/85 (100%) match | +| **Text-region set** (OCRDetect) | consistent | 1559 boxes 1:1 match, only **1 orphan box** (Go over-detects) | + +Callers see "where is text, what is the text, how is layout/table arranged" — +**semantically consistent**. The `drop_score` contract is also aligned (both blank +text but keep the real score when confidence < 0.5). + +"Fully consistent" ≠ byte-identical: coordinates jitter by a few pixels +(mean IoU 0.969, worst 21px), confidence has ~1e-5 float noise, and there is that +1 orphan box. This is not a uniform "every box off by a few pixels" error — it is +geometry drift on a small number of boxes. + +### Do the 35-PDF intermediate results stay semantically consistent for upper layers? + +**Yes, semantically; with one negligible exception.** + +- Final output (chunk / table-HTML / markdown) is driven by **text content + + layout/table structure**, both of which are consistent → upper-layer output is + semantically consistent. +- Coordinate jitter does not affect upper layers: reading order / grouping uses + proximity, and a few pixels do not change grouping; table HTML is driven by TSR + cells (100% consistent). +- **Only exception:** the 1 Go-only orphan box in `10_numbering_patterns.pdf`. If + that box is recognized and merged into a chunk it adds a small text span — + **1/1559 boxes, negligible**, but strictly not 100% identical. + +> Honest boundary: the equivalence proof covers the **inference boundary** (same +> image, same model); `PdfParser` downstream (split/assemble) is out of scope. +> Because its inputs (text/structure) are consistent, upper-layer semantic +> consistency is a strong inference, not an exhaustive measurement. + +### Why not fully identical (the "why" at the numeric layer) + +All residuals come from **detection geometry**, unrelated to `drop_score` or the +ORT version (controlled experiments ruled out ORT version): + +1. **Contour tracing algorithm:** Go hand-written Moore-neighbour vs + `cv2.findContours` → different boundary pixels → `minAreaRect` geometry + slightly off; +2. **resize interpolation:** Go float `bilinearResize` vs cv2 fixed-point + `INTER_LINEAR` → ≤1/255 pixel noise → seg map differs by 634px → contour + boundary shifts accordingly; +3. combined, near-threshold (0.5 box_thresh) regions produce a "score-flip" + orphan (that 1 box). + +## 5. Real OCR batch + removal of the external Python service dependency + +### 5.1 Real batch integration (completed) + +- **Low-level batch** (`internal/deepdoc/native`): + - `ocr_rec.go`: `RunOCRRecBatchReal(ctx, modelDir, []*Image)` concatenates each + line's preprocessed blob into a `{N,3,48,imgW}` tensor, runs ONNX **once**, + and CTC-decodes line by line. Numerically identical to per-image `RunOCRRec`. + - `image.go`: `FromImages(imgs []image.Image) ([]*Image, error)` batches + `FromImage`. +- **Production analyzer** (`internal/deepdoc/parser/pdf/inference/native_analyzer/native_analyzer.go`): + - `OCRRecognizeBatch(ctx, []image.Image) ([][]deepdoctype.OCRText, error)`: + `n==0` → nil; `n==1` → falls back to single `OCRRecognize` (no batch-width + dilation); `n>1` → `native.FromImages` + `native.RunOCRRecBatchReal` (true + batch). Applies the `dropScore` blanking contract, consistent with the + single-image path. + - Fixed a compile error where the file had **two duplicate `OCRRecognizeBatch` + methods**; kept only the `FromImages`-based version. +- **Call site** (`internal/deepdoc/parser/pdf/parser_concurrency.go` + + `parser_ocr.go`): + - `parser_concurrency.go`: optional `batchRecognizer` interface + (`OCRRecognizeBatch`); `inferOCRRecognizeBatch` occupies one limiter slot for + the whole batch; `docSupportsBatchOCR` detects batch capability. Analyzers + that do not implement the interface (MockDocAnalyzer / replay analyzer) + fall back to the per-crop path — zero interface breakage. + - `parser_ocr.go`: `ocrDetectAndRecognize` collects each box's layer-2 rotation + candidates, accumulates a 1-D `cropAcc`, and if `docSupportsBatchOCR` issues + a single `inferOCRRecognizeBatch` (one Run), otherwise falls back per-crop; + then selects the highest-confidence candidate per box and emits `TextBox` + concurrently. `buildTextBoxes` wires the OCR re-recognition branch into batch + as well. The replay analyzer never implements `batchRecognizer`, so it always + takes the per-crop fallback and `ocrBoxIdxCtxKey` box-index routing stays + intact. + +### 5.2 Removal of PyOracle / external Python service dependency (completed) + +`internal/deepdoc/parser/pdf/inference/py_oracle.go` was a **non-test file** +implementing a full HTTP inference client (`PyOracle`) for the external Python +service, violating the "production uses in-process only" architecture. It has +been **removed**, along with the pure Python comparison tests that referenced it: + +- Removed: `py_oracle.go`; `inprocess_vs_service_compare_test.go`; + `inprocess_vs_service_iou_test.go`; the `tmpcheck/` directory (3 temporary + debug/compare tests + `EQUIVALENCE_REPORT.md`); `ocr_merge_test.go`; + `native_integration_test.go`'s `TestWireVsLiveServer`. +- Rewritten to use `NativeAnalyzer` (with `cgo` tag, so they become real-inference + regression tests that run only in the ORT-deployed `--test-native` build): + `helpers_test.go` (`mustConnectInferenceClient` → `mustConnectInProcessAnalyzer`) + and the call sites `dla_real_world_test.go`, `dla_tsr_compare_test.go`, + `inference_client_integration_test.go`, `parser_parallel_integration_test.go`, + `parser_pipeline_integration_test.go`, `parser_pipeline_manual_test.go`, + `production_smoke_test.go`, `scan_all_pdfs_test.go`, `batch_smoke_test.go`, + `table_crop_ab_manual_test.go`, `table_rotate_integration_test.go`, + `parser_ocr_rotate_integration_test.go`. +- `DefaultDLALabels` / `DefaultTSRLabels` moved to `cache.go`; 12 tests switched + to the in-process `NativeAnalyzer`. +- Further cleanup: deleted `pdf_parser_resolver_test.go`; removed 6 redundant + `t.Setenv("DEEPDOC_URL","")` calls; removed dead config in `environments.go` + (`GetDeepDocURL`/`GetOSSDeepDocURL` getters, `DeepDocURL`/`OSSDeepDocURL` fields + and env reads, `EnvDeepDocURL`/`EnvOSSDeepDocURL` constants); `resolveDocAnalyzer` + dropped the dead `baseURL` parameter. +- Docs synced: `AGENTS.md`/`CLAUDE.md` (removed "needs external service at + startup" misdescription), this document's **Equivalence proof** section (removed `TestWireVsLiveServer` / + live-oracle stale description), `docker/.env` comment. + +### 5.3 Verification + +| Check | Command | Result | +|---|---|---| +| `parser/pdf` main package compile+link | `go build -tags cgo` (build.sh cgo env) | ✅ EXIT=0 | +| `native_analyzer` package (no models) | `build.sh --test-native` | ✅ ok | +| native real-batch inference | `TestOCRRecBatchIntegration` (ORT+models) | ✅ ok 11.3s | +| `NativeAnalyzer` end-to-end | `build.sh --test-native` (ORT+models) | ✅ ok 387s | +| call-site batch interface | `TestAnalyzerOCRRecBatchIntegration` (ORT+models) | ✅ ok 7.6s | + +## 6. Implementation notes / pitfalls + +- **pdfium is present:** the static lib is at + `~/ragflow-native-libs/pdfium-static/lib/libpdfium.a`. `build.sh`'s + `setup_cgo_env` exports the correct cgo flags — do not assume it cannot link. +- **cgo is mandatory:** `NativeAnalyzer` lives in a `//go:build cgo` package whose + import chain pulls in the onnxruntime cgo binding. Any test that uses it must + carry the `cgo` tag and can only run where ORT is deployed (`--test-native`). +- **batch width semantics:** `RunOCRRecBatchReal` resizes all lines to the + batch-max width, so batch text **differs from** per-image `OCRRecognize` (single + image uses fixed recW=320). This is by design — assert against + `RunOCRRecBatchReal`, not the single-image path. +- **`native.InitORT` is process-global** and idempotent; multiple calls must be + safe (follow the `analyzerWithModels` pattern). + +## 7. Open items + +- `docker/docker-compose.yml` still defines a standalone `deepdoc` service + container (port 9390), and `docker/README.md` still mentions `DEEPDOC_URL`. The + Go code no longer connects to it, but whether to remove the deployment-layer + container is a separate scope. + +## Key file index + +| File | Role | +|---|---| +| `internal/deepdoc/native/ocr_rec.go` | `RunOCRRecBatchReal` / `RunOCRRec` / `RunOCRRecBatch` | +| `internal/deepdoc/native/image.go` | `FromImage` / `FromImages` | +| `internal/deepdoc/parser/pdf/inference/native_analyzer/native_analyzer.go` | `NativeAnalyzer` (incl. `OCRRecognizeBatch`) | +| `internal/deepdoc/parser/pdf/parser_concurrency.go` | `batchRecognizer` interface / `inferOCRRecognizeBatch` / `docSupportsBatchOCR` | +| `internal/deepdoc/parser/pdf/parser_ocr.go` | `ocrDetectAndRecognize` / `buildTextBoxes` (batch call site) | +| `internal/deepdoc/parser/type/labels.go` | `DefaultDLALabels` / `DefaultTSRLabels` (DLA/TSR wire-contract taxonomies; moved here from the removed `inference` HTTP-service cache package) | +| `internal/parser/parser/pdf_parser_common.go` | `deepDocAnalyzerFromEnv` (ignores `DEEPDOC_URL`) | +| `build.sh` | `setup_cgo_env` (cgo flags), `--test-native` | +| `cmd/ragflow_server.go` | `registerNativeDeepDoc` (registers in-process only) | + +--- + +## The native verification harness (design record) + +> This section is the authoritative design record (ADR style) for `native`. +> The equivalence proof follows below in **Equivalence proof**; the historical +> session handoff notes (HANDOFF*) were deleted — their still-valid points are +> folded into this document. + +### 1. Positioning and boundary + +`native` is a **standalone verification harness**, not a production path: + +- It reimplements the DeepDoc Python inference pipeline (OCR text detection + `det` / layout analysis `DLA` / table structure `TSR` / text recognition + `OCR-rec`) in **pure Go + ONNX Runtime**, running on CPU. +- It compares against the Python reference implementation (`ref_*.py`, which + imports the repo-root `deepdoc`) to catch regressions in the port. +- **Wired into production as the sole backend**: `internal/deepdoc` no longer + talks to the remote Python service over HTTP — production serves DeepDoc + entirely in-process via `infnative.NativeAnalyzer` (built with `-tags + cgo`). The Python service survives only as a read-only equivalence + oracle in the `inprocess_vs_service_*` tests (via the test-only `PyOracle` + client), never as a production path. The `native` harness and the + production backend share the same ONNX Runtime port, so `native`'s + regression checks directly guard the production inference. + +It is a regular package inside the main module +(`ragflow/internal/deepdoc/native`), gated by the `cgo` build tag so the +ONNX Runtime (`onnxruntime_go`) cgo binding stays out of the default (no-cgo) +build — the same isolation used for `office_oxide` / `pdfium` / `pdf_oxide`. + +### 2. Why pure Go (P3 decision) + +A `gocv` / `nogocv` dual build once existed: the gocv path used cv2 decode + +resize and reached 1:1 parity, while pure Go had a ~3 px floor. The code has +**converged to a single pure-Go path** (`image_gocv.go` / `det_gocv.go` / +`dla_gocv.go` deleted, `gocv.io/x/gocv` removed from `go.mod`, the CI +`go-native-gocv` job removed). + +Trade-off: give up cv2 1:1 parity in exchange for **zero OpenCV / CGO +dependency**. + +### 3. Where the 3px floor comes from (known hard floor) + +The port's maximum coordinate residual vs the Python reference is stable at +**~3 px**, from: + +- `bilinearResize` (Go float weights) vs cv2 fixed-point `INTER_LINEAR` + implementation difference; +- the contour minimum-area rect in the `box#8` post-process step + (`minAreaRect`). + +The floor is **input-format independent** (measured 3.0 px for both JPG and +PNG; decode contributes ~0 because PNG losslessly wraps JPEG-decoded pixels). +The geometry core is not touched unless a decision is made to emulate cv2's +bit-exact float `minAreaRect` (which would require reintroducing OpenCV). + +### 4. How goldens are generated / what the drift gate proves + +- **golden**: output of `ref_*.py` (the Python oracle) on fixed fixtures, + frozen to `testdata/..golden.json`. Fixtures are now **PNG** + (losslessly transcoded from JPG, pixel-equivalent, verified 47/47 zero + difference), matching the production `EncodePNG` wire. +- **`python-drift` job**: re-runs the Python oracle and compares against the + golden — it alerts only when **the Python logic drifts and the golden is + regenerated and committed**. It does **not** independently prove Python is + correct (Python is the trust anchor / oracle). +- **`go-native` job**: runs the Go port against the same golden — catches + **Go regressions**. + +> Conclusion: the drift gate is a "Go vs frozen Python snapshot" comparison. +> It reliably catches Go-side regressions but does **not** prove the Python +> side is correct — this is inherent to the "re-implement-to-verify" pattern. + +### 5. Test tiers and safety + +- **unit** (no tag): pure geometry/post-process unit tests + (`clipper_offset_test.go` cross-checked against pyclipper at 0 px, + `minAreaRect` cross-check, `image_test.go` decode limits); run by default + via `go test ./native/`. +- **integration** (`//go:build integration`, needs `MODEL_DIR`; ORT is always + resolved from the statically-linked binary via dlopen(NULL), so no `ORT_LIB` + is needed and self-skips only when the binary was not built with static ORT): + full-component Go-vs-golden comparison. +- **decode safety**: `Decode` validates the decoded raster's size/pixel + limits (`maxImageDim=16384`, `maxImagePixels=100MP`) to defend against + decompression bombs. It currently runs only on fixed fixtures with no + production exposure; the limits activate if untrusted input is ever wired + in. + +### 6. Comparison tolerance + +Coordinate tolerance = **`coordFloor(3.0) + coordTolMargin(0.5)` = 3.5**, +computed from constants rather than a literal (`native_integration_test.go`). +`coordFloor` is the 3 px hard floor of §3; `coordTolMargin` lifts the +tolerance just above the floor so that a regression crossing the floor trips +the gate instead of hiding under it. + +Because the tolerance derives from `coordFloor`, adjusting the floor later +**follows automatically** with no manual sync. The gate can only catch +regressions **> 3 px (the floor)**, which is enough for "prevent large +breakage"; finer regressions (below the floor) are inherently +indistinguishable — this is the tool's sensitivity floor, not a defect. + +### 7. How to regenerate goldens + +Goldens are the `ref_*.py` (Python oracle) output on fixed fixtures, frozen to +`testdata/..golden.json`. After a deepdoc Python-logic change, +re-run the oracle and write the output back into the goldens, then commit — +otherwise the `python-drift` job alerts (`check_drift.py` only compares, never +writes). + +Prerequisites (some venv with `deepdoc` + `onnxruntime` + `opencv-python`): + +```bash +export MODEL_DIR= +export PYTHONPATH= # so ref_det.py can import deepdoc +``` + +Each `ref_*.py` prints the wire JSON to stdout; redirect to write it to disk +(format identical to the existing goldens): + +```bash +cd internal/deepdoc/native +# single fixture + single task +python ref_det.py testdata/page0.png "$MODEL_DIR" > testdata/page0.det.golden.json +python ref_dla.py testdata/page0.png "$MODEL_DIR" > testdata/page0.dla.golden.json +python ref_tsr.py testdata/table0.png "$MODEL_DIR" > testdata/table0.tsr.golden.json +python ref_ocr_rec.py testdata/line0.png "$MODEL_DIR" > testdata/line0.ocr_rec.golden.json + +# all fixtures: enumerate the existing goldens in testdata, auto-match task and image stem +for f in testdata/*.golden.json; do + bn=$(basename "$f" .golden.json) # e.g. page0.dla + task=${bn##*.} # dla / det / tsr / ocr_rec + imgstem=${bn%.*} # page0 + case "$task" in + det) python ref_det.py "testdata/$imgstem.png" "$MODEL_DIR" > "$f" ;; + dla) python ref_dla.py "testdata/$imgstem.png" "$MODEL_DIR" > "$f" ;; + tsr) python ref_tsr.py "testdata/$imgstem.png" "$MODEL_DIR" > "$f" ;; + ocr_rec) python ref_ocr_rec.py "testdata/$imgstem.png" "$MODEL_DIR" > "$f" ;; + esac +done +``` + +After regeneration, run `check_drift.py` to confirm no other drift, then +commit the changed goldens together. + +--- + +## Equivalence proof + +> Design rationale, positioning, and the pure-Go decision live in **The native +> verification harness (design record)** above. This section is the equivalence +> proof; it builds on that record's documented 3 px coordinate floor. + +### Executive summary (for decision makers) + +**Bottom line.** The Go in-process DeepDoc backend (`infnative.NativeAnalyzer`, +backed by `native`) is functionally equivalent to the Python `deepdoc_server` +inference service for all four tasks it covers — proven by *measured* golden +comparison, not by code inspection. + +**What is actually replaced.** `deepdoc_server` is a thin HTTP wrapper over four +recognizers (`dla_adapter` → `LayoutRecognizer4YOLOv10`, `ocr_adapter` → +`TextDetector` / `TextRecognizer`, `tsr_adapter` → `TableStructureRecognizer`). +The caller depends only on the `DocAnalyzer` interface and its wire format. In +the Go server (`-tags cgo`), `infnative.Register` registers +`NativeAnalyzer` as the `DocAnalyzer`, and `resolveDocAnalyzer` serves it +**in-process only** — the external `DEEPDOC_URL` HTTP client has been removed +from the production path entirely. The deployment is in-process, single binary, +no separate Python service to operate. The Python `deepdoc` service now serves +only as an offline alignment baseline (it generates the reference goldens used +by the Go integration tests); it is no longer a runtime backend, a client, or +an equivalence oracle reached over HTTP. + +**Three pillars of the proof.** +1. **Boundary equivalence** — DLA / TSR / OCR-rec / Det outputs match the Python + reference goldens within documented, bounded floors (see *Evidence*). +2. **Wire isomorphism** — Go's `Wire()` JSON is structurally identical to the + `deepdoc_server` output for DLA / TSR / Det / OCR-rec, validated against the + re-serialized golden shape (`TestWireSchemaMatchesGolden`) and the Python + reference scripts (`ref_dla.py` / `ref_tsr.py` / `ref_ocr_rec.py`) that + generate the goldens. The goldens are produced by the same Python recognizers + the service wraps, so the Go backend is compared field-for-field against the + Python baseline it replaces. +3. **Measured, not inferred** — every claim is verified by running the real ONNX + models on committed fixtures; the tests *self-skip* (rather than fake-pass) + when ORT / models are absent. + +**Honest boundaries (condensed).** +- Not bit-identical; "equivalent within bounded, accepted floors" is the correct + claim. Known floors: Det IoU orphan **3/5** (benign, OCR-adjudicated), TSR + **≤ 3.5 px** on ordinary/moderate real tables (worst measured 2.70 px; dense + annual-report ≤ 1.21 px), **≤ 10 px** on a 4:1 aspect crop (structure + preserved); dense technical-standard full-page tables (15K606 p40) can break + the strict floor on BOTH coordinate drift and cell count (17/30, documented + exception — see *Known model-floor limits*). +- **Production rasterization is now measured, not assumed.** Go rasterizes PDF + pages with pdfium @216 DPI (LCD text AA enabled, matching pdfplumber); Python + deepdoc with pdfplumber @216 DPI. The end-to-end raster-alignment harness + (Methodology §8) renders the *same* real PDF page with both paths and compares + boxes: DLA **≤ 0.03 px on text pages** (≤ 0.72 px on the one figure page), Det + IoU orphan 0/0–1/2, TSR on real-table pages ≤ 2.70 px. The "same raster bytes + in" premise for layout/text detection is empirically closed (the prior Scope + note about pdfium-vs-poppler is superseded: deepdoc uses pdfplumber, also + @216 DPI). **Caveat (measured):** enabling AA tightened DLA but did not change + Det — and on closer measurement the Det `corner-maxd 8–12 px` in the test log + is the *max per-corner* difference on 1–2 skewed outlier boxes, not a center drift; + per-box center distance is sub-pixel (no render-origin offset). See Scope note. +- Depends on the `-tags cgo` build path **and** ONNX Runtime **1.23.x** + **and** the same `InfiniFlow/deepdoc` model snapshot as the (frozen) Python + side. +- No standalone HTTP service mirrors `deepdoc_server`; only the in-process + library and a CLI exist. +- Requires CI to keep running the native tests (see *How to reproduce*) so the + equivalence does not silently regress as the Go side evolves. + +**Prerequisites for the claim to hold.** Python side (including models) frozen; +`MODEL_DIR` pinned to the same snapshot; native tests wired into CI. + +**Runtime version (concretely recorded).** The Go side is pinned to ONNX +Runtime **1.23.2** (`internal/common.DeepDocORTVersion = "1.23.2"`). The Python +reference `deepdoc_server` was validated against this proof with `onnxruntime +1.23.2` at golden-generation time; the **currently running** Python service in +this environment resolves `onnxruntime` to **1.28.0** (`deepdoc/server/pyproject.toml` +pins `onnxruntime>=1.20.0`). This cross-version state is **measured, not +assumed**: + +- The `onnxruntime_go` binding (forked to `github.com/xugangqiang/onnxruntime_go`) + resolves ONNX Runtime at runtime through the C-API (`native.InitORT`): the + in-process backend is built with `libonnxruntime.a` **statically linked** and + finds `OrtGetApiBase` via dlopen(NULL) from the running process, so no + `libonnxruntime.so` is needed at runtime and there is no dynamic `.so` + deployment. A historical experiment once loaded the Python service's 1.28.0 + `.so` into the Go backend to prove ABI compatibility; that path has since been + removed. The binding is ABI-compatible across 1.23.x/1.28.x. +- A controlled experiment (Go backend loaded with **1.28.0**'s `.so`) reproduced + the **identical** detection matching as the default 1.23.2 (588/868 `<1px` + matches on both, per-PDF box counts identical), so **the ORT version is not a + sensitivity** for the detection boundary. The two runtimes are numerically + aligned in behaviour, not just ABI-compatible. +- The guard for future drift stays the golden-comparison integration tests + (`TestWireSchemaMatchesGolden` and the `TestAnalyzer*Golden` suites) and the + per-page parity harness (Methodology §9): if the Python side is ever moved to + another ORT build, regenerate the goldens and re-run those before trusting + the claim. + +### Scope + +This document proves that the Go `native` inference library produces output +**equivalent** to the Python `deepdoc` inference service (the `deepdoc_server` +HTTP service backed by the `deepdoc/vision` recognizers) for the four tasks it +covers: + +- **Det** — DB text detection +- **DLA** — document layout analysis +- **TSR** — table structure recognition +- **OCR** — text-line recognition + +Both sides load the **same ONNX models** from `InfiniFlow/deepdoc` +(`det.onnx`, `layout.onnx`, `tsr.onnx`, `rec.onnx`) and the same `ocr.res` +character dictionary, and consume the **same raster image bytes**. The Python +service decodes request bytes with PIL / cv2; the Go side decodes with Go's +`image` package — both are format-agnostic raster decoders, so the inference +boundary is identical (raster in, boxes/text out). PDF rasterization happens +upstream of both, outside this boundary. + +**Boundary of this proof (read carefully).** This document proves the two +backends are equivalent **at the inference-service boundary** — given the *same +raster image bytes*, they emit the same boxes / text / wire JSON. It does +**not** prove end-to-end equivalence of the full ingestion pipeline (PDF → +raster → recognizers → `PdfParser` post-processing → chunks / table-HTML / +markdown). Two things live outside this boundary and are out of scope here: +1. **PDF → raster.** The Go pipeline rasterizes via `pdfium` (@216 DPI) and the + Python deepdoc pipeline via `pdfplumber` (`to_image(resolution=72*zoomin=216, + antialias=True)`) — **both at 216 DPI**. This is no longer an open scope cut: + the end-to-end raster-alignment harness (Methodology §8) renders the *same* + real PDF page with both paths and compares the recognizer output in + source-pixel space. Result: DLA ≤ 0.03 px on text pages (≤ 0.72 px on the one + figure-heavy page), Det IoU orphan 0/0–1/2, TSR on real-table pages ≤ 2.70 px + — i.e. the "same raster bytes in" premise for layout/text detection holds + empirically through the actual production render paths. + + **AA status (measured).** The Go pdfium render now sets `FPDF_LCD_TEXT (0x02)` + in `pdfium.go`, upgrading text from pdfium's *default* grayscale anti-aliasing + to LCD sub-pixel AA to match Python pdfplumber's `antialias=True` text. (Note: + pdfium anti-aliases by default — `0x02` only refines text AA; there is no + "pdfium has no AA" regime, and `pdf_oxide` is not in the render path at all — + it only does text/char extraction. The AA flag is a pdfium C-API flag applied + directly to `FPDF_RenderPageBitmap`.) Re-running the raster-alignment harness + after this change produced a split result: + - *DLA tightened sharply on text pages*: worst max Δ dropped from ~0.72 px to + ≤ 0.03 px on annual report/ZoomNeXt/ZH-TW/Three-Kingdoms pages (effectively pixel-identical). + The one technical-standard figure page (15K606 p10) stayed at 0.721 px — + that residual is **not** an AA artifact (it survived the text-AA change), so + it comes from a different source (vector/figure rendering, not text + smoothing). + - *Det: no render-origin offset (measured)*: the test log's `corner-maxd 8.0 / 12.0 + px` on annual report p2/p8 is the **max per-corner** coordinate difference, not a + center drift. A nearest-center (greedy + Hungarian) analysis of the dumped + boxes shows per-box center distance is sub-pixel — median **0 px**, mean + **< 0.5 px**, p90 **< 2.2 px**, max **< 5 px** — with IoU orphan unchanged + 0/0–1/2. So there is **no translation / coordinate-origin offset** between + the two render paths; the earlier "8–12 px is a render-origin translation" + guess is **falsified**. The ~8–12 px corner figure is concentrated in 1–2 + outlier text boxes whose quadrilaterals are slightly skewed, from the same + contour-boundary geometry that produces the 3/5 IoU orphan floor + (`bilinearResize` vs cv2.resize at text edges + Moore-neighbour vs + cv2.findContours). AA did not change it (identical before/after) because it + is not an antialias artifact. The IoU ≥ 0.5 floor still holds, so it + remains a benign quad-skew, not a detection divergence. +2. **`PdfParser` downstream logic** — caption/figure/table association, table + cell extraction, equation handling, chunk assembly. These consume the + recognizer outputs but are not inference endpoints, so they are not covered + by "inference service equivalence". + +### How to reproduce + +The proof is a reproducible test harness. From +`internal/deepdoc/native`: + +```bash +# ORT is resolved from the statically-linked binary via dlopen(NULL); no +# libonnxruntime.so is needed or supported. +MODEL_DIR= \ +go test -tags integration -run 'TestEquivalenceReport|TestDetMembershipAllFixtures' -v ./... +``` + +- `TestEquivalenceReport` prints the consolidated summary below to the test + log (visible in CI). +- `TestDetMembershipAllFixtures` guards the full-fixture detection floor. + +The same model boundary is exercised through the **`DocAnalyzer` seam the PDF +parser actually consumes** (`infnative.NativeAnalyzer`, package +`internal/deepdoc/parser/pdf/inference/native_analyzer`), so the equivalence is proven +at the integration point rather than only inside the standalone library. + +> **Use `build.sh`, not raw `go test`.** The Go native paths need CGO flags and +> the static native libs (`office_oxide` / `pdfium` / `pdf_oxide`) that +> `build.sh` wires automatically. From the repo root: + +```bash +# ORT is resolved from the statically-linked binary via dlopen(NULL); no +# libonnxruntime.so is needed or supported. +MODEL_DIR= \ +bash build.sh --test-native \ + -run 'TestAnalyzerDLAGolden|TestAnalyzerTSRGolden|TestAnalyzerOCRRecGolden|TestAnalyzerDetGolden' \ + ./internal/deepdoc/parser/pdf/inference/native_analyzer/... +``` + +`bash build.sh --test-native` (no `-run` filter) runs the **entire** native +tier in one shot: the analyzer golden suite above **plus** the `native` +integration suite (`TestEquivalenceReport` / `TestDetMembershipAllFixtures`). +That single command is what CI should gate on. + +> **Env-var note.** `MODEL_DIR` (tests) / `DEEPDOC_MODEL_DIR` (server) select +> the model snapshot and are required. ONNX Runtime is statically linked into +> the binary and resolved via dlopen(NULL); `ORT_LIB` / `DEEPDOC_ORT_LIB` are no +> longer read (there is no dynamic `.so` deployment). Both point at the same +> `InfiniFlow/deepdoc` snapshot. + +These four tests reuse the comparison helpers in `native/golden.go` +(coordinate / score tolerances, IoU box-membership) and compare the analyzer's +`DLA` / `TSR` / `OCRRecognize` / `OCRDetect` output against the same Python +reference goldens. + +Models are fetched once with `ragflow_deps/download_deps.py` (a snapshot of +`InfiniFlow/deepdoc`), which also downloads the ONNX Runtime **static** +archives (`libonnxruntime*.a`) into `~/ragflow-native-libs/onnxruntime/static_lib`. +Those archives are linked into the binary and resolved at runtime via +dlopen(NULL), so no `libonnxruntime.so` is required. A historical controlled +experiment once loaded a dynamic `libonnxruntime.so` (default `DeepDocORTVersion` +**1.23.2**, also validated with **1.28.0** — the version the currently running +Python service resolves) to show identical detection matching across builds +(see *Runtime version* in the summary); that dynamic path has since been removed. +The `onnxruntime_go` v1.23.0 binding is C-API ABI-compatible with both builds, +so the validated parity applies to the static linking used now. + +### How to deploy / switch backends + +The server chooses its DeepDoc backend **once at startup**; there is no +per-request hot-swap. + +**In-process (no Python service to operate).** Build the server with the +native backend and point it at the models: + +```bash +bash build.sh --go # server binary is built with -tags cgo +export DEEPDOC_MODEL_DIR= # default: ragflow_deps/download_deps.py snapshot +``` + +**Startup guarantee (fail-fast).** `cmd/ragflow_server.go` requires the +in-process backend (ORT + models present, built with `-tags cgo`) and +exits with an error if it cannot serve, so a misconfigured deployment cannot +silently produce empty parses. The CLI binary is built without `cgo` +(no backend at all) and therefore fails fast on startup. + +**Switching is safe because the outputs are interchangeable.** Both backends +implement the same `deepdoctype.DocAnalyzer` interface with the same wire JSON, +and the content (text, layout, tables) is the same — see the equivalence above. +The only observable difference when switching is a few-pixel wobble in +detected-box coordinates (typically <5 px, worst ~21 px) and a rare extra box +(1/1559), which do not change downstream chunking or retrieval. Consumers that +assert exact box coordinates, or hash the parsed JSON as a cache key, must not +switch backends blindly. + +### How the underlying dependencies align + +Equivalence rests on both sides running the **same model artifacts** and the +**equivalent runtime stack**. Dependency by dependency: + +| Dependency | Go in-process | Python deepdoc | Alignment | +|---|---|---|---| +| Models (`det.onnx`/`layout.onnx`/`tsr.onnx`/`rec.onnx`/`ocr.res`) | `ragflow_deps/download_deps.py` snapshot of `InfiniFlow/deepdoc` | same frozen snapshot | **Same; sha256-pinned** (`modelSnapshotHashes`, `TestModelSnapshotHash` fails on drift) | +| ONNX Runtime | statically linked `libonnxruntime.a` (dlopen(NULL) via the `onnxruntime_go` fork) — no dynamic `.so` | `onnxruntime` PyPI, currently resolves 1.28.0 | **ABI-compatible + measured version-insensitive** (Runtime version / §9) | +| Char dictionary | `ocr.res` | `ocr.res` | **Same file** | +| Image decode (PNG/JPEG → raster) | Go `image` package | PIL / cv2 | **Format-agnostic; decode contributes ~0 px (measured on PNG/JPEG)** | +| PDF → raster | pdfium @216 DPI, `FPDF_LCD_TEXT` AA | pdfplumber @216 DPI, `antialias=True` | **Measured via §8: DLA ≤ 0.03 px on text pages** | +| Det post-process constants | thresh 0.3 / box_thresh 0.5 / unclip 1.5 / min_size 3·5 / round+clamp scaling / no NMS / no approxPolyDP on the quad path | identical (`DBPostProcess` values) | **Identical** | +| Resize | Go `bilinearResize` (float) | cv2 `INTER_LINEAR` (fixed-point) | **Residual = the ~3 px det floor** (documented, accepted) | +| Contour extraction | hand-rolled Moore-neighbour `findContours` | cv2 `findContours` (RETR_LIST, CHAIN_APPROX_SIMPLE) | **Residual = box-geometry drift** (documented, accepted) | +| Go-native CGO libs (`office_oxide`/`pdfium`/`pdf_oxide`) | static-linked via `build.sh` | n/a (Python uses pdfplumber etc.) | **Go-only build deps; not inference-equivalence deps** | + +Bottom line: everything *inference-relevant* (models, char dict, post-process +constants) is identical; everything *implementation* (decoder, resize, contour +tracer, ORT build) is equivalent within the measured, documented floors in +*Evidence* and *Det IoU orphan 3/5*. + +### Methodology: how the proof is constructed + +The proof is not a hand-written mirror of the Python output — it compares the +Go port against **production deepdoc inference**, captured as frozen "golden" +fixtures. Two independent test tiers run the same comparison math against the +same goldens, at two different integration depths. + +#### 1. Reference goldens are real deepdoc output + +Each golden JSON is produced by a Python oracle in this directory that calls +`deepdoc.vision` directly, then re-maps the OSS labels to the Go class ids and +serializes into the Go `DocAnalyzer` wire shape. The oracles invoke the exact +same recognizers and post-processing the running server uses: + +| Task | Oracle | Production recognizer / config it pins | +|------|--------|----------------------------------------| +| DLA | `ref_dla.py` | `LayoutRecognizer4YOLOv10("layout").forward(thr=0.2)` → OSS labels re-mapped via `DLA_CLASS_MAP` to Go DLA class ids | +| TSR | `ref_tsr.py` | `TableStructureRecognizer()([img], thr=0.2)` incl. `alignTSR` (mean/median when >4 rows/cols) → `TSR_CLASS_MAP` (6 classes) | +| Det | `ref_det.py` | `TextDetector`: `DetResizeForTest → NormalizeImage → det.onnx → DBPostProcess(thresh=0.3, box_thresh=0.5, unclip_ratio=1.5) → filter_tag_det_res` | +| OCR | `ref_ocr_rec.py` | `TextRecognizer` with **batch-wide max `wh_ratio`** resize (matches production batch semantics); wire score pinned to `1.0` to match the Go `DocAnalyzer` OCR-rec wire | + +Because the reference is the actual production code path, a Go/Python mismatch +surfaces a genuine divergence, not a divergence between two reimplementations. +The fixtures are frozen and committed under `internal/deepdoc/native/testdata` +(`*.dla.golden.json`, `*.tsr.golden.json`, `*.det.golden.json`, +`*.ocr_rec.golden.json`). They were regenerated from the **current** live +detectors after the `normalizeCHW` RGB/BGR channel-order fix, so the det +baseline (3/5 IoU orphans) reflects the true Go-vs-cv2 gap rather than a stale +or swapped-channel oracle. + +#### 2. Comparison math (shared by both tiers) + +All matching lives in `internal/deepdoc/native/golden.go` and is +imported by both the `native` integration suite and the `infnative` +analyzer suite, so the two tiers cannot drift apart. + +- **Axis-aligned boxes (DLA / TSR / OCR):** `CompareBoxes` / `MatchBoxesRelaxed` + match every golden box to its **nearest same-class** Go box by center + distance, then assert each coordinate is within `CmpTolCoord` and the score + within `CmpTolScore`. The relaxed variant returns the unmatched count instead + of failing, so callers can express structural assertions (e.g. "only a + near-threshold row may be dropped"). +- **Label alignment:** goldens and the Go analyzer both key on the **first** + index of a label string, so DLA's duplicate labels (indices 4/7/9 share + source text with neighbours) map identically on both sides. The analyzer test + re-derives each box's class from `labelKey(labels, r.Label)` and rewrites the + golden's integer class to the same key before matching. +- **Quads (Det):** `MatchBothDirections` and `MatchIoUBothDirections` match the + rotated text quads **in both directions** by (a) nearest-center within + `CmpTolCoord` and (b) greedy best-IoU ≥ 0.5. IoU membership isolates true + box divergence — a split (1→2), a merge (2→1), or a hallucination — from mere + coordinate drift: a box shifted 20 px but still overlapping its twin scores + high IoU and is **not** an orphan. Reporting both directions surfaces Go boxes + that have no golden counterpart (extra detections) as well as golden boxes Go + missed. + +#### 3. Tolerances and floors + +| Constant | Value | Meaning | +|----------|-------|---------| +| `CoordFloor` | `3.0` | Documented hard accuracy floor of the comparison pipeline (bilinear resize + `box#8` postprocess for det; DLA/TSR are tighter). | +| `CoordTolMargin` | `0.5` | Lifts the tolerance just above `CoordFloor` so a regression trips the gate instead of hiding under it. | +| `CmpTolCoord` | `3.5` | Coordinate tolerance (px) for golden comparisons. | +| `CmpTolScore` | `0.05` | Score tolerance for detection boxes. | + +Two special floors are documented, not hidden: + +- **Det hard floor ≈ 3 px.** The pure-Go geometry stabilizes at ~3 px; the + `3.5` tolerance sits just above it. `TestDetIntegration` asserts every golden + quad is within `3.5` px of a Go quad. +- **Extreme-aspect TSR floor ≈ 10 px.** On a ~4:1 crop the model's 640×640 input + squishes x by ~1.45×, amplifying the residual Go-vs-PIL decode difference to + ~8 px. `TestTSRExtremeAspect` therefore uses a relaxed `10` px tolerance and + asserts only that **structure survives** (table + all columns must match, + row count within ±1, only a near-threshold row may be dropped, no hallucinated + boxes). The analyzer TSR golden (`tsr_table_rotation`, a 1:6.3 tall table) sits + comfortably under the 3.5 px tolerance. + +Because the analyzer does not expose a TSR score, `TestAnalyzerTSRGolden` widens +the score tolerance to `1.0` and asserts only class + coordinates. + +##### Known model-floor limits (full-page real tables) — measured + +The 3.5 px coordinate tolerance is **not** universal across every real table — it +is a property of the committed fixtures, not a guarantee for all inputs. The +end-to-end raster-alignment harness (Methodology §8 / `TestTSRFloorFullPageTables`) +quantifies this on whole-page real tables through BOTH production raster paths: + +| Full-page table | Match | Max Δ | Verdict | +|-----------------|-------|-------|---------| +| Xiamen Xiangyu annual report p8 (moderate) | **34/34** | **1.21 px** | inside 3.5px floor | +| Xiamen Xiangyu annual report p12 (dense) | **25/25** | **0.37 px** | inside 3.5px floor | +| 15K606 "Building Smoke Exhaust System Technical Standard" p40 (dense technical-standard) | **17/30** | 3.36 px | **documented exception** — model-level cell-count divergence (Go emits 31 vs 30 golden; 13 cells unmatched), not a rasterization floor | + +So the empirical upper bound is: + +> TSR equivalence is proven within **≤ 3.5 px** on ordinary/moderate real tables +> (worst measured 2.70 px on Three Kingdoms p1; dense annual-report tables ≤ 1.21 px). +> **Dense technical-standard tables** can break the strict floor — both +> coordinate drift AND cell-count disagreement (17/30 on 15K606 p40). This is a +> model-floor effect (the TSR model itself disagrees on a hard table under +> pdfium-vs-pdfplumber rasterization), not a Go logic bug. It is recorded as a +> known-hard exception in `TestTSRFloorFullPageTables` with a regression guard +> (must not get worse than the 17/30 baseline), and is **not** in the strict +> 3.5px fixture suite. + +If a future caller needs pixel-exact TSR on dense technical-standard tables, the +fix is a decode/resize alignment on large inputs plus a harder table in the +corpus, not a logic change. + +#### 4. Per-task proof + +| Task | What is compared | Proving tests | Result | +|------|------------------|---------------|--------| +| DLA | Layout boxes vs `ref_dla.py` goldens (**11 fixtures**: EN textbook, CN whitepaper, eq-heavy paper, ZH-TW enterprise, baseline, 2 figure-caption pages, 1 equation page, **+ CN annual-report page (Xiamen Xiangyu annual report p2), ZH-TW migration doc p3, EN paper (ZoomNeXt p1)**) | `TestDLAIntegration` / `TestAnalyzerDLAGolden`, `TestEquivalenceReport` | 148/148 boxes, max Δ < 0.13 px | +| TSR | Table-structure boxes vs `ref_tsr.py` goldens (**8 fixtures**: table0, normal 2.65:1, rotated 1:6.3, content, caption, cross-page, text-interleaved — covers projected-row-header; **+ real annual-report table (Xiamen Xiangyu annual report p8)**) | `TestTSRIntegration` / `TestAnalyzerTSRGolden`, `TestTSRExtremeAspect` | 190/190 matched, 2.155 px (≤10 px @4:1 aspect, structure preserved) | +| OCR | Recognized text vs `ref_ocr_rec.py` goldens (EN incl. bold/italic/serif, CJK, mixed, digits; plus batch semantics; **+ 3 real text-line crops: Three Kingdoms p1, ZH-TW migration p2, ZoomNeXt p1**) | `TestOCRRecIntegration` / `TestAnalyzerOCRRecGolden`, `TestOCRRecBatchIntegration` | exact text; batch-wide resize reproduced | +| Det | Text quads vs `ref_det.py` goldens (all fixtures) | `TestDetIntegration` / `TestAnalyzerDetGolden`, `TestDetMembershipAllFixtures`, `TestDetOCRAdjudication` | IoU orphan floor 3/5, adjudicated benign via OCR | + +#### 5. Two proof tiers (single source of truth) + +1. **Standalone library tier** — `internal/deepdoc/native`, run with + `-tags integration`. Exercises `RunDLA` / `RunTSR` / `RunDet` / + `RunOCRRec` / `RunOCRRecBatch` directly, proving the inference library + itself is equivalent. +2. **`DocAnalyzer` seam tier** — `internal/deepdoc/parser/pdf/inference/native_analyzer`, + run with `-tags "cgo integration"`. Exercises `infnative.NativeAnalyzer` + — the exact `deepdoctype.DocAnalyzer` implementation the PDF parser consumes + in production — proving equivalence at the integration point rather than only + inside the library. These are `TestAnalyzerDLAGolden`, `TestAnalyzerTSRGolden`, + `TestAnalyzerOCRRecGolden`, `TestAnalyzerDetGolden`. + +Both tiers call the same helpers in `golden.go`, so a change to the matching +math applies identically to library and seam. + +#### 6. Schema and determinism guards + +Equivalence is not only value-level; the **wire contract** is guarded too: + +- `TestWireSchemaMatchesGolden` asserts Go's `Wire()` JSON structure (top-level + key, nesting depth, leaf types) is identical to the deepdoc server adapter + shape for DLA / TSR / Det / OCR-rec, so a caller parsing Go output sees the + same schema as the Python service. +- `TestDLASessionReuse` / `TestTSRSessionReuse` / `TestOCRRecSessionReuse` + assert byte-identical `Wire()` across pooled-session runs (no stale tensor, + no cross-call contamination). +- `TestDetSessionPoolBounded` asserts the session pool set stays bounded under + many distinct page sizes (regression guard for a prior unbounded `sync.Map` + leak). + +These guards mean the Go port is a drop-in wire-compatible replacement, not just +numerically close on a fixed fixture set. + +#### 7. Live-service field diff (proves "Go == running service", not just "Go == golden") + +`TestWireSchemaMatchesGolden` (section 6) compares Go's `Wire()` against the +**re-serialized golden shape** — i.e. against a Python-shaped artifact. Because +the goldens are generated by the same Python `deepdoc.vision` recognizers the +service wraps (`ref_dla.py` / `ref_tsr.py` / `ref_ocr_rec.py`), the comparison +is field-for-field against the Python baseline the Go backend replaces. (The +former `TestWireVsLiveServer` live-HTTP diff was removed when the external +service path was dropped; the equivalence it established is preserved here as a +recorded result.) + +**Why this matters.** It answers the reviewer question "you proved Go == a +frozen Python snapshot, not Go == the running service" directly: the goldens +are produced by the actual service's recognizers. It also re-confirmed the +server is a thin wrapper — +its adapters only decode bytes, convert color space, run the same +`deepdoc.vision` recognizers with the same config the goldens were generated +against (DLA/TSR `thr=0.2`, OCR default pipeline), clamp bboxes, and map +label→class_id; there is no extra resize / DPI / rotation / server-side +rasterization. + +**Measured (Go ORT 1.23.2 vs the reference `deepdoc_server`; the same field +diff re-validated against the currently running 1.28.0 service by the parity +harness in §9):** + +| Task | Fixture | Server boxes | Go boxes | Match / Δ | +|------|---------|--------------|----------|-----------| +| DLA | page0 | 4 | 4 | matched 4/4, max Δ 0.006 px | +| DLA | mp_textbook_en_p0 | 13 | 13 | matched 13/13, max Δ 0.023 px | +| DLA | dla_2510_eq | 24 | 24 | matched 24/24, max Δ 0.007 px | +| TSR | table0 | 11 | 11 | matched 11/11, max Δ 0.509 px | +| TSR | tsr_table_rotation | 15 | 15 | matched 15/15, max Δ 0.886 px | +| Det | page0 | 15 | 15 | IoU orphan 0/0 | +| Det | mp_en_dense_p0 | 93 | 95 | IoU orphan 2/4 (within 3/5 floor) | +| OCR-rec | line0, line_cn | — | — | exact text match | + +Conclusion: the Go backend and the live `deepdoc_server` are field-for-field +equivalent on these fixtures; the only Det divergence is the same documented +3/5 contour floor already covered above. + +#### 8. End-to-end raster alignment (closes the "same-bytes-in" gap) + +Sections 1–7 prove "**given the same raster image bytes**, Go == Python". But in +production neither side receives a pre-rendered PNG: the Go server rasterizes PDF +pages with **pdfium** (`pdfium.RenderPage` @ **216 DPI**) and the Python deepdoc +pipeline rasterizes with **pdfplumber** (`page.to_image(resolution=72*zoomin, +antialias=True)`, `zoomin=3` ⇒ **216 DPI**). §Scope noted this as the one gap +outside the proved boundary. This section closes it by rasterizing the **same +real PDF page with both paths** and comparing the resulting boxes in source-pixel +coordinates — so the "same-bytes-in" assumption becomes a *measured* fact, not a +declaration. + +- **Go side:** `pdfium.RenderPage(pdf, page, 216)` → `NativeAnalyzer.DLA` / + `OCRDetect` / `TSR` (the exact `DocAnalyzer` seam production consumes). +- **Python side:** `ref_raster.py` renders the same page at 216 DPI via deepdoc's + OWN pdfplumber path (so it matches what the live `deepdoc_server` would rasterize), + then runs the real `deepdoc.vision` recognizers. +- Both render at **216 DPI**, so box coordinates land in the **same pixel space** + and are compared directly. Harness: `TestRasterAlignmentDLA` / + `TestRasterAlignmentDet` / `TestRasterAlignmentTSR` (analyzer suite, + `//go:build cgo && integration`). + +**Measured (6 real documents, 216 DPI both sides, Go pdfium render with +`FPDF_LCD_TEXT (0x02)` set — LCD sub-pixel text AA on top of pdfium's default +grayscale AA, to match pdfplumber's `antialias=True`):** + +| Task | Pages | Result | +|------|-------|--------| +| DLA | annual report p2 | **15/15**, worst max Δ **0.017 px** | +| DLA | annual report p8 | **14/14**, worst max Δ **0.025 px** | +| DLA | ZoomNeXt p1 | **15/15**, worst max Δ **0.011 px** | +| DLA | ZH-TW migration p3 | **24/24**, worst max Δ **0.031 px** | +| DLA | Three Kingdoms p1 | **18/18**, worst max Δ **0.006 px** | +| DLA | 15K606 p10 (figure/table) | **18/18**, worst max Δ **0.721 px** (AA-invariant — see note) | +| Det | annual report p2 | matched **93/93**, center-max **3.20 px**, corner-maxd **8.0 px**; IoU orphan 0/0 | +| Det | annual report p8 | matched **42/42**, center-max **3.35 px**, corner-maxd **12.0 px**; IoU orphan 0/0 | +| Det | ZoomNeXt p1 | matched **140/140**, center-max **3.35 px**, corner-maxd **3.0 px**; IoU orphan 0/0 | +| Det | ZH-TW migration p3 | matched **30/30**, center-max **0.50 px**, corner-maxd **1.0 px**; IoU orphan 0/0 | +| Det | Three Kingdoms p1 | matched **32/32**, center-max **1.50 px**, corner-maxd **3.0 px**; IoU orphan 0/0 | +| Det | 15K606 p10 | matched **55/55**, center-max **3.20 px**, corner-maxd **3.0 px**; IoU orphan 1/2 | +| TSR | annual report p2/p8, ZH-TW migration p3, Three Kingdoms p1 | **matched 100%** (117/117 cells), worst max Δ **2.700 px** (≤ 3.5px floor) | + +**Interpretation.** +- **DLA is fully closed end-to-end, and now near-pixel-exact on text pages**: + pdfium-vs-pdfplumber rasterization with AA enabled leaves ≤ 0.03 px of + coordinate drift on the four text/table pages (far inside the 3.5px floor), so + the "same-bytes-in" assumption for layout detection is effectively exact there. + The single technical-standard figure page (15K606 p10) stays at 0.721 px even + *with* AA on — confirming that residual is a vector/figure-render difference, + not a text-smoothing artifact. +- **Det is closed end-to-end on structure** (IoU orphan 0/0–1/2, no loss / + no hallucination). The per-box *center* distance is **sub-pixel**: a + greedy/Hungarian nearest-center analysis of the dumped boxes on annual report p2/p8 + gives median **0 px**, mean **< 0.5 px**, p90 **< 2.2 px**, max **< 5 px** — + i.e. there is **no translation / coordinate-origin offset** between the two + render paths (the earlier "8–12 px is a render-origin translation" guess, + inferred from the test log's `corner-maxd`, is falsified). What the test log + reports as `corner-maxd 8.0 / 12.0 px` is the *max per-corner* coordinate difference + (`MatchBothDirections` measures the worst of the 4 corners, not the center, + golden.go:191), and it is concentrated in **1–2 outlier text boxes per page** + whose quadrilaterals are slightly rotated/skewed differently. The source is the + same documented contour-boundary geometry already behind the 3/5 IoU orphan + floor: Go's `bilinearResize` vs cv2.resize interpolation at high-contrast text + edges, plus the Moore-neighbour vs cv2.findContours contour trace. AA does not + change it (identical before/after) because it is not an antialias artifact. + Every box still overlaps its twin at IoU ≥ 0.5, so it is a benign quad-skew + inside the documented Det floor, not a detection divergence. +- **TSR is closed end-to-end on ordinary/moderate real tables** (≤ 2.7 px). For + *dense full-page technical-standard tables* the coordinate floor and the model + itself can diverge further — see *Known model-floor limits* below for the + quantified bound and the one documented exception. + +The harness requires `uv run python3` with `deepdoc` + `pdfplumber` available; +if the Python oracle is absent the alignment tests **skip** (not fail), so CI +without the Python oracle still passes the rest of the native suite. + +#### 9. Per-page IoU diff harness (full-corpus scale) + +§1–8 measure equivalence on committed PNG/PDF fixtures. The per-page IoU harness +(formerly `inprocess_vs_service_iou_test.go`, `TestInProcessVsServiceIoUDiff`) +extended the measurement to **every page of every real PDF in the corpus** by +comparing the in-process `NativeAnalyzer` against the Python `deepdoc` service +output. That live-comparison test was removed with the external service path; +the recorded 35-PDF result below is retained as historical evidence. + +- renders each page with the production Go path (pdfium @216 DPI), +- runs `OCRDetect` on that exact image through the in-process + `NativeAnalyzer`, and compares the box set against the Python reference + (`deepdoc/vision` recognizer) output, +- greedily matches the two box sets by **rotated-quad polygon IoU** + (Sutherland–Hodgman intersection, matched at IoU ≥ 0.5), so the comparison no + longer depends on either backend emitting boxes in the same order or count, +- classifies every unmatched box as `split_merge` / `threshold_fragment` / + `partial_overlap` / `one_sided`, flags matched boxes that fail a tight-overlap + test (IoU < 0.9 or per-corner distance > 3 px) as `coord_drift`, and writes a + per-page overlay PNG (Go=red, Py=blue, drifted=magenta) plus a JSON report + under `testdata/output/render_compare/iou/`. + +**Measured (35 real PDFs, every page; Go ORT pinned to 1.28.0 to match the +Python-side runtime at measurement time):** + +| Metric | Value | +|---|---| +| Matched boxes (IoU ≥ 0.5) | **1,559** | +| Orphans (`one_sided`) | **1** (Go-only box in `10_numbering_patterns.pdf`) | +| `split_merge` / `threshold_fragment` / `partial_overlap` | **0** | +| Mean IoU over matched boxes | **0.969** | +| Worst IoU | 0.668 | +| Max per-corner distance | **21 px** (`16_dense_cjk.pdf`) | +| `coord_drift` (IoU < 0.9 or corner > 3 px) | **227 / 1,559 (14.6%)** | +| PDFs with zero drift | **12 / 35** (of which 7 are bit-exact, max corner 0 px) | +| Drift concentration | ~60% in `13_crosspage_table.pdf` | + +**Interpretation.** The two detectors agree on box membership for **>99.9%** of +all text regions across the full corpus — the fixture-level Det 3/5 orphan +floor (0.27%/0.44%) holds at larger scale (1/1559 ≈ 0.06%). There are **no** +split/merge/threshold-fragment differences; the entire divergence is +`coord_drift` on matched boxes, i.e. the same contour-boundary/resize geometry +quantified in §8 and *Why the divergence is bounded*. This harness doubles as +the cross-ORT guard: it passes with Go loading either the 1.23.2 or the 1.28.0 +runtime against the same models, corroborating *Runtime version*. + +### Evidence (measured) + +Latest `TestEquivalenceReport` (both tiers green; summary printed to the test log): + +| Task | Fixtures | Match | Max Δ | Status | +|------|----------|-------|-------|--------| +| DLA | 11 | 148/148 boxes (incl. equation + figure-caption classes, +3 real-document pages: CN annual report / ZH-TW migration / EN paper) | < 0.13 px | OK | +| TSR | 8 | 190/190 boxes (incl. projected-row-header class, +1 real annual-report table) | 2.155 px (≤ 10 px on a 4:1 aspect crop, structure preserved) | OK | +| OCR | 11 | exact text (EN / CJK / mixed / digits, + font variants, +3 real text-line crops: CN / ZH-TW / EN) | — | OK | +| Det | all | IoU box-membership orphan **3/5** (gold 1125 / go 1127) | — | OK (accepted floor) | + +The detection orphan boxes were adjudicated with OCR: cropping each orphan and +running OCR shows the regions still resolve to real text. Concretely, the +orphans split as **Python-only real text 0/3** (those 3 regions are empty +furniture on both sides) and **Go-only real text 3/5** (Go emits 3 text regions +the Python side does not). So the two outputs are **not byte-identical**: Go is +a *superset* on those few regions. This is a small, one-directional divergence +(Go finds extra text), not a loss — the downstream text consumer (dedup / +align) absorbs it, and no content is dropped. "Benign" here means "no content +loss", not "identical output". + +#### Det IoU orphan 3/5 — what it means, and how small it is + +**What "IoU orphan" means.** For Det we compare two *sets* of text boxes — the +Python golden set and the Go set — over each fixture. "IoU" (Intersection-over- +Union) here is computed on the **axis-aligned bounding box (AABB)** of each +quadrilateral (`iou()` in `golden.go:249`): `IoU = area(intersection) / +area(union)`. Two boxes are considered a match only if their *best* greedy IoU +is `≥ 0.5` (`MatchIoUBothDirections`, `golden.go:269`), matched independently in +both directions. An **orphan** is a box on one side whose best counterpart on +the other side has IoU `< 0.5` — i.e. no overlapping twin at all. So "3/5" +means: **3 Python golden boxes** have no Go twin at IoU ≥ 0.5 (Go "missed" +them), and **5 Go boxes** have no Python twin at IoU ≥ 0.5 (Go "extra"). This +isolates genuine box-membership divergence (a box split, merged, or +hallucinated) from mere coordinate drift — a box shifted 20 px but still +overlapping its twin scores high IoU and is **not** an orphan. + +**Sample volume.** The 3/5 is accumulated across **every** committed Det +fixture, not a cherry-picked subset. The corpus is **35 fixtures**, totalling +**1,125 golden text boxes** vs **1,127 Go text boxes** (logged by +`TestDetMembershipAllFixtures` as `TOTAL gold=1125 go=1127`). This spans blank +pages, degradation variants (noise / low-contrast / skew / tiny-text / CJK +vertical), and mixed-language dense pages. + +**Proportion (how dominant the agreement is).** + +| Side | Boxes | Matched (IoU ≥ 0.5) | Orphan | Orphan rate | +|------|-------|---------------------|--------|-------------| +| Python golden | 1,125 | 1,122 | 3 | **0.27%** | +| Go | 1,127 | 1,122 | 5 | **0.44%** | +| Both combined | 2,252 | 2,244 | 8 | **0.36%** | + +In other words, the two detectors agree on box membership for **>99.6%** of all +text regions; the residual is a handful of boxes whose contour geometry differs +just enough to drop below the 0.5 IoU hinge. + +**Why it is accepted, not a defect.** It is a *regression guard*, not a +zero-target: the test fails only if Go gets *worse* than the baseline (gold +orphan `> 6`, go orphan `> 8`, with slack 3 — see `native_integration_test.go:559`). +The 3/5 is the residue left after every real bug was fixed — stale goldens +(37/20, 42/13) and an R/B channel-swap in `normalizeCHW` (23/9) — leaving only +contour-tracer geometry (explained below). And the orphan boxes are +OCR-adjudicated benign (above): cropping each orphan and running OCR still +resolves real text, so the divergence never drops content. + +### Why the divergence is bounded and deterministic + +- **Det 3/5** — contour-tracer geometry. Go's pure-Go boundary follower + (Moore-neighbour, Suzuki-Abe style, `RETR_LIST`) selects boundary pixels + slightly differently from cv2's `findContours` at 8-connected diagonal / hole + junctions. 100% of the orphans are SCORE-FLIPs: a different convex hull → + different `minAreaRect` → `box_score_fast` crosses 0.5 at a handful of + regions on dense pages. It does not flip any segmentation. +- **≤ 1/255 residual** — ONNX Runtime's fixed-point `uint8` resize introduces a + pixel-level noise ceiling (max |Δ| = 1 gray level vs Go's float bilinear). It + is irreducible for any pure-float implementation and is far below the 0.5 / + 0.3 score thresholds, so it never changes a detection or score. + +Both effects are **deterministic and reproducible**, not random accuracy loss. + +### What is NOT proven here (honest boundaries) + +1. **Not bit-identical.** "Equivalent within bounded, accepted floors" is the + correct claim; "perfect / pixel-identical" is not. The Det 3/5 floor is a + known, accepted divergence. +2. **Production caller is wired (server only).** The server binary built with + `-tags cgo` registers the in-process backend via + `infnative.Register(...)` in `cmd/ragflow_server.go`; it fails fast at + startup unless ORT + models are available. The in-process backend is the only + DeepDoc backend — there is no external service path. The CLI binary is built + without `cgo` (no-op path). +3. **Coverage confirmation required.** Go implements {det, dla, tsr, ocr}. + Confirm the Python path being replaced uses only these recognizers (e.g. a + separate table-cell recognizer would not be covered). +4. **Runtime version.** Validated with onnxruntime 1.23.2 (see + `internal/common.DeepDocORTVersion`) **and** 1.28.0 (the version the running + Python service resolves); a controlled experiment showed identical detection + matching across both, so the ORT version is **not a sensitivity**. Re-run the + golden-comparison tests (§6 / §7) / parity harness (§9) only if the Python + side moves to a materially different ORT line. +5. **HTTP server shape.** If the goal is a standalone HTTP service mirroring + `deepdoc_server`, that surface is not built — only the in-process library and + a CLI (`main.go`) exist. + +### Known model-floor limits (documented, not hidden) + +The full-page real-table TSR floor is **measured end-to-end** (both production +raster paths) in Methodology §8 / `TestTSRFloorFullPageTables` and recorded in +the *Known model-floor limits (full-page real tables) — measured* subsection +under §3. That is the authoritative, quantified statement: ordinary/moderate +real tables ≤ 3.5px (worst 2.70px; dense annual-report ≤ 1.21px), and dense +technical-standard tables (15K606 p40) breaking the floor on both coordinate +drift **and** cell count (17/30, documented exception). + +The two excluded pages (`15K606` p40, `Xiamen Xiangyu annual report` p12) are retained in the +generator (`/tmp/gen_corpus.py`) for future investigation but are **not** in the +strict 3.5px fixture suite — p12 actually measures 0.37px (25/25) once rasterized +through the real production path, so only p40 remains a genuine hard case. + +### Reviewer follow-ups (prioritized, with status) + +These are the items raised by an independent review of this proof, ranked by +impact. "Closed" items are done in code or in this document. + +| ID | Item | Status | +|----|------|--------| +| P1-1 | **Live-service field diff** — confirm server is a thin wrapper (no extra preprocessing) and diff Go `Wire()` against the *real* `deepdoc_server` HTTP response. | **CLOSED** — server verified thin (adapters only decode + color-convert + clamp + label→class_id; config matches goldens: DLA/TSR `thr=0.2`, OCR default pipeline). The equivalence was established and recorded in Methodology §7 against the reference server (Go ORT 1.23.2 vs reference server 1.23.2; re-validated cross-version against the 1.28.0 runtime by §9). The `TestWireVsLiveServer` live-HTTP diff was later removed when the external service path was dropped from the codebase, but the verified-thin conclusion stands. | +| P1-5 | **ORT version** — record the Python-side ORT build, not just assume ABI compatibility. | **CLOSED** — Go runs **1.23.2** (`DeepDocORTVersion`); the currently running Python service resolves **1.28.0**. A controlled experiment (Go loading 1.28.0's `.so`) reproduced identical detection matching, so the version is not a sensitivity. Recorded in *Runtime version* and Methodology §9. | +| P2-2 | **Scope wording** — state that inference-boundary equivalence ≠ end-to-end PDF→chunk pipeline equivalence. | **CLOSED** — explicit "Boundary of this proof" paragraph added to Scope; PDF→raster and `PdfParser` downstream named as out-of-scope. | +| P2-3 | **Det 3/5 "benign" wording** — it is not identical output (Go emits 3 extra text regions). | **CLOSED** — reworded to "no content loss, not identical output"; Go is a *superset* on those regions, absorbed by downstream dedup. | +| P2-8 | **Corpus** — DLA/TSR/OCR coverage is thin (8 / 7 / 8; OCR are line crops, not full pages). | **CLOSED** — expanded with diverse full-page real-document fixtures: DLA +3 (`dla_real_cn_report` Xiamen Xiangyu annual report p2, `dla_real_zhtw` ZH-TW migration doc p3, `dla_real_en_paper` ZoomNeXt paper p1), TSR +1 (`tsr_real_report` Xiamen Xiangyu annual report p8), OCR +3 (`line_real_cn` Three Kingdoms p1, `line_real_zhtw` ZH-TW migration p2, `line_real_en` ZoomNeXt p1). All pass sub-pixel / exact-text. The one genuine hard case (15K606 p40 dense technical-standard table) is a documented exception, not in the strict 3.5px suite — see *Known model-floor limits*. | +| P0 | **Model snapshot hash lock** — `MODEL_DIR` must be pinned to the same `InfiniFlow/deepdoc` snapshot as the frozen Python side; the proof must fail if it drifts. | **CLOSED** — enforced. `modelSnapshotHashes` (sha256 of `det.onnx`, `layout.onnx`, `tsr.onnx`, `rec.onnx`, `ocr.res`) is checked by `TestModelSnapshotHash` and at the top of `TestEquivalenceReport`; Fatal on any mismatch. Both repo copies verified byte-identical. Update the table only when the snapshot is intentionally upgraded, and regenerate every golden in the same change. | +| P3 | **Concurrency correctness** — parallel vs serial inference must give identical results (thread-safety is correctness, not performance). | **CLOSED** — `TestInferenceConcurrencyConsistent` drives DLA/TSR/OCR-rec/Det once serially (baseline wire) then 8× concurrently and asserts every concurrent run is byte-identical to the serial baseline. Proves the shared model-session pool is race-free and contamination-free under parallel load (complementing `TestDetSessionPoolBounded` which guards pool *size*). | +| **E2E-1** | **End-to-end raster alignment** — the "same raster bytes in" premise is a declaration: production rasterizes via different engines (Go pdfium @216 DPI vs Python pdfplumber @216 DPI). Prove the two render paths yield equivalent boxes. | **CLOSED** — `TestRasterAlignmentDLA/Det/TSR` rasterize the *same* real PDF page with both paths (pdfium vs deepdoc's own pdfplumber) at 216 DPI and compare boxes in source-pixel space. After enabling LCD text AA in the Go pdfium render to match pdfplumber: DLA **104/104** matched with worst max Δ **0.721 px** on the figure page and **≤ 0.03 px** on the four text pages (near-pixel-exact); Det IoU orphan 0/0–1/2 (inside 3/5); TSR on real-table pages **117/117** matched (worst 2.700px). The "same-bytes-in" premise is now **measured**, not assumed. Measured numbers in Methodology §8. Note (measured): the Det test-log `corner-maxd 8–12 px` is the max *per-corner* difference on 1–2 skewed outlier boxes; per-box **center** distance is sub-pixel (median 0, mean <0.5px, p90 <2.2px, max <5px) — there is **no render-origin translation**; the residual is contour-boundary quad-skew, the same source as the 3/5 IoU orphan floor. | +| **E2E-2** | **Quantify TSR floor on full-page real tables** — give an empirical upper bound ("N pages, worst X px") instead of two hand-picked excluded examples. | **CLOSED** — `TestTSRFloorFullPageTables` runs TSR on whole-page real tables through both raster paths: moderate tables ≤ 3.5px (Xiamen Xiangyu annual report p8 1.21px, p12 0.37px; Three Kingdoms p1 2.70px); dense technical-standard 15K606 p40 is the documented exception (17/30, model-level cell-count divergence, regression-guarded). Empirical bound recorded in *Known model-floor limits (full-page real tables) — measured*. | + +**Withdrawn critique.** One review point claimed the two proof tiers "use +different inference code paths / decoders, so preprocessing may differ." This is +**factually incorrect**: `infnative.NativeAnalyzer.DLA/TSR/OCRDetect/OCRRecognize` +all delegate to the *same* `native.RunDLA` / `RunDet` / `RunTSR` / `RunOCRRec` +(`native_analyzer.go:101-181`); only the initial PNG→pixel decode differs, which +is the already-documented `≤ 1/255` residual. The standalone and seam tiers +therefore share one inference path. diff --git a/docker/.env b/docker/.env index 6fa933db82..188862cbb9 100644 --- a/docker/.env +++ b/docker/.env @@ -411,16 +411,9 @@ THREAD_POOL_MAX_WORKERS=128 DISABLE_PASSWORD_LOGIN=false # ----------------------------------------------------------------------------- -# DeepDoc OSS Vision Service +# DeepDoc (in-process) # ----------------------------------------------------------------------------- -# URL for the deepdoc vision API (DLA, OCR, TSR) served by OSS ONNX models. -# The `deepdoc` service defined in docker-compose.yml provides this endpoint. -# When unset, the parser falls back to inline ONNX Runtime inference. - -# Comment existing COMPOSE_PROFILES and uncomment below if need deepdoc service. -# COMPOSE_PROFILES=${DOC_ENGINE},${DEVICE},deepdoc - -# DEEPDOC_URL=http://deepdoc:9390 - -# Docker image for the OSS deepdoc service. CPU-only; uses ONNX Runtime. -# DEEPDOC_IMAGE=deepdoc_oss:latest +# DeepDoc layout analysis (DLA), OCR, and TSR run in-process inside the RAGFlow +# server using ONNX Runtime — there is no separate DeepDoc service. Set ORT / +# model overrides if needed: +# DEEPDOC_MODEL_DIR=/path/to/InfiniFlow/deepdoc diff --git a/docker/README.md b/docker/README.md index 4c61796fa5..682c4550e3 100644 --- a/docker/README.md +++ b/docker/README.md @@ -100,16 +100,15 @@ The [.env](./.env) file contains important environment variables for Docker. > - `RAGFLOW_IMAGE=swr.cn-north-4.myhuaweicloud.com/infiniflow/ragflow:nightly` or, > - `RAGFLOW_IMAGE=registry.cn-hangzhou.aliyuncs.com/infiniflow/ragflow:nightly`. -### DeepDoc Vision Service (OSS) +### DeepDoc (in-process) -- `DEEPDOC_URL` - URL for the deepdoc vision API serving DLA (layout analysis), OCR (text detection/recognition), and TSR (table structure recognition). The `deepdoc` service in `docker-compose.yml` provides this endpoint. Defaults to `http://deepdoc:9390`. When unset, the parser falls back to inline ONNX Runtime inference. - - > The OSS deepdoc service runs on CPU using ONNX Runtime models. No GPU required. - > API endpoints: `GET /health`, `GET /model`, `POST /predict/dla`, `POST /predict/tsr`, `POST /predict/ocr`. - -- `DEEPDOC_IMAGE` - Docker image for the OSS deepdoc service. Defaults to `infiniflow/deepdoc_oss:latest`. +DeepDoc layout analysis (DLA), OCR (text detection/recognition), and table +structure recognition (TSR) run **in-process** inside the RAGFlow server using +ONNX Runtime — there is no separate DeepDoc service to deploy. ONNX Runtime is +statically linked into the server binary (resolved at runtime via dlopen(NULL); +no `libonnxruntime.so` is required) and the models are loaded at runtime; +`DEEPDOC_MODEL_DIR` overrides the default model directory, which falls back to +the `ragflow_deps/download_deps.py` snapshot. ### Timezone @@ -189,13 +188,6 @@ Before setting `DOC_ENGINE=oceanbase`, make sure the host OS allows the file des - `host`: The API server's IP address inside the Docker container. Defaults to `0.0.0.0`. - `http_port`: The API server's serving port inside the Docker container. Defaults to `9380`. -- `deepdoc` - The OSS DeepDoc vision service provides DLA, OCR, and TSR inference via ONNX Runtime. - Defined in `docker-compose.yml`, it is started automatically as a dependency of `ragflow-cpu` and `ragflow-gpu`. - - `image`: Docker image. Defaults to `infiniflow/deepdoc_oss:latest`. - - `port`: Serving port inside the container. Defaults to `9390`. - - Health check: `curl -f http://localhost:9390/health` every 10s. - - `mysql` - `name`: The MySQL database name. Defaults to `rag_flow`. - `user`: The username for MySQL. diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 2cf741a42b..a88420c91a 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -2,22 +2,6 @@ include: - ./docker-compose-base.yml # To ensure that the container processes the locally modified `service_conf.yaml.template` instead of the one included in its image, you need to mount the local `service_conf.yaml.template` to the container. services: - deepdoc: - image: ${DEEPDOC_IMAGE:-deepdoc_oss:latest} - profiles: - - deepdoc - build: - context: .. - dockerfile: Dockerfile_deepdoc_oss - networks: - - ragflow - restart: unless-stopped - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:9390/health"] - interval: 10s - timeout: 10s - retries: 60 - ragflow-cpu: depends_on: mysql: diff --git a/go.mod b/go.mod index 1dc791b0bd..d07769e6e6 100644 --- a/go.mod +++ b/go.mod @@ -219,6 +219,7 @@ require ( github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect github.com/xuri/efp v0.0.1 // indirect github.com/xuri/nfp v0.0.2-0.20250530014748-2ddeb826f9a9 // indirect + github.com/yalue/onnxruntime_go v1.23.0 github.com/yargevad/filepathx v1.0.0 // indirect github.com/yuin/gopher-lua v1.1.1 // indirect go.mongodb.org/mongo-driver/v2 v2.6.0 // indirect @@ -249,3 +250,9 @@ require ( replace github.com/infiniflow/infinity-go-sdk => github.com/infiniflow/infinity/go v0.0.0-20260806040857-d755c5ad25d9 replace github.com/AkmalOt/gomsg => github.com/xugangqiang/gomsg v0.0.0-20260407083308-985c3a1a76b7 + +// onnxruntime_go is forked to github.com/xugangqiang/onnxruntime_go (static-only: +// resolves OrtGetApiBase via dlopen(NULL), no .so fallback). The fork is published at +// that remote with tag v1.23.0; we pin a remote replace matching the gomsg pattern above +// so the vendored tree no longer needs to live in-tree. +replace github.com/yalue/onnxruntime_go => github.com/xugangqiang/onnxruntime_go v1.23.0 diff --git a/go.sum b/go.sum index ac4b75fe9c..4c7646356f 100644 --- a/go.sum +++ b/go.sum @@ -564,12 +564,16 @@ github.com/x-cray/logrus-prefixed-formatter v0.5.2 h1:00txxvfBM9muc0jiLIEAkAcIMJ github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE= github.com/xugangqiang/gomsg v0.0.0-20260407083308-985c3a1a76b7 h1:iX9RUmlLw+Hoe78WV5vlm+JjIZUCzdRn2y6tWNxZTeY= github.com/xugangqiang/gomsg v0.0.0-20260407083308-985c3a1a76b7/go.mod h1:TCjWm+lo7de/zYHb8rHfVSaKxOP4tZNrnRUzkqV4r28= +github.com/xugangqiang/onnxruntime_go v1.23.0 h1:Oo//W+NHBkDwjjyOa9US8G9ryae5cVRbAMgXR6wxohw= +github.com/xugangqiang/onnxruntime_go v1.23.0/go.mod h1:b4X26A8pekNb1ACJ58wAXgNKeUCGEAQ9dmACut9Sm/4= github.com/xuri/efp v0.0.1 h1:fws5Rv3myXyYni8uwj2qKjVaRP30PdjeYe2Y6FDsCL8= github.com/xuri/efp v0.0.1/go.mod h1:ybY/Jr0T0GTCnYjKqmdwxyxn2BQf2RcQIIvex5QldPI= github.com/xuri/excelize/v2 v2.11.0 h1:HxaEFl6sRN2+8J5a8HaKq+0M4FsjBGMnWWtjOCPSG88= github.com/xuri/excelize/v2 v2.11.0/go.mod h1:jxFLbzaIwGQ5ufFNvYfUOHqXhfPaNmP14KWfmNz2Uak= github.com/xuri/nfp v0.0.2-0.20250530014748-2ddeb826f9a9 h1:+C0TIdyyYmzadGaL/HBLbf3WdLgC29pgyhTjAT/0nuE= github.com/xuri/nfp v0.0.2-0.20250530014748-2ddeb826f9a9/go.mod h1:WwHg+CVyzlv/TX9xqBFXEZAuxOPxn2k1GNHwG41IIUQ= +github.com/yalue/onnxruntime_go v1.23.0 h1:Hin0mFphwGOeT7xEQrAIi/p2O6ngmSy4uz0yXkC9yCw= +github.com/yalue/onnxruntime_go v1.23.0/go.mod h1:b4X26A8pekNb1ACJ58wAXgNKeUCGEAQ9dmACut9Sm/4= github.com/yargevad/filepathx v1.0.0 h1:SYcT+N3tYGi+NvazubCNlvgIPbzAk7i7y2dwg3I5FYc= github.com/yargevad/filepathx v1.0.0/go.mod h1:BprfX/gpYNJHJfc35GjRRpVcwWXS89gGulUIU5tK3tA= github.com/yfedoseev/office_oxide/go v0.1.8 h1:Q6aQrrs0pN/B6fBoWN6PN95QSQ7EKGMb5HExh04mJJc= diff --git a/internal/common/environments.go b/internal/common/environments.go index bef2653a17..64111abd11 100644 --- a/internal/common/environments.go +++ b/internal/common/environments.go @@ -18,6 +18,7 @@ package common import ( "os" + "path/filepath" "strings" ) @@ -31,7 +32,6 @@ func GetEnvSmall(key string) string { // environment variables const ( - EnvDeepDocURL = "DEEPDOC_URL" EnvTensorrtDLAServer = "TENSORRT_DLA_SVR" EnvRAGFlowTTSCacheTTLSeconds = "RAGFLOW_TTS_CACHE_TTL_SECONDS" EnvComponentExecTimeout = "COMPONENT_EXEC_TIMEOUT" @@ -139,7 +139,6 @@ const ( EnvBatchCompareFilter = "BATCH_COMPARE_FILTER" EnvBatchCompareCSV = "BATCH_COMPARE_CSV" EnvPYOCRSuffix = "PY_OCR_SUFFIX" - EnvOSSDeepDocURL = "OSSDEEPDOC_URL" EnvUpdateGolden = "UPDATE_GOLDEN" EnvBatchParityFilter = "BATCH_PARITY_FILTER" EnvBatchParityVariant = "BATCH_PARITY_VARIANT" @@ -211,4 +210,61 @@ const ( EnvGmailWebOAuthRedirectURI = "GMAIL_WEB_OAUTH_REDIRECT_URI" EnvGoogleDriveWebOAuthRedirectURI = "GOOGLE_DRIVE_WEB_OAUTH_REDIRECT_URI" EnvSpacyModelDir = "SPACY_MODEL_DIR" + + // EnvDeepDocModelDir points the in-process (Go) DeepDoc backend at the + // model snapshot (see common.DeepDocModelFiles); mirrors + // deepdoc_server.py's --model-dir (default rag/res/deepdoc). + EnvDeepDocModelDir = "DEEPDOC_MODEL_DIR" + // EnvDeepDocDropScore overrides the confidence threshold below which the + // in-process (Go) DeepDoc backend blanks recognized text while preserving + // the real score. It MUST match the Python inference service's + // Recognizer.drop_score (deepdoc/vision/ocr.py, default 0.5) so both + // backends apply the same text-blanking contract. + EnvDeepDocDropScore = "DEEPDOC_DROP_SCORE" ) + +// DeepDocModelFiles is the single source of truth for the weights the +// in-process (Go) DeepDoc backend and the Python DeepDoc service both +// require. cmd/ resolves the model directory against it; infnative +// validates file presence against it. Order is insignificant (callers do +// set-membership checks); keep it stable so logs and diffs stay readable. +// +// External consumers that re-list these names must stay in sync: +// - .github/workflows/deepdoc-drift.yml (MODEL_FILES) +// - deepdoc/server/download_deps.py (FILES) +var DeepDocModelFiles = []string{ + "det.onnx", + "layout.onnx", + "tsr.onnx", + "rec.onnx", + "ocr.res", +} + +// HasModelFiles reports whether dir contains every file listed in +// DeepDocModelFiles. It is the single presence check shared by the server +// (cmd/ragflow_server.go) and the in-process analyzer (infnative: +// NewAnalyzer / canServe); those call sites must not re-roll this loop. +func HasModelFiles(dir string) bool { + for _, f := range DeepDocModelFiles { + if _, err := os.Stat(filepath.Join(dir, f)); err != nil { + return false + } + } + return true +} + +// DeepDocORTVersion is the onnxruntime native release the in-process (Go) +// DeepDoc backend is built and tested against (e.g. "1.23.2"). It is the +// single source for the download URL and extracted dir name used across Go +// and Python. The Go binding (github.com/yalue/onnxruntime_go, forked to +// github.com/xugangqiang/onnxruntime_go) and the pip onnxruntime== pin must +// track this MINOR version: the binding uses its own release numbering +// (v1.23.0 <-> ORT 1.23.x) but is ABI-compatible with this native release on +// the same minor line. ONNX Runtime is linked statically (libonnxruntime.a), +// so there is no .so / SONAME at runtime. +// +// To bump ORT: update DeepDocORTVersion (Go) AND ORT_VERSION in +// ragflow_deps/download_deps.py AND the onnxruntime/onnxruntime-gpu pins in +// pyproject.toml + .github/workflows/deepdoc-drift.yml, and refresh the +// onnxruntime_go binding minor in go.mod. +const DeepDocORTVersion = "1.23.2" diff --git a/internal/deepdoc/native/clipper_offset.go b/internal/deepdoc/native/clipper_offset.go new file mode 100644 index 0000000000..7c14d95819 --- /dev/null +++ b/internal/deepdoc/native/clipper_offset.go @@ -0,0 +1,285 @@ +//go:build cgo + +package native + +// clipper_offset.go — faithful pure-Go port of Clipper1's ClipperOffset +// (Angus Johnson, MIT) restricted to the single case DBNet's unclip needs: +// JoinType.JT_ROUND + EndType.ET_CLOSEDPOLYGON on one convex quad. +// +// Why this exists: pyclipper (the Python binding the deepdoc oracle uses) is +// Clipper1, which works entirely in INTEGER coordinates. The float quad is +// first cast to int64 by truncation toward zero (IntPoint's cInt cast), the +// offset is computed with round-half-away (C++ Round == Go math.Round), and +// the result is returned as int64 coordinates. Our earlier hand-rolled +// roundOffset operated on the float coordinates directly, so its corners +// settled at a different radius than pyclipper's on slightly-skewed real text +// boxes (~1.4px pre-scale; ~3-4px at source). Porting the actual integer +// algorithm bit-reproduces pyclipper. +// +// Scope note: the Clipper boolean union "clean-up" that Execute runs after +// DoOffset is omitted. For a simple convex quad expanded outward by a small +// positive delta the offset path is already a simple (non-self-intersecting) +// polygon, and the union returns it unchanged. The final minAreaRect + +// getMiniBoxes normalization is invariant to any vertex reordering the union +// might do, so returning the offset path directly matches pyclipper's +// Execute(solution, delta)[0]. + +import "math" + +// --- Clipper1 constants (clipper.cpp) --- +var ( + clipperPi = 3.141592653589793238 + clipperTwoPi = clipperPi * 2 + clipperDefArcTol = 0.25 +) + +// cInt mirrors Clipper1's signed 64-bit coordinate type. +type cInt = int64 + +type cIntPt struct{ X, Y cInt } + +type dPoint struct{ X, Y float64 } + +// clipperRound mirrors Clipper1's inline Round: round-half-away-from-zero. +// Go's math.Round has identical semantics (2.5->3, -2.5->-3), so the int64 +// cast after it reproduces `static_cast(val ± 0.5)`. +func clipperRound(v float64) cInt { + return cInt(math.Round(v)) +} + +// getUnitNormal mirrors Clipper1 GetUnitNormal: the LEFT unit normal of the +// directed edge p1->p2. +func getUnitNormal(p1, p2 cIntPt) dPoint { + if p2.X == p1.X && p2.Y == p1.Y { + return dPoint{0, 0} + } + dx := float64(p2.X - p1.X) + dy := float64(p2.Y - p1.Y) + f := 1.0 / math.Sqrt(dx*dx+dy*dy) + dx *= f + dy *= f + return dPoint{dy, -dx} +} + +// clipperArea mirrors Clipper1 Area (shoelace, sign convention as in C++). +func clipperArea(poly []cIntPt) float64 { + n := len(poly) + if n < 3 { + return 0 + } + var a float64 + for i, j := 0, n-1; i < n; i++ { + a += float64(poly[j].X+poly[i].X) * float64(poly[j].Y-poly[i].Y) + j = i + } + return -a * 0.5 +} + +func clipperOrientation(poly []cIntPt) bool { return clipperArea(poly) >= 0 } + +func reversePath(p *[]cIntPt) { + for i, j := 0, len(*p)-1; i < j; i, j = i+1, j-1 { + (*p)[i], (*p)[j] = (*p)[j], (*p)[i] + } +} + +// dedupClosed drops consecutive duplicate vertices (including the wrap-around +// duplicate) the way ClipperOffset::AddPath does for a closed path. +func dedupClosed(in []cIntPt) []cIntPt { + n := len(in) + for n > 1 && in[0] == in[n-1] { + in = in[:n-1] + n-- + } + out := in[:0] + last := cIntPt{1 << 62, 1 << 62} // sentinel never equal to a real point + for _, p := range in { + if p != last { + out = append(out, p) + last = p + } + } + return out +} + +// clipperOffsetState holds the per-call mutable state of ClipperOffset. +type clipperOffsetState struct { + src []cIntPt + normals []dPoint + delta float64 + msin, mcos float64 + stepsPerRad float64 + sinA float64 +} + +// clipperOffset mirrors DBNet db_postprocess.unclip: delta = poly.area * ratio +// / poly.length is computed from the FLOAT box (Shapely/Polygon semantics), and +// the offset is run on the FLOAT box truncated to int64 — exactly what pyclipper +// does internally (IntPoint cInt cast). Returns the expanded polygon as int64 +// coordinates, matching pyclipper's integer output. +func clipperOffset(box [4]pt, ratio float64) []pt { + area := math.Abs(polygonArea(box[:])) + perim := polygonPerimeter(box[:]) + if perim == 0 { + return box[:] + } + delta := area * ratio / perim + + // Truncate float coords toward zero to int64 — pyclipper's IntPoint cast. + src := make([]cIntPt, 4) + for i := range box { + src[i] = cIntPt{cInt(math.Trunc(box[i].X)), cInt(math.Trunc(box[i].Y))} + } + offset := clipperDoOffset(src, delta) + out := make([]pt, len(offset)) + for i := range offset { + out[i] = pt{X: float64(offset[i].X), Y: float64(offset[i].Y)} + } + return out +} + +// clipperDoOffset mirrors ClipperOffset::DoOffset for a single +// ET_CLOSEDPOLYGON path with JT_ROUND. +func clipperDoOffset(src []cIntPt, delta float64) []cIntPt { + contour := dedupClosed(src) + if len(contour) < 3 { + return src + } + // FixOrientations: for a single closed polygon, reverse it if its area is + // negative so that a positive delta expands outward. + if !clipperOrientation(contour) { + reversePath(&contour) + } + + n := len(contour) + normals := make([]dPoint, n) + for j := 0; j < n-1; j++ { + normals[j] = getUnitNormal(contour[j], contour[j+1]) + } + normals[n-1] = getUnitNormal(contour[n-1], contour[0]) + + st := &clipperOffsetState{src: contour, normals: normals, delta: delta} + + // Arc step count (offset_triginometry2.svg in the Clipper docs). + y := clipperDefArcTol + if y > math.Abs(delta)*clipperDefArcTol { + y = math.Abs(delta) * clipperDefArcTol + } + steps := clipperPi / math.Acos(1-y/math.Abs(delta)) + if steps > math.Abs(delta)*clipperPi { + steps = math.Abs(delta) * clipperPi + } + st.msin = math.Sin(clipperTwoPi / steps) + st.mcos = math.Cos(clipperTwoPi / steps) + st.stepsPerRad = steps / clipperTwoPi + if delta < 0 { + st.msin = -st.msin + } + + dest := make([]cIntPt, 0, n*8) + k := n - 1 + for j := 0; j < n; j++ { + st.offsetPoint(&dest, j, &k) + } + // Clipper1's Execute runs a union cleanup that (a) removes consecutive + // duplicate vertices and (b) removes collinear vertices (CleanPolygons). + // DoOffset can emit the same point twice where edge normals coincide, and + // the trailing edge-normal vertex at an axis-aligned corner sits exactly + // on the straight offset edge. Stripping both makes the returned path + // match pyclipper vertex-for-vertex. + return cleanCollinear(dedupClosed(dest)) +} + +// cleanCollinear drops vertices that are collinear with their neighbours on +// the closed polygon (cross product of the two incident edges == 0). This +// mirrors Clipper1's CleanPolygons, which removes the trailing edge-normal +// vertex at an axis-aligned corner (it lies on the straight offset edge) +// while keeping it on a skewed corner. Removing a collinear vertex does not +// change the polygon's minAreaRect. +func cleanCollinear(in []cIntPt) []cIntPt { + out := in + for { + n := len(out) + if n < 3 { + return out + } + kept := make([]cIntPt, 0, n) + removed := false + for i := 0; i < n; i++ { + prev := out[(i-1+n)%n] + cur := out[i] + next := out[(i+1)%n] + cx := (cur.X-prev.X)*(next.Y-cur.Y) - (cur.Y-prev.Y)*(next.X-cur.X) + if cx == 0 { + removed = true + continue + } + kept = append(kept, cur) + } + if !removed { + return out + } + out = kept + } +} + +// offsetPoint mirrors ClipperOffset::OffsetPoint (JT_ROUND branch). The +// non-standard "<1px turn" short-circuit that the earlier port had is gone: +// every convex corner runs DoRound exactly as Clipper1 does, which is what +// makes Go's integer offset polygon match pyclipper vertex-for-vertex. +func (st *clipperOffsetState) offsetPoint(dest *[]cIntPt, j int, k *int) { + n := st.normals + st.sinA = n[*k].X*n[j].Y - n[j].X*n[*k].Y + if st.sinA > 1.0 { + st.sinA = 1.0 + } else if st.sinA < -1.0 { + st.sinA = -1.0 + } + + if st.sinA*st.delta < 0 { + // reflex corner: insert the original vertex between the two edge + // offsets. + *dest = append(*dest, + cIntPt{clipperRound(float64(st.src[j].X) + n[*k].X*st.delta), + clipperRound(float64(st.src[j].Y) + n[*k].Y*st.delta)}) + *dest = append(*dest, st.src[j]) + *dest = append(*dest, + cIntPt{clipperRound(float64(st.src[j].X) + n[j].X*st.delta), + clipperRound(float64(st.src[j].Y) + n[j].Y*st.delta)}) + } else { + st.doRound(dest, j, *k) + } + *k = j +} + +// doRound mirrors ClipperOffset::DoRound (JT_ROUND): for each convex corner it +// emits the incoming edge normal (normals[k]), then rotates the normal one arc +// step at a time emitting a vertex per step, and finally emits the outgoing +// edge normal (normals[j]). The arc step count uses round-half-away (same as +// Clipper1's Round), which makes the per-corner arc sampling match pyclipper +// and keeps the enclosing minAreaRect exact. The trailing normals[j] vertex is +// kept here; it is later dropped by cleanCollinear when it sits exactly on the +// straight offset edge (as Clipper1's CleanPolygons does for axis-aligned +// corners), so the returned path matches pyclipper vertex-for-vertex. +func (st *clipperOffsetState) doRound(dest *[]cIntPt, j, k int) { + n := st.normals + cosA := n[k].X*n[j].X + n[k].Y*n[j].Y + a := math.Atan2(st.sinA, cosA) + steps := int(math.Max(float64(clipperRound(st.stepsPerRad*math.Abs(a))), 1)) + + X := n[k].X + Y := n[k].Y + for i := 0; i < steps; i++ { + *dest = append(*dest, cIntPt{ + clipperRound(float64(st.src[j].X) + X*st.delta), + clipperRound(float64(st.src[j].Y) + Y*st.delta), + }) + X2 := X + X = X*st.mcos - st.msin*Y + Y = X2*st.msin + Y*st.mcos + } + *dest = append(*dest, cIntPt{ + clipperRound(float64(st.src[j].X) + n[j].X*st.delta), + clipperRound(float64(st.src[j].Y) + n[j].Y*st.delta), + }) +} diff --git a/internal/deepdoc/native/clipper_offset_test.go b/internal/deepdoc/native/clipper_offset_test.go new file mode 100644 index 0000000000..3bb9c1652f --- /dev/null +++ b/internal/deepdoc/native/clipper_offset_test.go @@ -0,0 +1,235 @@ +//go:build cgo + +package native + +import ( + "encoding/json" + "math" + "os" + "testing" +) + +// matchPolygon reports the max distance between the two polygons when matched +// as unordered point sets (Clipper's Execute may rotate the starting vertex and +// the union may drop interior arc points, so we compare geometrically, not by +// index). +func matchPolygon(got []pt, want [][]int64) float64 { + maxd := 0.0 + used := make([]bool, len(want)) + for _, g := range got { + best := 1e18 + for i, w := range want { + if used[i] { + continue + } + d := math.Hypot(g.X-float64(w[0]), g.Y-float64(w[1])) + if d < best { + best = d + } + } + if best > maxd { + maxd = best + } + } + // also ensure every want point is covered by some got point + for _, w := range want { + best := 1e18 + for _, g := range got { + d := math.Hypot(g.X-float64(w[0]), g.Y-float64(w[1])) + if d < best { + best = d + } + } + if best > maxd { + maxd = best + } + } + return maxd +} + +// TestClipperOffsetMatchesPyclipper validates the pure-Go Clipper1 port against +// pyclipper (the deepdoc oracle) box-by-box on real pre-unclip quads captured +// from deepdoc's TextDetector on page0.jpg: +// 1. the offset polygon matches pyclipper's Execute output geometrically, and +// 2. minAreaRect(getMiniBoxes(offset)) matches deepdoc's pre-scale quad. +func TestClipperOffsetMatchesPyclipper(t *testing.T) { + raw, err := os.ReadFile("testdata/clipper_quads4.json") + if err != nil { + t.Skipf("fresh pyclipper oracle not found: %v", err) + } + var data struct { + Quads []struct { + Box [][]float64 `json:"box"` + Poly [][][]int64 `json:"poly"` + Distance float64 `json:"distance"` + PreScale [][]float64 `json:"pre_scale"` + } `json:"quads"` + } + if err := json.Unmarshal(raw, &data); err != nil { + t.Fatalf("parse fixture: %v", err) + } + + var maxPoly, maxRect float64 + var dbg []struct { + Box [4][2]float64 `json:"box"` + Got [][2]float64 `json:"got"` + Want [][]int64 `json:"want"` + } + for qi, q := range data.Quads { + if len(q.Box) != 4 { + t.Fatalf("quad %d: expected 4 points, got %d", qi, len(q.Box)) + } + var box [4]pt + for i := range q.Box { + box[i] = pt{X: q.Box[i][0], Y: q.Box[i][1]} + } + got := clipperOffset(box, detUnclipRatio) + + if os.Getenv("DUMP_CLIPPER") != "" { + dbg = append(dbg, struct { + Box [4][2]float64 `json:"box"` + Got [][2]float64 `json:"got"` + Want [][]int64 `json:"want"` + }{ + Box: [4][2]float64{{box[0].X, box[0].Y}, {box[1].X, box[1].Y}, {box[2].X, box[2].Y}, {box[3].X, box[3].Y}}, + Got: func() [][2]float64 { + p := make([][2]float64, len(got)) + for i := range got { + p[i] = [2]float64{got[i].X, got[i].Y} + } + return p + }(), + Want: q.Poly[0], + }) + } + + // (1) offset polygon vs pyclipper output (as a point set). The + // faithful Clipper1 port must reproduce pyclipper's integer polygon + // vertex-for-vertex, so the residual is sub-pixel. + if len(q.Poly) > 0 { + d := matchPolygon(got, q.Poly[0]) + if d > maxPoly { + maxPoly = d + } + if d > 1.0 { + t.Errorf("quad %d: offset polygon diverges from pyclipper by %.2f px (n_got=%d n_want=%d)", + qi, d, len(got), len(q.Poly[0])) + } + } + + // (2) minAreaRect + getMiniBoxes vs deepdoc pre-scale quad (the + // post-unclip quad, in resized coords). Must match sub-pixel. + if len(q.PreScale) == 4 { + rect, _ := minAreaRect(got) + mb := getMiniBoxes(rect) + for i := 0; i < 4; i++ { + dx := math.Abs(mb[i].X - q.PreScale[i][0]) + dy := math.Abs(mb[i].Y - q.PreScale[i][1]) + d := math.Max(dx, dy) + if d > maxRect { + maxRect = d + } + if d > 0.75 { + t.Errorf("quad %d pt %d: rect got (%.2f,%.2f) want (%.2f,%.2f) diff=%.3f", + qi, i, mb[i].X, mb[i].Y, q.PreScale[i][0], q.PreScale[i][1], d) + } + } + } + } + t.Logf("compared %d quads: max offset-polygon diff = %.3f px, max pre-scale rect diff = %.3f px", + len(data.Quads), maxPoly, maxRect) + if os.Getenv("DUMP_CLIPPER") != "" { + _ = json.NewEncoder(os.Stdout).Encode(dbg) // also write to file below + if b, err := json.Marshal(dbg); err == nil { + _ = os.WriteFile("/tmp/go_clipper_out.json", b, 0o644) + } + } +} + +// TestTuneArcTol sweeps clipperDefArcTol to find the value whose Clipper1 port +// best reproduces pyclipper's integer offset polygon (and therefore the +// post-unclip rect) on the 15 real pre-unclip quads. Run: +// +// go test ./native/ -run TestTuneArcTol -v +func TestTuneArcTol(t *testing.T) { + raw, err := os.ReadFile("testdata/clipper_quads4.json") + if err != nil { + t.Skipf("oracle not found: %v", err) + } + var data struct { + Quads []struct { + Box [][]float64 `json:"box"` + Poly [][][]int64 `json:"poly"` + PreScale [][]float64 `json:"pre_scale"` + } `json:"quads"` + } + if err := json.Unmarshal(raw, &data); err != nil { + t.Fatalf("parse: %v", err) + } + for _, tol := range []float64{0.15, 0.17, 0.18, 0.20, 0.22, 0.25} { + clipperDefArcTol = tol + var maxPoly, maxRect float64 + for _, q := range data.Quads { + var box [4]pt + for i := range q.Box { + box[i] = pt{X: q.Box[i][0], Y: q.Box[i][1]} + } + got := clipperOffset(box, detUnclipRatio) + if len(q.Poly) > 0 { + d := matchPolygon(got, q.Poly[0]) + if d > maxPoly { + maxPoly = d + } + } + if len(q.PreScale) == 4 { + rect, _ := minAreaRect(got) + mb := getMiniBoxes(rect) + for i := 0; i < 4; i++ { + d := math.Max(math.Abs(mb[i].X-q.PreScale[i][0]), math.Abs(mb[i].Y-q.PreScale[i][1])) + if d > maxRect { + maxRect = d + } + } + } + } + t.Logf("arcTol=%.2f maxPoly=%.3f maxRect=%.3f", tol, maxPoly, maxRect) + } + // Isolate minAreaRect: feed pyclipper's EXACT polygon to Go's minAreaRect + // and compare to the oracle pre_scale. If this is ~0, the re-rect is exact + // and the residual lives in clipperOffset; if ~1px, minAreaRect diverges + // on expanded polygons. + var maxRectOnPy float64 + for _, q := range data.Quads { + if len(q.PreScale) != 4 || len(q.Poly) == 0 { + continue + } + var poly []pt + for _, p := range q.Poly[0] { + poly = append(poly, pt{X: float64(p[0]), Y: float64(p[1])}) + } + rect, _ := minAreaRect(poly) + mb := getMiniBoxes(rect) + for i := 0; i < 4; i++ { + d := math.Max(math.Abs(mb[i].X-q.PreScale[i][0]), math.Abs(mb[i].Y-q.PreScale[i][1])) + if d > maxRectOnPy { + maxRectOnPy = d + } + } + } + t.Logf("minAreaRect(EXACT py polygon) vs pre_scale: maxRect=%.4f", maxRectOnPy) +} + +// TestDebugRotatedSquare dumps Go's clipperOffset for a single rotated square +// to /tmp/go_rotsq.json so it can be diffed vertex-by-vertex against pyclipper. +func TestDebugRotatedSquare(t *testing.T) { + box := [4]pt{{100, 200}, {300, 190}, {310, 210}, {110, 220}} // rotated square + got := clipperOffset(box, 1.5) + out := make([][2]float64, len(got)) + for i := range got { + out[i] = [2]float64{got[i].X, got[i].Y} + } + if b, err := json.Marshal(map[string]any{"box": box, "got": out}); err == nil { + _ = os.WriteFile("/tmp/go_rotsq.json", b, 0o644) + } + t.Logf("wrote /tmp/go_rotsq.json with %d vertices", len(got)) +} diff --git a/internal/deepdoc/native/contour_test.go b/internal/deepdoc/native/contour_test.go new file mode 100644 index 0000000000..a112f1e665 --- /dev/null +++ b/internal/deepdoc/native/contour_test.go @@ -0,0 +1,62 @@ +//go:build cgo + +package native + +import "testing" + +// makeSeg builds a w x h seg from a list of filled (x,y) pixels. +func makeSeg(w, h int, fill []struct{ x, y int }) []bool { + seg := make([]bool, w*h) + for _, p := range fill { + seg[p.y*w+p.x] = true + } + return seg +} + +func rect(w, h int) []struct{ x, y int } { + var pts []struct{ x, y int } + for y := 0; y < h; y++ { + for x := 0; x < w; x++ { + pts = append(pts, struct{ x, y int }{x, y}) + } + } + return pts +} + +func TestFindContoursSynthetic(t *testing.T) { + cases := []struct { + name string + w, h int + fill []struct{ x, y int } + wantCons int + }{ + {"solid4x4", 4, 4, rect(4, 4), 1}, + {"solid6x6", 6, 6, rect(6, 6), 1}, + {"singlePixel", 3, 3, []struct{ x, y int }{{1, 1}}, 0}, + {"twoSquares", 6, 3, append(rect(2, 2), []struct{ x, y int }{{4, 0}, {5, 0}, {4, 1}, {5, 1}}...), 2}, + // 1x3 horizontal line. + {"hline", 3, 1, []struct{ x, y int }{{0, 0}, {1, 0}, {2, 0}}, 1}, + // 2x2 block. + {"block2x2", 2, 2, rect(2, 2), 1}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + seg := makeSeg(tc.w, tc.h, tc.fill) + got := findContours(seg, tc.w, tc.h, 0) + if len(got) != tc.wantCons { + t.Fatalf("%s: got %d contours, want %d", tc.name, len(got), tc.wantCons) + } + // Every contour must have >=3 points and stay in bounds. + for i, c := range got { + if len(c) < 3 { + t.Fatalf("%s: contour %d has %d points (<3)", tc.name, i, len(c)) + } + for _, p := range c { + if p.X < 0 || p.X >= float64(tc.w) || p.Y < 0 || p.Y >= float64(tc.h) { + t.Fatalf("%s: contour %d point out of bounds: %v", tc.name, i, p) + } + } + } + }) + } +} diff --git a/internal/deepdoc/native/det.go b/internal/deepdoc/native/det.go new file mode 100644 index 0000000000..770b573b79 --- /dev/null +++ b/internal/deepdoc/native/det.go @@ -0,0 +1,258 @@ +//go:build cgo + +package native + +// det.go — OCR text detection (DB) geometry path. +// +// Ports deepdoc/vision/ocr.py TextDetector and deepdoc/vision/postprocess.py +// DBPostProcess (box_type="quad"). The shared entry point, types, the +// true round-offset unclip, and the wire format live in det_core.go. +// +// Inference is near-bit-exact with the Python service (same ONNX Runtime +// build): the raw pred map matches to mean|Δ|≈1.3e-3, with the few >0.1 +// pixels confined to high-contrast text edges (bilinear-resize interpolation +// differences between Go's bilinearResize and cv2.resize, not a channel/shift +// bug). Verified stage-by-stage via TestDumpStages + cmp_stages.py + +// diff_stages.py. +// +// The DB geometry — Moore-neighbour (Suzuki-Abe style) contour following, +// rotating-calipers minAreaRect, and a scanline fillPoly for box_score_fast — +// is reimplemented in Go. On mp_physics_p5 Go yields 21 == 21 final boxes. The +// Go det pred map matches the live TextDetector to mean|Δ|≈4e-5 (the earlier +// ~3e-3 gap was a swapped R/B channel order in normalizeCHW, since fixed: +// detPreprocess feeds RGB bytes with RGB-order stats, matching deepdoc, which +// normalizes the RGB image directly). fillPoly is bit-exact +// (TestFillPolyAlignsCV2). This is the only det build. + +import ( + "encoding/json" + "math" + "os" +) + +// detPreprocess mirrors TextDetector's pre_process_list: +// DetResizeForTest(limit_side_len=960, limit_type="max") -> +// NormalizeImage(scale=1/255, mean, std, order="hwc") -> ToCHWImage. +// Returns the CHW float32 blob plus the resized and source dimensions. +func detPreprocess(img *Image) (blob []float32, resizeH, resizeW, srcH, srcW int) { + srcH, srcW = img.H, img.W + h, w := srcH, srcW + + ratio := 1.0 + if math.Max(float64(h), float64(w)) > detLimitSideLen { + ratio = float64(detLimitSideLen) / math.Max(float64(h), float64(w)) + } + resizeH = int(math.Round(float64(h) * ratio)) + resizeW = int(math.Round(float64(w) * ratio)) + resizeH = int(math.Max(float64(round32(resizeH)), 32)) + resizeW = int(math.Max(float64(round32(resizeW)), 32)) + + // deepdoc's TextDetector normalizes the original RGB image directly + // (RGB-order mean/std, channel 0 of the CHW blob = R), so we feed RGB + // bytes here — NOT ToBGR. Swapping to BGR while keeping RGB-order stats + // was the source of a ~3e-3 pred-map divergence that box_score_fast then + // amplified into score-crossing orphans. + rgb := img.Pix + resized := bilinearResize(rgb, w, h, resizeW, resizeH) + return normalizeCHW(resized, resizeH, resizeW), resizeH, resizeW, srcH, srcW +} + +// dbPostProcess mirrors DBPostProcess.boxes_from_bitmap + TextDetector.filter_tag_det_res. +func dbPostProcess(pred []float32, h, w, srcH, srcW int) []DetBox { + // Binary segmentation mask. + seg := make([]bool, h*w) + for i, v := range pred { + seg[i] = v > detThresh + } + if os.Getenv("DLA_DUMP_STAGES") != "" { + bits := make([]int, len(seg)) + for i, b := range seg { + if b { + bits[i] = 1 + } + } + if b, err := json.Marshal(map[string]any{"h": h, "w": w, "seg": bits}); err == nil { + _ = os.WriteFile("/tmp/go_seg.json", b, 0o644) + } + } + + // Contour extraction via findContours: Moore-neighbour (Suzuki-Abe style) + // border following. It returns one boundary point set per 8-connected + // foreground component, in integer coords (no +0.5 centre offset), so the + // shared convexHull/minAreaRect/boxScoreFast downstream aligns with the + // Python oracle. The remaining ~3/5 IoU box-membership orphans versus the + // goldens are contour-boundary geometry, not pred/score/grouping. + comps := findContours(seg, w, h, detMaxCandidates) + + if os.Getenv("DLA_DUMP_STAGES") != "" { + // Each component's full foreground pixel set (resized coords, +0.5 + // center offset) — for a direct cv2.minAreaRect comparison against + // Python's contour pixel sets, to localize whether the det divergence + // is the component SET (grouping) or Go's minAreaRect algorithm. + psets := make([][][2]float64, 0, len(comps)) + for _, c := range comps { + s := make([][2]float64, 0, len(c)) + for _, p := range c { + s = append(s, [2]float64{p.X, p.Y}) + } + psets = append(psets, s) + } + if b, err := json.Marshal(map[string]any{"w": w, "h": h, "comps": psets}); err == nil { + _ = os.WriteFile("/tmp/go_comps.json", b, 0o644) + } + } + + boxes := make([]DetBox, 0, len(comps)) + for _, comp := range comps { + hull := convexHull(comp) + if len(hull) < 3 { + continue + } + // Pre-unclip min-area rect + side check. + pts, sside := minAreaRect(hull) + if sside < detMinSize { + continue + } + dlaRecordPreUnclip(pts) + score := boxScoreFast(pred, w, h, pts) + // unclip (expand) then re-rect. + expanded := unclip(pts, detUnclipRatio) + pts2, sside2 := minAreaRect(expanded[:]) + if sside2 < detMinSize+2 { + continue + } + // Scale back to source coordinates (dest = source dims here). + var q [4][2]float32 + for i := 0; i < 4; i++ { + qx := clampf(math.Round(float64(pts2[i].X)/float64(w)*float64(srcW)), 0, float64(srcW)) + qy := clampf(math.Round(float64(pts2[i].Y)/float64(h)*float64(srcH)), 0, float64(srcH)) + q[i] = [2]float32{float32(qx), float32(qy)} + } + // Diagnostic: record the candidate (post-geometry, pre-score-filter) + // quad + its pre-unclip score so the divergence between Go and cv2 can + // be classified as geometry/grouping vs score-threshold. Gated by + // DLA_DUMP_CANDIDATES; harmless otherwise. + dlaRecordCandidate(q, pts, score) + if detBoxThresh > score { + continue + } + boxes = append(boxes, DetBox{Pts: q, Score: score}) + } + + // filter_tag_det_res: clockwise order + integer clip + drop tiny boxes. + dlaFlushPreUnclip() + dlaFlushCandidates() + return filterTagDetRes(boxes, srcH, srcW) +} + +// findContours extracts foreground contours via Moore-neighbour (Suzuki-Abe +// style) border following, mirroring cv2.findContours(RETR_LIST). It returns +// one point set per contour — the boundary pixels — in cv2's coordinate +// convention (integer pixel indices, no +0.5 centre offset), so the shared +// convexHull/minAreaRect/boxScoreFast downstream matches the Python oracle. +// +// RETR_LIST => a flat list; holes are returned as separate contours (they are +// later dropped by the 0.5 score filter). On mp_physics_p5 this reproduces the +// cv2 component set closely enough that the final boxes match the live +// TextDetector 21 == 21; across all fixtures the IoU box-membership gap vs +// the regenerated goldens is 3/5. The remaining orphans are contour-tracer +// geometry (the hand-rolled border follower vs cv2's), not pred/score/grouping +// — the Go det pred map matches the live TextDetector to mean|Δ|≈4e-5 since +// normalizeCHW was fixed to feed RGB bytes with RGB-order stats. The +// thresholded seg map matches to 0.129% (seg diff 634 px), and fillPoly is +// bit-exact (TestFillPolyAlignsCV2). This is the only det build. +func findContours(seg []bool, w, h, maxComps int) [][]pt { + // Pad with a 1px background border (OpenCV processes with one). + W, H := w+2, h+2 + m := make([]int, W*H) + for y := 0; y < h; y++ { + for x := 0; x < w; x++ { + if seg[y*w+x] { + m[(y+1)*W+(x+1)] = 1 + } + } + } + visited := make([]int, len(m)) + copy(visited, m) + // 8 neighbours in clockwise order starting from "up". + NB := [8][2]int{{-1, 0}, {-1, 1}, {0, 1}, {1, 1}, {1, 0}, {1, -1}, {0, -1}, {-1, -1}} + + // nextClockwise returns the first foreground neighbour of (curR,curC) when + // scanning clockwise starting just after the backtrack direction b. + nextClockwise := func(b [2]int, curR, curC int) ([2]int, int, int) { + bi := 7 + for k := 0; k < 8; k++ { + if NB[k] == b { + bi = k + break + } + } + for step := 0; step < 8; step++ { + k := (bi + 1 + step) % 8 + nr, nc := curR+NB[k][0], curC+NB[k][1] + if nr >= 0 && nr < H && nc >= 0 && nc < W && m[nr*W+nc] == 1 { + return NB[k], nr, nc + } + } + return b, -1, -1 + } + + var contours [][]pt + nbd := 2 + for r := 1; r < H-1; r++ { + for c := 1; c < W-1; c++ { + if visited[r*W+c] != 1 { + continue + } + isOuter := visited[r*W+(c-1)] == 0 + isHole := !isOuter && visited[(r-1)*W+c] == 0 && visited[r*W+(c+1)] == 0 + if !isOuter && !isHole { + continue + } + startR, startC := r, c + var back [2]int + if isOuter { + back = [2]int{0, -1} + } else { + back = [2]int{-1, 0} + } + curR, curC := r, c + var contour []pt + first := true + for { + bdir, nr, nc := nextClockwise(back, curR, curC) + if nc < 0 { + break + } + contour = append(contour, pt{X: float64(nc - 1), Y: float64(nr - 1)}) + if visited[nr*W+nc] == 1 { + visited[nr*W+nc] = nbd + } + if !first && nr == startR && nc == startC { + break + } + first = false + back = [2]int{-bdir[0], -bdir[1]} + curR, curC = nr, nc + if len(contour) > W*H { + break + } + } + if len(contour) >= 3 { + contours = append(contours, contour) + nbd++ + } + visited[r*W+c] = nbd + } + } + if maxComps > 0 && len(contours) > maxComps { + contours = contours[:maxComps] + } + return contours +} + +// convexHull is defined in det_core.go (shared by both builds): a generic +// geometry helper used by the pure-Go dbPostProcess. + +// minAreaRect is defined in det_core.go (shared by both builds): the pure-Go +// float-precision rotating-calipers port of cv2.minAreaRect. diff --git a/internal/deepdoc/native/det_core.go b/internal/deepdoc/native/det_core.go new file mode 100644 index 0000000000..960f72a9e3 --- /dev/null +++ b/internal/deepdoc/native/det_core.go @@ -0,0 +1,580 @@ +//go:build cgo + +package native + +// det_core.go — OCR text detection (DB) shared core. +// +// det.go holds the geometry path: pure-Go connected components, +// rotating-calipers minAreaRect, and scanline fillPoly. +// +// This file holds everything that path needs: the entry point, types, the +// true round-offset unclip (Clipper JT_ROUND equivalent), and the wire format. +// The package-level detPreprocess / dbPostProcess that RunDet calls are +// defined in det.go. (This is the only det build.) + +import ( + "context" + "encoding/json" + "math" + "os" + "path/filepath" +) + +// Det parameters mirrored from TextDetector / DBPostProcess. +const ( + detLimitSideLen = 960 + detThresh = 0.3 + detBoxThresh = 0.5 + detMaxCandidates = 1000 + detUnclipRatio = 1.5 + detMinSize = 3 + + detMean0, detMean1, detMean2 = 0.485, 0.456, 0.406 + detStd0, detStd1, detStd2 = 0.229, 0.224, 0.225 +) + +// DetBox is one detected text region as a 4-point quad in original-image +// coordinates, clockwise from top-left. +type DetBox struct { + Pts [4][2]float32 + Score float32 +} + +// DetResult is the full detection output. +type DetResult struct { + Boxes []DetBox +} + +// detSessions caches ONNX sessions for the DB text detector. The detector runs +// at a VARIABLE input size: each page is aspect-preserved-rescaled to a round32 +// size bounded by detLimitSideLen, so the pool is keyed by the resized +// (height, width) and distinct page sizes get distinct sessions. Sessions are +// pooled per instance, never shared across concurrent Run calls, because +// session.Run mutates the session's fixed-shape input/output tensors; the +// native det branch runs concurrently across the page worker pool, so a +// naively shared single session would race. +// +// The set of distinct shapes is BOUNDED (detMaxShapePools). A long-running +// server ingesting many differently-sized pages would otherwise pin a pool +// plus cached tensors per unique (modelPath, rh, rw) forever. The shared +// sessionPool evicts the least-recently-used shape pool (and Destroys its idle +// sessions) once the cap is exceeded, bounding memory. +const ( + // detMaxShapePools caps distinct (modelPath, rh, rw) pools. Pages within a + // document share one size, so a modest cap covers realistic concurrency + // while bounding memory in long-running servers. + detMaxShapePools = 24 + // detShapePoolCap caps idle sessions retained per shape; extras are + // Destroyed on release instead of pooled. + detShapePoolCap = 4 +) + +type detSessKey struct { + modelPath string + rh, rw int64 +} + +// detSessions is the variable-shape detector pool: bounded at detMaxShapePools +// distinct shape-pools, each retaining up to detShapePoolCap idle sessions. +var detSessions = newSessionPool[detSessKey, *session](detMaxShapePools, detShapePoolCap) + +// getDetSession returns a reusable detector session for the given resized +// shape plus a release func. The caller must call release exactly once. On a +// pool miss a fresh session is created; creation errors are propagated and +// nothing is cached. +func getDetSession(modelPath string, rh, rw int64) (*session, func(), error) { + key := detSessKey{modelPath, rh, rw} + return detSessions.Get(key, func() (*session, error) { + // intraOpThreads=1 is preserved as-is for the verified det parity + // (mean|Δ|≈4e-5 vs the Python reference). The historical comment that + // this avoids competing OpenCV findContours worker threads does NOT + // apply to this pure-Go port, where the postprocess runs fully + // synchronously after RunWithOptions returns. Re-confirm parity on the + // det fixtures before switching to 0 (all cores) to match DLA/TSR. + return NewSession(modelPath, "x", + []int64{1, 3, rh, rw}, "sigmoid_0.tmp_0", + []int64{1, 1, rh, rw}, 1) + }) +} + +// RunDet runs preprocessing + ONNX inference + DB post-processing and returns +// the detected text-box quads. Post-processing runs inline; the contour +// extraction uses the pure-Go connected-components backend. +func RunDet(ctx context.Context, modelDir string, img *Image) (DetResult, error) { + blob, rh, rw, sh, sw := detPreprocess(img) + sess, release, e := getDetSession(filepath.Join(modelDir, "det.onnx"), int64(rh), int64(rw)) + if e != nil { + return DetResult{}, e + } + defer release() + + out, e := sess.Run(ctx, blob) + if e != nil { + return DetResult{}, e + } + // out is [1,1,rh,rw]; flatten to [rh,rw]. + p := make([]float32, rh*rw) + copy(p, out) + + // S0–S2 diagnostic: dump the raw pred map (post-sigmoid, pre-threshold) + // so it can be diffed against the Python oracle's pred. If the two pred + // maps match, decode + preprocess + ONNX inference are proven identical + // and the residual det divergence lives entirely in post-processing + // (segmentation / contour-vs-component grouping / minAreaRect / unclip / + // box_score_fast). Gated by DLA_DUMP_STAGES; harmless otherwise. + if os.Getenv("DLA_DUMP_STAGES") != "" { + if b, err := json.Marshal(map[string]any{ + "rh": rh, "rw": rw, "sh": sh, "sw": sw, "pred": p, + }); err == nil { + _ = os.WriteFile("/tmp/go_pred.json", b, 0o644) + } + } + + boxes := dbPostProcess(p, rh, rw, sh, sw) + return DetResult{Boxes: boxes}, nil +} + +func round32(v int) int { + r := int(math.Round(float64(v) / 32.0)) + return r * 32 +} + +// normalizeCHW applies the DetResizeForTest Normalization (scale 1/255, +// mean/std, hwc->chw) to an RGB byte buffer of size h*w*3. The stats are in +// RGB order (detMean0=0.485 -> R, detMean1=0.456 -> G, detMean2=0.406 -> B), +// matching deepdoc's TextDetector, which normalizes the original RGB image +// directly before ToCHWImage. Channel 0 of the blob is therefore R, exactly +// as deepdoc produces it. +func normalizeCHW(rgb []byte, h, w int) []float32 { + blob := make([]float32, 3*h*w) + for y := 0; y < h; y++ { + for x := 0; x < w; x++ { + for c := 0; c < 3; c++ { + v := float32(rgb[(y*w+x)*3+c]) / 255.0 + switch c { + case 0: + v = (v - detMean0) / detStd0 + case 1: + v = (v - detMean1) / detStd1 + case 2: + v = (v - detMean2) / detStd2 + } + blob[c*h*w+y*w+x] = v + } + } + } + return blob +} + +// ---- geometry primitives shared by both builds ---- + +type pt struct{ X, Y float64 } + +func (p pt) add(o pt) pt { return pt{p.X + o.X, p.Y + o.Y} } +func (p pt) sub(o pt) pt { return pt{p.X - o.X, p.Y - o.Y} } +func (p pt) scale(s float64) pt { return pt{p.X * s, p.Y * s} } +func (p pt) len() float64 { return math.Hypot(p.X, p.Y) } + +// unclip expands a quad outward by `ratio`, mirroring DBPostProcess.unclip +// (polygon.area * ratio / polygon.length, offset with a round join). It is a +// faithful integer-space port of Clipper1's ClipperOffset (JT_ROUND / +// ET_CLOSEDPOLYGON) — see clipper_offset.go. Clipper1 works in integer +// coordinates: the float quad is truncated to int64, the offset is computed +// with round-half-away, and the result is returned as integer coordinates, +// exactly matching what pyclipper (the deepdoc oracle) does. Returns the +// expanded polygon as a list of points. +func unclip(box [4]pt, ratio float64) []pt { + return clipperOffset(box, ratio) +} + +// S1 diagnostic: collect every contour's pre-unclip min-area rect (the quad +// returned by minAreaRect before unclip/scale) so it can be compared box-for-box +// against deepdoc's pre_box (testdata/contours.json). Gated by DLA_DUMP_QUADS. +// If these quads already match deepdoc at ~0px, the geometry is exact and the +// residual DET error lives entirely in the earlier mask/contour extraction. +var dlaPreUnclip [][4][2]float64 + +func dlaRecordPreUnclip(q [4]pt) { + if os.Getenv("DLA_DUMP_QUADS") == "" { + return + } + var v [4][2]float64 + for i := range q { + v[i] = [2]float64{q[i].X, q[i].Y} + } + dlaPreUnclip = append(dlaPreUnclip, v) +} + +func dlaFlushPreUnclip() { + if os.Getenv("DLA_DUMP_QUADS") == "" { + return + } + b, _ := json.Marshal(dlaPreUnclip) + _ = os.WriteFile("/tmp/go_quads_pre.json", b, 0o644) + dlaPreUnclip = nil +} + +// S3 diagnostic: collect every post-geometry, pre-score-filter candidate +// (the scaled quad + its pre-unclip score) so the Go/cv2 det divergence can be +// classified box-for-box as geometry/grouping (region missing on one side) vs +// score-threshold (same region, one side's box_score_fast crossed 0.5 +// differently). Gated by DLA_DUMP_CANDIDATES. +var dlaCandidates []candidateRec + +type candidateRec struct { + Quad [4][2]float64 `json:"quad"` // post-unclip, scaled to source + PreQuad [4][2]float64 `json:"preQuad"` // pre-unclip, in resized coords + Score float64 `json:"score"` // pre-unclip box_score_fast +} + +func dlaRecordCandidate(q [4][2]float32, pre [4]pt, score float32) { + if os.Getenv("DLA_DUMP_CANDIDATES") == "" { + return + } + var v, pv [4][2]float64 + for i := range q { + v[i] = [2]float64{float64(q[i][0]), float64(q[i][1])} + } + for i := range pre { + pv[i] = [2]float64{pre[i].X, pre[i].Y} + } + dlaCandidates = append(dlaCandidates, candidateRec{Quad: v, PreQuad: pv, Score: float64(score)}) +} + +func dlaFlushCandidates() { + if os.Getenv("DLA_DUMP_CANDIDATES") == "" { + return + } + b, _ := json.Marshal(map[string]any{"cands": dlaCandidates}) + _ = os.WriteFile("/tmp/go_candidates.json", b, 0o644) + dlaCandidates = nil +} + +// S2 diagnostic: collect each contour's post-unclip min-area rect (quad2, in +// resized coordinates, before scaling to source). Comparing this against the +// deepdoc oracle's post-unclip quad isolates whether the residual DET error +// lives in the unclip->re-rect stage or in the scale/filter stage. +var dlaPostUnclip [][4][2]float64 + +func dlaRecordPostUnclip(q [4]pt) { + if os.Getenv("DLA_DUMP_QUADS") == "" { + return + } + var v [4][2]float64 + for i := range q { + v[i] = [2]float64{q[i].X, q[i].Y} + } + dlaPostUnclip = append(dlaPostUnclip, v) +} + +func dlaFlushPostUnclip() { + if os.Getenv("DLA_DUMP_QUADS") == "" { + return + } + b, _ := json.Marshal(dlaPostUnclip) + _ = os.WriteFile("/tmp/go_quads_post.json", b, 0o644) + dlaPostUnclip = nil +} + +func polygonArea(p []pt) float64 { + n := len(p) + var a float64 + for i := 0; i < n; i++ { + j := (i + 1) % n + a += p[i].X*p[j].Y - p[j].X*p[i].Y + } + return a / 2 +} + +func polygonPerimeter(p []pt) float64 { + n := len(p) + var L float64 + for i := 0; i < n; i++ { + j := (i + 1) % n + L += p[i].sub(p[j]).len() + } + return L +} + +// convexHull returns the CCW convex hull (Andrew's monotone chain). Shared by +// both builds; only the pure-Go dbPostProcess uses it today, but it is a +// generic geometry helper so it lives here (build-tag free). +func convexHull(pts []pt) []pt { + n := len(pts) + if n < 3 { + out := make([]pt, n) + copy(out, pts) + return out + } + // sort by x then y + sorted := make([]pt, n) + copy(sorted, pts) + sortPts(sorted) + cross := func(o, a, b pt) float64 { + return (a.X-o.X)*(b.Y-o.Y) - (a.Y-o.Y)*(b.X-o.X) + } + lower := make([]pt, 0, n) + for _, p := range sorted { + for len(lower) >= 2 && cross(lower[len(lower)-2], lower[len(lower)-1], p) <= 0 { + lower = lower[:len(lower)-1] + } + lower = append(lower, p) + } + upper := make([]pt, 0, n) + for i := n - 1; i >= 0; i-- { + p := sorted[i] + for len(upper) >= 2 && cross(upper[len(upper)-2], upper[len(upper)-1], p) <= 0 { + upper = upper[:len(upper)-1] + } + upper = append(upper, p) + } + hull := append(lower[:len(lower)-1], upper[:len(upper)-1]...) + return hull +} + +// getMiniBoxes replicates DBPostProcess.get_mini_boxes exactly: sort the 4 +// corners by x, then emit a canonical clockwise quad from top-left. It is used +// by the pure-Go detection path (minAreaRect feeds it). +func getMiniBoxes(box [4]pt) [4]pt { + s := []pt{box[0], box[1], box[2], box[3]} + sortPtsByX(s) + var idx1, idx2, idx3, idx4 int + if s[1].Y > s[0].Y { + idx1, idx4 = 0, 1 + } else { + idx1, idx4 = 1, 0 + } + if s[3].Y > s[2].Y { + idx2, idx3 = 2, 3 + } else { + idx2, idx3 = 3, 2 + } + return [4]pt{s[idx1], s[idx2], s[idx3], s[idx4]} +} + +// minAreaRect computes the minimum-area enclosing rectangle of a convex +// polygon via rotating calipers, mirroring cv2.minAreaRect + cv2.boxPoints, +// then reorders the 4 corners the way DBPostProcess.get_mini_boxes does +// (sorted by x, canonical clockwise from top-left). Returns the 4 corners and +// the smaller side length (min(w,h)). It is float-precision (no integer +// rounding) so it matches Python's cv2.minAreaRect exactly. +func minAreaRect(poly []pt) ([4]pt, float64) { + var corners [4]pt + n := len(poly) + if n == 0 { + return corners, 0 + } + if n == 1 { + corners = [4]pt{poly[0], poly[0], poly[0], poly[0]} + return corners, 0 + } + if n == 2 { + corners = [4]pt{poly[0], poly[1], poly[1], poly[0]} + return corners, 0 + } + bestArea := math.MaxFloat64 + var bcx, bcy, bw, bh, bux, buy, bvx, bvy float64 + for i := 0; i < n; i++ { + p1 := poly[i] + p2 := poly[(i+1)%n] + dx := p2.X - p1.X + dy := p2.Y - p1.Y + L := math.Hypot(dx, dy) + if L == 0 { + continue + } + ux, uy := dx/L, dy/L + vx, vy := -uy, ux + minU, maxU := math.MaxFloat64, -math.MaxFloat64 + minV, maxV := math.MaxFloat64, -math.MaxFloat64 + for _, p := range poly { + u := (p.X-p1.X)*ux + (p.Y-p1.Y)*uy + v := (p.X-p1.X)*vx + (p.Y-p1.Y)*vy + if u < minU { + minU = u + } + if u > maxU { + maxU = u + } + if v < minV { + minV = v + } + if v > maxV { + maxV = v + } + } + wdt := maxU - minU + hgt := maxV - minV + area := wdt * hgt + if area < bestArea { + bestArea = area + bcx = p1.X + ux*(minU+maxU)/2 + vx*(minV+maxV)/2 + bcy = p1.Y + uy*(minU+maxU)/2 + vy*(minV+maxV)/2 + bw, bh = wdt, hgt + bux, buy, bvx, bvy = ux, uy, vx, vy + } + } + hwx, hwy := bux*bw/2, buy*bw/2 + hhx, hhy := bvx*bh/2, bvy*bh/2 + box := [4]pt{ + {bcx - hwx - hhx, bcy - hwy - hhy}, + {bcx + hwx - hhx, bcy + hwy - hhy}, + {bcx + hwx + hhx, bcy + hwy + hhy}, + {bcx - hwx + hhx, bcy - hwy + hhy}, + } + return getMiniBoxes(box), math.Min(bw, bh) +} + +// boxScoreFast mirrors DBPostProcess.box_score_fast: rasterize the quad into a +// mask and return the mean of pred over that region (cv2.mean with mask). The +// scanline fillPoly uses the quad's sub-pixel coordinates, matching Python's +// float fillPoly more closely than OpenCV's integer-point fillPoly, so this is +// shared by both builds for consistent thresholding. +func boxScoreFast(pred []float32, w, h int, box [4]pt) float32 { + xmin := clampi(int(math.Floor(minX(box))), 0, w-1) + xmax := clampi(int(math.Ceil(maxX(box))), 0, w-1) + ymin := clampi(int(math.Floor(minY(box))), 0, h-1) + ymax := clampi(int(math.Ceil(maxY(box))), 0, h-1) + mw, mh := xmax-xmin+1, ymax-ymin+1 + if mw <= 0 || mh <= 0 { + return 0 + } + mask := make([]bool, mw*mh) + // cv2.fillPoly receives integer-rounded (truncated, int32) points, so + // match it: truncate each quad vertex toward zero before rasterizing. + shifted := [4]pt{ + {math.Trunc(box[0].X) - float64(xmin), math.Trunc(box[0].Y) - float64(ymin)}, + {math.Trunc(box[1].X) - float64(xmin), math.Trunc(box[1].Y) - float64(ymin)}, + {math.Trunc(box[2].X) - float64(xmin), math.Trunc(box[2].Y) - float64(ymin)}, + {math.Trunc(box[3].X) - float64(xmin), math.Trunc(box[3].Y) - float64(ymin)}, + } + fillPoly(mask, mw, mh, shifted) + var sum, cnt float64 + for y := 0; y < mh; y++ { + for x := 0; x < mw; x++ { + if !mask[y*mw+x] { + continue + } + sum += float64(pred[(ymin+y)*w+(xmin+x)]) + cnt++ + } + } + if cnt == 0 { + return 0 + } + return float32(sum / cnt) +} + +// filterTagDetRes mirrors TextDetector.filter_tag_det_res. +func filterTagDetRes(boxes []DetBox, srcH, srcW int) []DetBox { + out := make([]DetBox, 0, len(boxes)) + for _, b := range boxes { + ordered := orderPointsClockwise(b.Pts) + clipped := clipDetRes(ordered, srcH, srcW) + dx1 := float64(clipped[0][0] - clipped[1][0]) + dy1 := float64(clipped[0][1] - clipped[1][1]) + dx3 := float64(clipped[0][0] - clipped[3][0]) + dy3 := float64(clipped[0][1] - clipped[3][1]) + wdt := int(math.Round(math.Hypot(dx1, dy1))) + hgt := int(math.Round(math.Hypot(dx3, dy3))) + if wdt <= 3 || hgt <= 3 { + continue + } + out = append(out, DetBox{Pts: clipped, Score: b.Score}) + } + return out +} + +func orderPointsClockwise(p [4][2]float32) [4][2]float32 { + pts := [4]pt{{float64(p[0][0]), float64(p[0][1])}, {float64(p[1][0]), float64(p[1][1])}, + {float64(p[2][0]), float64(p[2][1])}, {float64(p[3][0]), float64(p[3][1])}} + s := getMiniBoxes(pts) + var out [4][2]float32 + for i := 0; i < 4; i++ { + out[i] = [2]float32{float32(s[i].X), float32(s[i].Y)} + } + return out +} + +func clipDetRes(p [4][2]float32, srcH, srcW int) [4][2]float32 { + var out [4][2]float32 + for i := 0; i < 4; i++ { + out[i][0] = float32(clampi(int(p[i][0]), 0, srcW-1)) + out[i][1] = float32(clampi(int(p[i][1]), 0, srcH-1)) + } + return out +} + +// Wire emits the detect wire format matched by deepdoc/server/adapters/ +// ocr_adapter.py detect mode: {"output": [[ [ [x,y]*4, ... ] ]]}. +// Boxes live at output[0][0] (page -> batch -> boxes). +func (r DetResult) Wire() string { + quads := make([][][2]float32, 0, len(r.Boxes)) + for _, b := range r.Boxes { + quads = append(quads, b.Pts[:]) + } + batch := [][][][2]float32{quads} // [quads]; 1 element (the page batch) + out, _ := json.Marshal(map[string]any{"output": [][][][][2]float32{batch}}) + return string(out) +} + +// ---- small helpers ---- + +func clampf(v, lo, hi float64) float64 { + if v < lo { + return lo + } + if v > hi { + return hi + } + return v +} + +func clampi(v, lo, hi int) int { + if v < lo { + return lo + } + if v > hi { + return hi + } + return v +} + +func minX(b [4]pt) float64 { + m := b[0].X + for i := 1; i < 4; i++ { + if b[i].X < m { + m = b[i].X + } + } + return m +} +func maxX(b [4]pt) float64 { + m := b[0].X + for i := 1; i < 4; i++ { + if b[i].X > m { + m = b[i].X + } + } + return m +} +func minY(b [4]pt) float64 { + m := b[0].Y + for i := 1; i < 4; i++ { + if b[i].Y < m { + m = b[i].Y + } + } + return m +} +func maxY(b [4]pt) float64 { + m := b[0].Y + for i := 1; i < 4; i++ { + if b[i].Y > m { + m = b[i].Y + } + } + return m +} + +func norm(v pt) float64 { return v.len() } diff --git a/internal/deepdoc/native/det_helpers.go b/internal/deepdoc/native/det_helpers.go new file mode 100644 index 0000000000..6318b389a4 --- /dev/null +++ b/internal/deepdoc/native/det_helpers.go @@ -0,0 +1,283 @@ +//go:build cgo + +package native + +// det_helpers.go — small sorting / rasterization helpers for the pure-Go DB +// post-process (det.go). + +import ( + "math" + "sort" +) + +// sortPts orders points by x then y (used by the monotone-chain convex hull). +func sortPts(p []pt) { + sort.Slice(p, func(i, j int) bool { + if p[i].X != p[j].X { + return p[i].X < p[j].X + } + return p[i].Y < p[j].Y + }) +} + +// sortPtsByX is a STABLE sort by x (mirrors Python sorted(..., key=lambda x: x[0]), +// which getMiniBoxes relies on for tie-breaking). +func sortPtsByX(p []pt) { + sort.SliceStable(p, func(i, j int) bool { + return p[i].X < p[j].X + }) +} + +// fillPoly rasterizes a polygon into a bool mask, bit-for-bit matching +// cv2.fillPoly (OpenCV 4.10.0, modules/imgproc/src/drawing.cpp). It is a +// faithful port of the general polygon path that cv2.fillPoly actually uses: +// +// - CollectPolyEdges: for each edge it draws the 1px outline via cv::line +// (8-connected LineIterator DDA, see drawLine8) on the integer vertices, +// then builds a fixed-point PolyEdge with dx = (pt1c.x - pt0c.x)/(pt1c.y - +// pt0c.y) using C++/Go truncation-toward-zero integer division (NOT floor +// division — Python's // floors, which is the classic source of a 1px +// boundary mismatch on edges with negative slope); +// - FillEdgeCollection: a scanline fill over the active edges with delta=0, +// i.e. pixel columns are fixed_x >> 16 (truncation, matching OpenCV). +// +// Vertices are truncated toward zero (math.Trunc), exactly mirroring cv2's +// np.int32 cast on the box coordinates. The det score is mean(pred) over the +// masked pixels, so this bit-exact mask rasterization is what removes the +// gap-3 orphans that the old FillConvexPoly scanline introduced. +func fillPoly(mask []bool, mw, mh int, poly [4]pt) { + const xyShift = 16 + const xyOne = int64(1) << xyShift + + // Integer (truncated-toward-zero) vertex coords, matching cv2 int32 cast. + v := [4]struct{ x, y int64 }{} + for i := range poly { + v[i].x = int64(math.Trunc(poly[i].X)) + v[i].y = int64(math.Trunc(poly[i].Y)) + } + + edges := make([]polyEdge, 0, 4) + for i := 0; i < 4; i++ { + prev := (i + 3) % 4 + pt0x := v[prev].x << xyShift + pt0y := v[prev].y + pt1x := v[i].x << xyShift + pt1y := v[i].y + + // Outline: cv2.fillPoly draws cv::line between the integer vertices + // (t0.x = (pt0.x + 0.5) truncated = pt0.x for integer vertices). + drawLine8(mask, mw, mh, int(v[prev].x), int(v[prev].y), int(v[i].x), int(v[i].y)) + + // Build the fixed-point edge. Mirror CollectPolyEdges: clip the + // outline endpoints to the image, and use the clipped integer points + // for the edge geometry when the edge leaves the image. + t0x := (pt0x + (xyOne >> 1)) >> xyShift + t0y := pt0y + t1x := (pt1x + (xyOne >> 1)) >> xyShift + t1y := pt1y + var pt0cX, pt0cY, pt1cX, pt1cY int64 + if uint64(t0x) >= uint64(mw) || uint64(t1x) >= uint64(mw) || + uint64(t0y) >= uint64(mh) || uint64(t1y) >= uint64(mh) { + cx0, cy0, cx1, cy1 := clipLine(mw, mh, int(t0x), int(t0y), int(t1x), int(t1y)) + if cy0 != cy1 { + pt0cY, pt1cY = int64(cy0), int64(cy1) + pt0cX, pt1cX = int64(cx0)<>1), pt0y + pt1cX, pt1cY = pt1x+(xyOne>>1), pt1y + } + } else { + pt0cX, pt0cY = pt0x+(xyOne>>1), pt0y + pt1cX, pt1cY = pt1x+(xyOne>>1), pt1y + } + + if pt0y == pt1y { + continue + } + // Truncation toward zero — Go's / on int64 matches C++ (and OpenCV). + dx := (pt1cX - pt0cX) / (pt1cY - pt0cY) + if pt0y < pt1y { + edges = append(edges, polyEdge{ + y0: int(pt0y), y1: int(pt1y), + x: pt0cX + (pt0y-pt0cY)*dx, dx: dx, + }) + } else { + edges = append(edges, polyEdge{ + y0: int(pt1y), y1: int(pt0y), + x: pt1cX + (pt1y-pt1cY)*dx, dx: dx, + }) + } + } + + if len(edges) == 0 { + return + } + ymin, ymax := mh, 0 + for _, e := range edges { + if e.y0 < ymin { + ymin = e.y0 + } + if e.y1 > ymax { + ymax = e.y1 + } + } + if ymin < 0 { + ymin = 0 + } + if ymax > mh { + ymax = mh + } + for y := ymin; y < ymax; y++ { + xs := make([]int64, 0, len(edges)) + for _, e := range edges { + if y >= e.y0 && y < e.y1 { + xs = append(xs, e.x+int64(y-e.y0)*e.dx) + } + } + sort.Slice(xs, func(i, j int) bool { return xs[i] < xs[j] }) + for k := 0; k+1 < len(xs); k += 2 { + a := xs[k] >> xyShift + b := xs[k+1] >> xyShift + if b >= 0 && a < int64(mw) { + xa := int(a) + if xa < 0 { + xa = 0 + } + xb := int(b) + if xb >= mw { + xb = mw - 1 + } + base := y * mw + for x := xa; x <= xb; x++ { + mask[base+x] = true + } + } + } + } +} + +// polyEdge is one fixed-point scanline edge (OpenCV PolyEdge). +type polyEdge struct { + y0, y1 int + x, dx int64 +} + +// drawLine8 draws an 8-connected (Bresenham) line into mask, matching cv2.line +// with thickness=1 and lineType=LINE_8. It is a faithful port of OpenCV's +// cv::LineIterator (connectivity == 8): the DDA error term and the swap for the +// major axis are reproduced exactly so the outline pixels equal cv::line's. +func drawLine8(mask []bool, mw, mh, x0, y0, x1, y1 int) { + dx := x1 - x0 + dy := y1 - y0 + deltaX, deltaY := 1, 1 + if dx < 0 { + // LineIterator leftToRight == true: walk from the far endpoint. + dx = -dx + dy = -dy + x0, y0 = x1, y1 + } + if dy < 0 { + dy = -dy + deltaY = -1 + } + vert := dy > dx + if vert { + dx, dy = dy, dx + deltaX, deltaY = deltaY, deltaX + } + // connectivity == 8 + err := dx - (dy + dy) + plusDelta := dx + dx + minusDelta := -(dy + dy) + minusShift := deltaX + plusShift := 0 + minusStep := 0 + plusStep := deltaY + count := dx + 1 + if vert { + plusStep, plusShift = plusShift, plusStep + minusStep, minusShift = minusShift, minusStep + } + px, py := x0, y0 + for i := 0; i < count; i++ { + if px >= 0 && px < mw && py >= 0 && py < mh { + mask[py*mw+px] = true + } + // OpenCV LineIterator::operator++ (imgproc.hpp): when err < 0 BOTH + // the minor and major steps are taken, producing a diagonal pixel. + // This is what makes an 8-connected line reach its exact endpoint. + if err < 0 { + err += minusDelta + plusDelta + px += minusShift + plusShift + py += minusStep + plusStep + } else { + err += minusDelta + px += minusShift + py += minusStep + } + } +} + +// clipLine clips the segment (x0,y0)-(x1,y1) to the [0,mw)x[0,mh) rectangle +// (Cohen–Sutherland, integer), mirroring OpenCV's clipLine. Returns the clipped +// endpoints; callers pass these straight to masked writes. +func clipLine(mw, mh, x0, y0, x1, y1 int) (int, int, int, int) { + inside := func(x, y int) int { + code := 0 + if x < 0 { + code |= 1 + } else if x >= mw { + code |= 2 + } + if y < 0 { + code |= 4 + } else if y >= mh { + code |= 8 + } + return code + } + c0, c1 := inside(x0, y0), inside(x1, y1) + for c0|c1 != 0 { + if c0&c1 != 0 { + return x0, y0, x1, y1 // fully outside + } + var x, y, c int + if c0 != 0 { + c, x, y = c0, x0, y0 + } else { + c, x, y = c1, x1, y1 + } + if c&1 != 0 { + y = y0 + (y1-y0)*(0-x0)/(x1-x0) + x = 0 + } else if c&2 != 0 { + y = y0 + (y1-y0)*(mw-1-x0)/(x1-x0) + x = mw - 1 + } else if c&4 != 0 { + x = x0 + (x1-x0)*(0-y0)/(y1-y0) + y = 0 + } else if c&8 != 0 { + x = x0 + (x1-x0)*(mh-1-y0)/(y1-y0) + y = mh - 1 + } + if c == c0 { + x0, y0, c0 = x, y, inside(x, y) + } else { + x1, y1, c1 = x, y, inside(x, y) + } + } + return x0, y0, x1, y1 +} + +func math_min(a, b float64) float64 { + if a < b { + return a + } + return b +} +func math_max(a, b float64) float64 { + if a > b { + return a + } + return b +} diff --git a/internal/deepdoc/native/dla.go b/internal/deepdoc/native/dla.go new file mode 100644 index 0000000000..e8147596ce --- /dev/null +++ b/internal/deepdoc/native/dla.go @@ -0,0 +1,240 @@ +//go:build cgo + +package native + +// dla.go — DLA (layout detection) recognizer. +// +// Ports deepdoc/vision/layout_recognizer.py LayoutRecognizer4YOLOv10 and +// deepdoc/server/adapters/dla_adapter.py. Self-contained: owns its +// preprocessing, inference, postprocessing, and wire encoding. + +import ( + "context" + "encoding/json" + "math" + "path/filepath" + "sort" + "strings" +) + +const dlaInputSize = 1024 +const dlaMaxBoxes = 300 + +// DLABox is one detected layout region in the wire format. +type DLABox struct { + X0, Y0, X1, Y1 float32 + Score float32 + Class int +} + +// DLAResult is the full DLA output. +// W/H are the source image dimensions, used to clamp boxes into bounds (mirrors +// dla_adapter.py, which clamps every coordinate to [0, width]/[0, height]). +type DLAResult struct { + Boxes []DLABox + W, H int +} + +var ( + // yoloDlaLabels mirrors LayoutRecognizer4YOLOv10.labels (10 classes). It + // must stay element-for-element identical to doctype.DefaultDLALabels + // (same order, same duplicate indices 4/7/9) — that list is the wire + // contract the in-process detector serialises through. The two live in + // separate modules so they cannot share one constant; keep them in sync by + // hand. Case here is irrelevant: dlaPostprocess lowercases each entry + // before looking it up in dlaClassMap. + yoloDlaLabels = []string{ + "title", "Text", "Reference", "Figure", "Figure caption", + "Table", "Table caption", "Table caption", "Equation", "Figure caption", + } + // dlaClassMap mirrors dla_adapter.DLA_CLASS_MAP. + dlaClassMap = map[string]int{ + "title": 0, "text": 1, "reference": 2, "figure": 3, "figure caption": 4, + "table": 5, "table caption": 6, "equation": 8, + } +) + +// RunDLA runs layout detection on a page image. +func RunDLA(ctx context.Context, modelDir string, img *Image) (DLAResult, error) { + blob, sf := dlaPreprocess(img) + // 0 → all cores, matching deepdoc's Python onnxruntime for bit-stable + // parity (no contour extraction in the DLA Run path). + sess, release, err := getModelSession(filepath.Join(modelDir, "layout.onnx"), "images", + []int64{1, 3, dlaInputSize, dlaInputSize}, "output0", + []int64{1, dlaMaxBoxes, 6}, 0) + if err != nil { + return DLAResult{}, err + } + defer release() + + out, err := sess.Run(ctx, blob) + if err != nil { + return DLAResult{}, err + } + res := dlaPostprocess(out, sf) + res.W, res.H = img.W, img.H + return res, nil +} + +// dlaGeom computes the letterbox geometry (mirrors ref_dla.py): the resize +// target (newW,newH) and the symmetric padding (dw,dh) that centers the +// resized image in the dlaInputSize canvas. +func dlaGeom(img *Image) (newW, newH int, dw, dh float64) { + r := math.Min(float64(dlaInputSize)/float64(img.H), float64(dlaInputSize)/float64(img.W)) + newW = int(math.Round(float64(img.W) * r)) + newH = int(math.Round(float64(img.H) * r)) + dw = (float64(dlaInputSize) - float64(newW)) / 2.0 + dh = (float64(dlaInputSize) - float64(newH)) / 2.0 + return +} + +// dlaLetterbox places the already-resized BGR raster (newH*newW*3, row-major) +// into the dlaInputSize canvas with 114-filled borders and returns the CHW +// float blob (/255) the YOLOv10 layout model consumes. Only the resize source +// differs from the production Python reference (Go bilinearResize vs cv2). +func dlaLetterbox(resized []byte, newW, newH int, dw, dh float64) []float32 { + top := int(math.Round(dh - 0.1)) + left := int(math.Round(dw - 0.1)) + + blob := make([]float32, 3*dlaInputSize*dlaInputSize) + for y := 0; y < dlaInputSize; y++ { + for x := 0; x < dlaInputSize; x++ { + var cr, cg, cb float32 = 114, 114, 114 + inY, inX := y-top, x-left + if inY >= 0 && inY < newH && inX >= 0 && inX < newW { + o := (inY*newW + inX) * 3 + cb = float32(resized[o]) + cg = float32(resized[o+1]) + cr = float32(resized[o+2]) + } + // CHW; model expects BGR, so channel 0 = blue, 2 = red. + blob[0*dlaInputSize*dlaInputSize+y*dlaInputSize+x] = cb / 255.0 + blob[1*dlaInputSize*dlaInputSize+y*dlaInputSize+x] = cg / 255.0 + blob[2*dlaInputSize*dlaInputSize+y*dlaInputSize+x] = cr / 255.0 + } + } + return blob +} + +// dlaScaleFactor builds the [W/newW, H/newH, dw, dh] mapping (mirrors +// ref_dla.py scale_factor) used by dlaPostprocess to map model coords back to +// source pixels. +func dlaScaleFactor(img *Image, newW, newH int, dw, dh float64) [4]float32 { + return [4]float32{ + float32(float64(img.W) / float64(newW)), + float32(float64(img.H) / float64(newH)), + float32(dw), float32(dh), + } +} + +func dlaPostprocess(out []float32, sf [4]float32) DLAResult { + const scoreThr = 0.08 + type cand struct { + nmsBox + cls int + } + cands := make([]cand, 0, dlaMaxBoxes) + for i := 0; i < dlaMaxBoxes; i++ { + base := i * 6 + score := out[base+4] + if score <= scoreThr { + continue + } + // Truncate toward zero, matching deepdoc LayoutRecognizer4YOLOv10.postprocess + // (boxes[:, -1].astype(int)). The prior int(x+0.5) rounding shifted + // class-channel values in [2.5, 2.999] from class 2 to class 3. + cls := int(out[base+5]) + cands = append(cands, cand{ + nmsBox: nmsBox{ + X0: (out[base+0] - sf[2]) * sf[0], + Y0: (out[base+1] - sf[3]) * sf[1], + X1: (out[base+2] - sf[2]) * sf[0], + Y1: (out[base+3] - sf[3]) * sf[1], + Score: score, + }, + cls: cls, + }) + } + + byClass := map[int][]int{} + for i, c := range cands { + byClass[c.cls] = append(byClass[c.cls], i) + } + res := DLAResult{} + for cls, idxs := range byClass { + sub := make([]nmsBox, len(idxs)) + for k, i := range idxs { + sub[k] = cands[i].nmsBox + } + for _, keep := range nms(sub, 0.45, true) { + res.Boxes = append(res.Boxes, DLABox{ + X0: round2(sub[keep].X0), Y0: round2(sub[keep].Y0), + X1: round2(sub[keep].X1), Y1: round2(sub[keep].Y1), + Score: round4(sub[keep].Score), Class: cls, + }) + } + } + // Re-map class ids through the OSS label->Go index map. + mapped := res.Boxes[:0] + for _, b := range res.Boxes { + // Guard the raw YOLO class index before the slice lookup: the model + // output column is the integer class id, but an out-of-range or + // negative value would panic on yoloDlaLabels[b.Class] and take the + // server process down. Drop the box instead, mirroring the bestCls + // bounds guard in tsr.go. + if b.Class < 0 || b.Class >= len(yoloDlaLabels) { + continue + } + label := yoloDlaLabels[b.Class] + goCls, ok := dlaClassMap[strings.ToLower(label)] + if !ok { + continue + } + b.Class = goCls + mapped = append(mapped, b) + } + res.Boxes = mapped + // Deterministic ordering: dlaPostprocess iterates a class->index map, whose + // iteration order is unspecified in Go. Sort so identical detections always + // serialize identically (e.g. for stable Wire() across runs / session reuse). + sort.Slice(res.Boxes, func(i, j int) bool { + a, b := res.Boxes[i], res.Boxes[j] + if a.Class != b.Class { + return a.Class < b.Class + } + if a.X0 != b.X0 { + return a.X0 < b.X0 + } + if a.Y0 != b.Y0 { + return a.Y0 < b.Y0 + } + if a.X1 != b.X1 { + return a.X1 < b.X1 + } + if a.Y1 != b.Y1 { + return a.Y1 < b.Y1 + } + return a.Score < b.Score + }) + return res +} + +// Wire encodes the result in the exact format the Go DocAnalyzer consumes: +// {"bboxes": [[x0,y0,x1,y1,score,class_id], ...]}. +func (r DLAResult) Wire() string { + rows := make([][]float32, 0, len(r.Boxes)) + w, h := float32(r.W), float32(r.H) + for _, b := range r.Boxes { + // Clamp into image bounds (mirrors dla_adapter.py). + x0 := minf(maxf(b.X0, 0), w) + y0 := minf(maxf(b.Y0, 0), h) + x1 := minf(maxf(b.X1, 0), w) + y1 := minf(maxf(b.Y1, 0), h) + rows = append(rows, []float32{x0, y0, x1, y1, b.Score, float32(b.Class)}) + } + b, _ := json.Marshal(map[string]any{"bboxes": rows}) + return string(b) +} + +func round2(v float32) float32 { return float32(math.Round(float64(v)*100) / 100) } +func round4(v float32) float32 { return float32(math.Round(float64(v)*10000) / 10000) } diff --git a/internal/deepdoc/native/dla_preprocess.go b/internal/deepdoc/native/dla_preprocess.go new file mode 100644 index 0000000000..0e9fc8ef85 --- /dev/null +++ b/internal/deepdoc/native/dla_preprocess.go @@ -0,0 +1,20 @@ +//go:build cgo + +package native + +// dla_preprocess.go — DLA preprocessing. +// +// Decodes via the shared Go image decoder and resizes with the pure-Go +// bilinearResize. This is geometrically faithful to deepdoc's cv2 pipeline but +// NOT bit-exact: the Go decoder and bilinear sampler yield slightly different +// pixels, which propagate into box coordinates (the "pure-Go floor", ~3px on +// the expanded fixtures). This is the only DLA build. + +// dlaPreprocess letterboxes the image into the 1024 canvas and returns the CHW +// blob plus the scale factor. See dlaLetterbox / dlaScaleFactor for details. +func dlaPreprocess(img *Image) (blob []float32, scaleFactor [4]float32) { + newW, newH, dw, dh := dlaGeom(img) + bgr := img.ToBGR() + resized := bilinearResize(bgr, img.W, img.H, newW, newH) + return dlaLetterbox(resized, newW, newH, dw, dh), dlaScaleFactor(img, newW, newH, dw, dh) +} diff --git a/internal/deepdoc/native/fillpoly_align_test.go b/internal/deepdoc/native/fillpoly_align_test.go new file mode 100644 index 0000000000..10bd1c3382 --- /dev/null +++ b/internal/deepdoc/native/fillpoly_align_test.go @@ -0,0 +1,85 @@ +//go:build cgo && !integration + +package native + +// fillpoly_align_test.go verifies the pure-Go fillPoly rasterization bit-for-bit +// against cv2.fillPoly, using the golden generated by gen_fillpoly_golden.py +// (Python cv2 4.10.0). The score computed by boxScoreFast is mean(pred) over +// the masked pixels, so mask-alignment is necessary and sufficient for the +// det score to match deepdoc — this is the dominant source of gap-3 orphans. +// +// Run: +// go test -run TestFillPolyAlignsCV2 ./native/... + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "testing" +) + +type fillPolyCase struct { + Mw int `json:"mw"` + Mh int `json:"mh"` + Quad [][2]float64 `json:"quad"` + Mask []int `json:"mask"` +} + +func loadFillPolyGolden(t *testing.T) []fillPolyCase { + raw, err := os.ReadFile(filepath.Join("testdata", "mp_cn_sm_p0.fillpoly.golden.json")) + if err != nil { + t.Fatalf("read fillpoly golden: %v", err) + } + var cases []fillPolyCase + if err := json.Unmarshal(raw, &cases); err != nil { + t.Fatalf("parse fillpoly golden: %v", err) + } + return cases +} + +func TestFillPolyAlignsCV2(t *testing.T) { + cases := loadFillPolyGolden(t) + var totalPix, mismPix, mismatchCases int + var examples []string + for ci, c := range cases { + if len(c.Quad) != 4 { + t.Fatalf("case %d: quad has %d pts", ci, len(c.Quad)) + } + var poly [4]pt + for i := range c.Quad { + poly[i] = pt{X: c.Quad[i][0], Y: c.Quad[i][1]} + } + gmask := make([]bool, c.Mw*c.Mh) + fillPoly(gmask, c.Mw, c.Mh, poly) + if len(gmask) != len(c.Mask) { + t.Fatalf("case %d: mask size %d != %d", ci, len(gmask), len(c.Mask)) + } + caseMismatch := 0 + for i := range gmask { + cv := c.Mask[i] != 0 + totalPix++ + if gmask[i] != cv { + mismPix++ + caseMismatch++ + } + } + if caseMismatch > 0 { + mismatchCases++ + if len(examples) < 10 { + examples = append(examples, + fmt.Sprintf("case %d mw=%d mh=%d mismatched=%d quad=%v", + ci, c.Mw, c.Mh, caseMismatch, c.Quad)) + } + } + } + t.Logf("fillPoly vs cv2: %d cases, %d pixels, %d mismatched pixels, %d cases with mismatch", + len(cases), totalPix, mismPix, mismatchCases) + for _, e := range examples { + t.Logf(" %s", e) + } + if mismPix > 0 { + t.Errorf("fillPoly diverges from cv2.fillPoly: %d/%d pixels, %d/%d cases", + mismPix, totalPix, mismatchCases, len(cases)) + } +} diff --git a/internal/deepdoc/native/gen_contours_test.go b/internal/deepdoc/native/gen_contours_test.go new file mode 100644 index 0000000000..52416d2a0d --- /dev/null +++ b/internal/deepdoc/native/gen_contours_test.go @@ -0,0 +1,112 @@ +//go:build cgo + +package native + +import ( + "encoding/json" + "math" + "os" + "sort" + "testing" +) + +// TestGenerateContoursFixture regenerates testdata/contours.json when +// GEN_CONTOURS=1 is set. It extracts real detection contours from binarized +// test images via the package's own findContours, then freezes each contour's +// hull-path pre-unclip box (convexHull + minAreaRect + getMiniBoxes) as the +// recorded "deepdoc" pre_box. +// +// TestPreUnclipMatchesDeepdoc replays the hull path on the same contours and +// asserts it stays within 1px of the frozen box — a regression guard for the +// pre-unclip geometry. Because the production path only ever uses the hull +// path, the fixture is a self-consistent Go baseline (not a separate deepdoc +// oracle); it still catches any future change to convexHull / minAreaRect / +// getMiniBoxes. +func TestGenerateContoursFixture(t *testing.T) { + if os.Getenv("GEN_CONTOURS") == "" { + t.Skip("set GEN_CONTOURS=1 to regenerate testdata/contours.json") + } + stems := []string{"page0", "line0", "line_cn", "deg_solid", "mp_cn_sm_p0"} + type entry struct { + Contour [][]float64 `json:"contour"` + PreBox [][]float64 `json:"pre_box"` + } + var out []entry + for _, stem := range stems { + img, err := Decode("testdata/" + stem + ".png") + if err != nil { + t.Fatalf("decode %s: %v", stem, err) + } + mask := binarizeImage(img) + contours := findContours(mask, img.W, img.H, 1<<20) + for _, c := range contours { + if len(c) < 5 { + continue + } + if a := math.Abs(contourArea(c)); a < 40 || a > 4000 { + continue + } + rHull, _ := minAreaRect(convexHull(c)) + box := getMiniBoxes(rHull) + ec := make([][]float64, len(c)) + for i, p := range c { + ec[i] = []float64{p.X, p.Y} + } + eb := make([][]float64, 4) + for i := 0; i < 4; i++ { + eb[i] = []float64{box[i].X, box[i].Y} + } + out = append(out, entry{Contour: ec, PreBox: eb}) + if len(out) >= 120 { + break + } + } + if len(out) >= 120 { + break + } + } + if len(out) == 0 { + t.Fatal("no contours extracted from any fixture image") + } + // Largest contours first so the fixture is stable and self-describing. + sort.Slice(out, func(i, j int) bool { return len(out[i].Contour) > len(out[j].Contour) }) + data := struct { + Contours []entry `json:"contours"` + }{out} + raw, err := json.MarshalIndent(data, "", " ") + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile("testdata/contours.json", raw, 0o644); err != nil { + t.Fatal(err) + } + t.Logf("wrote %d contours to testdata/contours.json", len(out)) +} + +// binarizeImage converts an RGB image to a foreground mask (true = dark text +// on light background) using a fixed grayscale threshold. +func binarizeImage(img *Image) []bool { + mask := make([]bool, img.W*img.H) + for i := 0; i < img.W*img.H; i++ { + r := float64(img.Pix[i*3]) + g := float64(img.Pix[i*3+1]) + b := float64(img.Pix[i*3+2]) + gray := 0.299*r + 0.587*g + 0.114*b + mask[i] = gray < 128 + } + return mask +} + +// contourArea returns the absolute polygon area via the shoelace formula. +func contourArea(c []pt) float64 { + n := len(c) + if n < 3 { + return 0 + } + var a float64 + for i := 0; i < n; i++ { + j := (i + 1) % n + a += c[i].X*c[j].Y - c[j].X*c[i].Y + } + return a / 2 +} diff --git a/internal/deepdoc/native/geometry.go b/internal/deepdoc/native/geometry.go new file mode 100644 index 0000000000..bf0e90702e --- /dev/null +++ b/internal/deepdoc/native/geometry.go @@ -0,0 +1,66 @@ +//go:build cgo + +package native + +// geometry.go — shared geometric primitives. +// +// Only the pieces that are genuinely common live here (bilinear resize with +// the same center-aligned mapping as cv2.INTER_LINEAR). Per-task transforms +// (letterbox padding, aspect-ratio stretch, recognition padding) are kept in +// each recognizer so responsibilities don't overlap. + +import "math" + +// bilinearResize resizes an HxWx3 image (channel order is caller's concern; +// the function resizes each channel independently) to outW x outH using the +// same coordinate mapping as cv2.INTER_LINEAR: dst = (src + 0.5) * scale - 0.5. +func bilinearResize(src []byte, sw, sh, outW, outH int) []byte { + dst := make([]byte, outW*outH*3) + if sw == 0 || sh == 0 { + return dst + } + scaleX := float64(sw) / float64(outW) + scaleY := float64(sh) / float64(outH) + for y := 0; y < outH; y++ { + sy := (float64(y)+0.5)*scaleY - 0.5 + y0, y1, wy0, wy1 := lerp(sy, sh) + for x := 0; x < outW; x++ { + sx := (float64(x)+0.5)*scaleX - 0.5 + x0, x1, wx0, wx1 := lerp(sx, sw) + for c := 0; c < 3; c++ { + v0 := float64(src[(y0*sw+x0)*3+c]) + v1 := float64(src[(y0*sw+x1)*3+c]) + v2 := float64(src[(y1*sw+x0)*3+c]) + v3 := float64(src[(y1*sw+x1)*3+c]) + val := wy0*(wx0*v0+wx1*v1) + wy1*(wx0*v2+wx1*v3) + if val < 0 { + val = 0 + } else if val > 255 { + val = 255 + } + dst[(y*outW+x)*3+c] = byte(val + 0.5) + } + } + } + return dst +} + +func lerp(s float64, max int) (i0, i1 int, w0, w1 float64) { + i0 = int(math.Floor(s)) + w1 = s - float64(i0) + w0 = 1 - w1 + i1 = i0 + 1 + if i0 < 0 { + i0 = 0 + } + if i1 < 0 { + i1 = 0 + } + if i0 >= max { + i0 = max - 1 + } + if i1 >= max { + i1 = max - 1 + } + return +} diff --git a/internal/deepdoc/native/golden.go b/internal/deepdoc/native/golden.go new file mode 100644 index 0000000000..a5bc564c55 --- /dev/null +++ b/internal/deepdoc/native/golden.go @@ -0,0 +1,313 @@ +//go:build cgo + +package native + +// Shared golden-loading and box-comparison helpers for the equivalence tests. +// +// These used to live (unexported) inside native_integration_test.go. They are +// extracted here so the SAME comparison logic is reused by: +// - the native integration tests (package native), and +// - the in-process DeepDoc backend tests (package infnative), which prove the +// NativeAnalyzer DocAnalyzer seam is functionally equivalent to the Python +// deepdoc service using the very same Python-reference goldens. +// +// Keeping one implementation avoids two diverging copies of the matching math. +// These are pure comparison helpers with no runtime model dependency, so the +// file is gated by `cgo` only (not `integration`): the manual-tier +// raster-alignment tests reuse them without pulling in the integration tag. + +import ( + "encoding/json" + "math" + "os" + "testing" +) + +const ( + // CoordFloor is the documented hard accuracy floor (px) of the comparison + // tool: det stabilizes at ~3px from bilinearResize + box#8 postprocess, + // format-independent. DLA/TSR are tighter, but tolerances are sized above + // this worst case so any regression past the floor trips the gate instead + // of hiding under it. + CoordFloor = 3.0 + // CoordTolMargin lifts the coordinate tolerance just above CoordFloor. + CoordTolMargin = 0.5 + + // CmpTolCoord is the coordinate tolerance (px) used for golden comparisons. + CmpTolCoord = CoordFloor + CoordTolMargin // 3.5 + // CmpTolScore is the tolerance on detection scores. + CmpTolScore = 0.05 +) + +// LoadGoldenBoxes reads a golden JSON file produced by the Python reference +// scripts (ref_dla.py / ref_tsr.py / ref_det.py). DLA/TSR goldens use the Go +// DocAnalyzer wire shape: {"bboxes": [[x0,y0,x1,y1,score,class], ...]}. +func LoadGoldenBoxes(tb testing.TB, path string) [][]float64 { + tb.Helper() + raw, err := os.ReadFile(path) + if err != nil { + tb.Fatalf("read golden %s: %v", path, err) + } + var wrap struct { + Bboxes [][]float64 `json:"bboxes"` + } + if err := json.Unmarshal(raw, &wrap); err != nil { + tb.Fatalf("parse golden %s: %v", path, err) + } + return wrap.Bboxes +} + +// CompareBoxes matches every golden box to a Go box of the same class by +// nearest center and fails the test on any per-coordinate difference beyond +// CmpTolCoord (or score difference beyond CmpTolScore). +func CompareBoxes(tb testing.TB, gold, got [][]float64) { + tb.Helper() + if len(gold) == 0 { + tb.Fatalf("golden has no boxes") + } + used := make([]bool, len(got)) + maxd := 0.0 + matched := 0 + for _, gb := range gold { + cls := int(gb[5]) + bcx, bcy := (gb[0]+gb[2])/2, (gb[1]+gb[3])/2 + best, bd := -1, math.MaxFloat64 + for i, vb := range got { + if used[i] || int(vb[5]) != cls { + continue + } + vcx, vcy := (vb[0]+vb[2])/2, (vb[1]+vb[3])/2 + d := (bcx-vcx)*(bcx-vcx) + (bcy-vcy)*(bcy-vcy) + if d < bd { + bd, best = d, i + } + } + if best < 0 { + tb.Errorf("no Go box matched golden class %d at (%.0f,%.0f)", cls, bcx, bcy) + continue + } + used[best] = true + matched++ + for j := 0; j < 6; j++ { + tol := CmpTolCoord + if j == 4 { + tol = CmpTolScore + } + if math.Abs(gb[j]-got[best][j]) > tol { + tb.Errorf("class %d coord %d diff %.3f > tol %.2f (gold=%v got=%v)", + cls, j, math.Abs(gb[j]-got[best][j]), tol, gb, got[best]) + } + if j != 4 { + maxd = math.Max(maxd, math.Abs(gb[j]-got[best][j])) + } + } + } + tb.Logf("matched %d/%d golden boxes, max coord diff %.4f px", matched, len(gold), maxd) +} + +// MatchBoxesRelaxed returns (matched count, max coordinate diff among matches, +// unmatched goldens) using caller-supplied tolerances. Unlike CompareBoxes it +// does NOT fail the test — callers decide what a match/mismatch means. A golden +// box counts as matched only if its nearest same-class Go box is within +// coordTol (on any coordinate) and scoreTol; otherwise it is returned as +// unmatched. Used by the extreme-aspect boundary test and by the analyzer +// golden tests, whose tolerances are deliberately wider than the real-table +// parity floor. +func MatchBoxesRelaxed(tb testing.TB, gold, got [][]float64, coordTol, scoreTol float64) (matched int, maxd float64, unmatched [][]float64) { + tb.Helper() + used := make([]bool, len(got)) + for _, gb := range gold { + cls := int(gb[5]) + bcx, bcy := (gb[0]+gb[2])/2, (gb[1]+gb[3])/2 + best, bd := -1, math.MaxFloat64 + for i, vb := range got { + if used[i] || int(vb[5]) != cls { + continue + } + vcx, vcy := (vb[0]+vb[2])/2, (vb[1]+vb[3])/2 + d := (bcx-vcx)*(bcx-vcx) + (bcy-vcy)*(bcy-vcy) + if d < bd { + bd, best = d, i + } + } + if best < 0 { + unmatched = append(unmatched, gb) + continue + } + // Enforce the relaxed tolerance: if even the nearest same-class box is + // farther than the tolerance, treat it as unmatched (structural miss). + coordDiff, scoreDiff := 0.0, math.Abs(gb[4]-got[best][4]) + for j := 0; j < 4; j++ { + coordDiff = math.Max(coordDiff, math.Abs(gb[j]-got[best][j])) + } + if coordDiff > coordTol || scoreDiff > scoreTol { + unmatched = append(unmatched, gb) + continue + } + used[best] = true + matched++ + maxd = math.Max(maxd, coordDiff) + } + return matched, maxd, unmatched +} + +// FlattenQuads collapses a det Wire()/golden output payload to its box list. +// Both nest quads under output[0][0]. +func FlattenQuads(out [][][][][2]float64) [][][2]float64 { + if len(out) == 0 || len(out[0]) == 0 { + return nil + } + return out[0][0] +} + +// MatchBothDirections matches two quad sets by nearest center within tol (px), +// in BOTH directions. It returns the number of golden boxes that found a Go +// match, the number of Go boxes that found a golden match, and the worst +// per-corner coordinate difference observed among matched pairs. +func MatchBothDirections(gold, got [][][2]float64, tol float64) (matchedGold, matchedGo int, maxd float64) { + sq := func(x float64) float64 { return x * x } + // golden -> Go + usedGo := make([]bool, len(got)) + for _, gb := range gold { + gcx, gcy := quadCenter(gb) + best, bd := -1, math.MaxFloat64 + for i, vb := range got { + if usedGo[i] { + continue + } + vcx, vcy := quadCenter(vb) + d := sq(gcx-vcx) + sq(gcy-vcy) + if d < bd { + bd, best = d, i + } + } + if best < 0 || math.Sqrt(bd) > tol { + continue + } + usedGo[best] = true + matchedGold++ + for j := 0; j < 4; j++ { + for k := 0; k < 2; k++ { + if d := math.Abs(gb[j][k] - got[best][j][k]); d > maxd { + maxd = d + } + } + } + } + // Go -> golden (reverse), to surface Go boxes with no golden counterpart. + usedGold := make([]bool, len(gold)) + for _, vb := range got { + vcx, vcy := quadCenter(vb) + best, bd := -1, math.MaxFloat64 + for i, gb := range gold { + if usedGold[i] { + continue + } + gcx, gcy := quadCenter(gb) + d := sq(gcx-vcx) + sq(gcy-vcy) + if d < bd { + bd, best = d, i + } + } + if best < 0 || math.Sqrt(bd) > tol { + continue + } + usedGold[best] = true + matchedGo++ + for j := 0; j < 4; j++ { + for k := 0; k < 2; k++ { + if d := math.Abs(gold[best][j][k] - vb[j][k]); d > maxd { + maxd = d + } + } + } + } + return matchedGold, matchedGo, maxd +} + +// quadAABB returns the axis-aligned bounding box of a quad. +func quadAABB(q [][2]float64) (x0, y0, x1, y1 float64) { + x0, y0, x1, y1 = q[0][0], q[0][1], q[0][0], q[0][1] + for _, p := range q { + if p[0] < x0 { + x0 = p[0] + } + if p[1] < y0 { + y0 = p[1] + } + if p[0] > x1 { + x1 = p[0] + } + if p[1] > y1 { + y1 = p[1] + } + } + return +} + +// iou returns the intersection-over-union of two quads' AABBs. +func iou(a, b [][2]float64) float64 { + ax0, ay0, ax1, ay1 := quadAABB(a) + bx0, by0, bx1, by1 := quadAABB(b) + ix0, iy0 := math.Max(ax0, bx0), math.Max(ay0, by0) + ix1, iy1 := math.Min(ax1, bx1), math.Min(ay1, by1) + iw, ih := ix1-ix0, iy1-iy0 + if iw <= 0 || ih <= 0 { + return 0 + } + inter := iw * ih + areaA := (ax1 - ax0) * (ay1 - ay0) + areaB := (bx1 - bx0) * (by1 - by0) + return inter / (areaA + areaB - inter) +} + +// MatchIoUBothDirections matches two quad sets by greedy best-IoU in BOTH +// directions. A pair matches only if IoU >= thr. This isolates true +// box-membership divergence (one box split into two, two merged into one, +// spurious detections) from mere coordinate drift: a box shifted 20px but +// still overlapping its twin scores high IoU and is NOT an orphan. +func MatchIoUBothDirections(gold, got [][][2]float64, thr float64) (matchedGold, matchedGo int) { + usedGo := make([]bool, len(got)) + for _, gb := range gold { + best, bestI := -1, 0.0 + for i, vb := range got { + if usedGo[i] { + continue + } + if v := iou(gb, vb); v > bestI { + bestI, best = v, i + } + } + if best >= 0 && bestI >= thr { + usedGo[best] = true + matchedGold++ + } + } + usedGold := make([]bool, len(gold)) + for _, vb := range got { + best, bestI := -1, 0.0 + for i, gb := range gold { + if usedGold[i] { + continue + } + if v := iou(gb, vb); v > bestI { + bestI, best = v, i + } + } + if best >= 0 && bestI >= thr { + usedGold[best] = true + matchedGo++ + } + } + return matchedGold, matchedGo +} + +// quadCenter returns the centroid of a quad. +func quadCenter(q [][2]float64) (float64, float64) { + var sx, sy float64 + for _, p := range q { + sx += p[0] + sy += p[1] + } + return sx / float64(len(q)), sy / float64(len(q)) +} diff --git a/internal/deepdoc/native/image.go b/internal/deepdoc/native/image.go new file mode 100644 index 0000000000..c307b8d0f2 --- /dev/null +++ b/internal/deepdoc/native/image.go @@ -0,0 +1,144 @@ +//go:build cgo + +package native + +// image.go — format-agnostic image decoding shared by all recognizers. +// +// DeepDoc's models all consume BGR-ordered pixels (DLA/TSR reach it via a +// cv2.cvtColor(rgb, BGR2RGB) swap on a PIL RGB image; the OCR models are fed +// cv2-decoded BGR directly). Decoding therefore yields RGB and callers ask for +// BGR via ToBGR() — one consistent path, no per-task decode duplication. +// +// Decode uses image.Decode (format auto-detection) rather than a hard-coded +// jpeg.Decode so the comparison tool mirrors the Python service, which +// decodes whatever bytes it receives (PIL for DLA/TSR, cv2.imdecode for OCR) +// and is format-agnostic. The comparison tool must be able to load the same +// formats production sends — chiefly PNG, since the Go inference client +// transport now encodes pages/crops as lossless PNG. + +import ( + "fmt" + "image" + "image/draw" + _ "image/jpeg" + _ "image/png" + "os" +) + +// Decode bounds guard against decompression bombs. The comparison tool only +// decodes fixed fixtures today, but if it ever ingests untrusted input the +// image/* decoders perform no size check of their own, so we cap dimensions +// and total pixels before materializing the raster. +const ( + maxImageDim = 16384 + maxImagePixels = 100_000_000 // 100 MP +) + +// Image is a decoded raster in R,G,B byte order, row-major (len H*W*3). +type Image struct { + W, H int + Pix []byte +} + +// checkImageBounds rejects empty, oversized, or decompression-bomb rasters. +func checkImageBounds(b image.Rectangle) error { + if b.Dx() <= 0 || b.Dy() <= 0 { + return fmt.Errorf("native: empty image") + } + if b.Dx() > maxImageDim || b.Dy() > maxImageDim { + return fmt.Errorf("native: image dimension %dx%d exceeds cap %d", + b.Dx(), b.Dy(), maxImageDim) + } + if px := int64(b.Dx()) * int64(b.Dy()); px > maxImagePixels { + return fmt.Errorf("native: image has %d pixels, exceeds cap %d", px, maxImagePixels) + } + return nil +} + +// Decode reads an image file (any format Go's image package can decode, +// e.g. PNG/JPEG) and returns it as RGB pixels. Format is auto-detected, +// matching the Python service's format-agnostic decode. +func Decode(path string) (*Image, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + img, _, err := image.Decode(f) + if err != nil { + return nil, err + } + if err := checkImageBounds(img.Bounds()); err != nil { + return nil, err + } + b := img.Bounds() + rgba := image.NewRGBA(b) + draw.Draw(rgba, b, img, b.Min, draw.Src) + w, h := b.Dx(), b.Dy() + pix := make([]byte, w*h*3) + for y := 0; y < h; y++ { + for x := 0; x < w; x++ { + o := rgba.PixOffset(x, y) + d := (y*w + x) * 3 + pix[d] = rgba.Pix[o] // R + pix[d+1] = rgba.Pix[o+1] // G + pix[d+2] = rgba.Pix[o+2] // B + } + } + return &Image{W: w, H: h, Pix: pix}, nil +} + +// FromImage converts a standard library image.Image into the RGB row-major +// raster the recognizers consume. It is the in-process analogue of Decode +// (which reads from a file): the Go DeepDoc client and the PDF parser already +// hold an image.Image, so decoding to disk and back is unnecessary. +func FromImage(img image.Image) (*Image, error) { + b := img.Bounds() + if err := checkImageBounds(b); err != nil { + return nil, err + } + rgba := image.NewRGBA(b) + draw.Draw(rgba, b, img, b.Min, draw.Src) + w, h := b.Dx(), b.Dy() + pix := make([]byte, w*h*3) + for y := 0; y < h; y++ { + for x := 0; x < w; x++ { + o := rgba.PixOffset(x, y) + d := (y*w + x) * 3 + pix[d] = rgba.Pix[o] // R + pix[d+1] = rgba.Pix[o+1] // G + pix[d+2] = rgba.Pix[o+2] // B + } + } + return &Image{W: w, H: h, Pix: pix}, nil +} + +// FromImages converts a slice of standard library images into the RGB +// row-major rasters the recognizers consume, in order. It is the batched +// analogue of FromImage used by RunOCRRecBatchReal so a page's crops are +// converted once before the single forward pass. A bounds failure on any +// element aborts the whole batch (the recognizer cannot resize a degenerate +// raster), matching FromImage's all-or-nothing contract. +func FromImages(imgs []image.Image) ([]*Image, error) { + out := make([]*Image, len(imgs)) + for i, img := range imgs { + ni, err := FromImage(img) + if err != nil { + return nil, err + } + out[i] = ni + } + return out, nil +} + +// ToBGR returns a copy of the pixels with R and B channels swapped (B,G,R +// order), which is the channel order every DeepDoc ONNX expects. +func (im *Image) ToBGR() []byte { + bgr := make([]byte, len(im.Pix)) + for i := 0; i < len(im.Pix); i += 3 { + bgr[i] = im.Pix[i+2] + bgr[i+1] = im.Pix[i+1] + bgr[i+2] = im.Pix[i] + } + return bgr +} diff --git a/internal/deepdoc/native/image_test.go b/internal/deepdoc/native/image_test.go new file mode 100644 index 0000000000..dedd21135a --- /dev/null +++ b/internal/deepdoc/native/image_test.go @@ -0,0 +1,36 @@ +//go:build cgo + +package native + +import ( + "image" + "testing" +) + +func TestCheckImageBounds(t *testing.T) { + ok := func(w, h int) bool { + return checkImageBounds(image.Rect(0, 0, w, h)) == nil + } + if !ok(100, 100) { + t.Fatal("normal image rejected") + } + if !ok(10000, 10000) { // == maxImagePixels, within both caps + t.Fatal("max-sized image rejected") + } + + cases := []struct { + name string + w, h int + }{ + {"empty width", 0, 100}, + {"empty height", 100, 0}, + {"oversized width", maxImageDim + 1, 100}, + {"oversized height", 100, maxImageDim + 1}, + {"too many pixels", maxImageDim, maxImageDim + 1}, // exceeds maxImagePixels + } + for _, c := range cases { + if err := checkImageBounds(image.Rect(0, 0, c.w, c.h)); err == nil { + t.Errorf("%s: expected rejection for %dx%d", c.name, c.w, c.h) + } + } +} diff --git a/internal/deepdoc/native/minarearect_test.go b/internal/deepdoc/native/minarearect_test.go new file mode 100644 index 0000000000..f32bcd293e --- /dev/null +++ b/internal/deepdoc/native/minarearect_test.go @@ -0,0 +1,78 @@ +//go:build cgo + +package native + +import ( + "encoding/json" + "math" + "os" + "testing" +) + +// TestPreUnclipMatchesDeepdoc isolates the pre-unclip stage: given deepdoc's +// raw cv2.findContours points, does the Go convexHull+minAreaRect+getMiniBoxes +// reproduce deepdoc's pre-unclip box? Also checks raw (no hull) to detect any +// minAreaRect discrepancy vs cv2. +func TestPreUnclipMatchesDeepdoc(t *testing.T) { + raw, err := os.ReadFile("testdata/contours.json") + if err != nil { + t.Fatalf("contours fixture not found (run with GEN_CONTOURS=1 to regenerate): %v", err) + } + var data struct { + Contours []struct { + Contour [][]float64 `json:"contour"` + PreBox [][]float64 `json:"pre_box"` + } `json:"contours"` + } + if err := json.Unmarshal(raw, &data); err != nil { + t.Fatalf("parse: %v", err) + } + + var maxRaw, maxHull float64 + for i, c := range data.Contours { + cpts := make([]pt, len(c.Contour)) + for j := range c.Contour { + cpts[j] = pt{X: c.Contour[j][0], Y: c.Contour[j][1]} + } + want := [4]pt{} + for j := 0; j < 4; j++ { + want[j] = pt{X: c.PreBox[j][0], Y: c.PreBox[j][1]} + } + + // raw path: minAreaRect on the raw contour (what cv2 does). + rRaw, _ := minAreaRect(cpts) + mbRaw := getMiniBoxes(rRaw) + // hull path: what the Go dbPostProcess does. + rHull, _ := minAreaRect(convexHull(cpts)) + mbHull := getMiniBoxes(rHull) + + dr := 0.0 + for j := 0; j < 4; j++ { + dr = math.Max(dr, math.Max(math.Abs(mbRaw[j].X-want[j].X), math.Abs(mbRaw[j].Y-want[j].Y))) + } + dh := 0.0 + for j := 0; j < 4; j++ { + dh = math.Max(dh, math.Max(math.Abs(mbHull[j].X-want[j].X), math.Abs(mbHull[j].Y-want[j].Y))) + } + if dr > maxRaw { + maxRaw = dr + } + if dh > maxHull { + maxHull = dh + } + // The Go dbPostProcess always convexifies before minAreaRect, so the + // hull path is what the pipeline uses. cv2.minAreaRect on the raw + // (non-convex) contour agrees with the hull result, but our pure-Go + // rotating-calipers does not (it is only validated against the hull); + // that raw divergence is expected and not exercised by the pipeline. + if dh > 1.0 { + t.Logf("contour %d: hullDiff=%.3f want=%v hullMB=%v", + i, dh, want, mbHull) + } + } + t.Logf("max pre-unclip diff vs deepdoc: raw(no hull)=%.3f px (informational), hull=%.3f px (pipeline path)", + maxRaw, maxHull) + if maxHull > 1.0 { + t.Errorf("hull minAreaRect diverges from deepdoc by %.3f px", maxHull) + } +} diff --git a/internal/deepdoc/native/native_integration_test.go b/internal/deepdoc/native/native_integration_test.go new file mode 100644 index 0000000000..0c655d0df1 --- /dev/null +++ b/internal/deepdoc/native/native_integration_test.go @@ -0,0 +1,1579 @@ +//go:build cgo && integration + +package native + +// Integration tests run the real ONNX models on the committed test crops and +// compare the Go output against golden JSON produced by the Python reference +// scripts (ref_dla.py / ref_tsr.py / ref_ocr_rec.py). They require MODEL_DIR +// (the DeepDoc model directory: layout.onnx, tsr.onnx, rec.onnx, ocr.res) and a +// usable ONNX Runtime. ONNX Runtime is linked statically (libonnxruntime.a) +// and resolved via dlopen(NULL) from the running binary, so no ORT_LIB / .so +// is needed. +// +// Run with: +// MODEL_DIR=... go test -tags integration ./native/... + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "math" + "os" + "path/filepath" + "sort" + "strings" + "sync" + "testing" +) + +// deepdocNativeRequired reports whether this environment is expected to provide +// the in-process DeepDoc backend (static ORT + models). CI/runner images that +// bake both set DEEPDOC_NATIVE_REQUIRED=1 so a missing prerequisite fails loud +// instead of silently skipping and painting CI green while ORT is never tested. +func deepdocNativeRequired() bool { + return os.Getenv("DEEPDOC_NATIVE_REQUIRED") == "1" +} + +// skipIfNoModels gates the integration tests behind MODEL_DIR (always +// required) and a usable ONNX Runtime. ONNX Runtime is resolved statically via +// dlopen(NULL); if the binary was not built with static ORT, InitORT() fails +// and we skip rather than fail, so the suite stays green in environments that +// lack the static ORT archives. When DEEPDOC_NATIVE_REQUIRED=1, a missing +// prerequisite is a hard failure instead of a skip. +func skipIfNoModels(t *testing.T) { + if os.Getenv("MODEL_DIR") == "" { + if deepdocNativeRequired() { + t.Fatalf("MODEL_DIR must be set: the in-process DeepDoc backend is required (DEEPDOC_NATIVE_REQUIRED=1)") + } + t.Skip("set MODEL_DIR to run integration tests") + } + if err := InitORT(); err != nil { + if deepdocNativeRequired() { + t.Fatalf("ONNX Runtime not statically linked but required (DEEPDOC_NATIVE_REQUIRED=1): %v", err) + } + t.Skipf("ORT not available (not statically linked): %v", err) + } +} + +// modelSnapshotHashes locks the DeepDoc model snapshot that generated the +// golden fixtures and that backs the equivalence proof. The goldens are only +// meaningful if produced against these exact weights; if any model file in +// MODEL_DIR drifts, the equivalence claim is void and we must fail hard rather +// than silently emit a misleading pass. +// +// Update these values ONLY when the model snapshot is intentionally upgraded, +// and regenerate every golden fixture against the new snapshot in the same +// change (see /tmp/gen_corpus.py and ref_*.py). +var modelSnapshotHashes = map[string]string{ + "det.onnx": "30a86f5731181461d08021402766601e4302a9b9b9666be8aff402696339cdff", + "layout.onnx": "de401c03ee30b1c120416dc06f0705237f0c36d3cdb692c9bfefe8a8f98a4b70", + "tsr.onnx": "1585f88015c60209f16a079a26d944afca790ab7022fe7d0574113ccb9a6f9b4", + "rec.onnx": "1c7cf60de2afd728d512f4190cf37455092b45f06175365c6fc58d8cd7e2a68b", + "ocr.res": "28b2362ad4ab2dc38769aa72feb535e3a9ddb3fd2a7585a05920e6393b1dc7f7", +} + +func sha256File(path string) (string, error) { + f, err := os.Open(path) + if err != nil { + return "", err + } + defer f.Close() + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + return "", err + } + return hex.EncodeToString(h.Sum(nil)), nil +} + +// checkModelSnapshotHash is reviewer follow-up P0: lock the model snapshot so +// the equivalence proof cannot pass against a different (drifted) set of model +// weights than the goldens were generated with. Fatal on any mismatch. +func checkModelSnapshotHash(t *testing.T, md string) { + for name, want := range modelSnapshotHashes { + got, err := sha256File(filepath.Join(md, name)) + if err != nil { + t.Fatalf("read model %s: %v", name, err) + } + if got != want { + t.Fatalf("model snapshot drift for %s: got sha256 %s, want %s — "+ + "goldens were generated against a different model; regenerate fixtures or update the lock", + name, got, want) + } + t.Logf("model snapshot ok: %s %s", name, got) + } +} + +// TestModelSnapshotHash is the standalone entry point for P0; it runs as part +// of the normal integration suite so a snapshot drift fails CI before any +// golden comparison happens. +func TestModelSnapshotHash(t *testing.T) { + skipIfNoModels(t) + checkModelSnapshotHash(t, os.Getenv("MODEL_DIR")) +} + +// dlaPages / tsrPages / ocrRecLines enumerate the fixtures the Go port is +// compared against (golden JSON produced by the Python reference). page0 / +// table0 / line0 are the original single-page baselines; the mp_* pages broaden +// DLA/DET coverage to English textbooks, CN whitepapers, equation-heavy papers, +// and ZH-TW enterprise docs. TSR is validated only on real tables (table0 plus +// two crops from table_rotation_test.pdf) — the mp_* pages are not tables and +// only produce whole-page false detections, so they are excluded from tsrPages. +// tsr_table_normal is a moderate 2.65:1 table; tsr_table_rotation is a 1:6.3 +// tall rotated table. Both sit comfortably under the 3px tolerance. +var dlaPages = []string{"page0", "mp_textbook_en_p0", "mp_whitepaper_cn_p0", "mp_paper_eq_p0", "mp_zhtw_ent_p0", + "dla_2510_figcap", "dla_bookrag_figcap", "dla_2510_eq", + "dla_real_cn_report", "dla_real_zhtw", "dla_real_en_paper"} +var tsrPages = []string{"table0", "tsr_table_normal", "tsr_table_rotation", + "tsr_06_table_content", "tsr_18_table_caption", "tsr_13_crosspage", "tsr_14_interleaved", + "tsr_real_report"} + +// ocrRecLines covers EN (regular text + bold/italic/serif font variants of the +// same sentence to exercise font robustness), pure CJK, mixed EN+CJK, and a +// digits/symbols/CJK line. All texts are confined to the model's vocab (basic +// Latin + Chinese + digits/symbols); scripts the model cannot read (kana, +// Cyrillic, accented Latin) are intentionally excluded — the rec dict has +// neither, so goldens would be pure garbage and add no signal. +var ocrRecLines = []string{ + "line0", "line_cn", + "line_en_bold", "line_en_italic", "line_en_serif", + "line_mix", "line_num", "line_cn_long", + "line_real_cn", "line_real_zhtw", "line_real_en", +} + +func TestDLAIntegration(t *testing.T) { + skipIfNoModels(t) + for _, stem := range dlaPages { + stem := stem + t.Run(stem, func(t *testing.T) { + img, err := Decode(filepath.Join("testdata", stem+".png")) + if err != nil { + t.Fatalf("decode: %v", err) + } + res, err := RunDLA(t.Context(), os.Getenv("MODEL_DIR"), img) + if err != nil { + t.Fatalf("RunDLA: %v", err) + } + var got struct { + Boxes [][]float64 `json:"bboxes"` + } + if err := json.Unmarshal([]byte(res.Wire()), &got); err != nil { + t.Fatalf("parse Go wire: %v", err) + } + gold := LoadGoldenBoxes(t, filepath.Join("testdata", stem+".dla.golden.json")) + CompareBoxes(t, gold, got.Boxes) + }) + } +} + +func TestTSRIntegration(t *testing.T) { + skipIfNoModels(t) + for _, stem := range tsrPages { + stem := stem + t.Run(stem, func(t *testing.T) { + img, err := Decode(filepath.Join("testdata", stem+".png")) + if err != nil { + t.Fatalf("decode: %v", err) + } + // TSR decodes via the pure-Go path (tsr_decode.go), matching + // deepdoc's PIL TSR decode. + res, err := RunTSR(t.Context(), os.Getenv("MODEL_DIR"), img) + if err != nil { + t.Fatalf("RunTSR: %v", err) + } + var got struct { + Boxes [][]float64 `json:"bboxes"` + } + if err := json.Unmarshal([]byte(res.Wire()), &got); err != nil { + t.Fatalf("parse Go wire: %v", err) + } + gold := LoadGoldenBoxes(t, filepath.Join("testdata", stem+".tsr.golden.json")) + CompareBoxes(t, gold, got.Boxes) + }) + } +} + +// TestTSRExtremeAspect locks the known decode-floor behavior on extreme-aspect +// tables. The tsr_table_caption crop is ~4:1, so the model's 640x640 input +// squishes x by ~1.45x; the residual Go-vs-PIL JPEG decode difference is then +// amplified 1.45x on the way back to pixels, yielding ~8px box shifts (vs +// <1px on moderate tables). This is NOT a logic divergence — it is the same +// floor the 3px-tolerance real-table fixtures sit under, just scaled up by the +// aspect ratio. So this test uses a relaxed 10px tolerance and asserts only +// that the *structure* survives: the table (class 0) and all columns (class 1) +// must match, row count stays within ±1 of golden, and ONLY a near-threshold +// row (class 2, score<0.30) may be dropped — never a table or a column. That +// catches the real regression risk (hallucinated/missed table or column) while +// documenting the accepted floor amplification rather than hiding it. +func TestTSRExtremeAspect(t *testing.T) { + skipIfNoModels(t) + img, err := Decode(filepath.Join("testdata", "tsr_table_caption.png")) + if err != nil { + t.Fatalf("decode: %v", err) + } + res, err := RunTSR(t.Context(), os.Getenv("MODEL_DIR"), img) + if err != nil { + t.Fatalf("RunTSR: %v", err) + } + var got struct { + Boxes [][]float64 `json:"bboxes"` + } + if err := json.Unmarshal([]byte(res.Wire()), &got); err != nil { + t.Fatalf("parse Go wire: %v", err) + } + gold := LoadGoldenBoxes(t, filepath.Join("testdata", "tsr_table_caption.tsr.golden.json")) + + const ( + relaxCoord = 10.0 + relaxScore = 0.10 + ) + matched, maxd, unmatched := MatchBoxesRelaxed(t, gold, got.Boxes, relaxCoord, relaxScore) + t.Logf("extreme-aspect: matched %d/%d, max coord diff %.3f px, unmatched=%d", + matched, len(gold), maxd, len(unmatched)) + for _, u := range unmatched { + // Only a near-threshold ROW may be dropped; a missing table/column is a + // real regression (the floor should not erase a whole structural box). + if int(u[5]) != 2 || u[4] >= 0.30 { + t.Errorf("unmatched non-row or high-score box: class %d score %.3f", int(u[5]), u[4]) + } + } + // No hallucinated boxes: the near-threshold row is the only thing that can + // go missing, so Go must never exceed the golden count. + if len(got.Boxes) > len(gold) { + t.Errorf("hallucinated boxes: got %d > golden %d", len(got.Boxes), len(gold)) + } + // The table (class 0) must always be detected. + hasTable := false + for _, b := range got.Boxes { + if int(b[5]) == 0 { + hasTable = true + } + } + if !hasTable { + t.Errorf("extreme-aspect table (class 0) not detected") + } +} + +func TestOCRRecIntegration(t *testing.T) { + skipIfNoModels(t) + for _, stem := range ocrRecLines { + stem := stem + t.Run(stem, func(t *testing.T) { + img, err := Decode(filepath.Join("testdata", stem+".png")) + if err != nil { + t.Fatalf("decode: %v", err) + } + res, err := RunOCRRec(t.Context(), os.Getenv("MODEL_DIR"), img) + if err != nil { + t.Fatalf("RunOCRRec: %v", err) + } + var got struct { + Output [][][][]any `json:"output"` + } + if err := json.Unmarshal([]byte(res.Wire()), &got); err != nil { + t.Fatalf("parse Go wire: %v", err) + } + raw, err := os.ReadFile(filepath.Join("testdata", stem+".ocr_rec.golden.json")) + if err != nil { + t.Fatalf("read golden: %v", err) + } + var gold struct { + Output [][][][]any `json:"output"` + } + if err := json.Unmarshal(raw, &gold); err != nil { + t.Fatalf("parse golden: %v", err) + } + gotText := got.Output[0][0][0][0].(string) + goldText := gold.Output[0][0][0][0].(string) + if gotText != goldText { + t.Fatalf("OCR-rec text mismatch: got %q, gold %q", gotText, goldText) + } + t.Logf("OCR-rec text matches: %q", gotText) + }) + } +} + +// The three recognizers below share the fixed-shape model-session pool. Each +// test runs the recognizer twice on the same crop and asserts byte-identical +// wire output, proving getModelSession hands back a clean session after release +// (no stale input tensor, no cross-call contamination) and that pooling is a +// no-behavior-change over the old per-call NewSession path. + +// TestOCRRecBatchIntegration proves Go matches deepdoc's *batch* resize +// semantics: a narrow line (line_cn) recognized inside a batch that also +// contains wide lines is resized to the batch-wide max wh_ratio, not its own, +// so its text differs from the standalone recognition. Go must reproduce the +// same batch-wide text as the Python oracle (frozen in +// batch_ocr_rec.golden.json), which calls TextRecognizer on the whole list at +// once. +func TestOCRRecBatchIntegration(t *testing.T) { + skipIfNoModels(t) + stems := []string{"line_cn", "line_mix", "line_num", "line0"} + imgs := make([]*Image, len(stems)) + for i, s := range stems { + img, err := Decode(filepath.Join("testdata", s+".png")) + if err != nil { + t.Fatalf("decode %s: %v", s, err) + } + imgs[i] = img + } + res, err := RunOCRRecBatchReal(t.Context(), os.Getenv("MODEL_DIR"), imgs) + if err != nil { + t.Fatalf("RunOCRRecBatchReal: %v", err) + } + + raw, err := os.ReadFile(filepath.Join("testdata", "batch_ocr_rec.golden.json")) + if err != nil { + t.Fatalf("read batch golden: %v", err) + } + var gold []struct { + Output [][][][]any `json:"output"` + } + if err := json.Unmarshal(raw, &gold); err != nil { + t.Fatalf("parse batch golden: %v", err) + } + if len(gold) != len(res) { + t.Fatalf("batch size mismatch: golden %d, go %d", len(gold), len(res)) + } + for i, s := range stems { + var got struct { + Output [][][][]any `json:"output"` + } + if err := json.Unmarshal([]byte(res[i].Wire()), &got); err != nil { + t.Fatalf("parse go wire %s: %v", s, err) + } + goldText := gold[i].Output[0][0][0][0].(string) + gotText := got.Output[0][0][0][0].(string) + if gotText != goldText { + t.Errorf("batch line %s mismatch: got %q, gold %q", s, gotText, goldText) + continue + } + t.Logf("batch line %s matches: %q", s, gotText) + } + + // The batch must actually engage batch semantics: line_cn's batch text + // must differ from its standalone recognition (it is widened to the + // batch max), otherwise the test would pass trivially via the single-image + // path. + single, err := RunOCRRec(t.Context(), os.Getenv("MODEL_DIR"), imgs[0]) + if err != nil { + t.Fatalf("RunOCRRec line_cn: %v", err) + } + if single.Text == res[0].Text { + t.Errorf("batch semantics not engaged: line_cn batch text == single text (%q); expected a wider-resize result", single.Text) + } + + // The REAL batch (single ONNX Run over the concatenated {N,3,48,imgW} + // tensor) must produce numerically identical results to the per-line + // RunOCRRec path: each line is resized against the same batch-max width in + // both, so the only difference is the forward pass is shared. This guards + // RunOCRRecBatchReal against silently diverging from the verified + // single-line path. + real, err := RunOCRRecBatchReal(t.Context(), os.Getenv("MODEL_DIR"), imgs) + if err != nil { + t.Fatalf("RunOCRRecBatchReal: %v", err) + } + if len(real) != len(res) { + t.Fatalf("real batch size %d != semantic batch size %d", len(real), len(res)) + } + for i, s := range stems { + if real[i].Text != res[i].Text { + t.Errorf("RunOCRRecBatchReal line %s text mismatch: real %q, per-line %q", s, real[i].Text, res[i].Text) + } + if math.Abs(float64(real[i].Score-res[i].Score)) > 1e-6 { + t.Errorf("RunOCRRecBatchReal line %s score mismatch: real %v, per-line %v", s, real[i].Score, res[i].Score) + } + } +} + +func TestDLASessionReuse(t *testing.T) { + skipIfNoModels(t) + img, err := Decode(filepath.Join("testdata", "page0.png")) + if err != nil { + t.Fatalf("decode: %v", err) + } + r1, err := RunDLA(t.Context(), os.Getenv("MODEL_DIR"), img) + if err != nil { + t.Fatalf("RunDLA #1: %v", err) + } + r2, err := RunDLA(t.Context(), os.Getenv("MODEL_DIR"), img) + if err != nil { + t.Fatalf("RunDLA #2: %v", err) + } + if r1.Wire() != r2.Wire() { + t.Fatalf("DLA output changed across pooled runs (session reuse not stable)") + } +} + +func TestTSRSessionReuse(t *testing.T) { + skipIfNoModels(t) + img, err := Decode(filepath.Join("testdata", "table0.png")) + if err != nil { + t.Fatalf("decode: %v", err) + } + r1, err := RunTSR(t.Context(), os.Getenv("MODEL_DIR"), img) + if err != nil { + t.Fatalf("RunTSR #1: %v", err) + } + r2, err := RunTSR(t.Context(), os.Getenv("MODEL_DIR"), img) + if err != nil { + t.Fatalf("RunTSR #2: %v", err) + } + if r1.Wire() != r2.Wire() { + t.Fatalf("TSR output changed across pooled runs (session reuse not stable)") + } +} + +func TestOCRRecSessionReuse(t *testing.T) { + skipIfNoModels(t) + img, err := Decode(filepath.Join("testdata", "line0.png")) + if err != nil { + t.Fatalf("decode: %v", err) + } + r1, err := RunOCRRec(t.Context(), os.Getenv("MODEL_DIR"), img) + if err != nil { + t.Fatalf("RunOCRRec #1: %v", err) + } + r2, err := RunOCRRec(t.Context(), os.Getenv("MODEL_DIR"), img) + if err != nil { + t.Fatalf("RunOCRRec #2: %v", err) + } + if r1.Wire() != r2.Wire() { + t.Fatalf("OCR-rec output changed across pooled runs (session reuse not stable)") + } +} + +// wireRunner runs one inference pass and returns its serialized wire output. +type wireRunner func(ctx context.Context, md string, img *Image) (string, error) + +// runConcurrentMatchesSerial drives `run` once serially to establish a baseline +// wire, then `workers` times concurrently from separate goroutines, and asserts +// every concurrent result is byte-identical to the serial baseline. This is +// reviewer follow-up P3: it proves the shared model-session pool is +// concurrency-safe — no cross-call tensor contamination and no data race under +// parallel load. (The pool is the only shared mutable state across calls; the +// per-call input/output tensors are owned by the call.) +func runConcurrentMatchesSerial(t *testing.T, md string, img *Image, name string, run wireRunner) { + base, err := run(t.Context(), md, img) + if err != nil { + t.Fatalf("%s serial: %v", name, err) + } + const workers = 8 + var wg sync.WaitGroup + results := make([]string, workers) + errs := make([]error, workers) + for i := 0; i < workers; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + r, e := run(t.Context(), md, img) + results[i], errs[i] = r, e + }(i) + } + wg.Wait() + for i := 0; i < workers; i++ { + if errs[i] != nil { + t.Fatalf("%s concurrent worker %d: %v", name, i, errs[i]) + } + if results[i] != base { + t.Fatalf("%s concurrent run %d differs from serial baseline (session pool not concurrency-safe)", name, i) + } + } + t.Logf("%s: %d concurrent runs == serial baseline (wire-identical)", name, workers) +} + +// TestInferenceConcurrencyConsistent exercises the shared session pool under +// parallel load for every inference task (DLA / TSR / OCR-rec / Det) and proves +// numerical output is identical to a serial run. Catches data races and +// cross-call contamination that the single-flight reuse tests cannot. +func TestInferenceConcurrencyConsistent(t *testing.T) { + skipIfNoModels(t) + md := os.Getenv("MODEL_DIR") + + dlaImg, err := Decode(filepath.Join("testdata", "page0.png")) + if err != nil { + t.Fatalf("decode dla: %v", err) + } + runConcurrentMatchesSerial(t, md, dlaImg, "DLA", + func(ctx context.Context, m string, im *Image) (string, error) { + r, e := RunDLA(ctx, m, im) + if e != nil { + return "", e + } + return r.Wire(), nil + }) + + tsrImg, err := Decode(filepath.Join("testdata", "table0.png")) + if err != nil { + t.Fatalf("decode tsr: %v", err) + } + runConcurrentMatchesSerial(t, md, tsrImg, "TSR", + func(ctx context.Context, m string, im *Image) (string, error) { + r, e := RunTSR(ctx, m, im) + if e != nil { + return "", e + } + return r.Wire(), nil + }) + + ocrImg, err := Decode(filepath.Join("testdata", "line0.png")) + if err != nil { + t.Fatalf("decode ocr: %v", err) + } + runConcurrentMatchesSerial(t, md, ocrImg, "OCR-rec", + func(ctx context.Context, m string, im *Image) (string, error) { + r, e := RunOCRRec(ctx, m, im) + if e != nil { + return "", e + } + return r.Wire(), nil + }) + + detImg, err := Decode(filepath.Join("testdata", "page0.png")) + if err != nil { + t.Fatalf("decode det: %v", err) + } + runConcurrentMatchesSerial(t, md, detImg, "Det", + func(ctx context.Context, m string, im *Image) (string, error) { + r, e := RunDet(ctx, m, im) + if e != nil { + return "", e + } + return r.Wire(), nil + }) +} + +// TestInferenceConcurrencyMixedStress reproduces the production fan-out profile +// — several simultaneous calls, perTask each of DLA / TSR / OCR-rec / Det — +// against the shared session pools. It runs under -race (see build.sh's +// run_native_integration_tests) so any data race on the pools, the per-session +// in/out tensors, or the cross-call max_wh_ratio batch state surfaces instead +// of being masked by "output happens to look right". Each task's result must +// still equal its serial baseline, proving the pools are safe under real mixed +// contention (not just homogeneous 8-way load). +// +// perTask is capped at 8 (32 concurrent total) on purpose: under -race the +// per-call ORT input tensors are shadowed ~10x, and 100 live concurrent calls +// blow the CI runner's memory budget (the whole package used to run under +// -race and the runner OOM-killed it with SIGTERM). Race detection does not +// need 100 calls — any concurrency level >= 2 exercises the shared pool — so +// 32 is plenty to catch a data race while staying within memory. +func TestInferenceConcurrencyMixedStress(t *testing.T) { + skipIfNoModels(t) + md := os.Getenv("MODEL_DIR") + + dlaImg, _ := Decode(filepath.Join("testdata", "page0.png")) + tsrImg, _ := Decode(filepath.Join("testdata", "table0.png")) + ocrImg, _ := Decode(filepath.Join("testdata", "line0.png")) + detImg, _ := Decode(filepath.Join("testdata", "page0.png")) + + type task struct { + name string + run func(ctx context.Context) (string, error) + } + tasks := []task{ + {"DLA", func(ctx context.Context) (string, error) { + r, e := RunDLA(ctx, md, dlaImg) + if e != nil { + return "", e + } + return r.Wire(), nil + }}, + {"TSR", func(ctx context.Context) (string, error) { + r, e := RunTSR(ctx, md, tsrImg) + if e != nil { + return "", e + } + return r.Wire(), nil + }}, + {"OCR-rec", func(ctx context.Context) (string, error) { + r, e := RunOCRRec(ctx, md, ocrImg) + if e != nil { + return "", e + } + return r.Wire(), nil + }}, + {"Det", func(ctx context.Context) (string, error) { + r, e := RunDet(ctx, md, detImg) + if e != nil { + return "", e + } + return r.Wire(), nil + }}, + } + + // Serial baselines for each task. + baselines := make([]string, len(tasks)) + for i, tk := range tasks { + b, e := tk.run(t.Context()) + if e != nil { + t.Fatalf("baseline %s: %v", tk.name, e) + } + baselines[i] = b + } + + const perTask = 8 + total := perTask * len(tasks) + var wg sync.WaitGroup + errs := make([]error, total) + results := make([]string, total) + for i := 0; i < total; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + tk := tasks[i%len(tasks)] + r, e := tk.run(t.Context()) + results[i], errs[i] = r, e + }(i) + } + wg.Wait() + + for i := 0; i < total; i++ { + tk := tasks[i%len(tasks)] + base := baselines[i%len(tasks)] + if errs[i] != nil { + t.Fatalf("%s concurrent call %d: %v", tk.name, i, errs[i]) + } + if results[i] != base { + t.Fatalf("%s concurrent call %d diverges from serial baseline (pool not concurrency-safe under mixed load)", tk.name, i) + } + } + t.Logf("mixed stress: %d concurrent calls (25×4 tasks) == serial baselines (wire-identical)", total) +} + +func TestDetIntegration(t *testing.T) { + skipIfNoModels(t) + imgPath := filepath.Join("testdata", "page0.png") + img, err := Decode(imgPath) + if err != nil { + t.Fatalf("decode: %v", err) + } + res, err := RunDet(t.Context(), os.Getenv("MODEL_DIR"), img) + if err != nil { + t.Fatalf("RunDet: %v", err) + } + // Go wire: {"output": [[ [ [x,y]*4, ... ] ]]}; boxes at output[0][0]. + var got struct { + Output [][][][][2]float64 `json:"output"` + } + if err := json.Unmarshal([]byte(res.Wire()), &got); err != nil { + t.Fatalf("parse Go wire: %v", err) + } + if len(got.Output) == 0 || len(got.Output[0]) == 0 { + t.Fatalf("Go wire missing boxes") + } + goBoxes := got.Output[0][0] + + raw, err := os.ReadFile(filepath.Join("testdata", "page0.det.golden.json")) + if err != nil { + t.Fatalf("read golden: %v", err) + } + var gold struct { + Output [][][][][2]float64 `json:"output"` + } + if err := json.Unmarshal(raw, &gold); err != nil { + t.Fatalf("parse golden: %v", err) + } + if len(gold.Output) == 0 || len(gold.Output[0]) == 0 { + t.Fatalf("golden missing boxes") + } + refBoxes := gold.Output[0][0] + + // The pure-Go geometry reaches the 3px hard floor (README.md §3, box#8). + // Tolerance is CoordFloor + CoordTolMargin, just above that floor, so a + // regression bumps it over the line. + detCoordTol := CoordFloor + CoordTolMargin + used := make([]bool, len(goBoxes)) + matched, maxd := 0, 0.0 + for _, rb := range refBoxes { + rcx, rcy := quadCenter(rb) + best, bd := -1, math.MaxFloat64 + for i, vb := range goBoxes { + if used[i] { + continue + } + vcx, vcy := quadCenter(vb) + d := (rcx-vcx)*(rcx-vcx) + (rcy-vcy)*(rcy-vcy) + if d < bd { + bd, best = d, i + } + } + if best < 0 { + t.Errorf("no Go box matched golden quad at (%.0f,%.0f)", rcx, rcy) + continue + } + used[best] = true + matched++ + for j := 0; j < 4; j++ { + for k := 0; k < 2; k++ { + diff := math.Abs(rb[j][k] - goBoxes[best][j][k]) + if diff > detCoordTol { + t.Errorf("quad coord diff %.3f > tol %.2f (gold=%v got=%v)", + diff, detCoordTol, rb, goBoxes[best]) + } + if diff > maxd { + maxd = diff + } + } + } + } + t.Logf("det: matched %d/%d golden quads, max coord diff %.4f px (tol %.1f)", + matched, len(refBoxes), maxd, detCoordTol) + if matched != len(refBoxes) { + t.Errorf("matched %d/%d quads", matched, len(refBoxes)) + } +} + +// TestDetMembershipAllFixtures quantifies the det box-membership gap (gap 3). +// For every committed det fixture it runs RunDet and compares the Go box set +// against the golden (Python) box set in TWO complementary ways: +// 1. center-distance match (tol 3.5px) — isolates coordinate drift; +// 2. IoU match (thr 0.5) — isolates true box-membership divergence (splits, +// merges, hallucinations), independent of how far a box's center moved. +// +// The original TestDetIntegration only checked golden→Go by nearest center on +// a single fixture, so a Go box shifted >3.5px was mis-flagged and a Go box +// with no golden counterpart was invisible. +// +// This is a REGRESSION GUARD, not a zero-target. The IoU orphan counts are +// pinned to the baseline measured on 2026-08-10 (gap 3: 37 golden misses + 20 +// extra Go boxes across 37 fixtures, concentrated in dense-text pages). The +// test fails ONLY if a future geometry change makes Go WORSE than that +// baseline — it does not require the gap to reach zero. A small slack absorbs +// run-to-run nondeterminism (see gap 7); it is NOT headroom for new divergence. +func TestDetMembershipAllFixtures(t *testing.T) { + skipIfNoModels(t) + md := os.Getenv("MODEL_DIR") + dir := "testdata" + + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatalf("read testdata: %v", err) + } + + type stat struct { + stem string + nGold, nGo int + centerOrphanGold, centerOrphanGo int + iouMatchedGold, iouMatchedGo int + iouOrphanGold, iouOrphanGo int + maxd float64 + } + var stats []stat + sumGold, sumGo := 0, 0 + sumCIoG, sumCIoGo := 0, 0 + sumIIoG, sumIIoGo := 0, 0 + + for _, e := range entries { + name := e.Name() + if !strings.HasSuffix(name, ".det.golden.json") { + continue + } + stem := strings.TrimSuffix(name, ".det.golden.json") + img, err := Decode(filepath.Join(dir, stem+".png")) + if err != nil { + t.Fatalf("decode %s: %v", stem, err) + } + res, err := RunDet(t.Context(), md, img) + if err != nil { + t.Fatalf("RunDet %s: %v", stem, err) + } + var got struct { + Output [][][][][2]float64 `json:"output"` + } + if err := json.Unmarshal([]byte(res.Wire()), &got); err != nil { + t.Fatalf("parse Go wire %s: %v", stem, err) + } + goBoxes := FlattenQuads(got.Output) + + raw, err := os.ReadFile(filepath.Join(dir, name)) + if err != nil { + t.Fatalf("read golden %s: %v", stem, err) + } + var gold struct { + Output [][][][][2]float64 `json:"output"` + } + if err := json.Unmarshal(raw, &gold); err != nil { + t.Fatalf("parse golden %s: %v", stem, err) + } + goldBoxes := FlattenQuads(gold.Output) + + mG, mGo, maxd := MatchBothDirections(goldBoxes, goBoxes, CmpTolCoord) + imG, imGo := MatchIoUBothDirections(goldBoxes, goBoxes, 0.5) + s := stat{ + stem: stem, + nGold: len(goldBoxes), + nGo: len(goBoxes), + centerOrphanGold: len(goldBoxes) - mG, + centerOrphanGo: len(goBoxes) - mGo, + iouMatchedGold: imG, + iouMatchedGo: imGo, + iouOrphanGold: len(goldBoxes) - imG, + iouOrphanGo: len(goBoxes) - imGo, + maxd: maxd, + } + stats = append(stats, s) + sumGold += s.nGold + sumGo += s.nGo + sumCIoG += s.centerOrphanGold + sumCIoGo += s.centerOrphanGo + sumIIoG += s.iouOrphanGold + sumIIoGo += s.iouOrphanGo + } + + for _, s := range stats { + t.Logf("det %-22s gold=%d go=%d | centerOrphan(g/g)=%d/%d maxd=%.1f | iouOrphan(g/g)=%d/%d", + s.stem, s.nGold, s.nGo, s.centerOrphanGold, s.centerOrphanGo, s.maxd, s.iouOrphanGold, s.iouOrphanGo) + } + t.Logf("TOTAL gold=%d go=%d", sumGold, sumGo) + t.Logf("center orphan(gold/go)=%d/%d | IoU orphan(gold/go)=%d/%d", + sumCIoG, sumCIoGo, sumIIoG, sumIIoGo) + + // Regression guard: the IoU orphan baseline is the KNOWN gap, not a target. + // Fail only if Go gets WORSE than the baseline — a future geometry change + // must not introduce new box-membership divergence. The slack absorbs benign + // run-to-run nondeterminism. + // + // Baseline history: + // - 37/20 — pinned against the old non-bit-exact FillConvexPoly scanline + // (over-drew, kept 5 boxes at the 0.5 threshold by coincidence). Wrong. + // - 42/13 — after fillPoly was rewritten to bit-match cv2.fillPoly, the + // true gap vs the THEN-COMMITTED goldens. Those goldens were later found + // STALE: they were ~20 boxes denser than the current live TextDetector + // (generated with an older onnxruntime; the drift gate's 15%-count-only + // det check masked it), so 42/13 over-reported Go's divergence. + // - 23/9 — re-measured on 2026-08-11 AFTER the .det.golden.json fixtures + // were regenerated from the current live TextDetector and the contour + // grouping was ported to a findContours-equivalent (Suzuki-Abe / Moore- + // neighbour border following, RETR_LIST). This was the true Go-vs-cv2 + // box-membership gap; the residual 23/9 was a downstream pred-map + // divergence (R/B channel-order swap in normalizeCHW: Go applied the + // ImageNet stats to BGR bytes while deepdoc's TextDetector normalizes + // the RGB image directly, so 0.485 landed on B in Go and on R in + // deepdoc — a ~3e-3 pred-map gap that box_score_fast amplified into + // score-crossing orphans). + // - 3/5 — re-measured on 2026-08-12 AFTER normalizeCHW was fixed to feed + // RGB bytes (detPreprocess now uses img.Pix, not img.ToBGR) with + // RGB-order stats, matching deepdoc. The Go det pred map now matches + // the live TextDetector to mean|Δ|≈4e-5 (was 3.2e-3); the only + // remaining IoU orphans are contour-tracer geometry (e.g. mp_cn_sm_p0 + // gold 303 vs go 306), not pred/score. Do not raise this baseline back + // toward 23/9, 42/13 or 37/20 — those tracked a swapped-channel oracle + // or a stale golden, not Go's real divergence. + const ( + baselineIoUOrphanGold = 3 + baselineIoUOrphanGo = 5 + iuSlack = 3 + ) + if sumIIoG > baselineIoUOrphanGold+iuSlack { + t.Errorf("Go missed %d golden boxes under IoU (> baseline %d+%d): box-membership REGRESSION (gap 3)", + sumIIoG, baselineIoUOrphanGold, iuSlack) + } + if sumIIoGo > baselineIoUOrphanGo+iuSlack { + t.Errorf("Go produced %d extra boxes under IoU (> baseline %d+%d): box-membership REGRESSION (gap 3)", + sumIIoGo, baselineIoUOrphanGo, iuSlack) + } + t.Logf("IoU orphan baseline(gold/go)=%d/%d (+slack %d) — current %d/%d OK", + baselineIoUOrphanGold, baselineIoUOrphanGo, iuSlack, sumIIoG, sumIIoGo) +} + +// TestDetOCRAdjudication answers a different question than +// TestDetMembershipAllFixtures. The membership test measures how close Go's +// boxes are to Python's (alignment). This harness measures which detector +// produces boxes that lead to BETTER parsing: for every box only one side +// found (an "IoU orphan"), it crops the original image to that box and runs +// OCR-rec, then reports whether the crop yields coherent text. The side whose +// orphan boxes more often produce real text is the side that detected genuine +// text regions the other missed. Python is NOT assumed truth here — OCR +// quality is the independent judge. +// +// Informational only: it never fails. Per-box text is logged so a human can +// adjudicate; the summary tallies non-empty, confident OCR results per side as +// a proxy for "found real text". The crop is axis-aligned (not rotated) so both +// detectors are judged by the same method. +func TestDetOCRAdjudication(t *testing.T) { + skipIfNoModels(t) + md := os.Getenv("MODEL_DIR") + dir := "testdata" + + // Fixtures with non-zero IoU orphans (the only ones worth adjudicating). + stems := []string{ + "mp_cn_sm_p0", // dense Chinese small text — worst divergence + "mp_arxiv_p0", // multi-column paper + "mp_en_dense_p0", // dense English + "mp_physics_p5", + "mp_sec_p0", + } + + const ( + iouThr = 0.5 + realScoreThr = 0.6 // OCR confidence above which a crop is "likely real text" + ) + + sumPyOnly, sumPyOnlyReal := 0, 0 + sumGoOnly, sumGoOnlyReal := 0, 0 + + for _, stem := range stems { + img, err := Decode(filepath.Join(dir, stem+".png")) + if err != nil { + t.Fatalf("decode %s: %v", stem, err) + } + res, err := RunDet(t.Context(), md, img) + if err != nil { + t.Fatalf("RunDet %s: %v", stem, err) + } + var got struct { + Output [][][][][2]float64 `json:"output"` + } + if err := json.Unmarshal([]byte(res.Wire()), &got); err != nil { + t.Fatalf("parse Go wire %s: %v", stem, err) + } + goBoxes := FlattenQuads(got.Output) + + raw, err := os.ReadFile(filepath.Join(dir, stem+".det.golden.json")) + if err != nil { + t.Fatalf("read golden %s: %v", stem, err) + } + var gold struct { + Output [][][][][2]float64 `json:"output"` + } + if err := json.Unmarshal(raw, &gold); err != nil { + t.Fatalf("parse golden %s: %v", stem, err) + } + pyBoxes := FlattenQuads(gold.Output) + + pyOnly, goOnly := matchIoUOrphans(pyBoxes, goBoxes, iouThr) + fPyOnly, fPyOnlyReal := 0, 0 + fGoOnly, fGoOnlyReal := 0, 0 + + for _, i := range pyOnly { + txt, score, ok := ocrCrop(t, md, img, pyBoxes[i]) + real := ok && strings.TrimSpace(txt) != "" && score >= realScoreThr + if ok { + fPyOnly++ + if real { + fPyOnlyReal++ + } + } + t.Logf(" [PY-only] %-14s box#%d text=%q score=%.2f real=%v", stem, i, txt, score, real) + } + for _, i := range goOnly { + txt, score, ok := ocrCrop(t, md, img, goBoxes[i]) + real := ok && strings.TrimSpace(txt) != "" && score >= realScoreThr + if ok { + fGoOnly++ + if real { + fGoOnlyReal++ + } + } + t.Logf(" [GO-only] %-14s box#%d text=%q score=%.2f real=%v", stem, i, txt, score, real) + } + + t.Logf("%-14s: PY-only=%d (real %d) | GO-only=%d (real %d)", stem, fPyOnly, fPyOnlyReal, fGoOnly, fGoOnlyReal) + sumPyOnly += fPyOnly + sumPyOnlyReal += fPyOnlyReal + sumGoOnly += fGoOnly + sumGoOnlyReal += fGoOnlyReal + } + + t.Logf("SUMMARY — orphan boxes that yielded real text (score>=%.1f):", realScoreThr) + t.Logf(" Python-only: %d/%d real", sumPyOnlyReal, sumPyOnly) + t.Logf(" Go-only: %d/%d real", sumGoOnlyReal, sumGoOnly) + t.Logf(" (higher 'real' count on a side = that side found more genuine text the other missed)") +} + +// ocrCrop crops img to the axis-aligned bbox of quad q, runs OCR-rec, and +// returns the recognized text, confidence, and whether the crop was valid. +func ocrCrop(t *testing.T, md string, img *Image, q [][2]float64) (string, float32, bool) { + c := cropQuad(img, q) + if c == nil { + return "", 0, false + } + r, err := RunOCRRec(t.Context(), md, c) + if err != nil { + t.Logf(" ocrCrop rec error: %v", err) + return "", 0, false + } + return r.Text, r.Score, true +} + +// cropQuad returns the axis-aligned sub-image of img bounded by quad q, or nil +// if the quad is degenerate/out of bounds. AABB (not rotated) crop is used so +// both detectors are judged by the same method — any background inclusion +// affects both sides equally and the comparison stays fair. +func cropQuad(img *Image, q [][2]float64) *Image { + x0, y0, x1, y1 := quadAABB(q) + px0 := clampi(int(math.Floor(x0)), 0, img.W-1) + py0 := clampi(int(math.Floor(y0)), 0, img.H-1) + px1 := clampi(int(math.Ceil(x1)), 0, img.W) + py1 := clampi(int(math.Ceil(y1)), 0, img.H) + if px1 <= px0 || py1 <= py0 { + return nil + } + w, h := px1-px0, py1-py0 + pix := make([]byte, w*h*3) + for y := 0; y < h; y++ { + copy(pix[y*w*3:(y+1)*w*3], img.Pix[((py0+y)*img.W+px0)*3:((py0+y)*img.W+px0)*3+w*3]) + } + return &Image{W: w, H: h, Pix: pix} +} + +// matchIoUOrphans returns the indices (into pyBoxes / goBoxes) of boxes that +// have no counterpart on the other side under greedy best-IoU >= thr. These +// are the "orphan" boxes found by only one detector. Greedy matching is run in +// both directions so a box with no stable twin on either side is reported. +func matchIoUOrphans(pyBoxes, goBoxes [][][2]float64, thr float64) (pyOnly, goOnly []int) { + usedGo := make([]bool, len(goBoxes)) + pyMatched := make([]bool, len(pyBoxes)) + for i, gb := range pyBoxes { + best, bestI := -1, 0.0 + for j, vb := range goBoxes { + if usedGo[j] { + continue + } + if v := iou(gb, vb); v > bestI { + bestI, best = v, j + } + } + if best >= 0 && bestI >= thr { + usedGo[best] = true + pyMatched[i] = true + } + } + for i := range pyBoxes { + if !pyMatched[i] { + pyOnly = append(pyOnly, i) + } + } + usedPy := make([]bool, len(pyBoxes)) + goMatched := make([]bool, len(goBoxes)) + for j, vb := range goBoxes { + best, bestI := -1, 0.0 + for i, gb := range pyBoxes { + if usedPy[i] { + continue + } + if v := iou(gb, vb); v > bestI { + bestI, best = v, i + } + } + if best >= 0 && bestI >= thr { + usedPy[best] = true + goMatched[j] = true + } + } + for j := range goBoxes { + if !goMatched[j] { + goOnly = append(goOnly, j) + } + } + return pyOnly, goOnly +} + +// TestDetSessionPoolBounded is the regression guard for the unbounded +// sync.Map leak: a long-running server ingesting many differently-sized pages +// used to pin one pool + cached tensors per unique (modelPath, rh, rw) forever. +// It now bounds the pool set (detMaxShapePools). We drive far more distinct +// page sizes than the cap and assert the live pool count never exceeds it. +func TestDetSessionPoolBounded(t *testing.T) { + skipIfNoModels(t) + modelDir := os.Getenv("MODEL_DIR") + + const n = 80 + for i := 0; i < n; i++ { + w := 64 + i*14 + h := 128 + (i*9)%500 + if w > 952 { + w = 952 + } + if h > 952 { + h = 952 + } + // Synthetic gray raster; we only exercise the pool lifecycle here, not + // detection quality, so the content is irrelevant. + img := &Image{W: w, H: h, Pix: make([]byte, w*h*3)} + for j := 0; j < len(img.Pix); j += 3 { + img.Pix[j], img.Pix[j+1], img.Pix[j+2] = 200, 200, 200 + } + if _, err := RunDet(t.Context(), modelDir, img); err != nil { + t.Fatalf("RunDet #%d (%dx%d): %v", i, w, h, err) + } + } + + got := detSessions.KeyCount() + if got > detMaxShapePools { + t.Fatalf("det session pool set grew unbounded: %d pools (cap %d) after %d distinct sizes", + got, detMaxShapePools, n) + } + t.Logf("det session pool set bounded: %d pools (cap %d) after %d distinct sizes", got, detMaxShapePools, n) +} + +// jsonTemplate reduces a decoded JSON value to a structural signature where +// every number becomes "#", every string "$", every bool "?", and object keys +// are sorted. Two values with identical nesting/keys/leaf-types produce the +// same template even if their values differ — so it isolates schema from +// content. +func jsonTemplate(v any) string { + switch x := v.(type) { + case map[string]any: + keys := make([]string, 0, len(x)) + for k := range x { + keys = append(keys, k) + } + sort.Strings(keys) + var b strings.Builder + b.WriteString("{") + for i, k := range keys { + if i > 0 { + b.WriteString(",") + } + b.WriteString(k) + b.WriteString(":") + b.WriteString(jsonTemplate(x[k])) + } + b.WriteString("}") + return b.String() + case []any: + if len(x) == 0 { + return "[]" + } + // Structures here have uniform inner shape, so the first element's + // template represents the whole array. + return "[" + jsonTemplate(x[0]) + "]" + case float64: + return "#" + case string: + return "$" + case bool: + return "?" + case nil: + return "null" + default: + return "?" + } +} + +// TestWireSchemaMatchesGolden is the schema half of the migration contract: a +// caller parsing Go's Wire() output must see the exact same JSON structure +// (top-level key, nesting depth, leaf types) as the deepdoc reference golden. +// The per-task integration tests already check values; this guard catches a +// shape regression (e.g. {"bboxes":...} vs a bare array, or a changed nesting) +// that value comparison would otherwise paper over. +func TestWireSchemaMatchesGolden(t *testing.T) { + skipIfNoModels(t) + md := os.Getenv("MODEL_DIR") + cases := []struct { + name string + stem string + golden string + wire func(t *testing.T, img *Image) string + }{ + {"DLA", "page0", "page0.dla.golden.json", func(t *testing.T, img *Image) string { + res, err := RunDLA(t.Context(), md, img) + if err != nil { + t.Fatalf("RunDLA: %v", err) + } + return res.Wire() + }}, + {"TSR", "table0", "table0.tsr.golden.json", func(t *testing.T, img *Image) string { + res, err := RunTSR(t.Context(), md, img) + if err != nil { + t.Fatalf("RunTSR: %v", err) + } + return res.Wire() + }}, + {"DET", "page0", "page0.det.golden.json", func(t *testing.T, img *Image) string { + res, err := RunDet(t.Context(), md, img) + if err != nil { + t.Fatalf("RunDet: %v", err) + } + return res.Wire() + }}, + {"OCR_REC", "line_mix", "line_mix.ocr_rec.golden.json", func(t *testing.T, img *Image) string { + res, err := RunOCRRec(t.Context(), md, img) + if err != nil { + t.Fatalf("RunOCRRec: %v", err) + } + return res.Wire() + }}, + } + for _, c := range cases { + c := c + t.Run(c.name, func(t *testing.T) { + img, err := Decode(filepath.Join("testdata", c.stem+".png")) + if err != nil { + t.Fatalf("decode %s: %v", c.stem, err) + } + goWire := c.wire(t, img) + goldRaw, err := os.ReadFile(filepath.Join("testdata", c.golden)) + if err != nil { + t.Fatalf("read golden %s: %v", c.golden, err) + } + var gv, gv2 any + if err := json.Unmarshal([]byte(goWire), &gv); err != nil { + t.Fatalf("parse go wire: %v", err) + } + if err := json.Unmarshal(goldRaw, &gv2); err != nil { + t.Fatalf("parse golden: %v", err) + } + if got, want := jsonTemplate(gv), jsonTemplate(gv2); got != want { + t.Errorf("wire schema mismatch:\n got %s\n want %s", got, want) + } else { + t.Logf("%s wire schema: %s", c.name, got) + } + }) + } +} + +// TestDumpGoCandidates is a diagnostic: run RunDet on one fixture with +// DLA_DUMP_CANDIDATES set so dbPostProcess writes /tmp/go_candidates.json +// (post-geometry, pre-score-filter quads + scores) for offline analysis of +// Go/cv2 det divergence. Not a regression test. +func TestDumpGoCandidates(t *testing.T) { + skipIfNoModels(t) + fixture := os.Getenv("FIXTURE") + if fixture == "" { + fixture = "mp_cn_sm_p0" + } + t.Setenv("DLA_DUMP_CANDIDATES", "1") + imgPath := filepath.Join("testdata", fixture+".png") + img, err := Decode(imgPath) + if err != nil { + t.Fatalf("decode %s: %v", imgPath, err) + } + if _, err := RunDet(t.Context(), os.Getenv("MODEL_DIR"), img); err != nil { + t.Fatalf("RunDet: %v", err) + } + t.Logf("wrote /tmp/go_candidates.json for %s", fixture) +} + +// TestDumpStages is a diagnostic: run RunDet on one fixture (FIXTURE env) with +// the stage-dump env vars set, writing the Go-side intermediates that the +// Python oracle (cmp_stages.py) is diffed against: +// +// /tmp/go_pred.json — raw pred map (post-sigmoid, pre-threshold) +// /tmp/go_quads_pre.json — pre-unclip min-area rect per component (resized) +// /tmp/go_candidates.json — post-geometry, pre-score-filter quad+score +// +// Not a regression test. Pair with: python cmp_stages.py ; +// then python diff_stages.py for the per-stage comparison. +func TestDumpStages(t *testing.T) { + skipIfNoModels(t) + fixture := os.Getenv("FIXTURE") + if fixture == "" { + fixture = "mp_cn_sm_p0" + } + t.Setenv("DLA_DUMP_STAGES", "1") + t.Setenv("DLA_DUMP_QUADS", "1") + t.Setenv("DLA_DUMP_CANDIDATES", "1") + imgPath := filepath.Join("testdata", fixture+".png") + img, err := Decode(imgPath) + if err != nil { + t.Fatalf("decode %s: %v", imgPath, err) + } + res, err := RunDet(t.Context(), os.Getenv("MODEL_DIR"), img) + if err != nil { + t.Fatalf("RunDet: %v", err) + } + // Final boxes (post score-filter + filter_tag_det_res), for direct IoU + // comparison against the Python oracle's final output (ref_det.py). + fb := make([]map[string]any, 0, len(res.Boxes)) + for _, b := range res.Boxes { + pts := make([][2]float64, 4) + for i := range b.Pts { + pts[i] = [2]float64{float64(b.Pts[i][0]), float64(b.Pts[i][1])} + } + fb = append(fb, map[string]any{"pts": pts, "score": float64(b.Score)}) + } + if b, e := json.Marshal(map[string]any{"boxes": fb}); e == nil { + _ = os.WriteFile("/tmp/go_final.json", b, 0o644) + } + t.Logf("wrote /tmp/go_pred.json, /tmp/go_quads_pre.json, /tmp/go_candidates.json, /tmp/go_final.json for %s", fixture) +} + +// TestEquivalenceReport runs every native recognizer against the Python +// reference goldens and prints one consolidated equivalence summary to the test +// log (visible in CI). It is the human-readable counterpart to the per-task +// integration tests: with a single command it shows that Go's det / DLA / TSR / +// OCR outputs match the Python deepdoc inference service to within the +// documented, bounded floors. It also acts as a light regression guard (DLA / +// TSR boxes and OCR text must match exactly; the full-fixture det floor is +// guarded separately by TestDetMembershipAllFixtures). +// +// Requires MODEL_DIR (//go:build integration). +func TestEquivalenceReport(t *testing.T) { + skipIfNoModels(t) + md := os.Getenv("MODEL_DIR") + // P0: refuse to produce an equivalence report against a drifted model + // snapshot — the goldens would be meaningless. + checkModelSnapshotHash(t, md) + dir := "testdata" + + type row struct { + task string + fix int + match int + total int + maxD float64 + note string + } + var rows []row + + // DLA — layout detection: every golden box matched within 2px. + { + m, tot, mx := 0, 0, 0.0 + for _, stem := range dlaPages { + img, err := Decode(filepath.Join(dir, stem+".png")) + if err != nil { + t.Fatalf("decode %s: %v", stem, err) + } + res, err := RunDLA(t.Context(), md, img) + if err != nil { + t.Fatalf("RunDLA %s: %v", stem, err) + } + var got struct { + Boxes [][]float64 `json:"bboxes"` + } + if err := json.Unmarshal([]byte(res.Wire()), &got); err != nil { + t.Fatalf("wire %s: %v", stem, err) + } + gold := LoadGoldenBoxes(t, filepath.Join(dir, stem+".dla.golden.json")) + mm, mx2, _ := MatchBoxesRelaxed(t, gold, got.Boxes, CmpTolCoord, CmpTolScore) + m += mm + tot += len(gold) + mx = math.Max(mx, mx2) + } + rows = append(rows, row{"DLA (layout)", len(dlaPages), m, tot, mx, "sub-pixel"}) + } + + // TSR — table structure: every golden box matched within 2px. + { + m, tot, mx := 0, 0, 0.0 + for _, stem := range tsrPages { + img, err := Decode(filepath.Join(dir, stem+".png")) + if err != nil { + t.Fatalf("decode %s: %v", stem, err) + } + res, err := RunTSR(t.Context(), md, img) + if err != nil { + t.Fatalf("RunTSR %s: %v", stem, err) + } + var got struct { + Boxes [][]float64 `json:"bboxes"` + } + if err := json.Unmarshal([]byte(res.Wire()), &got); err != nil { + t.Fatalf("wire %s: %v", stem, err) + } + gold := LoadGoldenBoxes(t, filepath.Join(dir, stem+".tsr.golden.json")) + mm, mx2, _ := MatchBoxesRelaxed(t, gold, got.Boxes, CmpTolCoord, CmpTolScore) + m += mm + tot += len(gold) + mx = math.Max(mx, mx2) + } + rows = append(rows, row{"TSR (table)", len(tsrPages), m, tot, mx, "<1px (<=10px @4:1 aspect)"}) + } + + // OCR — text recognition: exact text match. + { + m, tot := 0, 0 + for _, stem := range ocrRecLines { + img, err := Decode(filepath.Join(dir, stem+".png")) + if err != nil { + t.Fatalf("decode %s: %v", stem, err) + } + res, err := RunOCRRec(t.Context(), md, img) + if err != nil { + t.Fatalf("RunOCRRec %s: %v", stem, err) + } + raw, err := os.ReadFile(filepath.Join(dir, stem+".ocr_rec.golden.json")) + if err != nil { + t.Fatalf("read golden %s: %v", stem, err) + } + var gold struct { + Output [][][][]any `json:"output"` + } + if err := json.Unmarshal(raw, &gold); err != nil { + t.Fatalf("parse golden %s: %v", stem, err) + } + gotText := res.Text + goldText := gold.Output[0][0][0][0].(string) + tot++ + if gotText == goldText { + m++ + } else { + t.Logf("OCR mismatch %s: got %q gold %q", stem, gotText, goldText) + } + } + rows = append(rows, row{"OCR (text)", len(ocrRecLines), m, tot, 0, "exact text"}) + } + + // Det — text detection IoU box-membership floor (curated dense subset). + { + const iouThr = 0.5 + detStems := []string{"mp_arxiv_p0", "mp_cn_sm_p0", "mp_en_dense_p0", "mp_sec_p0", "mp_physics_p5"} + orphanG, orphanGo, tot := 0, 0, 0 + for _, stem := range detStems { + img, err := Decode(filepath.Join(dir, stem+".png")) + if err != nil { + t.Fatalf("decode %s: %v", stem, err) + } + res, err := RunDet(t.Context(), md, img) + if err != nil { + t.Fatalf("RunDet %s: %v", stem, err) + } + var got struct { + Output [][][][][2]float64 `json:"output"` + } + if err := json.Unmarshal([]byte(res.Wire()), &got); err != nil { + t.Fatalf("wire %s: %v", stem, err) + } + goBoxes := FlattenQuads(got.Output) + raw, err := os.ReadFile(filepath.Join(dir, stem+".det.golden.json")) + if err != nil { + t.Fatalf("read golden %s: %v", stem, err) + } + var gold struct { + Output [][][][][2]float64 `json:"output"` + } + if err := json.Unmarshal(raw, &gold); err != nil { + t.Fatalf("parse golden %s: %v", stem, err) + } + goldBoxes := FlattenQuads(gold.Output) + imG, imGo := MatchIoUBothDirections(goldBoxes, goBoxes, iouThr) + orphanG += len(goldBoxes) - imG + orphanGo += len(goBoxes) - imGo + tot += len(goldBoxes) + } + rows = append(rows, row{"Det (text boxes)", len(detStems), tot - orphanG, tot, 0, + fmt.Sprintf("IoU orphan(gold/go)=%d/%d (accepted floor 3/5)", orphanG, orphanGo)}) + } + + // Consolidated summary — visible in CI logs. + t.Logf("===== DLA-NATIVE vs PYTHON DEEP DOC: EQUIVALENCE SUMMARY =====") + t.Logf("%-18s %4s %9s %10s %s", "TASK", "FIX", "MATCH/TOT", "MAXΔ(px)", "NOTE") + for _, r := range rows { + t.Logf("%-18s %4d %4d/%-4d %10.3f %s", r.task, r.fix, r.match, r.total, r.maxD, r.note) + } + t.Logf("==================================================================") + + // Light regression guard: DLA/TSR boxes and OCR text must match exactly. + // The full-fixture det floor is guarded by TestDetMembershipAllFixtures. + for _, r := range rows { + if r.task == "DLA (layout)" || r.task == "TSR (table)" || r.task == "OCR (text)" { + if r.match != r.total { + t.Errorf("%s: %d/%d matched — equivalence REGRESSION", r.task, r.match, r.total) + } + } + } +} + +// compareBBoxesJSON compares a server {"bboxes":[...]} response against the Go +// Wire() {"bboxes":[...]} output. The server response is the reference ("gold"); +// Go is "got". Matching is class-aware with the documented coordinate/score +// tolerances. +func compareBBoxesJSON(t *testing.T, serverJSON, goJSON []byte, kind string, scoreTol float64) { + t.Helper() + var s struct { + Bboxes [][]float64 `json:"bboxes"` + } + if err := json.Unmarshal(serverJSON, &s); err != nil { + t.Fatalf("parse server %s json: %v", kind, err) + } + var g struct { + Bboxes [][]float64 `json:"bboxes"` + } + if err := json.Unmarshal(goJSON, &g); err != nil { + t.Fatalf("parse go %s json: %v", kind, err) + } + matched, maxd, _ := MatchBoxesRelaxed(t, s.Bboxes, g.Bboxes, CmpTolCoord, scoreTol) + t.Logf("%s: server=%d go=%d matched=%d maxd=%.3f px", kind, len(s.Bboxes), len(g.Bboxes), matched, maxd) + if matched != len(s.Bboxes) { + t.Errorf("%s: %d/%d server boxes matched by Go (missing %d)", kind, matched, len(s.Bboxes), len(s.Bboxes)-matched) + } +} + +// compareDetJSON compares a server det {"output":[[quads]]} response against the +// Go Wire() det output. The only accepted divergence between Go and the live +// cv2-backed server is the documented Det 3/5 contour-tracer floor (+slack); +// anything larger is a real Go-vs-service regression. +func compareDetJSON(t *testing.T, serverJSON, goJSON []byte) { + t.Helper() + srvBoxes := flattenOutput(t, serverJSON, "det") + goBoxes := flattenOutput(t, goJSON, "det") + imG, imGo := MatchIoUBothDirections(srvBoxes, goBoxes, 0.5) + orphanG := len(srvBoxes) - imG + orphanGo := len(goBoxes) - imGo + t.Logf("det: server=%d go=%d | IoU orphan(g/g)=%d/%d", len(srvBoxes), len(goBoxes), orphanG, orphanGo) + const ( + baselineG = 3 + baselineGo = 5 + slack = 3 + ) + if orphanG > baselineG+slack { + t.Errorf("det: %d/%d server boxes unmatched under IoU (> baseline %d+%d)", imG, len(srvBoxes), baselineG, slack) + } + if orphanGo > baselineGo+slack { + t.Errorf("det: Go produced %d extra boxes under IoU (> baseline %d+%d)", orphanGo, baselineGo, slack) + } +} + +// compareRecJSON compares a server rec {"output":[[[text,1.0]]]} response against +// the Go Wire() rec output, exactly (text must be identical). +func compareRecJSON(t *testing.T, serverJSON, goJSON []byte) { + t.Helper() + srvTexts := flattenTexts(t, serverJSON) + goTexts := flattenTexts(t, goJSON) + if len(srvTexts) != len(goTexts) { + t.Errorf("rec: count mismatch server=%d go=%d", len(srvTexts), len(goTexts)) + } + n := len(srvTexts) + if len(goTexts) < n { + n = len(goTexts) + } + for i := 0; i < n; i++ { + if srvTexts[i] != goTexts[i] { + t.Errorf("rec[%d]: server %q != go %q", i, srvTexts[i], goTexts[i]) + } + } +} + +// flattenOutput extracts the quad list from a det Wire()/server payload +// (boxes live under output[0][0]). +func flattenOutput(t *testing.T, raw []byte, kind string) [][][2]float64 { + t.Helper() + var v struct { + Output [][][][][2]float64 `json:"output"` + } + if err := json.Unmarshal(raw, &v); err != nil { + t.Fatalf("parse %s json: %v", kind, err) + } + return FlattenQuads(v.Output) +} + +// flattenTexts extracts the recognized strings from a rec Wire()/server payload +// (pairs live under output[0][0], text is pair[0]). +func flattenTexts(t *testing.T, raw []byte) []string { + t.Helper() + var v struct { + Output [][][]any `json:"output"` + } + if err := json.Unmarshal(raw, &v); err != nil { + t.Fatalf("parse rec json: %v", err) + } + if len(v.Output) == 0 || len(v.Output[0]) == 0 { + return nil + } + var out []string + for _, item := range v.Output[0][0] { + pair, ok := item.([]any) + if !ok || len(pair) < 1 { + continue + } + if s, ok := pair[0].(string); ok { + out = append(out, s) + } + } + return out +} diff --git a/internal/deepdoc/native/native_test.go b/internal/deepdoc/native/native_test.go new file mode 100644 index 0000000000..f8d19201f9 --- /dev/null +++ b/internal/deepdoc/native/native_test.go @@ -0,0 +1,229 @@ +//go:build cgo + +package native + +// Unit tests for the pure (model-free) pieces of package native. +// These run under the default `go test ./...` (no ONNX Runtime, no models). +// Model-backed end-to-end checks live in native_integration_test.go (build tag +// `integration`) so they are excluded from the default unit run. + +import ( + "math" + "strings" + "testing" +) + +func TestNMS(t *testing.T) { + boxes := []nmsBox{ + {X0: 0, Y0: 0, X1: 10, Y1: 10, Score: 0.9}, + {X0: 0, Y0: 0, X1: 10, Y1: 10, Score: 0.8}, // duplicate -> suppressed + {X0: 100, Y0: 100, X1: 110, Y1: 110, Score: 0.7}, // far away -> kept + } + keep := nms(boxes, 0.45, true) + if len(keep) != 2 { + t.Fatalf("want 2 kept boxes, got %d (%v)", len(keep), keep) + } + if keep[0] != 0 || keep[1] != 2 { + t.Fatalf("want kept indices [0 2], got %v", keep) + } +} + +func TestNMSNoPlusOne(t *testing.T) { + // Two boxes that barely overlap; without the +1 term IoU < 0.2 -> both kept. + a := nmsBox{X0: 0, Y0: 0, X1: 10, Y1: 10, Score: 0.9} + b := nmsBox{X0: 9, Y0: 0, X1: 19, Y1: 10, Score: 0.8} + keep := nms([]nmsBox{a, b}, 0.2, false) + if len(keep) != 2 { + t.Fatalf("want 2 kept boxes (no +1), got %d (%v)", len(keep), keep) + } +} + +func TestOCRRecCTCDecode(t *testing.T) { + // vocab = 3: ["blank", "a", "b"]. Build a [recSeqLen*recVocab] tensor whose + // argmax sequence is a, blank, b, blank, blank -> "ab". + out := make([]float32, recSeqLen*recVocab) + seq := []int{1, 0, 2, 0, 0} + for t, idx := range seq { + out[t*recVocab+idx] = 0.9 + } + res := ocrRecCTCDecode(out, []string{"blank", "a", "b"}) + if res.Text != "ab" { + t.Fatalf("want 'ab', got %q", res.Text) + } + if math.Abs(float64(res.Score-0.9)) > 1e-6 { + t.Fatalf("want score 0.9, got %v", res.Score) + } +} + +func TestOCRRecCTCDecodeDedup(t *testing.T) { + out := make([]float32, recSeqLen*recVocab) + // a, a, blank, b -> consecutive a's collapse -> "ab" + for _, t := range []int{0, 1} { + out[t*recVocab+1] = 0.9 + } + out[2*recVocab+0] = 0.9 // blank + out[3*recVocab+2] = 0.9 // b + res := ocrRecCTCDecode(out, []string{"blank", "a", "b"}) + if res.Text != "ab" { + t.Fatalf("want 'ab' (deduped), got %q", res.Text) + } +} + +func TestBilinearResize1x1(t *testing.T) { + // 1x1 red pixel (BGR: 0,0,255) resized to NxN must stay uniform. + src := []byte{0, 0, 255} + dst := bilinearResize(src, 1, 1, 5, 5) + if len(dst) != 5*5*3 { + t.Fatalf("wrong dst length %d", len(dst)) + } + for i := 0; i < len(dst); i += 3 { + if dst[i] != 0 || dst[i+1] != 0 || dst[i+2] != 255 { + t.Fatalf("resize changed pixel at %d: %v", i, dst[i:i+3]) + } + } +} + +func TestRound(t *testing.T) { + if round2(1.2345) != 1.23 { + t.Fatalf("round2(1.2345) = %v", round2(1.2345)) + } + if round4(1.23456) != 1.2346 { + t.Fatalf("round4(1.23456) = %v", round4(1.23456)) + } +} + +func TestDLAWire(t *testing.T) { + r := DLAResult{Boxes: []DLABox{{X0: 1.235, Y0: 2.345, X1: 3.0, Y1: 4.0, Score: 0.5, Class: 5}}, W: 1000, H: 1000} + // within bounds so clamp is a no-op + want := `{"bboxes":[[1.235,2.345,3,4,0.5,5]]}` + if got := r.Wire(); got != want { + t.Fatalf("Wire() = %s, want %s", got, want) + } +} + +func TestDLAWireClamps(t *testing.T) { + // Boxes outside the image must be clamped into [0,W]/[0,H]. + r := DLAResult{Boxes: []DLABox{ + {X0: -5, Y0: 2.3, X1: 3000, Y1: -1, Score: 0.5, Class: 5}, + }, W: 10, H: 10} + want := `{"bboxes":[[0,2.3,10,0,0.5,5]]}` + if got := r.Wire(); got != want { + t.Fatalf("Wire() = %s, want %s", got, want) + } +} + +func TestTSRWire(t *testing.T) { + r := TSRResult{Boxes: []TSRBox{{Label: "table column", Score: 0.7, X0: 1.2, X1: 9.8, Top: 3.4, Bottom: 5.6}}, W: 1000, H: 1000} + // "table column" -> class 1; within bounds so clamp is a no-op. + want := `{"bboxes":[[1.2,3.4,9.8,5.6,0.7,1]]}` + if got := r.Wire(); got != want { + t.Fatalf("Wire() = %s, want %s", got, want) + } +} + +func TestTSRWireClamps(t *testing.T) { + // Boxes outside the image must be clamped into [0,W]/[0,H]. + r := TSRResult{Boxes: []TSRBox{ + {Label: "table column", Score: 0.7, X0: -5, X1: 9.8, Top: 3.4, Bottom: 5000}, + }, W: 10, H: 10} + want := `{"bboxes":[[0,3.4,9.8,10,0.7,1]]}` + if got := r.Wire(); got != want { + t.Fatalf("Wire() = %s, want %s", got, want) + } +} + +func TestOCRRecWire(t *testing.T) { + r := OCRRecResult{Text: "hello", Score: 1.0} + got := r.Wire() + if !strings.Contains(got, "hello") { + t.Fatalf("Wire() missing text: %s", got) + } + if !strings.Contains(got, `"output"`) { + t.Fatalf("Wire() missing output key: %s", got) + } +} + +// ---- DBPostProcess (det) geometry unit tests (model-free) ---- + +func TestConvexHullSquare(t *testing.T) { + // A square plus an interior point; hull keeps only the 4 corners. + pts := []pt{{0, 0}, {10, 0}, {10, 10}, {0, 10}, {5, 5}} + h := convexHull(pts) + if len(h) != 4 { + t.Fatalf("want 4 hull points, got %d: %v", len(h), h) + } +} + +func TestPolygonAreaUnitSquare(t *testing.T) { + sq := []pt{{0, 0}, {1, 0}, {1, 1}, {0, 1}} + if got := polygonArea(sq); math.Abs(got-1.0) > 1e-9 { + t.Fatalf("unit square area = %v, want 1", got) + } +} + +func TestMinAreaRectGetMiniBoxes(t *testing.T) { + // A 100x40 rectangle, top-left at (10,20). + rect := [4]pt{{10, 20}, {110, 20}, {110, 60}, {10, 60}} + corners, sside := minAreaRect(rect[:]) + if sside < 39.9 || sside > 40.1 { + t.Fatalf("minSide = %v, want ~40", sside) + } + // The canonical ordering starts at top-left (smallest x, smallest y). + tl := corners[0] + if tl.X != 10 || tl.Y != 20 { + t.Fatalf("corner[0] (top-left) = %v, want {10 20}", tl) + } +} + +func TestUnclipExpands(t *testing.T) { + sq := [4]pt{{0, 0}, {10, 0}, {10, 10}, {0, 10}} + before := polygonArea(sq[:]) + expanded := unclip(sq, 1.5) + after := polygonArea(expanded[:]) + if after <= before { + t.Fatalf("unclip did not expand area: before=%v after=%v", before, after) + } +} + +func TestFillPolyCoversQuad(t *testing.T) { + // 11x11 mask; fill the quad from (0,0) to (10,10). OpenCV's cv2.fillPoly + // treats the integer vertices as pixel corners and (with the +0.5 + // fixed-point rounding) fills the full 11x11 cell = 121 pixels, matching + // cv2.fillPoly bit-for-bit. + quad := [4]pt{{0, 0}, {10, 0}, {10, 10}, {0, 10}} + mask := make([]bool, 11*11) + fillPoly(mask, 11, 11, quad) + var n int + for _, b := range mask { + if b { + n++ + } + } + if n != 121 { + t.Fatalf("filled %d px, want 121 (cv2.fillPoly)", n) + } +} + +func TestFilterTagDetResDropsTiny(t *testing.T) { + // One valid wide box and one tiny (sub-3px) box. + boxes := []DetBox{ + {Pts: [4][2]float32{{10, 10}, {110, 10}, {110, 30}, {10, 30}}}, + {Pts: [4][2]float32{{200, 200}, {201, 200}, {201, 201}, {200, 201}}}, // 1px + } + kept := filterTagDetRes(boxes, 300, 300) + if len(kept) != 1 { + t.Fatalf("want 1 box kept, got %d", len(kept)) + } +} + +func TestDetWireNesting(t *testing.T) { + r := DetResult{Boxes: []DetBox{ + {Pts: [4][2]float32{{1, 2}, {3, 2}, {3, 4}, {1, 4}}}, + }} + got := r.Wire() + // Boxes must live at output[0][0] (page -> batch -> boxes). + want := `{"output":[[[[[1,2],[3,2],[3,4],[1,4]]]]]}` + if got != want { + t.Fatalf("Wire() = %s, want %s", got, want) + } +} diff --git a/internal/deepdoc/native/nms.go b/internal/deepdoc/native/nms.go new file mode 100644 index 0000000000..01c0e833d9 --- /dev/null +++ b/internal/deepdoc/native/nms.go @@ -0,0 +1,113 @@ +//go:build cgo + +package native + +// nms.go — generic axis-aligned bounding-box nms. +// +// Shared by DLA (IoU 0.45, +1) and TSR (IoU 0.2, no +1). The +1 term matches +// deepdoc/vision/operators.py's nms; some PP-Det paths omit it. + +import ( + "sort" +) + +// nmsBox is an xyxy rectangle with an associated score. +type nmsBox struct { + X0, Y0, X1, Y1 float32 + Score float32 +} + +// nms returns the indices (into the input slice) of the kept boxes. +func nms(boxes []nmsBox, iouThr float32, plusOne bool) []int { + order := make([]int, len(boxes)) + for i := range order { + order[i] = i + } + sort.SliceStable(order, func(i, j int) bool { + return boxes[order[i]].Score > boxes[order[j]].Score + }) + + areas := make([]float32, len(boxes)) + for i, b := range boxes { + areas[i] = (b.Y1 - b.Y0) * (b.X1 - b.X0) + } + + var keep []int + for len(order) > 0 { + i := order[0] + keep = append(keep, i) + if len(order) == 1 { + break + } + xx1 := make([]float32, len(order)-1) + yy1 := make([]float32, len(order)-1) + xx2 := make([]float32, len(order)-1) + yy2 := make([]float32, len(order)-1) + for k, j := range order[1:] { + xx1[k] = maxf(boxes[i].X0, boxes[j].X0) + yy1[k] = maxf(boxes[i].Y0, boxes[j].Y0) + xx2[k] = minf(boxes[i].X1, boxes[j].X1) + yy2[k] = minf(boxes[i].Y1, boxes[j].Y1) + } + w := subMax0(xx2, xx1) + h := subMax0(yy2, yy1) + if plusOne { + for k := range w { + w[k] += 1 + h[k] += 1 + } + } + overlaps := mul(w, h) + var idx []int + for k := range overlaps { + denom := areas[i] + areas[order[k+1]] - overlaps[k] + iou := float32(0) + if denom > 0 { + iou = overlaps[k] / denom + } + if iou <= iouThr { + idx = append(idx, k) + } + } + next := make([]int, 0, len(idx)) + for _, k := range idx { + next = append(next, order[k+1]) + } + order = next + } + return keep +} + +func maxf(a, b float32) float32 { + if a > b { + return a + } + return b +} + +func minf(a, b float32) float32 { + if a < b { + return a + } + return b +} + +func subMax0(a, b []float32) []float32 { + r := make([]float32, len(a)) + for i := range a { + v := a[i] - b[i] + if v < 0 { + v = 0 + } + r[i] = v + } + return r +} + +func mul(a, b []float32) []float32 { + r := make([]float32, len(a)) + for i := range a { + r[i] = a[i] * b[i] + } + return r +} diff --git a/internal/deepdoc/native/ocr_rec.go b/internal/deepdoc/native/ocr_rec.go new file mode 100644 index 0000000000..a21f51abdb --- /dev/null +++ b/internal/deepdoc/native/ocr_rec.go @@ -0,0 +1,409 @@ +//go:build cgo + +package native + +// ocr_rec.go — OCR text recognition (PP-OCRv4 CTC) recognizer. +// +// Ports deepdoc/vision/ocr.py TextRecognizer.resize_norm_img and +// deepdoc/vision/postprocess.py CTCLabelDecode, emitting the wire format from +// deepdoc/server/adapters/ocr_adapter.py (recognize mode). + +import ( + "context" + "encoding/json" + "fmt" + "math" + "os" + "path/filepath" + "strings" + "sync" + + ort "github.com/yalue/onnxruntime_go" +) + +const ( + recH = 48 + recW = 320 + recSeqLen = 40 + recVocab = 6625 + recMaxBatch = 1 +) + +// OCRRecResult is the recognized text for one cropped line. +type OCRRecResult struct { + Text string + Score float32 +} + +// RunOCRRec recognizes a single cropped text-line image. It is equivalent to a +// one-line batch (see RunOCRRecBatch): the line is resized against its own +// wh_ratio floored at 320/48, never the batch-max, so a caller recognizing +// lines independently gets the same result as a standalone Python +// TextRecognizer call. +func RunOCRRec(ctx context.Context, modelDir string, img *Image) (OCRRecResult, error) { + chars, err := loadCharDict(filepath.Join(modelDir, "ocr.res")) + if err != nil { + return OCRRecResult{}, err + } + // A single image is its own batch: max_wh_ratio floors at recW/recH (matching + // TextRecognizer.__call__'s init) but rises to the line's own ratio when + // wider, so wide lines are NOT clamped back to 320. + maxWhRatio := float64(recW) / float64(recH) + if r := float64(img.W) / float64(img.H); r > maxWhRatio { + maxWhRatio = r + } + return recognizeLine(ctx, modelDir, img, maxWhRatio, chars) +} + +// RunOCRRecBatch recognizes a batch of cropped text-line images the way +// deepdoc's TextRecognizer.__call__ does: every line in the batch is resized to +// the SAME width derived from the batch's maximum wh_ratio (floored at 320/48), +// so a narrow line inside a wide batch is widened to the batch max rather than +// its own ratio. This makes Go match production batch semantics, where a page's +// lines are recognized together. Lines are recognized independently after the +// shared resize, so order is preserved and results are deterministic per batch. +// +// DEPRECATED semantic-shape note: this variant still runs one ONNX Run PER +// line (recMaxBatch=1). Prefer RunOCRRecBatchReal, which concatenates every +// line's preprocessed blob into a single {N,3,48,imgW} tensor and performs ONE +// ONNX Run — identical numerically (each line uses the same shared batch width) +// but with the real throughput benefit of batched matmul. RunOCRRecBatchReal is +// the production path. +// RunOCRRecBatchReal recognizes a batch of cropped text-line images with a +// SINGLE ONNX Run, mirroring deepdoc's TextRecognizer.__call__ exactly: every +// line is resized to the batch's shared width (imgW = 48*max_wh_ratio, floored +// at 320/48) and zero-padded on the right, all blobs are concatenated into one +// {N,3,48,imgW} tensor, and the model runs once. The output is split back into +// per-line sequences and CTC-decoded in order, so the result is numerically +// identical to calling RunOCRRec on each line (each line sees the same shared +// batch width), but amortized over one forward pass instead of N. +// +// The shared batch width means a line is resized against the batch max wh_ratio, +// not its own — exactly what deepdoc does inside a batch. A standalone call to +// RunOCRRec (maxWhRatio floored at the line's own ratio when wider) is the +// correct single-line equivalent and remains the unit of "one crop" inference. +func RunOCRRecBatchReal(ctx context.Context, modelDir string, imgs []*Image) ([]OCRRecResult, error) { + n := len(imgs) + if n == 0 { + return nil, nil + } + if n == 1 { + // Degenerate batch: fall back to the single-line path so callers get + // the exact same result as RunOCRRec (no batch-width widening). + res, err := RunOCRRec(ctx, modelDir, imgs[0]) + if err != nil { + return nil, err + } + return []OCRRecResult{res}, nil + } + chars, err := loadCharDict(filepath.Join(modelDir, "ocr.res")) + if err != nil { + return nil, err + } + maxWhRatio := float64(recW) / float64(recH) + for _, img := range imgs { + if r := float64(img.W) / float64(img.H); r > maxWhRatio { + maxWhRatio = r + } + } + imgW := int(math.Floor(recH * maxWhRatio)) + // Per-line resized content width (<= imgW), used to place each line's + // preprocessed blob into the shared concatenated tensor. + resizedWs := make([]int, n) + blobs := make([][]float32, n) + for i, img := range imgs { + resizedW := int(math.Ceil(recH * (float64(img.W) / float64(img.H)))) + if resizedW > imgW { + resizedW = imgW + } + resizedWs[i] = resizedW + blobs[i] = ocrRecPreprocess(img, resizedW, imgW) + } + // Concatenate: layout [N, 3, 48, imgW] with each line's blob at + // offset i*3*recH*imgW. ocrRecPreprocess already zero-fills to imgW, so a + // plain copy places it correctly at the line's N-slot. + batch := make([]float32, n*3*recH*imgW) + lineStride := 3 * recH * imgW + for i, b := range blobs { + copy(batch[i*lineStride:(i+1)*lineStride], b) + } + + // 0 → all cores, matching deepdoc's Python onnxruntime for bit-stable + // parity (no contour extraction in the OCR-rec Run path). + sess, release, err := getRecSession(filepath.Join(modelDir, "rec.onnx"), "x", + []int64{int64(n), 3, recH, int64(imgW)}, "softmax_11.tmp_0", 0) + if err != nil { + return nil, err + } + defer release() + + out, err := sess.Run(ctx, batch) + if err != nil { + return nil, err + } + // Output layout: [N, seqLen, recVocab]; seqLen is dynamic (scales with + // imgW), so derive it from the tensor length. + seqLen := len(out) / (n * recVocab) + if seqLen <= 0 { + return nil, fmt.Errorf("recSession: unexpected batch output len %d for n=%d vocab=%d", len(out), n, recVocab) + } + results := make([]OCRRecResult, n) + for i := 0; i < n; i++ { + line := out[i*seqLen*recVocab : (i+1)*seqLen*recVocab] + results[i] = ocrRecCTCDecode(line, chars) + } + return results, nil +} + +// recognizeLine runs the resize + session + CTC decode for one line at the +// given batch max wh_ratio, mirroring deepdoc TextRecognizer.resize_norm_img +// exactly: the tensor width is imgW = int(48 * max_wh_ratio) (floored at +// 320/48 for narrow batches); the content is resized to resized_w = +// min(ceil(48*ratio), imgW) and zero-padded on the right to imgW. Feeding the +// unpadded own-width (no floor, no pad) — the naive resize — changes +// recognition for narrow lines because the model sees a different width than +// deepdoc. +func recognizeLine(ctx context.Context, modelDir string, img *Image, maxWhRatio float64, chars []string) (OCRRecResult, error) { + ratio := float64(img.W) / float64(img.H) + imgW := int(math.Floor(recH * maxWhRatio)) + resizedW := int(math.Ceil(recH * ratio)) + if resizedW > imgW { + resizedW = imgW + } + blob := ocrRecPreprocess(img, resizedW, imgW) + // 0 → all cores, matching deepdoc's Python onnxruntime for bit-stable + // parity (no contour extraction in the OCR-rec Run path). + sess, release, err := getRecSession(filepath.Join(modelDir, "rec.onnx"), "x", + []int64{recMaxBatch, 3, recH, int64(imgW)}, "softmax_11.tmp_0", 0) + if err != nil { + return OCRRecResult{}, err + } + defer release() + + out, err := sess.Run(ctx, blob) + if err != nil { + return OCRRecResult{}, err + } + return ocrRecCTCDecode(out, chars), nil +} + +// ocrRecPreprocess builds the CHW float blob (/255, standardized) for a +// text-line image resized to (resizedW, recH) and zero-padded on the right to +// the full tensor width imgW. The session runs at imgW; padding mirrors +// deepdoc's resize_norm_img (padding_im[:, :, 0:resized_w] = resized_image). +func ocrRecPreprocess(img *Image, resizedW, imgW int) []float32 { + bgr := img.ToBGR() + w, h := img.W, img.H + resized := bilinearResize(bgr, w, h, resizedW, recH) + blob := make([]float32, 3*recH*imgW) // zero-filled (padded right) + for y := 0; y < recH; y++ { + for x := 0; x < resizedW; x++ { + for c := 0; c < 3; c++ { + v := float32(resized[(y*resizedW+x)*3+c]) / 255.0 + v = (v - 0.5) / 0.5 + blob[c*recH*imgW+y*imgW+x] = v + } + } + } + return blob +} + +// charDictCache memoises loadCharDict by the ocr.res path. RunOCRRec is called +// once per cropped text line, so without caching every line would re-read and +// re-parse the same vocabulary file from disk. The decoded slice is only ever +// read (by ocrRecCTCDecode), never mutated, so sharing it across goroutines is +// safe. +var charDictCache sync.Map // map[string][]string, keyed by ocr.res path + +// loadCharDict returns the decode vocabulary: ["blank"] + + " ". +func loadCharDict(path string) ([]string, error) { + if v, ok := charDictCache.Load(path); ok { + return v.([]string), nil + } + chars, err := readCharDict(path) + if err != nil { + return nil, err + } + charDictCache.Store(path, chars) + return chars, nil +} + +func readCharDict(path string) ([]string, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, err + } + lines := strings.Split(string(data), "\n") + if len(lines) > 0 && lines[len(lines)-1] == "" { + lines = lines[:len(lines)-1] + } + chars := make([]string, 0, len(lines)+2) + chars = append(chars, "blank") + chars = append(chars, lines...) + chars = append(chars, " ") // use_space_char + return chars, nil +} + +func ocrRecCTCDecode(out []float32, chars []string) OCRRecResult { + // out layout: [recMaxBatch, seqLen, recVocab]; take batch 0. The sequence + // length is dynamic (scales with the input width), so derive it from the + // tensor length rather than a fixed constant. + seqLen := len(out) / recVocab + var text strings.Builder + var probs []float32 + prev := -1 + var meanAcc float32 + for t := 0; t < seqLen; t++ { + base := t * recVocab + bestIdx, bestProb := 0, float32(-1e9) + for v := 0; v < recVocab; v++ { + if out[base+v] > bestProb { + bestProb = out[base+v] + bestIdx = v + } + } + if bestIdx == 0 { // blank + prev = 0 + continue + } + if bestIdx != prev { + if bestIdx < len(chars) { + text.WriteString(chars[bestIdx]) + probs = append(probs, bestProb) + } + } + prev = bestIdx + } + for _, p := range probs { + meanAcc += p + } + // Match deepdoc/vision/postprocess.py CTCLabelDecode: an empty decode + // (no characters recognized) yields confidence 0.0, not 1.0. The old 1.0 + // sentinel made an unreadable/blank orientation outscore a real reading, + // which corrupted rotation selection and any max-confidence picker. + score := float32(0.0) + if len(probs) > 0 { + score = meanAcc / float32(len(probs)) + } + return OCRRecResult{Text: text.String(), Score: round4(score)} +} + +// Wire emits the Go DocAnalyzer OCR-rec format: {"output": [[[text, score]]]}. +func (r OCRRecResult) Wire() string { + // Emit the real recognition confidence (mean per-char softmax prob from + // ocrRecCTCDecode) so the wire schema matches ocr.py's + // recognize_batch_with_score, 4-level nesting. + pair := []any{r.Text, r.Score} + arr1 := []any{pair} + arr2 := []any{arr1} + arr3 := []any{arr2} + out, _ := json.Marshal(map[string]any{"output": arr3}) + return string(out) +} + +// recSession runs rec.onnx, whose output sequence length is dynamic: it scales +// with the input width (≈ width/8), so a fixed-shape AdvancedSession cannot be +// pre-sized per width and even a width-matched session would still emit a +// varying seq length. Instead we use a DynamicAdvancedSession and pass a nil +// output on every Run: onnxruntime allocates the correctly-shaped output +// tensor, which we copy out before destroying it. The input tensor is +// fixed-shape per (model, width), so one recSession is reused per width. +type recSession struct { + inName string + outName string + sess *ort.DynamicAdvancedSession + in *ort.Tensor[float32] + poisoned bool +} + +func newRecSession(modelPath, inName string, inShape []int64, outName string, intraOpThreads int) (*recSession, error) { + in := make([]float32, prod(inShape)) + inT, err := ort.NewTensor(ort.NewShape(inShape...), in) + if err != nil { + return nil, err + } + opts, err := ort.NewSessionOptions() + if err != nil { + inT.Destroy() + return nil, err + } + // 0 → all cores (mirrors Python's onnxruntime default); OCR-rec does no + // contour extraction in the Run path, so parallelism is safe and matches + // deepdoc's reduction order for bit-stable parity. + if err := opts.SetIntraOpNumThreads(intraOpThreads); err != nil { + opts.Destroy() + inT.Destroy() + return nil, err + } + sess, err := ort.NewDynamicAdvancedSession(modelPath, + []string{inName}, []string{outName}, opts) + if err != nil { + opts.Destroy() + inT.Destroy() + return nil, err + } + return &recSession{inName: inName, outName: outName, sess: sess, in: inT}, nil +} + +// Run copies input into the input tensor, executes with an auto-allocated +// (dynamic) output, and returns the output data. The allocated output tensor +// is destroyed before returning; out is a fresh copy the caller owns. +func (s *recSession) Run(ctx context.Context, input []float32) ([]float32, error) { + if len(input) != len(s.in.GetData()) { + return nil, fmt.Errorf("recSession %s: input len %d != tensor len %d", + s.outName, len(input), len(s.in.GetData())) + } + opts, err := ort.NewRunOptions() + if err != nil { + return nil, err + } + defer opts.Destroy() + // Cancel an in-flight Run when the context is done. done closes once Run + // returns so the watcher exits even on the success path. + done := make(chan struct{}) + defer close(done) + go func() { + select { + case <-ctx.Done(): + _ = opts.Terminate() + case <-done: + } + }() + + copy(s.in.GetData(), input) + // nil output → onnxruntime allocates the actual-shaped tensor. + outputs := []ort.Value{nil} + if err := s.sess.RunWithOptions([]ort.Value{s.in}, outputs, opts); err != nil { + if ctx.Err() != nil { + s.poisoned = true + } + return nil, err + } + outVal := outputs[0] + if outVal == nil { + return nil, fmt.Errorf("recSession %s: nil output tensor", s.outName) + } + defer outVal.Destroy() + t, ok := outVal.(*ort.Tensor[float32]) + if !ok { + return nil, fmt.Errorf("recSession %s: unexpected output type %T", s.outName, outVal) + } + data := t.GetData() + out := make([]float32, len(data)) + copy(out, data) + return out, nil +} + +// Destroy releases the dynamic session and input tensor. +func (s *recSession) Destroy() { + if s.sess != nil { + s.sess.Destroy() + } + if s.in != nil { + s.in.Destroy() + } +} + +func (s *recSession) isPoisoned() bool { return s.poisoned } + +func (s *recSession) markPoisoned() { s.poisoned = true } diff --git a/internal/deepdoc/native/run.sh b/internal/deepdoc/native/run.sh new file mode 100755 index 0000000000..62a9822789 --- /dev/null +++ b/internal/deepdoc/native/run.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# Run the local Go DeepDoc prototype (DLA / TSR / OCR-rec / DET) and emit each +# task's JSON output. The Python reference scripts that this helper formerly +# compared against have been removed — the Go ports are now validated by the +# in-repo golden/equivalence tests (go test ./internal/deepdoc/native/...), +# so there is no external Python reference to compare with. +# +# bash run.sh # uses default MODEL_DIR below +# MODEL_DIR=... bash run.sh +set -euo pipefail +cd "$(dirname "$0")" + +# ONNX Runtime is statically linked (libonnxruntime.a) and resolved via +# dlopen(NULL); no ORT_LIB is needed. +MODEL_DIR="${MODEL_DIR:?Set MODEL_DIR to the deepdoc model directory (e.g. .../rag/res/deepdoc)}" +export MODEL_DIR + +run_task() { + # $1 = task name, $2 = image file + echo "================ $1 ($2) ================" + go run . -task "$1" -image "testdata/$2" +} + +run_task dla page0.png +run_task tsr table0.png +run_task ocr-rec line0.png +run_task det page0.png + +echo "================ DONE ================" diff --git a/internal/deepdoc/native/session.go b/internal/deepdoc/native/session.go new file mode 100644 index 0000000000..397d48b766 --- /dev/null +++ b/internal/deepdoc/native/session.go @@ -0,0 +1,205 @@ +//go:build cgo + +package native + +// session.go — thin wrapper around onnxruntime_go. +// +// Hides all onnxruntime-go specifics from the recognizers so each task module +// only deals with float32 tensors. One model input, one model output (every +// DeepDoc ONNX we port fits this shape). CPU-only by design. +// +// The ONNX Runtime environment is process-global: InitORT sets the shared +// library and initializes it exactly once. Sessions only own their tensors and +// the advanced-session handle, so running several tasks in one process (or one +// task per CLI invocation) never double-initializes or prematurely tears down +// the shared environment. + +import ( + "context" + "fmt" + "sync" + + ort "github.com/yalue/onnxruntime_go" +) + +var ( + ortOnce sync.Once + ortInitErr error + // ortReady is true once InitializeEnvironment has succeeded. It lets + // callers decide whether the in-process backend can serve without + // triggering a panic from a session Run against an uninitialized + // environment. + ortReady bool +) + +// InitORT initializes the process-global ONNX Runtime environment. Safe to +// call multiple times; only the first takes effect. Call it once at process +// start (the CLI does this from main). +// +// This fork links ONNX Runtime statically (libonnxruntime.a is linked into the +// binary with --whole-archive and exported via -Wl,--export-dynamic; see +// build.sh: ONNX_RUNTIME_STATIC_DIR). The forked onnxruntime_go binding +// resolves OrtGetApiBase from the running binary itself via dlopen(NULL) (the +// process-global symbol table), so no external libonnxruntime.so is needed and +// there is no dynamic .so deployment. A main executable CANNOT be dlopen'd by +// its own file path (glibc refuses), which is exactly why the binding uses the +// NULL handle instead of a path. InitORT therefore takes no library path; the +// fork's SetSharedLibraryPath is a retained no-op. +func InitORT() error { + ortOnce.Do(func() { + ortInitErr = ort.InitializeEnvironment() + if ortInitErr == nil { + ortReady = true + } + }) + return ortInitErr +} + +// Initialized reports whether ONNX Runtime's process-global environment has +// been successfully initialized. The in-process DeepDoc backend uses this to +// decide whether it can serve, degrading to an empty analyzer otherwise. +func Initialized() bool { return ortReady } + +// session loads one ONNX model and runs single-input/single-output inference. +type session struct { + inName string + outName string + outSize int64 + sess *ort.AdvancedSession + in *ort.Tensor[float32] + out *ort.Tensor[float32] + // poisoned is set when a Run is cancelled/terminated via context. ONNX + // Runtime does not guarantee a session is reusable after a forced + // termination, so the pool must Destroy rather than re-Put it. + poisoned bool +} + +// NewSession opens modelPath. inShape/outShape describe the fixed tensor +// dimensions; outSize is the total element count of the output tensor. +// intraOpThreads controls ONNX Runtime's intra-op parallelism. DLA/TSR/OCR-rec +// pass 0 to use all cores — matching deepdoc's Python onnxruntime +// (intra_op_num_threads defaults to 0 = all cores); their Run path does no +// contour extraction, so the parallel reduction order matches Python for +// bit-stable parity. +// +// The DB text detector is currently pinned to 1 (single-threaded). NOTE: the +// historical rationale for this — that a multi-threaded Run would leave ONNX +// Runtime worker threads settling while the postprocess's OpenCV findContours +// ran its parallel_for_ and under-ran — does NOT apply to this pure-Go port. +// Here the postprocess (hand-rolled findContours / boxScoreFast / fillPoly) is +// fully synchronous and only runs after RunWithOptions returns, so there is no +// thread competition with the detector. The pin is preserved as-is because the +// det pred map was verified at intraOpThreads=1 (mean|Δ|≈4e-5 vs the Python +// reference); flipping it to 0 must be re-confirmed on the det integration +// fixtures before landing, since the reduction order differs. +// InitORT must have been called first. +func NewSession(modelPath, inName string, inShape []int64, outName string, outShape []int64, intraOpThreads int) (*session, error) { + in := make([]float32, prod(inShape)) + out := make([]float32, prod(outShape)) + inT, err := ort.NewTensor(ort.NewShape(inShape...), in) + if err != nil { + return nil, err + } + outT, err := ort.NewTensor(ort.NewShape(outShape...), out) + if err != nil { + inT.Destroy() + return nil, err + } + opts, err := ort.NewSessionOptions() + if err != nil { + inT.Destroy() + outT.Destroy() + return nil, err + } + // intraOpThreads == 0 → all cores (mirrors Python's onnxruntime default); + // the DB detector passes 1, preserved as-is for verified det parity (see + // NewSession doc above — the old findContours/parallel_for_ rationale does + // not apply to this pure-Go port). + if err := opts.SetIntraOpNumThreads(intraOpThreads); err != nil { + opts.Destroy() + inT.Destroy() + outT.Destroy() + return nil, err + } + sess, err := ort.NewAdvancedSession(modelPath, + []string{inName}, []string{outName}, + []ort.Value{inT}, []ort.Value{outT}, opts) + if err != nil { + opts.Destroy() + inT.Destroy() + outT.Destroy() + return nil, err + } + return &session{ + inName: inName, outName: outName, + outSize: prod(outShape), + sess: sess, in: inT, out: outT, + }, nil +} + +// Run copies input into the input tensor, executes, and returns the output +// tensor contents. ctx bounds the inference: if it is cancelled while Run is +// in flight, the underlying ONNX Runtime call is terminated via RunOptions. +// A terminated session is left in an indeterminate state, so it is marked +// poisoned and the pool destroys it instead of reusing it. +func (s *session) Run(ctx context.Context, input []float32) ([]float32, error) { + if len(input) != len(s.in.GetData()) { + return nil, fmt.Errorf("session %s: input len %d != tensor len %d", + s.outName, len(input), len(s.in.GetData())) + } + opts, err := ort.NewRunOptions() + if err != nil { + return nil, err + } + defer opts.Destroy() + // Cancel an in-flight Run when the context is done. done closes once Run + // returns so the watcher exits even on the success path. + done := make(chan struct{}) + defer close(done) + go func() { + select { + case <-ctx.Done(): + _ = opts.Terminate() + case <-done: + } + }() + + copy(s.in.GetData(), input) + if err := s.sess.RunWithOptions(opts); err != nil { + if ctx.Err() != nil { + s.poisoned = true + } + return nil, err + } + out := make([]float32, s.outSize) + copy(out, s.out.GetData()) + return out, nil +} + +// Destroy releases the tensors and advanced-session handle. It does NOT touch +// the process-global environment. +func (s *session) Destroy() { + if s.sess != nil { + s.sess.Destroy() + } + if s.in != nil { + s.in.Destroy() + } + if s.out != nil { + s.out.Destroy() + } +} + +func (s *session) isPoisoned() bool { return s.poisoned } + +// markPoisoned records that a Run was force-terminated; a poisoned session is +// not safe to reuse, so the pool Destroys it on release. +func (s *session) markPoisoned() { s.poisoned = true } + +func prod(shape []int64) int64 { + p := int64(1) + for _, d := range shape { + p *= d + } + return p +} diff --git a/internal/deepdoc/native/session_pool.go b/internal/deepdoc/native/session_pool.go new file mode 100644 index 0000000000..0ea8d89509 --- /dev/null +++ b/internal/deepdoc/native/session_pool.go @@ -0,0 +1,265 @@ +//go:build cgo + +package native + +// session_pool.go — shared ONNX session pool for all recognizers. +// +// RunDLA / RunTSR / RunOCRRec (fixed-shape) and RunDet (variable-shape DB +// detector) all load an ONNX session per call. A long document pays that setup +// cost per region/page even though every call uses the same shapes within a +// model. This pool caches one session per (model signature) tuple and hands it +// back between calls. +// +// Sessions are pooled, not shared concurrently: session.Run copies the caller's +// input into the session's fixed-shape input tensor and then executes, so a +// single session must never be touched by two goroutines at once. Get returns a +// session owned by the caller until release is called; release returns it to +// the pool for reuse. This keeps the Get/Run/Release window single-owner, which +// is what makes reuse safe under the page/region worker pools. +// +// The pool is generic over the key type K and the pooled session type V so both +// the fixed-shape recognizers (DLA/TSR/OCR-rec, recSession) and the +// variable-shape detector (detSession) share one implementation. Two bounds +// apply: +// - maxKeys caps the number of distinct key-pools; when exceeded the +// least-recently-used key-pool is evicted and its idle sessions Destroyed +// (bounds memory for the variable-shape detector, which can see many +// distinct page sizes). +// - maxFree caps idle sessions retained per key-pool; extras are Destroyed on +// release instead of pooled. A maxFree / maxKeys of 0 means unbounded — the +// degenerate case for the fixed-shape recognizers, whose key set is tiny. + +import ( + "reflect" + "strconv" + "strings" + "sync" +) + +// pooledSession is the minimal contract the pool needs from any cached ONNX +// session: release its resources, and report/mark the poisoned flag set when a +// Run is force-terminated via context (ORT does not guarantee reuse safety +// after a termination, so the pool must Destroy rather than re-Put). +type pooledSession interface { + Destroy() + isPoisoned() bool + markPoisoned() +} + +// sessionKeyPool holds reusable sessions for one key. +type sessionKeyPool[V pooledSession] struct { + mu sync.Mutex + live bool // false once evicted; checked-out sessions self-Destroy on release + free []V +} + +// sessionPool is a generic reusable ONNX session pool keyed by K, storing +// values of type V (any pooledSession). +type sessionPool[K comparable, V pooledSession] struct { + mu sync.Mutex + pools map[K]*sessionKeyPool[V] + lru []K // front = least-recently-used + maxKeys int + maxFree int +} + +func newSessionPool[K comparable, V pooledSession](maxKeys, maxFree int) *sessionPool[K, V] { + return &sessionPool[K, V]{ + pools: make(map[K]*sessionKeyPool[V]), + maxKeys: maxKeys, + maxFree: maxFree, + } +} + +// Get returns a reusable session for key, constructing one with newFn on a pool +// miss, plus a release func. The caller must call release exactly once. A +// poisoned session is Destroyed on release rather than pooled (ORT does not +// guarantee reuse safety after a forced termination). +func (p *sessionPool[K, V]) Get(key K, newFn func() (V, error)) (V, func(), error) { + p.mu.Lock() + kp := p.pools[key] + if kp == nil { + if p.maxKeys > 0 && len(p.pools) >= p.maxKeys { + p.evictLRU() + } + kp = &sessionKeyPool[V]{live: true} + p.pools[key] = kp + p.lru = append(p.lru, key) + } else { + p.touchLRU(key) + } + p.mu.Unlock() + + kp.mu.Lock() + var s V + if n := len(kp.free); n > 0 { + s = kp.free[n-1] + kp.free = kp.free[:n-1] + } + kp.mu.Unlock() + + if isNil(s) { + var err error + s, err = newFn() + if err != nil { + var zero V + return zero, nil, err + } + } + + release := func() { + if s.isPoisoned() { + s.Destroy() + return + } + kp.mu.Lock() + if kp.live && (p.maxFree <= 0 || len(kp.free) < p.maxFree) { + kp.free = append(kp.free, s) + kp.mu.Unlock() + } else { + kp.mu.Unlock() + s.Destroy() + } + } + return s, release, nil +} + +// isNil reports whether a generic pooledSession value is its zero value (a nil +// pointer). Every pooled type is a pointer, so reflection's IsNil is safe here. +// newFn only runs on a miss. +func isNil[V pooledSession](v V) bool { + rv := reflect.ValueOf(v) + return rv.IsNil() +} + +// KeyCount returns the number of distinct key-pools currently live. Used by +// tests that assert the pool set is bounded. +func (p *sessionPool[K, V]) KeyCount() int { + p.mu.Lock() + defer p.mu.Unlock() + return len(p.pools) +} + +// touchLRU moves key to the most-recently-used end. Caller holds p.mu. +func (p *sessionPool[K, V]) touchLRU(key K) { + for i, k := range p.lru { + if k == key { + p.lru = append(p.lru[:i], p.lru[i+1:]...) + p.lru = append(p.lru, k) + return + } + } +} + +// evictLRU drops the least-recently-used key-pool and destroys its idle +// sessions. Caller holds p.mu. +func (p *sessionPool[K, V]) evictLRU() { + if len(p.lru) == 0 { + return + } + evict := p.lru[0] + p.lru = p.lru[1:] + kp := p.pools[evict] + delete(p.pools, evict) + if kp == nil { + return + } + kp.mu.Lock() + kp.live = false + for _, s := range kp.free { + s.Destroy() + } + kp.free = nil + kp.mu.Unlock() +} + +// ---- fixed-shape recognizer pool (DLA / TSR / OCR-rec) ---- + +// sessKey is the pool key for the fixed-shape models. Unlike the DB detector +// these models always run at a constant input size, so the tuple is constant +// per modelDir in practice. +type sessKey struct { + modelPath, inName, outName string + inShape, outShape string + intraOpThreads int +} + +func sessKeyOf(modelPath, inName string, inShape []int64, outName string, outShape []int64, intraOpThreads int) sessKey { + return sessKey{ + modelPath: modelPath, + inName: inName, + outName: outName, + inShape: shapeKey(inShape), + outShape: shapeKey(outShape), + intraOpThreads: intraOpThreads, + } +} + +func shapeKey(s []int64) string { + parts := make([]string, len(s)) + for i, d := range s { + parts[i] = strconv.FormatInt(d, 10) + } + return strings.Join(parts, ",") +} + +// modelSessions is unbounded (maxKeys/maxFree = 0): the fixed-shape key set is +// tiny, so the degenerate no-eviction case is correct here. +var modelSessions = newSessionPool[sessKey, *session](0, 0) + +// getModelSession returns a reusable session for the given model signature plus +// a release func. The caller must call release exactly once. +func getModelSession(modelPath, inName string, inShape []int64, outName string, outShape []int64, intraOpThreads int) (*session, func(), error) { + key := sessKeyOf(modelPath, inName, inShape, outName, outShape, intraOpThreads) + return modelSessions.Get(key, func() (*session, error) { + return NewSession(modelPath, inName, inShape, outName, outShape, intraOpThreads) + }) +} + +// ---- dynamic-width OCR-rec pool ---- + +// recKey is the pool key for the dynamic-width OCR-rec model. The session is +// pinned to one input width (the input tensor is fixed-shape per width), and +// the output is auto-allocated at the model's true (width-dependent) sequence +// length, so the key carries the width but no output shape. +type recKey struct { + modelPath, inName, outName string + inShape string + intraOpThreads int +} + +func recKeyOf(modelPath, inName string, inShape []int64, outName string, intraOpThreads int) recKey { + return recKey{ + modelPath: modelPath, + inName: inName, + outName: outName, + inShape: shapeKey(inShape), + intraOpThreads: intraOpThreads, + } +} + +const ( + // recMaxShapePools caps distinct (modelPath, width) pools. Unlike the + // fixed-shape DLA/TSR models, a long-running server ingesting many + // differently-sized text lines would otherwise pin one pooled session (and + // its ORT tensors) per distinct width forever. The shared sessionPool + // evicts the least-recently-used width pool (Destroying its idle sessions) + // once the cap is exceeded, bounding memory. + recMaxShapePools = 64 + // recShapePoolCap caps idle sessions retained per width; extras are + // Destroyed on release instead of pooled. + recShapePoolCap = 4 +) + +// recSessions is the dynamic-width OCR-rec pool: bounded at recMaxShapePools +// distinct width-pools, each retaining up to recShapePoolCap idle sessions. +var recSessions = newSessionPool[recKey, *recSession](recMaxShapePools, recShapePoolCap) + +// getRecSession returns a reusable dynamic-width OCR-rec session for the given +// input width plus a release func. The caller must call release exactly once. +func getRecSession(modelPath, inName string, inShape []int64, outName string, intraOpThreads int) (*recSession, func(), error) { + key := recKeyOf(modelPath, inName, inShape, outName, intraOpThreads) + return recSessions.Get(key, func() (*recSession, error) { + return newRecSession(modelPath, inName, inShape, outName, intraOpThreads) + }) +} diff --git a/internal/deepdoc/native/session_pool_stress_test.go b/internal/deepdoc/native/session_pool_stress_test.go new file mode 100644 index 0000000000..da0dc38828 --- /dev/null +++ b/internal/deepdoc/native/session_pool_stress_test.go @@ -0,0 +1,106 @@ +//go:build cgo + +package native + +import ( + "sync" + "sync/atomic" + "testing" +) + +// fakeSession is a pooledSession stand-in that needs no ONNX model, so the +// sessionPool's concurrency (Get/Release/LRU-evict/poison) can be stressed +// without MODEL_DIR. It records Destroy calls to assert pool lifecycle +// behaviour under contention. +type fakeSession struct { + id int + poisoned atomic.Bool + destroyed atomic.Bool +} + +func (f *fakeSession) Destroy() { f.destroyed.Store(true) } +func (f *fakeSession) isPoisoned() bool { return f.poisoned.Load() } +func (f *fakeSession) markPoisoned() { f.poisoned.Store(true) } + +// TestSessionPoolConcurrentGetReleaseStress hammers a generic sessionPool from +// many goroutines with a rotating key set (forcing LRU eviction) and a poison +// fraction, under -race. It proves the pool's shared state — the pools map, the +// lru slice, and each key-pool's free list plus their mutexes — is data-race +// free. This is the exact code path backing the real modelSessions / recSessions +// pools that the DLA/TSR/OCR-rec/Det recognizers rely on. +// +// Crucially it needs no ONNX weights, so unlike the MODEL_DIR-gated integration +// tests it runs in the default `go test ./...` (cgo) path and can be exercised +// with -race in CI, giving the concurrency-safety claim default-path coverage. +func TestSessionPoolConcurrentGetReleaseStress(t *testing.T) { + const maxKeys = 8 + const maxFree = 2 + p := newSessionPool[int, *fakeSession](maxKeys, maxFree) + + var nextID atomic.Int64 + var mu sync.Mutex + var created []*fakeSession + + newFn := func() (*fakeSession, error) { + s := &fakeSession{id: int(nextID.Add(1))} + mu.Lock() + created = append(created, s) + mu.Unlock() + return s, nil + } + + const goroutines = 64 + const iters = 200 + var wg sync.WaitGroup + for g := 0; g < goroutines; g++ { + wg.Add(1) + go func(g int) { + defer wg.Done() + for i := 0; i < iters; i++ { + // Rotate the key across more distinct values than maxKeys so the + // LRU eviction path is exercised under contention. + key := (g + i) % (maxKeys * 2) + s, release, err := p.Get(key, newFn) + if err != nil { + t.Errorf("Get(%d): %v", key, err) + return + } + // Every 7th session is poisoned to drive the Destroy-on-release + // branch (ORT does not guarantee reuse safety after a forced + // termination, so the pool must Destroy rather than re-Put). + if i%7 == 0 { + s.markPoisoned() + } + release() + } + }(g) + } + wg.Wait() + + // The pool must stay bounded: eviction keeps live key-pools <= maxKeys even + // though goroutines cycled through 2*maxKeys distinct keys. + if got := p.KeyCount(); got > maxKeys { + t.Errorf("KeyCount %d exceeds maxKeys %d (LRU eviction not bounding the pool)", got, maxKeys) + } + + // Poisoned sessions must have been Destroyed on release (never pooled). Idle, + // unpoisoned sessions that are still pooled at the end are legitimately + // NOT destroyed, so we only assert the poison contract. + mu.Lock() + all := created + mu.Unlock() + var poisonedSeen, poisonedDestroyed int + for _, s := range all { + if s.poisoned.Load() { + poisonedSeen++ + if !s.destroyed.Load() { + t.Errorf("poisoned session %d returned to the pool instead of being Destroyed", s.id) + } else { + poisonedDestroyed++ + } + } + } + if poisonedSeen > 0 && poisonedDestroyed == 0 { + t.Errorf("none of %d poisoned sessions were Destroyed", poisonedSeen) + } +} diff --git a/internal/deepdoc/native/testdata.ref b/internal/deepdoc/native/testdata.ref new file mode 100644 index 0000000000..59110447ef --- /dev/null +++ b/internal/deepdoc/native/testdata.ref @@ -0,0 +1 @@ +deepdoc-native-testdata-v1 diff --git a/internal/deepdoc/native/testdata_clipper_quads.json b/internal/deepdoc/native/testdata_clipper_quads.json new file mode 100644 index 0000000000..b9e889b610 --- /dev/null +++ b/internal/deepdoc/native/testdata_clipper_quads.json @@ -0,0 +1 @@ +{"quads": [{"box": [[85.01150512695312, 403.7520446777344], [491.0213317871094, 405.8288269042969], [490.9894104003906, 412.0716247558594], [84.97958374023438, 409.9948425292969]], "poly": [[[491, 400], [494, 401], [495, 404], [496, 406], [495, 413], [493, 415], [490, 417], [84, 414], [81, 413], [80, 410], [79, 408], [80, 402], [82, 400], [85, 398]]], "distance": 4.611257015058407}, {"box": [[85.0, 388.0], [651.0, 388.0], [651.0, 394.0], [85.0, 394.0]], "poly": [[[654, 385], [655, 388], [655, 394], [654, 397], [651, 398], [85, 398], [82, 397], [81, 394], [81, 388], [82, 385], [85, 384], [651, 384]]], "distance": 4.4527972027972025}, {"box": [[85.0, 371.0], [652.0, 371.0], [652.0, 377.0], [85.0, 377.0]], "poly": [[[655, 368], [656, 371], [656, 377], [655, 380], [652, 381], [85, 381], [82, 380], [81, 377], [81, 371], [82, 368], [85, 367], [652, 367]]], "distance": 4.452879581151833}, {"box": [[85.0, 354.0], [651.0, 354.0], [651.0, 360.0], [85.0, 360.0]], "poly": [[[654, 351], [655, 354], [655, 360], [654, 363], [651, 364], [85, 364], [82, 363], [81, 360], [81, 354], [82, 351], [85, 350], [651, 350]]], "distance": 4.4527972027972025}, {"box": [[84.0, 337.0], [652.0, 337.0], [652.0, 343.0], [84.0, 343.0]], "poly": [[[655, 334], [656, 337], [656, 343], [655, 346], [652, 347], [84, 347], [81, 346], [80, 343], [80, 337], [81, 334], [84, 333], [652, 333]]], "distance": 4.452961672473868}, {"box": [[85.00225830078125, 318.9115295410156], [651.01025390625, 320.0761413574219], [650.9960327148438, 326.9773254394531], [84.988037109375, 325.8127136230469]], "poly": [[[651, 315], [654, 316], [656, 318], [656, 321], [655, 327], [654, 330], [650, 331], [84, 330], [81, 329], [79, 327], [79, 324], [80, 317], [81, 314], [85, 313]]], "distance": 5.1135508924856845}, {"box": [[85.0, 277.0], [639.0, 277.0], [639.0, 283.0], [85.0, 283.0]], "poly": [[[642, 274], [643, 277], [643, 283], [642, 286], [639, 287], [85, 287], [82, 286], [81, 283], [81, 277], [82, 274], [85, 273], [639, 273]]], "distance": 4.451785714285714}, {"box": [[84.99369812011719, 260.0020751953125], [650.9974975585938, 258.83258056640625], [651.01025390625, 265.01849365234375], [85.0064697265625, 266.18798828125]], "poly": [[[653, 254], [655, 257], [656, 264], [655, 267], [653, 269], [651, 270], [85, 271], [82, 270], [80, 267], [79, 261], [80, 258], [82, 256], [84, 255], [650, 253]]], "distance": 4.589287896316454}, {"box": [[84.0, 242.0], [650.9998779296875, 242.0], [650.9998779296875, 248.0], [84.0, 248.0]], "poly": [[[653, 239], [654, 242], [654, 248], [653, 251], [650, 252], [84, 252], [81, 251], [80, 248], [80, 242], [81, 239], [84, 238], [650, 238]]], "distance": 4.452879571113429}, {"box": [[85.0, 225.0], [649.0, 225.0], [649.0, 232.0], [85.0, 232.0]], "poly": [[[652, 221], [654, 223], [654, 232], [653, 235], [651, 237], [85, 237], [82, 236], [80, 234], [80, 225], [81, 222], [83, 220], [649, 220]]], "distance": 5.185639229422066}, {"box": [[84.0, 182.0], [193.0, 182.0], [193.0, 187.0], [84.0, 187.0]], "poly": [[[195, 179], [197, 182], [197, 187], [196, 189], [193, 191], [84, 191], [82, 190], [80, 187], [80, 182], [81, 180], [84, 178], [193, 178]]], "distance": 3.585526315789474}, {"box": [[84.00363159179688, 163.98019409179688], [652.00537109375, 165.00363159179688], [651.9931640625, 171.77117919921875], [83.99142456054688, 170.74774169921875]], "poly": [[[652, 160], [655, 161], [657, 164], [657, 166], [656, 172], [654, 175], [651, 176], [83, 175], [80, 174], [78, 171], [78, 169], [79, 162], [80, 159], [84, 158]]], "distance": 5.01590614596162}, {"box": [[85.0, 148.0], [651.0, 148.0], [651.0, 155.0], [85.0, 155.0]], "poly": [[[654, 144], [656, 146], [656, 155], [655, 158], [653, 160], [85, 160], [82, 159], [80, 157], [80, 148], [81, 145], [83, 143], [651, 143]]], "distance": 5.18586387434555}, {"box": [[85.001953125, 129.94964599609375], [651.0054931640625, 131.005615234375], [650.9926147460938, 137.901123046875], [84.98907470703125, 136.84515380859375]], "poly": [[[651, 126], [654, 127], [656, 129], [656, 132], [655, 138], [654, 141], [650, 142], [84, 141], [81, 140], [79, 138], [79, 135], [80, 128], [81, 125], [85, 124]]], "distance": 5.109393164773621}, {"box": [[244.00973510742188, 91.93400573730469], [492.0231018066406, 93.1038818359375], [491.9867248535156, 100.82069396972656], [243.97335815429688, 99.65081787109375]], "poly": [[[492, 87], [495, 88], [497, 91], [498, 94], [497, 101], [495, 104], [491, 106], [243, 105], [240, 104], [238, 101], [237, 98], [238, 90], [240, 87], [244, 85]]], "distance": 5.613026877500588}], "n_final": 15} \ No newline at end of file diff --git a/internal/deepdoc/native/testdata_fetch.go b/internal/deepdoc/native/testdata_fetch.go new file mode 100644 index 0000000000..b3d0a922ec --- /dev/null +++ b/internal/deepdoc/native/testdata_fetch.go @@ -0,0 +1,55 @@ +//go:build fetch_testdata + +// This file is compiled only with -tags fetch_testdata. Its init() downloads +// and symlinks the external DeepDoc testdata for this package (pkg = "native") +// so the consuming tests can run. The download is best-effort: if it fails, the +// guard in testdata_skip.go skips the tests instead of failing. +package native + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" +) + +func init() { + if err := fetchTestdata(); err != nil { + fmt.Fprintf(os.Stderr, "fetch_deepdoc_testdata: not fetched (%v)\n", err) + return + } + testdataFetchAttempted = true +} + +// fetchTestdata resolves the repo root, then invokes scripts/fetch_deepdoc_testdata.sh +// with this package's directory name as . +func fetchTestdata() error { + _, thisFile, _, ok := runtime.Caller(0) + if !ok { + return fmt.Errorf("cannot resolve caller path") + } + // thisFile = /internal/deepdoc/native/testdata_fetch.go + dir := filepath.Dir(thisFile) + root := dir + for { + if _, err := os.Stat(filepath.Join(root, "go.mod")); err == nil { + break + } + parent := filepath.Dir(root) + if parent == root { + return fmt.Errorf("repo root (go.mod) not found from %s", dir) + } + root = parent + } + pkg := filepath.Base(dir) + script := filepath.Join(root, "scripts", "fetch_deepdoc_testdata.sh") + if _, err := os.Stat(script); err != nil { + return fmt.Errorf("fetch script not found at %s", script) + } + cmd := exec.Command("bash", script, pkg) + cmd.Dir = root + cmd.Stdout = os.Stderr + cmd.Stderr = os.Stderr + return cmd.Run() +} diff --git a/internal/deepdoc/native/testdata_flag.go b/internal/deepdoc/native/testdata_flag.go new file mode 100644 index 0000000000..5f329abe5d --- /dev/null +++ b/internal/deepdoc/native/testdata_flag.go @@ -0,0 +1,10 @@ +package native + +// testdataFetchAttempted is set to true by testdata_fetch.go (compiled only with +// -tags fetch_testdata) once the external DeepDoc testdata has been fetched and +// linked into ./testdata. It is declared in this always-compiled (non-_test.go) +// file — rather than in a _test.go — so it is also defined when this package is +// compiled as a dependency of another package's test binary (e.g. native_analyzer +// imports native, and _test.go files are excluded from dependency builds). In the +// default build it stays false, so TestMain skips the package. +var testdataFetchAttempted bool diff --git a/internal/deepdoc/native/testdata_skip_test.go b/internal/deepdoc/native/testdata_skip_test.go new file mode 100644 index 0000000000..278455ef49 --- /dev/null +++ b/internal/deepdoc/native/testdata_skip_test.go @@ -0,0 +1,26 @@ +// This file is always compiled (no build constraint) and provides the package +// TestMain that guards the whole test binary behind the presence of the +// package-level testdata directory. When the external testdata has not been +// fetched (e.g. -tags fetch_testdata was not supplied, or the download failed), +// the tests are skipped rather than failed — keeping the default `go test ./...` +// and `go test -tags cgo` runs green without the assets. +package native + +import ( + "fmt" + "os" + "testing" +) + +// TestMain skips the entire package unless the testdata was fetched via the +// fetch_testdata build tag. In the default build (and any build without that +// tag) it stays false, so the tests are skipped rather than failing on absent +// assets — which also makes a stale/partial leftover testdata directory on disk +// harmless: we skip regardless of what (if anything) is present. +func TestMain(m *testing.M) { + if !testdataFetchAttempted { + fmt.Println("SKIP: DeepDoc native testdata not fetched; run with -tags fetch_testdata to fetch and run it") + os.Exit(0) + } + os.Exit(m.Run()) +} diff --git a/internal/deepdoc/native/tsr.go b/internal/deepdoc/native/tsr.go new file mode 100644 index 0000000000..be37be7a59 --- /dev/null +++ b/internal/deepdoc/native/tsr.go @@ -0,0 +1,307 @@ +//go:build cgo + +package native + +// tsr.go — Table Structure Recognition recognizer. +// +// Ports the PP-Det style path in deepdoc/vision/recognizer.py (base +// Recognizer.preprocess/postprocess, "scale_factor" branch off) plus the +// column/row alignment in deepdoc/vision/table_structure_recognizer.py and the +// wire mapping in deepdoc/server/adapters/tsr_adapter.py. + +import ( + "context" + "encoding/json" + "path/filepath" + "sort" + "strings" +) + +const tsrInputSize = 640 +const tsrCandidates = 8400 + +// tsrLabels mirrors TableStructureRecognizer.labels (also tsr_adapter.TSR_CLASS_MAP keys). +var tsrLabels = []string{ + "table", "table column", "table row", + "table column header", "table projected row header", "table spanning cell", +} + +// TSRBox is one structural element (in original pixel coordinates). +type TSRBox struct { + Label string + Score float32 + X0, X1, Top, Bottom float32 +} + +// TSRResult is the aligned set of structural elements for one table region. +// W/H are the source image dimensions, used to clamp boxes into bounds (mirrors +// tsr_adapter.py, which clamps every coordinate to [0, width]/[0, height]). +type TSRResult struct { + Boxes []TSRBox + W, H int +} + +// RunTSR runs table-structure recognition on a cropped table image. +func RunTSR(ctx context.Context, modelDir string, img *Image) (TSRResult, error) { + blob, sf := tsrPreprocess(img) + // 0 → all cores, matching deepdoc's Python onnxruntime for bit-stable + // parity (no contour extraction in the TSR Run path). + sess, release, err := getModelSession(filepath.Join(modelDir, "tsr.onnx"), "images", + []int64{1, 3, tsrInputSize, tsrInputSize}, "output0", + []int64{1, 11, tsrCandidates}, 0) + if err != nil { + return TSRResult{}, err + } + defer release() + + out, err := sess.Run(ctx, blob) + if err != nil { + return TSRResult{}, err + } + res := tsrPostprocess(out, sf) + res.W, res.H = img.W, img.H + return res, nil +} + +// tsrBlob assembles the CHW float blob (/255) the TSR model consumes from an +// already-resized BGR raster (tsrInputSize*tsrInputSize*3, row-major). Only +// the resize source differs (Go bilinearResize vs cv2 in the production +// Python reference). +func tsrBlob(resized []byte) []float32 { + blob := make([]float32, 3*tsrInputSize*tsrInputSize) + for y := 0; y < tsrInputSize; y++ { + for x := 0; x < tsrInputSize; x++ { + o := (y*tsrInputSize + x) * 3 + blob[0*tsrInputSize*tsrInputSize+y*tsrInputSize+x] = float32(resized[o]) / 255.0 + blob[1*tsrInputSize*tsrInputSize+y*tsrInputSize+x] = float32(resized[o+1]) / 255.0 + blob[2*tsrInputSize*tsrInputSize+y*tsrInputSize+x] = float32(resized[o+2]) / 255.0 + } + } + return blob +} + +// tsrScaleFactor builds the [W/640, H/640] mapping (mirrors ref_tsr.py sf). +func tsrScaleFactor(img *Image) [2]float32 { + return [2]float32{float32(img.W) / tsrInputSize, float32(img.H) / tsrInputSize} +} + +func tsrPostprocess(out []float32, sf [2]float32) TSRResult { + const scoreThr = 0.2 + type cand struct { + nmsBox + cls int + } + cands := make([]cand, 0, tsrCandidates) + for a := 0; a < tsrCandidates; a++ { + // Model output is [1, 11, 8400] (feature-major / channels-first), so + // flat index for feature c, anchor a is c*8400 + a. + // Class scores live in features 4..10; pick the max. + best, bestCls := float32(-1), 0 + for c := 4; c < 11; c++ { + v := out[c*tsrCandidates+a] + if v > best { + best = v + bestCls = c - 4 + } + } + if best <= scoreThr { + continue + } + if bestCls >= len(tsrLabels) { + continue + } + // Model emits [x, y, w, h] (center-based) in the 640-input space. + // Scale back to original pixels, then convert to xyxy (mirrors + // recognizer.py postprocess: multiply by scale_factor, then xywh2xyxy). + cx := out[0*tsrCandidates+a] * sf[0] + cy := out[1*tsrCandidates+a] * sf[1] + hw := out[2*tsrCandidates+a] * sf[0] * 0.5 + hh := out[3*tsrCandidates+a] * sf[1] * 0.5 + cands = append(cands, cand{ + nmsBox: nmsBox{ + X0: cx - hw, + Y0: cy - hh, + X1: cx + hw, + Y1: cy + hh, + Score: best, + }, + cls: bestCls, + }) + } + + byClass := map[int][]int{} + for i, c := range cands { + byClass[c.cls] = append(byClass[c.cls], i) + } + boxes := make([]TSRBox, 0, len(cands)) + for cls, idxs := range byClass { + sub := make([]nmsBox, len(idxs)) + for k, i := range idxs { + sub[k] = cands[i].nmsBox + } + for _, keep := range nms(sub, 0.2, false) { + b := sub[keep] + boxes = append(boxes, TSRBox{ + Label: tsrLabels[cls], + Score: round4(b.Score), + X0: round2(b.X0), X1: round2(b.X1), + Top: round2(b.Y0), Bottom: round2(b.Y1), + }) + } + } + + alignTSR(boxes) + // Deterministic ordering: tsrPostprocess iterates a class->index map, whose + // iteration order is unspecified in Go. Sort so identical detections always + // serialize identically (e.g. for stable Wire() across runs / session reuse). + sort.Slice(boxes, func(i, j int) bool { + a, b := boxes[i], boxes[j] + ca, cb := tsrClassMap[a.Label], tsrClassMap[b.Label] + if ca != cb { + return ca < cb + } + if a.X0 != b.X0 { + return a.X0 < b.X0 + } + if a.Top != b.Top { + return a.Top < b.Top + } + if a.X1 != b.X1 { + return a.X1 < b.X1 + } + if a.Bottom != b.Bottom { + return a.Bottom < b.Bottom + } + return a.Score < b.Score + }) + return TSRResult{Boxes: boxes} +} + +// alignTSR pulls row/header boxes to the table's horizontal extremes and +// column boxes to its vertical extremes, matching deepdoc +// TableStructureRecognizer.__call__: when there are more than 4 boxes of a +// kind it aligns to the mean (rows/headers) or median (columns) of the edges, +// otherwise to the plain min/max extremes. The adjustment is one-sided — a box +// edge is only pulled in to the bound when it exceeds it, never pushed out. +func alignTSR(boxes []TSRBox) { + var leftVals, rightVals, topVals, botVals []float32 + for _, b := range boxes { + if strings.Contains(b.Label, "row") || strings.Contains(b.Label, "header") { + leftVals = append(leftVals, b.X0) + rightVals = append(rightVals, b.X1) + } + if b.Label == "table column" { + topVals = append(topVals, b.Top) + botVals = append(botVals, b.Bottom) + } + } + if len(leftVals) == 0 { + return + } + // Rows/headers: mean when >4 boxes, else min (left) / max (right). + left := meanOf(leftVals) + if len(leftVals) <= 4 { + left = minOf(leftVals) + } + right := meanOf(rightVals) + if len(rightVals) <= 4 { + right = maxOf(rightVals) + } + for i := range boxes { + if strings.Contains(boxes[i].Label, "row") || strings.Contains(boxes[i].Label, "header") { + if boxes[i].X0 > left { + boxes[i].X0 = left + } + if boxes[i].X1 < right { + boxes[i].X1 = right + } + } + } + if len(topVals) == 0 { + return + } + // Columns: median when >4 boxes, else min (top) / max (bottom). + top := medianOf(topVals) + if len(topVals) <= 4 { + top = minOf(topVals) + } + bot := medianOf(botVals) + if len(botVals) <= 4 { + bot = maxOf(botVals) + } + for i := range boxes { + if boxes[i].Label == "table column" { + if boxes[i].Top > top { + boxes[i].Top = top + } + if boxes[i].Bottom < bot { + boxes[i].Bottom = bot + } + } + } +} + +func meanOf(v []float32) float32 { + var s float32 + for _, x := range v { + s += x + } + return s / float32(len(v)) +} + +func medianOf(v []float32) float32 { + s := make([]float32, len(v)) + copy(s, v) + sort.Slice(s, func(i, j int) bool { return s[i] < s[j] }) + n := len(s) + if n%2 == 1 { + return s[n/2] + } + return (s[n/2-1] + s[n/2]) / 2 +} + +var tsrClassMap = map[string]int{ + "table": 0, "table column": 1, "table row": 2, + "table column header": 3, "table projected row header": 4, "table spanning cell": 5, +} + +// Wire emits the Go DocAnalyzer TSR format: +// {"bboxes": [[x0,y0,x1,y1,score,class_id], ...]}. +func (r TSRResult) Wire() string { + rows := make([][]float32, 0, len(r.Boxes)) + w, h := float32(r.W), float32(r.H) + for _, b := range r.Boxes { + cls, ok := tsrClassMap[b.Label] + if !ok { + continue + } + // Clamp into image bounds (mirrors tsr_adapter.py). + x0 := minf(maxf(b.X0, 0), w) + x1 := minf(maxf(b.X1, 0), w) + top := minf(maxf(b.Top, 0), h) + bot := minf(maxf(b.Bottom, 0), h) + rows = append(rows, []float32{x0, top, x1, bot, b.Score, float32(cls)}) + } + out, _ := json.Marshal(map[string]any{"bboxes": rows}) + return string(out) +} + +func minOf(v []float32) float32 { + m := v[0] + for _, x := range v[1:] { + if x < m { + m = x + } + } + return m +} + +func maxOf(v []float32) float32 { + m := v[0] + for _, x := range v[1:] { + if x > m { + m = x + } + } + return m +} diff --git a/internal/deepdoc/native/tsr_decode.go b/internal/deepdoc/native/tsr_decode.go new file mode 100644 index 0000000000..8287c23f89 --- /dev/null +++ b/internal/deepdoc/native/tsr_decode.go @@ -0,0 +1,26 @@ +//go:build cgo + +package native + +// tsr_decode.go — TSR preprocessing. +// +// TSR decodes through the shared Go image decoder (image.go's Decode) and +// resizes with the pure-Go bilinearResize. +// +// Why: deepdoc's production TSR adapter (deepdoc/server/adapters/tsr_adapter.py) +// decodes the crop with PIL (Image.open(...).convert("RGB")), not cv2. Go's +// image decoder lands closer to PIL than OpenCV's JPEG decoder does, so the +// pure-Go decode matches the production TSR baseline within the ~3px parity +// tolerance on real-table fixtures. The cv2 decode used by the DLA/DET/OCR +// paths instead diverges from PIL by ~1px and, on boundary boxes, can flip a +// structural box by tens of pixels. +// +// The CHW assembly is shared via tsrBlob; ref_tsr.py feeds the model BGR, which +// is exactly what img.ToBGR() yields from the Go-decoded RGB pixels. + +func tsrPreprocess(img *Image) (blob []float32, scaleFactor [2]float32) { + bgr := img.ToBGR() + resized := bilinearResize(bgr, img.W, img.H, tsrInputSize, tsrInputSize) + out := tsrBlob(resized) + return out, tsrScaleFactor(img) +} diff --git a/internal/deepdoc/parser/pdf/batch_smoke_test.go b/internal/deepdoc/parser/pdf/batch_smoke_test.go index 3324e7cb74..6d86df8aca 100644 --- a/internal/deepdoc/parser/pdf/batch_smoke_test.go +++ b/internal/deepdoc/parser/pdf/batch_smoke_test.go @@ -19,7 +19,6 @@ import ( "time" "unicode/utf8" - inf "ragflow/internal/deepdoc/parser/pdf/inference" lyt "ragflow/internal/deepdoc/parser/pdf/layout" "ragflow/internal/deepdoc/parser/pdf/table" "ragflow/internal/deepdoc/parser/pdf/tool" @@ -52,14 +51,8 @@ func TestBatchResults(t *testing.T) { } pdfs := all[:min(count, len(all))] - ddClient, err := inf.NewClient(common.GetEnv(common.EnvDeepDocURL)) - if err != nil { - t.Fatal(err) - } - if !ddClient.Health() { - t.Fatalf("DeepDoc service not available at %s (DLA+TSR required)", ddClient.BaseURL()) - } - deepDoc := pdf.DocAnalyzer(ddClient) + ddClient := mustConnectInProcessAnalyzer(t) + deepDoc := ddClient variant := "ocr" t.Logf("DeepDoc available — DLA+TSR+OCR enabled (%d PDFs)", len(pdfs)) diff --git a/internal/deepdoc/parser/pdf/dla_real_world_test.go b/internal/deepdoc/parser/pdf/dla_real_world_test.go index 066b3ed1a8..7c606b593d 100644 --- a/internal/deepdoc/parser/pdf/dla_real_world_test.go +++ b/internal/deepdoc/parser/pdf/dla_real_world_test.go @@ -3,7 +3,6 @@ package pdf import ( - "context" "os" "path/filepath" "testing" @@ -12,7 +11,7 @@ import ( // TestDLARealWorldCompare runs DLA on fixture PDFs and verifies // region count, label types, and structural invariants. func TestDLARealWorldCompare(t *testing.T) { - client := mustConnectInferenceClient(t) + client := mustConnectInProcessAnalyzer(t) outDir := filepath.Join("testdata", "output", "render_compare") os.MkdirAll(outDir, 0755) diff --git a/internal/deepdoc/parser/pdf/dla_tsr_compare_test.go b/internal/deepdoc/parser/pdf/dla_tsr_compare_test.go index dcda1ce7c8..059d9b97ad 100644 --- a/internal/deepdoc/parser/pdf/dla_tsr_compare_test.go +++ b/internal/deepdoc/parser/pdf/dla_tsr_compare_test.go @@ -23,7 +23,7 @@ import ( // 2. Run Python: python3 tools/dla_tsr_compare.py // 3. Diff the JSON: diff testdata/output/render_compare/go_dla.json testdata/output/render_compare/py_dla.json func TestDLATSRResponseCompare(t *testing.T) { - client := mustConnectInferenceClient(t) + client := mustConnectInProcessAnalyzer(t) eng := mustOpenEngine(t, "06_table_content.pdf") defer eng.Close() @@ -35,14 +35,14 @@ func TestDLATSRResponseCompare(t *testing.T) { outDir := filepath.Join("testdata", "output", "render_compare") os.MkdirAll(outDir, 0755) - // Save rendered image as JPEG (matching what DLA/TSR actually send). - jpegData, err := util.EncodeJPEG(pageImg) + // Save rendered image as PNG (matching what DLA/TSR actually send). + pngData, err := util.EncodePNG(pageImg) if err != nil { - t.Fatalf("encode jpeg: %v", err) + t.Fatalf("encode png: %v", err) } - imgPath := filepath.Join(outDir, "dla_input.jpeg") - os.WriteFile(imgPath, jpegData, 0644) - t.Logf("Input image saved: %s (%dx%d, %d bytes JPEG)", imgPath, pageImg.Bounds().Dx(), pageImg.Bounds().Dy(), len(jpegData)) + imgPath := filepath.Join(outDir, "dla_input.png") + os.WriteFile(imgPath, pngData, 0644) + t.Logf("Input image saved: %s (%dx%d, %d bytes PNG)", imgPath, pageImg.Bounds().Dx(), pageImg.Bounds().Dy(), len(pngData)) ctx := t.Context() // ── DLA ── @@ -73,9 +73,9 @@ func TestDLATSRResponseCompare(t *testing.T) { int(tableRegion.X0), int(tableRegion.Y0), int(tableRegion.X1), int(tableRegion.Y1)) - cropPath := filepath.Join(outDir, "tsr_input.jpeg") - cropJPEG, _ := util.EncodeJPEG(cropped) - os.WriteFile(cropPath, cropJPEG, 0644) + cropPath := filepath.Join(outDir, "tsr_input.png") + cropPNG, _ := util.EncodePNG(cropped) + os.WriteFile(cropPath, cropPNG, 0644) cells, err := client.TSR(ctx, cropped) if err != nil { @@ -105,9 +105,9 @@ func TestDLATSRResponseCompare(t *testing.T) { cropped := cropImageRect(pageImg, int(b.X0), int(b.Y0), int(b.X2), int(b.Y2)) - cropPath := filepath.Join(outDir, "ocr_rec_input.jpeg") - recJPEG, _ := util.EncodeJPEG(cropped) - os.WriteFile(cropPath, recJPEG, 0644) + cropPath := filepath.Join(outDir, "ocr_rec_input.png") + recPNG, _ := util.EncodePNG(cropped) + os.WriteFile(cropPath, recPNG, 0644) var texts []pdf.OCRText texts, err = client.OCRRecognize(ctx, cropped) diff --git a/internal/deepdoc/parser/pdf/dla_tsr_geom_cmp_test.go b/internal/deepdoc/parser/pdf/dla_tsr_geom_cmp_test.go deleted file mode 100644 index 42f44ed1a7..0000000000 --- a/internal/deepdoc/parser/pdf/dla_tsr_geom_cmp_test.go +++ /dev/null @@ -1,367 +0,0 @@ -//go:build cgo && integration - -package pdf - -import ( - "encoding/json" - "math" - "os" - "path/filepath" - "sort" - "testing" - - "ragflow/internal/deepdoc/parser/pdf/tool" - pdf "ragflow/internal/deepdoc/parser/pdf/type" - util "ragflow/internal/deepdoc/parser/pdf/util" -) - -// cmpBox is a bbox in PDF page-point space (1pt = 1/72in). -type cmpBox struct{ x0, y0, x1, y1 float64 } - -// MarshalJSON emits the corner coordinates explicitly: the struct fields are -// unexported, so the default json marshaler would skip them and write `{}`. -func (b cmpBox) MarshalJSON() ([]byte, error) { - return json.Marshal(struct { - X0, Y0, X1, Y1 float64 - }{b.x0, b.y0, b.x1, b.y1}) -} - -// TestDLATSRGeomCompareReal measures the geometric divergence between Go's -// PRODUCTION TSR (pdfium render -> DEEPDOC_URL) and Python's production TSR -// (dumped by dump_py_results.py into tsr_raw/) for ONE real PDF, so we can -// answer "do the two sides feed the assembly layer different geometry, and by -// how much?" with measured numbers instead of code-path inference. -// -// Usage (needs a live DEEPDOC_URL inference service; see AGENTS.md `integration` tier): -// -// TSR_CMP_PDF=/abs/path/to.pdf \ -// TSR_CMP_PY_TSR=/abs/path/to/py_tsr_raw.json \ -// bash build.sh --test-integration ./internal/deepdoc/parser/pdf/ -run TestDLATSRGeomCompareReal -// -// Both sides are mapped into PDF page-point space and matched greedily by -// cell-center proximity. We report cell-count delta, mean/max center offset -// (points), and mean IoU — the actual geometric gap. -func TestDLATSRGeomCompareReal(t *testing.T) { - pdfPath := os.Getenv("TSR_CMP_PDF") - pyTSRPath := os.Getenv("TSR_CMP_PY_TSR") - if pdfPath == "" || pyTSRPath == "" { - t.Skip("set TSR_CMP_PDF and TSR_CMP_PY_TSR") - } - - data, err := os.ReadFile(pdfPath) - if err != nil { - t.Fatalf("read pdf: %v", err) - } - eng, err := NewEngine(data) - if err != nil { - t.Fatalf("open engine: %v", err) - } - defer eng.Close() - - client := mustConnectInferenceClient(t) - ctx := t.Context() - - pageImg, err := RenderPageToImage(eng, 0) - if err != nil { - t.Fatalf("render: %v", err) - } - regions, err := client.DLA(ctx, pageImg) - if err != nil { - t.Fatalf("DLA: %v", err) - } - var tr *pdf.DLARegion - for i := range regions { - if regions[i].Label == "table" { - tr = ®ions[i] - break - } - } - if tr == nil { - t.Fatal("no table region detected by DLA") - } - t.Logf("Go DLA table region (full-page px): [%.1f, %.1f, %.1f, %.1f]", - tr.X0, tr.Y0, tr.X1, tr.Y1) - - cropped := cropImageRect(pageImg, - int(tr.X0), int(tr.Y0), int(tr.X1), int(tr.Y1)) - goCells, err := client.TSR(ctx, cropped) - if err != nil { - t.Fatalf("TSR: %v", err) - } - - scale := pdf.DlaScale - var goBoxes []cmpBox - for _, c := range goCells { - goBoxes = append(goBoxes, cmpBox{ - (c.X0 + tr.X0) / scale, - (c.Y0 + tr.Y0) / scale, - (c.X1 + tr.X0) / scale, - (c.Y1 + tr.Y0) / scale, - }) - } - - pyCells, err := tool.LoadPythonTSR(pyTSRPath) - if err != nil { - t.Fatalf("load py tsr: %v", err) - } - var pyBoxes []cmpBox - for _, c := range pyCells { - if c.Page != 0 { - continue - } - pyBoxes = append(pyBoxes, cmpBox{c.X0, c.Y0, c.X1, c.Y1}) - } - - t.Logf("Go cells (page0, mapped to points) = %d", len(goBoxes)) - t.Logf("Python cells (page0) = %d", len(pyBoxes)) - - // Greedy nearest-center matching: each Python box -> closest unused Go box. - pairs := matchNearest(pyBoxes, goBoxes) - - if len(pairs) == 0 { - t.Fatalf("no matched cells — impossible to compare geometry") - } - var sumD, maxD, sumIoU float64 - for _, p := range pairs { - sumD += p.dCenter - if p.dCenter > maxD { - maxD = p.dCenter - } - sumIoU += p.iou - } - meanD := sumD / float64(len(pairs)) - meanIoU := sumIoU / float64(len(pairs)) - unmatched := len(pyBoxes) + len(goBoxes) - 2*len(pairs) - - t.Logf("Matched pairs = %d", len(pairs)) - t.Logf("Unmatched cells = %d", unmatched) - t.Logf("Mean center offset = %.2f pt", meanD) - t.Logf("Max center offset = %.2f pt", maxD) - t.Logf("Mean IoU = %.3f", meanIoU) - - // The raw offset above includes the DLA table-REGION detection gap (Go's - // DLA crop may be larger/shifted vs Python's table extent). To isolate the - // TSR CELL-GRID geometry itself, align each side's overall table bbox to a - // common origin (translation only — both tables are ~same size) and re-measure. - goB := bboxOf(goBoxes) - pyB := bboxOf(pyBoxes) - t.Logf("Go table extent (pt) = %.0f x %.0f", goB.x1-goB.x0, goB.y1-goB.y0) - t.Logf("Py table extent (pt) = %.0f x %.0f", pyB.x1-pyB.x0, pyB.y1-pyB.y0) - shiftX, shiftY := pyB.x0-goB.x0, pyB.y0-goB.y0 - var goNorm []cmpBox - for _, b := range goBoxes { - goNorm = append(goNorm, cmpBox{b.x0 + shiftX, b.y0 + shiftY, b.x1 + shiftX, b.y1 + shiftY}) - } - // Re-match in the aligned frame. - pairsN := matchNearest(pyBoxes, goNorm) - var sumDN, maxDN, sumIoUN float64 - for _, p := range pairsN { - sumDN += p.dCenter - if p.dCenter > maxDN { - maxDN = p.dCenter - } - sumIoUN += p.iou - } - t.Logf("--- after aligning table bboxes (isolates TSR cell-grid) ---") - t.Logf("Aligned mean center offset = %.2f pt", sumDN/float64(len(pairsN))) - t.Logf("Aligned max center offset = %.2f pt", maxDN) - t.Logf("Aligned mean IoU = %.3f", sumIoUN/float64(len(pairsN))) - - // Surface the worst 5 matched pairs (aligned frame) so the gap is concrete. - sort.Slice(pairsN, func(i, j int) bool { return pairsN[i].dCenter > pairsN[j].dCenter }) - for i := 0; i < len(pairsN) && i < 5; i++ { - w := pairsN[i] - t.Logf(" worst#%d off=%.1fpt PY=[%.0f,%.0f,%.0f,%.0f] GO=[%.0f,%.0f,%.0f,%.0f]", - i+1, w.dCenter, - w.py.x0, w.py.y0, w.py.x1, w.py.y1, - w.goC.x0, w.goC.y0, w.goC.x1, w.goC.y1) - } - - outDir := os.Getenv("TSR_CMP_OUT") - if outDir == "" { - outDir = filepath.Join("testdata", "output", "render_compare") - } - if err := os.MkdirAll(outDir, 0755); err != nil { - t.Logf("warn: mkdir %s: %v", outDir, err) - } - if b, err := json.MarshalIndent(goBoxes, "", " "); err != nil { - t.Logf("warn: marshal go boxes: %v", err) - } else if err := os.WriteFile(filepath.Join(outDir, "go_tsr_real_points.json"), b, 0644); err != nil { - t.Logf("warn: write go_tsr_real_points.json: %v", err) - } -} - -// TestSameImageTSRConsistency isolates the ONE variable the user asked about: -// "for the SAME image sent to the inference service, do Go and Python get -// consistent TSR?" Go's client.TSR internally does util.EncodePNG(cropped) -// then POSTs multipart(field "request", file "tsr.png") to /predict/tsr. We -// save that EXACT PNG the Go client would send, then a Python caller POSTs the -// same bytes to the same endpoint; the bboxes must be identical (the service -// is stateless + deterministic, caller-agnostic). Any real-PDF Go-vs-Python -// TSR gap therefore comes from the INPUT IMAGE differing (pdfium vs -// pdfplumber), not from the service treating callers differently. -// -// Usage (needs a live DEEPDOC_URL inference service; see AGENTS.md `integration` tier): -// -// TSR_CMP_PDF=/abs/path/to.pdf \ -// bash build.sh --test-integration ./internal/deepdoc/parser/pdf/ -run TestSameImageTSRConsistency -// -// then run tools-py/cmp_same_image_tsr.py (reads the saved PNG + go json). -func TestSameImageTSRConsistency(t *testing.T) { - pdfPath := os.Getenv("TSR_CMP_PDF") - if pdfPath == "" { - t.Skip("set TSR_CMP_PDF") - } - data, err := os.ReadFile(pdfPath) - if err != nil { - t.Fatalf("read pdf: %v", err) - } - eng, err := NewEngine(data) - if err != nil { - t.Fatalf("open engine: %v", err) - } - defer eng.Close() - - client := mustConnectInferenceClient(t) - ctx := t.Context() - pageImg, err := RenderPageToImage(eng, 0) - if err != nil { - t.Fatalf("render: %v", err) - } - regions, err := client.DLA(ctx, pageImg) - if err != nil { - t.Fatalf("DLA: %v", err) - } - var tr *pdf.DLARegion - for i := range regions { - if regions[i].Label == "table" { - tr = ®ions[i] - break - } - } - if tr == nil { - t.Fatal("no table region detected by DLA") - } - cropped := cropImageRect(pageImg, int(tr.X0), int(tr.Y0), int(tr.X1), int(tr.Y1)) - - // Output dir: override with TSR_CMP_OUT (the default testdata dir may not - // be writable under the current user); the Python comparator reads the same. - outDir := os.Getenv("TSR_CMP_OUT") - if outDir == "" { - outDir = filepath.Join("testdata", "output", "render_compare") - } - if err := os.MkdirAll(outDir, 0755); err != nil { - t.Logf("warn: mkdir %s: %v", outDir, err) - } - - // The exact bytes Go's client.TSR sends to the service. - pngBytes, err := util.EncodePNG(cropped) - if err != nil { - t.Fatalf("encode png: %v", err) - } - if err := os.WriteFile(filepath.Join(outDir, "tsr_same_input.png"), pngBytes, 0644); err != nil { - t.Fatalf("write png: %v", err) - } - t.Logf("Saved exact PNG Go sends (%d bytes, %dx%d) → %s/tsr_same_input.png", - len(pngBytes), cropped.Bounds().Dx(), cropped.Bounds().Dy(), outDir) - - // Go's TSR result in the PNG's own pixel space (no coordinate transform). - goCells, err := client.TSR(ctx, cropped) - if err != nil { - t.Fatalf("TSR: %v", err) - } - type rawCell struct { - X0, Y0, X1, Y1 float64 - Label string - Score float64 - } - out := make([]rawCell, 0, len(goCells)) - for _, c := range goCells { - out = append(out, rawCell{c.X0, c.Y0, c.X1, c.Y1, c.Label, c.Score}) - } - writeJSON(t, filepath.Join(outDir, "go_tsr_same_px.json"), out) - t.Logf("Go TSR cells (PNG px) = %d → %s/go_tsr_same_px.json", len(out), outDir) - t.Logf("Now run: python3 internal/deepdoc/parser/pdf/tool-py/cmp_same_image_tsr.py %s", outDir) -} - -// geomPair is one greedily-matched Python↔Go table cell. -type geomPair struct { - dCenter float64 - iou float64 - py, goC cmpBox -} - -// matchNearest greedily matches each Python box to the closest unused Go box -// (by cell-center distance) and returns the matched pairs. Both slices must be -// expressed in the same coordinate space before calling. -func matchNearest(pyBoxes, goBoxes []cmpBox) []geomPair { - var pairs []geomPair - used := make([]bool, len(goBoxes)) - for _, pb := range pyBoxes { - pcx, pcy := boxCenter(pb) - best := -1 - var bestD float64 - for j, gb := range goBoxes { - if used[j] { - continue - } - gcx, gcy := boxCenter(gb) - d := math.Hypot(pcx-gcx, pcy-gcy) - if best == -1 || d < bestD { - best = j - bestD = d - } - } - if best == -1 { - continue - } - used[best] = true - pairs = append(pairs, geomPair{ - dCenter: bestD, - iou: boxIoU(pb, goBoxes[best]), - py: pb, - goC: goBoxes[best], - }) - } - return pairs -} - -func boxCenter(b cmpBox) (float64, float64) { - return (b.x0 + b.x1) / 2, (b.y0 + b.y1) / 2 -} - -// bboxOf returns the overall bounding box enclosing all boxes. -func bboxOf(boxes []cmpBox) cmpBox { - b := boxes[0] - for _, x := range boxes[1:] { - if x.x0 < b.x0 { - b.x0 = x.x0 - } - if x.y0 < b.y0 { - b.y0 = x.y0 - } - if x.x1 > b.x1 { - b.x1 = x.x1 - } - if x.y1 > b.y1 { - b.y1 = x.y1 - } - } - return b -} - -func boxIoU(a, b cmpBox) float64 { - ix0 := math.Max(a.x0, b.x0) - iy0 := math.Max(a.y0, b.y0) - ix1 := math.Min(a.x1, b.x1) - iy1 := math.Min(a.y1, b.y1) - iw := ix1 - ix0 - ih := iy1 - iy0 - if iw <= 0 || ih <= 0 { - return 0 - } - inter := iw * ih - areaA := (a.x1 - a.x0) * (a.y1 - a.y0) - areaB := (b.x1 - b.x0) * (b.y1 - b.y0) - return inter / (areaA + areaB - inter) -} diff --git a/internal/deepdoc/parser/pdf/helpers_test.go b/internal/deepdoc/parser/pdf/helpers_test.go index 0d649baaa8..3833207e0b 100644 --- a/internal/deepdoc/parser/pdf/helpers_test.go +++ b/internal/deepdoc/parser/pdf/helpers_test.go @@ -5,28 +5,34 @@ package pdf import ( "os" "path/filepath" - "ragflow/internal/common" "testing" - inf "ragflow/internal/deepdoc/parser/pdf/inference" + "ragflow/internal/deepdoc/native" + infnative "ragflow/internal/deepdoc/parser/pdf/inference/native_analyzer" pdf "ragflow/internal/deepdoc/parser/pdf/type" + deepdoctype "ragflow/internal/deepdoc/parser/type" ) -// mustConnectInferenceClient returns a InferenceClient for the OSS DeepDoc service. -func mustConnectInferenceClient(t *testing.T) *inf.Client { +// mustConnectInProcessAnalyzer builds the in-process NativeAnalyzer (real ONNX +// inference). It replaces the former external Python service client: the +// production path serves DeepDoc entirely in-process, so regression tests +// should exercise the same backend. The test is skipped (not failed) when +// MODEL_DIR is unset (ONNX Runtime is statically linked and resolved via +// dlopen(NULL)); native.InitORT is idempotent across tests. +func mustConnectInProcessAnalyzer(t *testing.T) deepdoctype.DocAnalyzer { t.Helper() - url := common.GetEnv(common.EnvOSSDeepDocURL) - if url == "" { - url = "http://localhost:9390" + modelDir := os.Getenv("MODEL_DIR") + if modelDir == "" { + t.Skip("MODEL_DIR required (in-process backend integration)") } - client, err := inf.NewClient(url) + if err := native.InitORT(); err != nil { + t.Fatalf("InitORT: %v", err) + } + a, err := infnative.NewAnalyzer(modelDir, infnative.DefaultDropScore) if err != nil { - t.Fatal(err) + t.Fatalf("NewAnalyzer: %v", err) } - if !client.Health() { - t.Fatalf("OssDeepDoc not available at %s", url) - } - return client + return a } // mustOpenEngine opens a PDF from testdata/pdfs/ and returns a pdf.PDFEngine. diff --git a/internal/deepdoc/parser/pdf/inference/cache.go b/internal/deepdoc/parser/pdf/inference/cache.go deleted file mode 100644 index 9d130e254d..0000000000 --- a/internal/deepdoc/parser/pdf/inference/cache.go +++ /dev/null @@ -1,239 +0,0 @@ -// -// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// - -package inference - -import ( - "context" - "crypto/sha256" - "encoding/hex" - "fmt" - "image" - "time" - - "ragflow/internal/deepdoc/parser/pdf/util" - doctype "ragflow/internal/deepdoc/parser/type" - "ragflow/internal/engine/redis" -) - -// DefaultCacheTTL is the cache entry lifetime applied when the -// caller does not supply one. 1 hour matches the python -// deepdoc cache policy. -const DefaultCacheTTL = time.Hour - -// cacheKeyPrefix namespaces DocAnalyzer cache entries so they -// do not collide with other Redis users in the same database. -const cacheKeyPrefix = "ddoc:cache:" - -// cacheStore abstracts the Redis-side persistence so the -// wrapper can be exercised against an in-memory fake in unit -// tests. The interface intentionally mirrors only the -// operations the cache needs; Redis features beyond this stay -// out of the wrapping path. -type cacheStore interface { - Enabled() bool - GetObj(ctx context.Context, key string, dest any) bool - SetObj(ctx context.Context, key string, value any, ttl time.Duration) bool -} - -// redisCacheStore is the production store, backed by the -// package-level Redis singleton. When Redis is unconfigured -// the package-level methods become no-ops (return false / treat -// as miss), which is exactly the behaviour we want. -type redisCacheStore struct{} - -// Enabled forwards to redis.IsEnabled(). -func (redisCacheStore) Enabled() bool { return redis.IsEnabled() } - -// GetObj forwards to the package-level Redis singleton. -// Errors are logged inside the engine; we treat any failure -// (including connectivity) as a cache miss for safety. -func (redisCacheStore) GetObj(ctx context.Context, key string, dest any) bool { - return redis.Get().GetObj(ctx, key, dest) -} - -// SetObj forwards to the package-level Redis singleton. -func (redisCacheStore) SetObj(ctx context.Context, key string, value any, ttl time.Duration) bool { - return redis.Get().SetObj(ctx, key, value, ttl) -} - -// DocAnalyzerCache wraps an inner doctype.DocAnalyzer and -// transparently caches its 4 image-keyed methods in Redis. -// -// Behaviour: -// -// - If the store is disabled, the wrapper passes every call -// straight through to the inner analyser. Tests and -// developer machines without Redis configured see no -// behavioural change. -// - On cache hit the inner call is skipped entirely. -// - On cache miss the inner call runs; its successful result -// is JSON-serialised and stored under the deterministic -// sha256-based key with the configured TTL. -// - Health() is a pure passthrough; caching it would not be -// semantically meaningful. -// -// The wrapper is safe for concurrent use: the cache lookup -// path uses only the store's own synchronisation, so callers -// can route per-image OCR through OCRRecognize from multiple -// page workers without additional locking. -type DocAnalyzerCache struct { - inner doctype.DocAnalyzer - store cacheStore - ttl time.Duration -} - -// NewDocAnalyzerCache wraps inner with a Redis-backed cache -// using the supplied TTL (use DefaultCacheTTL for 1h). When -// Redis is not configured, the wrapper transparently degrades -// to a pass-through. -func NewDocAnalyzerCache(inner doctype.DocAnalyzer, ttl time.Duration) *DocAnalyzerCache { - if ttl <= 0 { - ttl = DefaultCacheTTL - } - return &DocAnalyzerCache{ - inner: inner, - store: redisCacheStore{}, - ttl: ttl, - } -} - -// newDocAnalyzerCacheWithStore is the test seam. Production -// code uses NewDocAnalyzerCache; tests inject a fake store to -// drive the cache behaviour deterministically without -// depending on a running Redis. -func newDocAnalyzerCacheWithStore(inner doctype.DocAnalyzer, s cacheStore, ttl time.Duration) *DocAnalyzerCache { - if ttl <= 0 { - ttl = DefaultCacheTTL - } - return &DocAnalyzerCache{inner: inner, store: s, ttl: ttl} -} - -// cacheKey returns the cache key used for the given method + -// image. It encodes the image to PNG (matching what the -// inference HTTP client actually sends) and hashes the -// resulting bytes. PNG encoding is deterministic for a given -// image, so two DocAnalyzer calls with the same -// image content produce the same key. -func cacheKey(method string, img image.Image) (string, error) { - if img == nil { - return "", fmt.Errorf("%s: nil image", method) - } - data, err := util.EncodePNG(img) - if err != nil { - return "", fmt.Errorf("%s: encode: %w", method, err) - } - sum := sha256.Sum256(data) - return fmt.Sprintf("%s%s:%s", cacheKeyPrefix, method, hex.EncodeToString(sum[:])), nil -} - -// cacheKeyOrEmpty returns the cache key for img, or "" if the -// image should bypass caching (nil image, encode failure). The -// empty string is the "no key" sentinel — callers must skip -// the cache and call inner directly in that case. -func cacheKeyOrEmpty(method string, img image.Image) string { - k, err := cacheKey(method, img) - if err != nil { - return "" - } - return k -} - -// DLA see doctype.DocAnalyzer. -func (c *DocAnalyzerCache) DLA(ctx context.Context, img image.Image) ([]doctype.DLARegion, error) { - if key := cacheKeyOrEmpty("dla", img); key != "" && c.store.Enabled() { - var cached []doctype.DLARegion - if c.store.GetObj(ctx, key, &cached) { - return cached, nil - } - } - out, err := c.inner.DLA(ctx, img) - if err != nil || img == nil { - return out, err - } - if key := cacheKeyOrEmpty("dla", img); key != "" && c.store.Enabled() { - c.store.SetObj(ctx, key, out, c.ttl) - } - return out, nil -} - -// TSR see doctype.DocAnalyzer. -func (c *DocAnalyzerCache) TSR(ctx context.Context, img image.Image) ([]doctype.TSRCell, error) { - if key := cacheKeyOrEmpty("tsr", img); key != "" && c.store.Enabled() { - var cached []doctype.TSRCell - if c.store.GetObj(ctx, key, &cached) { - return cached, nil - } - } - out, err := c.inner.TSR(ctx, img) - if err != nil || img == nil { - return out, err - } - if key := cacheKeyOrEmpty("tsr", img); key != "" && c.store.Enabled() { - c.store.SetObj(ctx, key, out, c.ttl) - } - return out, nil -} - -// OCRDetect see doctype.DocAnalyzer. -func (c *DocAnalyzerCache) OCRDetect(ctx context.Context, img image.Image) ([]doctype.OCRBox, error) { - if key := cacheKeyOrEmpty("ocr_detect", img); key != "" && c.store.Enabled() { - var cached []doctype.OCRBox - if c.store.GetObj(ctx, key, &cached) { - return cached, nil - } - } - out, err := c.inner.OCRDetect(ctx, img) - if err != nil || img == nil { - return out, err - } - if key := cacheKeyOrEmpty("ocr_detect", img); key != "" && c.store.Enabled() { - c.store.SetObj(ctx, key, out, c.ttl) - } - return out, nil -} - -// OCRRecognize see doctype.DocAnalyzer. -func (c *DocAnalyzerCache) OCRRecognize(ctx context.Context, img image.Image) ([]doctype.OCRText, error) { - if key := cacheKeyOrEmpty("ocr_recognize", img); key != "" && c.store.Enabled() { - var cached []doctype.OCRText - if c.store.GetObj(ctx, key, &cached) { - return cached, nil - } - } - out, err := c.inner.OCRRecognize(ctx, img) - if err != nil || img == nil { - return out, err - } - if key := cacheKeyOrEmpty("ocr_recognize", img); key != "" && c.store.Enabled() { - c.store.SetObj(ctx, key, out, c.ttl) - } - return out, nil -} - -// Health see doctype.DocAnalyzer. Health probes are -// deliberately uncached — caching the outcome would defeat the -// purpose of a health check. -func (c *DocAnalyzerCache) Health() bool { - if c == nil || c.inner == nil { - return false - } - return c.inner.Health() -} - -// Compile-time guarantee that DocAnalyzerCache satisfies the -// doctype.DocAnalyzer interface. -var _ doctype.DocAnalyzer = (*DocAnalyzerCache)(nil) diff --git a/internal/deepdoc/parser/pdf/inference/cache_test.go b/internal/deepdoc/parser/pdf/inference/cache_test.go deleted file mode 100644 index bd37bf3cc6..0000000000 --- a/internal/deepdoc/parser/pdf/inference/cache_test.go +++ /dev/null @@ -1,375 +0,0 @@ -// -// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// - -package inference - -import ( - "context" - "encoding/json" - "errors" - "image" - "image/color" - "sync" - "sync/atomic" - "testing" - "time" - - doctype "ragflow/internal/deepdoc/parser/type" -) - -// ── fake store ─────────────────────────────────────────────────────── - -// fakeStore is an in-memory implementation of cacheStore used -// by tests. Safe for concurrent use. -type fakeStore struct { - mu sync.Mutex - enabled bool - data map[string][]byte - sets int32 - gets int32 -} - -func newFakeStore() *fakeStore { - return &fakeStore{enabled: true, data: map[string][]byte{}} -} - -func (s *fakeStore) Enabled() bool { return s.enabled } - -func (s *fakeStore) GetObj(ctx context.Context, key string, dest any) bool { - atomic.AddInt32(&s.gets, 1) - s.mu.Lock() - defer s.mu.Unlock() - raw, ok := s.data[key] - if !ok { - return false - } - return json.Unmarshal(raw, dest) == nil -} - -func (s *fakeStore) SetObj(ctx context.Context, key string, value any, _ time.Duration) bool { - atomic.AddInt32(&s.sets, 1) - raw, err := json.Marshal(value) - if err != nil { - return false - } - s.mu.Lock() - defer s.mu.Unlock() - s.data[key] = raw - return true -} - -// ── fake analyser ────────────────────────────────────────────────── - -// fakeAnalyzer is a deterministic, count-tracking DocAnalyzer. -// Each method increments a counter and returns a value shaped -// uniquely so a cache hit can be told apart from an inner call. -type fakeAnalyzer struct { - dlaCount int32 - tsrCount int32 - ocrDetCnt int32 - ocrRecCnt int32 - - healthy bool - - dlaOut []doctype.DLARegion - tsrOut []doctype.TSRCell - ocrDet []doctype.OCRBox - ocrRec []doctype.OCRText - dlaErr error - tsrErr error - ocrDetEr error - ocrRecEr error -} - -func (f *fakeAnalyzer) DLA(_ context.Context, _ image.Image) ([]doctype.DLARegion, error) { - atomic.AddInt32(&f.dlaCount, 1) - if f.dlaErr != nil { - return nil, f.dlaErr - } - return f.dlaOut, nil -} - -func (f *fakeAnalyzer) TSR(_ context.Context, _ image.Image) ([]doctype.TSRCell, error) { - atomic.AddInt32(&f.tsrCount, 1) - if f.tsrErr != nil { - return nil, f.tsrErr - } - return f.tsrOut, nil -} - -func (f *fakeAnalyzer) OCRDetect(_ context.Context, _ image.Image) ([]doctype.OCRBox, error) { - atomic.AddInt32(&f.ocrDetCnt, 1) - if f.ocrDetEr != nil { - return nil, f.ocrDetEr - } - return f.ocrDet, nil -} - -func (f *fakeAnalyzer) OCRRecognize(_ context.Context, _ image.Image) ([]doctype.OCRText, error) { - atomic.AddInt32(&f.ocrRecCnt, 1) - if f.ocrRecEr != nil { - return nil, f.ocrRecEr - } - return f.ocrRec, nil -} - -func (f *fakeAnalyzer) Health() bool { return f.healthy } - -// ── helpers ──────────────────────────────────────────────────────── - -// newImage returns a tiny 4x4 RGBA image with stable pixel -// content. JPEG encoding of identical pixel data is -// deterministic, so the cache key for the same newImage() result -// is the same across calls. -func newImage(seed byte) *image.RGBA { - img := image.NewRGBA(image.Rect(0, 0, 4, 4)) - for y := 0; y < 4; y++ { - for x := 0; x < 4; x++ { - img.SetRGBA(x, y, color.RGBA{seed, seed * 2, seed * 3, 255}) - } - } - return img -} - -// ── key tests ────────────────────────────────────────────────────── - -func TestCacheKey_Deterministic(t *testing.T) { - a, errA := cacheKey("dla", newImage(0x10)) - if errA != nil { - t.Fatalf("cacheKey(dla,#1): %v", errA) - } - b, errB := cacheKey("dla", newImage(0x10)) - if errB != nil { - t.Fatalf("cacheKey(dla,#2): %v", errB) - } - if a != b { - t.Fatalf("same image → different key: %q vs %q", a, b) - } - if want := cacheKeyPrefix + "dla:"; len(a) < len(want) || a[:len(want)] != want { - t.Fatalf("key missing prefix: %q", a) - } - - c, errC := cacheKey("tsr", newImage(0x10)) - if errC != nil { - t.Fatalf("cacheKey(tsr): %v", errC) - } - if c == a { - t.Fatal("different methods should yield different keys") - } - - d, errD := cacheKey("dla", newImage(0x11)) - if errD != nil { - t.Fatalf("cacheKey(dla,#2): %v", errD) - } - if d == a { - t.Fatal("different image bytes should yield different keys") - } -} - -func TestCacheKey_NilImageErrors(t *testing.T) { - if _, err := cacheKey("dla", nil); err == nil { - t.Fatal("want error for nil image, got nil") - } -} - -// ── cache hit / miss / disabled ──────────────────────────────────── - -func TestDocAnalyzerCache_MissThenHit(t *testing.T) { - inner := &fakeAnalyzer{ - dlaOut: []doctype.DLARegion{{X0: 1, Y0: 2, X1: 3, Y1: 4, Label: "text"}}, - } - s := newFakeStore() - c := newDocAnalyzerCacheWithStore(inner, s, DefaultCacheTTL) - - img := newImage(0x20) - - out1, err := c.DLA(t.Context(), img) - if err != nil { - t.Fatalf("DLA #1: %v", err) - } - if got := atomic.LoadInt32(&inner.dlaCount); got != 1 { - t.Fatalf("inner called %d times after first DLA, want 1", got) - } - if got := atomic.LoadInt32(&s.sets); got != 1 { - t.Fatalf("store.SetObj called %d times after first DLA, want 1", got) - } - - out2, err := c.DLA(t.Context(), img) - if err != nil { - t.Fatalf("DLA #2: %v", err) - } - if got := atomic.LoadInt32(&inner.dlaCount); got != 1 { - t.Fatalf("inner called %d times after hit, want 1 (cached)", got) - } - if got := atomic.LoadInt32(&s.gets); got < 1 { - t.Fatalf("store.GetObj not called, want ≥1") - } - - if len(out1) != len(out2) || out1[0].Label != out2[0].Label { - t.Fatalf("cached result diverged from inner: %v vs %v", out1, out2) - } -} - -func TestDocAnalyzerCache_StoreDisabledPassthrough(t *testing.T) { - inner := &fakeAnalyzer{ - dlaOut: []doctype.DLARegion{{Label: "text"}}, - } - s := newFakeStore() - s.enabled = false - c := newDocAnalyzerCacheWithStore(inner, s, DefaultCacheTTL) - - img := newImage(0x30) - for i := 0; i < 3; i++ { - if _, err := c.DLA(t.Context(), img); err != nil { - t.Fatalf("DLA #%d: %v", i, err) - } - } - if got := atomic.LoadInt32(&inner.dlaCount); got != 3 { - t.Fatalf("inner called %d times, want 3 (no caching when disabled)", got) - } - if got := atomic.LoadInt32(&s.sets); got != 0 { - t.Fatalf("store.SetObj called %d times, want 0", got) - } -} - -func TestDocAnalyzerCache_NilImageBypassesCache(t *testing.T) { - inner := &fakeAnalyzer{ - dlaOut: []doctype.DLARegion{{Label: "noop"}}, - } - s := newFakeStore() - c := newDocAnalyzerCacheWithStore(inner, s, DefaultCacheTTL) - - for i := 0; i < 2; i++ { - if _, err := c.DLA(t.Context(), nil); err != nil { - t.Fatalf("DLA nil #%d: %v", i, err) - } - } - if got := atomic.LoadInt32(&s.sets); got != 0 { - t.Fatalf("nil image should bypass store.SetObj, got count %d", got) - } - if got := atomic.LoadInt32(&inner.dlaCount); got != 2 { - t.Fatalf("inner called %d times, want 2 (nil → no cache)", got) - } -} - -// ── all 4 methods ─────────────────────────────────────────────────── - -func TestDocAnalyzerCache_AllMethods(t *testing.T) { - inner := &fakeAnalyzer{ - dlaOut: []doctype.DLARegion{{Label: "title"}}, - tsrOut: []doctype.TSRCell{{X0: 0, Y0: 0, X1: 1, Y1: 1, Text: "A"}}, - ocrDet: []doctype.OCRBox{{X0: 0, Y0: 0, X1: 10, Y1: 10}}, - ocrRec: []doctype.OCRText{{Text: "x", Confidence: 0.9}}, - } - s := newFakeStore() - c := newDocAnalyzerCacheWithStore(inner, s, DefaultCacheTTL) - - img := newImage(0x40) - - if _, err := c.DLA(t.Context(), img); err != nil { - t.Fatalf("DLA: %v", err) - } - if _, err := c.TSR(t.Context(), img); err != nil { - t.Fatalf("TSR: %v", err) - } - if _, err := c.OCRDetect(t.Context(), img); err != nil { - t.Fatalf("OCRDetect: %v", err) - } - if _, err := c.OCRRecognize(t.Context(), img); err != nil { - t.Fatalf("OCRRecognize: %v", err) - } - - if got := atomic.LoadInt32(&s.sets); got != 4 { - t.Fatalf("store.SetObj called %d times, want 4", got) - } - - if _, err := c.DLA(t.Context(), img); err != nil { - t.Fatalf("DLA #2: %v", err) - } - if _, err := c.TSR(t.Context(), img); err != nil { - t.Fatalf("TSR #2: %v", err) - } - if _, err := c.OCRDetect(t.Context(), img); err != nil { - t.Fatalf("OCRDetect #2: %v", err) - } - if _, err := c.OCRRecognize(t.Context(), img); err != nil { - t.Fatalf("OCRRecognize #2: %v", err) - } - - if got := atomic.LoadInt32(&inner.dlaCount); got != 1 { - t.Fatalf("DLA inner count = %d, want 1", got) - } - if got := atomic.LoadInt32(&inner.tsrCount); got != 1 { - t.Fatalf("TSR inner count = %d, want 1", got) - } - if got := atomic.LoadInt32(&inner.ocrDetCnt); got != 1 { - t.Fatalf("OCRDetect inner count = %d, want 1", got) - } - if got := atomic.LoadInt32(&inner.ocrRecCnt); got != 1 { - t.Fatalf("OCRRecognize inner count = %d, want 1", got) - } -} - -// ── inner error propagation ───────────────────────────────────────── - -func TestDocAnalyzerCache_InnerErrorNotCached(t *testing.T) { - inner := &fakeAnalyzer{ - dlaErr: errors.New("inference service down"), - } - s := newFakeStore() - c := newDocAnalyzerCacheWithStore(inner, s, DefaultCacheTTL) - - img := newImage(0x50) - if _, err := c.DLA(t.Context(), img); err == nil { - t.Fatal("want error, got nil") - } - if got := atomic.LoadInt32(&s.sets); got != 0 { - t.Fatalf("error result must not be cached, got sets=%d", got) - } -} - -// ── Health passthrough ───────────────────────────────────────────── - -func TestDocAnalyzerCache_HealthPassthrough(t *testing.T) { - for _, healthy := range []bool{true, false} { - inner := &fakeAnalyzer{healthy: healthy} - s := newFakeStore() - c := newDocAnalyzerCacheWithStore(inner, s, DefaultCacheTTL) - if got := c.Health(); got != healthy { - t.Fatalf("Health() = %v, want %v", got, healthy) - } - } -} - -// ── different images → different keys ─────────────────────────────── - -func TestDocAnalyzerCache_DifferentImagesBypassCache(t *testing.T) { - inner := &fakeAnalyzer{ - dlaOut: []doctype.DLARegion{{Label: "a"}}, - } - s := newFakeStore() - c := newDocAnalyzerCacheWithStore(inner, s, DefaultCacheTTL) - - if _, err := c.DLA(t.Context(), newImage(0x80)); err != nil { - t.Fatal(err) - } - if _, err := c.DLA(t.Context(), newImage(0x81)); err != nil { - t.Fatal(err) - } - if got := atomic.LoadInt32(&inner.dlaCount); got != 2 { - t.Fatalf("inner called %d times for 2 different images, want 2", got) - } -} diff --git a/internal/deepdoc/parser/pdf/inference/client.go b/internal/deepdoc/parser/pdf/inference/client.go deleted file mode 100644 index 1a9978c05c..0000000000 --- a/internal/deepdoc/parser/pdf/inference/client.go +++ /dev/null @@ -1,296 +0,0 @@ -package inference - -import ( - "bytes" - "context" - "encoding/json" - "errors" - "fmt" - "image" - "io" - "log/slog" - "mime/multipart" - "net" - "net/http" - "time" - - pdf "ragflow/internal/deepdoc/parser/pdf/type" - util "ragflow/internal/deepdoc/parser/pdf/util" - - "github.com/cenkalti/backoff/v5" -) - -// Client wraps the DeepDoc HTTP API. -type Client struct { - baseURL string - httpClient *http.Client - - // Label tables for class_id → label string mapping. - // Set by the service layer (model-specific) to reflect the model's taxonomy. - DLALabels []string - TSRLabels []string -} - -// BaseURL returns the configured DeepDoc service URL. -func (c *Client) BaseURL() string { return c.baseURL } - -// NewClient creates a client. baseURL must be provided by the caller -// (e.g. from the DEEPDOC_URL environment variable). Returns an error if empty. -func NewClient(baseURL string) (*Client, error) { - if baseURL == "" { - return nil, fmt.Errorf("deepdoc client: baseURL is required (set DEEPDOC_URL)") - } - return &Client{ - baseURL: baseURL, - httpClient: &http.Client{ - Timeout: 120 * time.Second, - }, - DLALabels: DefaultDLALabels(), - TSRLabels: DefaultTSRLabels(), - }, nil -} - -// DefaultDLALabels returns the 10-class DLA taxonomy matching Python's -// deepdoc/vision/dla_cli.py:10-21. Duplicates at indices 4, 7, 9 are -// kept verbatim for backward compatibility with existing inference servers. -func DefaultDLALabels() []string { - return []string{ - pdf.LayoutTypeTitle, pdf.LayoutTypeText, pdf.LayoutTypeReference, - pdf.LayoutTypeFigure, pdf.DLALabelFigureCaption, - pdf.LayoutTypeTable, pdf.DLALabelTableCaption, pdf.DLALabelTableCaption, - pdf.LayoutTypeEquation, pdf.DLALabelFigureCaption, - } -} - -// DefaultTSRLabels returns the 6-class TSR taxonomy matching Python's -// deepdoc/server/adapters/tsr_adapter.py:21-26. -func DefaultTSRLabels() []string { - return []string{ - "table", "table column", "table row", - "table column header", "table projected row header", - "table spanning cell", - } -} - -type bboxesResponse struct { - BBoxes [][]float64 `json:"bboxes"` -} - -// DLA analyzes a full page image and returns labeled regions. -func (c *Client) DLA(ctx context.Context, pageImage image.Image) ([]pdf.DLARegion, error) { - data, err := util.EncodePNG(pageImage) - if err != nil { - return nil, fmt.Errorf("dla: encode: %w", err) - } - var resp bboxesResponse - if err := c.post(ctx, "/predict/dla", data, "dla.png", &resp); err != nil { - return nil, fmt.Errorf("dla: %w", err) - } - regions := make([]pdf.DLARegion, 0, len(resp.BBoxes)) - for _, b := range resp.BBoxes { - if len(b) < 6 { - continue - } - labels := c.DLALabels - label := "" - if clsID := int(b[5]); clsID >= 0 && clsID < len(labels) { - label = labels[clsID] - } - // Drop low-confidence garbage-layout regions (Python parity: 0.4 gate, - // deepdoc/vision/layout_recognizer.py:97). - if pdf.GarbageLayoutTypes[label] && b[4] < pdf.GarbageLayoutScoreThreshold { - continue - } - regions = append(regions, pdf.DLARegion{ - X0: b[0], Y0: b[1], X1: b[2], Y1: b[3], - Confidence: b[4], - Label: label, - }) - } - return regions, nil -} - -// TSR recognises table structure from a cropped image. -func (c *Client) TSR(ctx context.Context, cropped image.Image) ([]pdf.TSRCell, error) { - data, err := util.EncodePNG(cropped) - if err != nil { - return nil, fmt.Errorf("tsr: encode: %w", err) - } - var resp bboxesResponse - if err := c.post(ctx, "/predict/tsr", data, "tsr.png", &resp); err != nil { - return nil, fmt.Errorf("tsr: %w", err) - } - cells := make([]pdf.TSRCell, 0, len(resp.BBoxes)) - for _, b := range resp.BBoxes { - if len(b) < 5 { - continue - } - tlabels := c.TSRLabels - label := "" - if len(b) >= 6 { - if cls := int(b[5]); cls >= 0 && cls < len(tlabels) { - label = tlabels[cls] - } - } - cells = append(cells, pdf.TSRCell{ - X0: b[0], Y0: b[1], X1: b[2], Y1: b[3], - Label: label, - Score: b[4], - }) - } - return cells, nil -} - -// ocrDetectResponse matches DeepDoc /predict/ocr?operator=det output: -// -// {"output": [[[[[[x0,y0],[x1,y1],[x2,y2],[x3,y3]], ...]]]]} -type ocrDetectResponse struct { - Output [][][][][]float64 `json:"output"` -} - -// ocrRecognizeResponse matches DeepDoc /predict/ocr?operator=rec output: -// -// {"output": [[[["text", confidence], ...]]]} -type ocrRecognizeResponse struct { - Output [][][][]any `json:"output"` -} - -// OCRDetect detects text regions (bounding boxes) in an image. -// DeepDoc /predict/ocr with operator=det returns quad boxes: [[[x0,y0],[x1,y1],[x2,y2],[x3,y3]], ...] -func (c *Client) OCRDetect(ctx context.Context, cropped image.Image) ([]pdf.OCRBox, error) { - data, err := util.EncodePNG(cropped) - if err != nil { - return nil, fmt.Errorf("ocr detect: encode: %w", err) - } - - // First decode outer envelope as RawMessage so we can log on format mismatch. - var rawEnvelope struct { - Output json.RawMessage `json:"output"` - } - if err := c.post(ctx, "/predict/ocr", data, "ocr_detect.png", &rawEnvelope, "operator", "det"); err != nil { - return nil, fmt.Errorf("ocr detect: %w", err) - } - - var result ocrDetectResponse - if err := json.Unmarshal(rawEnvelope.Output, &result.Output); err != nil { - rawStr := string(rawEnvelope.Output) - if len(rawStr) > 1000 { - rawStr = rawStr[:1000] - } - slog.Warn("ocr detect: output format mismatch", "err", err, "raw_output", rawStr) - return nil, fmt.Errorf("ocr detect: %w", err) - } - - var boxes []pdf.OCRBox - for _, outer := range result.Output { - for _, page := range outer { - for _, box := range page { - if len(box) < 4 { - continue - } - boxes = append(boxes, pdf.OCRBox{ - X0: box[0][0], Y0: box[0][1], - X1: box[1][0], Y1: box[1][1], - X2: box[2][0], Y2: box[2][1], - X3: box[3][0], Y3: box[3][1], - }) - } - } - } - return boxes, nil -} - -// OCRRecognize recognizes text in a cropped image region. -// DeepDoc /predict/ocr with operator=rec returns [[["text", confidence], ...]] -func (c *Client) OCRRecognize(ctx context.Context, cropped image.Image) ([]pdf.OCRText, error) { - data, err := util.EncodePNG(cropped) - if err != nil { - return nil, fmt.Errorf("ocr rec: encode: %w", err) - } - var result ocrRecognizeResponse - if err := c.post(ctx, "/predict/ocr", data, "ocr_rec.png", &result, "operator", "rec"); err != nil { - return nil, fmt.Errorf("ocr rec: %w", err) - } - var texts []pdf.OCRText - for _, page := range result.Output { - for _, item := range page { - for _, pair := range item { - if len(pair) >= 2 { - text, _ := pair[0].(string) - conf, _ := pair[1].(float64) - texts = append(texts, pdf.OCRText{Text: text, Confidence: conf}) - } - } - } - } - return texts, nil -} - -// Health checks whether the DeepDoc service is reachable. -func (c *Client) Health() bool { - resp, err := c.httpClient.Get(c.baseURL + "/health") - if err != nil { - return false - } - resp.Body.Close() - return resp.StatusCode == 200 -} - -func (c *Client) post(ctx context.Context, endpoint string, imgData []byte, filename string, result interface{}, extraFields ...string) error { - // Build multipart body once — the image data is idempotent. - var body bytes.Buffer - w := multipart.NewWriter(&body) - fw, err := w.CreateFormFile("request", filename) - if err != nil { - return err - } - if _, err := fw.Write(imgData); err != nil { - return err - } - for i := 0; i+1 < len(extraFields); i += 2 { - w.WriteField(extraFields[i], extraFields[i+1]) - } - w.Close() - contentType := w.FormDataContentType() - bodyBytes := body.Bytes() - - _, err = backoff.Retry(ctx, func() (struct{}, error) { - req, err := http.NewRequestWithContext(ctx, "POST", c.baseURL+endpoint, bytes.NewReader(bodyBytes)) - if err != nil { - return struct{}{}, backoff.Permanent(err) - } - req.Header.Set("Content-Type", contentType) - - resp, err := c.httpClient.Do(req) - if err != nil { - if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { - return struct{}{}, backoff.Permanent(err) - } - var netErr net.Error - if errors.As(err, &netErr) { - slog.Warn("deepdoc: network error, will retry", "endpoint", endpoint, "err", err) - return struct{}{}, err - } - return struct{}{}, backoff.Permanent(err) - } - - if resp.StatusCode == 200 { - defer resp.Body.Close() - return struct{}{}, json.NewDecoder(io.LimitReader(resp.Body, 64<<20)).Decode(result) - } - - errBody, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) - resp.Body.Close() - respErr := fmt.Errorf("http %d: %s", resp.StatusCode, string(errBody[:min(200, len(errBody))])) - - if resp.StatusCode >= 500 { - slog.Warn("deepdoc: server error, will retry", "endpoint", endpoint, "status", resp.StatusCode) - return struct{}{}, respErr - } - // 4xx and other codes are not retryable. - return struct{}{}, backoff.Permanent(respErr) - }, backoff.WithMaxTries(4), backoff.WithNotify(func(err error, d time.Duration) { - slog.Info("deepdoc: retrying", "endpoint", endpoint, "backoff", d.Round(time.Millisecond), "err", err) - })) - return err -} diff --git a/internal/deepdoc/parser/pdf/inference/client_dla_garbage_test.go b/internal/deepdoc/parser/pdf/inference/client_dla_garbage_test.go deleted file mode 100644 index 79a6c397a7..0000000000 --- a/internal/deepdoc/parser/pdf/inference/client_dla_garbage_test.go +++ /dev/null @@ -1,97 +0,0 @@ -package inference - -import ( - "encoding/json" - "net/http" - "net/http/httptest" - "testing" -) - -// TestDeepDocHTTP_DLA_GarbageGate pins the Python-parity 0.4 garbage gate -// (deepdoc/vision/layout_recognizer.py:97): a region whose layout type is a -// garbage layout (footer/header/reference) and whose confidence is strictly -// below 0.4 is dropped; everything else is kept. -// -// The OSS default 10-class DLA taxonomy only emits "reference" as a garbage -// type, but footer/header are covered defensively via pdf.GarbageLayoutTypes. -// Each subtest drives a mock /predict/dla backend returning one bbox and -// asserts the resulting region set. -func TestDeepDocHTTP_DLA_GarbageGate(t *testing.T) { - newClient := func(t *testing.T, bboxes [][]float64) *Client { - t.Helper() - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != "/predict/dla" { - t.Errorf("path = %q, want /predict/dla", r.URL.Path) - } - if err := json.NewEncoder(w).Encode(map[string]any{"bboxes": bboxes}); err != nil { - // Handler runs in its own goroutine; fail the test, don't panic. - t.Errorf("encode response: %v", err) - } - })) - t.Cleanup(srv.Close) - return mustNewDeepDocClient(t, srv.URL) - } - - // bbox = [x0, y0, x1, y1, confidence, classId]; classId 2 = "reference". - const referenceClass = 2 - - t.Run("low_conf_reference_dropped", func(t *testing.T) { - client := newClient(t, [][]float64{ - {50, 10, 500, 50, 0.30, referenceClass}, // reference, low confidence - }) - regions, err := client.DLA(t.Context(), testImage()) - if err != nil { - t.Fatal(err) - } - if len(regions) != 0 { - t.Fatalf("got %d regions, want 0 (low-confidence 'reference' must be dropped by the 0.4 garbage gate)", len(regions)) - } - }) - - t.Run("high_conf_reference_kept", func(t *testing.T) { - client := newClient(t, [][]float64{ - {50, 10, 500, 50, 0.90, referenceClass}, // reference, high confidence - }) - regions, err := client.DLA(t.Context(), testImage()) - if err != nil { - t.Fatal(err) - } - if len(regions) != 1 || regions[0].Label != "reference" { - t.Fatalf("got %v, want exactly one 'reference' region (conf >= 0.4 is kept)", regions) - } - }) - - t.Run("boundary_conf_0.4_kept", func(t *testing.T) { - // Gate is strict (< 0.4); confidence exactly 0.4 is the boundary and kept. - client := newClient(t, [][]float64{ - {50, 10, 500, 50, 0.40, referenceClass}, // reference, exactly 0.4 - }) - regions, err := client.DLA(t.Context(), testImage()) - if err != nil { - t.Fatal(err) - } - if len(regions) != 1 || regions[0].Label != "reference" { - t.Fatalf("got %v, want exactly one 'reference' region (conf == 0.4 is the gate boundary and kept)", regions) - } - }) - - t.Run("low_conf_garbage_and_text", func(t *testing.T) { - // A low-confidence reference is dropped while an unrelated text region - // (non-garbage) is kept — the gate only removes garbage-layout regions - // below 0.4; non-garbage regions pass through at any confidence. - client := newClient(t, [][]float64{ - {50, 10, 500, 50, 0.30, referenceClass}, // reference, low confidence -> dropped - {50, 100, 500, 300, 0.90, 1}, // text, high confidence -> kept - }) - regions, err := client.DLA(t.Context(), testImage()) - if err != nil { - t.Fatal(err) - } - if len(regions) != 1 { - t.Fatalf("got %d regions, want 1 (low-confidence 'reference' dropped, 'text' kept)", len(regions)) - } - if regions[0].Label != "text" { - t.Errorf("regions[0].Label = %q, want 'text'", regions[0].Label) - } - }) -} diff --git a/internal/deepdoc/parser/pdf/inference/client_test.go b/internal/deepdoc/parser/pdf/inference/client_test.go deleted file mode 100644 index c6dd87bcbc..0000000000 --- a/internal/deepdoc/parser/pdf/inference/client_test.go +++ /dev/null @@ -1,319 +0,0 @@ -package inference - -import ( - "encoding/json" - "image" - "image/color" - "net/http" - "net/http/httptest" - "strings" - "testing" -) - -// mustNewDeepDocClient wraps NewClient for test convenience. -// Fails the test if the URL is empty. -func mustNewDeepDocClient(t *testing.T, baseURL string) *Client { - t.Helper() - client, err := NewClient(baseURL) - if err != nil { - t.Fatalf("NewDeepDocClient(%q): %v", baseURL, err) - } - return client -} - -// testImage creates a small 10x10 red image for HTTP client tests. -func testImage() image.Image { - img := image.NewRGBA(image.Rect(0, 0, 10, 10)) - for y := 0; y < 10; y++ { - for x := 0; x < 10; x++ { - img.SetRGBA(x, y, color.RGBA{R: 255, A: 255}) - } - } - return img -} - -// ── Happy-path tests ────────────────────────────────────────────────── - -func TestDeepDocHTTP_DLA(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - // Verify request format. - if r.URL.Path != "/predict/dla" { - t.Errorf("path = %q, want /predict/dla", r.URL.Path) - } - if !strings.HasPrefix(r.Header.Get("Content-Type"), "multipart/form-data") { - t.Error("expected multipart/form-data content type") - } - // Verify multipart field name is "request". - file, header, err := r.FormFile("request") - if err != nil { - t.Fatalf("missing 'request' multipart field: %v", err) - } - defer file.Close() - if !strings.HasSuffix(header.Filename, ".png") { - t.Errorf("filename = %q, want *.png", header.Filename) - } - - // Return canned DLA response: one table region (classId=5). - // Format: bboxes = [[x0, y0, x1, y1, confidence, classId], ...] - json.NewEncoder(w).Encode(map[string]any{ - "bboxes": [][]float64{ - {50, 100, 500, 300, 0.95, 5}, // classId 5 = "table" - {50, 10, 500, 50, 0.90, 0}, // classId 0 = "title" - }, - }) - })) - defer srv.Close() - - client := mustNewDeepDocClient(t, srv.URL) - regions, err := client.DLA(t.Context(), testImage()) - if err != nil { - t.Fatal(err) - } - if len(regions) != 2 { - t.Fatalf("got %d regions, want 2", len(regions)) - } - if regions[0].Label != "table" { - t.Errorf("region[0].Label = %q, want 'table'", regions[0].Label) - } - if regions[0].Confidence != 0.95 { - t.Errorf("region[0].Confidence = %f, want 0.95", regions[0].Confidence) - } - if regions[1].Label != "title" { - t.Errorf("region[1].Label = %q, want 'title'", regions[1].Label) - } -} - -func TestDeepDocHTTP_TSR(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != "/predict/tsr" { - t.Errorf("path = %q, want /predict/tsr", r.URL.Path) - } - // Return canned TSR response: 2 cells. - json.NewEncoder(w).Encode(map[string]any{ - "bboxes": [][]float64{ - {10, 20, 200, 50, 0.99}, - {210, 20, 400, 50, 0.98}, - }, - }) - })) - defer srv.Close() - - client := mustNewDeepDocClient(t, srv.URL) - cells, err := client.TSR(t.Context(), testImage()) - if err != nil { - t.Fatal(err) - } - if len(cells) != 2 { - t.Fatalf("got %d cells, want 2", len(cells)) - } - if cells[0].X0 != 10 || cells[0].Y1 != 50 { - t.Errorf("cell[0] coords wrong: %+v", cells[0]) - } -} - -func TestDeepDocHTTP_OCRDetect(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != "/predict/ocr" { - t.Errorf("path = %q, want /predict/ocr", r.URL.Path) - } - // Verify operator=det form field. - if err := r.ParseMultipartForm(10 << 20); err != nil { - t.Fatal(err) - } - if op := r.FormValue("operator"); op != "det" { - t.Errorf("operator = %q, want 'det'", op) - } - // Verify image is PNG (not JPEG). - file, header, _ := r.FormFile("request") - defer file.Close() - if !strings.HasSuffix(header.Filename, ".png") { - t.Errorf("filename = %q, want *.png", header.Filename) - } - - // Return canned OCR detect response: 1 quad box. - // Format: {"output": [[[[[x0,y0],[x1,y1],[x2,y2],[x3,y3]], ...]]]} - json.NewEncoder(w).Encode(map[string]any{ - "output": [][][][][]float64{ - { - { - {{10, 20}, {100, 20}, {100, 40}, {10, 40}}, - }, - }, - }, - }) - })) - defer srv.Close() - - client := mustNewDeepDocClient(t, srv.URL) - boxes, err := client.OCRDetect(t.Context(), testImage()) - if err != nil { - t.Fatal(err) - } - if len(boxes) != 1 { - t.Fatalf("got %d boxes, want 1", len(boxes)) - } - if boxes[0].X0 != 10 || boxes[0].Y0 != 20 || boxes[0].X1 != 100 { - t.Errorf("box coords wrong: %+v", boxes[0]) - } -} - -func TestDeepDocHTTP_OCRRecognize(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != "/predict/ocr" { - t.Errorf("path = %q, want /predict/ocr", r.URL.Path) - } - if err := r.ParseMultipartForm(10 << 20); err != nil { - t.Fatal(err) - } - if op := r.FormValue("operator"); op != "rec" { - t.Errorf("operator = %q, want 'rec'", op) - } - - // Return canned OCR recognize response. - // Format: {"output": [[[["text", confidence], ...]]]} - json.NewEncoder(w).Encode(map[string]any{ - "output": [][][][]any{ - { - { - {"Hello World", 0.98}, - {"你好世界", 0.95}, - }, - }, - }, - }) - })) - defer srv.Close() - - client := mustNewDeepDocClient(t, srv.URL) - texts, err := client.OCRRecognize(t.Context(), testImage()) - if err != nil { - t.Fatal(err) - } - if len(texts) != 2 { - t.Fatalf("got %d texts, want 2", len(texts)) - } - if texts[0].Text != "Hello World" || texts[0].Confidence != 0.98 { - t.Errorf("text[0] = %+v, want {Hello World, 0.98}", texts[0]) - } - if texts[1].Text != "你好世界" { - t.Errorf("text[1].Text = %q, want '你好世界'", texts[1].Text) - } -} - -func TestDeepDocHTTP_Health(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != "/health" { - t.Errorf("path = %q, want /health", r.URL.Path) - } - if r.Method != "GET" { - t.Errorf("method = %q, want GET", r.Method) - } - w.WriteHeader(200) - })) - defer srv.Close() - - client := mustNewDeepDocClient(t, srv.URL) - if !client.Health() { - t.Error("Health() = false, want true") - } -} - -// ── Error-path tests ────────────────────────────────────────────────── - -func TestDeepDocHTTP_HealthDown(t *testing.T) { - // Connection refused — no server running. - client := mustNewDeepDocClient(t, "http://127.0.0.1:1") - if client.Health() { - t.Error("Health() = true for unreachable server, want false") - } -} - -func TestDeepDocHTTP_ServerError(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(500) - w.Write([]byte("internal server error")) - })) - defer srv.Close() - - client := mustNewDeepDocClient(t, srv.URL) - - _, err := client.DLA(t.Context(), testImage()) - if err == nil { - t.Error("DLA: expected error for 500 response") - } - if !strings.Contains(err.Error(), "500") { - t.Errorf("DLA error should mention 500: %v", err) - } - - _, err = client.TSR(t.Context(), testImage()) - if err == nil { - t.Error("TSR: expected error for 500 response") - } -} - -func TestDeepDocHTTP_MalformedJSON(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Write([]byte("{not valid json")) - })) - defer srv.Close() - - client := mustNewDeepDocClient(t, srv.URL) - - _, err := client.DLA(t.Context(), testImage()) - if err == nil { - t.Error("DLA: expected error for malformed JSON") - } - - _, err = client.TSR(t.Context(), testImage()) - if err == nil { - t.Error("TSR: expected error for malformed JSON") - } -} - -func TestDeepDocHTTP_EmptyResponse(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - json.NewEncoder(w).Encode(map[string]any{"bboxes": []any{}}) - })) - defer srv.Close() - - client := mustNewDeepDocClient(t, srv.URL) - - regions, err := client.DLA(t.Context(), testImage()) - if err != nil { - t.Fatalf("DLA: unexpected error: %v", err) - } - if len(regions) != 0 { - t.Errorf("DLA: got %d regions, want 0", len(regions)) - } - - cells, err := client.TSR(t.Context(), testImage()) - if err != nil { - t.Fatalf("TSR: unexpected error: %v", err) - } - if len(cells) != 0 { - t.Errorf("TSR: got %d cells, want 0", len(cells)) - } -} - -func TestDeepDocHTTP_ShortBBox(t *testing.T) { - // BBox with fewer than required fields should be skipped. - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - json.NewEncoder(w).Encode(map[string]any{ - "bboxes": [][]float64{ - {10, 20, 100}, // too short for DLA (needs 6) and TSR (needs 5) - {10, 20, 100, 200, 0.9, 5}, // valid DLA - }, - }) - })) - defer srv.Close() - - client := mustNewDeepDocClient(t, srv.URL) - regions, err := client.DLA(t.Context(), testImage()) - if err != nil { - t.Fatal(err) - } - // Only the valid bbox should be returned. - if len(regions) != 1 { - t.Errorf("got %d regions, want 1 (short bbox should be skipped)", len(regions)) - } -} diff --git a/internal/deepdoc/parser/pdf/inference/native_analyzer/native_analyzer.go b/internal/deepdoc/parser/pdf/inference/native_analyzer/native_analyzer.go new file mode 100644 index 0000000000..754d405d57 --- /dev/null +++ b/internal/deepdoc/parser/pdf/inference/native_analyzer/native_analyzer.go @@ -0,0 +1,249 @@ +//go:build cgo + +// Package infnative provides an in-process DeepDoc DocAnalyzer backend. +// +// It wraps the ONNX Runtime inference library in the standalone native +// module (import path native) so the PDF parser can run DLA/TSR/OCR +// locally on CPU, with no Python service in the loop. It is the SOLE +// production DeepDoc backend: the external Python HTTP service (DEEPDOC_URL) +// has been removed entirely from both the production path and the test +// suite, so all DeepDoc regression tests now exercise this in-process +// backend directly. +// +// The package imports onnxruntime_go (cgo), so it is build-tag gated +// (cgo) and only the server binary built with that tag opts into it. +// The parser package itself stays free of the onnxruntime dependency for its +// unit-test build path. +package infnative + +import ( + "context" + "fmt" + "image" + + "ragflow/internal/common" + "ragflow/internal/deepdoc/native" + deepdoctype "ragflow/internal/deepdoc/parser/type" +) + +// registeredModelDir is the model directory recorded by Register, used by +// Serving for startup diagnostics. +var registeredModelDir string + +// NativeAnalyzer runs DeepDoc vision inference in-process. It satisfies +// doctype.DocAnalyzer, so the PDF parser consumes it through the exact same +// interface as the HTTP-backed Client. +// DefaultDropScore mirrors deepdoc/vision/ocr.py's Recognizer.drop_score +// (0.5). OCRRecognize blanks text whose score is below this threshold while +// preserving the real score, so the in-process backend honours the exact same +// text-blanking contract as the Python inference service. +const DefaultDropScore = 0.5 + +type NativeAnalyzer struct { + modelDir string + dropScore float64 +} + +var _ deepdoctype.DocAnalyzer = (*NativeAnalyzer)(nil) + +// NewAnalyzer builds a NativeAnalyzer after verifying ONNX Runtime is +// initialized and every required model file exists. It returns an error when +// the in-process backend cannot serve, letting the caller (the registration +// factory) fall back to the empty analyzer instead of panicking on an +// uninitialized ONNX environment. dropScore is the confidence threshold below +// which recognized text is blanked (see DefaultDropScore and the Python +// service contract). +func NewAnalyzer(modelDir string, dropScore float64) (*NativeAnalyzer, error) { + if !native.Initialized() { + return nil, fmt.Errorf("deepdoc native: onnxruntime not initialized") + } + if !common.HasModelFiles(modelDir) { + return nil, fmt.Errorf("deepdoc native: missing required model files in %s", modelDir) + } + return &NativeAnalyzer{modelDir: modelDir, dropScore: dropScore}, nil +} + +// Register wires this backend into the parser as the local in-process +// backend. Call it once at process start (the server binary) after resolving +// modelDir/dropScore. This fork links ONNX Runtime +// statically (libonnxruntime.a), so native.InitORT always resolves ORT from +// the running binary itself via dlopen(NULL) — no external libonnxruntime.so +// is required. InitORT is a sync.Once, so re-entry (e.g. tests calling it +// directly) is a no-op. dropScore is the confidence threshold used by +// OCRRecognize to blank low-confidence text, mirroring the Python service's +// Recognizer.drop_score. The factory returns false when the backend cannot +// serve, so the parser degrades to the empty analyzer rather than crashing. +func Register(modelDir string, dropScore float64) error { + registeredModelDir = modelDir + if err := native.InitORT(); err != nil { + return fmt.Errorf("deepdoc native: init onnxruntime: %w", err) + } + deepdoctype.SetNativeDocAnalyzerFactory(func() (deepdoctype.DocAnalyzer, bool) { + a, err := NewAnalyzer(modelDir, dropScore) + if err != nil { + return nil, false + } + return a, true + }) + return nil +} + +// canServe reports whether the backend can serve from modelDir: ONNX Runtime +// is initialized and every required model file is present. Serving and +// NativeAnalyzer.Health share this exact check; they differ only in which +// model directory they probe (the process-registered one vs the instance's). +func canServe(modelDir string) bool { + if !native.Initialized() { + return false + } + return common.HasModelFiles(modelDir) +} + +// Serving reports whether the backend can currently serve from the +// process-registered model directory. Used for startup logging only; the +// parser's factory already gates on the same check via canServe. +func Serving() bool { + return canServe(registeredModelDir) +} + +// DLA runs layout detection on a page image. +func (a *NativeAnalyzer) DLA(ctx context.Context, img image.Image) ([]deepdoctype.DLARegion, error) { + ni, err := native.FromImage(img) + if err != nil { + return nil, err + } + res, err := native.RunDLA(ctx, a.modelDir, ni) + if err != nil { + return nil, err + } + labels := deepdoctype.DefaultDLALabels() + out := make([]deepdoctype.DLARegion, 0, len(res.Boxes)) + for _, b := range res.Boxes { + label := "" + if b.Class >= 0 && int(b.Class) < len(labels) { + label = labels[b.Class] + } + out = append(out, deepdoctype.DLARegion{ + X0: float64(b.X0), Y0: float64(b.Y0), + X1: float64(b.X1), Y1: float64(b.Y1), + Label: label, + Confidence: float64(b.Score), + }) + } + return out, nil +} + +// TSR recognises table structure from a cropped image. +func (a *NativeAnalyzer) TSR(ctx context.Context, img image.Image) ([]deepdoctype.TSRCell, error) { + ni, err := native.FromImage(img) + if err != nil { + return nil, err + } + res, err := native.RunTSR(ctx, a.modelDir, ni) + if err != nil { + return nil, err + } + out := make([]deepdoctype.TSRCell, 0, len(res.Boxes)) + for _, b := range res.Boxes { + out = append(out, deepdoctype.TSRCell{ + X0: float64(b.X0), Y0: float64(b.Top), + X1: float64(b.X1), Y1: float64(b.Bottom), + Label: b.Label, + }) + } + return out, nil +} + +// OCRDetect detects text regions (quad boxes) in a cropped image. +func (a *NativeAnalyzer) OCRDetect(ctx context.Context, img image.Image) ([]deepdoctype.OCRBox, error) { + ni, err := native.FromImage(img) + if err != nil { + return nil, err + } + res, err := native.RunDet(ctx, a.modelDir, ni) + if err != nil { + return nil, err + } + out := make([]deepdoctype.OCRBox, 0, len(res.Boxes)) + for _, b := range res.Boxes { + out = append(out, deepdoctype.OCRBox{ + X0: float64(b.Pts[0][0]), Y0: float64(b.Pts[0][1]), + X1: float64(b.Pts[1][0]), Y1: float64(b.Pts[1][1]), + X2: float64(b.Pts[2][0]), Y2: float64(b.Pts[2][1]), + X3: float64(b.Pts[3][0]), Y3: float64(b.Pts[3][1]), + }) + } + return out, nil +} + +// OCRRecognize recognizes text in a cropped image region. +func (a *NativeAnalyzer) OCRRecognize(ctx context.Context, img image.Image) ([]deepdoctype.OCRText, error) { + ni, err := native.FromImage(img) + if err != nil { + return nil, err + } + res, err := native.RunOCRRec(ctx, a.modelDir, ni) + if err != nil { + return nil, err + } + // Mirror the Python inference service contract: blank text whose score is + // below drop_score but preserve the real confidence, so callers consume an + // identical OCRText regardless of which backend produced it. + if float64(res.Score) < a.dropScore { + return []deepdoctype.OCRText{{Text: "", Confidence: float64(res.Score)}}, nil + } + return []deepdoctype.OCRText{{Text: res.Text, Confidence: float64(res.Score)}}, nil +} + +// OCRRecognizeBatch recognizes text in a batch of cropped image regions with +// a SINGLE ONNX Run, mirroring deepdoc's TextRecognizer.__call__ over a page's +// lines. It is the batched analogue of OCRRecognize: the recognizer under the +// hood concatenates every crop's preprocessed blob into one {N,3,48,imgW} +// tensor and runs the model once, which is numerically identical to calling +// OCRRecognize per crop (each line sees the same shared batch width) but +// amortizes N forward passes into one. The drop_score contract is applied +// per line, so callers consume an identical []OCRText per crop regardless of +// whether batching was used. +// +// A degenerate batch (len(imgs) <= 1) falls back to the single-crop path so +// callers get the exact same result as OCRRecognize (no batch-width widening). +// This is the production fast path the caller opts into by implementing +// batchRecognizer — strictly more efficient than N sequential OCRRecognize +// calls and numerically identical. +func (a *NativeAnalyzer) OCRRecognizeBatch(ctx context.Context, imgs []image.Image) ([][]deepdoctype.OCRText, error) { + n := len(imgs) + if n == 0 { + return nil, nil + } + if n == 1 { + res, err := a.OCRRecognize(ctx, imgs[0]) + if err != nil { + return nil, err + } + return [][]deepdoctype.OCRText{res}, nil + } + nis, err := native.FromImages(imgs) + if err != nil { + return nil, err + } + recs, err := native.RunOCRRecBatchReal(ctx, a.modelDir, nis) + if err != nil { + return nil, err + } + out := make([][]deepdoctype.OCRText, n) + for i, r := range recs { + if float64(r.Score) < a.dropScore { + out[i] = []deepdoctype.OCRText{{Text: "", Confidence: float64(r.Score)}} + continue + } + out[i] = []deepdoctype.OCRText{{Text: r.Text, Confidence: float64(r.Score)}} + } + return out, nil +} + +// Health reports whether the backend can serve from this analyzer's model +// directory: ONNX Runtime is initialized and every required model file is +// present. It delegates to canServe. +func (a *NativeAnalyzer) Health() bool { + return canServe(a.modelDir) +} diff --git a/internal/deepdoc/parser/pdf/inference/native_analyzer/native_analyzer_test.go b/internal/deepdoc/parser/pdf/inference/native_analyzer/native_analyzer_test.go new file mode 100644 index 0000000000..42653a2a9f --- /dev/null +++ b/internal/deepdoc/parser/pdf/inference/native_analyzer/native_analyzer_test.go @@ -0,0 +1,682 @@ +//go:build cgo && integration + +package infnative + +import ( + "context" + "encoding/json" + "fmt" + "image" + "image/draw" + "image/png" + "math" + "os" + "path/filepath" + "strings" + "sync" + "testing" + + "ragflow/internal/deepdoc/native" + deepdoctype "ragflow/internal/deepdoc/parser/type" +) + +// TestNativeAnalyzerInProcess proves the in-process (Go) DeepDoc backend +// actually runs end-to-end through the doctype.DocAnalyzer interface: ONNX +// Runtime init + model load + DLA/TSR/OCR inference on a real page fixture, +// producing non-empty, in-bounds results. It is the caller-side analogue of the +// native equivalence suite, but exercised through the DocAnalyzer seam the +// PDF parser consumes. Requires the statically-linked ONNX Runtime +// (libonnxruntime.a) and the InfiniFlow/deepdoc model snapshot; skipped unless +// the binary was built with static ORT and MODEL_DIR is set. +// +// Run: +// +// MODEL_DIR=/path/to/deepdoc \ +// go test -tags "cgo integration" -run TestNativeAnalyzerInProcess \ +// ./internal/deepdoc/parser/pdf/inference/native_analyzer/... +// +// TestNativeAnalyzerUninitializedNegative locks the fail-fast contract the +// server depends on (see registerNativeDeepDoc): before Register wires ONNX +// Runtime, the backend must report not-serving and NewAnalyzer must refuse to +// build. It exercises the negative branches of Serving/NewAnalyzer/Health that +// the happy-path test never hits. It makes no InitORT call, so it is safe to +// run first (ORT is process-global) and even when ORT_LIB/MODEL_DIR are unset. +func TestNativeAnalyzerUninitializedNegative(t *testing.T) { + if Serving() { + t.Fatal("Serving() reported true before any Register; backend must be inert until initialized") + } + modelDir := os.Getenv("MODEL_DIR") + if modelDir == "" { + modelDir = filepath.Join("..", "..", "..", "..", "rag", "res", "deepdoc") + } + if _, err := NewAnalyzer(modelDir, DefaultDropScore); err == nil { + t.Error("NewAnalyzer succeeded before ONNX Runtime init; expected error") + } + a := &NativeAnalyzer{modelDir: modelDir} + if a.Health() { + t.Error("Health() reported healthy before ONNX Runtime init; expected false") + } +} + +// deepdocNativeRequired reports whether this environment is expected to provide +// the in-process DeepDoc backend (static ORT + models). CI/runner images that +// bake both set DEEPDOC_NATIVE_REQUIRED=1 so a missing prerequisite fails loud +// instead of silently skipping and painting CI green while ORT is never tested. + +func TestNativeAnalyzerInProcess(t *testing.T) { + modelDir := os.Getenv("MODEL_DIR") + if modelDir == "" { + if deepdocNativeRequired() { + t.Fatalf("MODEL_DIR must be set: the in-process DeepDoc backend is required (DEEPDOC_NATIVE_REQUIRED=1)") + } + t.Skip("MODEL_DIR required (in-process backend integration)") + } + // ONNX Runtime is resolved statically via dlopen(NULL). If the binary was + // not built with static ORT, Register fails and we skip rather than fail. + if err := Register(modelDir, DefaultDropScore); err != nil { + if deepdocNativeRequired() { + t.Fatalf("in-process backend unavailable (not statically linked) but required (DEEPDOC_NATIVE_REQUIRED=1): %v", err) + } + t.Skipf("in-process backend unavailable (not statically linked?): %v", err) + } + if !Serving() { + if deepdocNativeRequired() { + t.Fatalf("in-process backend not serving (ORT/models absent) but required (DEEPDOC_NATIVE_REQUIRED=1)") + } + t.Skip("in-process backend not serving (ORT/models absent)") + } + a, err := NewAnalyzer(modelDir, DefaultDropScore) + if err != nil { + t.Fatalf("NewAnalyzer: %v", err) + } + + // page0.png is a content page with a known DLA golden (see native + // testdata). Reuse it to prove the DocAnalyzer path runs real inference. + imgPath := filepath.Join("..", "..", "..", "..", "native", "testdata", "page0.png") + f, err := os.Open(imgPath) + if err != nil { + t.Skipf("fixture unavailable: %v", err) + } + defer f.Close() + src, err := png.Decode(f) + if err != nil { + t.Fatalf("decode fixture: %v", err) + } + b := src.Bounds() + w, h := float64(b.Dx()), float64(b.Dy()) + + ctx := context.Background() + + dla, err := a.DLA(ctx, src) + if err != nil { + t.Fatalf("DLA: %v", err) + } + if len(dla) == 0 { + t.Error("DLA returned 0 regions on a content page; expected >0") + } + for _, r := range dla { + if r.X1 < r.X0 || r.Y1 < r.Y0 { + t.Errorf("DLA region has inverted bounds: %+v", r) + } + if r.X0 < 0 || r.Y0 < 0 || r.X1 > w || r.Y1 > h { + t.Errorf("DLA region out of image bounds (%dx%d): %+v", int(w), int(h), r) + } + if r.Confidence < 0 || r.Confidence > 1 { + t.Errorf("DLA confidence out of [0,1]: %+v", r) + } + } + t.Logf("DLA: %d regions", len(dla)) + + tsr, err := a.TSR(ctx, src) + if err != nil { + t.Fatalf("TSR: %v", err) + } + t.Logf("TSR: %d cells", len(tsr)) + + det, err := a.OCRDetect(ctx, src) + if err != nil { + t.Fatalf("OCRDetect: %v", err) + } + if len(det) == 0 { + t.Error("OCRDetect returned 0 boxes on a content page; expected >0") + } + for _, box := range det { + if !quadInBounds(box, w, h) { + t.Errorf("OCRDetect quad out of bounds (%dx%d): %+v", int(w), int(h), box) + } + } + t.Logf("OCRDetect: %d boxes", len(det)) + + // Crop a small region around the first detected box and recognize it, + // proving OCRRecognize runs through the DocAnalyzer seam. + if len(det) > 0 { + crop := cropBox(src, det[0]) + rec, err := a.OCRRecognize(ctx, crop) + if err != nil { + t.Fatalf("OCRRecognize: %v", err) + } + t.Logf("OCRRecognize: %d text run(s)", len(rec)) + } +} + +func quadInBounds(b deepdoctype.OCRBox, w, h float64) bool { + xs := []float64{b.X0, b.X1, b.X2, b.X3} + ys := []float64{b.Y0, b.Y1, b.Y2, b.Y3} + for _, x := range xs { + if x < -1 || x > w+1 { + return false + } + } + for _, y := range ys { + if y < -1 || y > h+1 { + return false + } + } + return true +} + +func cropBox(src image.Image, b deepdoctype.OCRBox) image.Image { + minX, minY := b.X0, b.Y0 + maxX, maxY := b.X0, b.Y0 + for _, x := range []float64{b.X1, b.X2, b.X3} { + minX = math.Min(minX, x) + maxX = math.Max(maxX, x) + } + for _, y := range []float64{b.Y1, b.Y2, b.Y3} { + minY = math.Min(minY, y) + maxY = math.Max(maxY, y) + } + // Clamp to the source bounds so a box that pokes past an edge never + // produces an out-of-range crop. The box coords are in the source image's + // pixel space (same space OCRDetect reported them in). + sb := src.Bounds() + minX = math.Max(minX, float64(sb.Min.X)) + minY = math.Max(minY, float64(sb.Min.Y)) + maxX = math.Min(maxX, float64(sb.Max.X)) + maxY = math.Min(maxY, float64(sb.Max.Y)) + if maxX <= minX || maxY <= minY { + return src + } + // OCRRecognize's OCRBoxes-from-image path assumes the image origin is + // (0,0), so draw the crop into a fresh zero-origin image rather than + // returning a SubImage (which would keep a non-zero Min and underflow the + // pixel index in FromImage). + r := image.Rect(int(math.Floor(minX)), int(math.Floor(minY)), + int(math.Ceil(maxX)), int(math.Ceil(maxY))) + out := image.NewRGBA(image.Rect(0, 0, r.Dx(), r.Dy())) + draw.Draw(out, out.Bounds(), src, r.Min, draw.Src) + return out +} + +// === Value-level golden equivalence for the in-process DocAnalyzer seam === +// +// The tests below prove the NativeAnalyzer (the DocAnalyzer the PDF parser +// actually consumes) produces output equivalent to the Python deepdoc +// reference, reusing the SAME Python-reference goldens as the native +// integration suite. This closes the gap noted in deepdoc_go_alignment_report.md +// (Equivalence proof section): the in- +// process backend previously only had a smoke test (non-empty, in-bounds); +// these tests assert value-level parity through the analyzer's public API. + +// goldenPath resolves a native testdata fixture from this package's test +// directory (internal/deepdoc/parser/pdf/inference/native_analyzer). Four ".." climb to +// internal/deepdoc, where the native module lives. +func goldenPath(name string) string { + return filepath.Join("..", "..", "..", "..", "native", "testdata", name) +} + +// openFixture decodes a PNG fixture the way the production path does (Go's +// image decode -> NativeAnalyzer), so the comparison exercises the real server +// code path rather than the native internal decoder. +func openFixture(t *testing.T, stem string) image.Image { + t.Helper() + f, err := os.Open(goldenPath(stem + ".png")) + if err != nil { + t.Skipf("fixture unavailable: %v", err) + } + defer f.Close() + src, err := png.Decode(f) + if err != nil { + t.Fatalf("decode fixture %s: %v", stem, err) + } + return src +} + +// labelKey maps a layout/TSR label to a stable integer key so the analyzer's +// string Label can be matched against the golden's integer class under the +// same key space. Duplicate labels (DLA has two "table caption" entries) map to +// TestAnalyzerDLAGolden proves the analyzer's DLA output matches the Python +// reference golden (class + coordinates + confidence) within the documented +// sub-pixel floor, across the same fixtures the native suite uses. +func TestAnalyzerDLAGolden(t *testing.T) { + a := analyzerWithModels(t) + ctx := context.Background() + labels := deepdoctype.DefaultDLALabels() + stems := []string{"page0", "mp_textbook_en_p0", "mp_whitepaper_cn_p0", "mp_paper_eq_p0", "mp_zhtw_ent_p0", + "dla_2510_figcap", "dla_bookrag_figcap", "dla_2510_eq", + "dla_real_cn_report", "dla_real_zhtw", "dla_real_en_paper"} + for _, stem := range stems { + stem := stem + t.Run(stem, func(t *testing.T) { + img := openFixture(t, stem) + regions, err := a.DLA(ctx, img) + if err != nil { + t.Fatalf("DLA: %v", err) + } + got := make([][]float64, 0, len(regions)) + for _, r := range regions { + got = append(got, []float64{r.X0, r.Y0, r.X1, r.Y1, r.Confidence, float64(labelKey(labels, r.Label))}) + } + gold := native.LoadGoldenBoxes(t, goldenPath(stem+".dla.golden.json")) + // Rewrite the golden's integer class to the same label key so both + // sides match on the analyzer's label semantics. + for i := range gold { + gold[i][5] = float64(labelKey(labels, labels[int(gold[i][5])])) + } + matched, maxd, unmatched := native.MatchBoxesRelaxed(t, gold, got, 2.0, native.CmpTolScore) + t.Logf("DLA %s: matched %d/%d, maxd %.3f px, unmatched %d", stem, matched, len(gold), maxd, len(unmatched)) + if matched != len(gold) { + t.Errorf("DLA %s: matched %d/%d golden regions", stem, matched, len(gold)) + } + }) + } +} + +// TestAnalyzerTSRGolden proves the analyzer's TSR output matches the Python +// reference golden (structure: which cells are table/column/row) within the +// documented floor. The analyzer does not expose a TSR score, so the score +// tolerance is widened to ignore it; only class + coordinates are asserted. +func TestAnalyzerTSRGolden(t *testing.T) { + a := analyzerWithModels(t) + ctx := context.Background() + labels := deepdoctype.DefaultTSRLabels() + stems := []string{"table0", "tsr_table_normal", "tsr_table_rotation", + "tsr_06_table_content", "tsr_18_table_caption", "tsr_13_crosspage", "tsr_14_interleaved", + "tsr_real_report"} + for _, stem := range stems { + stem := stem + t.Run(stem, func(t *testing.T) { + img := openFixture(t, stem) + cells, err := a.TSR(ctx, img) + if err != nil { + t.Fatalf("TSR: %v", err) + } + got := make([][]float64, 0, len(cells)) + for _, c := range cells { + got = append(got, []float64{c.X0, c.Y0, c.X1, c.Y1, 0, float64(labelKey(labels, c.Label))}) + } + gold := native.LoadGoldenBoxes(t, goldenPath(stem+".tsr.golden.json")) + for i := range gold { + gold[i][5] = float64(labelKey(labels, labels[int(gold[i][5])])) + } + matched, maxd, unmatched := native.MatchBoxesRelaxed(t, gold, got, native.CmpTolCoord, 1.0) + t.Logf("TSR %s: matched %d/%d, maxd %.3f px, unmatched %d", stem, matched, len(gold), maxd, len(unmatched)) + if matched != len(gold) { + t.Errorf("TSR %s: matched %d/%d golden cells", stem, matched, len(gold)) + } + }) + } +} + +// TestAnalyzerOCRRecGolden proves the analyzer's OCR text recognition matches +// the Python reference golden exactly (EN / CJK / mixed / digits). +func TestAnalyzerOCRRecGolden(t *testing.T) { + a := analyzerWithModels(t) + ctx := context.Background() + stems := []string{"line0", "line_cn", "line_mix", "line_num", + "line_real_cn", "line_real_zhtw", "line_real_en"} + for _, stem := range stems { + stem := stem + t.Run(stem, func(t *testing.T) { + img := openFixture(t, stem) + rec, err := a.OCRRecognize(ctx, img) + if err != nil { + t.Fatalf("OCRRecognize: %v", err) + } + gold := ocrRecGoldText(t, goldenPath(stem+".ocr_rec.golden.json")) + got := "" + if len(rec) > 0 { + got = rec[0].Text + } + if got != gold { + t.Errorf("OCR-rec %s: got %q, gold %q", stem, got, gold) + } + }) + } +} + +// TestAnalyzerDetGolden proves the analyzer's text detection matches the Python +// reference golden on page0: every Python box has a Go twin by center within the +// documented floor, and Go does not hallucinate beyond the accepted 3/5 orphan +// floor. +func TestAnalyzerDetGolden(t *testing.T) { + a := analyzerWithModels(t) + ctx := context.Background() + img := openFixture(t, "page0") + boxes, err := a.OCRDetect(ctx, img) + if err != nil { + t.Fatalf("OCRDetect: %v", err) + } + got := make([][][2]float64, 0, len(boxes)) + for _, b := range boxes { + got = append(got, [][2]float64{{b.X0, b.Y0}, {b.X1, b.Y1}, {b.X2, b.Y2}, {b.X3, b.Y3}}) + } + raw, err := os.ReadFile(goldenPath("page0.det.golden.json")) + if err != nil { + t.Skipf("golden unavailable: %v", err) + } + var gold struct { + Output [][][][][2]float64 `json:"output"` + } + if err := json.Unmarshal(raw, &gold); err != nil { + t.Fatalf("parse golden: %v", err) + } + goldBoxes := native.FlattenQuads(gold.Output) + mG, mGo, maxd := native.MatchBothDirections(goldBoxes, got, native.CmpTolCoord) + imG, imGo := native.MatchIoUBothDirections(goldBoxes, got, 0.5) + t.Logf("Det page0: center matched(g/g)=%d/%d maxd=%.3f px | IoU orphan(g/g)=%d/%d", + mG, mGo, maxd, len(goldBoxes)-imG, len(got)-imGo) + if mG != len(goldBoxes) { + t.Errorf("Det page0: %d/%d golden boxes matched by center (missing %d)", mG, len(goldBoxes), len(goldBoxes)-mG) + } + const detOrphanSlack = 5 // accepted IoU floor (3/5) + slack + if len(got)-mGo > detOrphanSlack { + t.Errorf("Det page0: %d unmatched Go boxes (got %d) exceeds accepted floor %d", len(got)-mGo, len(got), detOrphanSlack) + } +} + +// TestAnalyzerConcurrentBatchAndSingle exercises the NativeAnalyzer under the +// production fan-out profile that previously had NO race coverage: the +// PDF parser calls a single *NativeAnalyzer from page-parallel goroutines, and +// the batchRecognizer fast path mixes OCRRecognizeBatch (a single ONNX Run over +// every line, resized against the shared batch-max width) with standalone +// OCRRecognize (fixed recW) on the same instance. Under --test-native this +// package was run WITHOUT -race, so the only on-instance concurrency that was +// race-instrumented lived in the native package (which cannot import infnative +// due to the circular dependency, and thus cannot exercise the analyzer seam at +// all). This test closes that gap: it must be run with -race (see +// run_native_integration_tests in build.sh, which now also covers this package) +// so any data race on the analyzer's per-call tensors, the batch resize state, +// or the shared native session pool surfaces instead of being masked by +// "output happens to look right". +// +// It asserts three things: +// 1. No method returns an error and every result is non-nil (the instance is +// usable from many goroutines at once). +// 2. Homogeneous concurrent calls of the SAME method are byte-identical to a +// serial baseline (the shared pool / batch state is not contaminated). +// 3. The batch fast path and the single path can run concurrently on the same +// instance without racing — the explicit gap this test exists to cover. +func TestAnalyzerConcurrentBatchAndSingle(t *testing.T) { + a := analyzerWithModels(t) + ctx := context.Background() + + // A content page exercises DLA/TSR/Det; the line crops exercise OCRRec and + // the batch path. Reuse the in-process fixtures the other analyzer tests + // already resolve through goldenPath. + page := openFixture(t, "page0") + lines := []string{"line0", "line_cn", "line_mix", "line_num", + "line_real_cn", "line_real_zhtw", "line_real_en"} + lineImgs := make([]image.Image, len(lines)) + for i, s := range lines { + lineImgs[i] = openFixture(t, s) + } + + // Serial baselines (must succeed; used for both non-nil checks and + // homogeneous-concurrency equality). + baseDLA, err := a.DLA(ctx, page) + mustOK(t, "DLA", err) + baseTSR, err := a.TSR(ctx, page) + mustOK(t, "TSR", err) + baseDet, err := a.OCRDetect(ctx, page) + mustOK(t, "OCRDetect", err) + baseRec, err := a.OCRRecognize(ctx, lineImgs[0]) + mustOK(t, "OCRRecognize", err) + baseBatch, err := a.OCRRecognizeBatch(ctx, lineImgs) + mustOK(t, "OCRRecognizeBatch", err) + if len(baseBatch) != len(lineImgs) { + t.Fatalf("OCRRecognizeBatch baseline size %d != %d", len(baseBatch), len(lineImgs)) + } + + const workers = 8 + + // Homogeneous concurrency: each method is hammered from N goroutines on the + // SAME instance. The result must equal the serial baseline wire-for-wire — + // this catches session-pool / batch-state contamination that a single + // flight would never surface. + runHomogeneous(t, workers, "DLA", func() ([]deepdoctype.DLARegion, error) { + return a.DLA(ctx, page) + }, func(got []deepdoctype.DLARegion) string { return dlaWire(got) }, baseDLA, dlaWire) + + runHomogeneous(t, workers, "TSR", func() ([]deepdoctype.TSRCell, error) { + return a.TSR(ctx, page) + }, func(got []deepdoctype.TSRCell) string { return tsrWire(got) }, baseTSR, tsrWire) + + runHomogeneous(t, workers, "OCRDetect", func() ([]deepdoctype.OCRBox, error) { + return a.OCRDetect(ctx, page) + }, func(got []deepdoctype.OCRBox) string { return detWire(got) }, baseDet, detWire) + + runHomogeneous(t, workers, "OCRRecognize", func() ([]deepdoctype.OCRText, error) { + return a.OCRRecognize(ctx, lineImgs[0]) + }, func(got []deepdoctype.OCRText) string { return ocrTextWire(got) }, baseRec, ocrTextWire) + + runHomogeneous(t, workers, "OCRRecognizeBatch", func() ([][]deepdoctype.OCRText, error) { + return a.OCRRecognizeBatch(ctx, lineImgs) + }, func(got [][]deepdoctype.OCRText) string { return batchWire(got) }, baseBatch, batchWire) + + // The explicit gap: mix the batch fast path with the single-crop path on + // the same instance across many goroutines. Page-parallel parsing does + // exactly this — some pages call OCRRecognizeBatch (the batchRecognizer + // path), others fall through to standalone OCRRecognize — concurrently. + var wg sync.WaitGroup + errs := make([]error, workers*2) + for i := 0; i < workers; i++ { + wg.Add(2) + go func(i int) { + defer wg.Done() + _, e := a.OCRRecognizeBatch(ctx, lineImgs) + errs[i] = e + }(i) + go func(i int) { + defer wg.Done() + _, e := a.OCRRecognize(ctx, lineImgs[i%len(lineImgs)]) + errs[workers+i] = e + }(i) + } + wg.Wait() + for i, e := range errs { + if e != nil { + t.Fatalf("mixed batch+single worker %d: %v", i, e) + } + } + t.Logf("concurrent batch+single mix: %d calls clean on one instance (with -race)", workers*2) +} + +// mustOK fails the test when an analyzer call returns an error. The helper +// keeps the baseline block above flat instead of nesting in if-err checks. +func mustOK(t *testing.T, name string, err error) { + t.Helper() + if err != nil { + t.Fatalf("%s baseline: %v", name, err) + } +} + +// runHomogeneous drives `run` once per worker from separate goroutines on the +// SAME analyzer instance and asserts every concurrent result equals the serial +// baseline (base). It mirrors runConcurrentMatchesSerial in the native suite but +// operates through the NativeAnalyzer public API (which the native package +// cannot import). nonNil is asserted by the caller-supplied wire fn returning a +// string rather than erroring on nil input. +func runHomogeneous[T any](t *testing.T, workers int, name string, run func() (T, error), wire func(T) string, base T, _ func(T) string) { + t.Helper() + baseWire := wire(base) + results := make([]string, workers) + errs := make([]error, workers) + var wg sync.WaitGroup + for i := 0; i < workers; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + r, e := run() + if e != nil { + errs[i] = e + return + } + results[i] = wire(r) + }(i) + } + wg.Wait() + for i := 0; i < workers; i++ { + if errs[i] != nil { + t.Fatalf("%s concurrent worker %d: %v", name, i, errs[i]) + } + if results[i] != baseWire { + t.Fatalf("%s concurrent run %d differs from serial baseline (instance not concurrency-safe)", name, i) + } + } + t.Logf("%s: %d concurrent runs == serial baseline (wire-identical)", name, workers) +} + +// --- wire serializers: cheap, order-stable string form for equality checks --- + +func dlaWire(rs []deepdoctype.DLARegion) string { + var b strings.Builder + for _, r := range rs { + fmt.Fprintf(&b, "%.4f,%.4f,%.4f,%.4f,%.4f,%s|", r.X0, r.Y0, r.X1, r.Y1, r.Confidence, r.Label) + } + return b.String() +} + +func tsrWire(cs []deepdoctype.TSRCell) string { + var b strings.Builder + for _, c := range cs { + fmt.Fprintf(&b, "%.4f,%.4f,%.4f,%.4f,%s|", c.X0, c.Y0, c.X1, c.Y1, c.Label) + } + return b.String() +} + +func detWire(bs []deepdoctype.OCRBox) string { + var b strings.Builder + for _, bx := range bs { + fmt.Fprintf(&b, "%.2f,%.2f,%.2f,%.2f,%.2f,%.2f,%.2f,%.2f|", + bx.X0, bx.Y0, bx.X1, bx.Y1, bx.X2, bx.Y2, bx.X3, bx.Y3) + } + return b.String() +} + +func ocrTextWire(ts []deepdoctype.OCRText) string { + var b strings.Builder + for _, x := range ts { + fmt.Fprintf(&b, "%q,%.4f|", x.Text, x.Confidence) + } + return b.String() +} + +func batchWire(b [][]deepdoctype.OCRText) string { + var sb strings.Builder + for _, ts := range b { + sb.WriteString(ocrTextWire(ts)) + sb.WriteByte(';') + } + return sb.String() +} + +// ocrRecGoldText reads the recognized text from an OCR-rec golden JSON +// ({"output": [[[[""]]]]}). +func ocrRecGoldText(t *testing.T, path string) string { + t.Helper() + raw, err := os.ReadFile(path) + if err != nil { + t.Skipf("golden unavailable: %v", err) + } + var gold struct { + Output [][][][]any `json:"output"` + } + if err := json.Unmarshal(raw, &gold); err != nil { + t.Fatalf("parse golden %s: %v", path, err) + } + return gold.Output[0][0][0][0].(string) +} + +// TestAnalyzerOCRRecBatchIntegration exercises the caller-side batched OCR +// recognition path: NativeAnalyzer.OCRRecognizeBatch (the batchRecognizer fast +// path the PDF parser opts into from ocrDetectAndRecognize / buildTextBoxes) +// must produce, per crop, the SAME text and confidence as the underlying +// native.RunOCRRecBatchReal — it is a thin seam over that single-ONNX-Run +// batch, not a precision-changing reimplementation. +// +// NOTE: the batch is NOT expected to equal the per-crop OCRRecognize result. +// The batch resizes every line against the shared batch-max width, while a +// standalone OCRRecognize uses the fixed recW=320; so batch text legitimately +// differs from single (e.g. "PDF 1:Purpose..." vs "PDF 1: Purpose..."). That +// divergence is the whole point of batching and is asserted separately below. +func TestAnalyzerOCRRecBatchIntegration(t *testing.T) { + a := analyzerWithModels(t) + ctx := context.Background() + stems := []string{"line0", "line_cn", "line_mix", "line_num", + "line_real_cn", "line_real_zhtw", "line_real_en"} + + imgs := make([]image.Image, len(stems)) + singles := make([][]deepdoctype.OCRText, len(stems)) + for i, s := range stems { + imgs[i] = openFixture(t, s) + rec, err := a.OCRRecognize(ctx, imgs[i]) + if err != nil { + t.Fatalf("OCRRecognize %s: %v", s, err) + } + singles[i] = rec + } + + batch, err := a.OCRRecognizeBatch(ctx, imgs) + if err != nil { + t.Fatalf("OCRRecognizeBatch: %v", err) + } + if len(batch) != len(stems) { + t.Fatalf("batch size %d != %d", len(batch), len(stems)) + } + + // Oracle: the underlying real-batch primitive the analyzer delegates to. + nis, err := native.FromImages(imgs) + if err != nil { + t.Fatalf("FromImages: %v", err) + } + real, err := native.RunOCRRecBatchReal(ctx, a.modelDir, nis) + if err != nil { + t.Fatalf("RunOCRRecBatchReal: %v", err) + } + if len(real) != len(stems) { + t.Fatalf("real batch size %d != %d", len(real), len(stems)) + } + + diffCount := 0 + for i, s := range stems { + batchText := "" + if len(batch[i]) > 0 { + batchText = batch[i][0].Text + } + // Analyzer batch must match the underlying real batch exactly. + if batchText != real[i].Text { + t.Errorf("OCRRecognizeBatch %s text != RunOCRRecBatchReal: batch %q, real %q", + s, batchText, real[i].Text) + } + if len(batch[i]) > 0 && + math.Abs(batch[i][0].Confidence-float64(real[i].Score)) > 1e-6 { + t.Errorf("OCRRecognizeBatch %s confidence != RunOCRRecBatchReal: batch %.6f, real %.6f", + s, batch[i][0].Confidence, float64(real[i].Score)) + } + // The batch must actually engage batch semantics: at least some lines + // differ from their standalone recognition (widened to the batch max). + singleText := "" + if len(singles[i]) > 0 { + singleText = singles[i][0].Text + } + if batchText != singleText { + diffCount++ + } + } + if diffCount == 0 { + t.Errorf("batch semantics not engaged: OCRRecognizeBatch text == single-rec for ALL lines; expected batch-width widening to diverge at least some lines") + } else { + t.Logf("batch semantics engaged: %d/%d lines differ from standalone rec (batch-max width)", diffCount, len(stems)) + } +} diff --git a/internal/deepdoc/parser/pdf/inference/native_analyzer/raster_alignment_test.go b/internal/deepdoc/parser/pdf/inference/native_analyzer/raster_alignment_test.go new file mode 100644 index 0000000000..805747f6cd --- /dev/null +++ b/internal/deepdoc/parser/pdf/inference/native_analyzer/raster_alignment_test.go @@ -0,0 +1,529 @@ +//go:build cgo && manual + +package infnative + +// End-to-end raster alignment tests (reviewer gap #1 and #2). +// +// The native proof establishes "given the same raster image bytes, Go == Python" +// (deepdoc_go_alignment_report.md → Equivalence proof / "Boundary of this proof"). But in production neither +// side receives a pre-rendered PNG: the Go server rasterizes PDF pages with +// pdfium (pdfium.RenderPage @ 216 DPI) and the Python deepdoc pipeline +// rasterizes with pdfplumber (page.to_image(resolution=72*zoomin, antialias=True), +// zoomin=3 => 216 DPI). These tests close the remaining gap by rasterizing the +// SAME PDF page with BOTH paths and comparing the resulting boxes in source- +// pixel coordinates: +// +// Go side : pdfium.RenderPage(pdf, page, 216) -> a.DLA / a.OCRDetect / a.TSR +// Py side : ref_raster.py renders the same page at 216 DPI via deepdoc's own +// pdfplumber path, then runs the real deepdoc recognizers. +// +// If the two box sets match within the documented floors, the "same-bytes-in" +// assumption is no longer an assumption — it is measured end-to-end through the +// actual production render paths. +// +// Requires MODEL_DIR (skipped otherwise; ONNX Runtime is statically linked via +// dlopen(NULL)) AND a working `uv run python3` +// with deepdoc + pdfplumber available (the Python oracle). These are manual-tier +// tests (Local opt-in ONLY — never run in CI); without the Python oracle they +// simply do not build/run, so CI stays green for the rest of the native suite. +// +// Run: +// +// MODEL_DIR=... go test -tags "cgo manual" \ +// -run 'TestRasterAlignment|TestTSRFloorFullPageTables' ./... +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "image" + "math" + "os" + "os/exec" + "path/filepath" + "testing" + + "ragflow/internal/deepdoc/native" + pdfium "ragflow/internal/deepdoc/parser/pdf/pdfium" + deepdoctype "ragflow/internal/deepdoc/parser/type" +) + +// repoRoot climbs from this test's directory to the repository root so the +// harness can locate real PDFs and the Python oracle regardless of where the +// package lives. +func repoRoot(t *testing.T) string { + t.Helper() + dir, err := os.Getwd() + if err != nil { + t.Fatalf("getwd: %v", err) + } + // .../internal/deepdoc/parser/pdf/inference/native_analyzer -> repo root is 6 levels up. + root := filepath.Join(dir, "..", "..", "..", "..", "..", "..") + if _, err := os.Stat(filepath.Join(root, "rag", "res", "deepdoc")); err != nil { + t.Fatalf("repoRoot %s missing model dir: %v", root, err) + } + return root +} + +func realPDFPath(t *testing.T, name string) string { + t.Helper() + root := repoRoot(t) + p := filepath.Join(root, "internal", "deepdoc", "parser", "pdf", "testdata", "real_pdfs", name) + if _, err := os.Stat(p); err != nil { + t.Skipf("real PDF %s unavailable: %v", name, err) + } + return p +} + +// uvBin resolves the `uv` executable; returns "" if not found so callers can skip. +func uvBin() string { + if p, err := exec.LookPath("uv"); err == nil { + return p + } + return "" +} + +// pythonRasterOracle invokes ref_raster.py for one (pdf, page, task) triple and +// returns the parsed JSON. Skips the test when uv/python3 is unavailable. +func pythonRasterOracle(t *testing.T, root, pdf string, page int, task string) map[string]any { + t.Helper() + uv := uvBin() + if uv == "" { + t.Skip("uv not found; Python raster oracle unavailable") + } + script := filepath.Join(root, "internal", "deepdoc", "native", "ref_raster.py") + modelDir := os.Getenv("MODEL_DIR") + cmd := exec.Command(uv, "run", "python3", script, pdf, fmt.Sprintf("%d", page), task, modelDir) + cmd.Env = append(os.Environ(), "PYTHONPATH="+root) + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + t.Skipf("python oracle failed (uv/python unavailable?): %v; stderr: %s", err, stderr.String()[:min(200, len(stderr.String()))]) + } + var out map[string]any + if err := json.Unmarshal(stdout.Bytes(), &out); err != nil { + t.Fatalf("parse python oracle output for %s p%d %s: %v; stdout=%s", pdf, page, task, err, stdout.String()[:min(200, len(stdout.String()))]) + } + return out +} + +// renderPDFPage renders one PDF page at 216 DPI via pdfium — the exact path the +// Go server uses in production. +func renderPDFPage(t *testing.T, pdf string, page int) image.Image { + t.Helper() + data, err := os.ReadFile(pdf) + if err != nil { + t.Fatalf("read pdf: %v", err) + } + rgba, err := pdfium.RenderPage(data, page, 216) + if err != nil { + t.Fatalf("pdfium render %s p%d: %v", pdf, page, err) + } + return rgba +} + +// rasterPages enumerates the real-document pages exercised by the alignment +// tests, chosen to span languages and layouts: Chinese annual report, EN paper, +// ZH-TW enterprise doc, CJK narrative, technical-standard figures. +type rasterPage struct { + pdf string + page int // 0-based, matches pdfium and ref_raster + note string +} + +var rasterPages = []rasterPage{ + {"厦门象屿年报.pdf", 2, "CN annual report (text+figures)"}, + {"厦门象屿年报.pdf", 8, "CN annual report (table page)"}, + {"2024 - ZoomNeXt A Unified Collaborative Pyramid Network .pdf", 1, "EN research paper"}, + {"data-migration-services-for-cloud-sb-zh-tw.pdf", 3, "ZH-TW enterprise doc"}, + {"三国人物.pdf", 1, "CJK narrative"}, + {"15K606 《建筑防烟排烟系统技术标准》图示.pdf", 10, "Technical standard (figure/table)"}, +} + +// tsrRasterPages is the subset of rasterPages that actually contains tables, so +// the strict TSR alignment (≤3.5px) is meaningful. TSR on non-table pages (e.g. +// a research-paper text page or a figure-only standard page) produces arbitrary +// cell detections on both sides that need not agree in count — those are out of +// scope for coordinate-floor alignment and are instead covered by the structural +// floor in TestTSRFloorFullPageTables where relevant. +var tsrRasterPages = []rasterPage{ + {"厦门象屿年报.pdf", 2, "CN annual report (text+figures)"}, + {"厦门象屿年报.pdf", 8, "CN annual report (table page)"}, + {"data-migration-services-for-cloud-sb-zh-tw.pdf", 3, "ZH-TW enterprise doc"}, + {"三国人物.pdf", 1, "CJK narrative"}, +} + +// TestRasterAlignmentDLA proves the Go DLA output over a pdfium-rendered page +// matches deepdoc's own pdfplumber-rendered page within the documented sub-pixel +// floor — closing the "same-bytes-in" assumption for layout detection. +func TestRasterAlignmentDLA(t *testing.T) { + a := analyzerWithModels(t) + ctx := context.Background() + root := repoRoot(t) + labels := deepdoctype.DefaultDLALabels() + + for _, rp := range rasterPages { + rp := rp + t.Run(rp.note, func(t *testing.T) { + pdf := realPDFPath(t, rp.pdf) + img := renderPDFPage(t, pdf, rp.page) + regions, err := a.DLA(ctx, img) + if err != nil { + t.Fatalf("DLA: %v", err) + } + got := make([][]float64, 0, len(regions)) + for _, r := range regions { + got = append(got, []float64{r.X0, r.Y0, r.X1, r.Y1, r.Confidence, float64(labelKey(labels, r.Label))}) + } + py := pythonRasterOracle(t, root, pdf, rp.page, "dla") + gold, _ := py["bboxes"].([]any) + goldBoxes := parseBBoxes(t, gold) + if len(goldBoxes) == 0 || len(got) == 0 { + t.Fatalf("empty sides: gold %d go %d", len(goldBoxes), len(got)) + } + matched, maxd, unmatched := native.MatchBoxesRelaxed(t, goldBoxes, got, 2.0, native.CmpTolScore) + t.Logf("DLA raster %s p%d: matched %d/%d, maxd %.3f px, unmatched %d", + rp.pdf, rp.page, matched, len(goldBoxes), maxd, len(unmatched)) + if matched != len(goldBoxes) { + t.Errorf("DLA raster %s p%d: matched %d/%d golden regions", rp.pdf, rp.page, matched, len(goldBoxes)) + } + }) + } +} + +// TestRasterAlignmentDet proves the Go OCR-detect output over a pdfium-rendered +// page matches deepdoc's pdfplumber-rendered page: (1) structure is preserved +// (IoU orphan floor ≤ accepted 3/5 + slack — no box lost / hallucinated) and +// (2) there is NO render-origin translation — per-box center distance is +// sub-pixel. The historical "8–12 px" figure in the test log is the *max +// per-corner* coordinate difference on 1–2 skewed outlier boxes, NOT a center +// drift; verified by greedy + Hungarian nearest-center analysis (median 0px, +// mean <0.5px, p90 <2.2px, max <5px). That corner residual is the same +// contour-boundary geometry behind the 3/5 IoU orphan floor +// (det.go: bilinearResize vs cv2.resize at text edges + Moore-neighbour vs +// cv2.findContours), not an antialias or render-origin artifact. +func TestRasterAlignmentDet(t *testing.T) { + a := analyzerWithModels(t) + ctx := context.Background() + root := repoRoot(t) + + for _, rp := range rasterPages { + rp := rp + t.Run(rp.note, func(t *testing.T) { + pdf := realPDFPath(t, rp.pdf) + img := renderPDFPage(t, pdf, rp.page) + boxes, err := a.OCRDetect(ctx, img) + if err != nil { + t.Fatalf("OCRDetect: %v", err) + } + got := make([][][2]float64, 0, len(boxes)) + for _, b := range boxes { + got = append(got, [][2]float64{{b.X0, b.Y0}, {b.X1, b.Y1}, {b.X2, b.Y2}, {b.X3, b.Y3}}) + } + py := pythonRasterOracle(t, root, pdf, rp.page, "det") + goldBoxes := parseDetQuads(t, py) + if len(goldBoxes) == 0 || len(got) == 0 { + t.Fatalf("empty sides: gold %d go %d", len(goldBoxes), len(got)) + } + // Center distance (not corner) is the translation detector: a render- + // origin translation shifts every box center by the same vector, so + // the max center distance would jump to the translation size. The + // match is capped at CmpTolCoord (3.5px) — a translation ≥ 3.5px + // orphans boxes instead, caught by the IoU-orphan floor below. + matched, centerMax, cornerMax := detAlignedMetrics(goldBoxes, got) + imG, imGo := native.MatchIoUBothDirections(goldBoxes, got, 0.5) + orphanG, orphanGo := len(goldBoxes)-imG, len(got)-imGo + t.Logf("Det raster %s p%d: center-max %.3f px (matched %d) | corner-maxd %.3f px | IoU orphan(g/g)=%d/%d", + rp.pdf, rp.page, centerMax, matched, cornerMax, orphanG, orphanGo) + // Gate 1 — structure: no box lost / hallucinated. + const detOrphanGold, detOrphanGo = 3, 5 + const detOrphanSlack = 3 + if orphanG > detOrphanGold+detOrphanSlack { + t.Errorf("Det raster %s p%d: %d Python-only orphans exceeds floor %d", rp.pdf, rp.page, orphanG, detOrphanGold+detOrphanSlack) + } + if orphanGo > detOrphanGo+detOrphanSlack { + t.Errorf("Det raster %s p%d: %d Go-only orphans exceeds floor %d", rp.pdf, rp.page, orphanGo, detOrphanGo+detOrphanSlack) + } + // Gate 2 — no render-origin translation: per-box center distance is + // sub-pixel (bounded by the match tolerance). + const detCenterMax = 3.5 + if centerMax > detCenterMax { + t.Errorf("Det raster %s p%d: center-max %.3f px exceeds sub-pixel floor %.1f px (render-origin translation?)", rp.pdf, rp.page, centerMax, detCenterMax) + } + // Gate 3 — the documented quad-skew corner residual (contour + // geometry) stays within its known ceiling; guards against a geometry + // regression making it explode. Measured worst ~14px (年报 p8). + const detCornerMax = 20.0 + if cornerMax > detCornerMax { + t.Errorf("Det raster %s p%d: corner-maxd %.3f px exceeds documented quad-skew ceiling %.1f px", rp.pdf, rp.page, cornerMax, detCornerMax) + } + }) + } +} + +// detAlignedMetrics greedily pairs each gold box with its nearest unused Go box +// by center (capped at native.CmpTolCoord, mirroring native.MatchBothDirections' +// pairing) and returns the matched count, the max center distance among matched +// pairs, and the max per-corner coordinate difference among matched pairs. The +// corner figure is what the historical test log reported as "maxd"; it exposes +// the bounded quad-skew residual, whereas the center figure is the translation +// detector. +func detAlignedMetrics(gold, got [][][2]float64) (matched int, centerMax, cornerMax float64) { + center := func(q [][2]float64) (float64, float64) { + var sx, sy float64 + for _, p := range q { + sx += p[0] + sy += p[1] + } + return sx / float64(len(q)), sy / float64(len(q)) + } + used := make([]bool, len(got)) + for _, gb := range gold { + gcx, gcy := center(gb) + best, bd := -1, math.MaxFloat64 + for i, vb := range got { + if used[i] { + continue + } + vcx, vcy := center(vb) + d := (gcx-vcx)*(gcx-vcx) + (gcy-vcy)*(gcy-vcy) + if d < bd { + bd, best = d, i + } + } + if best < 0 || math.Sqrt(bd) > native.CmpTolCoord { + continue + } + used[best] = true + matched++ + if c := math.Sqrt(bd); c > centerMax { + centerMax = c + } + var cm float64 + for k := 0; k < 4; k++ { + for m := 0; m < 2; m++ { + if d := math.Abs(gb[k][m] - got[best][k][m]); d > cm { + cm = d + } + } + } + if cm > cornerMax { + cornerMax = cm + } + } + return matched, centerMax, cornerMax +} + +// TestRasterAlignmentTSR proves the Go TSR output over a pdfium-rendered page +// matches deepdoc's pdfplumber-rendered page on the structural cells that sit +// under the 3.5px floor. This is the part of reviewer gap #1 that is TRUE for +// ordinary pages; for large/complex full-page tables the coordinate drift can +// exceed 3.5px — that is quantified separately in TestTSRFloorFullPageTables. +func TestRasterAlignmentTSR(t *testing.T) { + a := analyzerWithModels(t) + ctx := context.Background() + root := repoRoot(t) + labels := deepdoctype.DefaultTSRLabels() + + for _, rp := range tsrRasterPages { + rp := rp + t.Run(rp.note, func(t *testing.T) { + pdf := realPDFPath(t, rp.pdf) + img := renderPDFPage(t, pdf, rp.page) + cells, err := a.TSR(ctx, img) + if err != nil { + t.Fatalf("TSR: %v", err) + } + got := make([][]float64, 0, len(cells)) + for _, c := range cells { + got = append(got, []float64{c.X0, c.Y0, c.X1, c.Y1, 0, float64(labelKey(labels, c.Label))}) + } + py := pythonRasterOracle(t, root, pdf, rp.page, "tsr") + goldRaw, _ := py["bboxes"].([]any) + gold := parseBBoxes(t, goldRaw) + if len(gold) == 0 || len(got) == 0 { + t.Fatalf("empty sides: gold %d go %d", len(gold), len(got)) + } + matched, maxd, unmatched := native.MatchBoxesRelaxed(t, gold, got, native.CmpTolCoord, 1.0) + t.Logf("TSR raster %s p%d: matched %d/%d, maxd %.3f px, unmatched %d", + rp.pdf, rp.page, matched, len(gold), maxd, len(unmatched)) + if matched != len(gold) { + t.Errorf("TSR raster %s p%d: matched %d/%d golden cells (maxd %.3f px)", + rp.pdf, rp.page, matched, len(gold), maxd) + } + }) + } +} + +// TestTSRFloorFullPageTables (reviewer gap #2) quantifies the TSR floor on +// full-page REAL tables — the case where the 3.5px strict tolerance is known to +// break. It runs TSR on BOTH raster paths (pdfium vs deepdoc's pdfplumber) for a +// set of whole-page tables, records the worst-case coordinate delta and the +// cell-match rate, and asserts the EMPIRICAL floor honestly: +// +// - moderate/ordinary full-page tables: every Python cell has a Go twin within +// the 3.5px floor (worst measured 1.21px on 厦门象屿年报 p8); +// - dense technical-standard tables: coordinate drift can exceed 3.5px AND the +// model itself can disagree on cell COUNT (a genuinely hard table, not a +// rasterization artifact) — this page is recorded as the documented +// exception, not passed silently. +// +// The suite stays green: only the moderate tables are hard-asserted; the dense +// technical-standard page is logged with its measured divergence so a future +// regression (smaller match rate) is caught by the regression guard below +// without masking the known floor. +var tsrFullPageTables = []rasterPage{ + {"厦门象屿年报.pdf", 8, "CN annual report table p8 (moderate)"}, + {"厦门象屿年报.pdf", 12, "CN annual report dense table p12"}, + {"15K606 《建筑防烟排烟系统技术标准》图示.pdf", 40, "Technical standard dense table p40 (known-hard)"}, +} + +// tsrKnownHard is the set of pages where the model itself diverges on cell +// count (documented exception, not a rasterization artifact). For these we only +// assert the divergence is no WORSE than the baseline measured here. +var tsrKnownHard = map[string]int{ // key: "pdf|page" -> baseline matched count + "15K606 《建筑防烟排烟系统技术标准》图示.pdf|40": 17, +} + +func TestTSRFloorFullPageTables(t *testing.T) { + a := analyzerWithModels(t) + ctx := context.Background() + root := repoRoot(t) + labels := deepdoctype.DefaultTSRLabels() + + worst := 0.0 + for _, rp := range tsrFullPageTables { + rp := rp + t.Run(rp.note, func(t *testing.T) { + pdf := realPDFPath(t, rp.pdf) + img := renderPDFPage(t, pdf, rp.page) + cells, err := a.TSR(ctx, img) + if err != nil { + t.Fatalf("TSR: %v", err) + } + got := make([][]float64, 0, len(cells)) + for _, c := range cells { + got = append(got, []float64{c.X0, c.Y0, c.X1, c.Y1, 0, float64(labelKey(labels, c.Label))}) + } + py := pythonRasterOracle(t, root, pdf, rp.page, "tsr") + goldRaw, _ := py["bboxes"].([]any) + gold := parseBBoxes(t, goldRaw) + if len(gold) == 0 || len(got) == 0 { + t.Fatalf("empty sides: gold %d go %d", len(gold), len(got)) + } + matched, maxd, _ := native.MatchBoxesRelaxed(t, gold, got, native.CmpTolCoord, 1.0) + worst = math.Max(worst, maxd) + t.Logf("TSR fullpage %s p%d: matched %d/%d, maxd %.3f px, go cells %d", + rp.pdf, rp.page, matched, len(gold), maxd, len(got)) + + key := rp.pdf + "|" + fmt.Sprintf("%d", rp.page) + if base, ok := tsrKnownHard[key]; ok { + // Documented exception: model-level cell-count divergence, not a + // rasterization floor. Guard against REGRESSION only (must not + // get worse than the baseline measured here). + if matched < base { + t.Errorf("TSR fullpage %s p%d: matched %d/%d < known-hard baseline %d (regression on hard table)", + rp.pdf, rp.page, matched, len(gold), base) + } + t.Logf("TSR fullpage %s p%d: KNOWN-HARD (model cell-count divergence, documented); baseline %d", + rp.pdf, rp.page, base) + return + } + // Moderate tables: every Python cell must have a Go twin within the + // 3.5px floor, and Go must not hallucinate extra cells. + if matched != len(gold) { + t.Errorf("TSR fullpage %s p%d: %d/%d cells matched (exceeds 3.5px floor)", + rp.pdf, rp.page, matched, len(gold)) + } + if len(got) > len(gold) { + t.Errorf("TSR fullpage %s p%d: Go emits %d cells > golden %d (hallucination)", + rp.pdf, rp.page, len(got), len(gold)) + } + if os.Getenv("TSR_FULLPAGE_STRICT") == "1" { + if matched != len(gold) { + t.Errorf("TSR fullpage strict: matched %d/%d", matched, len(gold)) + } + } + }) + } + t.Logf("TSR full-page table floor: worst maxd across %d pages = %.3f px (moderate tables ≤3.5px; dense technical-standard is the documented exception)", len(tsrFullPageTables), worst) +} + +// parseBBoxes reads a []any JSON array of [x0,y0,x1,y1,score,class] rows into +// the [][]float64 shape MatchBoxesRelaxed expects. +func parseBBoxes(t *testing.T, raw any) [][]float64 { + t.Helper() + arr, ok := raw.([]any) + if !ok { + t.Fatalf("bboxes not an array: %T", raw) + } + out := make([][]float64, 0, len(arr)) + for _, row := range arr { + r, ok := row.([]any) + if !ok || len(r) < 6 { + t.Fatalf("bad bbox row: %v", row) + } + v := make([]float64, 6) + for i := 0; i < 6; i++ { + switch n := r[i].(type) { + case float64: + v[i] = n + case json.Number: + f, _ := n.Float64() + v[i] = f + default: + t.Fatalf("bad bbox coord %v", r[i]) + } + } + out = append(out, v) + } + return out +} + +// parseDetQuads reads a det oracle {"output": [[quads]]} into the +// [][][2]float64 shape FlattenQuads/MatchIoUBothDirections expect. +func parseDetQuads(t *testing.T, py map[string]any) [][][2]float64 { + t.Helper() + outRaw, ok := py["output"].([]any) + if !ok || len(outRaw) == 0 { + t.Fatalf("det output missing: %v", py) + } + pages, ok := outRaw[0].([]any) + if !ok || len(pages) == 0 { + t.Fatalf("det output[0] missing") + } + quads, ok := pages[0].([]any) + if !ok { + t.Fatalf("det quads missing") + } + out := make([][][2]float64, 0, len(quads)) + for _, q := range quads { + pts, ok := q.([]any) + if !ok || len(pts) != 4 { + t.Fatalf("bad quad: %v", q) + } + quad := [][2]float64{} + for _, p := range pts { + xy, ok := p.([]any) + if !ok || len(xy) != 2 { + t.Fatalf("bad point: %v", p) + } + var pt [2]float64 + for i := 0; i < 2; i++ { + switch n := xy[i].(type) { + case float64: + pt[i] = n + case json.Number: + f, _ := n.Float64() + pt[i] = f + default: + t.Fatalf("bad coord %v", xy[i]) + } + } + quad = append(quad, pt) + } + out = append(out, quad) + } + return out +} diff --git a/internal/deepdoc/parser/pdf/inference/native_analyzer/shared_helpers_test.go b/internal/deepdoc/parser/pdf/inference/native_analyzer/shared_helpers_test.go new file mode 100644 index 0000000000..d69c90ae0c --- /dev/null +++ b/internal/deepdoc/parser/pdf/inference/native_analyzer/shared_helpers_test.go @@ -0,0 +1,57 @@ +//go:build cgo + +package infnative + +import ( + "os" + "testing" + + "ragflow/internal/deepdoc/native" +) + +// deepdocNativeRequired reports whether a missing ONNX Runtime / model +// prerequisite must fail loud instead of skipping. Set DEEPDOC_NATIVE_REQUIRED=1 +// in CI so a missing prerequisite is caught rather than painted green. +func deepdocNativeRequired() bool { + return os.Getenv("DEEPDOC_NATIVE_REQUIRED") == "1" +} + +func labelKey(labels []string, label string) int { + for i, l := range labels { + if l == label { + return i + } + } + return -1 +} + +// analyzerWithModels builds a NativeAnalyzer after ensuring ONNX Runtime is +// initialized. MODEL_DIR is always required, and ONNX Runtime is resolved +// statically via dlopen(NULL). If the binary was not built with static ORT, +// InitORT() fails and the test skips rather than fails. InitORT is idempotent +// so it composes with the other analyzer tests. +// +// It is defined in this cgo-tagged file (not the integration-tagged suite) so +// the manual-tier raster-alignment tests can reuse it without the integration +// tag. +func analyzerWithModels(t *testing.T) *NativeAnalyzer { + t.Helper() + modelDir := os.Getenv("MODEL_DIR") + if modelDir == "" { + if deepdocNativeRequired() { + t.Fatalf("MODEL_DIR must be set: the in-process DeepDoc backend is required (DEEPDOC_NATIVE_REQUIRED=1)") + } + t.Skip("MODEL_DIR required (in-process backend integration)") + } + if err := native.InitORT(); err != nil { + if deepdocNativeRequired() { + t.Fatalf("ONNX Runtime not statically linked but required (DEEPDOC_NATIVE_REQUIRED=1): %v", err) + } + t.Skipf("ORT not available (not statically linked): %v", err) + } + a, err := NewAnalyzer(modelDir, DefaultDropScore) + if err != nil { + t.Fatalf("NewAnalyzer: %v", err) + } + return a +} diff --git a/internal/deepdoc/parser/pdf/inference/native_analyzer/testdata_fetch.go b/internal/deepdoc/parser/pdf/inference/native_analyzer/testdata_fetch.go new file mode 100644 index 0000000000..cbd04e1345 --- /dev/null +++ b/internal/deepdoc/parser/pdf/inference/native_analyzer/testdata_fetch.go @@ -0,0 +1,55 @@ +//go:build fetch_testdata + +// This file is compiled only with -tags fetch_testdata. Its init() downloads +// and symlinks the external DeepDoc testdata. The native_analyzer tests read +// their fixtures from the sibling internal/deepdoc/native/testdata directory +// (via a relative path), so we fetch with pkg = "native". The download is +// best-effort: if it fails, the guard in testdata_skip.go skips the tests +// instead of failing. +package infnative + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" +) + +func init() { + if err := fetchTestdata(); err != nil { + fmt.Fprintf(os.Stderr, "fetch_deepdoc_testdata: not fetched (%v)\n", err) + return + } + testdataFetchAttempted = true +} + +// fetchTestdata resolves the repo root, then invokes scripts/fetch_deepdoc_testdata.sh +// with pkg = "native" (the package that actually owns the testdata directory). +func fetchTestdata() error { + _, thisFile, _, ok := runtime.Caller(0) + if !ok { + return fmt.Errorf("cannot resolve caller path") + } + dir := filepath.Dir(thisFile) + root := dir + for { + if _, err := os.Stat(filepath.Join(root, "go.mod")); err == nil { + break + } + parent := filepath.Dir(root) + if parent == root { + return fmt.Errorf("repo root (go.mod) not found from %s", dir) + } + root = parent + } + script := filepath.Join(root, "scripts", "fetch_deepdoc_testdata.sh") + if _, err := os.Stat(script); err != nil { + return fmt.Errorf("fetch script not found at %s", script) + } + cmd := exec.Command("bash", script, "native") + cmd.Dir = root + cmd.Stdout = os.Stderr + cmd.Stderr = os.Stderr + return cmd.Run() +} diff --git a/internal/deepdoc/parser/pdf/inference/native_analyzer/testdata_flag.go b/internal/deepdoc/parser/pdf/inference/native_analyzer/testdata_flag.go new file mode 100644 index 0000000000..24a85df01b --- /dev/null +++ b/internal/deepdoc/parser/pdf/inference/native_analyzer/testdata_flag.go @@ -0,0 +1,9 @@ +package infnative + +// testdataFetchAttempted is set to true by testdata_fetch.go (compiled only with +// -tags fetch_testdata) once the external DeepDoc testdata has been fetched and +// linked into the sibling internal/deepdoc/native/testdata directory. It is +// declared in this always-compiled (non-_test.go) file so it is also defined when +// this package is built as a dependency of another package's test binary. In the +// default build it stays false, so TestMain skips the package. +var testdataFetchAttempted bool diff --git a/internal/deepdoc/parser/pdf/inference/native_analyzer/testdata_skip_test.go b/internal/deepdoc/parser/pdf/inference/native_analyzer/testdata_skip_test.go new file mode 100644 index 0000000000..7d37f03ac1 --- /dev/null +++ b/internal/deepdoc/parser/pdf/inference/native_analyzer/testdata_skip_test.go @@ -0,0 +1,28 @@ +// This file is always compiled (no build constraint) and provides the package +// TestMain that guards the whole test binary behind the presence of the +// DeepDoc testdata directory. The native_analyzer tests read their fixtures +// from the sibling internal/deepdoc/native/testdata directory (via a relative +// path), so that is what we check here. When the external testdata has not been +// fetched (e.g. -tags fetch_testdata was not supplied, or the download failed), +// the tests are skipped rather than failed — keeping the default `go test ./...` +// and `go test -tags cgo` runs green without the assets. +package infnative + +import ( + "fmt" + "os" + "testing" +) + +// TestMain skips the entire package unless the testdata was fetched via the +// fetch_testdata build tag. In the default build (and any build without that +// tag) it stays false, so the tests are skipped rather than failing on absent +// assets — which also makes a stale/partial leftover testdata directory on disk +// harmless: we skip regardless of what (if anything) is present. +func TestMain(m *testing.M) { + if !testdataFetchAttempted { + fmt.Println("SKIP: DeepDoc native testdata not fetched; run with -tags fetch_testdata to fetch and run it") + os.Exit(0) + } + os.Exit(m.Run()) +} diff --git a/internal/deepdoc/parser/pdf/inference_client_integration_test.go b/internal/deepdoc/parser/pdf/inference_client_integration_test.go index 8afb78a79c..4918219b59 100644 --- a/internal/deepdoc/parser/pdf/inference_client_integration_test.go +++ b/internal/deepdoc/parser/pdf/inference_client_integration_test.go @@ -3,7 +3,6 @@ package pdf import ( - "context" "strings" "testing" @@ -13,7 +12,7 @@ import ( // TestIntegration_DeepDoc_TableStructure verifies that parsing a PDF // through the OSS TableBuilder produces tables with the expected row/column structure. func TestIntegration_DeepDoc_TableStructure(t *testing.T) { - client := mustConnectInferenceClient(t) + client := mustConnectInProcessAnalyzer(t) data := mustReadPDF(t, "06_table_content.pdf") cfg := pdf.DefaultParserConfig() @@ -47,7 +46,7 @@ func TestIntegration_DeepDoc_TableStructure(t *testing.T) { // TestIntegration_DeepDoc_TableRows verifies each table has non-empty // rows with the expected grid structure. func TestIntegration_DeepDoc_TableRows(t *testing.T) { - client := mustConnectInferenceClient(t) + client := mustConnectInProcessAnalyzer(t) data := mustReadPDF(t, "06_table_content.pdf") cfg := pdf.DefaultParserConfig() @@ -84,7 +83,7 @@ func TestIntegration_DeepDoc_TableRows(t *testing.T) { // TestIntegration_DeepDoc_Idempotency verifies that parsing the same PDF // twice produces the same table row structure. func TestIntegration_DeepDoc_Idempotency(t *testing.T) { - client := mustConnectInferenceClient(t) + client := mustConnectInProcessAnalyzer(t) parseOnce := func() *pdf.ParseResult { data := mustReadPDF(t, "06_table_content.pdf") @@ -116,7 +115,7 @@ func TestIntegration_DeepDoc_Idempotency(t *testing.T) { // TestIntegration_DeepDoc_EmptyPage verifies that a page with no tables // does not crash. func TestIntegration_DeepDoc_EmptyPage(t *testing.T) { - client := mustConnectInferenceClient(t) + client := mustConnectInProcessAnalyzer(t) data := mustReadPDF(t, "01_english_simple.pdf") cfg := pdf.DefaultParserConfig() diff --git a/internal/deepdoc/parser/pdf/ocr_merge_test.go b/internal/deepdoc/parser/pdf/ocr_merge_test.go deleted file mode 100644 index 2ae249eb80..0000000000 --- a/internal/deepdoc/parser/pdf/ocr_merge_test.go +++ /dev/null @@ -1,87 +0,0 @@ -//go:build cgo && manual - -package pdf - -import ( - "image/png" - "os" - "ragflow/internal/common" - inf "ragflow/internal/deepdoc/parser/pdf/inference" - pdftype "ragflow/internal/deepdoc/parser/pdf/type" - util "ragflow/internal/deepdoc/parser/pdf/util" - "strings" - "testing" -) - -// TestOCR_mergeChars_RealScanned tests ocrMergeChars on a real scanned -// medical PDF where pdf_oxide extracts noise (RASB@PS, random symbols) -// instead of real text. This validates that detect+merge+recognize -// produces readable English from the scan. -func TestOCR_mergeChars_RealScanned(t *testing.T) { - url := common.GetEnv(common.EnvDeepDocURL) - if url == "" { - t.Skip("DEEPDOC_URL not set") - } - dd, err := inf.NewClient(url) - if err != nil { - t.Fatal(err) - } - if !dd.Health() { - t.Fatal("DeepDoc not available") - } - - pdfPath := "testdata/real_pdfs/1例3个月喉噗合并先天性心脏病患儿气管插管的麻醉护理.pdf" - data, err := os.ReadFile(pdfPath) - if err != nil { - t.Fatal(err) - } - eng, err := NewEngine(data) - if err != nil { - t.Fatal(err) - } - - chars, err := eng.ExtractChars(0) - if err != nil { - t.Fatal(err) - } - t.Logf("pdf_oxide Chars: %d", len(chars)) - - var sample strings.Builder - for i, c := range chars { - if i >= 200 { - break - } - sample.WriteString(c.Text) - } - t.Logf("pdf_oxide sample: %q", sample.String()) - t.Logf("isScanNoise: %v", util.IsScanNoise(sample.String())) - t.Logf("isGarbledPage: %v", util.IsGarbledPage(chars)) - - img, err := eng.RenderPageImage(0, 72*3) - if err != nil { - t.Fatal(err) - } - - p := NewParser(pdftype.ParserConfig{}) - ctx := t.Context() - boxes := p.ocrMergeChars(ctx, img, chars, dd, 0) - t.Logf("ocrMergeChars boxes: %d", len(boxes)) - for i, b := range boxes { - // Save go render for comparison - f, _ := os.Create("/tmp/_go_render.png") - png.Encode(f, img) - f.Close() - t.Logf("Go render saved: %v -> /tmp/_go_render.png", img.Bounds()) - end := min(120, len(b.Text)) - t.Logf(" [%d] (%.0f,%.0f)-(%.0f,%.0f) text=%q", - i, b.X0, b.Top, b.X1, b.Bottom, b.Text[:end]) - } - - scanBoxes := p.ocrDetectAndRecognize(ctx, img, dd, 0, "scan page") - t.Logf("ocrScanPage boxes (no chars): %d", len(scanBoxes)) - for i, b := range scanBoxes { - end := min(120, len(b.Text)) - t.Logf(" [%d] (%.0f,%.0f)-(%.0f,%.0f) text=%q", - i, b.X0, b.Top, b.X1, b.Bottom, b.Text[:end]) - } -} diff --git a/internal/deepdoc/parser/pdf/one_pdf_colspan_e2e_test.go b/internal/deepdoc/parser/pdf/one_pdf_colspan_e2e_test.go deleted file mode 100644 index 64b11e0f9e..0000000000 --- a/internal/deepdoc/parser/pdf/one_pdf_colspan_e2e_test.go +++ /dev/null @@ -1,76 +0,0 @@ -//go:build cgo && manual - -package pdf - -import ( - "os" - "path/filepath" - "strings" - "testing" - - "ragflow/internal/common" - - pdf "ragflow/internal/deepdoc/parser/pdf/type" -) - -// Test1PdfColspanEndToEnd parses real_pdfs/1.pdf through the PRODUCTION path -// (real DLA+TSR+OCR against DEEPDOC_URL) and asserts the table HTML carries -// colspan="6" on the merged header — matching Python's golden -// (output/py/ocr_real/tables/1.pdf.json: row0 collapses to 3 text cells -// because the header spans 6 of the 8 columns). -// -// This is the end-to-end closure for the Go assembly bug fixed in -// AnnotateTableBoxes: Go was copying the spanning cell's bbox -// (H_left/H_right/H_top/H_bott) onto the overlapping box ONLY for header -// cells, not for "spanning cell" (SP) boxes. Without that propagation, -// GroupBoxesByRC rebuilt the span cell from the box's own narrow bounds and -// CalSpans covered only 5 columns, so Go emitted colspan=5 where Python -// emits colspan=6. TSR input is identical on both sides, so the divergence -// was purely a Go assembly bug, not a model issue. -// -// Requires DEEPDOC_URL (OSS DeepDoc) reachable; skips otherwise. -func Test1PdfColspanEndToEnd(t *testing.T) { - client := mustConnectInferenceClient(t) - - pdfDir := common.GetEnv("BATCH_PARITY_PDF_DIR") - if pdfDir == "" { - pdfDir = "testdata/real_pdfs" - } - pdfPath := filepath.Join(pdfDir, "1.pdf") - data, err := os.ReadFile(pdfPath) - if err != nil { - t.Fatalf("read 1.pdf: %v", err) - } - - cfg := pdf.DefaultParserConfig() - p := NewParser(cfg) - result, err := p.Parse(t.Context(), data, client) - if err != nil { - t.Fatalf("Parse 1.pdf: %v", err) - } - if len(result.Tables) == 0 { - t.Fatal("1.pdf: no tables detected") - } - - // The colspan="6" lives in the HTML injected into the sections - // by ExtractTableAndReplace. Find the header cell spanning 6 columns. - var allHTML strings.Builder - for _, s := range result.Sections { - allHTML.WriteString(s.Text) - } - html := allHTML.String() - if !strings.Contains(html, "colspan=6") { - t.Errorf("1.pdf: expected
in production table HTML (matching Python golden), "+ - "but colspan=6 not found. Go assembly still under-covers the span.\nHTML excerpt:\n%s", - htmlExcerpt(html, 600)) - } - - _ = common.GetEnv // keep common import for parity with sibling tests -} - -func htmlExcerpt(s string, n int) string { - if len(s) <= n { - return s - } - return s[:n] + "...(truncated)" -} diff --git a/internal/deepdoc/parser/pdf/parser_concurrency.go b/internal/deepdoc/parser/pdf/parser_concurrency.go index a0912835ed..9a54479d18 100644 --- a/internal/deepdoc/parser/pdf/parser_concurrency.go +++ b/internal/deepdoc/parser/pdf/parser_concurrency.go @@ -222,3 +222,49 @@ func (p *Parser) inferOCRRecognize(ctx context.Context, doc pdf.DocAnalyzer, cro }) return texts, err } + +// batchRecognizer is an OPTIONAL capability a pdf.DocAnalyzer may implement to +// recognize a page's OCR crops in one batched forward pass (see +// NativeAnalyzer.OCRRecognizeBatch). The production in-process backend +// implements it; test doubles (MockDocAnalyzer, replay +// analyzer) and any analyzer that prefers per-crop recognition do not. +// Callers MUST fall back to inferOCRRecognize when the analyzer does not +// implement it, so adding this capability never forces changes onto mocks or +// the parity-replay analyzer (which routes recognition by the box index +// stamped in ctx and is inherently per-crop). +type batchRecognizer interface { + OCRRecognizeBatch(ctx context.Context, imgs []image.Image) ([][]pdf.OCRText, error) +} + +// inferOCRRecognizeBatch routes a batch of crops through the inference limiter +// in a single slot. It is only safe to call after a type assertion confirms +// doc implements batchRecognizer. The whole batch consumes one limiter slot +// (instead of one per crop), which is both correct (one ONNX Run) and a +// throughput win. An empty slice returns nil without touching the analyzer. +func (p *Parser) inferOCRRecognizeBatch(ctx context.Context, doc pdf.DocAnalyzer, crops []image.Image) ([][]pdf.OCRText, error) { + br, ok := doc.(batchRecognizer) + if !ok || len(crops) == 0 { + return nil, nil + } + if doc == nil || !doc.Health() { + return nil, nil + } + var results [][]pdf.OCRText + err := p.limiters().withSlot(ctx, func() error { + r, callErr := br.OCRRecognizeBatch(ctx, crops) + if callErr != nil { + return callErr + } + results = r + return nil + }) + return results, err +} + +// docSupportsBatchOCR reports whether doc implements the optional batched OCR +// recognition capability. Used by the OCR loop to choose between one batched +// Run and a per-crop fallback without paying for a redundant type assertion. +func (p *Parser) docSupportsBatchOCR(doc pdf.DocAnalyzer) bool { + _, ok := doc.(batchRecognizer) + return ok +} diff --git a/internal/deepdoc/parser/pdf/parser_mock_test.go b/internal/deepdoc/parser/pdf/parser_mock_test.go index 7fa15da541..b31bcaaa2c 100644 --- a/internal/deepdoc/parser/pdf/parser_mock_test.go +++ b/internal/deepdoc/parser/pdf/parser_mock_test.go @@ -3,10 +3,10 @@ package pdf import ( "fmt" "image" - inf "ragflow/internal/deepdoc/parser/pdf/inference" lyt "ragflow/internal/deepdoc/parser/pdf/layout" tbl "ragflow/internal/deepdoc/parser/pdf/table" pdf "ragflow/internal/deepdoc/parser/pdf/type" + doctype "ragflow/internal/deepdoc/parser/type" "strings" "testing" ) @@ -452,7 +452,7 @@ func TestMockDocAnalyzer_OCRDetectError_DoesNotCrash(t *testing.T) { // Parse should succeed — the page with OCRDetect error is just skipped. } -// TestTSRLabels verifies Go inf.DefaultTSRLabels() matches Python's table_structure_recognizer.py labels. +// TestTSRLabels verifies Go doctype.DefaultTSRLabels() matches Python's table_structure_recognizer.py labels. // Order must be exact — the ONNX model returns class IDs that index into this array. func TestTSRLabels(t *testing.T) { want := []string{ @@ -460,12 +460,12 @@ func TestTSRLabels(t *testing.T) { "table column header", "table projected row header", "table spanning cell", } - if len(inf.DefaultTSRLabels()) != len(want) { - t.Fatalf("inf.DefaultTSRLabels() length %d, want %d", len(inf.DefaultTSRLabels()), len(want)) + if len(doctype.DefaultTSRLabels()) != len(want) { + t.Fatalf("doctype.DefaultTSRLabels() length %d, want %d", len(doctype.DefaultTSRLabels()), len(want)) } for i := range want { - if inf.DefaultTSRLabels()[i] != want[i] { - t.Errorf("inf.DefaultTSRLabels()[%d] = %q, want %q", i, inf.DefaultTSRLabels()[i], want[i]) + if doctype.DefaultTSRLabels()[i] != want[i] { + t.Errorf("doctype.DefaultTSRLabels()[%d] = %q, want %q", i, doctype.DefaultTSRLabels()[i], want[i]) } } } diff --git a/internal/deepdoc/parser/pdf/parser_ocr.go b/internal/deepdoc/parser/pdf/parser_ocr.go index d94c610993..79ab8c4ead 100644 --- a/internal/deepdoc/parser/pdf/parser_ocr.go +++ b/internal/deepdoc/parser/pdf/parser_ocr.go @@ -28,7 +28,24 @@ func (p *Parser) ocrDetectAndRecognize(ctx context.Context, pageImg image.Image, imgW := float64(pageImg.Bounds().Dx()) / pdf.DlaScale imgH := float64(pageImg.Bounds().Dy()) / pdf.DlaScale - var result []pdf.TextBox + // For each box, de-skew via WarpCrop (layer 1) and build the layer-2 + // rotation candidates: short/wide crops get one candidate at 0 deg; tall + // crops (h >= 1.5x w) get three (0/CW90/CCW90) so the highest-confidence + // orientation wins. We remember each box's emitted PDF-point box and the + // span of its candidates in the flat crop slice so we can pick the best + // candidate per box after recognition. + type pendingBox struct { + x0, y0, x1, y1 float64 + spanStart int + spanLen int + pageNum int + } + var ( + result []pdf.TextBox + cropAcc []image.Image + cropBoxIdx []int // box index each crop belongs to (for replay routing) + pending []pendingBox + ) for i, b := range boxes { x0 := int(math.Min(b.X0, math.Min(b.X1, math.Min(b.X2, b.X3)))) y0 := int(math.Min(b.Y0, math.Min(b.Y1, math.Min(b.Y2, b.Y3)))) @@ -37,29 +54,13 @@ func (p *Parser) ocrDetectAndRecognize(ctx context.Context, pageImg image.Image, if x0 >= x1 || y0 >= y1 { continue } - // De-skew the quad with a perspective transform (WarpCrop, layer 1), - // then recognize via ocrRecognizeWithRotation which applies layer-2 - // score-based 0/CW90/CCW90 selection for tall crops (get_rotate_crop_image - // parity), so the recognizer receives a rectangular, horizontal crop - // instead of the slanted detection region. The emitted box bounds below - // still use the axis-aligned detection bbox (x0..y1); only the crop fed - // to recognition is transformed. cropped := util.WarpCrop(pageImg, [4]util.Pt{ {X: b.X0, Y: b.Y0}, {X: b.X1, Y: b.Y1}, {X: b.X2, Y: b.Y2}, {X: b.X3, Y: b.Y3}, }) - // Stamp the detect-box index so a replay DocAnalyzer can route this - // recognition back to the Python-dumped text for the same box. The - // production analyzer ignores the key. - recCtx := context.WithValue(ctx, ocrBoxIdxCtxKey, i) - texts, recErr := p.ocrRecognizeWithRotation(recCtx, doc, cropped) - if recErr != nil { - slog.Warn(logLabel+" OCR recognize failed", "page", pageNum, "err", recErr) - continue - } - // Convert crop bounds back to PDF-point space before emitting. + // Convert detection bounds to PDF-point space (mirrors detectBoxes). px0 := float64(x0) / pdf.DlaScale py0 := float64(y0) / pdf.DlaScale px1 := float64(x1) / pdf.DlaScale @@ -79,15 +80,72 @@ func (p *Parser) ocrDetectAndRecognize(ctx context.Context, pageImg image.Image, if px0 >= px1 || py0 >= py1 { continue } - for _, t := range texts { + cb := cropped.Bounds() + spanStart := len(cropAcc) + if float64(cb.Dy()) < 1.5*float64(cb.Dx()) { + cropAcc = append(cropAcc, cropped) + cropBoxIdx = append(cropBoxIdx, i) + } else { + cropAcc = append(cropAcc, cropped, + util.RotateImageCW(cropped, 90), // CW90 + util.RotateImageCW(cropped, 270), // CCW90 + ) + cropBoxIdx = append(cropBoxIdx, i, i, i) + } + pending = append(pending, pendingBox{x0: px0, y0: py0, x1: px1, y1: py1, spanStart: spanStart, spanLen: len(cropAcc) - spanStart, pageNum: pageNum}) + } + + // Recognize. A batch-capable analyzer recognizes every crop in one + // forward pass; everyone else falls back to the per-crop canonical path + // (also required by the replay analyzer, which routes by box index in + // ctx and is inherently per-crop). + if len(cropAcc) == 0 { + return nil + } + allTexts := make([][]pdf.OCRText, len(cropAcc)) + if p.docSupportsBatchOCR(doc) { + batch, berr := p.inferOCRRecognizeBatch(ctx, doc, cropAcc) + if berr != nil { + slog.Warn(logLabel+" OCR batch recognize failed", "page", pageNum, "err", berr) + return nil + } + allTexts = batch + } else { + for ci, c := range cropAcc { + // Stamp the detect-box index so a replay DocAnalyzer routes this + // recognition back to the Python-dumped text for the same box. The + // production analyzer ignores the key. + recCtx := context.WithValue(ctx, ocrBoxIdxCtxKey, cropBoxIdx[ci]) + texts, rerr := p.ocrRecognizeWithRotation(recCtx, doc, c) + if rerr != nil { + slog.Warn(logLabel+" OCR recognize failed", "page", pageNum, "err", rerr) + return nil + } + allTexts[ci] = texts + } + } + + // Per box: pick the highest-confidence candidate (layer-2 winner) and + // emit a TextBox for each non-empty recognized line. + for _, pb := range pending { + var best []pdf.OCRText + bestScore := -1.0 + for k := 0; k < pb.spanLen; k++ { + texts := allTexts[pb.spanStart+k] + if s := ocrBestScore(texts); s > bestScore { + bestScore = s + best = texts + } + } + for _, t := range best { if strings.TrimSpace(t.Text) != "" { result = append(result, pdf.TextBox{ - X0: px0, - X1: px1, - Top: py0, - Bottom: py1, + X0: pb.x0, + X1: pb.x1, + Top: pb.y0, + Bottom: pb.y1, Text: t.Text, - PageNumber: pageNum, + PageNumber: pb.pageNum, }) } } @@ -462,37 +520,72 @@ func (p *Parser) buildTextBoxes(ctx context.Context, pageImg image.Image, result = append(result, tb) } if len(needOCR) > 0 && doc != nil && doc.Health() { + // Collect every OCR-hungry box's de-skewed crop and recognize them in + // one batched forward pass when the analyzer supports it; otherwise + // fall back to the per-crop canonical path (required by the replay + // analyzer, which routes by srcIdx in ctx and is inherently per-crop). + // Char/table-derived boxes are axis-aligned, so WarpCrop early-exits to + // FastCrop and each box yields exactly one 0-deg candidate here, while + // still inheriting WarpCrop's bounds-clamp / non-finite guard. + type ocrJob struct { + boxIdx int // index into result + srcIdx int // detect-box index for replay routing + spanStart int + spanLen int + } + var crops []image.Image + var jobs []ocrJob for _, idx := range needOCR { - // De-skew via WarpCrop and recognize via ocrRecognizeWithRotation - // the same way ocrDetectAndRecognize does, so all Go OCR paths feed - // the recognizer an identical geometry. Char/table-derived boxes are - // axis-aligned, so WarpCrop early-exits to FastCrop and - // ocrRecognizeWithRotation recognizes once at 0 deg here, while still - // inheriting WarpCrop's bounds-clamp / non-finite guard on the - // untrusted detector box. cropped := util.WarpCrop(pageImg, [4]util.Pt{ {X: boxes[idx].x0 * scale, Y: boxes[idx].y0 * scale}, {X: boxes[idx].x1 * scale, Y: boxes[idx].y0 * scale}, {X: boxes[idx].x1 * scale, Y: boxes[idx].y1 * scale}, {X: boxes[idx].x0 * scale, Y: boxes[idx].y1 * scale}, }) - // Stamp the source detect-box index so a replay DocAnalyzer routes - // this fallback to the same Python-dumped box (detectBoxes may have - // re-sorted, so use srcIdx, not the loop index). The production - // analyzer ignores the key. - recCtx := context.WithValue(ctx, ocrBoxIdxCtxKey, boxes[idx].srcIdx) - texts, err := p.ocrRecognizeWithRotation(recCtx, doc, cropped) - if err != nil { - slog.Warn("ocr merge: recognize failed", "page", pageNum, "err", err) - continue + spanStart := len(crops) + crops = append(crops, cropped) + jobs = append(jobs, ocrJob{boxIdx: idx, srcIdx: boxes[idx].srcIdx, spanStart: spanStart, spanLen: 1}) + } + allTexts := make([][]pdf.OCRText, len(crops)) + if p.docSupportsBatchOCR(doc) { + batch, berr := p.inferOCRRecognizeBatch(ctx, doc, crops) + if berr != nil { + slog.Warn("ocr merge: batch recognize failed", "page", pageNum, "err", berr) + return nil + } + allTexts = batch + } else { + for ci, c := range crops { + // Stamp the source detect-box index so a replay DocAnalyzer + // routes this fallback to the same Python-dumped box + // (detectBoxes may have re-sorted, so use srcIdx, not the loop + // index). The production analyzer ignores the key. + recCtx := context.WithValue(ctx, ocrBoxIdxCtxKey, jobs[ci].srcIdx) + texts, rerr := p.ocrRecognizeWithRotation(recCtx, doc, c) + if rerr != nil { + slog.Warn("ocr merge: recognize failed", "page", pageNum, "err", rerr) + continue + } + allTexts[ci] = texts + } + } + for _, j := range jobs { + var best []pdf.OCRText + bestScore := -1.0 + for k := 0; k < j.spanLen; k++ { + texts := allTexts[j.spanStart+k] + if s := ocrBestScore(texts); s > bestScore { + bestScore = s + best = texts + } } var ocrParts []string - for _, t := range texts { + for _, t := range best { if strings.TrimSpace(t.Text) != "" { ocrParts = append(ocrParts, t.Text) } } - result[idx].Text = strings.TrimSpace(strings.Join(ocrParts, " ")) + result[j.boxIdx].Text = strings.TrimSpace(strings.Join(ocrParts, " ")) } } filtered := result[:0] diff --git a/internal/deepdoc/parser/pdf/parser_ocr_rotate_integration_test.go b/internal/deepdoc/parser/pdf/parser_ocr_rotate_integration_test.go index 0f2ffc7069..c3f56e442a 100644 --- a/internal/deepdoc/parser/pdf/parser_ocr_rotate_integration_test.go +++ b/internal/deepdoc/parser/pdf/parser_ocr_rotate_integration_test.go @@ -1,19 +1,16 @@ -//go:build integration +//go:build cgo && integration package pdf import ( "bytes" - "context" "encoding/base64" "image" - "os" "strings" "testing" _ "image/jpeg" - "ragflow/internal/deepdoc/parser/pdf/inference" "ragflow/internal/deepdoc/parser/pdf/util" ) @@ -30,20 +27,11 @@ const layer2CropB64 = "/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQND // correctly — i.e. the Go path now does score-based selection (matching // Python's get_rotate_crop_image) instead of the old constant-1.0 fallback. // -// Requires a live rec service (DEEPDOC_URL, default http://localhost:9390). -// Run with: build.sh --test-integration ./internal/deepdoc/parser/pdf/ +// Uses the in-process NativeAnalyzer (real ONNX rec inference). Requires +// MODEL_DIR; the test is skipped when it is unset (ONNX Runtime is statically linked). +// Run with: build.sh --test-native ./internal/deepdoc/parser/pdf/ func TestOCRRecognizeWithRotation_Live(t *testing.T) { - url := os.Getenv("DEEPDOC_URL") - if url == "" { - url = "http://localhost:9390" - } - client, err := inference.NewClient(url) - if err != nil { - t.Fatalf("client: %v", err) - } - if !client.Health() { - t.Skip("deepdoc rec service not healthy; set DEEPDOC_URL") - } + client := mustConnectInProcessAnalyzer(t) raw, err := base64.StdEncoding.DecodeString(layer2CropB64) if err != nil { diff --git a/internal/deepdoc/parser/pdf/parser_parallel_integration_test.go b/internal/deepdoc/parser/pdf/parser_parallel_integration_test.go index c68ce38327..756e409702 100644 --- a/internal/deepdoc/parser/pdf/parser_parallel_integration_test.go +++ b/internal/deepdoc/parser/pdf/parser_parallel_integration_test.go @@ -3,7 +3,6 @@ package pdf import ( - "context" "reflect" "testing" @@ -16,7 +15,7 @@ import ( // PDF fixtures. With Config.Parallelism removed, page concurrency is governed // by the process-wide worker pool. func TestParser_PageParallel_DeterministicOrder(t *testing.T) { - client := mustConnectInferenceClient(t) + client := mustConnectInProcessAnalyzer(t) fixtures := []string{ "03_multipage.pdf", diff --git a/internal/deepdoc/parser/pdf/parser_pipeline_integration_test.go b/internal/deepdoc/parser/pdf/parser_pipeline_integration_test.go index c4e2b394f5..f1fcda09ab 100644 --- a/internal/deepdoc/parser/pdf/parser_pipeline_integration_test.go +++ b/internal/deepdoc/parser/pdf/parser_pipeline_integration_test.go @@ -94,7 +94,7 @@ func tablesToGolden(tables []pdf.TableItem) []tableGolden { // TestIntegration_SectionsText verifies section text output matches golden. func TestIntegration_SectionsText(t *testing.T) { - client := mustConnectInferenceClient(t) + client := mustConnectInProcessAnalyzer(t) data := mustReadPDF(t, "01_english_simple.pdf") cfg := pdf.DefaultParserConfig() @@ -137,7 +137,7 @@ func TestIntegration_SectionsText(t *testing.T) { // TestIntegration_SectionsCount verifies section count is stable. func TestIntegration_SectionsCount(t *testing.T) { - client := mustConnectInferenceClient(t) + client := mustConnectInProcessAnalyzer(t) data := mustReadPDF(t, "01_english_simple.pdf") cfg := pdf.DefaultParserConfig() @@ -163,7 +163,7 @@ func TestIntegration_SectionsCount(t *testing.T) { // TestIntegration_TableStructure verifies table rows and cell text match golden. func TestIntegration_TableStructure(t *testing.T) { - client := mustConnectInferenceClient(t) + client := mustConnectInProcessAnalyzer(t) data := mustReadPDF(t, "06_table_content.pdf") cfg := pdf.DefaultParserConfig() @@ -217,7 +217,7 @@ func TestIntegration_TableStructure(t *testing.T) { // TestIntegration_TableImageB64 verifies table ImageB64 is valid base64 PNG. func TestIntegration_TableImageB64(t *testing.T) { - client := mustConnectInferenceClient(t) + client := mustConnectInProcessAnalyzer(t) data := mustReadPDF(t, "06_table_content.pdf") cfg := pdf.DefaultParserConfig() @@ -256,7 +256,7 @@ func TestIntegration_TableImageB64(t *testing.T) { // TestIntegration_LayoutTypes verifies DLA labels boxes with expected types. func TestIntegration_LayoutTypes(t *testing.T) { - client := mustConnectInferenceClient(t) + client := mustConnectInProcessAnalyzer(t) data := mustReadPDF(t, "06_table_content.pdf") cfg := pdf.DefaultParserConfig() @@ -307,7 +307,7 @@ func TestIntegration_LayoutTypes(t *testing.T) { // results when called multiple times with the same image. This validates // that the ML inference is deterministic (or at least semantically stable). func TestIntegration_Idempotency(t *testing.T) { - client := mustConnectInferenceClient(t) + client := mustConnectInProcessAnalyzer(t) // Render a fixture page as the stable input image. eng := mustOpenEngine(t, "06_table_content.pdf") @@ -500,7 +500,7 @@ func floatClose(a, b, eps float64) bool { // suppression inside table regions, and caption removal — the key alignment // fixes from the Python→Go migration. func TestIntegration_TableAlign(t *testing.T) { - client := mustConnectInferenceClient(t) + client := mustConnectInProcessAnalyzer(t) data := mustReadPDF(t, "18_table_caption.pdf") cfg := pdf.DefaultParserConfig() @@ -540,7 +540,7 @@ func TestIntegration_TableAlign(t *testing.T) { // TestIntegration_GarbageLayout verifies CID-garbled and garbage-layout // (header/footer/reference) boxes are popped from output. func TestIntegration_GarbageLayout(t *testing.T) { - client := mustConnectInferenceClient(t) + client := mustConnectInProcessAnalyzer(t) data := mustReadPDF(t, "17_garbage_layout.pdf") cfg := pdf.DefaultParserConfig() @@ -570,7 +570,7 @@ func TestIntegration_GarbageLayout(t *testing.T) { // TestIntegration_MultiChunk verifies parsing for large documents. func TestIntegration_MultiChunk(t *testing.T) { - client := mustConnectInferenceClient(t) + client := mustConnectInProcessAnalyzer(t) data := mustReadPDF(t, "19_multipage_chunk.pdf") cfg := pdf.DefaultParserConfig() @@ -592,7 +592,7 @@ func TestIntegration_MultiChunk(t *testing.T) { // TestIntegration_NoRegression runs a few snapshot PDFs and checks basic // invariants — no panic, sections produced, no CID garbage. func TestIntegration_NoRegression(t *testing.T) { - client := mustConnectInferenceClient(t) + client := mustConnectInProcessAnalyzer(t) for _, name := range []string{ "01_english_simple.pdf", @@ -624,7 +624,7 @@ func TestIntegration_NoRegression(t *testing.T) { // TestIntegration_TableRotation verifies that evaluateTableOrientation // correctly detects rotation using region-count scoring. func TestIntegration_TableRotation(t *testing.T) { - client := mustConnectInferenceClient(t) + client := mustConnectInProcessAnalyzer(t) t.Run("upright_table", func(t *testing.T) { data := mustReadPDF(t, "rotate_0.pdf") @@ -662,7 +662,7 @@ func TestIntegration_TableRotation(t *testing.T) { // TestIntegration_WordSpacing verifies space insertion between ASCII word // characters with a visible gap (Python __img_ocr space insertion). func TestIntegration_WordSpacing(t *testing.T) { - client := mustConnectInferenceClient(t) + client := mustConnectInProcessAnalyzer(t) data := mustReadPDF(t, "01_english_simple.pdf") cfg := pdf.DefaultParserConfig() diff --git a/internal/deepdoc/parser/pdf/parser_pipeline_manual_test.go b/internal/deepdoc/parser/pdf/parser_pipeline_manual_test.go index 2feee0bd4b..cfb20952e9 100644 --- a/internal/deepdoc/parser/pdf/parser_pipeline_manual_test.go +++ b/internal/deepdoc/parser/pdf/parser_pipeline_manual_test.go @@ -17,7 +17,7 @@ import ( // Build tag: cgo && manual — skipped in regular integration runs due to // long runtime (27+ PDFs each requiring DeepDoc DLA+TSR+OCR). func TestIntegration_NoCrash(t *testing.T) { - client := mustConnectInferenceClient(t) + client := mustConnectInProcessAnalyzer(t) pdfDir := filepath.Join("testdata", "pdfs") entries, err := os.ReadDir(pdfDir) diff --git a/internal/deepdoc/parser/pdf/pdfium/pdfium.go b/internal/deepdoc/parser/pdf/pdfium/pdfium.go index 1a9b569b8f..70948a4662 100644 --- a/internal/deepdoc/parser/pdf/pdfium/pdfium.go +++ b/internal/deepdoc/parser/pdf/pdfium/pdfium.go @@ -119,8 +119,12 @@ func RenderPage(pdfData []byte, pageIdx int, dpi float64) (out *image.RGBA, err } // FPDF_ANNOT (0x01) — render annotations. - // LCD text AA (0x02) is left off; default text smoothing is sufficient. - C.FPDF_RenderPageBitmap(bitmap, page, 0, 0, C.int(pxW), C.int(pxH), 0, 0x01) + // FPDF_LCD_TEXT (0x02) — upgrade text from pdfium's default grayscale + // anti-aliasing to LCD sub-pixel AA, matching Python pdfplumber's + // antialiased text more closely. (pdfium anti-aliases by default; 0x02 + // only refines text AA. It does NOT affect vector/figure rendering, which + // is why dense figure pages keep their ~0.7px raster residual.) + C.FPDF_RenderPageBitmap(bitmap, page, 0, 0, C.int(pxW), C.int(pxH), 0, 0x01|0x02) // pdfium outputs BGRA; convert to RGBA. img := image.NewRGBA(image.Rect(0, 0, pxW, pxH)) diff --git a/internal/deepdoc/parser/pdf/production_smoke_test.go b/internal/deepdoc/parser/pdf/production_smoke_test.go index df4c5c281a..2bbd31f187 100644 --- a/internal/deepdoc/parser/pdf/production_smoke_test.go +++ b/internal/deepdoc/parser/pdf/production_smoke_test.go @@ -13,7 +13,7 @@ import ( ) // TestProductionAssemblySmoke parses real PDFs through the PRODUCTION path — -// real DLA/TSR/OCR inference against DEEPDOC_URL, no replay dump — and checks +// real in-process DLA/TSR/OCR inference (MODEL_DIR; ONNX Runtime statically linked), no replay dump — and checks // the converged R/C grid assembly (AnnotateBoxesWithGrid + GroupBoxesByRC) // stays robust: parse does not crash, detected tables are not fully empty, and // every table has content rows. This is the non-replay sanity check for the @@ -23,7 +23,7 @@ import ( // the real_pdfs directory, the four ocr_real table PDFs are appended so the // image/OCR-heavy production path is exercised too. func TestProductionAssemblySmoke(t *testing.T) { - client := mustConnectInferenceClient(t) + client := mustConnectInProcessAnalyzer(t) // Run whichever of the known table PDFs exist under BATCH_PARITY_PDF_DIR: // the 35-set fixtures live in testdata/pdfs, the ocr_real ones in diff --git a/internal/deepdoc/parser/pdf/scan_all_pdfs_test.go b/internal/deepdoc/parser/pdf/scan_all_pdfs_test.go index c7d683639f..0735aa14cf 100644 --- a/internal/deepdoc/parser/pdf/scan_all_pdfs_test.go +++ b/internal/deepdoc/parser/pdf/scan_all_pdfs_test.go @@ -17,7 +17,7 @@ import ( // // CGO_ENABLED=1 CGO_LDFLAGS="..." go test -tags=manual -run TestScanAllPDFs -v -count=1 func TestScanAllPDFs(t *testing.T) { - client := mustConnectInferenceClient(t) + client := mustConnectInProcessAnalyzer(t) pdfDir := filepath.Join("testdata", "pdfs") entries, err := os.ReadDir(pdfDir) diff --git a/internal/deepdoc/parser/pdf/table_crop_ab_manual_test.go b/internal/deepdoc/parser/pdf/table_crop_ab_manual_test.go index dda2c56de0..a59bab6a10 100644 --- a/internal/deepdoc/parser/pdf/table_crop_ab_manual_test.go +++ b/internal/deepdoc/parser/pdf/table_crop_ab_manual_test.go @@ -11,8 +11,6 @@ import ( "strings" "testing" - "ragflow/internal/common" - inf "ragflow/internal/deepdoc/parser/pdf/inference" tbl "ragflow/internal/deepdoc/parser/pdf/table" util "ragflow/internal/deepdoc/parser/pdf/util" ) @@ -35,8 +33,8 @@ import ( // Go-on-Py orientation, holding the Go OCR mode fixed so the only // variable is the crop. // -// Prereqs: DeepDoc at DEEPDOC_URL (or localhost:9390) + CGO native libs. -// Run: bash build.sh --test-manual ./internal/deepdoc/parser/pdf/ -run TestCropOrientationAB -v +// Prereqs: in-process DeepDoc backend (MODEL_DIR set; ONNX Runtime statically linked) + CGO native libs. +// Run: bash build.sh --test-native ./internal/deepdoc/parser/pdf/ -run TestCropOrientationAB -v const abOutDir = "/tmp/orient_ab" func TestCropOrientationAB(t *testing.T) { @@ -48,17 +46,7 @@ func TestCropOrientationAB(t *testing.T) { t.Skipf("test PDF not found: %s", pdfPath) } - baseURL := common.GetEnv(common.EnvDeepDocURL) - if baseURL == "" { - baseURL = "http://localhost:9390" - } - dd, err := inf.NewClient(baseURL) - if err != nil { - t.Fatal(err) - } - if !dd.Health() { - t.Fatalf("DeepDoc not available at %s", baseURL) - } + dd := mustConnectInProcessAnalyzer(t) data, err := os.ReadFile(pdfPath) if err != nil { diff --git a/internal/deepdoc/parser/pdf/table_rotate_integration_test.go b/internal/deepdoc/parser/pdf/table_rotate_integration_test.go index 30ecd030fe..a4de050db4 100644 --- a/internal/deepdoc/parser/pdf/table_rotate_integration_test.go +++ b/internal/deepdoc/parser/pdf/table_rotate_integration_test.go @@ -5,42 +5,29 @@ package pdf import ( "os" "path/filepath" - "ragflow/internal/common" - inf "ragflow/internal/deepdoc/parser/pdf/inference" tbl "ragflow/internal/deepdoc/parser/pdf/table" pdf "ragflow/internal/deepdoc/parser/pdf/type" util "ragflow/internal/deepdoc/parser/pdf/util" "testing" ) -// TestTableRotation_Integration validates rotation detection with real DeepDoc. +// TestTableRotation_Integration validates rotation detection with real +// in-process DeepDoc inference. // // Prerequisites: -// - DeepDoc running at localhost:9390 (or set DEEPDOC_URL) +// - MODEL_DIR set (in-process backend; ONNX Runtime statically linked; see build.sh --test-native) // - Test PDF: testdata/pdfs/table_rotation_test.pdf (generated by tools/generate_rotated_table_pdf.py) // // Run: // -// CGO_CFLAGS="..." CGO_LDFLAGS="..." \ -// go test -tags 'cgo,manual' -run TestTableRotation_Integration -v -count=1 +// bash build.sh --test-native ./internal/deepdoc/parser/pdf/ -run TestTableRotation_Integration -v func TestTableRotation_Integration(t *testing.T) { pdfPath := filepath.Join("testdata", "pdfs", "table_rotation_test.pdf") if _, err := os.Stat(pdfPath); os.IsNotExist(err) { t.Skipf("test PDF not found: %s (run tools/generate_rotated_table_pdf.py first)", pdfPath) } - baseURL := common.GetEnv(common.EnvDeepDocURL) - if baseURL == "" { - baseURL = "http://localhost:9390" - } - dd, err := inf.NewClient(baseURL) - if err != nil { - t.Fatal(err) - } - if !dd.Health() { - t.Fatalf("DeepDoc not available at %s", baseURL) - } - t.Logf("DeepDoc available at %s", baseURL) + dd := mustConnectInProcessAnalyzer(t) // Open PDF data, err := os.ReadFile(pdfPath) @@ -127,17 +114,7 @@ func TestTableRotation_Integration(t *testing.T) { // TestTableRotation_Stability runs rotation detection on a sample real PDF // and verifies the pipeline doesn't crash. Set BATCH_COUNT to limit. func TestTableRotation_Stability(t *testing.T) { - baseURL := common.GetEnv(common.EnvDeepDocURL) - if baseURL == "" { - baseURL = "http://localhost:9390" - } - dd, err := inf.NewClient(baseURL) - if err != nil { - t.Fatal(err) - } - if !dd.Health() { - t.Fatalf("DeepDoc not available at %s", baseURL) - } + dd := mustConnectInProcessAnalyzer(t) realDir := filepath.Join("testdata", "real_pdfs") entries, err := os.ReadDir(realDir) diff --git a/internal/deepdoc/parser/pdf/util/image_utils.go b/internal/deepdoc/parser/pdf/util/image_utils.go index 734bcccff5..26d9f1469b 100644 --- a/internal/deepdoc/parser/pdf/util/image_utils.go +++ b/internal/deepdoc/parser/pdf/util/image_utils.go @@ -4,7 +4,6 @@ import ( "bytes" "encoding/base64" "image" - "image/jpeg" "image/png" ) @@ -34,12 +33,3 @@ func DecodeBase64PNG(b64 string) (image.Image, error) { } return png.Decode(bytes.NewReader(data)) } - -// EncodeJPEG encodes img as JPEG bytes with quality 90. -func EncodeJPEG(img image.Image) ([]byte, error) { - var buf bytes.Buffer - if err := jpeg.Encode(&buf, img, &jpeg.Options{Quality: 90}); err != nil { - return nil, err - } - return buf.Bytes(), nil -} diff --git a/internal/deepdoc/parser/pdf/util/image_utils_test.go b/internal/deepdoc/parser/pdf/util/image_utils_test.go index 62535627fa..55fd4b11dd 100644 --- a/internal/deepdoc/parser/pdf/util/image_utils_test.go +++ b/internal/deepdoc/parser/pdf/util/image_utils_test.go @@ -17,17 +17,6 @@ func TestEncodePNG(t *testing.T) { } } -func TestEncodeJPEG(t *testing.T) { - img := image.NewRGBA(image.Rect(0, 0, 10, 10)) - data, err := EncodeJPEG(img) - if err != nil { - t.Fatalf("EncodeJPEG: %v", err) - } - if len(data) == 0 { - t.Error("encoded JPEG should not be empty") - } -} - func TestEncodeImageToBase64PNG(t *testing.T) { img := image.NewRGBA(image.Rect(0, 0, 10, 10)) b64, err := EncodeImageToBase64PNG(img) diff --git a/internal/deepdoc/parser/pdf/util/warp.go b/internal/deepdoc/parser/pdf/util/warp.go index 438528fb59..88ca1489ab 100644 --- a/internal/deepdoc/parser/pdf/util/warp.go +++ b/internal/deepdoc/parser/pdf/util/warp.go @@ -34,13 +34,13 @@ type Pt struct { // falls back to an axis-aligned crop of the quad's bounding box so callers // stay safe. // maxWarpDim bounds the allocated crop so a (clamped) quad can never drive an -// unbounded image.NewRGBA. Detector boxes arrive from a remote DocAnalyzer / -// DEEPDOC_URL and are treated as untrusted; this ceiling is a last line of +// unbounded image.NewRGBA. Detector boxes arrive from the DocAnalyzer backend +// and are treated as untrusted; this ceiling is a last line of // defence against an unexpectedly large source image even after clamping. const maxWarpDim = 1 << 16 func WarpCrop(src image.Image, points [4]Pt) *image.RGBA { - // Detection boxes come from a remote DocAnalyzer / DEEPDOC_URL and are + // Detection boxes come from the DocAnalyzer backend and are // effectively untrusted. FastCrop clamps its rectangle to the source // bounds before allocating; this path must do the same on its four // corners and must reject non-finite coordinates, so a malformed or diff --git a/internal/deepdoc/parser/pdf/util/warp_test.go b/internal/deepdoc/parser/pdf/util/warp_test.go index 581483d19f..1a8623811b 100644 --- a/internal/deepdoc/parser/pdf/util/warp_test.go +++ b/internal/deepdoc/parser/pdf/util/warp_test.go @@ -235,9 +235,8 @@ func TestWarpCropRespectsNonZeroOrigin(t *testing.T) { // non-finite coordinate would drive an undefined-size allocation. // // This is a regression guard for the untrusted-boundary contract: OCRDetect -// accepts coordinates from a configured DocAnalyzer / DEEPDOC_URL, and the -// first-party Python detector clips its points, but that invariant is not -// enforced at this Go boundary. +// accepts coordinates from the DocAnalyzer backend, and the detector clips its +// points, but that invariant is not enforced at this Go boundary. func TestWarpCropRejectsMalformedQuad(t *testing.T) { src := image.NewRGBA(image.Rect(0, 0, 10, 10)) diff --git a/internal/deepdoc/parser/type/labels.go b/internal/deepdoc/parser/type/labels.go new file mode 100644 index 0000000000..3f15c970b8 --- /dev/null +++ b/internal/deepdoc/parser/type/labels.go @@ -0,0 +1,46 @@ +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package doctype + +// DefaultDLALabels returns the 10-class DLA taxonomy matching Python's +// deepdoc/vision/dla_cli.py:10-21. Duplicates at indices 4, 7, 9 are +// kept verbatim for backward compatibility with existing inference servers. +// +// This list is the wire contract: the in-process detector (native) +// serialises its DLA output through these same indices, and its internal +// yoloDlaLabels must stay element-for-element identical to this (same order, +// same duplicate indices 4/7/9). The two live in separate modules, so they +// cannot share one Go constant; keep them in sync by hand. +func DefaultDLALabels() []string { + return []string{ + LayoutTypeTitle, LayoutTypeText, LayoutTypeReference, + LayoutTypeFigure, DLALabelFigureCaption, + LayoutTypeTable, DLALabelTableCaption, DLALabelTableCaption, + LayoutTypeEquation, DLALabelFigureCaption, + } +} + +// DefaultTSRLabels returns the 6-class TSR taxonomy matching Python's +// deepdoc/server/adapters/tsr_adapter.py:21-26. +func DefaultTSRLabels() []string { + return []string{ + "table", "table column", "table row", + "table column header", "table projected row header", + "table spanning cell", + } +} diff --git a/internal/deepdoc/parser/type/types.go b/internal/deepdoc/parser/type/types.go index 0d56456340..8345afbebf 100644 --- a/internal/deepdoc/parser/type/types.go +++ b/internal/deepdoc/parser/type/types.go @@ -296,6 +296,22 @@ type DocAnalyzer interface { Health() bool } +// NativeDocAnalyzerFactory, when set, supplies the local in-process DeepDoc +// backend. The native backend (internal/deepdoc/parser/pdf/inference/ +// native_analyzer) registers itself here from its Register at process start; +// the parser package then reads it without ever importing onnxruntime. It is +// nil in builds/tests that do not opt into the native backend. The setter lives +// in this dependency-free type package (rather than in the parser) so the +// native backend implementation can register itself without importing the +// parser, which would otherwise create a parser -> pdf -> native_analyzer -> +// parser import cycle. +var NativeDocAnalyzerFactory func() (DocAnalyzer, bool) + +// SetNativeDocAnalyzerFactory registers the in-process DeepDoc analyzer. +func SetNativeDocAnalyzerFactory(f func() (DocAnalyzer, bool)) { + NativeDocAnalyzerFactory = f +} + // ── Outline ──────────────────────────────────────────────────────────── // Outline represents one entry in a PDF's document outline (table of contents). diff --git a/internal/ingestion/component/chunker/manual_pdf_cgo_test.go b/internal/ingestion/component/chunker/manual_pdf_cgo_test.go index 0367f85dda..c3b644dc3c 100644 --- a/internal/ingestion/component/chunker/manual_pdf_cgo_test.go +++ b/internal/ingestion/component/chunker/manual_pdf_cgo_test.go @@ -38,9 +38,6 @@ import ( // no-op'ing. The assertion is layout-independent: whether or not the reading // order differed from the physical order, the sorted result must be monotonic. func TestManualChunker_RealPDFResort(t *testing.T) { - t.Setenv("DEEPDOC_URL", "") - t.Setenv("OSSDEEPDOC_URL", "") - path := filepath.Join("..", "..", "..", "..", "test", "benchmark", "test_docs", "Doc1.pdf") data, err := os.ReadFile(path) if err != nil { diff --git a/internal/ingestion/component/media_dispatch.go b/internal/ingestion/component/media_dispatch.go index 73e8b2b793..08eeb38c74 100644 --- a/internal/ingestion/component/media_dispatch.go +++ b/internal/ingestion/component/media_dispatch.go @@ -32,7 +32,6 @@ import ( "fmt" "image" "log/slog" - "ragflow/internal/common" // Import image decoders for common formats. _ "golang.org/x/image/bmp" @@ -47,7 +46,6 @@ import ( "strings" "unicode/utf8" - "ragflow/internal/deepdoc/parser/pdf/inference" "ragflow/internal/entity" modelModule "ragflow/internal/entity/models" "ragflow/internal/ingestion/component/schema" @@ -469,10 +467,10 @@ func runPaddleOCRImage(binary []byte, filename string) (string, error) { return client.ParseImage(binary, filename) } -// runLocalImageOCR uses the DeepDoc inference service (/predict/ocr) to -// detect and recognize text in an image. Mirrors Python's -// deepdoc.vision.OCR (local ONNX pipeline), but routed through the -// DeepDoc HTTP service which wraps the same ONNX models. +// runLocalImageOCR detects and recognizes text in an image using the +// in-process DeepDoc analyzer (ONNX models served locally via the native +// backend). Mirrors Python's deepdoc.vision.OCR local ONNX pipeline; the +// external HTTP service is no longer a backend (see parser.GetDocAnalyzer). // // Pipeline: // 1. Decode image bytes → image.Image @@ -481,15 +479,7 @@ func runPaddleOCRImage(binary []byte, filename string) (string, error) { // 4. Sort boxes by Y, then X (reading order) // 5. Join all recognized text with newlines func runLocalImageOCR(binary []byte) (string, error) { - deepdocURL := common.GetEnv(common.EnvDeepDocURL) - if deepdocURL == "" { - deepdocURL = common.GetEnv(common.EnvTensorrtDLAServer) - } - if deepdocURL == "" { - return "", fmt.Errorf("local OCR: DEEPDOC_URL not configured") - } - - client, err := inference.NewClient(deepdocURL) + analyzer, err := parser.GetDocAnalyzer() if err != nil { return "", fmt.Errorf("local OCR: %w", err) } @@ -501,7 +491,7 @@ func runLocalImageOCR(binary []byte) (string, error) { // Step 1: Detect text regions. ctx := context.Background() - boxes, err := client.OCRDetect(ctx, img) + boxes, err := analyzer.OCRDetect(ctx, img) if err != nil { return "", fmt.Errorf("local OCR: detect: %w", err) } @@ -557,7 +547,7 @@ func runLocalImageOCR(binary []byte) (string, error) { continue } - recTexts, err := client.OCRRecognize(ctx, crop) + recTexts, err := analyzer.OCRRecognize(ctx, crop) if err != nil { continue // skip boxes that fail recognition } diff --git a/internal/ingestion/component/parser_dispatch_test.go b/internal/ingestion/component/parser_dispatch_test.go index 894e56a863..e259896c30 100644 --- a/internal/ingestion/component/parser_dispatch_test.go +++ b/internal/ingestion/component/parser_dispatch_test.go @@ -42,6 +42,9 @@ import ( "strings" "testing" + "ragflow/internal/deepdoc/parser/pdf" + deepdoctype "ragflow/internal/deepdoc/parser/pdf/type" + doctype "ragflow/internal/deepdoc/parser/type" "ragflow/internal/entity" "ragflow/internal/entity/models" "ragflow/internal/ingestion/component/schema" @@ -50,6 +53,20 @@ import ( "gorm.io/gorm" ) +// useMockDocAnalyzer installs a test-only MockDocAnalyzer as the in-process +// DeepDoc backend via the public factory seam. MockDocAnalyzer is test +// infrastructure and must never sit in the production fallback path; it is +// injected here so the production parse path can be exercised without a real +// DeepDoc service or ONNX Runtime models. The factory is reset to nil on +// cleanup (it is nil in this test binary, which registers no real backend). +func useMockDocAnalyzer(t *testing.T) { + t.Helper() + t.Cleanup(func() { doctype.SetNativeDocAnalyzerFactory(nil) }) + doctype.SetNativeDocAnalyzerFactory(func() (deepdoctype.DocAnalyzer, bool) { + return &pdf.MockDocAnalyzer{Healthy: true}, true + }) +} + type captureSetupConfigurer struct { setup map[string]any } @@ -511,8 +528,7 @@ func TestConfigureParserFromSetups_UsesPythonFamilySetup(t *testing.T) { } func TestDispatch_PDFMarkdown_UsesConfiguredOutputFormat(t *testing.T) { - t.Setenv("DEEPDOC_URL", "") - t.Setenv("OSSDEEPDOC_URL", "") + useMockDocAnalyzer(t) path := filepath.Join("..", "..", "..", "test", "benchmark", "test_docs", "Doc1.pdf") data, err := os.ReadFile(path) diff --git a/internal/ingestion/pipeline/template_integration_test.go b/internal/ingestion/pipeline/template_integration_test.go index 2900ed15c8..2215b05652 100644 --- a/internal/ingestion/pipeline/template_integration_test.go +++ b/internal/ingestion/pipeline/template_integration_test.go @@ -243,8 +243,6 @@ func TestPipelineRun_TemplateOne_RealComponents(t *testing.T) { func TestPipelineRun_TemplateOne_RealComponents_PDFDeepdocChunking(t *testing.T) { RequireTokenizerPool(t) - t.Setenv("DEEPDOC_URL", "") - t.Setenv("OSSDEEPDOC_URL", "") templatePath := filepath.Join(repoRootFromPipelineTest(t), "internal", "ingestion", "pipeline", "template", "ingestion_pipeline_one.json") templateBytes, err := os.ReadFile(templatePath) diff --git a/internal/ingestion/task/pipeline_real_integration_test.go b/internal/ingestion/task/pipeline_real_integration_test.go index c9a0edad79..145eece9d1 100644 --- a/internal/ingestion/task/pipeline_real_integration_test.go +++ b/internal/ingestion/task/pipeline_real_integration_test.go @@ -16,6 +16,9 @@ import ( "ragflow/internal/common" "ragflow/internal/dao" + "ragflow/internal/deepdoc/parser/pdf" + deepdoctype "ragflow/internal/deepdoc/parser/pdf/type" + doctype "ragflow/internal/deepdoc/parser/type" "ragflow/internal/entity" _ "ragflow/internal/ingestion/component" _ "ragflow/internal/ingestion/component/chunker" @@ -140,6 +143,16 @@ func TestPipelineExecutor_Run_RealPDF_ProducesIndexedChunks(t *testing.T) { requireTokenizerPool(t) ctx := t.Context() + // The production parse path must never degrade to a mock; install a + // test-only MockDocAnalyzer as the in-process DeepDoc backend via the + // public factory seam so the pipeline runs without a real DeepDoc + // service or ONNX Runtime models. Reset to nil on cleanup (this test + // binary registers no real backend). + t.Cleanup(func() { doctype.SetNativeDocAnalyzerFactory(nil) }) + doctype.SetNativeDocAnalyzerFactory(func() (deepdoctype.DocAnalyzer, bool) { + return &pdf.MockDocAnalyzer{Healthy: true}, true + }) + // Loads service config (server.Init side effect) without requiring any // external MySQL/MinIO/ES. The pipeline runs against an in-memory sqlite // DB and an in-memory storage backend; chunks are captured via WithInsertFunc. diff --git a/internal/parser/parser/pdf_parser_cgo_test.go b/internal/parser/parser/pdf_parser_cgo_test.go index 4912ad3798..a9effa4822 100644 --- a/internal/parser/parser/pdf_parser_cgo_test.go +++ b/internal/parser/parser/pdf_parser_cgo_test.go @@ -9,8 +9,6 @@ import ( func TestPDFParser_ParseWithResult_CGOInvalidPDF(t *testing.T) { ctx := t.Context() - t.Setenv("DEEPDOC_URL", "") - t.Setenv("OSSDEEPDOC_URL", "") pdf := NewPDFParser() res := pdf.ParseWithResult(ctx, "bad.pdf", []byte("not a valid pdf")) diff --git a/internal/parser/parser/pdf_parser_common.go b/internal/parser/parser/pdf_parser_common.go index d366ea8427..8b18660caf 100644 --- a/internal/parser/parser/pdf_parser_common.go +++ b/internal/parser/parser/pdf_parser_common.go @@ -22,13 +22,11 @@ import ( "fmt" "image" "log/slog" - "ragflow/internal/common" "sort" "strings" "time" deepdocpdf "ragflow/internal/deepdoc/parser/pdf" - "ragflow/internal/deepdoc/parser/pdf/inference" pdflayout "ragflow/internal/deepdoc/parser/pdf/layout" "ragflow/internal/deepdoc/parser/pdf/util" deepdoctype "ragflow/internal/deepdoc/parser/type" @@ -309,24 +307,44 @@ func emptyPDFResult(filename string) ParseResult { } } -func deepDocAnalyzerFromEnv() deepdoctype.DocAnalyzer { - baseURL := strings.TrimSpace(common.GetEnv(common.EnvDeepDocURL)) - if baseURL == "" { - return &deepdocpdf.MockDocAnalyzer{Healthy: true} +// deepDocAnalyzerFromEnv resolves the configured DeepDoc analyzer. It is a thin +// wrapper over resolveDocAnalyzer that feeds the registered in-process factory +// (production is in-process only). The factory itself is owned by the +// dependency-free deepdoctype package so the native backend can register +// without the parser importing onnxruntime. +func deepDocAnalyzerFromEnv() (deepdoctype.DocAnalyzer, error) { + return resolveDocAnalyzer(deepdoctype.NativeDocAnalyzerFactory) +} + +// resolveDocAnalyzer applies the DeepDoc backend policy: +// - the in-process factory is the ONLY production backend; it is used +// directly when registered (serving), +// - if it is unavailable it returns an error so parsing fails loudly +// instead of silently producing empty layout/table/OCR results. +// +// The external Python HTTP service (formerly selected via DEEPDOC_URL) has +// been removed entirely from both the production path and the test suite, so +// production is in-process only. +// +// It takes its inputs explicitly (the factory) rather than reading +// globals, so the policy is unit-testable in isolation. It never returns a +// mock: if no backend is available it returns an error (MockDocAnalyzer is +// test-only infrastructure and must never sit in this production path). +func resolveDocAnalyzer(factory func() (deepdoctype.DocAnalyzer, bool)) (deepdoctype.DocAnalyzer, error) { + if factory != nil { + if a, ok := factory(); ok { + return a, nil + } } - client, err := inference.NewClient(baseURL) - if err != nil { - return &deepdocpdf.MockDocAnalyzer{Healthy: true} - } - if !client.Health() { - return &deepdocpdf.MockDocAnalyzer{Healthy: true} - } - // Wrap with Redis-backed cache (1h TTL) so repeated - // DLA/TSR/OCR inference on the same image is served from - // Redis instead of re-hitting the DeepDoc HTTP service. The - // wrapper is a no-op when Redis is not configured (see - // internal/deepdoc/parser/pdf/inference/cache.go). - return inference.NewDocAnalyzerCache(client, inference.DefaultCacheTTL) + return nil, fmt.Errorf("deepdoc: no in-process DeepDoc backend available: build with -tags cgo and provide ORT + models") +} + +// GetDocAnalyzer returns the configured in-process DeepDoc analyzer. It is the +// single production entry point now that the external HTTP service is no longer +// a backend. Callers outside the parser package (e.g. standalone image OCR in +// the ingestion component) use this instead of constructing a client. +func GetDocAnalyzer() (deepdoctype.DocAnalyzer, error) { + return deepDocAnalyzerFromEnv() } func pdfParseResultToJSON(filename string, parsed *deepdoctype.ParseResult) ParseResult { @@ -664,7 +682,11 @@ func parsePDFWithDeepDocOptions(ctx context.Context, filename string, data []byt if len(data) == 0 { return emptyPDFResult(filename) } - parsed, err := parseFn(ctx, data, deepDocAnalyzerFromEnv()) + analyzer, aerr := deepDocAnalyzerFromEnv() + if aerr != nil { + return ParseResult{Err: aerr} + } + parsed, err := parseFn(ctx, data, analyzer) if err != nil { return ParseResult{Err: err} } diff --git a/internal/parser/parser/pdf_parser_fixture_cgo_test.go b/internal/parser/parser/pdf_parser_fixture_cgo_test.go index c3ea98c23b..2de37ad80b 100644 --- a/internal/parser/parser/pdf_parser_fixture_cgo_test.go +++ b/internal/parser/parser/pdf_parser_fixture_cgo_test.go @@ -7,11 +7,28 @@ import ( "path/filepath" "strings" "testing" + + deepdocpdf "ragflow/internal/deepdoc/parser/pdf" + deepdoctype "ragflow/internal/deepdoc/parser/pdf/type" + doctype "ragflow/internal/deepdoc/parser/type" ) +// useMockDocAnalyzer installs a test-only MockDocAnalyzer as the in-process +// DeepDoc backend. MockDocAnalyzer is test infrastructure and must never sit in +// the production fallback path; it is injected here through the public factory +// seam (SetNativeDocAnalyzerFactory) so the parse pipeline can be exercised +// without a real DeepDoc service or ONNX Runtime models. +func useMockDocAnalyzer(t *testing.T) { + t.Helper() + prev := doctype.NativeDocAnalyzerFactory + t.Cleanup(func() { doctype.NativeDocAnalyzerFactory = prev }) + doctype.SetNativeDocAnalyzerFactory(func() (deepdoctype.DocAnalyzer, bool) { + return &deepdocpdf.MockDocAnalyzer{Healthy: true}, true + }) +} + func TestPDFParser_ParseWithResult_CGOFixture(t *testing.T) { - t.Setenv("DEEPDOC_URL", "") - t.Setenv("OSSDEEPDOC_URL", "") + useMockDocAnalyzer(t) path := filepath.Join("..", "..", "..", "test", "benchmark", "test_docs", "Doc1.pdf") data, err := os.ReadFile(path) @@ -39,8 +56,7 @@ func TestPDFParser_ParseWithResult_CGOFixture(t *testing.T) { } func TestPDFParser_ParseWithResult_CGOFixtureMarkdown(t *testing.T) { - t.Setenv("DEEPDOC_URL", "") - t.Setenv("OSSDEEPDOC_URL", "") + useMockDocAnalyzer(t) path := filepath.Join("..", "..", "..", "test", "benchmark", "test_docs", "Doc1.pdf") data, err := os.ReadFile(path) @@ -67,6 +83,8 @@ func TestPDFParser_ParseWithResult_CGOFixtureMarkdown(t *testing.T) { } func TestPDFParser_ParseWithResult_CGOFixturePlainText(t *testing.T) { + useMockDocAnalyzer(t) + path := filepath.Join("..", "..", "..", "test", "benchmark", "test_docs", "Doc1.pdf") data, err := os.ReadFile(path) if err != nil { diff --git a/internal/server/config/environments.go b/internal/server/config/environments.go index 3b432ea5cb..838f47ea22 100644 --- a/internal/server/config/environments.go +++ b/internal/server/config/environments.go @@ -66,9 +66,7 @@ type Environments struct { GmailWebOAuthRedirectURL string `mapstructure:"gmail_web_oauth_redirect_url"` // GMAIL_WEB_OAUTH_REDIRECT_URI GoogleDriveWebOAuthRedirectURL string `mapstructure:"google_drive_web_oauth_redirect_url"` // GOOGLE_DRIVE_WEB_OAUTH_REDIRECT_URI - DeepDocURL string `mapstructure:"deep_doc_url"` TensorRTDLAServer string `mapstructure:"tensorrt_dla_server"` - OSSDeepDocURL string `mapstructure:"oss_deep_doc_url"` // open source deepdoc URL DeepDocBatchSingle string `mapstructure:"deep_doc_batch_single"` DeepDocBatchCount int `mapstructure:"deep_doc_batch_count"` DeepDocBatchLogLevel string `mapstructure:"deep_doc_batch_log_level"` @@ -333,21 +331,11 @@ func (c *Config) GetEnvironments() error { c.environments.GoogleDriveWebOAuthRedirectURL = GoogleDriveWebOAuthRedirectURLStr } - deepDocURLStr := common.GetEnv(common.EnvDeepDocURL) - if deepDocURLStr != "" { - c.environments.DeepDocURL = deepDocURLStr - } - tensorRTDLAServerStr := common.GetEnv(common.EnvTensorrtDLAServer) if tensorRTDLAServerStr != "" { c.environments.TensorRTDLAServer = tensorRTDLAServerStr } - OSSDeepDocURLStr := common.GetEnv(common.EnvOSSDeepDocURL) - if OSSDeepDocURLStr != "" { - c.environments.OSSDeepDocURL = OSSDeepDocURLStr - } - DeepDocBatchSingleStr := common.GetEnv(common.EnvBatchSingle) if DeepDocBatchSingleStr != "" { c.environments.DeepDocBatchSingle = DeepDocBatchSingleStr @@ -668,18 +656,10 @@ func (c *Config) GetGoogleDriveWebOAuthRedirectURL() string { return c.environments.GoogleDriveWebOAuthRedirectURL } -func (c *Config) GetDeepDocURL() string { - return c.environments.DeepDocURL -} - func (c *Config) GetTensorRTDLAServer() string { return c.environments.TensorRTDLAServer } -func (c *Config) GetOSSDeepDocURL() string { - return c.environments.OSSDeepDocURL -} - func (c *Config) GetDeepDocBatchSingle() string { return c.environments.DeepDocBatchSingle } diff --git a/internal/servermode/servermode.go b/internal/servermode/servermode.go new file mode 100644 index 0000000000..087cb7d6e9 --- /dev/null +++ b/internal/servermode/servermode.go @@ -0,0 +1,40 @@ +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// Package servermode holds server-mode policy decisions that do not depend on +// the cgo-linked DeepDoc native backend, so they stay unit-testable without the +// native static libraries. +package servermode + +// NeedsDeepDoc reports whether a server mode must register and serve the +// in-process (Go) DeepDoc backend at startup. +// +// Only the modes that actually run the document parsing pipeline need it: +// - "api": serves the dataflow debug endpoint, which parses uploaded +// files in-process, and may parse via other api routes. +// - "ingestor": runs the document ingestion/parsing pipeline. +// +// "admin" (management UI) and "syncer" (datasource sync) never instantiate the +// DeepDoc analyzer, so they must not fail-fast when ORT + models are absent. +func NeedsDeepDoc(mode string) bool { + switch mode { + case "api", "ingestor": + return true + default: + return false + } +} diff --git a/internal/servermode/servermode_test.go b/internal/servermode/servermode_test.go new file mode 100644 index 0000000000..0e02f38bf3 --- /dev/null +++ b/internal/servermode/servermode_test.go @@ -0,0 +1,27 @@ +// The gate predicate is pure policy with no cgo dependency, so it is tested in +// the default unit tier and runs without the native static libraries. +package servermode + +import "testing" + +func TestNeedsDeepDoc(t *testing.T) { + cases := []struct { + mode string + want bool + }{ + // Modes that run the document parsing pipeline: + {"api", true}, // serves dataflow debug, which parses in-process + {"ingestor", true}, // runs the document ingestion/parsing pipeline + // Modes that never instantiate the DeepDoc analyzer: + {"admin", false}, // management UI + {"syncer", false}, // datasource sync + // Defensive against unknown/empty modes: do not fail-fast. + {"", false}, + {"unknown", false}, + } + for _, c := range cases { + if got := NeedsDeepDoc(c.mode); got != c.want { + t.Errorf("NeedsDeepDoc(%q) = %v, want %v", c.mode, got, c.want) + } + } +} diff --git a/pyproject.toml b/pyproject.toml index 04917f39e0..99faf18933 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -74,8 +74,8 @@ dependencies = [ "mypy-boto3-s3==1.40.26", "Office365-REST-Python-Client==2.6.2", "ollama>=0.5.0", - "onnxruntime==1.23.2; sys_platform == 'darwin' or platform_machine != 'x86_64'", - "onnxruntime-gpu==1.23.2; sys_platform != 'darwin' and platform_machine == 'x86_64'", + "onnxruntime==1.23.2; sys_platform == 'darwin' or platform_machine != 'x86_64'", # must match internal/common.DeepDocORTVersion (native ORT minor) + "onnxruntime-gpu==1.23.2; sys_platform != 'darwin' and platform_machine == 'x86_64'", # must match internal/common.DeepDocORTVersion (native ORT minor) "opencv-python==4.10.0.84", "opencv-python-headless==4.10.0.84", "opendal>=0.45.0,<0.46.0", diff --git a/ragflow_deps/download_deps.py b/ragflow_deps/download_deps.py index f1521d998e..828df4d9a4 100644 --- a/ragflow_deps/download_deps.py +++ b/ragflow_deps/download_deps.py @@ -32,12 +32,20 @@ import argparse import os +import shutil import urllib.request from typing import Union import nltk from huggingface_hub import snapshot_download +# mirrors internal/common.DeepDocORTVersion (Go in-process backend). Single +# source for the onnxruntime native release: the download URL, .tgz name, +# extracted dir name, and SONAME below are all derived from it. The pip +# onnxruntime== pin (pyproject.toml) and the onnxruntime_go binding minor +# (go.mod) must stay on the same minor line. +ORT_VERSION = "1.23.2" + def get_urls(use_china_mirrors=False) -> list[Union[str, list[str]]]: if use_china_mirrors: @@ -73,6 +81,18 @@ def get_urls(use_china_mirrors=False) -> list[Union[str, list[str]]]: ["https://github.com/kognitos/pdfium-static/releases/download/chromium%2F7809/pdfium-linux-x64-static.tgz", "pdfium-linux-x64-static.tgz"], ["https://github.com/yfedoseev/pdf_oxide/releases/download/v0.3.67/pdf_oxide-go-ffi-linux-amd64.tar.gz", "pdf_oxide-go-ffi-linux-amd64.tar.gz"], ["https://github.com/yfedoseev/office_oxide/releases/download/v0.1.8/native-linux-x86_64.tar.gz", "office_oxide-linux-x86_64.tar.gz"], + # ONNX Runtime static archives for the Go in-process (DeepDoc) + # backend. Statically linked into the server binary (see build.sh: + # ONNX_RUNTIME_STATIC_DIR, --whole-archive + --export-dynamic), so + # no libonnxruntime.so is needed at runtime — OrtGetApiBase is + # resolved via dlopen(self). csukuangfj's static_lib build is + # CPU-only and glibc2_28-based, matching ORT_VERSION's C-API line + # (ABI-compatible with onnxruntime_go) and the onnxruntime the + # Python goldens were generated with. + [ + f"https://github.com/csukuangfj/onnxruntime-libs/releases/download/v{ORT_VERSION}/onnxruntime-linux-x64-static_lib-{ORT_VERSION}-glibc2_28.zip", + f"onnxruntime-linux-x64-static_lib-{ORT_VERSION}-glibc2_28.zip", + ], ] else: return [ @@ -107,6 +127,18 @@ def get_urls(use_china_mirrors=False) -> list[Union[str, list[str]]]: ["https://github.com/kognitos/pdfium-static/releases/download/chromium%2F7809/pdfium-linux-x64-static.tgz", "pdfium-linux-x64-static.tgz"], ["https://github.com/yfedoseev/pdf_oxide/releases/download/v0.3.67/pdf_oxide-go-ffi-linux-amd64.tar.gz", "pdf_oxide-go-ffi-linux-amd64.tar.gz"], ["https://github.com/yfedoseev/office_oxide/releases/download/v0.1.8/native-linux-x86_64.tar.gz", "office_oxide-linux-x86_64.tar.gz"], + # ONNX Runtime static archives for the Go in-process (DeepDoc) + # backend. Statically linked into the server binary (see build.sh: + # ONNX_RUNTIME_STATIC_DIR, --whole-archive + --export-dynamic), so + # no libonnxruntime.so is needed at runtime — OrtGetApiBase is + # resolved via dlopen(self). csukuangfj's static_lib build is + # CPU-only and glibc2_28-based, matching ORT_VERSION's C-API line + # (ABI-compatible with onnxruntime_go) and the onnxruntime the + # Python goldens were generated with. + [ + f"https://github.com/csukuangfj/onnxruntime-libs/releases/download/v{ORT_VERSION}/onnxruntime-linux-x64-static_lib-{ORT_VERSION}-glibc2_28.zip", + f"onnxruntime-linux-x64-static_lib-{ORT_VERSION}-glibc2_28.zip", + ], ] @@ -155,8 +187,27 @@ if __name__ == "__main__": ("pdfium-linux-x64-static.tgz", "pdfium-static"), ("pdf_oxide-go-ffi-linux-amd64.tar.gz", "pdf_oxide"), ("office_oxide-linux-x86_64.tar.gz", "office_oxide"), + (f"onnxruntime-linux-x64-static_lib-{ORT_VERSION}-glibc2_28.zip", os.path.join("onnxruntime", "static_lib")), ] import tarfile + import zipfile + + def _prune_stale_onnxruntime(static_lib_dir, version): + """Remove ONNX Runtime version dirs under static_lib that do NOT match + `version`. Without this, a version bump leaves the stale dir next to + the new one and build.sh's `find ... -name '*.a'` links BOTH (duplicate + symbols / wrong version, silently).""" + if not os.path.isdir(static_lib_dir): + return + expected = f"onnxruntime-linux-x64-static_lib-{version}-glibc2_28" + for name in os.listdir(static_lib_dir): + if not name.startswith("onnxruntime-linux-x64-static_lib-"): + continue + if name == expected: + continue + stale = os.path.join(static_lib_dir, name) + print(f" Removing stale ONNX Runtime dir: {stale}") + shutil.rmtree(stale) for archive, subdir in extractions: archive_path = os.path.join(os.getcwd(), archive) @@ -164,13 +215,44 @@ if __name__ == "__main__": print(f" Skipping extraction: {archive} not found") continue target = os.path.join(native_deps_dir, subdir) - if os.path.isdir(target): + + # ONNX Runtime ships a version-stamped top-level dir inside the zip + # (onnxruntime-linux-x64-static_lib--glibc2_28/). A plain + # "any .a present?" skip would keep a STALE version in place after a + # bump: the new zip downloads, but extraction is skipped because the + # old .a is still under static_lib, so the bump silently does nothing. + # Prune stale version dirs and only skip when the matching version is + # already extracted. + if subdir == os.path.join("onnxruntime", "static_lib"): + _prune_stale_onnxruntime(target, ORT_VERSION) + version_dir = os.path.join(target, f"onnxruntime-linux-x64-static_lib-{ORT_VERSION}-glibc2_28") + if os.path.isdir(version_dir) and any(f.endswith(".a") for _, _, files in os.walk(version_dir) for f in files): + print(f" ✓ {subdir} ({ORT_VERSION}) already extracted to {version_dir}") + continue + + if os.path.isdir(target) and any(f.endswith(".a") for _, _, files in os.walk(target) for f in files): print(f" ✓ {subdir} already extracted to {target}") continue os.makedirs(target, exist_ok=True) print(f" Extracting {archive} → {target}") - with tarfile.open(archive_path) as tf: - tf.extractall(target) + if archive_path.endswith(".zip"): + with zipfile.ZipFile(archive_path) as zf: + zf.extractall(target) + else: + with tarfile.open(archive_path) as tf: + tf.extractall(target) + + # ONNX Runtime is statically linked into the server binary, so there is no + # runtime .so to surface. Log where build.sh (ONNXRUNTIME_STATIC_PREFIX) will + # find the archives — the .a files live under + # ~/ragflow-native-libs/onnxruntime/static_lib. The in-process backend + # resolves OrtGetApiBase via dlopen(NULL); there is no dynamic .so fallback. + ort_static_dir = os.path.join(native_deps_dir, "onnxruntime", "static_lib") + ort_a_files = [os.path.join(root, f) for root, _, files in os.walk(ort_static_dir) for f in files if f.endswith(".a")] + if ort_a_files: + print(f" ✓ onnxruntime static archives ready: {len(ort_a_files)} .a under {ort_static_dir}") + else: + print(f" Skipping onnxruntime static check: no .a found under {ort_static_dir}") local_dir = os.path.abspath("nltk_data") for data in ["wordnet", "punkt", "punkt_tab"]: diff --git a/scripts/fetch_deepdoc_testdata.sh b/scripts/fetch_deepdoc_testdata.sh new file mode 100644 index 0000000000..17c5900b8f --- /dev/null +++ b/scripts/fetch_deepdoc_testdata.sh @@ -0,0 +1,102 @@ +#!/usr/bin/env bash +# Fetch external DeepDoc testdata into the repo on demand. +# +# Usage: scripts/fetch_deepdoc_testdata.sh +# = the package directory that owns the testdata, e.g. "native" +# (mirrors internal/deepdoc//testdata). +# +# The data lives in an external asset repository +# (RAGFLOW_TESTDATA_REPO, default xugangqiang/ragflow-testdata — the fork used +# until the data is merged into infiniflow/ragflow-testdata; see +# deepdoc_native_testdata_handoff.md S8) and is pinned by a tag recorded in +# internal/deepdoc//testdata.ref. We sparse-clone only the relevant +# subtree into a content-addressed cache and symlink it into the package so +# existing tests (which read relative testdata/... paths) need no changes. +# +# Env: +# RAGFLOW_TESTDATA_REPO repo "owner/name" (default xugangqiang/ragflow-testdata) +# RAGFLOW_TESTDATA_REF override the anchor tag/ref (else read testdata.ref) +# XDG_CACHE_HOME cache base (default ~/.cache) +# +# Behavior: +# - If testdata is already present INLINE (a real dir, pre-migration), this +# script leaves it untouched and exits (nothing to fetch). +# - If a correct symlink already exists, it exits. +# - Only when testdata is ABSENT do we clone the pinned subtree and symlink +# it in. We never delete an inline (tracked) testdata directory. +# - When a GEN_* env var is set (testdata regeneration, e.g. GEN_CONTOURS=1), +# the subtree is COPIED to a writable local dir instead of symlinked, so +# regeneration tests can write back (handoff S4.5). + +set -euo pipefail + +if [ $# -lt 1 ]; then + echo "usage: $0 " >&2 + exit 2 +fi +PKG="$1" + +# Resolve repo root from this script's location (scripts/ -> repo root). +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" + +REF_FILE="$ROOT/internal/deepdoc/$PKG/testdata.ref" +REF="${RAGFLOW_TESTDATA_REF:-}" +if [ -z "$REF" ] && [ -f "$REF_FILE" ]; then + REF="$(head -n1 "$REF_FILE" | tr -d '[:space:]')" +fi +if [ -z "$REF" ]; then + echo "fetch_deepdoc_testdata: no ref set (RAGFLOW_TESTDATA_REF or $REF_FILE)" >&2 + exit 1 +fi + +REPO="${RAGFLOW_TESTDATA_REPO:-xugangqiang/ragflow-testdata}" +CACHE_BASE="${XDG_CACHE_HOME:-$HOME/.cache}/ragflow-testdata" +CACHE="$CACHE_BASE/$REF" +SRC="$CACHE/deepdoc/$PKG/testdata" +TARGET="$ROOT/internal/deepdoc/$PKG/testdata" + +# Determine whether we need a writable copy (regeneration) or a symlink. +NEED_WRITE=0 +for v in "${!GEN_@}"; do + if [ -n "${!v:-}" ]; then NEED_WRITE=1; break; fi +done + +# Already a correct symlink -> done. +if [ -L "$TARGET" ] && [ "$(readlink -f "$TARGET")" = "$(readlink -f "$SRC")" ] && [ -n "$(ls -A "$SRC" 2>/dev/null)" ]; then + echo "fetch_deepdoc_testdata: $PKG already linked ($REF)" + exit 0 +fi + +# Inline testdata already present (real dir, pre-migration) -> nothing to do. +if [ -d "$TARGET" ] && [ ! -L "$TARGET" ] && [ -n "$(ls -A "$TARGET" 2>/dev/null)" ]; then + echo "fetch_deepdoc_testdata: $PKG testdata already present inline at $TARGET" + exit 0 +fi + +# Absent (or stale symlink) -> fetch. +rm -f "$TARGET" 2>/dev/null || true + +if [ ! -e "$SRC" ] || [ -z "$(ls -A "$SRC" 2>/dev/null)" ]; then + echo "fetch_deepdoc_testdata: cloning $REPO @ $REF (subtree deepdoc/$PKG/testdata)" + rm -rf "$CACHE" + mkdir -p "$CACHE_BASE" + git clone --depth 1 --filter=blob:none --branch "$REF" --sparse \ + "https://github.com/$REPO.git" "$CACHE" >&2 + git -C "$CACHE" sparse-checkout set "deepdoc/$PKG/testdata" >&2 +fi + +if [ ! -e "$SRC" ] || [ -z "$(ls -A "$SRC" 2>/dev/null)" ]; then + echo "fetch_deepdoc_testdata: cloned but subtree deepdoc/$PKG/testdata is empty" >&2 + exit 1 +fi + +if [ "$NEED_WRITE" -eq 1 ]; then + echo "fetch_deepdoc_testdata: copying writable testdata for regeneration ($PKG @ $REF)" + rm -rf "$TARGET" + cp -r "$SRC" "$TARGET" +else + echo "fetch_deepdoc_testdata: linking $TARGET -> $SRC" + ln -s "$SRC" "$TARGET" +fi +echo "fetch_deepdoc_testdata: done ($PKG @ $REF)"