diff --git a/.gitignore b/.gitignore index 77c6acb579..907ec1485e 100644 --- a/.gitignore +++ b/.gitignore @@ -31,7 +31,7 @@ Cargo.lock docker/ragflow-logs/ /flask_session /logs -rag/res/deepdoc +internal/rag/res/deepdoc/ # Exclude sdk generated files sdk/python/ragflow.egg-info/ diff --git a/AGENTS.md b/AGENTS.md index c30473070c..102adab8d7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -18,7 +18,7 @@ Use this file as the local operating guide for the current codebase. Prefer the ## Go-only target - Remove all Python code and Python-specific dependencies, scripts, containers, CI jobs, documentation, SDKs, and tests from the repository. Do not add new Python code while this removal is in progress. Replace Python-based preparation of Go native libraries, DeepDoc models, and the BPE table before removing the existing download scripts. - Move any still-required behavior and test coverage into the Go backend or the frontend before deleting the old implementation. Do not preserve a Python server, compatibility layer, fallback, or dual-backend path. -- Before deleting `rag/`, move the Go DeepDoc `.ort` models and `ocr.res` out of `rag/res/deepdoc/`, then update model discovery, dependency preparation, Docker packaging, and tests together. +- Before deleting `rag/`, move the Go DeepDoc `.ort` models and `ocr.res` out of `internal/rag/res/deepdoc/`, then update model discovery, dependency preparation, Docker packaging, and tests together. - Remove the frontend's Go/Python backend variants and update `web/CLAUDE.md` when the frontend no longer needs them. Until then, treat its dual-backend instructions as migration-only guidance; new frontend work should target the Go API. - Update Docker's default entrypoint, dependency image, and CI workflows when removing their Python paths. Until then, select Go explicitly with `API_PROXY_SCHEME=go` where that variable controls startup. - Keep this guide aligned with the checked-in tree. Once the Python directories are deleted, remove their migration inventory below. diff --git a/Dockerfile b/Dockerfile index 55e2b74958..162d16363f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,7 +16,7 @@ WORKDIR /ragflow # layout.laws/manual/paper.onnx are byte-identical to layout.onnx, so we # exclude them from the tar extract and symlink them to layout.onnx instead, # saving ~219MB in the image. -RUN mkdir -p /ragflow/rag/res/deepdoc /root/.ragflow +RUN mkdir -p /ragflow/internal/rag/res/deepdoc /root/.ragflow RUN --mount=type=bind,from=infiniflow/ragflow_deps:latest,source=/huggingface.co,target=/huggingface.co \ tar --exclude='.*' \ --exclude='layout.laws.onnx' \ @@ -32,13 +32,13 @@ RUN --mount=type=bind,from=infiniflow/ragflow_deps:latest,source=/huggingface.co -cf - \ /huggingface.co/InfiniFlow/text_concat_xgb_v1.0 \ /huggingface.co/InfiniFlow/deepdoc \ - | tar -xf - --strip-components=3 -C /ragflow/rag/res/deepdoc && \ - #ln -s layout.onnx /ragflow/rag/res/deepdoc/layout.laws.onnx && \ - #ln -s layout.onnx /ragflow/rag/res/deepdoc/layout.manual.onnx && \ - #ln -s layout.onnx /ragflow/rag/res/deepdoc/layout.paper.onnx - ln -s layout.ort /ragflow/rag/res/deepdoc/layout.laws.ort && \ - ln -s layout.ort /ragflow/rag/res/deepdoc/layout.manual.ort && \ - ln -s layout.ort /ragflow/rag/res/deepdoc/layout.paper.ort + | tar -xf - --strip-components=3 -C /ragflow/internal/rag/res/deepdoc && \ + #ln -s layout.onnx /ragflow/internal/rag/res/deepdoc/layout.laws.onnx && \ + #ln -s layout.onnx /ragflow/internal/rag/res/deepdoc/layout.manual.onnx && \ + #ln -s layout.onnx /ragflow/internal/rag/res/deepdoc/layout.paper.onnx + ln -s layout.ort /ragflow/internal/rag/res/deepdoc/layout.laws.ort && \ + ln -s layout.ort /ragflow/internal/rag/res/deepdoc/layout.manual.ort && \ + ln -s layout.ort /ragflow/internal/rag/res/deepdoc/layout.paper.ort # Copy the cl100k_base BPE table used by the Go tokenizer (tiktoken-go # cl100k_base). The deps image ships it at its root; the Go image previously diff --git a/Dockerfile_base b/Dockerfile_base index e67bfd6bb0..ae2b0be2e2 100644 --- a/Dockerfile_base +++ b/Dockerfile_base @@ -10,12 +10,12 @@ ARG GITEE_TOKEN="" WORKDIR /ragflow # copy models downloaded via download_deps.py -RUN mkdir -p /ragflow/rag/res/deepdoc /root/.ragflow +RUN mkdir -p /ragflow/internal/rag/res/deepdoc /root/.ragflow RUN --mount=type=bind,from=infiniflow/ragflow_deps:latest,source=/huggingface.co,target=/huggingface.co \ tar --exclude='.*' -cf - \ /huggingface.co/InfiniFlow/text_concat_xgb_v1.0 \ /huggingface.co/InfiniFlow/deepdoc \ - | tar -xf - --strip-components=3 -C /ragflow/rag/res/deepdoc + | tar -xf - --strip-components=3 -C /ragflow/internal/rag/res/deepdoc # https://github.com/chrismattmann/tika-python # This is the only way to run python-tika without internet access. Without this set, the default is to check the tika version and pull latest every time from Apache. diff --git a/README.md b/README.md index 4b9a48fd29..b53c059436 100644 --- a/README.md +++ b/README.md @@ -264,7 +264,7 @@ Install the Go version specified in `go.mod` (currently Go 1.27), Clang 20, LLD bash build.sh --all ``` - The script prepares native libraries and model resources required for the Go build and needs `requests` and `huggingface-hub`. Skip this step if you have prepared the same resources by other means. When started from the repository root, Go services automatically find `rag/res/deepdoc`; to start from another directory, set `DEEPDOC_MODEL_DIR` to its absolute path. + The script prepares native libraries and model resources required for the Go build and needs `requests` and `huggingface-hub`. Skip this step if you have prepared the same resources by other means. When started from the repository root, Go services automatically find `internal/rag/res/deepdoc`; to start from another directory, set `DEEPDOC_MODEL_DIR` to its absolute path. 3. Start the local dependencies and make sure the hosts and ports in **conf/service_conf.yaml** point to addresses accessible from the host. Go source services connect to Compose-exposed Kvrocks at `localhost:6379`, while Go Docker services connect to Kvrocks on the container network. If using the default Elasticsearch engine, set `vm.max_map_count` on the Docker host to at least `262144` first. diff --git a/README_ar.md b/README_ar.md index 023b3b6856..cfa7b18e6c 100644 --- a/README_ar.md +++ b/README_ar.md @@ -285,7 +285,7 @@ bash build.sh --all ``` - يجهّز البرنامج النصي المكتبات الأصلية وموارد النماذج اللازمة لبناء Go، ويحتاج إلى `requests` و`huggingface-hub`. يمكن تخطي هذه الخطوة إذا جُهزت الموارد نفسها بطريقة أخرى. عند التشغيل من جذر المستودع، تعثر خدمات Go تلقائيًا على `rag/res/deepdoc`؛ وللتشغيل من دليل آخر، اضبط `DEEPDOC_MODEL_DIR` على المسار المطلق لذلك الدليل. + يجهّز البرنامج النصي المكتبات الأصلية وموارد النماذج اللازمة لبناء Go، ويحتاج إلى `requests` و`huggingface-hub`. يمكن تخطي هذه الخطوة إذا جُهزت الموارد نفسها بطريقة أخرى. عند التشغيل من جذر المستودع، تعثر خدمات Go تلقائيًا على `internal/rag/res/deepdoc`؛ وللتشغيل من دليل آخر، اضبط `DEEPDOC_MODEL_DIR` على المسار المطلق لذلك الدليل. 3. ابدأ الخدمات التابعة المطلوبة (Elasticsearch وMySQL وMinIO وNATS وKvrocks وClickHouse) باستخدام Docker Compose: ```bash diff --git a/README_fr.md b/README_fr.md index 73b177fbd5..d9707706be 100644 --- a/README_fr.md +++ b/README_fr.md @@ -273,7 +273,7 @@ Pour changer de moteur de documents, modifier la configuration, redémarrer les bash build.sh --all ``` - Le script prépare les bibliothèques natives et les ressources de modèles nécessaires à la compilation Go et requiert `requests` et `huggingface-hub`. Ignorez cette étape si vous avez préparé les mêmes ressources autrement. Depuis la racine du dépôt, les services Go trouvent automatiquement `rag/res/deepdoc` ; depuis un autre répertoire, définissez `DEEPDOC_MODEL_DIR` sur son chemin absolu. + Le script prépare les bibliothèques natives et les ressources de modèles nécessaires à la compilation Go et requiert `requests` et `huggingface-hub`. Ignorez cette étape si vous avez préparé les mêmes ressources autrement. Depuis la racine du dépôt, les services Go trouvent automatiquement `internal/rag/res/deepdoc` ; depuis un autre répertoire, définissez `DEEPDOC_MODEL_DIR` sur son chemin absolu. 3. Lancez les dépendances requises (Elasticsearch, MySQL, MinIO, NATS, Kvrocks et ClickHouse) avec Docker Compose : ```bash diff --git a/README_id.md b/README_id.md index d74b3ccfc9..8baf7f0442 100644 --- a/README_id.md +++ b/README_id.md @@ -281,7 +281,7 @@ Untuk mengganti mesin dokumen, mengubah konfigurasi, memulai ulang layanan, sert bash build.sh --all ``` - Skrip tersebut menyiapkan pustaka native dan sumber daya model yang diperlukan untuk build Go serta memerlukan `requests` dan `huggingface-hub`. Lewati langkah ini jika sumber daya yang sama telah disiapkan dengan cara lain. Saat dijalankan dari root repositori, layanan Go otomatis menemukan `rag/res/deepdoc`; untuk menjalankan dari direktori lain, atur `DEEPDOC_MODEL_DIR` ke path absolutnya. + Skrip tersebut menyiapkan pustaka native dan sumber daya model yang diperlukan untuk build Go serta memerlukan `requests` dan `huggingface-hub`. Lewati langkah ini jika sumber daya yang sama telah disiapkan dengan cara lain. Saat dijalankan dari root repositori, layanan Go otomatis menemukan `internal/rag/res/deepdoc`; untuk menjalankan dari direktori lain, atur `DEEPDOC_MODEL_DIR` ke path absolutnya. 3. Jalankan dependensi yang diperlukan (Elasticsearch, MySQL, MinIO, NATS, Kvrocks, dan ClickHouse) menggunakan Docker Compose: ```bash diff --git a/README_ja.md b/README_ja.md index 90cd828485..bb2d434ede 100644 --- a/README_ja.md +++ b/README_ja.md @@ -263,7 +263,7 @@ Go版のDockerデプロイでは `docker/.env` と `docker/docker-compose.yml` bash build.sh --all ``` - このスクリプトはGoビルドに必要なネイティブライブラリとモデルリソースを準備し、`requests`と`huggingface-hub`を使用します。同じリソースを別の方法で準備済みの場合は、この手順を省略できます。リポジトリルートから起動するとGoサービスは`rag/res/deepdoc`を自動検出します。別のディレクトリから起動する場合は、`DEEPDOC_MODEL_DIR`にその絶対パスを設定してください。 + このスクリプトはGoビルドに必要なネイティブライブラリとモデルリソースを準備し、`requests`と`huggingface-hub`を使用します。同じリソースを別の方法で準備済みの場合は、この手順を省略できます。リポジトリルートから起動するとGoサービスは`internal/rag/res/deepdoc`を自動検出します。別のディレクトリから起動する場合は、`DEEPDOC_MODEL_DIR`にその絶対パスを設定してください。 3. Docker Composeで必要な依存サービス(Elasticsearch、MySQL、MinIO、NATS、Kvrocks、ClickHouse)を起動します: ```bash diff --git a/README_ko.md b/README_ko.md index 3df5f47b55..68b7044c03 100644 --- a/README_ko.md +++ b/README_ko.md @@ -263,7 +263,7 @@ Go Docker 배포는 `docker/.env`와 `docker/docker-compose.yml`을 사용하며 bash build.sh --all ``` - 이 스크립트는 Go 빌드에 필요한 네이티브 라이브러리와 모델 리소스를 준비하며 `requests`와 `huggingface-hub`가 필요합니다. 같은 리소스를 다른 방법으로 준비했다면 이 단계를 건너뛸 수 있습니다. 저장소 루트에서 시작하면 Go 서비스가 `rag/res/deepdoc`을 자동으로 찾습니다. 다른 디렉터리에서 시작하려면 `DEEPDOC_MODEL_DIR`을 해당 디렉터리의 절대 경로로 설정하세요. + 이 스크립트는 Go 빌드에 필요한 네이티브 라이브러리와 모델 리소스를 준비하며 `requests`와 `huggingface-hub`가 필요합니다. 같은 리소스를 다른 방법으로 준비했다면 이 단계를 건너뛸 수 있습니다. 저장소 루트에서 시작하면 Go 서비스가 `internal/rag/res/deepdoc`을 자동으로 찾습니다. 다른 디렉터리에서 시작하려면 `DEEPDOC_MODEL_DIR`을 해당 디렉터리의 절대 경로로 설정하세요. 3. Docker Compose를 사용하여 필요한 의존 서비스(Elasticsearch, MySQL, MinIO, NATS, Kvrocks, ClickHouse)를 시작합니다: diff --git a/README_pt_br.md b/README_pt_br.md index 65a4271d62..49bec0c864 100644 --- a/README_pt_br.md +++ b/README_pt_br.md @@ -269,7 +269,7 @@ Para trocar o mecanismo de documentos, alterar configurações, reiniciar servi bash build.sh --all ``` - O script prepara as bibliotecas nativas e os recursos de modelo necessários para o build Go e requer `requests` e `huggingface-hub`. Ignore esta etapa se você já preparou os mesmos recursos de outra forma. Quando iniciados na raiz do repositório, os serviços Go encontram automaticamente `rag/res/deepdoc`; para iniciar em outro diretório, defina `DEEPDOC_MODEL_DIR` como o caminho absoluto correspondente. + O script prepara as bibliotecas nativas e os recursos de modelo necessários para o build Go e requer `requests` e `huggingface-hub`. Ignore esta etapa se você já preparou os mesmos recursos de outra forma. Quando iniciados na raiz do repositório, os serviços Go encontram automaticamente `internal/rag/res/deepdoc`; para iniciar em outro diretório, defina `DEEPDOC_MODEL_DIR` como o caminho absoluto correspondente. 3. Inicie as dependências necessárias (Elasticsearch, MySQL, MinIO, NATS, Kvrocks e ClickHouse) usando Docker Compose: ```bash diff --git a/README_ru.md b/README_ru.md index ec49faa9bd..d09cc90d22 100644 --- a/README_ru.md +++ b/README_ru.md @@ -279,7 +279,7 @@ bash build.sh --all ``` - Скрипт подготавливает нативные библиотеки и ресурсы моделей для сборки Go и требует `requests` и `huggingface-hub`. Пропустите этот шаг, если те же ресурсы подготовлены другим способом. При запуске из корня репозитория сервисы Go автоматически находят `rag/res/deepdoc`; для запуска из другого каталога задайте `DEEPDOC_MODEL_DIR` как абсолютный путь к нему. + Скрипт подготавливает нативные библиотеки и ресурсы моделей для сборки Go и требует `requests` и `huggingface-hub`. Пропустите этот шаг, если те же ресурсы подготовлены другим способом. При запуске из корня репозитория сервисы Go автоматически находят `internal/rag/res/deepdoc`; для запуска из другого каталога задайте `DEEPDOC_MODEL_DIR` как абсолютный путь к нему. 3. Запустите необходимые зависимости (Elasticsearch, MySQL, MinIO, NATS, Kvrocks и ClickHouse): diff --git a/README_tr.md b/README_tr.md index dacd373645..0395c24b43 100644 --- a/README_tr.md +++ b/README_tr.md @@ -281,7 +281,7 @@ Belge motorunu değiştirme, yapılandırma değişikliklerinden sonra hizmetler bash build.sh --all ``` - Betik, Go derlemesi için gerekli yerel kitaplıkları ve model kaynaklarını hazırlar ve `requests` ile `huggingface-hub` gerektirir. Aynı kaynakları başka bir yöntemle hazırladıysanız bu adımı atlayabilirsiniz. Depo kökünden başlatıldığında Go hizmetleri `rag/res/deepdoc` dizinini otomatik olarak bulur; başka bir dizinden başlatmak için `DEEPDOC_MODEL_DIR` değişkenini mutlak yola ayarlayın. + Betik, Go derlemesi için gerekli yerel kitaplıkları ve model kaynaklarını hazırlar ve `requests` ile `huggingface-hub` gerektirir. Aynı kaynakları başka bir yöntemle hazırladıysanız bu adımı atlayabilirsiniz. Depo kökünden başlatıldığında Go hizmetleri `internal/rag/res/deepdoc` dizinini otomatik olarak bulur; başka bir dizinden başlatmak için `DEEPDOC_MODEL_DIR` değişkenini mutlak yola ayarlayın. 3. Gerekli bağımlılıkları (Elasticsearch, MySQL, MinIO, NATS, Kvrocks ve ClickHouse) Docker Compose ile başlatın: ```bash diff --git a/README_tzh.md b/README_tzh.md index 0b03cd4187..ade350faf6 100644 --- a/README_tzh.md +++ b/README_tzh.md @@ -281,7 +281,7 @@ Go 版 Docker 部署使用 `docker/.env` 和 `docker/docker-compose.yml`,以 K bash build.sh --all ``` - 下載腳本使用 `requests` 和 `huggingface-hub` 來準備 Go 建置所需的原生程式庫和模型資源;如已透過其他方式準備好相同資源,可略過此步驟。從倉庫根目錄啟動時,Go 服務會自動尋找 `rag/res/deepdoc`;如需從其他目錄啟動,請將 `DEEPDOC_MODEL_DIR` 設為該目錄的絕對路徑。 + 下載腳本使用 `requests` 和 `huggingface-hub` 來準備 Go 建置所需的原生程式庫和模型資源;如已透過其他方式準備好相同資源,可略過此步驟。從倉庫根目錄啟動時,Go 服務會自動尋找 `internal/rag/res/deepdoc`;如需從其他目錄啟動,請將 `DEEPDOC_MODEL_DIR` 設為該目錄的絕對路徑。 3. 啟動本機依賴服務,並確認 **conf/service_conf.yaml** 中的主機與連接埠可從主機存取: ```bash diff --git a/README_zh.md b/README_zh.md index d63005044d..65a18287da 100644 --- a/README_zh.md +++ b/README_zh.md @@ -288,7 +288,7 @@ Go 版 Docker 部署使用 `docker/.env` 和 `docker/docker-compose.yml`,使 bash build.sh --all ``` - 下载脚本负责准备 Go 构建所需的原生库和模型资源,并需要 `requests` 和 `huggingface-hub`。如果已通过其他方式准备好相同资源,可跳过这一步。Go 服务从仓库根目录启动时会自动查找 `rag/res/deepdoc`;如需从其他目录启动,请将 `DEEPDOC_MODEL_DIR` 设为该目录的绝对路径。 + 下载脚本负责准备 Go 构建所需的原生库和模型资源,并需要 `requests` 和 `huggingface-hub`。如果已通过其他方式准备好相同资源,可跳过这一步。Go 服务从仓库根目录启动时会自动查找 `internal/rag/res/deepdoc`;如需从其他目录启动,请将 `DEEPDOC_MODEL_DIR` 设为该目录的绝对路径。 3. 启动本地依赖服务,并确认 **conf/service_conf.yaml** 中的主机与端口对应宿主机可访问的地址。Go 源码服务通过 `localhost:6379` 连接 Compose 暴露的 Kvrocks;Go Docker 服务则在容器网络中连接 Kvrocks。如果使用默认 Elasticsearch,还需先将 Docker 主机的 `vm.max_map_count` 设为至少 `262144`。 diff --git a/build.sh b/build.sh index d20d05b5b9..e3d2ecf309 100755 --- a/build.sh +++ b/build.sh @@ -1181,10 +1181,10 @@ main() { pkgs=(./internal/deepdoc/parser/pdf/inference/native_analyzer/...) fi # The in-process (Go) DeepDoc backend needs the .ort weights. Default - # MODEL_DIR to the canonical repo model dir (rag/res/deepdoc) so a + # MODEL_DIR to the canonical repo model dir (internal/rag/res/deepdoc) so a # local run needs no MODEL_DIR export after `download_go_deps.py`. REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - export MODEL_DIR="${MODEL_DIR:-$REPO_ROOT/rag/res/deepdoc}" + export MODEL_DIR="${MODEL_DIR:-$REPO_ROOT/internal/rag/res/deepdoc}" run_go_tests_tagged "cgo integration" "${pkgs[@]}" run_native_integration_tests ;; diff --git a/cmd/ragflow_server.go b/cmd/ragflow_server.go index 2f1f8ad409..c4d36d5fe4 100644 --- a/cmd/ragflow_server.go +++ b/cmd/ragflow_server.go @@ -1678,10 +1678,6 @@ func logTokenizerCounters() { } } -// resolveDeepDocModelDir picks the model directory: the explicit DEEPDOC_MODEL_DIR -// env, else the RAGFlow default model dir (rag/res/deepdoc), -// else the snapshot fetched by ragflow_deps/download_deps.py. The first -// candidate that actually contains the required weights wins. func resolveDeepDocModelDir() string { if v := strings.TrimSpace(common.GetEnv(common.EnvDeepDocModelDir)); v != "" { return v @@ -1691,7 +1687,7 @@ func resolveDeepDocModelDir() string { // variable can point at the DeepDoc weights as well as the embedding tokenizers. candidates := append([]string(nil), common.ModelAssetCandidates("huggingface.co/InfiniFlow/deepdoc")...) candidates = append(candidates, - filepath.Join(wd, "rag", "res", "deepdoc"), + filepath.Join(wd, "internal", "rag", "res", "deepdoc"), filepath.Join(wd, "huggingface.co", "InfiniFlow", "deepdoc"), ) for _, c := range candidates { @@ -1701,7 +1697,7 @@ func resolveDeepDocModelDir() string { } // None verified; return the canonical default so any error message points // at the conventional location. - return filepath.Join(wd, "rag", "res", "deepdoc") + return filepath.Join(wd, "internal", "rag", "res", "deepdoc") } // resolveDeepDocDropScore returns the explicit DEEPDOC_DROP_SCORE env, else the diff --git a/docker/README.md b/docker/README.md index 60640fe5ab..096dd117c5 100644 --- a/docker/README.md +++ b/docker/README.md @@ -154,7 +154,7 @@ The optional `tei-cpu` and `tei-gpu` profiles start a local text-embeddings-infe ### DeepDoc (in-process) -DeepDoc layout analysis (DLA), OCR (text detection/recognition), and table structure recognition (TSR) run **in-process** inside the RAGFlow server using ONNX Runtime — there is no separate DeepDoc service to deploy. ONNX Runtime is statically linked into the server binary (resolved at runtime via dlopen(NULL); no `libonnxruntime.so` is required) and the models are loaded at runtime; `DEEPDOC_MODEL_DIR` overrides the default model directory. `Dockerfile` copies the required model assets into `/ragflow/rag/res/deepdoc`. +DeepDoc layout analysis (DLA), OCR (text detection/recognition), and table structure recognition (TSR) run **in-process** inside the RAGFlow server using ONNX Runtime — there is no separate DeepDoc service to deploy. ONNX Runtime is statically linked into the server binary (resolved at runtime via dlopen(NULL); no `libonnxruntime.so` is required) and the models are loaded at runtime; `DEEPDOC_MODEL_DIR` overrides the default model directory. `Dockerfile` copies the required model assets into `/ragflow/internal/rag/res/deepdoc`. In the RAGFlow open-source 1.0 release, DeepDoc uses CPU inference. diff --git a/docs/develop/launch_ragflow_from_source.md b/docs/develop/launch_ragflow_from_source.md index 4779490a04..d22e488772 100644 --- a/docs/develop/launch_ragflow_from_source.md +++ b/docs/develop/launch_ragflow_from_source.md @@ -62,7 +62,7 @@ python3 -m venv /tmp/ragflow-go-download-venv /tmp/ragflow-go-download-venv/bin/python ragflow_deps/download_go_deps.py ``` -The downloader fetches the static libraries needed by `build.sh`, plus `det.ort`, `layout.ort`, `tsr.ort`, `rec.ort`, and `ocr.res` into `rag/res/deepdoc/`. These files are required by the in-process Go DeepDoc backend. Keep the server's working directory at the repository root so it can find them automatically; if you launch it elsewhere, set `DEEPDOC_MODEL_DIR` to the absolute path of `rag/res/deepdoc`. +The downloader fetches the static libraries needed by `build.sh`, plus `det.ort`, `layout.ort`, `tsr.ort`, `rec.ort`, and `ocr.res` into `internal/rag/res/deepdoc/`. These files are required by the in-process Go DeepDoc backend. Keep the server's working directory at the repository root so it can find them automatically; if you launch it elsewhere, set `DEEPDOC_MODEL_DIR` to the absolute path of `rag/res/deepdoc`. The isolated download environment can be removed after the resources have been prepared. @@ -147,7 +147,7 @@ If you changed a dependency's published port or credentials, update `conf/servic `RAGFLOW_DEV_MODE=true` bypasses the code-versus-database version check. Use it only for development, not to run an older production binary against a newer database. The standalone `--migrate` action must still complete first. -If startup reports `no in-process DeepDoc backend serving`, check that all five model files above are in `rag/res/deepdoc/`, then re-run the Go dependency downloader and rebuild if necessary. If the tokenizer reports a missing `cl100k_base.tiktoken`, ensure the BPE table is in `ragflow_deps/`. +If startup reports `no in-process DeepDoc backend serving`, check that all five model files above are in `internal/rag/res/deepdoc/`, then re-run the Go dependency downloader and rebuild if necessary. If the tokenizer reports a missing `cl100k_base.tiktoken`, ensure the BPE table is in `ragflow_deps/`. ## 4. Start the frontend diff --git a/internal/common/environments.go b/internal/common/environments.go index f0562e0a6e..522a236ea2 100644 --- a/internal/common/environments.go +++ b/internal/common/environments.go @@ -249,7 +249,7 @@ const ( // EnvDeepDocModelDir points the in-process (Go) DeepDoc backend at the // model snapshot (see common.DeepDocModelFiles); mirrors - // the RAGFlow default model dir (rag/res/deepdoc). + // the RAGFlow default model dir (internal/rag/res/deepdoc). EnvDeepDocModelDir = "DEEPDOC_MODEL_DIR" // EnvDeepDocDropScore overrides the confidence threshold below which the // in-process (Go) DeepDoc backend blanks recognized text while preserving diff --git a/internal/deepdoc/native/run.sh b/internal/deepdoc/native/run.sh index 6b82e94cbc..8dd6d7b06e 100755 --- a/internal/deepdoc/native/run.sh +++ b/internal/deepdoc/native/run.sh @@ -14,11 +14,11 @@ cd "$SCRIPT_DIR" # ONNX Runtime is statically linked (libonnxruntime.a) and resolved via # dlopen(NULL); no ORT_LIB is needed. # -# Default MODEL_DIR to the canonical repo model dir (rag/res/deepdoc) so running +# Default MODEL_DIR to the canonical repo model dir (internal/rag/res/deepdoc) so running # `download_go_deps.py` is enough — no MODEL_DIR export is required for local # Go DeepDoc runs. Override with `MODEL_DIR=... bash run.sh` if needed. REPO_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)" -MODEL_DIR="${MODEL_DIR:-$REPO_ROOT/rag/res/deepdoc}" +MODEL_DIR="${MODEL_DIR:-$REPO_ROOT/internal/rag/res/deepdoc}" export MODEL_DIR run_task() { diff --git a/internal/deepdoc/parser/pdf/inference/native_analyzer/native_analyzer_test.go b/internal/deepdoc/parser/pdf/inference/native_analyzer/native_analyzer_test.go index 61c5c21d85..d80c4762a7 100644 --- a/internal/deepdoc/parser/pdf/inference/native_analyzer/native_analyzer_test.go +++ b/internal/deepdoc/parser/pdf/inference/native_analyzer/native_analyzer_test.go @@ -47,7 +47,7 @@ func TestNativeAnalyzerUninitializedNegative(t *testing.T) { } modelDir := os.Getenv("MODEL_DIR") if modelDir == "" { - modelDir = filepath.Join("..", "..", "..", "..", "rag", "res", "deepdoc") + modelDir = filepath.Join("..", "..", "..", "..", "internal", "rag", "res", "deepdoc") } if _, err := NewAnalyzer(modelDir, DefaultDropScore); err == nil { t.Error("NewAnalyzer succeeded before ONNX Runtime init; expected error") diff --git a/internal/deepdoc/parser/pdf/inference/native_analyzer/raster_alignment_test.go b/internal/deepdoc/parser/pdf/inference/native_analyzer/raster_alignment_test.go index d8b0f70b44..ee7a14fcb2 100644 --- a/internal/deepdoc/parser/pdf/inference/native_analyzer/raster_alignment_test.go +++ b/internal/deepdoc/parser/pdf/inference/native_analyzer/raster_alignment_test.go @@ -58,7 +58,7 @@ func repoRoot(t *testing.T) string { } // .../internal/deepdoc/parser/pdf/inference/native_analyzer -> repo root is 6 levels up. root := filepath.Join(dir, "..", "..", "..", "..", "..", "..") - if _, err := os.Stat(filepath.Join(root, "rag", "res", "deepdoc")); err != nil { + if _, err := os.Stat(filepath.Join(root, "internal", "rag", "res", "deepdoc")); err != nil { t.Fatalf("repoRoot %s missing model dir: %v", root, err) } return root diff --git a/internal/deepdoc/parser/pdf/util/garbled.go b/internal/deepdoc/parser/pdf/util/garbled.go index ad28caa0b3..8355088111 100644 --- a/internal/deepdoc/parser/pdf/util/garbled.go +++ b/internal/deepdoc/parser/pdf/util/garbled.go @@ -215,13 +215,13 @@ var ( ) // loadOCRAlphabet reads the recogniser's character dictionary once -// (rag/res/deepdoc/ocr.res, provisioned next to the OCR model). On any error it +// (internal/rag/res/deepdoc/ocr.res, provisioned next to the OCR model). On any error it // returns an empty set, which OcrCanRepresent treats as "unknown alphabet" and // so allows the existing fallback behaviour. func loadOCRAlphabet() map[rune]struct{} { ocrAlphabetOnce.Do(func() { ocrAlphabet = map[rune]struct{}{} - data, err := os.ReadFile(filepath.Join(utility.GetProjectRoot(), "rag", "res", "deepdoc", "ocr.res")) + data, err := os.ReadFile(filepath.Join(utility.GetProjectRoot(), "internal", "rag", "res", "deepdoc", "ocr.res")) if err != nil { return } diff --git a/internal/development.md b/internal/development.md index 41efe2586f..f40b1b7d37 100644 --- a/internal/development.md +++ b/internal/development.md @@ -166,7 +166,7 @@ binary carries it and registers the backend at startup. #### Model weights The Go backend loads **`.ort`** (FlatBuffer) weights; the Python side loads -**`.onnx`**. Both formats live side by side in `rag/res/deepdoc/` — neither +**`.onnx`**. Both formats live side by side in `internal/rag/res/deepdoc/` — neither supersedes the other, so do not delete one to "clean up". | | Go (in-process) | Python | @@ -175,12 +175,12 @@ supersedes the other, so do not delete one to "clean up". | Files | `det.ort`, `layout.ort`, `tsr.ort`, `rec.ort`, `ocr.res` | `det.onnx`, `layout.onnx`, `tsr.onnx`, `rec.onnx`, `ocr.res` | `download_go_deps.py` (§1.4) fetches the five required files — four `.ort` plus -`ocr.res` — into `rag/res/deepdoc/`; `download_deps.py` snapshots the whole +`ocr.res` — into `internal/rag/res/deepdoc/`; `download_deps.py` snapshots the whole `InfiniFlow/deepdoc` repo and therefore carries both formats. Auto-discovery is **relative to the server process's working directory**: `resolveDeepDocModelDir()` (`cmd/ragflow_server.go`) probes -`/rag/res/deepdoc`, then `/huggingface.co/InfiniFlow/deepdoc`. +`/internal/rag/res/deepdoc`, then `/huggingface.co/InfiniFlow/deepdoc`. Launching `./bin/ragflow_server` from the repo root therefore needs no `MODEL_DIR` / `DEEPDOC_MODEL_DIR` export; from any other CWD — or an image with a different WORKDIR — set `MODEL_DIR` explicitly. @@ -189,7 +189,7 @@ a different WORKDIR — set `MODEL_DIR` explicitly. authoritative list — `HasModelFiles()` refuses to serve when any file in it is missing from the model directory. -> **Note**: A `rag/res/deepdoc/` populated before the `.ort` switch holds only +> **Note**: A `internal/rag/res/deepdoc/` populated before the `.ort` switch holds only > `.onnx` and will NOT serve the Go backend, even though the directory looks > fully populated. Re-run `download_go_deps.py` after updating. diff --git a/rag/res/ner.json b/internal/rag/res/ner.json similarity index 100% rename from rag/res/ner.json rename to internal/rag/res/ner.json diff --git a/rag/res/synonym.json b/internal/rag/res/synonym.json similarity index 100% rename from rag/res/synonym.json rename to internal/rag/res/synonym.json diff --git a/internal/service/nlp/synonym.go b/internal/service/nlp/synonym.go index 4f86183661..f34eae83f5 100644 --- a/internal/service/nlp/synonym.go +++ b/internal/service/nlp/synonym.go @@ -61,7 +61,7 @@ func NewSynonym(redis Client, resPath string, wordnetDir string) *Synonym { s.lookupNum.Store(100000000) if resPath == "" { - s.resPath = "rag/res" + s.resPath = "internal/rag/res" } // Initialize WordNet with provided path diff --git a/internal/service/nlp/term_weight.go b/internal/service/nlp/term_weight.go index 984b2df613..2579726dbb 100644 --- a/internal/service/nlp/term_weight.go +++ b/internal/service/nlp/term_weight.go @@ -54,12 +54,12 @@ func NewTermWeightDealer(resPath string) *TermWeightDealer { // Load named entity dictionary if resPath == "" { - resPath = "rag/res" + resPath = "internal/rag/res" } nerPath := filepath.Join(resPath, "ner.json") if data, err := os.ReadFile(nerPath); err == nil { - if err := json.Unmarshal(data, &d.ne); err != nil { + if err = json.Unmarshal(data, &d.ne); err != nil { common.Warn("Failed to load ner.json", zap.Error(err)) } } else { diff --git a/ragflow_deps/download_go_deps.py b/ragflow_deps/download_go_deps.py index a333c4bc39..90d5f1cf81 100755 --- a/ragflow_deps/download_go_deps.py +++ b/ragflow_deps/download_go_deps.py @@ -35,7 +35,7 @@ # # Go DeepDoc weights: in addition to the native libs, this script downloads the # five Go model files (internal/common.DeepDocModelFiles) from InfiniFlow/deepdoc -# straight into the repo's canonical model directory `rag/res/deepdoc/` (one level +# straight into the repo's canonical model directory `internal/rag/res/deepdoc/` (one level # up from this script). The Go server auto-discovers that directory via # resolveDeepDocModelDir(), and build.sh --test-native / internal/deepdoc/native/run.sh # default MODEL_DIR there too — so after running this script NO MODEL_DIR env needs @@ -287,7 +287,7 @@ def download_go_models(use_china_mirrors=False): backend with no further setup. The files are written into the repo's canonical model directory - `rag/res/deepdoc/` (relative to the repo root, one level up from this + `internal/rag/res/deepdoc/` (relative to the repo root, one level up from this script). The Go server auto-discovers that directory via resolveDeepDocModelDir(), and build.sh --test-native / run.sh default MODEL_DIR there too — so after this script runs, no MODEL_DIR env needs to @@ -305,7 +305,7 @@ def download_go_models(use_china_mirrors=False): # Canonical local-dev model dir the Go backend auto-discovers. repo_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - target_dir = os.path.join(repo_root, "rag", "res", "deepdoc") + target_dir = os.path.join(repo_root, "internal", "rag", "res", "deepdoc") os.makedirs(target_dir, exist_ok=True) missing = [] diff --git a/tools/render_diff/warp_align.py b/tools/render_diff/warp_align.py index 2a8eb2467f..bcfd33ee1e 100755 --- a/tools/render_diff/warp_align.py +++ b/tools/render_diff/warp_align.py @@ -41,7 +41,7 @@ Run WARP_PDF=test/benchmark/test_docs/Doc1.pdf \ bash build.sh --test-manual ./internal/deepdoc/parser/pdf/ -run TestWarpAlignGo .venv/bin/python tools/render_diff/warp_align.py compare - # (needs HF weights in rag/res/deepdoc) + # (needs HF weights in internal/rag/res/deepdoc) .venv/bin/python tools/render_diff/warp_align.py rec """